diff --git a/Cargo.lock b/Cargo.lock index e138a9f0f..ceb79ef37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -830,6 +830,10 @@ version = "0.1.0" dependencies = [ "omicron-common", "oxnet", + "schemars 0.8.22", + "serde", + "serde_json", + "thiserror 2.0.20", ] [[package]] @@ -1367,10 +1371,12 @@ dependencies = [ "chrono", "ddm-api", "ddm-api-types", + "ddm-api-types-versions", "ddm-protocol", "dpd-client", "dropshot", "expectorate", + "futures", "hostname 0.4.2", "http-body-util", "hyper", @@ -1386,6 +1392,7 @@ dependencies = [ "oxnet", "port-file", "pretty_assertions", + "reqwest 0.13.4", "schemars 0.8.22", "serde", "serde_json", @@ -1393,6 +1400,7 @@ dependencies = [ "slog", "slog-error-chain", "socket2", + "tempfile", "thiserror 2.0.20", "tokio", "uuid", @@ -1432,11 +1440,14 @@ dependencies = [ name = "ddm-api-types-versions" version = "0.1.0" dependencies = [ + "client-common", "ddm-protocol", "oxnet", "schemars 0.8.22", "serde", + "serde_json", "serde_repr", + "thiserror 2.0.20", "uuid", ] @@ -3920,13 +3931,16 @@ dependencies = [ "backoff", "clap", "ddm-api-types", + "dpd-client", "libc", "libnet", "oximeter", "oximeter-producer", "oxnet", + "proptest", "schemars 0.8.22", "serde", + "serde_json", "slog", "slog-async", "slog-bunyan", diff --git a/Cargo.toml b/Cargo.toml index d1df039e5..9dd5f2a5e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -97,6 +97,7 @@ dropshot-api-manager-types = "0.7.2" expectorate = "1.3.0" schemars = { version = "0.8.22", features = [ "uuid1", "chrono" ] } tokio = { version = "1.52.1", features = ["full"] } +futures = "0.3" serde_repr = "0.1" anyhow = "1.0.104" port-file = "0.1.0" diff --git a/client-common/Cargo.toml b/client-common/Cargo.toml index 545ef8cb1..272287858 100644 --- a/client-common/Cargo.toml +++ b/client-common/Cargo.toml @@ -5,6 +5,10 @@ edition = "2024" [dependencies] oxnet.workspace = true +schemars.workspace = true +serde.workspace = true +thiserror.workspace = true [dev-dependencies] omicron-common.workspace = true +serde_json.workspace = true diff --git a/client-common/src/address.rs b/client-common/src/address.rs index 1d6f9ce19..fc5b57cdd 100644 --- a/client-common/src/address.rs +++ b/client-common/src/address.rs @@ -2,8 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -// Copyright 2026 Oxide Computer Company - //! Multicast addressing constants shared across the routing suite. //! //! These mirror the canonical definitions in `omicron_common::address`. diff --git a/client-common/src/lib.rs b/client-common/src/lib.rs index f653328c9..601b08f92 100644 --- a/client-common/src/lib.rs +++ b/client-common/src/lib.rs @@ -3,6 +3,8 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. pub mod address; +pub mod multicast; +pub mod vni; /// Like `println!`, but silently exits on broken pipe (EPIPE) instead of /// panicking. Other I/O errors still panic. diff --git a/client-common/src/multicast.rs b/client-common/src/multicast.rs new file mode 100644 index 000000000..c2c16a10c --- /dev/null +++ b/client-common/src/multicast.rs @@ -0,0 +1,237 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Validated underlay multicast address shared across the routing suite. +//! +//! Lives in the cycle-free leaf crate so the API-types crates consumed by +//! Omicron can share a single definition without depending on +//! `omicron_common`, which would form a dependency cycle. + +// TODO: Consolidate these types into `oxnet`, the cycle-free leaf crate that +// maghemite, dendrite, and omicron already share, so the duplication can be +// removed. `dpd_types::mcast::UnderlayMulticastIpv6` in dendrite carries an +// independent copy today. + +use crate::address::UNDERLAY_MULTICAST_SUBNET; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::fmt; +use std::net::{IpAddr, Ipv6Addr}; +use std::str::FromStr; +use thiserror::Error; + +/// Error constructing an [`UnderlayMulticastIpv6`] address. +#[derive(Debug, Clone, Error)] +pub enum UnderlayMulticastError { + /// The address is not within the underlay multicast subnet (ff04::/64). + #[error( + "underlay address {addr} is not within {UNDERLAY_MULTICAST_SUBNET}" + )] + NotInSubnet { addr: Ipv6Addr }, + + /// The string could not be parsed as an IPv6 address. + #[error("invalid IPv6 address: {0}")] + InvalidIpv6(#[from] std::net::AddrParseError), +} + +/// Error constructing an [`OverlayMulticast`] address. +#[derive(Debug, Clone, Error)] +pub enum OverlayMulticastError { + /// The address is not a multicast address. + #[error("overlay address {addr} is not a multicast address")] + NotMulticast { addr: IpAddr }, + + /// The string could not be parsed as an IP address. + #[error("invalid IP address: {0}")] + InvalidIp(#[from] std::net::AddrParseError), +} + +/// A validated overlay multicast group address (IPv4 or IPv6). +/// +/// The application-visible group an operator announces via DDM, e.g. +/// `233.252.0.1` or `ff0e::1`. The overlay group spans both address +/// families, so this type wraps [`IpAddr`] and enforces only that the +/// address is multicast: IPv4 `224.0.0.0/4` per [RFC 1112 §4] or IPv6 +/// `ff00::/8` per [RFC 4291 §2.7]. Its admin-local underlay mapping is the +/// separately validated [`UnderlayMulticastIpv6`]. The mapping is defined by +/// [RFD 488]. +/// +/// [RFC 1112 §4]: https://www.rfc-editor.org/rfc/rfc1112#section-4 +/// [RFC 4291 §2.7]: https://www.rfc-editor.org/rfc/rfc4291#section-2.7 +/// [RFD 488]: https://rfd.shared.oxide.computer/rfd/488 +#[derive( + Debug, + Copy, + Clone, + Eq, + PartialEq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + JsonSchema, +)] +#[serde(try_from = "IpAddr", into = "IpAddr")] +#[schemars(transparent)] +pub struct OverlayMulticast(IpAddr); + +impl OverlayMulticast { + /// Create a new validated overlay multicast address. + /// + /// # Errors + /// + /// Returns [`OverlayMulticastError::NotMulticast`] if the address is not + /// a multicast address. + pub fn new(value: IpAddr) -> Result { + if !value.is_multicast() { + return Err(OverlayMulticastError::NotMulticast { addr: value }); + } + Ok(Self(value)) + } + + /// Return the underlying IP address. + #[inline] + pub const fn ip(&self) -> IpAddr { + self.0 + } +} + +impl fmt::Display for OverlayMulticast { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl TryFrom for OverlayMulticast { + type Error = OverlayMulticastError; + + fn try_from(value: IpAddr) -> Result { + Self::new(value) + } +} + +impl From for IpAddr { + fn from(addr: OverlayMulticast) -> Self { + addr.0 + } +} + +impl FromStr for OverlayMulticast { + type Err = OverlayMulticastError; + + fn from_str(s: &str) -> Result { + let addr: IpAddr = s.parse()?; + Self::new(addr) + } +} + +/// A validated underlay multicast IPv6 address within `ff04::/64`. +/// +/// The Oxide rack maps overlay multicast groups 1:1 to admin-local scoped +/// IPv6 multicast addresses in `UNDERLAY_MULTICAST_SUBNET` (`ff04::/64`, +/// admin-local scope per [RFC 4291 §2.7]). The mapping is defined by +/// [RFD 488]. This type enforces the subnet invariant at construction +/// time. +/// +/// [RFC 4291 §2.7]: https://www.rfc-editor.org/rfc/rfc4291#section-2.7 +/// [RFD 488]: https://rfd.shared.oxide.computer/rfd/488 +#[derive( + Debug, + Copy, + Clone, + Eq, + PartialEq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + JsonSchema, +)] +#[serde(try_from = "Ipv6Addr", into = "Ipv6Addr")] +#[schemars(transparent)] +pub struct UnderlayMulticastIpv6(Ipv6Addr); + +impl UnderlayMulticastIpv6 { + /// Create a new validated underlay multicast address. + /// + /// # Errors + /// + /// Returns [`UnderlayMulticastError::NotInSubnet`] if the address is + /// not within `UNDERLAY_MULTICAST_SUBNET` (ff04::/64). + pub fn new(value: Ipv6Addr) -> Result { + if !UNDERLAY_MULTICAST_SUBNET.contains(value) { + return Err(UnderlayMulticastError::NotInSubnet { addr: value }); + } + Ok(Self(value)) + } + + /// Return the underlying IPv6 address. + #[inline] + pub const fn ip(&self) -> Ipv6Addr { + self.0 + } +} + +impl fmt::Display for UnderlayMulticastIpv6 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl TryFrom for UnderlayMulticastIpv6 { + type Error = UnderlayMulticastError; + + fn try_from(value: Ipv6Addr) -> Result { + Self::new(value) + } +} + +impl From for Ipv6Addr { + fn from(addr: UnderlayMulticastIpv6) -> Self { + addr.0 + } +} + +impl From for IpAddr { + fn from(addr: UnderlayMulticastIpv6) -> Self { + IpAddr::V6(addr.0) + } +} + +impl FromStr for UnderlayMulticastIpv6 { + type Err = UnderlayMulticastError; + + fn from_str(s: &str) -> Result { + let addr: Ipv6Addr = s.parse()?; + Self::new(addr) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn overlay_serde_rejects_unicast() { + let json = + serde_json::to_string(&"192.0.2.1".parse::().unwrap()) + .unwrap(); + let result: Result = serde_json::from_str(&json); + assert!(result.is_err()); + } + + #[test] + fn underlay_serde_rejects_invalid() { + // ff0e::1 serialized as an Ipv6Addr, then deserialized as + // UnderlayMulticastIpv6 should fail via try_from. + let json = + serde_json::to_string(&Ipv6Addr::new(0xff0e, 0, 0, 0, 0, 0, 0, 1)) + .unwrap(); + let result: Result = + serde_json::from_str(&json); + assert!(result.is_err()); + } +} diff --git a/client-common/src/vni.rs b/client-common/src/vni.rs new file mode 100644 index 000000000..1c515b645 --- /dev/null +++ b/client-common/src/vni.rs @@ -0,0 +1,141 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Geneve Virtual Network Identifier (VNI). +//! +//! Lives in the cycle-free leaf crate so the API-types crates consumed by +//! Omicron can share a single definition without depending on +//! `omicron_common`, which would form a dependency cycle. + +// TODO: `omicron_common::api::external::Vni` and `oxide_vpc::api::Vni` carry +// independent copies, forcing consumers to assert their constants stay equal. +// Consolidate into `oxnet`, the cycle-free leaf crate that maghemite, +// dendrite, and omicron already share, so the duplication can be removed. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::fmt::{self, Formatter}; + +/// Error raised while validating a [`Vni`]. +#[derive(thiserror::Error, Debug)] +pub enum VniError { + /// The value exceeds the 24-bit Geneve maximum. + #[error("VNI {value} exceeds the maximum 24-bit value {}", Vni::MAX_VNI)] + OutOfRange { value: u32 }, +} + +/// A validated Geneve Virtual Network Identifier. +/// +/// Wraps a 24-bit VNI, rejecting any value above [`Vni::MAX_VNI`] at +/// construction and deserialization so an out-of-range identifier is +/// unrepresentable. +#[derive( + Debug, + Copy, + Clone, + Eq, + PartialEq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + JsonSchema, +)] +#[serde(try_from = "u32", into = "u32")] +#[schemars(transparent)] +pub struct Vni(u32); + +impl Vni { + /// Maximum Geneve VNI value. + /// + /// Virtual Network Identifiers are constrained to 24-bit values per the + /// Geneve specification (RFC 8926 Section 3.3). + pub const MAX_VNI: u32 = 0xFF_FFFF; + + /// Default VNI for fleet-wide multicast routing. + /// + /// A low-numbered VNI chosen to avoid colliding with user VNIs, though + /// it is not yet within the Oxide-reserved range. + pub const DEFAULT_MULTICAST: Self = Self(77); + + /// Create a validated VNI. + /// + /// # Errors + /// + /// Returns [`VniError::OutOfRange`] if `value` exceeds [`Vni::MAX_VNI`], + /// the largest 24-bit Geneve VNI. + /// + /// # Examples + /// + /// ``` + /// use client_common::vni::Vni; + /// + /// assert!(Vni::new(77).is_ok()); + /// assert!(Vni::new(Vni::MAX_VNI + 1).is_err()); + /// ``` + pub fn new(value: u32) -> Result { + if value > Self::MAX_VNI { + return Err(VniError::OutOfRange { value }); + } + Ok(Self(value)) + } + + /// Return the underlying 24-bit value. + #[inline] + pub const fn as_u32(self) -> u32 { + self.0 + } +} + +impl fmt::Display for Vni { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl TryFrom for Vni { + type Error = VniError; + + fn try_from(value: u32) -> Result { + Self::new(value) + } +} + +impl From for u32 { + fn from(vni: Vni) -> Self { + vni.0 + } +} + +#[cfg(test)] +mod tests { + use omicron_common::api::external::Vni as CanonicalVni; + + use super::*; + + /// Assert the locally copied VNI literals equal their + /// `omicron_common::api::external::Vni` originals so they cannot drift. + /// + /// `omicron_common` is a dev-dependency only, so it does not appear in the + /// normal dependency tree the no-omicron CI check inspects. + #[test] + fn vni_constants_match_canonical_values() { + assert_eq!(Vni::MAX_VNI, CanonicalVni::MAX_VNI); + assert_eq!( + Vni::DEFAULT_MULTICAST.as_u32(), + CanonicalVni::DEFAULT_MULTICAST_VNI.as_u32() + ); + } + + /// The [`Vni`] newtype accepts in-range values and rejects values above + /// [`Vni::MAX_VNI`], enforcing the 24-bit invariant at construction. + #[test] + fn vni_rejects_out_of_range() { + assert_eq!(Vni::new(0).unwrap().as_u32(), 0); + assert_eq!(Vni::new(Vni::MAX_VNI).unwrap().as_u32(), Vni::MAX_VNI); + assert!(Vni::new(Vni::MAX_VNI + 1).is_err()); + assert!(Vni::new(u32::MAX).is_err()); + } +} diff --git a/ddm-admin-client/src/lib.rs b/ddm-admin-client/src/lib.rs index 77988121c..7159b21ec 100644 --- a/ddm-admin-client/src/lib.rs +++ b/ddm-admin-client/src/lib.rs @@ -20,6 +20,12 @@ progenitor::generate_api!( }), replace = { TunnelOrigin = ddm_api_types_versions::latest::net::TunnelOrigin, + MulticastOrigin = ddm_api_types_versions::latest::net::MulticastOrigin, + UnderlayMulticastIpv6 = ddm_api_types_versions::latest::net::UnderlayMulticastIpv6, + Vni = ddm_api_types_versions::latest::net::Vni, + MulticastRoute = ddm_api_types_versions::latest::db::MulticastRoute, + MulticastPathHop = ddm_api_types_versions::latest::exchange::MulticastPathHop, + MulticastPathVector = ddm_api_types_versions::latest::exchange::MulticastPathVector, PeerInfo = ddm_api_types_versions::latest::db::PeerInfo, PeerStatus = ddm_api_types_versions::latest::db::PeerStatus, Duration = std::time::Duration, diff --git a/ddm-api-types/versions/Cargo.toml b/ddm-api-types/versions/Cargo.toml index eac3d8d3f..9ff5e7ddd 100644 --- a/ddm-api-types/versions/Cargo.toml +++ b/ddm-api-types/versions/Cargo.toml @@ -4,9 +4,12 @@ version = "0.1.0" edition = "2024" [dependencies] +client-common.workspace = true +ddm-protocol.workspace = true oxnet.workspace = true schemars.workspace = true serde.workspace = true +serde_json.workspace = true serde_repr.workspace = true +thiserror.workspace = true uuid.workspace = true -ddm-protocol.workspace = true diff --git a/ddm-api-types/versions/src/latest.rs b/ddm-api-types/versions/src/latest.rs index a2c1b47f0..0eaade4fe 100644 --- a/ddm-api-types/versions/src/latest.rs +++ b/ddm-api-types/versions/src/latest.rs @@ -13,15 +13,21 @@ pub mod admin { pub mod db { pub use crate::v1::db::RouterKind; pub use crate::v1::db::TunnelRoute; - - pub use crate::v2::db::PeerInfo; pub use crate::v2::db::PeerStatus; + pub use crate::v3::db::MulticastRoute; + pub use crate::v3::db::PeerInfo; } pub mod exchange { pub use crate::v1::exchange::PathVector; + pub use crate::v3::exchange::MulticastPathHop; + pub use crate::v3::exchange::MulticastPathVector; } pub mod net { pub use crate::v1::net::TunnelOrigin; + pub use crate::v3::net::MulticastOrigin; + pub use crate::v3::net::OverlayMulticast; + pub use crate::v3::net::UnderlayMulticastIpv6; + pub use crate::v3::net::Vni; } diff --git a/ddm-api-types/versions/src/lib.rs b/ddm-api-types/versions/src/lib.rs index 7d88b9274..bd9be63b5 100644 --- a/ddm-api-types/versions/src/lib.rs +++ b/ddm-api-types/versions/src/lib.rs @@ -34,3 +34,5 @@ pub mod latest; pub mod v1; #[path = "peer_durations/mod.rs"] pub mod v2; +#[path = "multicast_support/mod.rs"] +pub mod v3; diff --git a/ddm-api-types/versions/src/multicast_support/db.rs b/ddm-api-types/versions/src/multicast_support/db.rs new file mode 100644 index 000000000..4f21e3b39 --- /dev/null +++ b/ddm-api-types/versions/src/multicast_support/db.rs @@ -0,0 +1,161 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Database types changed in API version 3 (MULTICAST_SUPPORT). +//! +//! Adds `MulticastRoute` for routes learned via DDM and extends +//! `PeerInfo` with an optional discovery interface name. + +use std::net::Ipv6Addr; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use ddm_protocol::v4::MulticastPathHop; + +use super::net::MulticastOrigin; +use crate::v1::db::RouterKind; +use crate::v2::db::PeerStatus; + +/// A multicast route learned via DDM. +/// +/// Carries a `MulticastOrigin` (overlay group + ff04::/64 underlay +/// mapping) and the path vector from the originating subscriber +/// through intermediate transit routers. +// The path enables loop detection and (in multi-rack topologies) +// replication optimizations (RFD 488) in the future. +// +// Equality and hashing consider only `origin` and `nexthop`, so the path +// is not part of a route's identity. Updating a stored route's path +// therefore requires replacing the existing entry (e.g. `HashSet::replace`). +// `HashSet::insert` leaves the stored path unchanged. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct MulticastRoute { + /// The multicast group origin information. + pub origin: MulticastOrigin, + + /// Underlay nexthop address (DDM peer that advertised this route). + /// Used to associate the route with a peer for expiration. + pub nexthop: Ipv6Addr, + + /// Path vector from the originating subscriber outward. + /// Each hop records the router that redistributed this + /// subscription announcement. Used for loop detection on pull + /// and for future replication optimization in multi-rack + /// topologies. + #[serde(default)] + pub path: Vec, +} + +impl MulticastRoute { + /// Identity used for equality and hashing: which group, from which peer. + /// Excludes `path`, so equality and hashing both key on the same fields + /// and cannot drift as the struct grows. + fn identity(&self) -> (&MulticastOrigin, &Ipv6Addr) { + (&self.origin, &self.nexthop) + } +} + +impl PartialEq for MulticastRoute { + fn eq(&self, other: &Self) -> bool { + self.identity() == other.identity() + } +} + +impl Eq for MulticastRoute {} + +impl std::hash::Hash for MulticastRoute { + fn hash(&self, state: &mut H) { + self.identity().hash(state); + } +} + +impl From for MulticastOrigin { + fn from(x: MulticastRoute) -> Self { + x.origin + } +} + +/// Peer information with an optional interface name. +/// +// Adds the `if_name` field to identify which underlay interface the peer +// was discovered on. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)] +pub struct PeerInfo { + pub status: PeerStatus, + pub addr: Ipv6Addr, + pub host: String, + pub kind: RouterKind, + /// Interface name the peer was discovered on (e.g., "tfportrear0_0"). + #[serde(default)] + pub if_name: Option, +} + +/// Downconvert v3 `PeerInfo` to v2 `PeerInfo` by dropping `if_name`. +impl From for crate::v2::db::PeerInfo { + fn from(p: PeerInfo) -> Self { + Self { + status: p.status, + addr: p.addr, + host: p.host, + kind: p.kind, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn origin(overlay: &str) -> MulticastOrigin { + serde_json::from_value(serde_json::json!({ + "overlay_group": overlay, + "underlay_group": "ff04::1", + "vni": 77, + "metric": 0 + })) + .unwrap() + } + + // The path is excluded from a route's identity, so two routes sharing an + // origin and nexthop but carrying different paths are equal. + #[test] + fn route_identity_excludes_path() { + let base = MulticastRoute { + origin: origin("233.252.0.1"), + nexthop: Ipv6Addr::LOCALHOST, + path: vec![MulticastPathHop::new( + "router-1".into(), + Ipv6Addr::LOCALHOST, + )], + }; + let mut other = base.clone(); + other.path = vec![ + MulticastPathHop::new("router-1".into(), Ipv6Addr::LOCALHOST), + MulticastPathHop::new("router-2".into(), Ipv6Addr::LOCALHOST), + ]; + assert_eq!(base, other); + } + + #[test] + fn route_identity_keys_on_origin_and_nexthop() { + let base = MulticastRoute { + origin: origin("233.252.0.1"), + nexthop: Ipv6Addr::LOCALHOST, + path: vec![], + }; + let other_origin = MulticastRoute { + origin: origin("233.252.0.2"), + nexthop: Ipv6Addr::LOCALHOST, + path: vec![], + }; + let other_nexthop = MulticastRoute { + origin: origin("233.252.0.1"), + nexthop: Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1), + path: vec![], + }; + assert_ne!(base, other_origin); + assert_ne!(base, other_nexthop); + } +} diff --git a/ddm-api-types/versions/src/multicast_support/exchange.rs b/ddm-api-types/versions/src/multicast_support/exchange.rs new file mode 100644 index 000000000..59359b8e6 --- /dev/null +++ b/ddm-api-types/versions/src/multicast_support/exchange.rs @@ -0,0 +1,14 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Exchange (wire) types added in API version 3 (MULTICAST_SUPPORT), +//! which carries the `ddm_protocol::v4` wire types. +//! +//! These are re-exports of the plain wire types defined in the +//! [`ddm_protocol`] crate, mirroring how `PathVector` and `TunnelOrigin` +//! re-export their `ddm_protocol::v3` counterparts. Keeping a single +//! definition avoids a rich/wire split for path vectors, and peer-supplied +//! routes are stored in this plain form without re-validation. + +pub use ddm_protocol::v4::{MulticastPathHop, MulticastPathVector}; diff --git a/ddm-api-types/versions/src/multicast_support/mod.rs b/ddm-api-types/versions/src/multicast_support/mod.rs new file mode 100644 index 000000000..bb95299fb --- /dev/null +++ b/ddm-api-types/versions/src/multicast_support/mod.rs @@ -0,0 +1,12 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Types added in API version 3 (MULTICAST_SUPPORT). +//! +//! Adds multicast group origination, route distribution, and per-peer +//! discovery interface name tracking. + +pub mod db; +pub mod exchange; +pub mod net; diff --git a/ddm-api-types/versions/src/multicast_support/net.rs b/ddm-api-types/versions/src/multicast_support/net.rs new file mode 100644 index 000000000..aa0820be6 --- /dev/null +++ b/ddm-api-types/versions/src/multicast_support/net.rs @@ -0,0 +1,227 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Multicast origin and validated underlay address types added in +//! API version 3 (MULTICAST_SUPPORT). + +pub use client_common::multicast::{ + OverlayMulticast, OverlayMulticastError, UnderlayMulticastError, + UnderlayMulticastIpv6, +}; +pub use client_common::vni::{Vni, VniError}; +use ddm_protocol::v4; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::net::IpAddr; + +/// Error promoting a wire [`v4::MulticastOrigin`] to the validated +/// [`MulticastOrigin`]. +#[derive(Debug, thiserror::Error)] +pub enum MulticastOriginError { + /// The overlay group is not a multicast address. + #[error(transparent)] + OverlayGroup(#[from] OverlayMulticastError), + + /// The underlay group is not within ff04::/64. + #[error(transparent)] + UnderlayGroup(#[from] UnderlayMulticastError), + + /// The VNI exceeds the 24-bit Geneve maximum. + #[error(transparent)] + Vni(#[from] VniError), +} + +fn default_multicast_vni() -> Vni { + Vni::DEFAULT_MULTICAST +} + +/// Origin information for a multicast group announcement. +/// +/// Analogous to `TunnelOrigin` but for multicast groups. Represents a +/// subscription to a multicast group that should be advertised via DDM. +/// `overlay_group` is the application-visible multicast address (e.g., +/// 233.252.0.1 or ff0e::1), while `underlay_group` is the mapped +/// admin-local scoped IPv6 address (ff04::X) used in the underlay network. +#[derive(Debug, Clone, Eq, Serialize, Deserialize, JsonSchema)] +pub struct MulticastOrigin { + /// The overlay multicast group address (IPv4 or IPv6). + /// This is the group address visible to applications. + /// Validated at construction to be a multicast address. + pub overlay_group: OverlayMulticast, + + /// The underlay multicast group address (ff04::X). + /// Validated at construction to be within ff04::/64. + pub underlay_group: UnderlayMulticastIpv6, + + /// VNI for this multicast group (identifies the VPC/network context). + #[serde(default = "default_multicast_vni")] + pub vni: Vni, + + /// Metric for path selection (lower is better). + /// + /// Used for multi-rack replication optimization. + /// Excluded from identity (Hash/Eq) so that metric changes update an + /// existing entry rather than creating a duplicate. + #[serde(default)] + pub metric: u64, + + /// Optional source address for Source-Specific Multicast (S,G) routes. + /// `None` for Any-Source Multicast (*,G) routes. + #[serde(default)] + pub source: Option, +} + +impl MulticastOrigin { + /// Identity used for equality and hashing: the group, its underlay + /// mapping, VNI, and source. Excludes `metric`, a mutable path-selection + /// attribute, so a metric change updates an existing entry rather than + /// creating a duplicate. Routing both `PartialEq` and `Hash` through this + /// accessor keeps the field set defined once so the two cannot drift. + /// + /// This type is not used in ordered collections (BTreeSet). See #649 for + /// why adding `Ord` here would require more care. + fn identity( + &self, + ) -> ( + &OverlayMulticast, + &UnderlayMulticastIpv6, + &Vni, + &Option, + ) { + ( + &self.overlay_group, + &self.underlay_group, + &self.vni, + &self.source, + ) + } + + /// Return a stable string key for this origin's identity. + /// + /// Serializes only the identity fields, matching `PartialEq`/`Hash`, so a + /// keyed store overwrites the entry for an origin whose `metric` changed + /// rather than leaving a stale entry under the prior metric. Deriving the + /// key from [`MulticastOrigin::identity`] keeps it from drifting from + /// equality. + /// + /// # Errors + /// + /// Returns an error if the identity fields fail to serialize. + pub fn identity_key(&self) -> Result { + serde_json::to_string(&self.identity()) + } +} + +impl PartialEq for MulticastOrigin { + fn eq(&self, other: &Self) -> bool { + self.identity() == other.identity() + } +} + +impl std::hash::Hash for MulticastOrigin { + fn hash(&self, state: &mut H) { + self.identity().hash(state); + } +} + +// Locally-originated groups are operator-supplied and validated at the admin +// boundary. Before propagation on the wire the validated form is lowered to the +// plain `v4::MulticastOrigin`. The reverse promotion is deliberately absent: +// peer-supplied routes are stored in their plain form, +// trusted like tunnel and underlay routes. The one exception is the underlay +// group, which reaches DPD directly, so the exchange handler enforces its +// ff04::/64 invariant at import. +impl From<&MulticastOrigin> for v4::MulticastOrigin { + fn from(o: &MulticastOrigin) -> Self { + Self { + overlay_group: o.overlay_group.ip(), + underlay_group: o.underlay_group.ip(), + vni: o.vni.as_u32(), + metric: o.metric, + source: o.source, + } + } +} + +impl From for v4::MulticastOrigin { + fn from(o: MulticastOrigin) -> Self { + Self::from(&o) + } +} + +// Promote a peer-supplied wire origin to the validated form. Peer routes +// arrive as the plain `v4::MulticastOrigin`. Promotion enforces the invariants +// the rich type guarantees, the overlay group being multicast, the underlay +// group within ff04::/64, and the VNI within the 24-bit range, so an invalid +// origin is rejected at the exchange boundary before it can be stored or reach +// DPD. +impl TryFrom<&v4::MulticastOrigin> for MulticastOrigin { + type Error = MulticastOriginError; + + fn try_from(o: &v4::MulticastOrigin) -> Result { + Ok(Self { + overlay_group: OverlayMulticast::new(o.overlay_group)?, + underlay_group: UnderlayMulticastIpv6::new(o.underlay_group)?, + vni: Vni::new(o.vni)?, + metric: o.metric, + source: o.source, + }) + } +} + +impl TryFrom for MulticastOrigin { + type Error = MulticastOriginError; + + fn try_from(o: v4::MulticastOrigin) -> Result { + Self::try_from(&o) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::Ipv6Addr; + + #[test] + fn multicast_origin_rejects_bad_underlay() { + let json = serde_json::json!({ + "overlay_group": "233.252.0.1", + "underlay_group": "ff0e::1", + "vni": 77 + }); + let result: Result = serde_json::from_value(json); + assert!(result.is_err()); + } + + #[test] + fn multicast_origin_accepts_valid() { + let json = serde_json::json!({ + "overlay_group": "233.252.0.1", + "underlay_group": "ff04::1", + "vni": 77 + }); + let origin: MulticastOrigin = serde_json::from_value(json).unwrap(); + assert_eq!( + origin.underlay_group.ip(), + Ipv6Addr::new(0xff04, 0, 0, 0, 0, 0, 0, 1), + ); + } + + // Metric is excluded from the identity so a metric-only change updates an + // existing entry rather than duplicating it, mirroring the wire-type + // contract in `v4::MulticastOrigin`. + #[test] + fn multicast_origin_identity_excludes_metric() { + let base: MulticastOrigin = serde_json::from_value(serde_json::json!({ + "overlay_group": "233.252.0.1", + "underlay_group": "ff04::1", + "vni": 77, + "metric": 0 + })) + .unwrap(); + let mut other = base.clone(); + other.metric = 100; + assert_eq!(base, other); + } +} diff --git a/ddm-api-types/versions/src/peer_durations/mod.rs b/ddm-api-types/versions/src/peer_durations/mod.rs index 7a9776a99..a60e9bd95 100644 --- a/ddm-api-types/versions/src/peer_durations/mod.rs +++ b/ddm-api-types/versions/src/peer_durations/mod.rs @@ -2,7 +2,8 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! Version `PEER_DURATIONS` of the DDM Admin API. +//! Types from API version 2 (PEER_DURATIONS) that changed in version 3 +//! (MULTICAST_SUPPORT). //! //! Tracks how long each DDM peer has been in its current state and exposes //! that through the `/peers` endpoint with duration information. diff --git a/ddm-api/src/lib.rs b/ddm-api/src/lib.rs index 623b8376b..1a77a257a 100644 --- a/ddm-api/src/lib.rs +++ b/ddm-api/src/lib.rs @@ -4,6 +4,7 @@ use ddm_api_types_versions::latest; use ddm_api_types_versions::v1; +use ddm_api_types_versions::v2; use dropshot::HttpError; use dropshot::HttpResponseOk; use dropshot::HttpResponseUpdatedNoContent; @@ -26,6 +27,7 @@ api_versions!([ // | example for the next person. // v // (next_int, IDENT), + (3, MULTICAST_SUPPORT), (2, PEER_DURATIONS), (1, INITIAL), ]); @@ -49,23 +51,45 @@ pub trait DdmAdminApi { #[endpoint { method = GET, path = "/peers", - versions = VERSION_PEER_DURATIONS.., + versions = VERSION_MULTICAST_SUPPORT.. }] async fn get_peers( ctx: RequestContext, ) -> Result>, HttpError>; + /// Returns peers without interface name information. #[endpoint { method = GET, path = "/peers", - versions = ..VERSION_PEER_DURATIONS, + versions = VERSION_PEER_DURATIONS..VERSION_MULTICAST_SUPPORT + }] + async fn get_peers_v2( + ctx: RequestContext, + ) -> Result>, HttpError> { + let resp = Self::get_peers(ctx).await?; + let converted: HashMap = + resp.0.into_iter().map(|(k, v)| (k, v.into())).collect(); + Ok(HttpResponseOk(converted)) + } + + /// Returns peers without per-state duration or interface name information. + #[endpoint { + method = GET, + path = "/peers", + versions = ..VERSION_PEER_DURATIONS }] async fn get_peers_v1( ctx: RequestContext, ) -> Result>, HttpError> { let resp = Self::get_peers(ctx).await?; - let converted: HashMap = - resp.0.into_iter().map(|(k, v)| (k, v.into())).collect(); + let converted: HashMap = resp + .0 + .into_iter() + .map(|(k, v)| { + let v2_info: v2::db::PeerInfo = v.into(); + (k, v2_info.into()) + }) + .collect(); Ok(HttpResponseOk(converted)) } @@ -146,10 +170,45 @@ pub trait DdmAdminApi { request: TypedBody>, ) -> Result; + #[endpoint { + method = GET, + path = "/originated_multicast_groups", + versions = VERSION_MULTICAST_SUPPORT.. + }] + async fn get_originated_multicast_groups( + ctx: RequestContext, + ) -> Result>, HttpError>; + + #[endpoint { + method = GET, + path = "/multicast_groups", + versions = VERSION_MULTICAST_SUPPORT.. + }] + async fn get_multicast_groups( + ctx: RequestContext, + ) -> Result>, HttpError>; + #[endpoint { method = PUT, - path = "/sync", + path = "/multicast_group", + versions = VERSION_MULTICAST_SUPPORT.. }] + async fn advertise_multicast_groups( + ctx: RequestContext, + request: TypedBody>, + ) -> Result; + + #[endpoint { + method = DELETE, + path = "/multicast_group", + versions = VERSION_MULTICAST_SUPPORT.. + }] + async fn withdraw_multicast_groups( + ctx: RequestContext, + request: TypedBody>, + ) -> Result; + + #[endpoint { method = PUT, path = "/sync" }] async fn sync( ctx: RequestContext, ) -> Result; diff --git a/ddm-protocol/src/lib.rs b/ddm-protocol/src/lib.rs index 7585766ec..8c5c4bdd3 100644 --- a/ddm-protocol/src/lib.rs +++ b/ddm-protocol/src/lib.rs @@ -8,6 +8,52 @@ use oxnet::{IpNet, Ipv4Net, Ipv6Net}; pub mod v2; pub mod v3; +pub mod v4; + +impl From for v4::Update { + fn from(value: v3::Update) -> Self { + Self { + underlay: value.underlay, + tunnel: value.tunnel, + // V3 has no multicast section. + multicast: None, + } + } +} + +impl From for v3::Update { + fn from(value: v4::Update) -> Self { + // The multicast section is not representable in the V3 wire form and is + // dropped here. A V3 peer would likewise ignore the unknown field. + Self { + underlay: value.underlay, + tunnel: value.tunnel, + } + } +} + +impl From for v4::PullResponse { + fn from(value: v3::PullResponse) -> Self { + Self { + underlay: value.underlay, + tunnel: value.tunnel, + multicast: None, + // Only the multicast section pages, and V3 has no multicast + // section, so an upconverted response is always a complete + // snapshot. + next_page_token: None, + } + } +} + +impl From for v3::PullResponse { + fn from(value: v4::PullResponse) -> Self { + Self { + underlay: value.underlay, + tunnel: value.tunnel, + } + } +} impl From for v3::Update { fn from(value: v2::Update) -> Self { diff --git a/ddm-protocol/src/v3.rs b/ddm-protocol/src/v3.rs index 80cfcb925..688b76ff2 100644 --- a/ddm-protocol/src/v3.rs +++ b/ddm-protocol/src/v3.rs @@ -20,8 +20,8 @@ pub struct Update { impl Update { /// Build an `Update` whose underlay/tunnel halves carry the announcements - /// from `pr`. Used by [`pull`] to project a pull response back into the - /// update event stream. + /// from the [`PullResponse`] `pr`. Used by [`pull`] to project a pull + /// response back into the update event stream. pub fn announce(pr: PullResponse) -> Self { Self { underlay: pr.underlay.map(UnderlayUpdate::announce), diff --git a/ddm-protocol/src/v4.rs b/ddm-protocol/src/v4.rs new file mode 100644 index 000000000..14637fc5a --- /dev/null +++ b/ddm-protocol/src/v4.rs @@ -0,0 +1,470 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! ALL TYPES IN THIS FILE ARE FOR DDM PROTOCOL VERSION 4. THEY SHALL NEVER +//! CHANGE. THESE TYPES CAN BE REMOVED WHEN DDMV4 CLIENTS AND SERVERS NO LONGER +//! EXIST BUT THEIR DEFINITIONS SHALL NEVER CHANGE. +//! +//! Version 4 extends version 3 with multicast group subscription propagation +//! (RFD 488). The underlay and tunnel sections are unchanged from version 3 +//! are reused directly. The multicast wire types are defined here as plain, +//! self-contained structures: this crate must stay free of `omicron-common`, +//! so the validated forms (`UnderlayMulticastIpv6`, `Vni`) used by the admin +//! and database layers are converted to and from these wire types at the +//! exchange boundary. + +use std::{ + collections::HashSet, + net::{IpAddr, Ipv6Addr}, +}; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::v3; + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct Update { + pub underlay: Option, + pub tunnel: Option, + pub multicast: Option, +} + +impl Update { + /// Build an `Update` whose sections carry the announcements from the + /// [`PullResponse`] `pr`. + pub fn announce(pr: PullResponse) -> Self { + Self { + underlay: pr.underlay.map(v3::UnderlayUpdate::announce), + tunnel: pr.tunnel.map(v3::TunnelUpdate::announce), + multicast: pr.multicast.map(MulticastUpdate::announce), + } + } +} + +impl From for Update { + fn from(u: v3::UnderlayUpdate) -> Self { + Update { + underlay: Some(u), + tunnel: None, + multicast: None, + } + } +} + +impl From for Update { + fn from(t: v3::TunnelUpdate) -> Self { + Update { + underlay: None, + tunnel: Some(t), + multicast: None, + } + } +} + +impl From for Update { + fn from(m: MulticastUpdate) -> Self { + Update { + underlay: None, + tunnel: None, + multicast: Some(m), + } + } +} + +/// One page of a peer's route snapshot. +/// +/// The underlay and tunnel sections are bounded by the fabric's prefix and +/// tunnel endpoint counts, so they are carried whole on the first page. The +/// multicast section has no such bound, since it grows with the product of +/// groups, VNIs, sources, and path length, and is therefore paged. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct PullResponse { + pub underlay: Option>, + pub tunnel: Option>, + pub multicast: Option>, + + /// Token for resuming multicast keyset pagination, or `None` on the final + /// page. + /// + /// A responder that never pages omits the field, so an unpaged snapshot + /// deserializes as a single final page. The token is produced and + /// interpreted by the same responder and is opaque to the reader, so its + /// encoding is not part of the negotiated protocol. + /// + /// Reading every page matters because a withdrawal is synthesized from + /// the difference between what the reader has imported from this peer and + /// what the snapshot announces. A partial snapshot would read as a + /// withdrawal of everything the unread pages hold. + #[serde(default)] + pub next_page_token: Option, +} + +/// Multicast group subscription updates. +/// +/// Each entry carries a [`MulticastPathVector`] with the group origin and the +/// path vector used for loop detection. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct MulticastUpdate { + pub announce: HashSet, + pub withdraw: HashSet, +} + +impl MulticastUpdate { + pub fn announce(groups: HashSet) -> Self { + Self { + announce: groups, + ..Default::default() + } + } + pub fn withdraw(groups: HashSet) -> Self { + Self { + withdraw: groups, + ..Default::default() + } + } + + /// Add a hop to all path vectors in this update. + pub fn with_hop(&self, hop: MulticastPathHop) -> Self { + Self { + announce: self + .announce + .iter() + .map(|pv| pv.with_hop(hop.clone())) + .collect(), + withdraw: self + .withdraw + .iter() + .map(|pv| pv.with_hop(hop.clone())) + .collect(), + } + } +} + +/// Wire form of a multicast group origin. +/// +/// The validated counterpart (`ddm_api_types::net::MulticastOrigin`) carries an +/// `UnderlayMulticastIpv6` and a `Vni`. As a frozen wire type this form stays +/// unvalidated, a plain `Ipv6Addr` for the underlay group and a plain `u32` for +/// the VNI. Validation happens when converting into that counterpart at the +/// exchange boundary. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +pub struct MulticastOrigin { + /// The overlay multicast group address (IPv4 or IPv6). + pub overlay_group: IpAddr, + + /// The underlay multicast group address (ff04::X on the wire). + pub underlay_group: Ipv6Addr, + + /// VNI identifying the VPC/network context for this group. + #[serde(default)] + pub vni: u32, + + /// Metric for path selection (lower is better). Excluded from identity so + /// that metric changes update an existing entry rather than duplicating it. + #[serde(default)] + pub metric: u64, + + /// Optional source address for Source-Specific Multicast (S,G) routes. + /// `None` for Any-Source Multicast (*,G) routes. + #[serde(default)] + pub source: Option, +} + +impl MulticastOrigin { + /// Identity used for equality and hashing: the group, its underlay mapping, + /// VNI, and source. Excludes `metric`, a mutable path-selection attribute, + /// matching the validated `MulticastOrigin`. Routing both `PartialEq` and + /// `Hash` through this accessor keeps the field set defined once so the two + /// cannot drift. + fn identity(&self) -> (&IpAddr, &Ipv6Addr, &u32, &Option) { + ( + &self.overlay_group, + &self.underlay_group, + &self.vni, + &self.source, + ) + } +} + +impl PartialEq for MulticastOrigin { + fn eq(&self, other: &Self) -> bool { + self.identity() == other.identity() + } +} + +impl Eq for MulticastOrigin {} + +impl std::hash::Hash for MulticastOrigin { + fn hash(&self, state: &mut H) { + self.identity().hash(state); + } +} + +/// Total order over group identities, used for multicast keyset pagination. +/// +/// Ordering by identity rather than by position is what makes keyset +/// pagination safe against concurrent change. A cursor names the last group +/// read, so a group present for the whole scan is read exactly once even if +/// groups are added or removed around it. An index-based cursor would let an +/// insertion shift a later group backwards past the cursor, and the reader +/// would treat the group it never saw as withdrawn. +/// +/// Ordering agrees with [`PartialEq`], comparing the same identity that +/// equality and hashing use, so distinct entries never compare equal and a +/// cursor cannot skip one. +impl Ord for MulticastOrigin { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.identity().cmp(&other.identity()) + } +} + +impl PartialOrd for MulticastOrigin { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +/// A single hop in the multicast path, carrying metadata for replication +/// optimization (RFD 488). +#[derive( + Debug, Clone, PartialEq, Eq, Hash, Deserialize, Serialize, JsonSchema, +)] +pub struct MulticastPathHop { + /// Router identifier (hostname). + pub router_id: String, + + /// The underlay address of this router (for replication targeting). + pub underlay_addr: Ipv6Addr, + + /// Number of downstream subscribers reachable via this hop. + #[serde(default)] + pub downstream_subscriber_count: u32, +} + +impl MulticastPathHop { + /// Create a hop with the given router identity and a zero subscriber count. + pub fn new(router_id: String, underlay_addr: Ipv6Addr) -> Self { + Self { + router_id, + underlay_addr, + downstream_subscriber_count: 0, + } + } +} + +/// Multicast group subscription announcement propagating through DDM. +#[derive( + Debug, Clone, PartialEq, Eq, Hash, Deserialize, Serialize, JsonSchema, +)] +pub struct MulticastPathVector { + /// The multicast group origin information. + pub origin: MulticastOrigin, + + /// The path from the original subscriber to the current router, ordered + /// from subscriber outward (subscriber router first). + pub path: Vec, +} + +impl MulticastPathVector { + /// Append a hop to this path vector. + pub fn with_hop(&self, hop: MulticastPathHop) -> Self { + let mut path = self.path.clone(); + path.push(hop); + Self { + origin: self.origin.clone(), + path, + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + // Write out the JSON schema for the DDMv4 protocol to a file for + // validation. This should not change. + #[test] + fn test_ddm_v4_protocol() { + #[derive(JsonSchema)] + #[allow(dead_code)] + struct Protocol { + update: Update, + pull_response: PullResponse, + } + + let schema = schemars::schema_for!(Protocol); + expectorate::assert_contents( + "tests/output/ddm_v4_protocol.json", + &serde_json::to_string_pretty(&schema).unwrap(), + ); + } + + fn mcast_origin(overlay: &str, underlay: &str) -> MulticastOrigin { + MulticastOrigin { + overlay_group: overlay.parse().unwrap(), + underlay_group: underlay.parse().unwrap(), + vni: 77, + metric: 0, + source: None, + } + } + + fn multicast_update() -> MulticastUpdate { + let pv = MulticastPathVector { + origin: mcast_origin("233.252.0.1", "ff04::1"), + path: vec![MulticastPathHop::new( + "router-1".into(), + Ipv6Addr::LOCALHOST, + )], + }; + MulticastUpdate::announce([pv].into_iter().collect()) + } + + #[test] + fn v4_update_round_trips() { + let update = Update { + underlay: None, + tunnel: None, + multicast: Some(multicast_update()), + }; + let json = serde_json::to_string(&update).unwrap(); + let back: Update = serde_json::from_str(&json).unwrap(); + assert!(back.multicast.is_some()); + assert_eq!(back.multicast.unwrap().announce.len(), 1); + } + + #[test] + fn v4_update_deserializes_as_v3_drops_multicast() { + let update = Update { + underlay: None, + tunnel: None, + multicast: Some(multicast_update()), + }; + let json = serde_json::to_string(&update).unwrap(); + // A v3 peer deserializes this as a v3 update, silently dropping the + // unknown multicast field. + let v3: v3::Update = serde_json::from_str(&json).unwrap(); + assert!(v3.underlay.is_none()); + assert!(v3.tunnel.is_none()); + } + + // A v4 node reading a populated v3 update keeps the underlay and tunnel + // sections and defaults the absent multicast section to None. + #[test] + fn populated_v3_update_deserializes_as_v4() { + let v3 = v3::Update { + underlay: Some(underlay_update()), + tunnel: Some(tunnel_update()), + }; + let json = serde_json::to_string(&v3).unwrap(); + let update: Update = serde_json::from_str(&json).unwrap(); + assert!(update.underlay.is_some()); + assert!(update.tunnel.is_some()); + assert!(update.multicast.is_none()); + } + + #[test] + fn v4_pull_response_round_trips() { + let pv = MulticastPathVector { + origin: mcast_origin("ff0e::1", "ff04::2"), + path: vec![], + }; + let resp = PullResponse { + underlay: None, + tunnel: None, + multicast: Some([pv].into_iter().collect()), + next_page_token: None, + }; + let json = serde_json::to_string(&resp).unwrap(); + let back: PullResponse = serde_json::from_str(&json).unwrap(); + assert!(back.multicast.is_some()); + } + + #[test] + fn v4_pull_response_deserializes_as_v3() { + let pv = MulticastPathVector { + origin: mcast_origin("233.252.0.1", "ff04::1"), + path: vec![], + }; + let resp = PullResponse { + underlay: None, + tunnel: None, + multicast: Some([pv].into_iter().collect()), + next_page_token: None, + }; + let json = serde_json::to_string(&resp).unwrap(); + // A v3 peer drops the multicast field. + let v3: v3::PullResponse = serde_json::from_str(&json).unwrap(); + assert!(v3.underlay.is_none()); + assert!(v3.tunnel.is_none()); + } + + #[test] + fn from_conversions_strip_multicast() { + let update = Update { + underlay: None, + tunnel: None, + multicast: Some(multicast_update()), + }; + // Downconvert to v3 for an older peer, then back. Multicast has no v3 + // representation, so the round trip drops it. + let v3 = v3::Update::from(update); + let back = Update::from(v3); + assert!(back.multicast.is_none()); + } + + fn underlay_update() -> v3::UnderlayUpdate { + let pv = v3::PathVector { + destination: "fd00::/64".parse().unwrap(), + path: vec!["router-1".into()], + }; + v3::UnderlayUpdate::announce([pv].into_iter().collect()) + } + + fn tunnel_update() -> v3::TunnelUpdate { + let origin = v3::TunnelOrigin { + overlay_prefix: "10.0.0.0/24".parse().unwrap(), + boundary_addr: Ipv6Addr::LOCALHOST, + vni: 77, + metric: 0, + }; + v3::TunnelUpdate::announce([origin].into_iter().collect()) + } + + // A v4 update carrying all three sections must keep its underlay and + // tunnel sections intact when downconverted for older peers, while the + // multicast section (which has no v3 or v2 wire form) is dropped. + #[test] + fn mixed_update_down_conversion_preserves_underlay_and_tunnel() { + let update = Update { + underlay: Some(underlay_update()), + tunnel: Some(tunnel_update()), + multicast: Some(multicast_update()), + }; + + let v3 = v3::Update::from(update.clone()); + assert!(v3.underlay.is_some()); + assert!(v3.tunnel.is_some()); + assert_eq!(v3.underlay.as_ref().unwrap().announce.len(), 1); + assert_eq!(v3.tunnel.as_ref().unwrap().announce.len(), 1); + + let v2 = crate::v2::Update::from(v3); + assert!(v2.underlay.is_some()); + assert!(v2.tunnel.is_some()); + assert_eq!(v2.underlay.unwrap().announce.len(), 1); + assert_eq!(v2.tunnel.unwrap().announce.len(), 1); + } + + // Metric is excluded from the wire identity so a metric-only change updates + // an existing entry rather than duplicating it inside a HashSet. + #[test] + fn multicast_origin_identity_excludes_metric() { + let mut a = mcast_origin("233.252.0.1", "ff04::1"); + let mut b = a.clone(); + a.metric = 0; + b.metric = 100; + assert_eq!(a, b); + } +} diff --git a/ddm-protocol/tests/output/ddm_v4_protocol.json b/ddm-protocol/tests/output/ddm_v4_protocol.json new file mode 100644 index 000000000..8e301211f --- /dev/null +++ b/ddm-protocol/tests/output/ddm_v4_protocol.json @@ -0,0 +1,361 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Protocol", + "type": "object", + "required": [ + "pull_response", + "update" + ], + "properties": { + "pull_response": { + "$ref": "#/definitions/PullResponse" + }, + "update": { + "$ref": "#/definitions/Update" + } + }, + "definitions": { + "IpNet": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/definitions/Ipv4Net" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/definitions/Ipv6Net" + } + ] + } + ], + "x-rust-type": { + "crate": "oxnet", + "path": "oxnet::IpNet", + "version": "0.1.0" + } + }, + "Ipv4Net": { + "title": "An IPv4 subnet", + "description": "An IPv4 subnet, including prefix and prefix length", + "examples": [ + "192.168.1.0/24" + ], + "type": "string", + "pattern": "^(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])/([0-9]|1[0-9]|2[0-9]|3[0-2])$", + "x-rust-type": { + "crate": "oxnet", + "path": "oxnet::Ipv4Net", + "version": "0.1.0" + } + }, + "Ipv6Net": { + "title": "An IPv6 subnet", + "description": "An IPv6 subnet, including prefix and subnet mask", + "examples": [ + "fd12:3456::/64" + ], + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$", + "x-rust-type": { + "crate": "oxnet", + "path": "oxnet::Ipv6Net", + "version": "0.1.0" + } + }, + "MulticastOrigin": { + "description": "Wire form of a multicast group origin.\n\nThe validated counterpart (`ddm_api_types::net::MulticastOrigin`) carries an `UnderlayMulticastIpv6` and a `Vni`. As a frozen wire type this form stays unvalidated, a plain `Ipv6Addr` for the underlay group and a plain `u32` for the VNI. Validation happens when converting into that counterpart at the exchange boundary.", + "type": "object", + "required": [ + "overlay_group", + "underlay_group" + ], + "properties": { + "metric": { + "description": "Metric for path selection (lower is better). Excluded from identity so that metric changes update an existing entry rather than duplicating it.", + "default": 0, + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "overlay_group": { + "description": "The overlay multicast group address (IPv4 or IPv6).", + "type": "string", + "format": "ip" + }, + "source": { + "description": "Optional source address for Source-Specific Multicast (S,G) routes. `None` for Any-Source Multicast (*,G) routes.", + "default": null, + "type": [ + "string", + "null" + ], + "format": "ip" + }, + "underlay_group": { + "description": "The underlay multicast group address (ff04::X on the wire).", + "type": "string", + "format": "ipv6" + }, + "vni": { + "description": "VNI identifying the VPC/network context for this group.", + "default": 0, + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + "MulticastPathHop": { + "description": "A single hop in the multicast path, carrying metadata for replication optimization (RFD 488).", + "type": "object", + "required": [ + "router_id", + "underlay_addr" + ], + "properties": { + "downstream_subscriber_count": { + "description": "Number of downstream subscribers reachable via this hop.", + "default": 0, + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "router_id": { + "description": "Router identifier (hostname).", + "type": "string" + }, + "underlay_addr": { + "description": "The underlay address of this router (for replication targeting).", + "type": "string", + "format": "ipv6" + } + } + }, + "MulticastPathVector": { + "description": "Multicast group subscription announcement propagating through DDM.", + "type": "object", + "required": [ + "origin", + "path" + ], + "properties": { + "origin": { + "description": "The multicast group origin information.", + "allOf": [ + { + "$ref": "#/definitions/MulticastOrigin" + } + ] + }, + "path": { + "description": "The path from the original subscriber to the current router, ordered from subscriber outward (subscriber router first).", + "type": "array", + "items": { + "$ref": "#/definitions/MulticastPathHop" + } + } + } + }, + "MulticastUpdate": { + "description": "Multicast group subscription updates.\n\nEach entry carries a [`MulticastPathVector`] with the group origin and the path vector used for loop detection.", + "type": "object", + "required": [ + "announce", + "withdraw" + ], + "properties": { + "announce": { + "type": "array", + "items": { + "$ref": "#/definitions/MulticastPathVector" + }, + "uniqueItems": true + }, + "withdraw": { + "type": "array", + "items": { + "$ref": "#/definitions/MulticastPathVector" + }, + "uniqueItems": true + } + } + }, + "PathVector": { + "type": "object", + "required": [ + "destination", + "path" + ], + "properties": { + "destination": { + "$ref": "#/definitions/Ipv6Net" + }, + "path": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "PullResponse": { + "description": "One page of a peer's route snapshot.\n\nThe underlay and tunnel sections are bounded by the fabric's prefix and tunnel endpoint counts, so they are carried whole on the first page. The multicast section has no such bound, since it grows with the product of groups, VNIs, sources, and path length, and is therefore paged.", + "type": "object", + "properties": { + "multicast": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/MulticastPathVector" + }, + "uniqueItems": true + }, + "next_page_token": { + "description": "Token for resuming multicast keyset pagination, or `None` on the final page.\n\nA responder that never pages omits the field, so an unpaged snapshot deserializes as a single final page. The token is produced and interpreted by the same responder and is opaque to the reader, so its encoding is not part of the negotiated protocol.\n\nReading every page matters because a withdrawal is synthesized from the difference between what the reader has imported from this peer and what the snapshot announces. A partial snapshot would read as a withdrawal of everything the unread pages hold.", + "default": null, + "type": [ + "string", + "null" + ] + }, + "tunnel": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/TunnelOrigin" + }, + "uniqueItems": true + }, + "underlay": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/PathVector" + }, + "uniqueItems": true + } + } + }, + "TunnelOrigin": { + "type": "object", + "required": [ + "boundary_addr", + "overlay_prefix", + "vni" + ], + "properties": { + "boundary_addr": { + "type": "string", + "format": "ipv6" + }, + "metric": { + "default": 0, + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "overlay_prefix": { + "$ref": "#/definitions/IpNet" + }, + "vni": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + "TunnelUpdate": { + "type": "object", + "required": [ + "announce", + "withdraw" + ], + "properties": { + "announce": { + "type": "array", + "items": { + "$ref": "#/definitions/TunnelOrigin" + }, + "uniqueItems": true + }, + "withdraw": { + "type": "array", + "items": { + "$ref": "#/definitions/TunnelOrigin" + }, + "uniqueItems": true + } + } + }, + "UnderlayUpdate": { + "type": "object", + "required": [ + "announce", + "withdraw" + ], + "properties": { + "announce": { + "type": "array", + "items": { + "$ref": "#/definitions/PathVector" + }, + "uniqueItems": true + }, + "withdraw": { + "type": "array", + "items": { + "$ref": "#/definitions/PathVector" + }, + "uniqueItems": true + } + } + }, + "Update": { + "type": "object", + "properties": { + "multicast": { + "anyOf": [ + { + "$ref": "#/definitions/MulticastUpdate" + }, + { + "type": "null" + } + ] + }, + "tunnel": { + "anyOf": [ + { + "$ref": "#/definitions/TunnelUpdate" + }, + { + "type": "null" + } + ] + }, + "underlay": { + "anyOf": [ + { + "$ref": "#/definitions/UnderlayUpdate" + }, + { + "type": "null" + } + ] + } + } + } + } +} \ No newline at end of file diff --git a/ddm/Cargo.toml b/ddm/Cargo.toml index 1168489a9..919a093a5 100644 --- a/ddm/Cargo.toml +++ b/ddm/Cargo.toml @@ -5,6 +5,7 @@ edition = "2024" [dev-dependencies] pretty_assertions.workspace = true +tempfile = "3" expectorate.workspace = true [dependencies] @@ -18,6 +19,7 @@ thiserror.workspace = true dropshot.workspace = true schemars.workspace = true tokio.workspace = true +futures.workspace = true anyhow.workspace = true camino.workspace = true hyper.workspace = true @@ -28,6 +30,7 @@ sled.workspace = true mg-common.workspace = true port-file.workspace = true ddm-api-types.workspace = true +ddm-api-types-versions.workspace = true ddm-protocol.workspace = true chrono.workspace = true omicron-common.workspace = true @@ -44,7 +47,8 @@ libnet = { workspace = true, optional = true } dpd-client = { workspace = true, optional = true } opte-ioctl = { workspace = true, optional = true } oxide-vpc = { workspace = true, optional = true } +reqwest = { workspace = true, optional = true } [features] default = ["backend"] -backend = ["dep:libnet", "dep:dpd-client", "dep:opte-ioctl", "dep:oxide-vpc"] +backend = ["dep:libnet", "dep:dpd-client", "dep:opte-ioctl", "dep:oxide-vpc", "dep:reqwest"] diff --git a/ddm/src/admin.rs b/ddm/src/admin.rs index 21d373f4d..27610e2cc 100644 --- a/ddm/src/admin.rs +++ b/ddm/src/admin.rs @@ -8,9 +8,9 @@ use camino::Utf8PathBuf; use ddm_api::DdmAdminApi; use ddm_api::ddm_admin_api_mod; use ddm_api_types::admin::{EnableStatsRequest, ExpirePathParams, PrefixMap}; -use ddm_api_types::db::{PeerInfo, TunnelRoute}; +use ddm_api_types::db::{MulticastRoute, PeerInfo, TunnelRoute}; use ddm_api_types::exchange::PathVector; -use ddm_api_types::net::TunnelOrigin; +use ddm_api_types::net::{MulticastOrigin, TunnelOrigin}; use dropshot::ApiDescription; use dropshot::ApiDescriptionBuildErrors; use dropshot::ConfigDropshot; @@ -50,6 +50,11 @@ pub struct HandlerContext { pub event_channels: Vec>, pub db: Db, pub stats: Arc, + /// Per-interface state machine contexts shared with the multicast sweep, + /// seeded from the running state machines and read by the `/peers` view. + /// + /// Under the `--api-only` flag there are no state machines, so the set is + /// empty. pub peers: Vec, pub stats_handler: Arc>>>, pub log: Logger, @@ -129,27 +134,7 @@ impl DdmAdminApi for DdmAdminApiImpl { async fn get_peers( ctx: RequestContext, ) -> Result>, HttpError> { - let ctx = lock!(ctx.context()); - let mut result = HashMap::new(); - for sm in &ctx.peers { - // Compute status first so peer_status() never runs while we hold - // any of the InterfaceState mutexes below. - let status = sm.iface.peer_status(); - let if_index = *lock!(sm.iface.if_index); - let Some(peer) = lock!(sm.iface.peer_identity).clone() else { - continue; - }; - result.insert( - if_index, - PeerInfo { - status, - addr: peer.addr, - host: peer.hostname, - kind: peer.kind, - }, - ); - } - Ok(HttpResponseOk(result)) + Ok(HttpResponseOk(do_get_peers(ctx.context()))) } async fn expire_peer( @@ -369,6 +354,76 @@ impl DdmAdminApi for DdmAdminApiImpl { Ok(HttpResponseUpdatedNoContent()) } + async fn get_originated_multicast_groups( + ctx: RequestContext, + ) -> Result>, HttpError> { + let ctx = lock!(ctx.context()); + let originated = ctx + .db + .originated_mcast() + .map_err(|e| HttpError::for_internal_error(e.to_string()))?; + Ok(HttpResponseOk(originated)) + } + + async fn get_multicast_groups( + ctx: RequestContext, + ) -> Result>, HttpError> { + let ctx = lock!(ctx.context()); + let imported = ctx.db.imported_mcast(); + Ok(HttpResponseOk(imported)) + } + + async fn advertise_multicast_groups( + ctx: RequestContext, + request: TypedBody>, + ) -> Result { + let ctx = lock!(ctx.context()); + let groups = request.into_inner(); + slog::info!(ctx.log, "advertise multicast groups: {groups:#?}"); + ctx.db + .originate_mcast(&groups) + .map_err(|e| HttpError::for_internal_error(e.to_string()))?; + + for e in &ctx.event_channels { + e.send(Event::Admin(AdminEvent::AnnounceMulticast(groups.clone()))) + .map_err(|e| { + HttpError::for_internal_error(format!( + "admin event send: {e}" + )) + })?; + } + + Ok(HttpResponseUpdatedNoContent()) + } + + async fn withdraw_multicast_groups( + ctx: RequestContext, + request: TypedBody>, + ) -> Result { + let ctx = lock!(ctx.context()); + let groups = request.into_inner(); + slog::info!(ctx.log, "withdraw multicast groups: {groups:#?}"); + // The modification is applied before any event is enqueued, and each + // state machine revalidates reachability when it processes the + // event. An import racing this request cannot be withdrawn against + // stale state, since the revalidation reads post-modification database + // state. The modification is idempotent, so a client retry is safe. + ctx.db + .withdraw_mcast(&groups) + .map_err(|e| HttpError::for_internal_error(e.to_string()))?; + + for e in &ctx.event_channels { + e.send(Event::Admin(AdminEvent::WithdrawMulticast(groups.clone()))) + .map_err(|e| { + HttpError::for_internal_error(format!( + "admin event send: {e}" + )) + })?; + } + + Ok(HttpResponseUpdatedNoContent()) + } + async fn sync( ctx: RequestContext, ) -> Result { @@ -436,3 +491,35 @@ pub fn api_description() { ddm_admin_api_mod::api_description::() } + +/// Snapshot the current peers, keyed by interface index. +/// +/// Reads the per-interface state machine contexts. +/// +/// Under the `--api-only` flag there are no state machines, so the map is empty. +pub(crate) fn do_get_peers( + ctx: &Arc>, +) -> HashMap { + let ctx = lock!(ctx); + ctx.peers + .iter() + .filter_map(|sm| { + // Compute status first so peer_status() never runs while we hold + // any of the InterfaceState mutexes below. + let status = sm.iface.peer_status(); + let peer = lock!(sm.iface.peer_identity).clone()?; + let if_index = *lock!(sm.iface.if_index); + let if_name = lock!(sm.iface.if_name).clone(); + Some(( + if_index, + PeerInfo { + status, + addr: peer.addr, + host: peer.hostname, + kind: peer.kind, + if_name: (!if_name.is_empty()).then_some(if_name), + }, + )) + }) + .collect() +} diff --git a/ddm/src/db.rs b/ddm/src/db.rs index d72fd7b20..ee568b1a5 100644 --- a/ddm/src/db.rs +++ b/ddm/src/db.rs @@ -2,8 +2,8 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use ddm_api_types::db::TunnelRoute; -use ddm_api_types::net::TunnelOrigin; +use ddm_api_types::db::{MulticastRoute, TunnelRoute}; +use ddm_api_types::net::{MulticastOrigin, TunnelOrigin}; use mg_common::lock; use oxnet::{IpNet, Ipv6Net}; use schemars::JsonSchema; @@ -21,6 +21,10 @@ const ORIGINATE: &str = "originate"; /// tunnel endpoints. const TUNNEL_ORIGINATE: &str = "tunnel_originate"; +/// The handle used to open a persistent key-value tree for originated +/// multicast groups. +const MCAST_ORIGINATE: &str = "mcast_originate"; + #[derive(thiserror::Error, Debug)] pub enum Error { #[error("datastore error {0}")] @@ -43,10 +47,27 @@ pub struct Db { log: Logger, } +/// The realized change to the imported multicast set after applying an update. +/// +/// This holds the routes that actually became present or absent, computed as +/// the set difference between the pre- and post-update state rather than the +/// raw request. Downstream consumers reconcile only these groups, so an update +/// that re-imports an existing route or withdraws an absent one yields an empty +/// delta and triggers no work. +#[derive(Debug, Default, Clone)] +pub struct McastRibDelta { + /// Routes newly present after the update. + pub added: HashSet, + + /// Routes no longer present after the update. + pub removed: HashSet, +} + #[derive(Default, Clone)] pub struct DbData { pub imported: HashSet, pub imported_tunnel: HashSet, + pub imported_mcast: HashSet, } const _: () = { @@ -62,6 +83,7 @@ impl Db { log, }) } + pub fn dump(&self) -> DbData { lock!(self.data).clone() } @@ -82,6 +104,30 @@ impl Db { lock!(self.data).imported_tunnel.len() } + pub fn imported_mcast(&self) -> HashSet { + lock!(self.data).imported_mcast.clone() + } + + pub fn imported_mcast_count(&self) -> usize { + lock!(self.data).imported_mcast.len() + } + + /// Underlay groups imported via `nexthop`, deduplicated. + pub fn mcast_groups_for_nexthop( + &self, + nexthop: Ipv6Addr, + ) -> HashSet { + // Filter under the lock so the caller never clones the full imported + // set just to keep one peer's routes. Non-destructive analog of the + // next-hop filter in `remove_nexthop_routes`. + lock!(self.data) + .imported_mcast + .iter() + .filter(|route| route.nexthop == nexthop) + .map(|route| route.origin.underlay_group.ip()) + .collect() + } + pub fn import(&self, r: &HashSet) { lock!(self.data).imported.extend(r.clone()); } @@ -104,6 +150,73 @@ impl Db { } } + /// Atomically import and delete multicast routes under a single lock, + /// returning the effective [`McastRibDelta`] against the state before + /// any modification. + /// + /// The single lock avoids a TOCTOU race where concurrent modifications + /// between separate lock acquisitions could produce an incorrect delta. + /// Callers that redistribute the update also need post-modification + /// reachability and use + /// [`Db::update_imported_mcast_with_reachability`] instead. + pub fn update_imported_mcast( + &self, + import: &HashSet, + remove: &HashSet, + ) -> McastRibDelta { + Self::apply_imported_mcast(&mut lock!(self.data), import, remove) + } + + /// [`Db::update_imported_mcast`] variant that also captures a + /// [`MulticastReachability`] snapshot of post-modification reachability. + /// + /// The imported set is captured under the same lock as the modification, + /// so downstream withdrawal reconciliation cannot observe imported state + /// older than the modification that produced it. Callers that do not + /// redistribute have no reconciliation to feed and skip this variant's + /// imported-set clone and persistent origin read. + pub fn update_imported_mcast_with_reachability( + &self, + import: &HashSet, + remove: &HashSet, + ) -> (McastRibDelta, MulticastReachability) { + let (delta, imported) = { + let mut data = lock!(self.data); + let delta = Self::apply_imported_mcast(&mut data, import, remove); + (delta, data.imported_mcast.clone()) + }; + + // Persistent origins are not touched by this method, so reading them + // outside the lock still yields a post-modification snapshot. The + // added tree scan is acceptable, sled caches the tree and it stays + // small. + (delta, self.reachability_snapshot(imported)) + } + + /// Apply `import` and `remove` to the imported multicast set under the + /// caller-held lock, returning the effective delta. + fn apply_imported_mcast( + data: &mut DbData, + import: &HashSet, + remove: &HashSet, + ) -> McastRibDelta { + let before = data.imported_mcast.clone(); + // Route identity excludes the path, so `insert` would keep a stale + // path on re-import. `replace` lets the newest path win. + for x in import { + data.imported_mcast.replace(x.clone()); + } + + for x in remove { + data.imported_mcast.remove(x); + } + + let added = data.imported_mcast.difference(&before).cloned().collect(); + let removed = + before.difference(&data.imported_mcast).cloned().collect(); + McastRibDelta { added, removed } + } + pub fn originate(&self, prefixes: &HashSet) -> Result<(), Error> { let tree = self.persistent_data.open_tree(ORIGINATE)?; for p in prefixes { @@ -126,76 +239,117 @@ impl Db { Ok(()) } - pub fn originated(&self) -> Result, Error> { - let tree = self.persistent_data.open_tree(ORIGINATE)?; + /// Persist multicast origins for advertisement to peers. + pub fn originate_mcast( + &self, + origins: &HashSet, + ) -> Result<(), Error> { + let tree = self.persistent_data.open_tree(MCAST_ORIGINATE)?; + for o in origins { + // Key by the metric-excluded identity, storing the full origin as + // the value. `MulticastOrigin` equality ignores `metric`, so keying + // by identity lets a re-origination with a changed metric overwrite + // the stored entry instead of leaving a stale one under the old + // metric. + tree.insert( + o.identity_key()?.as_str(), + serde_json::to_string(o)?.as_str(), + )?; + } + tree.flush()?; + Ok(()) + } + + /// Scan a persistent origin tree with `parse`, skipping entries that + /// fail to read or parse. `kind` names the entry kind for log context. + fn scan_origin_tree( + &self, + tree: &str, + kind: &str, + parse: impl Fn(&[u8], &[u8]) -> Result, + ) -> Result, Error> + where + T: Eq + std::hash::Hash, + { + let tree = self.persistent_data.open_tree(tree)?; let result = tree - .scan_prefix(vec![]) + .iter() .filter_map(|item| { - let (key, _value) = match item { + let (key, value) = match item { Ok(item) => item, Err(e) => { error!( self.log, - "db: error ddm originated prefix: {e}" + "db: error fetching ddm {kind} entry: {e}" ); return None; } }; - Some(match Ipv6Net::from_db_key(&key) { - Ok(item) => item, + match parse(key.as_ref(), value.as_ref()) { + Ok(item) => Some(item), Err(e) => { - error!( - self.log, - "db: error parsing ddm origin entry value: {e}" - ); - return None; + error!(self.log, "db: error parsing ddm {kind}: {e}"); + None } - }) + } }) .collect(); Ok(result) } + pub fn originated(&self) -> Result, Error> { + self.scan_origin_tree(ORIGINATE, "origin prefix", |key, _value| { + Ipv6Net::from_db_key(key).map_err(|e| Error::DbKey(e.to_string())) + }) + } + pub fn originated_count(&self) -> Result { Ok(self.originated()?.len()) } pub fn originated_tunnel(&self) -> Result, Error> { - let tree = self.persistent_data.open_tree(TUNNEL_ORIGINATE)?; - let result = tree - .scan_prefix(vec![]) - .filter_map(|item| { - let (key, _value) = match item { - Ok(item) => item, - Err(e) => { - error!( - self.log, - "db: error fetching ddm tunnel origin entry: {e}" - ); - return None; - } - }; - let value = String::from_utf8_lossy(&key); - let value: TunnelOrigin = match serde_json::from_str(&value) { - Ok(item) => item, - Err(e) => { - error!( - self.log, - "db: error parsing ddm tunnel origin: {e}" - ); - return None; - } - }; - Some(value) - }) - .collect(); - Ok(result) + self.scan_origin_tree(TUNNEL_ORIGINATE, "tunnel origin", |key, _v| { + Ok(serde_json::from_slice(key)?) + }) } pub fn originated_tunnel_count(&self) -> Result { Ok(self.originated_tunnel()?.len()) } + /// Multicast origins originated locally. + /// + /// Each origin is keyed by its metric-excluded identity and stored as the + /// value, so the current metric is read back from the value rather than the + /// key. + /// + /// This iterates the tree directly rather than going through + /// [`Db::scan_origin_tree`], which skips entries that fail to read or + /// parse. Withdrawal reconciliation treats the result as the complete + /// origin set, so a silently skipped entry could become a false final + /// withdrawal. Here any per-entry failure fails the whole read. The + /// caller surfaces that by degrading the snapshot's origin set, and + /// reconciliation then drops the withdrawal rather than treating the + /// missing origins as truly gone. + /// + /// # Errors + /// + /// Returns an error if the tree cannot be opened or any entry fails to + /// read or parse. + pub fn originated_mcast(&self) -> Result, Error> { + let tree = self.persistent_data.open_tree(MCAST_ORIGINATE)?; + tree.iter() + .map(|item| { + let (_key, value) = item?; + Ok(serde_json::from_slice(&value)?) + }) + .collect() + } + + pub fn originated_mcast_count(&self) -> Result { + Ok(self.originated_mcast()?.len()) + } + pub fn withdraw(&self, prefixes: &HashSet) -> Result<(), Error> { let tree = self.persistent_data.open_tree(ORIGINATE)?; for p in prefixes { @@ -218,33 +372,116 @@ impl Db { Ok(()) } + /// Remove persisted multicast origins. + /// + /// State machines revalidate reachability at processing time via + /// [`Db::multicast_reachability`] rather than from a snapshot captured + /// here. + /// + /// The modification lands before any event is enqueued, so the + /// processing-time snapshot is guaranteed to reflect this removal. + pub fn withdraw_mcast( + &self, + origins: &HashSet, + ) -> Result<(), Error> { + let tree = self.persistent_data.open_tree(MCAST_ORIGINATE)?; + for o in origins { + // Remove by identity so a withdraw matches regardless of the metric + // the origin was advertised with (e.g. the CLI's default metric). + tree.remove(o.identity_key()?.as_str())?; + } + tree.flush()?; + Ok(()) + } + + /// Capture current multicast reachability for processing-time + /// revalidation. An event is enqueued only after its modification completes, + /// so a snapshot taken while processing that event is post-modification. It + /// also observes any later modification, which lets a state machine avoid + /// acting on reachability that has since been restored. + pub fn multicast_reachability(&self) -> MulticastReachability { + // Same lock-ordering discipline as `update_imported_mcast`: clone + // `imported_mcast` under the data lock and read persistent origins + // after dropping it, since the two sources are not co-modified. + let imported = lock!(self.data).imported_mcast.clone(); + self.reachability_snapshot(imported) + } + pub fn remove_nexthop_routes( &self, nexthop: Ipv6Addr, - ) -> (HashSet, HashSet) { - let mut data = lock!(self.data); - // Routes are generally held in sets to prevent duplication and provide - // handy set-algebra operations. - let mut removed = HashSet::new(); - for x in &data.imported { - if x.nexthop == nexthop { - removed.insert(x.clone()); + ) -> RemovedNexthopRoutes { + let (removed, tnl_removed, mcast_removed, imported_mcast) = { + let mut data = lock!(self.data); + let mut removed = HashSet::new(); + for x in &data.imported { + if x.nexthop == nexthop { + removed.insert(x.clone()); + } + } + for x in &removed { + data.imported.remove(x); + } + + let mut tnl_removed = HashSet::new(); + for x in &data.imported_tunnel { + if x.nexthop == nexthop { + tnl_removed.insert(*x); + } + } + for x in &tnl_removed { + data.imported_tunnel.remove(x); } - } - for x in &removed { - data.imported.remove(x); - } - let mut tnl_removed = HashSet::new(); - for x in &data.imported_tunnel { - if x.nexthop == nexthop { - tnl_removed.insert(*x); + let mut mcast_removed = HashSet::new(); + for x in &data.imported_mcast { + if x.nexthop == nexthop { + mcast_removed.insert(x.clone()); + } + } + for x in &mcast_removed { + data.imported_mcast.remove(x); } + + let imported_mcast = data.imported_mcast.clone(); + (removed, tnl_removed, mcast_removed, imported_mcast) + }; + + // Persistent origins are not touched by this method, so reading them + // outside the lock still yields a post-modification snapshot. + RemovedNexthopRoutes { + underlay: removed, + tunnel: tnl_removed, + multicast: mcast_removed, + mcast_reachability: self.reachability_snapshot(imported_mcast), } - for x in &tnl_removed { - data.imported_tunnel.remove(x); + } + + /// Build a post-modification reachability snapshot around an imported + /// set captured under the data lock, taking ownership so no further + /// clone is needed. A persistent origin read failure degrades + /// `originated` to the empty set and marks the snapshot, so consumers + /// know the empty origin set is a read failure rather than a real + /// absence. + fn reachability_snapshot( + &self, + imported: HashSet, + ) -> MulticastReachability { + match self.originated_mcast() { + Ok(originated) => MulticastReachability { + imported, + originated, + origins_degraded: false, + }, + Err(e) => { + error!(self.log, "read remaining multicast origins: {e}"); + MulticastReachability { + imported, + originated: HashSet::new(), + origins_degraded: true, + } + } } - (removed, tnl_removed) } pub fn routes_by_vector( @@ -263,6 +500,48 @@ impl Db { } } +/// Routes withdrawn for a next hop, grouped by route family. +pub struct RemovedNexthopRoutes { + pub underlay: HashSet, + pub tunnel: HashSet, + pub multicast: HashSet, + /// Post-modification multicast reachability snapshot captured under the same + /// removal that produced `multicast`. Downstream reconciliation reads + /// viable paths from here rather than reading the database again. + pub mcast_reachability: MulticastReachability, +} + +/// A snapshot of multicast reachability: imported routes and local origins. +/// +/// Only `Db` methods construct this. A snapshot is either captured by the +/// modification that produced a set of withdrawals or read at processing time +/// via [`Db::multicast_reachability`]. In both cases, it describes state no +/// older than that modification, since the event that triggers a +/// processing-time read is enqueued only after the modification completes. +#[derive(Debug, Clone)] +pub struct MulticastReachability { + imported: HashSet, + originated: HashSet, + origins_degraded: bool, +} + +impl MulticastReachability { + pub fn imported(&self) -> &HashSet { + &self.imported + } + + pub fn originated(&self) -> &HashSet { + &self.originated + } + + /// Whether the persistent origin read failed, degrading `originated` to + /// the empty set. A degraded snapshot cannot distinguish a withdrawn + /// origin from an unread one. + pub fn origins_degraded(&self) -> bool { + self.origins_degraded + } +} + #[derive( Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema, )] diff --git a/ddm/src/discovery/mod.rs b/ddm/src/discovery/mod.rs index 3f0167e59..0ada12d65 100644 --- a/ddm/src/discovery/mod.rs +++ b/ddm/src/discovery/mod.rs @@ -14,10 +14,9 @@ //! //! [`Version`] and [`DiscoveryError`] are platform-agnostic and stay in this //! module so the state machine type definitions in [`crate::sm`] continue to -//! compile when the routing runtime is gated out (e.g. Linux test fixtures -//! running `ddmd` with `--api-only`). The runtime helpers that drive -//! the protocol over UDPv6 sockets live in the [`runtime`] submodule and -//! are illumos-only. +//! compile when the routing runtime is gated out (e.g. a non-illumos `ddmd` +//! running with `--api-only`). The runtime helpers that drive the protocol +//! over UDPv6 sockets live in the `runtime` submodule and are illumos-only. //! //! ## Protocol //! @@ -80,7 +79,7 @@ //! 1 2 3 //! 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 //! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -//! | version |S A r r r r r r| router kind | hostname len | +//! | version |S A C r r r r r| router kind | hostname len | //! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ //! | hostname : //! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ @@ -88,13 +87,15 @@ //! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ //! ``` //! -//! The first byte indicates the version. The only valid version at present is -//! version 1. The second byte is a flags bitfield. The first position `S` -//! indicates a solicitation. The second position `A` indicates and -//! advertisement. All other positions are reserved for future use. The third -//! byte indicates the kind of router. Current values are 0 for a server router -//! and 1 for a transit routers. The fourth byte is a hostname length followed -//! directly by a hostname of up to 255 bytes in length. +//! The first byte indicates the version. The second byte is a flags bitfield. +//! The first position `S` indicates a solicitation. The second position `A` +//! indicates an advertisement. The third position `C` indicates DDMv4 +//! (multicast) capability, advertised independently of the version byte so +//! peers that ignore it still peer at the floor version. All other positions +//! are reserved for future use. The third byte indicates the kind of router. +//! Current values are 0 for a server router and 1 for a transit router. The +//! fourth byte is a hostname length followed directly by a hostname of up to +//! 255 bytes in length. use thiserror::Error; @@ -104,11 +105,15 @@ mod runtime; #[cfg(all(feature = "backend", target_os = "illumos"))] pub(crate) use runtime::handler; -#[derive(Debug, Copy, Clone)] +// Ordering follows the ascending discriminants, so version-dependent +// behavior can use range checks (`>= Version::V4`) that remain correct +// as newer versions are added. +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] #[repr(u8)] pub enum Version { V2 = 2, V3 = 3, + V4 = 4, } #[derive(Error, Debug)] diff --git a/ddm/src/discovery/runtime.rs b/ddm/src/discovery/runtime.rs index 2cd8c53fc..c0259f2b4 100644 --- a/ddm/src/discovery/runtime.rs +++ b/ddm/src/discovery/runtime.rs @@ -29,6 +29,10 @@ const DDM_MADDR: Ipv6Addr = Ipv6Addr::new(0xff02, 0, 0, 0, 0, 0, 0, 0xdd); const DDM_PORT: u16 = 0xddd; const SOLICIT: u8 = 1; const ADVERTISE: u8 = 1 << 1; +// Advertises DDMv4 (multicast) capability without raising the discovery +// version byte past what old peers accept. Following RFC 5492, peers ignore +// capability bits they do not understand. +const MCAST_CAPABLE: u8 = 1 << 2; /// Reinterpret an initialized prefix of `[MaybeUninit]` as `[u8]`. /// @@ -51,7 +55,7 @@ impl DiscoveryPacket { fn new_solicitation(hostname: String, kind: RouterKind) -> Self { Self { version: Version::V2 as u8, - flags: SOLICIT, + flags: SOLICIT | MCAST_CAPABLE, hostname, kind, } @@ -59,7 +63,7 @@ impl DiscoveryPacket { fn new_advertisement(hostname: String, kind: RouterKind) -> Self { Self { version: Version::V2 as u8, - flags: ADVERTISE, + flags: ADVERTISE | MCAST_CAPABLE, hostname, kind, } @@ -89,6 +93,9 @@ struct Neighbor { hostname: String, kind: RouterKind, last_seen: Instant, + /// Last negotiated exchange version, tracked so a capability change on an + /// otherwise stable peer re-emits a neighbor update. + version: Version, } pub(crate) fn handler( @@ -324,6 +331,7 @@ fn handle_msg( msg.hostname, msg.kind, msg.version, + msg.flags, stats, ); } @@ -343,12 +351,52 @@ fn handle_solicitation( } } +/// The outcome of negotiating the session version from a peer's advertisement. +#[derive(Debug, PartialEq)] +enum NegotiatedVersion { + /// A usable version was negotiated directly from the advertised byte and + /// capability flags. + Use(Version), + /// The advertised byte was outside the range this node speaks; the + /// advertisement is rejected and must not replace a valid neighbor. + Rejected, +} + +/// Negotiate the session version from a peer's advertised version byte and +/// capability flags. +/// +/// The version byte is a backward-compatible floor ([`Version::V2`]) that all +/// deployed peers accept. Capability rides flags rather than the byte: +/// [`MCAST_CAPABLE`] signals [`Version::V4`], so a peer at the floor that sets +/// it negotiates V4 while older peers that ignore the flag still session at the +/// floor. A conforming peer never advertises a byte above the known maximum. +/// +/// A byte outside the known range is therefore a malformed or incompatible +/// peer, not a capability hint, so it is rejected rather than capped. See +/// [RFC 5492] for the capability-negotiation model. +/// +/// [RFC 5492]: https://www.rfc-editor.org/rfc/rfc5492 +fn negotiate_version(version: u8, flags: u8) -> NegotiatedVersion { + let base = match version { + 2 => Version::V2, + 3 => Version::V3, + 4 => Version::V4, + _ => return NegotiatedVersion::Rejected, + }; + if (flags & MCAST_CAPABLE) != 0 { + NegotiatedVersion::Use(Version::V4) + } else { + NegotiatedVersion::Use(base) + } +} + fn handle_advertisement( ctx: &HandlerContext, sender: &Ipv6Addr, hostname: String, kind: RouterKind, version: u8, + flags: u8, stats: &Arc, ) { trc!(&ctx.log, ctx.config.if_name, "advert from {}", &hostname); @@ -356,25 +404,15 @@ fn handle_advertisement( .advertisements_received .fetch_add(1, Ordering::Relaxed); - // TODO: version negotiation - // - // Things currently work because ddm v1 does no version checking at all. - // So ddm v2 speakers can send out discovery packets with the version set to - // 2, and ddm v1 speakers can send out discovery packets with the version - // set to 1, and as long a v2 router speaks version 1 after discovering a v1 - // peer, things will work. However, this will not work for version 3. So we - // need to implement version negotiation. This would also not work for - // changes in the discovery protocol, if we were to have changes there. So - // we need to come up with a general way for both protocols to evolve. - let version = match version { - 2 => Version::V2, - 3 => Version::V3, - x => { + let version = match negotiate_version(version, flags) { + NegotiatedVersion::Use(v) => v, + NegotiatedVersion::Rejected => { err!( ctx.log, ctx.config.if_name, - "unknown protocol version {}, known versions are: 1, 2", - x + "unsupported protocol version {version}, this node supports {}..={}", + Version::V2 as u8, + Version::V4 as u8 ); return; } @@ -387,7 +425,7 @@ fn handle_advertisement( return; } }; - match &mut *guard { + let version_changed = match &mut *guard { Some(nbr) => { if *sender != nbr.addr { inf!( @@ -407,6 +445,9 @@ fn handle_advertisement( stats.peer_address_changes.fetch_add(1, Ordering::Relaxed); } nbr.last_seen = Instant::now(); + let version_changed = nbr.version != version; + nbr.version = version; + version_changed } None => { inf!( @@ -422,8 +463,10 @@ fn handle_advertisement( hostname: hostname.clone(), last_seen: Instant::now(), kind, + version, }); stats.peer_established.fetch_add(1, Ordering::Relaxed); + false } }; drop(guard); @@ -433,7 +476,11 @@ fn handle_advertisement( kind, }; let mut info = lock!(ctx.iface.peer_identity); - if info.as_ref() != Some(&new_peer) { + // Emit when the peer's identity changes or when its negotiated version + // changes. A version change on a stable identity (for example a peer that + // begins advertising MCAST_CAPABLE after a restart) must reach the state + // machine, otherwise the exchange keeps using the prior version. + if info.as_ref() != Some(&new_peer) || version_changed { *info = Some(new_peer); drop(info); emit_nbr_update(ctx, sender, version); @@ -494,3 +541,46 @@ fn advertise( stats.advertisements_sent.fetch_add(1, Ordering::Relaxed); Ok(n) } + +#[cfg(test)] +mod tests { + use super::*; + + // Pins the negotiation contract. + // + // Without the capability flag the byte is the floor and maps straight + // through, MCAST_CAPABLE raises any accepted byte to V4, and out-of-range + // bytes are rejected even with the flag set so a malformed advertisement + // cannot replace a valid neighbor. + #[test] + fn negotiate_version_contract() { + use NegotiatedVersion::{Rejected, Use}; + + let cases = [ + // Byte alone is the floor. + (2u8, 0u8, Use(Version::V2)), + (3, 0, Use(Version::V3)), + (4, 0, Use(Version::V4)), + // Capability flag raises any accepted byte to V4. + (2, MCAST_CAPABLE, Use(Version::V4)), + (3, MCAST_CAPABLE, Use(Version::V4)), + (4, MCAST_CAPABLE, Use(Version::V4)), + // Reserved flags are ignored rather than rejected. + (2, 1 << 3, Use(Version::V2)), + // Out-of-range bytes are rejected, flag or not. + (0, 0, Rejected), + (1, 0, Rejected), + (1, MCAST_CAPABLE, Rejected), + (5, 0, Rejected), + (255, MCAST_CAPABLE, Rejected), + ]; + + for (version, flags, expected) in cases { + assert_eq!( + negotiate_version(version, flags), + expected, + "negotiate_version({version}, {flags:#05b})" + ); + } + } +} diff --git a/ddm/src/exchange/mod.rs b/ddm/src/exchange/mod.rs index d9f842ebd..6a4c66604 100644 --- a/ddm/src/exchange/mod.rs +++ b/ddm/src/exchange/mod.rs @@ -15,21 +15,31 @@ //! model of a ddm router is defined in the state machine implementation in //! [`crate::sm`]. //! -//! The wire types ([`Update`], [`UnderlayUpdate`], [`TunnelUpdate`], and -//! their versioned counterparts) are platform-agnostic and stay in this -//! module. The runtime helpers that drive the HTTP exchange protocol and -//! program forwarding state live in the [`runtime`] submodule and are -//! illumos-only, since they call into [`crate::sys`] to install routes. +//! The wire types (`Update`, `UnderlayUpdate`, `TunnelUpdate`, +//! `MulticastUpdate`, and their versioned counterparts) live in the +//! [`ddm_protocol`] crate. The runtime helpers that drive the HTTP exchange +//! protocol and program forwarding state live in the `runtime` submodule and +//! are illumos-only, since they call into `crate::sys` to install routes. use thiserror::Error; +#[cfg(any(test, all(feature = "backend", target_os = "illumos")))] +mod paging; + +#[cfg(any(test, all(feature = "backend", target_os = "illumos")))] +mod reconcile; + #[cfg(all(feature = "backend", target_os = "illumos"))] mod runtime; +#[cfg(all(feature = "backend", target_os = "illumos"))] +pub(crate) use reconcile::reconcile_multicast_withdrawals; + #[cfg(all(feature = "backend", target_os = "illumos"))] pub(crate) use runtime::{ - announce_tunnel, announce_underlay, do_pull, handler, pull, - withdraw_tunnel, withdraw_underlay, + ExchangeHandle, UpdateMode, announce_multicast, announce_tunnel, + announce_underlay, do_pull_v2, do_pull_v3, do_pull_v4, handler, pull, + withdraw_multicast, withdraw_tunnel, withdraw_underlay, }; #[derive(Error, Debug)] @@ -46,6 +56,101 @@ pub enum ExchangeError { #[error("timeout error: {0}")] Timeout(#[from] tokio::time::error::Elapsed), + #[error("peer returned status {0}")] + Status(hyper::StatusCode), + #[error("json error: {0}")] SerdeJson(#[from] serde_json::Error), + + /// A single multicast path vector exceeds the exchange body limit, so no + /// approach to batching can make it sendable. + #[error("multicast vector for {group} is {size} bytes, limit is {limit}")] + MulticastVectorTooLarge { + group: std::net::IpAddr, + size: usize, + limit: usize, + }, + + /// A peer's pull response exceeded the bound on the client's read. + #[error("pull response exceeds the {limit} byte limit")] + ResponseTooLarge { limit: usize }, + + /// A peer handed back a continuation token on every page, meaning that a + /// single pull never reached the end of its snapshot. + #[error("pull did not terminate within {limit} pages")] + PullTooManyPages { limit: usize }, + + /// A continuation token could not be encoded for the next page. + #[error("could not construct a pull page token: {0}")] + PageToken(String), + + /// An exchange request URI could not be constructed. + #[error("invalid exchange request URI: {0}")] + InvalidUri(String), +} + +impl ExchangeError { + /// Whether a failed exchange operation should expire the peer session. + /// + /// Expiry is a liveness response, so it applies when a failure suggests + /// the peer is unreachable or unhealthy. Two cases are deterministic + /// instead, where retrying the same payload against a fresh session + /// changes nothing. + /// + /// A `400 Bad Request` from the push endpoint means Dropshot rejected the + /// request body before the handler ran, since the handlers themselves + /// surface every failure as a `500`. Other 4xx responses, such as `404 Not + /// Found` or `405 Method Not Allowed`, can indicate an endpoint or + /// protocol mismatch and still expire the peer. + /// + /// A size limit is reached locally, before anything reaches the wire. + /// + /// Dropping the update is safe for multicast because the peer's periodic + /// V4 pull reads our full advertisable multicast set and reconciles + /// against it, repairing both a missing announcement and a missed + /// withdrawal. Callers should use this exception only for multicast + /// updates. Underlay and tunnel updates have no equivalent full + /// reconciliation path. + pub fn expires_peer(&self) -> bool { + match self { + Self::Status(status) => *status != hyper::StatusCode::BAD_REQUEST, + Self::MulticastVectorTooLarge { .. } + | Self::ResponseTooLarge { .. } => false, + Self::Io(_) + | Self::Hyper(_) + | Self::HyperClient(_) + | Self::Timeout(_) + | Self::PullTooManyPages { .. } + | Self::PageToken(_) + | Self::InvalidUri(_) + | Self::SerdeJson(_) => true, + } + } +} + +#[cfg(test)] +mod tests { + use super::ExchangeError; + + #[test] + fn deterministic_size_errors_do_not_expire_peers() { + assert!( + !ExchangeError::MulticastVectorTooLarge { + group: "ff04::1".parse().unwrap(), + size: 16, + limit: 8, + } + .expires_peer() + ); + assert!(!ExchangeError::ResponseTooLarge { limit: 8 }.expires_peer()); + } + + #[test] + fn transport_and_protocol_errors_expire_peers() { + assert!( + ExchangeError::Io(std::io::Error::other("connection lost")) + .expires_peer() + ); + assert!(ExchangeError::PullTooManyPages { limit: 64 }.expires_peer()); + } } diff --git a/ddm/src/exchange/paging.rs b/ddm/src/exchange/paging.rs new file mode 100644 index 000000000..3f8c8faa3 --- /dev/null +++ b/ddm/src/exchange/paging.rs @@ -0,0 +1,787 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Splitting a multicast exchange set so that it fits the paged V4 body limit. +//! +//! The underlay and tunnel sections of an exchange are bounded by the fabric's +//! prefix and tunnel endpoint counts. The multicast section is not: it grows +//! with the product of groups, VNIs, sources, and path length. Both directions +//! of the exchange therefore need a way to carry that set in more than one +//! HTTP body. +//! +//! A push splits with [`batch_multicast`], whose batches may be sent in any +//! order. Nothing else depends on where the boundaries fall, because +//! announcements and withdrawals within one update are already disjoint by +//! design. +//! +//! A pull uses keyset pagination with [`page_multicast`], where the boundaries +//! are not free. The reader synthesizes a withdrawal for every route it +//! imported from a peer that the peer's snapshot did not announce, so a group +//! that goes unread reads as withdrawn and drains that group's replication +//! members. +//! +//! Therefore, paging is ordered by group identity rather than by position, and +//! a [`MulticastPageSelector`] names the last group read. Per [Dropshot +//! pagination], a reader that scans a collection to the end sees every item +//! that existed both before and after the scan and was not renamed during it. +//! That is not true of a scheme whose cursor is a numeric offset. An insertion +//! ahead of an offset cursor shifts a later group backwards past it, and the +//! reader would treat the group it never saw as withdrawn. +//! +//! A keyset walk still races with concurrent change(s), but both outcomes are +//! harmless. A group added at a key the keyset walk has already passed is +//! missed until the next keyset walk. A group removed at a key the keyset walk +//! has not yet reached is correctly absent. Neither invents a withdrawal for a +//! group the peer still holds. +//! +//! [Dropshot pagination]: https://github.com/oxidecomputer/dropshot/blob/4ff9cb3f7fb29e477190dfccb100e244ed8562cf/dropshot/src/pagination.rs + +use std::collections::{BTreeMap, HashSet}; +use std::net::{IpAddr, Ipv6Addr}; +use std::ops::Bound; + +use ddm_protocol::v3; +use ddm_protocol::v4::{ + MulticastOrigin, MulticastPathVector, MulticastUpdate, PullResponse, Update, +}; +use dropshot::{EmptyScanParams, ResultsPage}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use super::ExchangeError; + +/// Bound on a paging-capable V4 exchange request or response. +/// +/// It is a per-request memory guard rather than a multicast capacity limit: an +/// exchange set too large for one body is split across several requests, so the +/// limit constrains how much arrives at once, not how many groups the protocol +/// can carry. Legacy unpaged pulls keep their prior behavior and are not +/// subject to this client-side response bound. +pub(crate) const MAX_EXCHANGE_BODY_BYTES: usize = 10 * 1024 * 1024; + +/// Maximum encoded length of a Dropshot page token. +/// +/// This reserves room for a continuation token while sizing a response, and +/// bounds a token read back off the wire. Dropshot's own `MAX_TOKEN_LENGTH` is +/// private, so the value is mirrored here rather than imported. +pub(crate) const MAX_PAGE_TOKEN_BYTES: usize = 512; + +/// Validate a Dropshot page token before placing it in a request URI. +/// +/// Dropshot emits URL-safe base64 with optional padding. Pull responses are +/// peer input, so the client must not trust an arbitrary string to remain a +/// valid URI component. +pub(crate) fn page_token_query(token: &str) -> Result { + if token.is_empty() + || token.len() > MAX_PAGE_TOKEN_BYTES + || !token + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '=')) + { + return Err(ExchangeError::PageToken(String::from( + "peer returned an invalid page token", + ))); + } + + Ok(format!("page_token={token}")) +} + +/// Multicast entries grouped under the identity that paging orders by. +pub(crate) type GroupedVectors = + BTreeMap>; + +/// The sortable identity used by the multicast page token. +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub(crate) struct MulticastOriginKey { + pub overlay_group: IpAddr, + pub underlay_group: Ipv6Addr, + pub vni: u32, + pub source: Option, +} + +impl From<&MulticastOrigin> for MulticastOriginKey { + fn from(origin: &MulticastOrigin) -> Self { + Self { + overlay_group: origin.overlay_group, + underlay_group: origin.underlay_group, + vni: origin.vni, + source: origin.source, + } + } +} + +impl From for MulticastOrigin { + fn from(key: MulticastOriginKey) -> Self { + Self { + overlay_group: key.overlay_group, + underlay_group: key.underlay_group, + vni: key.vni, + metric: 0, + source: key.source, + } + } +} + +/// Dropshot-compatible selector for multicast keyset pagination. +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub(crate) struct MulticastPageSelector { + pub last_seen: MulticastOriginKey, +} + +impl MulticastPageSelector { + pub(crate) fn after(&self) -> MulticastOrigin { + self.last_seen.clone().into() + } +} + +/// Serialized length a value contributes to a set, including the separator +/// that follows it. Charging the separator to every element keeps a batch +/// within budget once its elements are joined into an array. +fn element_len(value: &T) -> Result { + Ok(serde_json::to_string(value)?.len() + 1) +} + +/// Measure the fixed portion of an update with an empty multicast section. +/// +/// The section is present rather than absent, since an absent one serializes +/// as `null` and would leave the object's own braces uncharged. +fn update_envelope_len() -> Result { + Ok(serde_json::to_string(&Update::from(MulticastUpdate::default()))?.len()) +} + +/// Measure the fixed portion of a pull response with an empty multicast page. +/// +/// The caller supplies the underlay and tunnel sections because they are +/// present only on the first page. The multicast array is included even when +/// empty so the returned length remains valid when that page contains entries. +pub(crate) fn response_envelope_len( + underlay: Option<&HashSet>, + tunnel: Option<&HashSet>, +) -> Result { + Ok(serde_json::to_string(&PullResponse { + underlay: underlay.cloned(), + tunnel: tunnel.cloned(), + multicast: Some(HashSet::new()), + next_page_token: None, + })? + .len()) +} + +/// Split a multicast exchange set into batches that each serialize within the +/// `limit`. +/// +/// Batches are sized by measured serialized length rather than entry count, +/// because a path vector grows with the length of its path and an entry count +/// would either waste headroom or overshoot the limit. +/// +/// An empty input yields a single empty batch, preserving the send that +/// callers already make for an empty set. +/// +/// # Errors +/// +/// Returns [`ExchangeError::MulticastVectorTooLarge`] when one vector exceeds +/// the budget on its own, since no split can make it sendable. +pub(crate) fn batch_multicast( + groups: HashSet, + limit: usize, +) -> Result>, ExchangeError> { + if groups.is_empty() { + return Ok(vec![HashSet::new()]); + } + + let budget = limit.saturating_sub(update_envelope_len()?); + + let mut batches = Vec::new(); + let mut batch = HashSet::new(); + let mut used = 0usize; + + for group in groups { + let size = element_len(&group)?; + if size > budget { + return Err(ExchangeError::MulticastVectorTooLarge { + group: group.origin.overlay_group, + size, + limit: budget, + }); + } + if used + size > budget { + batches.push(std::mem::take(&mut batch)); + used = 0; + } + used += size; + batch.insert(group); + } + + if !batch.is_empty() { + batches.push(batch); + } + + Ok(batches) +} + +/// Take the page of `groups` that follows `after`, and a selector for the next +/// page if any groups remain. +/// +/// A page breaks only between groups, never inside one. Reconciliation is +/// keyed on the group, so a group split across two pages would read as a +/// partial announcement on the first of them. +/// +/// # Errors +/// +/// Returns [`ExchangeError::MulticastVectorTooLarge`] when one group's +/// vectors exceed the budget together, since no page boundary can make that +/// group servable. +pub(crate) fn page_multicast( + groups: &GroupedVectors, + after: Option<&MulticastOrigin>, + limit: usize, + response_envelope: usize, +) -> Result< + (HashSet, Option), + ExchangeError, +> { + let budget = limit + .saturating_sub(response_envelope.saturating_add(MAX_PAGE_TOKEN_BYTES)); + + let remaining = match after { + Some(origin) => { + groups.range((Bound::Excluded(origin), Bound::Unbounded)) + } + None => groups.range(..), + }; + + let mut page = HashSet::new(); + let mut used = 0usize; + let mut last = None; + let mut truncated = false; + + for (origin, vectors) in remaining { + let size = vectors + .iter() + .map(element_len) + .sum::>()?; + if size > budget { + return Err(ExchangeError::MulticastVectorTooLarge { + group: origin.overlay_group, + size, + limit: budget, + }); + } + if used + size > budget { + truncated = true; + break; + } + used += size; + page.extend(vectors.iter().cloned()); + last = Some(origin); + } + + // A group that is too large to serve on its own is rejected above, so the + // first group of a page always fits and `last` is set whenever pagination + // breaks. + let next = if truncated { + last.map(|origin| MulticastPageSelector { + last_seen: origin.into(), + }) + } else { + None + }; + + Ok((page, next)) +} + +/// Group multicast entries under the identity that [`page_multicast`] orders +/// by. +pub(crate) fn group_by_origin( + vectors: impl IntoIterator, +) -> GroupedVectors { + let mut grouped = GroupedVectors::new(); + for vector in vectors { + grouped + .entry(vector.origin.clone()) + .or_default() + .push(vector); + } + grouped +} + +/// Render a Dropshot page token for the next multicast page. +/// +/// `ResultsPage` is used only for its token encoder. The exchange response has +/// additional fields, so it cannot use `ResultsPage` as its wire shape. +pub(crate) fn encode_page_token( + selector: &MulticastPageSelector, +) -> Result { + let page = ResultsPage::new( + vec![selector.clone()], + &EmptyScanParams {}, + |selector, _| selector.clone(), + ) + .map_err(|e| ExchangeError::PageToken(e.to_string()))?; + + page.next_page.ok_or_else(|| { + ExchangeError::PageToken(String::from( + "Dropshot did not create a token for a non-empty page", + )) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use ddm_protocol::v4::MulticastPathHop; + + fn origin(index: u16) -> MulticastOrigin { + MulticastOrigin { + overlay_group: IpAddr::V6(Ipv6Addr::new( + 0xff04, + 0, + 0, + 0, + 0, + 0, + 0, + index + 1, + )), + underlay_group: Ipv6Addr::new(0xff04, 0, 0, 0, 0, 0, 1, index + 1), + vni: 100, + metric: 0, + source: None, + } + } + + fn vector(index: u16, hops: usize) -> MulticastPathVector { + MulticastPathVector { + origin: origin(index), + path: (0..hops) + .map(|hop| MulticastPathHop { + router_id: format!("router-{index}-{hop}"), + underlay_addr: Ipv6Addr::new( + 0xfd00, + 0, + 0, + 0, + 0, + 0, + index + 1, + hop as u16, + ), + downstream_subscriber_count: 0, + }) + .collect(), + } + } + + fn response_envelope_len() -> usize { + serde_json::to_string(&PullResponse { + multicast: Some(HashSet::new()), + ..Default::default() + }) + .unwrap() + .len() + } + + fn body_len( + page: &HashSet, + next_page_token: Option<&str>, + ) -> usize { + body_len_with_sections(page, next_page_token, None, None) + } + + fn body_len_with_sections( + page: &HashSet, + next_page_token: Option<&str>, + underlay: Option<&HashSet>, + tunnel: Option<&HashSet>, + ) -> usize { + serde_json::to_string(&PullResponse { + underlay: underlay.cloned(), + tunnel: tunnel.cloned(), + multicast: Some(page.clone()), + next_page_token: next_page_token.map(String::from), + }) + .unwrap() + .len() + } + + /// Read every page of `groups`, as a peer would, and return what the + /// keyset walk announced along with the number of pages it took. + fn walk( + groups: &GroupedVectors, + limit: usize, + ) -> (HashSet, usize) { + let mut seen = HashSet::new(); + let mut selector: Option = None; + let mut pages = 0; + + loop { + let after = selector.as_ref().map(MulticastPageSelector::after); + let (page, next) = page_multicast( + groups, + after.as_ref(), + limit, + response_envelope_len(), + ) + .unwrap(); + pages += 1; + let next_token = + next.as_ref().map(encode_page_token).transpose().unwrap(); + assert!(body_len(&page, next_token.as_deref()) <= limit); + seen.extend(page); + match next { + Some(next) => selector = Some(next), + None => return (seen, pages), + } + } + } + + #[test] + fn empty_exchange_set_yields_one_empty_batch() { + let batches = + batch_multicast(HashSet::new(), MAX_EXCHANGE_BODY_BYTES).unwrap(); + assert_eq!(batches.len(), 1); + assert!(batches[0].is_empty()); + } + + #[test] + fn exchange_set_within_budget_stays_in_one_batch() { + let groups: HashSet<_> = (0..8).map(|i| vector(i, 3)).collect(); + let batches = + batch_multicast(groups.clone(), MAX_EXCHANGE_BODY_BYTES).unwrap(); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0], groups); + } + + #[test] + fn oversized_exchange_set_splits_and_preserves_every_vector() { + let groups: HashSet<_> = (0..10).map(|i| vector(i, 3)).collect(); + let per = groups + .iter() + .map(|v| element_len(v).unwrap()) + .max() + .unwrap(); + let limit = update_envelope_len().unwrap() + per * 3; + + let batches = batch_multicast(groups.clone(), limit).unwrap(); + assert!(batches.len() > 1); + + // Measure the update each batch actually serializes to, rather than + // re-deriving the sizing the implementation used to build it. + for batch in &batches { + let body = serde_json::to_string(&Update::from( + MulticastUpdate::announce(batch.clone()), + )) + .unwrap(); + assert!(body.len() <= limit, "batch body {} > {limit}", body.len()); + } + + let flattened: HashSet<_> = batches.into_iter().flatten().collect(); + assert_eq!(flattened, groups); + } + + #[test] + fn vector_larger_than_budget_is_an_error() { + let group = vector(0, 3); + let limit = + update_envelope_len().unwrap() + element_len(&group).unwrap() - 1; + + let err = batch_multicast(HashSet::from([group.clone()]), limit) + .expect_err("a vector exceeding the budget cannot be sent"); + assert!(matches!( + err, + ExchangeError::MulticastVectorTooLarge { group: g, .. } + if g == group.origin.overlay_group + )); + } + + #[test] + fn oversized_group_is_rejected_by_page_multicast() { + let group = vector(0, 3); + let groups = group_by_origin([group.clone()]); + let limit = response_envelope_len() + + MAX_PAGE_TOKEN_BYTES + + element_len(&group).unwrap() + - 1; + + let err = page_multicast(&groups, None, limit, response_envelope_len()) + .expect_err("a group exceeding the page budget cannot be served"); + assert!(matches!( + err, + ExchangeError::MulticastVectorTooLarge { group: g, .. } + if g == group.origin.overlay_group + )); + } + + #[test] + fn snapshot_within_budget_is_one_page() { + let groups = group_by_origin((0..8).map(|i| vector(i, 3))); + let (seen, pages) = walk(&groups, MAX_EXCHANGE_BODY_BYTES); + assert_eq!(pages, 1); + assert_eq!(seen.len(), 8); + } + + #[test] + fn page_budget_includes_unicast_sections_and_cursor() { + let underlay = HashSet::from([v3::PathVector { + destination: "2001:db8::/64".parse().unwrap(), + path: vec![String::from("router")], + }]); + let tunnel = HashSet::from([v3::TunnelOrigin { + overlay_prefix: "192.0.2.0/24".parse().unwrap(), + boundary_addr: "2001:db8::1".parse().unwrap(), + vni: 100, + metric: 0, + }]); + let groups = group_by_origin((0..6).map(|i| vector(i, 3))); + let per = groups + .values() + .flatten() + .map(|v| element_len(v).unwrap()) + .max() + .unwrap(); + let envelope = + super::response_envelope_len(Some(&underlay), Some(&tunnel)) + .unwrap(); + let limit = envelope + MAX_PAGE_TOKEN_BYTES + per * 2; + + let (page, next) = + page_multicast(&groups, None, limit, envelope).unwrap(); + let token = next.as_ref().map(encode_page_token).transpose().unwrap(); + + assert!(next.is_some(), "the test set should require another page"); + assert!( + body_len_with_sections( + &page, + token.as_deref(), + Some(&underlay), + Some(&tunnel), + ) <= limit + ); + } + + #[test] + fn keyset_walk_reads_every_group_exactly_once() { + let vectors: Vec<_> = (0..10).map(|i| vector(i, 3)).collect(); + let groups = group_by_origin(vectors.clone()); + let per = vectors + .iter() + .map(|v| element_len(v).unwrap()) + .max() + .unwrap(); + let limit = response_envelope_len() + MAX_PAGE_TOKEN_BYTES + per * 3; + + let (seen, pages) = walk(&groups, limit); + assert!(pages > 1); + assert_eq!(seen, vectors.into_iter().collect::>()); + } + + /// A group with several path vectors is announced as a unit. Splitting it + /// would read as a partial announcement, since reconciliation is keyed on + /// the group rather than on the vector. + #[test] + fn a_group_is_never_split_across_pages() { + let mut groups = GroupedVectors::new(); + for i in 0..6u16 { + groups.insert( + origin(i), + (0..3).map(|hop| vector(i, hop + 1)).collect(), + ); + } + let limit = response_envelope_len() + + MAX_PAGE_TOKEN_BYTES + + groups + .values() + .map(|v| { + v.iter().map(|v| element_len(v).unwrap()).sum::() + }) + .max() + .unwrap() + * 2; + + let mut selector: Option = None; + let mut pages = 0; + loop { + let after = selector.as_ref().map(MulticastPageSelector::after); + let (page, next) = page_multicast( + &groups, + after.as_ref(), + limit, + response_envelope_len(), + ) + .unwrap(); + pages += 1; + for (origin, vectors) in &groups { + let present = + vectors.iter().filter(|v| page.contains(*v)).count(); + assert!( + present == 0 || present == vectors.len(), + "group {} split: {present} of {} vectors", + origin.overlay_group, + vectors.len(), + ); + } + match next { + Some(next) => selector = Some(next), + None => break, + } + } + assert!(pages > 1); + } + + /// A group inserted beyond the cursor during a keyset walk is still read, + /// and one removed beyond the cursor is correctly absent. Neither disturbs + /// the groups the keyset walk already passed. + #[test] + fn change_beyond_the_cursor_is_picked_up_by_the_same_keyset_walk() { + let mut groups = group_by_origin((0..6).map(|i| vector(i, 2))); + let per = groups + .values() + .flatten() + .map(|v| element_len(v).unwrap()) + .max() + .unwrap(); + let limit = response_envelope_len() + MAX_PAGE_TOKEN_BYTES + per * 2; + + let (first, next) = + page_multicast(&groups, None, limit, response_envelope_len()) + .unwrap(); + let selector = next.expect("six groups exceed a two-group page"); + + // The last group sorts beyond any page boundary this keyset walk has + // reached, so removing it now is a change the keyset walk has not + // passed. + let removed = groups.keys().next_back().unwrap().clone(); + groups.remove(&removed); + + let mut seen = first; + let mut selector = Some(selector); + while let Some(current) = selector { + let after = current.after(); + let (page, next) = page_multicast( + &groups, + Some(&after), + limit, + response_envelope_len(), + ) + .unwrap(); + seen.extend(page); + selector = next; + } + + assert!(!seen.iter().any(|v| v.origin == removed)); + assert_eq!(seen.len(), 5); + } + + /// A group present for the whole keyset walk appears in some page even when + /// groups are inserted ahead of the cursor, which is what keeps the + /// reader from synthesizing a withdrawal for it. + #[test] + fn insertion_before_the_cursor_cannot_drop_a_present_group() { + let stable: Vec<_> = (10..16u16).map(|i| vector(i, 2)).collect(); + let mut groups = group_by_origin(stable.clone()); + let per = stable + .iter() + .map(|v| element_len(v).unwrap()) + .max() + .unwrap(); + let limit = response_envelope_len() + MAX_PAGE_TOKEN_BYTES + per * 2; + + let mut seen = HashSet::new(); + let mut selector: Option = None; + let mut inserted = 0u16; + + loop { + let after = selector.as_ref().map(MulticastPageSelector::after); + let (page, next) = page_multicast( + &groups, + after.as_ref(), + limit, + response_envelope_len(), + ) + .unwrap(); + seen.extend(page); + + // Insert a group that sorts ahead of every stable group, so it + // lands behind the cursor on each subsequent request. + let early = vector(inserted, 2); + groups.insert(early.origin.clone(), vec![early]); + inserted += 1; + + match next { + Some(next) => selector = Some(next), + None => break, + } + } + + for vector in &stable { + assert!( + seen.contains(vector), + "group {} was present throughout but went unread", + vector.origin.overlay_group, + ); + } + } + + #[test] + fn a_group_round_trips_through_its_token() { + let mut source_specific = origin(3); + source_specific.source = Some("192.0.2.7".parse().unwrap()); + let selector = MulticastPageSelector { + last_seen: (&source_specific).into(), + }; + let token = encode_page_token(&selector).unwrap(); + assert!(token.len() <= MAX_PAGE_TOKEN_BYTES); + + let params: dropshot::PaginationParams< + EmptyScanParams, + MulticastPageSelector, + > = serde_json::from_value(serde_json::json!({ + "page_token": token, + })) + .unwrap(); + match params.page { + dropshot::WhichPage::Next(decoded) => { + assert_eq!(decoded, selector); + assert_eq!(decoded.after(), source_specific); + } + dropshot::WhichPage::First(_) => { + panic!("a page token must select the next page") + } + } + + let any_source = origin(3); + let selector = MulticastPageSelector { + last_seen: (&any_source).into(), + }; + assert_eq!(selector.after(), any_source); + } + + /// A token rides in a query value, so its URL-safe encoding needs no + /// additional escaping. Dropshot retains base64 padding. + #[test] + fn a_token_is_query_safe() { + let selector = MulticastPageSelector { + last_seen: (&origin(3)).into(), + }; + let token = encode_page_token(&selector).unwrap(); + assert!( + token.chars().all(|c| { + c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '=' + }), + "token {token} would need escaping", + ); + } + + #[test] + fn invalid_page_tokens_are_rejected_before_uri_construction() { + for token in ["", "not valid", "%", "a/b", &"a".repeat(513)] { + assert!( + page_token_query(token).is_err(), + "token {token:?} should be rejected", + ); + } + + assert_eq!(page_token_query("abc-_==").unwrap(), "page_token=abc-_==",); + } +} diff --git a/ddm/src/exchange/reconcile.rs b/ddm/src/exchange/reconcile.rs new file mode 100644 index 000000000..cc2e67d6e --- /dev/null +++ b/ddm/src/exchange/reconcile.rs @@ -0,0 +1,262 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Side-effect-free reconciliation management shared by exchange +//! runtime paths, taking state in as arguments and returning updates +//! without database or network effects. + +use ddm_api_types::net::MulticastOrigin; +use ddm_protocol::v4::{ + MulticastPathHop, MulticastPathVector, MulticastUpdate, +}; + +/// Rewrite each withdrawal as a replacement announcement when another path to +/// the origin remains, and as a final withdrawal otherwise. +/// +/// Downstream peers keep exactly one route per `(origin, nexthop)`, with this +/// router as the nexthop, regardless of how many paths line up behind it. +/// Blindly forwarding a withdrawal would drop the peer's only route through +/// this router even when the origin is still reachable in another way. For each +/// withdrawal, this checks the local origins first, then the best entry in +/// the imported set. A remaining path produces an announcement with a +/// refreshed path vector. +/// +/// Only when nothing remains is the final withdrawal emitted with `local_hop` +/// appended. +/// +/// The reachability snapshot always describes post-modification state. Peer +/// expiry and renumber capture it in `remove_nexthop_routes`, exchange +/// updates in `update_imported_mcast`, and the admin withdraws read it at +/// processing time via `multicast_reachability`, which is still +/// post-modification because the event is enqueued only after the +/// modification lands. +pub(crate) fn reconcile_multicast_withdrawals<'a>( + withdrawals: impl IntoIterator, + reachability: &crate::db::MulticastReachability, + local_hop: &MulticastPathHop, +) -> MulticastUpdate { + let mut update = MulticastUpdate::default(); + + for withdrawal in withdrawals { + let replacement = MulticastOrigin::try_from(&withdrawal.origin) + .ok() + .and_then(|origin| { + if let Some(local_origin) = + reachability.originated().get(&origin) + { + return Some(MulticastPathVector { + origin: local_origin.into(), + path: vec![local_hop.clone()], + }); + } + + reachability + .imported() + .iter() + .filter(|route| route.origin == origin) + // Route identity guarantees one entry per nexthop for this + // origin. Address order is only a stable tie-breaker (it + // does not assign semantics to multicast metric). + .min_by_key(|route| route.nexthop) + .map(|route| { + let mut path = route.path.clone(); + path.push(local_hop.clone()); + MulticastPathVector { + origin: (&route.origin).into(), + path, + } + }) + }); + + match replacement { + Some(replacement) => { + update.announce.insert(replacement); + } + // A degraded snapshot cannot confirm the absence of a local + // origin, so a final withdrawal here could tear down a route to + // an origin that is still reachable. Dropping the withdrawal is + // the safe direction. The worst case is a transient stale route + // to an origin that really is gone, while a false withdrawal + // could remove a peer's only path through this router. The + // periodic exchange resync repairs the drift either way. + None if reachability.origins_degraded() => {} + None => { + update + .withdraw + .insert(withdrawal.with_hop(local_hop.clone())); + } + } + } + + update +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::{Db, MulticastReachability}; + use ddm_api_types::db::MulticastRoute; + use slog::Logger; + use std::collections::HashSet; + use std::net::Ipv6Addr; + use tempfile::TempDir; + + fn origin(metric: u64) -> MulticastOrigin { + serde_json::from_value(serde_json::json!({ + "overlay_group": "233.252.0.1", + "underlay_group": "ff04::1", + "vni": 77, + "metric": metric, + })) + .unwrap() + } + + fn hop(router_id: &str, last: u16) -> MulticastPathHop { + MulticastPathHop::new( + router_id.to_string(), + Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, last), + ) + } + + /// Build a `MulticastReachability` snapshot through `Db` persistence + /// rather than constructing it by hand, so tests exercise the same + /// snapshot path production uses. Any origins in `originated` are + /// persisted before the imported set is applied, so the captured + /// snapshot reflects both sources of reachability. + fn snapshot( + imported: HashSet, + originated: HashSet, + ) -> (TempDir, MulticastReachability) { + let dir = TempDir::new().unwrap(); + let log = Logger::root(slog::Discard, slog::o!()); + let db = Db::new(dir.path().to_str().unwrap(), log).unwrap(); + if !originated.is_empty() { + db.originate_mcast(&originated).unwrap(); + } + let (_delta, reachability) = db + .update_imported_mcast_with_reachability( + &imported, + &HashSet::new(), + ); + (dir, reachability) + } + + #[test] + fn remaining_import_replaces_withdrawal_and_refreshes_path() { + let origin = origin(10); + let withdrawn = MulticastPathVector { + origin: (&origin).into(), + path: vec![hop("withdrawn", 1)], + }; + let remaining_hop = hop("remaining", 2); + let imported = HashSet::from([MulticastRoute { + origin: origin.clone(), + nexthop: "fe80::2".parse().unwrap(), + path: vec![remaining_hop.clone()], + }]); + let local_hop = hop("local", 3); + let (_dir, remaining) = snapshot(imported, HashSet::new()); + + let update = reconcile_multicast_withdrawals( + [&withdrawn], + &remaining, + &local_hop, + ); + + assert!(update.withdraw.is_empty()); + let replacement = update.announce.iter().next().unwrap(); + assert_eq!(replacement.path, vec![remaining_hop, local_hop]); + } + + #[test] + fn remaining_local_origin_replaces_withdrawal() { + let origin = origin(10); + let withdrawn = MulticastPathVector { + origin: (&origin).into(), + path: vec![hop("withdrawn", 1)], + }; + let local_hop = hop("local", 3); + let (_dir, remaining) = + snapshot(HashSet::new(), HashSet::from([origin])); + + let update = reconcile_multicast_withdrawals( + [&withdrawn], + &remaining, + &local_hop, + ); + + assert!(update.withdraw.is_empty()); + let replacement = update.announce.iter().next().unwrap(); + assert_eq!(replacement.path, vec![local_hop]); + } + + #[test] + fn final_withdrawal_preserves_path_and_appends_local_hop() { + let origin = origin(10); + let withdrawn_hop = hop("withdrawn", 1); + let withdrawn = MulticastPathVector { + origin: (&origin).into(), + path: vec![withdrawn_hop.clone()], + }; + let local_hop = hop("local", 3); + let (_dir, remaining) = snapshot(HashSet::new(), HashSet::new()); + + let update = reconcile_multicast_withdrawals( + [&withdrawn], + &remaining, + &local_hop, + ); + + assert!(update.announce.is_empty()); + let forwarded = update.withdraw.iter().next().unwrap(); + assert_eq!(forwarded.path, vec![withdrawn_hop, local_hop]); + } + + /// The processing-time read path used by admin withdraw + /// revalidation must observe both persisted origins and the current + /// imported set. + #[test] + fn multicast_reachability_reads_current_imported_and_originated() { + let dir = TempDir::new().unwrap(); + let log = Logger::root(slog::Discard, slog::o!()); + let db = Db::new(dir.path().to_str().unwrap(), log).unwrap(); + + // Distinct overlay/underlay groups so identity-based equality on + // `MulticastOrigin` keeps the persisted and imported origins apart. + let local_origin: MulticastOrigin = + serde_json::from_value(serde_json::json!({ + "overlay_group": "233.252.0.1", + "underlay_group": "ff04::1", + "vni": 77, + "metric": 10, + })) + .unwrap(); + + let imported_origin: MulticastOrigin = + serde_json::from_value(serde_json::json!({ + "overlay_group": "233.252.0.2", + "underlay_group": "ff04::2", + "vni": 77, + "metric": 20, + })) + .unwrap(); + + let route = MulticastRoute { + origin: imported_origin, + nexthop: "fe80::2".parse().unwrap(), + path: vec![hop("remote", 2)], + }; + + db.originate_mcast(&HashSet::from([local_origin.clone()])) + .unwrap(); + db.update_imported_mcast( + &HashSet::from([route.clone()]), + &HashSet::new(), + ); + + let reachability = db.multicast_reachability(); + assert_eq!(reachability.imported(), &HashSet::from([route])); + assert_eq!(reachability.originated(), &HashSet::from([local_origin])); + } +} diff --git a/ddm/src/exchange/runtime.rs b/ddm/src/exchange/runtime.rs index 653824e43..568fa12b0 100644 --- a/ddm/src/exchange/runtime.rs +++ b/ddm/src/exchange/runtime.rs @@ -7,28 +7,38 @@ //! plumbing that drains received updates into the local DB and the //! forwarding platform via [`crate::sys`]. illumos-only. -use super::ExchangeError; +use super::{ExchangeError, paging, reconcile_multicast_withdrawals}; use crate::db::{Route, effective_route_set}; use crate::discovery::Version; use crate::sm::{Config, Event, PeerEvent, SmContext}; -use crate::{dbg, err, inf, wrn}; -use ddm_api_types::db::{RouterKind, TunnelRoute}; +use crate::{dbg, err, inf, trc, wrn}; +use ddm_api_types::db::{MulticastRoute, RouterKind, TunnelRoute}; +use ddm_api_types::net::MulticastOrigin; +use ddm_protocol::v3::{PathVector, TunnelOrigin}; +use ddm_protocol::v4::{ + MulticastPathHop, MulticastPathVector, MulticastUpdate, PullResponse, + Update, +}; use ddm_protocol::{v2, v3}; use dropshot::ApiDescription; use dropshot::ConfigDropshot; use dropshot::ConfigLogging; use dropshot::ConfigLoggingLevel; +use dropshot::EmptyScanParams; use dropshot::HttpError; use dropshot::HttpResponseOk; use dropshot::HttpResponseUpdatedNoContent; use dropshot::HttpServerStarter; +use dropshot::PaginationParams; use dropshot::RequestContext; use dropshot::TypedBody; +use dropshot::WhichPage; use dropshot::{ApiDescriptionRegisterError, endpoint}; -use http_body_util::BodyExt; +use http_body_util::{BodyExt, Limited}; use hyper::body::Bytes; use hyper_util::client::legacy::Client; use hyper_util::rt::TokioExecutor; +use mg_common::lock; use slog::{Logger, o}; use std::collections::HashSet; use std::net::{Ipv6Addr, SocketAddrV6}; @@ -47,6 +57,47 @@ pub struct HandlerContext { log: Logger, } +/// How an update's imported routes propagate beyond the local DB. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum UpdateMode { + /// Import into the local DB only. + ImportOnly, + /// Import and re-announce to this router's other peers. Only transit + /// routers act on this. Server routers treat it as [`Self::ImportOnly`]. + Redistribute, +} + +/// A handle to a running exchange server, pairing the server task with the +/// shared request context so the state machine can rebind the peer address +/// on renumber without restarting the server. +/// +/// A renumber occurs when a peer's link-local unicast address changes. +/// [`crate::discovery`] detects the change and re-advertises the neighbor +/// under the new address. The neighbor is still the same router, so the +/// exchange server keeps running and only the nexthop address it assigns to +/// imports changes. +pub struct ExchangeHandle { + thread: tokio::task::JoinHandle<()>, + context: Arc>, +} + +impl ExchangeHandle { + pub fn abort(&self) { + self.thread.abort(); + } + + /// Rebind the peer address used by the handler after a renumber. + /// + /// The handler assigns this address as the nexthop on every route it + /// imports, so it must track the state machine's view of the peer or else, + /// post-renumber imports leak under the prior address. + pub fn renumber_peer(&self, peer: Ipv6Addr) { + // Safe to block: callers run on state machine threads, outside + // the runtime. + self.context.blocking_lock().peer = peer; + } +} + pub(crate) fn announce_underlay( ctx: &SmContext, config: Config, @@ -57,7 +108,17 @@ pub(crate) fn announce_underlay( log: Logger, ) -> Result<(), ExchangeError> { let update = v3::UnderlayUpdate::announce(prefixes); - send_update(ctx, config, update.into(), addr, version, rt, log) + send_update( + SendCtx { + ctx, + config, + addr, + version, + rt, + log, + }, + update.into(), + ) } pub(crate) fn announce_tunnel( @@ -70,7 +131,17 @@ pub(crate) fn announce_tunnel( log: Logger, ) -> Result<(), ExchangeError> { let update = v3::TunnelUpdate::announce(endpoints.into_iter().collect()); - send_update(ctx, config, update.into(), addr, version, rt, log) + send_update( + SendCtx { + ctx, + config, + addr, + version, + rt, + log, + }, + update.into(), + ) } pub(crate) fn withdraw_underlay( @@ -83,7 +154,17 @@ pub(crate) fn withdraw_underlay( log: Logger, ) -> Result<(), ExchangeError> { let update = v3::UnderlayUpdate::withdraw(prefixes); - send_update(ctx, config, update.into(), addr, version, rt, log) + send_update( + SendCtx { + ctx, + config, + addr, + version, + rt, + log, + }, + update.into(), + ) } pub(crate) fn withdraw_tunnel( @@ -96,19 +177,154 @@ pub(crate) fn withdraw_tunnel( log: Logger, ) -> Result<(), ExchangeError> { let update = v3::TunnelUpdate::withdraw(endpoints.into_iter().collect()); - send_update(ctx, config, update.into(), addr, version, rt, log) + send_update( + SendCtx { + ctx, + config, + addr, + version, + rt, + log, + }, + update.into(), + ) } -pub(crate) fn do_pull( +pub(crate) fn announce_multicast( + ctx: &SmContext, + config: Config, + groups: HashSet, + addr: Ipv6Addr, + version: Version, + rt: Arc, + log: Logger, +) -> Result<(), ExchangeError> { + send_multicast( + SendCtx { + ctx, + config, + addr, + version, + rt, + log, + }, + groups, + MulticastUpdate::announce, + ) +} + +pub(crate) fn withdraw_multicast( + ctx: &SmContext, + config: Config, + groups: HashSet, + addr: Ipv6Addr, + version: Version, + rt: Arc, + log: Logger, +) -> Result<(), ExchangeError> { + send_multicast( + SendCtx { + ctx, + config, + addr, + version, + rt, + log, + }, + groups, + MulticastUpdate::withdraw, + ) +} + +/// The peer an update is addressed to, and the resources used to send it. +#[derive(Clone)] +struct SendCtx<'a> { + ctx: &'a SmContext, + config: Config, + addr: Ipv6Addr, + version: Version, + rt: Arc, + log: Logger, +} + +/// Send a multicast exchange set to a peer, splitting it across as many pushes +/// as the body limit requires. +/// +/// Chunking makes an update non-atomic at the receiver, which is safe here +/// because `reconcile_multicast_withdrawals` partitions origins so that the +/// announce and withdraw sets are disjoint. No chunk can contradict another, +/// so a partially applied update converges rather than flapping, and the +/// peer's periodic V4 pull repairs whatever a failed chunk left behind. +fn send_multicast( + send: SendCtx<'_>, + groups: HashSet, + build: fn(HashSet) -> MulticastUpdate, +) -> Result<(), ExchangeError> { + // Multicast first appears on the wire in V4. For an earlier peer + // `send_update` downconverts to an empty payload and skips the send, so + // batching would only produce repeated skips. + if send.version < Version::V4 { + let update = build(groups); + return send_update(send, update.into()); + } + + let batches = + paging::batch_multicast(groups, paging::MAX_EXCHANGE_BODY_BYTES)?; + let total = batches.len(); + for (i, batch) in batches.into_iter().enumerate() { + let update = build(batch); + let chunk = i + 1; + if let Err(e) = send_update(send.clone(), update.into()) { + err!( + send.log, + send.config.if_name, + "multicast chunk {chunk} of {total} to {}: {e}", + send.addr, + ); + return Err(e); + } + } + Ok(()) +} + +/// Fetch one page of a peer's V4 snapshot, resuming after `page_token` when the +/// preceding page reported more to read. +/// +/// The token is echoed back to the peer that issued it. It uses Dropshot's +/// URL-safe page-token alphabet, including its optional padding. +pub(crate) fn do_pull_v4( + ctx: &SmContext, + addr: &Ipv6Addr, + rt: &Arc, + page_token: Option<&str>, +) -> Result { + let if_index = ctx.config.if_index; + let port = ctx.config.exchange_port; + let base = format!("http://[{addr}%{if_index}]:{port}/v4/pull"); + let uri = match page_token { + Some(token) => format!("{base}?{}", paging::page_token_query(token)?), + None => base, + }; + let timeout = Duration::from_millis(ctx.config.exchange_timeout); + let body = do_pull_common( + uri, + timeout, + Some(paging::MAX_EXCHANGE_BODY_BYTES), + rt, + )?; + Ok(serde_json::from_slice(&body)?) +} + +pub(crate) fn do_pull_v3( ctx: &SmContext, addr: &Ipv6Addr, rt: &Arc, ) -> Result { - let uri = format!( - "http://[{}%{}]:{}/v3/pull", - addr, ctx.config.if_index, ctx.config.exchange_port, - ); - let body = do_pull_common(uri, rt)?; + let if_index = ctx.config.if_index; + let port = ctx.config.exchange_port; + let uri = format!("http://[{addr}%{if_index}]:{port}/v3/pull"); + let timeout = Duration::from_millis(ctx.config.exchange_timeout); + let body = do_pull_common(uri, timeout, None, rt)?; Ok(serde_json::from_slice(&body)?) } @@ -121,12 +337,30 @@ pub(crate) fn do_pull_v2( "http://[{}%{}]:{}/v2/pull", addr, ctx.config.if_index, ctx.config.exchange_port, ); - let body = do_pull_common(uri, rt)?; + let timeout = Duration::from_millis(ctx.config.exchange_timeout); + let body = do_pull_common(uri, timeout, None, rt)?; Ok(serde_json::from_slice(&body)?) } +fn require_success( + response: hyper::Response, +) -> Result, ExchangeError> { + if response.status().is_success() { + Ok(response) + } else { + Err(ExchangeError::Status(response.status())) + } +} + +/// Fetch a pull response body, accepting only successful HTTP responses. +/// +/// The status is checked before the body reaches a versioned decoder. This is +/// especially important for V4, whose optional response fields could otherwise +/// make a Dropshot JSON error look like an empty route set. fn do_pull_common( uri: String, + timeout_duration: Duration, + max_body_bytes: Option, rt: &Arc, ) -> Result { let client = Client::builder(TokioExecutor::new()).build_http(); @@ -135,92 +369,241 @@ fn do_pull_common( .method(hyper::Method::GET) .uri(&uri) .body(http_body_util::Empty::::new()) - .unwrap(); + .map_err(|e| ExchangeError::InvalidUri(e.to_string()))?; let resp = client.request(req); + // The timeout covers reading the body too, since a peer that stalls + // mid-response would otherwise block indefinitely. V4 responses are capped + // below. V2 and V3 retain the legacy behavior and are read without a size + // limit, since neither carries the unbounded multicast section that the + // cap exists for. rt.block_on(async move { - let body = timeout(Duration::from_millis(250), resp) - .await?? - .into_body() - .collect() - .await? - .to_bytes(); - Ok(body) + timeout(timeout_duration, async { + let resp = require_success(resp.await?)?; + // The Dropshot request cap bounds what a peer may PUT to us. It + // does not constrain a pull response, so the V4 reader applies + // its own bound here. + let body = resp.into_body(); + match max_body_bytes { + Some(limit) => Limited::new(body, limit) + .collect() + .await + .map(|b| b.to_bytes()) + .map_err(|e| match e.downcast::() { + Ok(e) => ExchangeError::Hyper(*e), + Err(_) => ExchangeError::ResponseTooLarge { limit }, + }), + None => body + .collect() + .await + .map(|b| b.to_bytes()) + .map_err(ExchangeError::Hyper), + } + }) + .await? }) } +/// Pull the peer's routes and import them. +/// +/// When `mode` is [`UpdateMode::Redistribute`] and this router is a transit, +/// the imported set is also announced to the other peers. The initial pull on +/// entering the exchange state redistributes. The periodic pull imports only. +/// Each router runs its own periodic pull, so a route learned here still +/// reaches every router through that router's pull. Redistributing on every +/// cycle would only resend updates transit peers already hold in steady +/// state. +/// +/// A non-successful HTTP response aborts the pull before decoding or +/// reconciliation, leaving the routes previously imported from that peer +/// unchanged. +/// +/// Only V4 carries multicast and pages. Earlier versions take the whole +/// snapshot in one response and have no withdrawals to reconcile, since their +/// underlay and tunnel imports keep push-only withdraw semantics. pub(crate) fn pull( ctx: SmContext, addr: Ipv6Addr, version: Version, rt: Arc, log: Logger, + mode: UpdateMode, + stop: Option<&std::sync::atomic::AtomicBool>, ) -> Result<(), ExchangeError> { - let pr: v3::PullResponse = match version { - Version::V2 => do_pull_v2(&ctx, &addr, &rt)?.into(), - Version::V3 => do_pull(&ctx, &addr, &rt)?, - }; + if stop.is_some_and(|stop| stop.load(Ordering::Relaxed)) { + return Ok(()); + } - let update = v3::Update::announce(pr); + let pr: PullResponse = match version { + Version::V2 => { + v3::PullResponse::from(do_pull_v2(&ctx, &addr, &rt)?).into() + } + Version::V3 => do_pull_v3(&ctx, &addr, &rt)?.into(), + Version::V4 => { + return pull_v4(ctx, addr, rt, log, mode, stop); + } + }; - let hctx = HandlerContext { + let handler = HandlerContext { ctx, peer: addr, - log: log.clone(), + log, }; - handle_update(&update, &hctx); + handle_update(&Update::announce(pr), &handler, mode); Ok(()) } -fn send_update( - ctx: &SmContext, - config: Config, - update: v3::Update, +/// A bound on the pages one V4 pull will read. +/// +/// A responder that keeps issuing tokens would otherwise hold the reader in a +/// pagination sequence that never ends. At the body limit, this allows a +/// multicast exchange set far beyond any rack's, so reaching it means the peer +/// is faulty rather than large. +const MAX_PULL_PAGES: usize = 64; + +/// Read a peer's V4 snapshot using keyset pagination, importing each page and +/// reconciling withdrawals once pagination completes. +/// +/// Each page's announcements are imported on arrival rather than accumulated. +/// Announcements are additive, so aborted pagination leaves a subset of the +/// peer's snapshot imported and synthesizes no withdrawals, which the next +/// cycle then repairs. Only the announced group identities are carried across +/// pages, so the reader holds one page plus that set rather than the whole +/// snapshot. +/// +/// Withdrawal synthesis waits for the final page because it reads an imported +/// route the peer did not announce as one whose withdraw was missed. Against a +/// partial snapshot that treats every unread group as withdrawn, which would +/// drain the replication members those groups still need. +fn pull_v4( + ctx: SmContext, addr: Ipv6Addr, - version: Version, rt: Arc, log: Logger, + mode: UpdateMode, + stop: Option<&std::sync::atomic::AtomicBool>, ) -> Result<(), ExchangeError> { - ctx.stats.updates_sent.fetch_add(1, Ordering::Relaxed); - match version { - Version::V2 => { - send_update_v2(ctx, config, update.into(), addr, rt, log) + let handler = HandlerContext { + ctx: ctx.clone(), + peer: addr, + log: log.clone(), + }; + + let mut announced: HashSet = HashSet::new(); + let mut page_token: Option = None; + let mut pages = 0usize; + + loop { + if stop.is_some_and(|stop| stop.load(Ordering::Relaxed)) { + return Ok(()); + } + + let pr = do_pull_v4(&ctx, &addr, &rt, page_token.as_deref())?; + if stop.is_some_and(|stop| stop.load(Ordering::Relaxed)) { + return Ok(()); + } + pages += 1; + + announced.extend( + pr.multicast + .iter() + .flatten() + .filter_map(|pv| MulticastOrigin::try_from(&pv.origin).ok()), + ); + + page_token = pr.next_page_token.clone(); + handle_update(&Update::announce(pr), &handler, mode); + + if page_token.is_none() { + break; + } + if pages >= MAX_PULL_PAGES { + return Err(ExchangeError::PullTooManyPages { + limit: MAX_PULL_PAGES, + }); } - Version::V3 => send_update_v3(ctx, config, update, addr, rt, log), } -} -fn send_update_v2( - ctx: &SmContext, - config: Config, - update: v2::Update, - addr: Ipv6Addr, - rt: Arc, - log: Logger, -) -> Result<(), ExchangeError> { - let payload = serde_json::to_string(&update)?; - let uri = format!( - "http://[{}%{}]:{}/v2/push", - addr, config.if_index, config.exchange_port, - ); - send_update_common(ctx, uri, payload, config, rt, log) + if pages > 1 { + dbg!( + log, + ctx.config.if_name, + "pull read {pages} pages from {addr}" + ); + } + + if stop.is_some_and(|stop| stop.load(Ordering::Relaxed)) { + return Ok(()); + } + + let withdraw: HashSet = ctx + .db + .imported_mcast() + .iter() + .filter(|route| { + route.nexthop == addr && !announced.contains(&route.origin) + }) + .map(|route| MulticastPathVector { + origin: (&route.origin).into(), + path: Vec::new(), + }) + .collect(); + + if !withdraw.is_empty() { + dbg!( + log, + ctx.config.if_name, + "pull reconcile: withdrawing {} stale multicast routes", + withdraw.len(), + ); + handle_update( + &MulticastUpdate::withdraw(withdraw).into(), + &handler, + mode, + ); + } + + Ok(()) } -fn send_update_v3( - ctx: &SmContext, - config: Config, - update: v3::Update, - addr: Ipv6Addr, - rt: Arc, - log: Logger, -) -> Result<(), ExchangeError> { - let payload = serde_json::to_string(&update)?; - let uri = format!( - "http://[{}%{}]:{}/v3/push", - addr, config.if_index, config.exchange_port, - ); +fn send_update(send: SendCtx<'_>, update: Update) -> Result<(), ExchangeError> { + let SendCtx { + ctx, + config, + addr, + version, + rt, + log, + } = send; + // The update arrives in the latest wire form. Downconvert through + // consecutive versions when a peer negotiated an older protocol. + // Conversion drops content the peer's version cannot represent (multicast + // did not exist before V4, for example), so the downconverted form can + // be empty. We skip the send in that case rather than emit an empty + // payload. + let (payload, path) = match version { + Version::V2 => { + let update = v2::Update::from(v3::Update::from(update)); + if update.underlay.is_none() && update.tunnel.is_none() { + return Ok(()); + } + (serde_json::to_string(&update)?, "v2") + } + Version::V3 => { + let update = v3::Update::from(update); + if update.underlay.is_none() && update.tunnel.is_none() { + return Ok(()); + } + (serde_json::to_string(&update)?, "v3") + } + Version::V4 => (serde_json::to_string(&update)?, "v4"), + }; + ctx.stats.updates_sent.fetch_add(1, Ordering::Relaxed); + let if_index = config.if_index; + let port = config.exchange_port; + let uri = format!("http://[{addr}%{if_index}]:{port}/{path}/push"); send_update_common(ctx, uri, payload, config, rt, log) } @@ -243,23 +626,23 @@ fn send_update_common( let resp = client.request(req); + // A completed request only counts as delivered when the peer's handler + // reported success. Connection failures and error statuses must surface as + // errors so the state machine can expire the peer. rt.block_on(async move { - match timeout(Duration::from_millis(config.exchange_timeout), resp) - .await - { - Ok(_) => Ok(()), - Err(e) => { - err!( - log, - config.if_name, - "peer request timeout to {}: {}", - uri, - e, - ); - ctx.stats.update_send_fail.fetch_add(1, Ordering::Relaxed); - Err(e.into()) - } + let result: Result<(), ExchangeError> = async { + let resp = + timeout(Duration::from_millis(config.exchange_timeout), resp) + .await??; + require_success(resp)?; + Ok(()) + } + .await; + if let Err(e) = &result { + err!(log, config.if_name, "peer update to {uri} failed: {e}"); + ctx.stats.update_send_fail.fetch_add(1, Ordering::Relaxed); } + result }) } @@ -268,17 +651,21 @@ pub fn handler( addr: Ipv6Addr, peer: Ipv6Addr, log: Logger, -) -> Result, String> { +) -> Result { let context = Arc::new(Mutex::new(HandlerContext { ctx: ctx.clone(), log: log.clone(), peer, })); + let handler_ctx = Arc::clone(&context); let sa = SocketAddrV6::new(addr, ctx.config.exchange_port, 0, 0); let config = ConfigDropshot { bind_address: sa.into(), + // Dropshot's default request body cap is 1 KiB, which is too small for + // batched V4 multicast updates. + default_request_body_max_bytes: paging::MAX_EXCHANGE_BODY_BYTES, ..Default::default() }; @@ -307,7 +694,7 @@ pub fn handler( } })?; - Ok(ctx.rt.spawn(async move { + let thread = ctx.rt.spawn(async move { match server.start().await { Ok(_) => wrn!( log, @@ -321,7 +708,12 @@ pub fn handler( e ), } - })) + }); + + Ok(ExchangeHandle { + thread, + context: handler_ctx, + }) } pub fn api_description() -> Result< @@ -330,9 +722,11 @@ pub fn api_description() -> Result< > { let mut api = ApiDescription::new(); api.register(push_handler_v2)?; - api.register(push_handler)?; + api.register(push_handler_v3)?; + api.register(push_handler_v4)?; api.register(pull_handler_v2)?; - api.register(pull_handler)?; + api.register(pull_handler_v3)?; + api.register(pull_handler_v4)?; Ok(api) } @@ -345,17 +739,23 @@ async fn push_handler_v2( request: TypedBody, ) -> Result { let update_v2 = request.into_inner(); - let update = v3::Update::from(update_v2); + let update = Update::from(v3::Update::from(update_v2)); push_handler_common(ctx, update).await } -#[endpoint { - method = PUT, - path = "/v3/push", -}] -async fn push_handler( +#[endpoint { method = PUT, path = "/v3/push" }] +async fn push_handler_v3( ctx: RequestContext>>, request: TypedBody, +) -> Result { + let update = Update::from(request.into_inner()); + push_handler_common(ctx, update).await +} + +#[endpoint { method = PUT, path = "/v4/push" }] +async fn push_handler_v4( + ctx: RequestContext>>, + request: TypedBody, ) -> Result { let update = request.into_inner(); push_handler_common(ctx, update).await @@ -363,11 +763,11 @@ async fn push_handler( async fn push_handler_common( ctx: RequestContext>>, - update: v3::Update, + update: Update, ) -> Result { let ctx = ctx.context().lock().await.clone(); tokio::task::spawn_blocking(move || { - handle_update(&update, &ctx); + handle_update(&update, &ctx, UpdateMode::Redistribute); }) .await .map_err(|e| { @@ -396,19 +796,18 @@ async fn pull_handler_v2( if route.nexthop == ctx.peer { continue; } - let mut pv = v3::PathVector { + let mut path_vector = PathVector { destination: route.destination, path: route.path.clone(), }; - pv.path.push(ctx.ctx.hostname.clone()); - underlay.insert(pv); + path_vector.path.push(ctx.ctx.hostname.clone()); + underlay.insert(path_vector); } for route in &ctx.ctx.db.imported_tunnel() { if route.nexthop == ctx.peer { continue; } - let tv = route.origin; - tunnel.insert(tv); + tunnel.insert(route.origin); } } let originated = ctx @@ -417,11 +816,11 @@ async fn pull_handler_v2( .originated() .map_err(|e| HttpError::for_internal_error(e.to_string()))?; for prefix in &originated { - let pv = v3::PathVector { + let path_vector = PathVector { destination: *prefix, path: vec![ctx.ctx.hostname.clone()], }; - underlay.insert(pv); + underlay.insert(path_vector); } let originated_tunnel = ctx @@ -453,51 +852,43 @@ async fn pull_handler_v2( })) } -#[endpoint { - method = GET, - path = "/v3/pull", -}] -async fn pull_handler( - ctx: RequestContext>>, -) -> Result, HttpError> { - let ctx = ctx.context().lock().await.clone(); - +/// Collect underlay and tunnel routes for pull responses (shared by V3/V4). +fn collect_underlay_tunnel( + ctx: &HandlerContext, +) -> Result<(HashSet, HashSet), HttpError> { let mut underlay = HashSet::new(); let mut tunnel = HashSet::new(); - // Only transit routers redistribute prefixes if ctx.ctx.config.kind == RouterKind::Transit { for route in &ctx.ctx.db.imported() { - // don't redistribute prefixes to their originators if route.nexthop == ctx.peer { continue; } - let mut pv = v3::PathVector { + let mut path_vector = PathVector { destination: route.destination, path: route.path.clone(), }; - pv.path.push(ctx.ctx.hostname.clone()); - underlay.insert(pv); + path_vector.path.push(ctx.ctx.hostname.clone()); + underlay.insert(path_vector); } for route in &ctx.ctx.db.imported_tunnel() { if route.nexthop == ctx.peer { continue; } - let tv = route.origin; - tunnel.insert(tv); + tunnel.insert(route.origin); } } + let originated = ctx .ctx .db .originated() .map_err(|e| HttpError::for_internal_error(e.to_string()))?; for prefix in &originated { - let pv = v3::PathVector { + underlay.insert(PathVector { destination: *prefix, path: vec![ctx.ctx.hostname.clone()], - }; - underlay.insert(pv); + }); } let originated_tunnel = ctx @@ -506,35 +897,152 @@ async fn pull_handler( .originated_tunnel() .map_err(|e| HttpError::for_internal_error(e.to_string()))?; for prefix in &originated_tunnel { - let tv = v3::TunnelOrigin { + tunnel.insert(TunnelOrigin { overlay_prefix: prefix.overlay_prefix, boundary_addr: prefix.boundary_addr, vni: prefix.vni, metric: prefix.metric, - }; - tunnel.insert(tv); + }); + } + + Ok((underlay, tunnel)) +} + +/// Collect multicast routes for V4 pull responses. +fn collect_multicast( + ctx: &HandlerContext, +) -> Result, HttpError> { + let mut multicast = HashSet::new(); + + if ctx.ctx.config.kind == RouterKind::Transit { + for route in &ctx.ctx.db.imported_mcast() { + if route.nexthop == ctx.peer { + continue; + } + let hop = MulticastPathHop::new( + ctx.ctx.hostname.clone(), + ctx.ctx.config.addr, + ); + let mut path = route.path.clone(); + path.push(hop); + multicast.insert(MulticastPathVector { + origin: (&route.origin).into(), + path, + }); + } + } + + let originated_mcast = ctx + .ctx + .db + .originated_mcast() + .map_err(|e| HttpError::for_internal_error(e.to_string()))?; + for origin in &originated_mcast { + let hop = MulticastPathHop::new( + ctx.ctx.hostname.clone(), + ctx.ctx.config.addr, + ); + multicast.insert(MulticastPathVector { + origin: origin.into(), + path: vec![hop], + }); } + Ok(multicast) +} + +#[endpoint { method = GET, path = "/v3/pull" }] +async fn pull_handler_v3( + ctx: RequestContext>>, +) -> Result, HttpError> { + let ctx = ctx.context().lock().await.clone(); + let (underlay, tunnel) = collect_underlay_tunnel(&ctx)?; Ok(HttpResponseOk(v3::PullResponse { - underlay: if underlay.is_empty() { - None - } else { - Some(underlay) - }, - tunnel: if tunnel.is_empty() { - None - } else { - Some(tunnel) - }, + underlay: crate::non_empty(underlay), + tunnel: crate::non_empty(tunnel), })) } -fn handle_update(update: &v3::Update, ctx: &HandlerContext) { +#[endpoint { method = GET, path = "/v4/pull" }] +async fn pull_handler_v4( + ctx: RequestContext>>, + query: dropshot::Query< + PaginationParams, + >, +) -> Result, HttpError> { + let page = query.into_inner(); + let ctx = ctx.context().lock().await.clone(); + let after = match page.page { + WhichPage::First(_) => None, + WhichPage::Next(selector) => Some(selector.after()), + }; + + // The bounded sections ride on the first page. Repeating the whole underlay + // and tunnel table on every page would cost more than the multicast + // continuation it accompanies. + let (underlay, tunnel) = if after.is_none() { + collect_underlay_tunnel(&ctx)? + } else { + (HashSet::new(), HashSet::new()) + }; + + let underlay = crate::non_empty(underlay); + let tunnel = crate::non_empty(tunnel); + + // The sections already placed on this page leave that much less room for + // the multicast page accompanying them. + let envelope = + paging::response_envelope_len(underlay.as_ref(), tunnel.as_ref()) + .map_err(|e| HttpError::for_internal_error(e.to_string()))?; + + let groups = paging::group_by_origin(collect_multicast(&ctx)?); + let (multicast, next_page_token) = paging::page_multicast( + &groups, + after.as_ref(), + paging::MAX_EXCHANGE_BODY_BYTES, + envelope, + ) + .map_err(|e| HttpError::for_internal_error(e.to_string()))?; + let next_page_token = next_page_token + .as_ref() + .map(paging::encode_page_token) + .transpose() + .map_err(|e| HttpError::for_internal_error(e.to_string()))?; + + Ok(HttpResponseOk(PullResponse { + underlay, + tunnel, + multicast: crate::non_empty(multicast), + next_page_token, + })) +} + +fn handle_update(update: &Update, ctx: &HandlerContext, mode: UpdateMode) { ctx.ctx .stats .updates_received .fetch_add(1, Ordering::Relaxed); + // Route application and peer cleanup take the same per-interface lock. + // This lets discovery publish identity and liveness changes without + // waiting for DPD, OPTE, or datastore work below. Once the lock is held, + // a brief identity check rejects an update whose peer has already expired + // or renumbered. Otherwise, the subsequent cleanup waits and removes + // anything this update imports. + let _route_update = lock!(ctx.ctx.iface.route_update); + let current_peer = lock!(ctx.ctx.iface.peer_identity) + .as_ref() + .map(|peer| peer.addr); + if current_peer != Some(ctx.peer) { + inf!( + ctx.log, + ctx.ctx.config.if_name, + "discarding update from stale peer {}", + ctx.peer, + ); + return; + } + if let Some(underlay_update) = &update.underlay { handle_underlay_update(underlay_update, ctx); } @@ -543,9 +1051,25 @@ fn handle_update(update: &v3::Update, ctx: &HandlerContext) { handle_tunnel_update(tunnel_update, ctx); } - // distribute updates - - if ctx.ctx.config.kind == RouterKind::Transit { + // Only transit routers redistribute, so demote the mode on a server + // before it reaches the multicast handler. Only the redistribution path + // reconciles against a reachability snapshot, so only it pays for + // capturing one. + let mode = if ctx.ctx.config.kind == RouterKind::Transit { + mode + } else { + UpdateMode::ImportOnly + }; + let mcast_reachability = update + .multicast + .as_ref() + .and_then(|mu| handle_multicast_update(mu, ctx, mode)); + + // Event delivery from different interfaces is intentionally not globally + // ordered. A reversed pair can expose an older multicast view until the + // next successful V4 pull, whose complete response repairs missing and + // stale imports. This avoids a global lock on every multicast change. + if mode == UpdateMode::Redistribute { dbg!( ctx.log, ctx.ctx.config.if_name, @@ -558,13 +1082,69 @@ fn handle_update(update: &v3::Update, ctx: &HandlerContext) { .as_ref() .map(|update| update.with_path_element(ctx.ctx.hostname.clone())); - let push = v3::Update { + // Multicast loop prevention is asymmetric with the underlay. The + // underlay filters on send, skipping any route whose nexthop is the + // destination peer. Multicast drops, on receipt, any + // announcement whose path already carries our router_id. The path + // check is required because a replacement announcement goes to every + // peer, so a peer can appear mid-path rather than as the nexthop, + // and paths can cross several transits, forming loops longer than + // the immediate echo. + // + // The same filter applies here before redistributing. A peer already + // in a vector's path would drop it anyway, but a peer that is not + // would import a looped path. + let hostname = &ctx.ctx.hostname; + + // The snapshot came from the `handle_multicast_update` modification, so + // reconciliation reads state consistent with the local application. + let multicast = + update + .multicast + .as_ref() + .zip(mcast_reachability.as_ref()) + .map(|(update, reachability)| { + let hop = MulticastPathHop::new( + hostname.clone(), + ctx.ctx.config.addr, + ); + + let is_loop_free = |path_vector: &&MulticastPathVector| { + !path_vector + .path + .iter() + .any(|hop| &hop.router_id == hostname) + }; + + let mut reconciled = reconcile_multicast_withdrawals( + update.withdraw.iter().filter(is_loop_free), + reachability, + &hop, + ); + reconciled.announce.extend( + update.announce.iter().filter(is_loop_free).map( + |path_vector| path_vector.with_hop(hop.clone()), + ), + ); + reconciled + }); + + let push = Arc::new(Update { underlay, tunnel: update.tunnel.clone(), - }; + multicast, + }); for ec in &ctx.ctx.event_channels { - ec.send(Event::Peer(PeerEvent::Push(push.clone()))).unwrap(); + if let Err(e) = + ec.send(Event::Peer(PeerEvent::Push(Arc::clone(&push)))) + { + err!( + ctx.log, + ctx.ctx.config.if_name, + "deliver redistributed update: {e}", + ); + } } } } @@ -711,3 +1291,136 @@ fn handle_underlay_update(update: &v3::UnderlayUpdate, ctx: &HandlerContext) { .imported_underlay_prefixes .store(ctx.ctx.db.imported_count() as u64, Ordering::Relaxed); } + +fn handle_multicast_update( + update: &MulticastUpdate, + ctx: &HandlerContext, + mode: UpdateMode, +) -> Option { + let db = &ctx.ctx.db; + let hostname = &ctx.ctx.hostname; + + let mut import = HashSet::new(); + let mut remove = HashSet::new(); + // A replacement is broadcast to every peer, including peers already in + // its path. For such a peer, the looped announce implicitly invalidates + // its old route through the sender. A clean vector for the same + // `(origin, peer)` in this update takes precedence. + for path_vector in &update.announce { + // Promote the wire origin to the validated form. Peer-supplied routes + // are otherwise trusted, but the underlay group reaches DPD directly, + // so a promotion enforces its ff04::/64 invariant (and the VNI range) + // before the route can be stored. An invalid origin is dropped rather + // than tracking a group DPD would refuse to program. + let origin = match MulticastOrigin::try_from(&path_vector.origin) { + Ok(origin) => origin, + Err(e) => { + wrn!( + ctx.log, + ctx.ctx.config.if_name, + "dropping multicast announce for {}; {e}", + path_vector.origin.overlay_group, + ); + continue; + } + }; + + let route = MulticastRoute { + origin, + nexthop: ctx.peer, + path: path_vector.path.clone(), + }; + if path_vector + .path + .iter() + .any(|hop| &hop.router_id == hostname) + { + if !import.contains(&route) { + dbg!( + ctx.log, + ctx.ctx.config.if_name, + "removing multicast route for {} via {}; \ + looped announce (path length {})", + path_vector.origin.overlay_group, + ctx.peer, + path_vector.path.len(), + ); + remove.insert(route); + } + } else { + // This also cancels an implicit removal if a looped vector for + // the same route appeared earlier in the unordered announce set. + remove.remove(&route); + import.insert(route); + } + } + + for path_vector in &update.withdraw { + // A withdrawal whose path already contains this router is an echo of + // a local redistribution and must not be acted on. + if path_vector + .path + .iter() + .any(|hop| &hop.router_id == hostname) + { + trc!( + ctx.log, + ctx.ctx.config.if_name, + "dropping multicast withdraw for {}; loop detected \ + (path length {})", + path_vector.origin.overlay_group, + path_vector.path.len(), + ); + continue; + } + + // A withdraw carrying an invalid origin cannot match a stored route, + // since storage only admits promoted origins, so drop it here too. + let origin = match MulticastOrigin::try_from(&path_vector.origin) { + Ok(origin) => origin, + Err(e) => { + wrn!( + ctx.log, + ctx.ctx.config.if_name, + "dropping multicast withdraw for {}; {e}", + path_vector.origin.overlay_group, + ); + continue; + } + }; + + // Route identity is (origin, nexthop), so an empty path matches. + remove.insert(MulticastRoute { + origin, + nexthop: ctx.peer, + path: Vec::new(), + }); + } + + // Atomic import + delete + diff under a single lock. The redistribution + // path also reconciles against a post-modification reachability snapshot, + // captured under that same lock scope. + let (delta, reachability) = match mode { + UpdateMode::Redistribute => { + let (delta, reachability) = + db.update_imported_mcast_with_reachability(&import, &remove); + (delta, Some(reachability)) + } + UpdateMode::ImportOnly => { + (db.update_imported_mcast(&import, &remove), None) + } + }; + + // Notify the multicast sweep of each affected underlay group so it + // reconciles the group's DPD members. Only the sweep writes to DPD. + // + // This handler records the import and signals, deriving the notification + // from the effective diff rather than the requested sets avoids waking the + // sweep for routes that were already present or already absent. + crate::mcast::notify_affected_groups( + delta.added.iter().chain(delta.removed.iter()), + &ctx.ctx.mcast_notify, + ); + + reachability +} diff --git a/ddm/src/lib.rs b/ddm/src/lib.rs index 6a5e2a68a..2bfb631e9 100644 --- a/ddm/src/lib.rs +++ b/ddm/src/lib.rs @@ -6,6 +6,8 @@ pub mod admin; pub mod db; pub mod discovery; pub mod exchange; +#[cfg(feature = "backend")] +pub mod mcast; pub mod oxstats; pub mod sm; #[cfg(all(feature = "backend", target_os = "illumos"))] @@ -15,6 +17,29 @@ pub const COMPONENT_DDM: &str = "ddm"; pub const MOD_ADMIN: &str = "admin"; pub const MOD_EXCHANGE: &str = "exchange"; +/// Capacity of the channel carrying wake hints to the multicast sweep. +/// +/// Wake messages are best-effort: senders drop a hint when the channel is full, +/// and the sweep's periodic reconciliation of its full tracked set repairs the +/// omission. A depth of one would suffice semantically, while this small buffer +/// absorbs routine bursts. +/// +/// This constant lives outside the `backend`-feature-gated `mcast` module +/// because `ddmd` wires the channel in every supported feature configuration. +pub const MCAST_NOTIFY_CHANNEL_DEPTH: usize = 8; + +/// Wrap a set in `Some`, treating an empty set as absence. +/// +/// # Returns +/// +/// `None` if `set` is empty, otherwise `Some(set)`. +#[cfg(all(feature = "backend", target_os = "illumos"))] +pub(crate) fn non_empty( + set: std::collections::HashSet, +) -> Option> { + (!set.is_empty()).then_some(set) +} + #[macro_export] macro_rules! err { ($log:expr, $index:expr, $($args:tt)+) => { diff --git a/ddm/src/mcast.rs b/ddm/src/mcast.rs new file mode 100644 index 000000000..b4691570e --- /dev/null +++ b/ddm/src/mcast.rs @@ -0,0 +1,1248 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Underlay multicast membership programming. +//! +//! Programs the local switch's underlay multicast replication members in the +//! Dendrite (DPD) data plane from the multicast routes DDM has imported from +//! peers. This is the multicast analog of the unicast import-to-DPD path that +//! [`crate::sys::add_underlay_routes`] performs in-process: `ddmd` owns both +//! inputs locally, the imported multicast set ([`Db::imported_mcast`]) and the +//! peer table, so it programs DPD without any cross-daemon coordination. +//! +//! Membership is reconciled by a single periodic sweep over every active +//! underlay group, like the unicast lower-half's resync and `rdb`'s reaper. The +//! control plane never writes multicast members to DPD itself. +//! +//! When a peer's subscription to a group changes, the exchange update handler +//! and peer expiry send the group's address down a notify channel to wake the +//! sweep early. Peer-link resolution does the same for any import that raced +//! ahead of the link. Absent a trigger, the sweep self-ticks on +//! `RECONCILE_INTERVAL`, which bounds how long drift persists. The address +//! on a trigger is only a wake hint: the sweep always reconciles the full set, +//! so a coalesced or missed trigger costs at most one interval of latency. +//! +//! Each sweep recomputes the full, desired member set for every tracked group, +//! repairs any drift, and programs members it could not previously (because the +//! group did not yet exist or a peer link had not resolved). A group whose +//! imports are withdrawn stays in the sweep until its DPD member list is +//! confirmed empty, then drops out, so a withdrawn group is emptied exactly +//! once and the tracked set stays bounded to active and recently active groups. +//! Discovering pre-existing DPD-only groups is deferred for a startup grace so +//! a restart does not empty groups whose imports have not yet been re-learned. +//! +//! DPD's only member-write surface is a full-list replace, so every member edit +//! is a read-modify-write. Groups reconcile concurrently within a pass, but the +//! sweep is the sole writer and each group is a distinct DPD object, so +//! concurrent edits cannot clobber one another. +//! +//! Each imported [`MulticastRoute`] names the peer (`nexthop`) that advertised +//! a group subscription and that peer is a replication target on this sled. +//! Every next hop is resolved to its switch `(PortId, LinkId)` through the +//! interface the peer was discovered on, and members are aggregated per +//! underlay group. +//! +//! Aggregation keys solely on the underlay group address and discards the +//! overlay group. This is sound because Omicron maps each overlay group to a +//! distinct underlay group, so the mapping is one to one and the underlay +//! address alone identifies a group's replication set. A route's overlay group +//! is carried for diagnostics, not for keying. +//! +//! Omicron owns each underlay group's create and delete. `ddmd` programs only +//! the member set of groups that already exist and authorizes each write +//! against the group's tag, read back from DPD, so it never changes the tag or +//! deletes the group. +//! +//! See [RFD 488] for the multicast architecture. +//! +//! [RFD 488]: https://rfd.shared.oxide.computer/rfd/0488 + +use crate::db::Db; +use crate::sm::{DpdConfig, SmContext}; +use crate::sys::DDM_DPD_TAG; +use ddm_api_types::db::MulticastRoute; +use dpd_client::types::{ + Direction, LinkId, MulticastGroupMember, MulticastGroupUpdateUnderlayEntry, + MulticastTag, PortId, UnderlayMulticastIpv6, +}; +use dpd_client::{Client, ClientState}; +use futures::TryStreamExt; +use futures::stream::{self, StreamExt}; +use mg_common::lock; +use reqwest::StatusCode; +use slog::{Logger, debug, error, info, warn}; +use std::collections::{HashMap, HashSet}; +use std::net::Ipv6Addr; +use std::time::{Duration, Instant}; +use tokio::sync::mpsc::{Receiver, Sender}; + +/// Interval between the sweep's periodic membership reconcile passes. +/// +/// A trigger wakes the sweep immediately, so this interval governs only the +/// periodic resync: how quickly drift and any change not delivered as a +/// trigger converge into DPD. It is kept coarse to bound idle DPD churn, since +/// membership changes themselves arrive as triggers. +pub(crate) const RECONCILE_INTERVAL: Duration = Duration::from_secs(10); + +/// Per-request timeout for a single DPD member operation. +/// +/// Bounds one GET or PUT against an unresponsive DPD so a stalled request +/// cannot delay the rest of a sweep pass indefinitely. It is set above the +/// expected DPD member operation latency so that it fires only on a genuine +/// stall, and low enough that a group's sequential fetch-then-write pair +/// (`2 * DPD_REQUEST_TIMEOUT`) stays under [`RECONCILE_INTERVAL`], so a single +/// stalled group cannot extend a pass beyond one reconcile interval. This +/// stall-detection threshold is reasoned about independently of the +/// convergence cadence set by [`RECONCILE_INTERVAL`]. A timed-out operation is +/// logged distinctly and retried on the next pass. +const DPD_REQUEST_TIMEOUT: Duration = Duration::from_secs(3); + +/// Sets how long after startup the sweep waits before discovering DPD-only +/// groups. +/// +/// A restart races the sweep against the exchange machinery: the first pass +/// runs before peers have re-advertised their subscriptions. Seeding the +/// tracked set from DPD immediately would make those groups look withdrawn and +/// drain them, cutting live replication until the imports return. Two +/// reconcile intervals cover peer discovery and the initial pulls. Imported +/// groups are still reconciled immediately, and a group withdrawn after +/// startup drains normally because it is already tracked. +const STARTUP_SEED_GRACE: Duration = RECONCILE_INTERVAL.saturating_mul(2); + +/// Cap on concurrently reconciling groups within a single pass. +/// +/// Bounds the in-flight DPD requests a pass can generate so a large tracked +/// set cannot flood DPD with an unbounded burst of GETs and PUTs. +const MAX_CONCURRENT_GROUP_RECONCILES: usize = 16; + +/// Run the multicast membership sweep. +/// +/// This loops forever, so callers spawn it as a dedicated task on the runtime. +/// +/// We track the set of active underlay groups and reconcile them on each pass. +/// `notify_rx` is a wake hint only. A trigger wakes the sweep early, and absent +/// a trigger it self-ticks on `RECONCILE_INTERVAL`. Every pass reconciles the +/// full tracked set, so the group address carried by a trigger is not consulted +/// and a coalesced trigger costs at most one interval of latency. +/// +/// The tracked set is the union of every currently imported group and any group +/// still being drained. `reconcile_group` returns `false` only once a +/// withdrawn group's DPD members are confirmed empty, so that a group leaves +/// the set exactly once its drain is complete. A re-import re-adds it on the +/// next pass since the control plane writes to the DB before sending its +/// trigger. +/// +/// `peers` is the set of per-interface state machine contexts, fixed at +/// startup. Peer identity lives behind interior mutability, so each pass +/// resolves against whatever peers have been discovered when it reads. +/// +/// Under `--api-only` there are no state machines, so the set is empty. +pub async fn run( + db: Db, + peers: Vec, + dpd: DpdConfig, + mut notify_rx: Receiver, + log: Logger, +) { + let client_state = ClientState { + tag: DDM_DPD_TAG.into(), + log: log.clone(), + }; + // Build the inner HTTP client explicitly to bound each request at + // DPD_REQUEST_TIMEOUT. The progenitor-generated dpd_client defaults to a + // 15s connect and request timeout, which exceeds RECONCILE_INTERVAL. A + // single stalled GET could outlast the whole reconcile interval, and a + // sequential fetch-then-write pair could run three times it. Stepping down + // to DPD_REQUEST_TIMEOUT keeps a stalled group's pair under one interval. + let http = match reqwest::ClientBuilder::new() + .connect_timeout(DPD_REQUEST_TIMEOUT) + .timeout(DPD_REQUEST_TIMEOUT) + .build() + { + Ok(http) => http, + Err(e) => { + error!(log, "failed to build DPD HTTP client, stopping sweep: {e}"); + return; + } + }; + let client = Client::new_with_client( + &format!("http://{}:{}", dpd.host, dpd.port), + http, + client_state, + ); + + // On each pass, the sweep reconciles every imported group plus any group + // still draining withdrawn members, so the set stays bounded to active and + // recently active groups rather than growing without limit. We seed it from + // the underlay groups DPD already has members for. + // + // On a fresh start, the imported set and triggers only reference groups + // with live subscriptions, so a group whose imports were withdrawn while + // `ddmd` was down would never re-enter the sweep and its stale replication + // members would persist. After a startup grace, folding those groups in + // lets a pass drain any that no peer still imports, while groups still + // imported simply reconcile as usual. A failed listing is retried after + // another reconcile interval, since ordinary passes cannot discover + // orphans. + let mut tracked: HashSet = HashSet::new(); + let mut seeded = false; + let mut next_seed_attempt = Instant::now() + STARTUP_SEED_GRACE; + + loop { + if !seeded && Instant::now() >= next_seed_attempt { + match client.member_group_ips(&log).await { + Some(groups) => { + tracked.extend(groups); + seeded = true; + } + None => { + next_seed_attempt = Instant::now() + RECONCILE_INTERVAL; + } + } + } + + // The imported set and resolved peer links are the same for every group + // in a pass, so compute them once here rather than per group. + let imported = db.imported_mcast(); + let peer_links = resolve_peer_links(&peers, &log); + + tracked = + reconcile_pass(tracked, imported, peer_links, &client, &log).await; + + // Wait for a trigger or one idle reconcile interval, whichever comes + // first, then drain any burst since the next pass reconciles + // everything. A fresh sleep is sufficient because every trigger also + // runs a full pass, and avoids catch-up timer semantics after a slow + // pass. Both arms are cancel-safe leaf futures with no `.await` in + // their bodies, so the sweep cannot futurelock. + tokio::select! { + trigger = notify_rx.recv() => match trigger { + Some(_) => while notify_rx.try_recv().is_ok() {}, + None => { + // Unreachable while `ddmd` runs: `main()` owns the original + // `notify_tx` and parks for the daemon's lifetime, so the + // channel cannot close even if every per-peer sender clone + // is torn down. We stop the sweep rather than spin on a + // closed channel if that invariant ever changes. + error!(log, "multicast notify channel closed, stopping sweep"); + break; + } + }, + _ = tokio::time::sleep(RECONCILE_INTERVAL) => {} + } + } +} + +/// Signal the multicast sweep for each distinct underlay group in `routes`. +/// +/// The route iterator may repeat a group for multiple next hops, so group +/// addresses are deduplicated before notification. +pub(crate) fn notify_affected_groups<'a>( + routes: impl IntoIterator, + notify: &Sender, +) { + let groups = routes + .into_iter() + .map(|route| route.origin.underlay_group.ip()) + .collect(); + notify_groups(groups, notify); +} + +/// Wake the multicast sweep once per group in `groups`. +fn notify_groups(groups: HashSet, notify: &Sender) { + for group in groups { + // A full channel or closed receiver is harmless because triggers are + // wake hints and the periodic pass reconciles the full tracked set. + let _ = notify.try_send(group); + } +} + +/// Wake the multicast sweep for every group `peer` advertised once that peer's +/// link resolves. +/// +/// A multicast import already wakes the sweep, but a route imported before the +/// peer link resolved cannot be programmed yet, so it waits out the reconcile +/// interval. Waking the peer's groups on resolution closes that window. The +/// imported set is read, not consumed, so this is the non-destructive analog of +/// the [`Db::remove_nexthop_routes`] removal on peer expiry. +pub(crate) fn notify_peer_groups( + db: &Db, + peer: Ipv6Addr, + notify: &Sender, +) { + notify_groups(db.mcast_groups_for_nexthop(peer), notify); +} + +/// Reconcile every tracked group against DPD once, returning the next tracked +/// set. +/// +/// Folds every currently imported group into `tracked`, reconciles the whole +/// set concurrently, and returns only the groups `reconcile_group` reports as +/// still active. A withdrawn group lingers for exactly one pass to empty its +/// DPD members, then drops out on the following pass. Re-importing a dropped +/// group re-adds it here, since Omicron writes to the DB before triggering the +/// sweep. +/// +/// The per-group futures run concurrently on this task rather than being +/// spawned, capped at [`MAX_CONCURRENT_GROUP_RECONCILES`] in flight, so a +/// group whose DPD call stalls does not serialize the others behind it and a +/// large tracked set cannot flood DPD. +/// +/// The pass still returns only once its slowest group completes. Each request +/// is bounded by [`DPD_REQUEST_TIMEOUT`], but stalled groups beyond the +/// concurrency cap execute in waves, so a pass is bounded by one timeout per +/// wave of stalled groups rather than one timeout overall. +async fn reconcile_pass( + mut tracked: HashSet, + imported: HashSet, + peer_links: HashMap, + client: &C, + log: &Logger, +) -> HashSet { + for route in imported.iter() { + tracked.insert(route.origin.underlay_group.ip()); + } + + // Borrow once so every per-group future shares the same imports and + // resolved links by reference. + let imported = &imported; + let peer_links = &peer_links; + stream::iter(tracked) + .map(|group_ip| async move { + ( + group_ip, + reconcile_group(group_ip, imported, peer_links, client, log) + .await, + ) + }) + .buffer_unordered(MAX_CONCURRENT_GROUP_RECONCILES) + .filter_map(|(group_ip, keep)| async move { keep.then_some(group_ip) }) + .collect() + .await +} + +/// Whether a DPD client error is a request timeout. +/// +/// A timeout surfaces as a transport-level error with no HTTP status, so it is +/// distinguished by inspecting the underlying `reqwest::Error` rather than by +/// status code. +fn is_timeout(e: &dpd_client::Error) -> bool { + matches!(e, dpd_client::Error::CommunicationError(re) if re.is_timeout()) +} + +/// Outcome of writing a group's member list to DPD. +#[derive(Clone, Copy)] +enum WriteOutcome { + /// Members were written. + Updated, + /// The group disappeared or its tag changed after the preceding read. + Stale, + /// The write failed and should be retried on the next pass. + Retry, +} + +/// Outcome of reading a group's state from DPD. +#[derive(Clone)] +enum FetchOutcome { + /// The group exists and its authorization tag and current members were read. + Found(String, Vec), + /// The group does not exist in DPD, either because Omicron has not created + /// it yet or because it has been deleted. + Absent, + /// The read failed, so the group's state is unknown this pass. + Retry, +} + +/// DPD group operations the reconcile loop depends on. +trait GroupClient { + /// Read an underlay group's current members and authorization tag. + async fn fetch_group( + &self, + log: &Logger, + group_ip: Ipv6Addr, + ) -> FetchOutcome; + + /// Write `members` to an underlay group, authorized by its current `tag`. + async fn write_members( + &self, + log: &Logger, + group_ip: Ipv6Addr, + tag: &str, + members: Vec, + ) -> WriteOutcome; + + /// Underlay groups that currently have members programmed in DPD, or + /// `None` if the listing failed. + /// + /// Read after the startup grace to seed the sweep's tracked set. `ddmd` is + /// the sole writer of underlay members on this switch, so every group + /// returned was programmed by `ddmd`, possibly before a restart, and is + /// safe to fold in. A failed listing is retried later, since ordinary + /// passes reconcile only tracked and imported groups and cannot otherwise + /// discover orphans. + async fn member_group_ips(&self, log: &Logger) -> Option>; +} + +impl GroupClient for Client { + /// Distinguishes a group that is genuinely absent (`FetchOutcome::Absent`) + /// from one whose state could not be read (`FetchOutcome::Retry`), so a + /// withdrawn group is not dropped from the sweep on a transient read failure + /// before its members are confirmed drained. + async fn fetch_group( + &self, + log: &Logger, + group_ip: Ipv6Addr, + ) -> FetchOutcome { + let underlay_ip = UnderlayMulticastIpv6::from(group_ip); + match self.multicast_group_get_underlay(&underlay_ip).await { + Ok(resp) => { + let resp = resp.into_inner(); + FetchOutcome::Found(resp.tag, resp.members) + } + // The underlay group's create and delete are owned by Omicron, which + // creates the group before traffic flows. Until the group exists + // there are no members to program, so skip it. + Err(e) if e.status() == Some(StatusCode::NOT_FOUND) => { + debug!( + log, + "underlay group {group_ip} does not exist yet, skipping \ + until Omicron creates it" + ); + FetchOutcome::Absent + } + + // Surface a stalled read distinctly from other failures. The sweep + // retries the group on its next pass regardless. + Err(e) if is_timeout(&e) => { + warn!( + log, + "get of underlay group {group_ip} timed out after \ + {DPD_REQUEST_TIMEOUT:?}, retrying next pass" + ); + FetchOutcome::Retry + } + Err(e) => { + warn!(log, "failed to get underlay group {group_ip}: {e}"); + FetchOutcome::Retry + } + } + } + + /// The expected races, a tag change (403) or a deleted group (404), are + /// returned as outcomes rather than logged, leaving the handling to the + /// caller. + async fn write_members( + &self, + log: &Logger, + group_ip: Ipv6Addr, + tag: &str, + members: Vec, + ) -> WriteOutcome { + let underlay_ip = UnderlayMulticastIpv6::from(group_ip); + let tag = match MulticastTag::try_from(tag.to_string()) { + Ok(tag) => tag, + Err(e) => { + error!( + log, + "tag for underlay group {group_ip} is invalid, skipping \ + update: {e}" + ); + return WriteOutcome::Retry; + } + }; + + let body = MulticastGroupUpdateUnderlayEntry { members }; + match self + .multicast_group_update_underlay(&underlay_ip, &tag, &body) + .await + { + Ok(_) => WriteOutcome::Updated, + Err(e) + if matches!( + e.status(), + Some(StatusCode::FORBIDDEN | StatusCode::NOT_FOUND) + ) => + { + WriteOutcome::Stale + } + // Log a stalled write distinctly from other failures, while both + // share the same retry outcome. + Err(e) if is_timeout(&e) => { + warn!( + log, + "update of underlay group {group_ip} members timed out \ + after {DPD_REQUEST_TIMEOUT:?}, retrying next pass" + ); + WriteOutcome::Retry + } + Err(e) => { + warn!( + log, + "failed to update underlay group {group_ip} members: {e}" + ); + WriteOutcome::Retry + } + } + } + + async fn member_group_ips(&self, log: &Logger) -> Option> { + let groups: Vec = match self + .multicast_groups_list_stream(None) + .try_collect() + .await + { + Ok(groups) => groups, + Err(e) => { + warn!( + log, + "could not list multicast groups to seed sweep, retrying \ + later: {e}" + ); + return None; + } + }; + + Some( + groups + .into_iter() + .filter_map(|group| match group { + dpd_client::types::MulticastGroupResponse::Underlay { + group_ip, + members, + .. + } if !members.is_empty() => Some(*group_ip), + _ => None, + }) + .collect(), + ) + } +} + +/// Resolve each established peer's underlay address to a switch +/// `(PortId, LinkId)` through the interface it was discovered on. +/// +/// Peers without an established identity, without an interface name, or whose +/// interface does not resolve to a switch link are omitted. +/// +/// A peer omitted here is seen by `group_members` as an unresolved next hop, so +/// a transient resolution failure neither drops a previously programmed member +/// nor blocks a newly resolved one. +/// +/// The map keys on the peer's link-local address alone, with no interface +/// scope, since an imported route's `nexthop` carries no interface either. +/// This relies on the rack deriving link-local addresses from EUI-64, which +/// makes them unique across links rather than only within one ([RFC 4007], +/// section 5). A duplicate address on distinct links would collapse to one +/// entry, so a collision is logged as a warning. +/// +/// [RFC 4007]: https://www.rfc-editor.org/rfc/rfc4007#section-5 +fn resolve_peer_links( + peers: &[SmContext], + log: &Logger, +) -> HashMap { + let mut peer_links: HashMap = HashMap::new(); + for sm in peers { + let Some(peer) = lock!(sm.iface.peer_identity).clone() else { + continue; + }; + let if_name = lock!(sm.iface.if_name).clone(); + if if_name.is_empty() { + warn!( + log, + "peer {} has no interface name; omitting as multicast member", + peer.addr + ); + continue; + } + + match mg_common::tfport::port_link_from_ifname(&if_name) { + Ok(port_link) => { + if let Some(prev) = + peer_links.insert(peer.addr, port_link.clone()) + && prev != port_link + { + warn!( + log, + "peer link-local address {} resolves to multiple \ + switch links ({prev:?} and {port_link:?}), violating \ + EUI-64 uniqueness; keeping the latter", + peer.addr + ); + } + } + Err(e) => warn!( + log, + "cannot resolve peer {} interface {if_name} to a switch link, \ + omitting as multicast member: {e}", + peer.addr + ), + } + } + peer_links +} + +/// Aggregate one underlay group's desired replication members. +/// +/// Returns the member list for `group_ip` and whether any of its next hops +/// failed to resolve this pass, in which case the derived set may be incomplete. +/// +/// Members are derived from each route's `nexthop`. Distinct downstream peers +/// carry distinct next hops, so each becomes its own member. Subscribers reached +/// through the same downstream peer collapse to one member, because a single +/// egress port toward that peer suffices and the next hop handles further +/// fan-out. The path vector is not needed, only the per-node egress set. +fn group_members( + group_ip: Ipv6Addr, + imported: &HashSet, + peer_links: &HashMap, +) -> (Vec, bool) { + let mut members: Vec = Vec::new(); + let mut has_unresolved = false; + for route in imported + .iter() + .filter(|route| route.origin.underlay_group.ip() == group_ip) + { + let Some((port_id, link_id)) = peer_links.get(&route.nexthop) else { + has_unresolved = true; + continue; + }; + // The single (port, link) here is per next hop, not per group: one peer + // is reached over one tfport link. A group fans out to as many links as + // it has distinct downstream peers. + let member = MulticastGroupMember { + port_id: port_id.clone(), + link_id: *link_id, + direction: Direction::Underlay, + }; + if !members.contains(&member) { + members.push(member); + } + } + (members, has_unresolved) +} + +/// Reconcile a single underlay group's members in DPD against the multicast +/// routes DDM has imported, returning whether the group is still active. +/// +/// The group's current members are read fresh from DPD and diffed against the +/// desired set, so the periodic resync repairs member drift. +/// +/// Returns `true` to keep the group tracked, either while it still has imports +/// (so resync repairs drift) or whenever its DPD state could not be read this +/// pass, and `false` only once the group has no imports and its DPD member list +/// is confirmed empty, so it drops out of the sweep. +async fn reconcile_group( + group_ip: Ipv6Addr, + imported: &HashSet, + peer_links: &HashMap, + client: &C, + log: &Logger, +) -> bool { + let has_imports = imported + .iter() + .any(|route| route.origin.underlay_group.ip() == group_ip); + let (members, has_unresolved) = + group_members(group_ip, imported, peer_links); + + let (tag, existing) = match client.fetch_group(log, group_ip).await { + FetchOutcome::Found(tag, existing) => (tag, existing), + // The group is absent from DPD, Omicron has not created it or it is + // deleted. There is nothing to program or drain, so keep it tracked + // only while it still has imports, so a later pass programs it once it + // exists. + FetchOutcome::Absent => return has_imports, + // The state is unknown at this pass. Keep the group tracked so the next + // pass retries, whether it is active or still draining withdrawn + // members. A withdrawn group must not drop out here, or stale + // replication would stay programmed until some later re-import tracked + // it again. + FetchOutcome::Retry => return true, + }; + + if !has_imports { + // Withdrawn: empty the member list to stop replication, leaving the + // group for Omicron to delete. + if existing.is_empty() { + return false; + } + return match client.write_members(log, group_ip, &tag, Vec::new()).await + { + WriteOutcome::Updated => { + info!( + log, + "emptied withdrawn underlay group {group_ip} members" + ); + false + } + // Already gone, or recreated under a tag that is no longer what + // we've seen. Either way `ddmd` no longer programs this group. + WriteOutcome::Stale => false, + // Retry the empty on the next pass. + WriteOutcome::Retry => true, + }; + } + + // When a next hop did not resolve this pass, the derived set may be missing + // members. Merge it with the group's current members so a transient + // resolution failure neither drops a previously programmed member nor + // blocks adding a newly resolved one. With every next hop resolved, the + // derived set replaces the current members. + // + // Fail open: preserve every current member, not only those of unresolved + // routes, since a member cannot be attributed to a next hop without + // resolving it. A stale member, even one unrelated to the unresolved next + // hop, persists until every next hop resolves. Extra replication is + // preferred over dropping a live member. + let to_write = if has_unresolved { + let merged = union_members(&members, &existing); + if merged.len() > members.len() { + debug!( + log, + "underlay group {group_ip} has unresolved next hops, preserving \ + {} current DPD member(s) beyond the {} derived this pass", + merged.len() - members.len(), + members.len() + ); + } + merged + } else { + members + }; + + if !members_eq(&existing, &to_write) { + match client.write_members(log, group_ip, &tag, to_write).await { + WriteOutcome::Updated => { + info!(log, "updated underlay group {group_ip} members") + } + WriteOutcome::Stale | WriteOutcome::Retry => {} + } + } + + true +} + +/// Union of two multicast member lists, preserving order and dropping +/// duplicates. Used to merge a derived member set with the group's current DPD +/// members when a next hop did not resolve this pass. +fn union_members( + base: &[MulticastGroupMember], + extra: &[MulticastGroupMember], +) -> Vec { + let mut merged = base.to_vec(); + for member in extra { + if !merged.contains(member) { + merged.push(member.clone()); + } + } + merged +} + +/// Compare two multicast member lists for set equality, ignoring order and +/// duplicates. +/// +/// `ddmd` never writes duplicates, but the list read back from DPD is not +/// trusted to be duplicate-free. A length check alone would be fooled by +/// duplicates, e.g. `[A, A]` against `[A, B]`, skipping the write that would +/// repair the drift, so containment is checked in both directions. +fn members_eq(a: &[MulticastGroupMember], b: &[MulticastGroupMember]) -> bool { + a.iter().all(|member| b.contains(member)) + && b.iter().all(|member| a.contains(member)) +} + +#[cfg(test)] +mod tests { + use super::*; + use ddm_api_types::net::{ + MulticastOrigin, OverlayMulticast, UnderlayMulticastIpv6, Vni, + }; + use std::net::IpAddr; + + fn underlay(last: u16) -> Ipv6Addr { + Ipv6Addr::new(0xff04, 0, 0, 0, 0, 0, 0, last) + } + + fn route(nexthop: Ipv6Addr, group: Ipv6Addr) -> MulticastRoute { + MulticastRoute { + origin: MulticastOrigin { + overlay_group: OverlayMulticast::new(IpAddr::V6( + Ipv6Addr::new(0xff0e, 0, 0, 0, 0, 0, 0, 1), + )) + .unwrap(), + underlay_group: UnderlayMulticastIpv6::new(group).unwrap(), + vni: Vni::DEFAULT_MULTICAST, + metric: 0, + source: None, + }, + nexthop, + path: Vec::new(), + } + } + + fn rear(port: &str, link: u8) -> (PortId, LinkId) { + ( + PortId::Rear(dpd_client::types::Rear::try_from(port).unwrap()), + LinkId(link), + ) + } + + fn member(port: &str, link: u8) -> MulticastGroupMember { + let (port_id, link_id) = rear(port, link); + MulticastGroupMember { + port_id, + link_id, + direction: Direction::Underlay, + } + } + + #[test] + fn distinct_peers_become_distinct_members() { + let peer_a = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let peer_b = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + let group = underlay(1); + + let imported = + HashSet::from([route(peer_a, group), route(peer_b, group)]); + let peer_links = HashMap::from([ + (peer_a, rear("rear0", 0)), + (peer_b, rear("rear1", 0)), + ]); + + let (members, unresolved) = + group_members(group, &imported, &peer_links); + assert!(!unresolved); + assert_eq!(members.len(), 2); + assert!(members.contains(&member("rear0", 0))); + assert!(members.contains(&member("rear1", 0))); + } + + #[test] + fn same_link_peers_share_member() { + let peer_a = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let peer_b = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + let group = underlay(1); + + // Two distinct peers that resolve to the same switch link. Replicating + // twice out one (PortId, LinkId) would duplicate delivery on that link, + // so the members collapse to one. The dedup keys on the resolved link, + // not on the next hop. + let imported = + HashSet::from([route(peer_a, group), route(peer_b, group)]); + let peer_links = HashMap::from([ + (peer_a, rear("rear0", 0)), + (peer_b, rear("rear0", 0)), + ]); + + let (members, unresolved) = + group_members(group, &imported, &peer_links); + assert!(!unresolved); + assert_eq!(members, vec![member("rear0", 0)]); + } + + #[test] + fn unresolved_nexthop_returns_empty_and_unresolved() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(7); + + let imported = HashSet::from([route(peer, group)]); + // No peer_links entry: next hop is unresolved. + let peer_links = HashMap::new(); + + let (members, unresolved) = + group_members(group, &imported, &peer_links); + assert!(members.is_empty()); + assert!(unresolved); + } + + #[test] + fn mixed_resolution_returns_members_and_unresolved() { + let resolved = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let unresolved_peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + let group = underlay(3); + + // One next hop resolves and one does not. The resolved peer contributes a + // member, and the group is still flagged so the reconcile merges with the + // group's current DPD members rather than dropping the unresolved one. + let imported = HashSet::from([ + route(resolved, group), + route(unresolved_peer, group), + ]); + let peer_links = HashMap::from([(resolved, rear("rear0", 0))]); + + let (members, unresolved) = + group_members(group, &imported, &peer_links); + assert!(unresolved); + assert_eq!(members, vec![member("rear0", 0)]); + } + + /// Mock DPD that returns preset fetch and write outcomes, and records every + /// member list written so a test can assert the reconcile's keep/drop + /// decision and whether it wrote at all. + /// + /// A landed write (`WriteOutcome::Updated`) updates the stored fetch state + /// to the members just written, so a later fetch reflects it. This models + /// DPD's read-after-write semantics and lets a multi-pass test observe a + /// group's drain across passes. + struct MockDpd { + fetch: std::sync::Mutex, + write_outcome: WriteOutcome, + writes: std::sync::Mutex>>, + member_groups: Vec, + } + + impl MockDpd { + fn new(fetch: FetchOutcome, write_outcome: WriteOutcome) -> Self { + Self { + fetch: std::sync::Mutex::new(fetch), + write_outcome, + writes: std::sync::Mutex::new(Vec::new()), + member_groups: Vec::new(), + } + } + + fn with_member_groups(mut self, groups: Vec) -> Self { + self.member_groups = groups; + self + } + + fn writes(&self) -> Vec> { + self.writes.lock().unwrap().clone() + } + } + + impl GroupClient for MockDpd { + async fn fetch_group( + &self, + _log: &Logger, + _group_ip: Ipv6Addr, + ) -> FetchOutcome { + self.fetch.lock().unwrap().clone() + } + + async fn write_members( + &self, + _log: &Logger, + _group_ip: Ipv6Addr, + tag: &str, + members: Vec, + ) -> WriteOutcome { + self.writes.lock().unwrap().push(members.clone()); + if matches!(self.write_outcome, WriteOutcome::Updated) { + *self.fetch.lock().unwrap() = + FetchOutcome::Found(tag.to_string(), members); + } + self.write_outcome + } + + async fn member_group_ips( + &self, + _log: &Logger, + ) -> Option> { + Some(self.member_groups.clone()) + } + } + + fn found(members: Vec) -> FetchOutcome { + FetchOutcome::Found("tag".to_string(), members) + } + + fn reconcile( + group: Ipv6Addr, + imported: &HashSet, + peer_links: &HashMap, + mock: &MockDpd, + ) -> bool { + let log = Logger::root(slog::Discard, slog::o!()); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(reconcile_group(group, imported, peer_links, mock, &log)) + } + + fn run_pass( + tracked: HashSet, + imported: &HashSet, + peer_links: &HashMap, + mock: &MockDpd, + ) -> HashSet { + let log = Logger::root(slog::Discard, slog::o!()); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(reconcile_pass( + tracked, + imported.clone(), + peer_links.clone(), + mock, + &log, + )) + } + + /// Drives the sweep's cross-pass carry-over invariant: an active group is + /// tracked, a withdraw takes one pass to empty its members and drop it, and + /// a re-import adds and reconciles it again. + #[test] + fn group_drains_and_readds_across_passes() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let active = HashSet::from([route(peer, group)]); + let withdrawn = HashSet::new(); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + + // DPD already holds the derived member, so the group starts active and + // in sync with the imported set. + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ); + + // Pass 1: imported and already in sync, so the group is tracked without + // a write. + let tracked = run_pass(HashSet::new(), &active, &peer_links, &mock); + assert_eq!(tracked, HashSet::from([group])); + assert!(mock.writes().is_empty()); + + // Pass 2: the withdrawn group carries over from pass 1, its members are + // emptied, and then it drops out of the tracked set. + let tracked = run_pass(tracked, &withdrawn, &peer_links, &mock); + assert!(tracked.is_empty()); + assert_eq!(mock.writes(), vec![Vec::::new()]); + + // Pass 3: the dropped group is re-imported and reprogrammed, since DPD + // now holds no members for it. + let tracked = run_pass(tracked, &active, &peer_links, &mock); + assert_eq!(tracked, HashSet::from([group])); + assert_eq!(mock.writes(), vec![Vec::new(), vec![member("rear0", 0)]]); + } + + /// A group whose imports were withdrawn while `ddmd` was down has no entry + /// in the imported set or any trigger, so only the startup seed can + /// re-initialize it. + #[test] + fn startup_seed_drains_orphans() { + let group = underlay(9); + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ) + .with_member_groups(vec![group]); + let log = Logger::root(slog::Discard, slog::o!()); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let seeded: HashSet = rt + .block_on(mock.member_group_ips(&log)) + .unwrap() + .into_iter() + .collect(); + assert!(seeded.contains(&group)); + + let next = run_pass(seeded, &HashSet::new(), &HashMap::new(), &mock); + assert_eq!(mock.writes(), vec![Vec::::new()]); + assert!(next.is_empty()); + } + + #[test] + fn absent_group_with_imports_stays_tracked() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new(FetchOutcome::Absent, WriteOutcome::Updated); + + // Omicron has not created the group yet, so there is nothing to program, + // but it stays tracked so a later pass programs it once it exists. + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn absent_group_without_imports_drops() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = MockDpd::new(FetchOutcome::Absent, WriteOutcome::Updated); + + assert!(!reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn read_retry_keeps_withdrawn_group() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = MockDpd::new(FetchOutcome::Retry, WriteOutcome::Updated); + + // A withdrawn group must not drop out on a transient read failure, or + // its stale replication would stay programmed until a later re-import. + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn withdrawn_group_with_no_members_drops_without_writing() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = MockDpd::new(found(Vec::new()), WriteOutcome::Updated); + + assert!(!reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn withdrawn_group_drains_then_drops() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ); + + assert!(!reconcile(group, &imported, &peer_links, &mock)); + assert_eq!(mock.writes(), vec![Vec::::new()]); + } + + #[test] + fn empty_write_retry_keeps_withdrawn_group() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = + MockDpd::new(found(vec![member("rear0", 0)]), WriteOutcome::Retry); + + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert_eq!(mock.writes(), vec![Vec::::new()]); + } + + #[test] + fn stale_empty_write_drops_withdrawn_group() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = + MockDpd::new(found(vec![member("rear0", 0)]), WriteOutcome::Stale); + + // The group disappeared or was reassigned after the read, so `ddmd` + // abandons the withdrawn group rather than retrying. + assert!(!reconcile(group, &imported, &peer_links, &mock)); + assert_eq!(mock.writes(), vec![Vec::::new()]); + } + + #[test] + fn active_group_with_matching_members_skips_write() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ); + + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn active_group_with_drifted_members_writes_derived_set() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new(found(Vec::new()), WriteOutcome::Updated); + + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert_eq!(mock.writes(), vec![vec![member("rear0", 0)]]); + } + + #[test] + fn unresolved_active_group_preserves_members() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + // No peer_links entry: the next hop is unresolved this pass. + let peer_links = HashMap::new(); + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ); + + // The derived set is empty because the next hop did not resolve, but + // the merge with current DPD members keeps the programmed member, so no + // destructive write occurs. + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + /// Encodes the fail-open merge policy: while any next hop is unresolved, + /// a stale DPD member that no import accounts for, even one unrelated to + /// the unresolved next hop, persists rather than being dropped. + #[test] + fn unresolved_nexthop_retains_unrelated_stale_member() { + let resolved = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let unresolved_peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + let group = underlay(1); + let imported = HashSet::from([ + route(resolved, group), + route(unresolved_peer, group), + ]); + let peer_links = HashMap::from([(resolved, rear("rear0", 0))]); + + // DPD holds the resolved member plus a stale one ("rear1") that no + // current import accounts for. + let mock = MockDpd::new( + found(vec![member("rear0", 0), member("rear1", 0)]), + WriteOutcome::Updated, + ); + + // The stale member cannot be distinguished from one owned by the + // unresolved next hop, so the merge preserves it and no write occurs. + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn stale_write_keeps_active_group() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new(found(Vec::new()), WriteOutcome::Stale); + + // The group changed after the read, but remains imported, so it stays + // tracked to retry with a fresh read next pass. + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert_eq!(mock.writes(), vec![vec![member("rear0", 0)]]); + } + + #[test] + fn notify_deduplicates_groups() { + let group_a = underlay(1); + let group_b = underlay(2); + let peer_a = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let peer_b = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + + let routes = [ + route(peer_a, group_a), + route(peer_b, group_a), + route(peer_a, group_b), + ]; + + let (tx, mut rx) = + tokio::sync::mpsc::channel(crate::MCAST_NOTIFY_CHANNEL_DEPTH); + notify_affected_groups(routes.iter(), &tx); + + let mut signalled = Vec::new(); + while let Ok(group) = rx.try_recv() { + signalled.push(group); + } + assert_eq!(signalled.len(), 2); + assert_eq!( + signalled.into_iter().collect::>(), + HashSet::from([group_a, group_b]) + ); + } +} diff --git a/ddm/src/sm/mod.rs b/ddm/src/sm/mod.rs index 4a52efb55..160cabd0b 100644 --- a/ddm/src/sm/mod.rs +++ b/ddm/src/sm/mod.rs @@ -4,13 +4,13 @@ //! State machine type definitions and the [`StateMachine`] handle. The //! routing state machine implementation (discovery, solicit, exchange) lives -//! in the [`state`] submodule and is illumos-only, since it programs kernel -//! routes via [`crate::sys`] and reads interface addressing through `libnet`. +//! in the `state` submodule and is illumos-only, since it programs kernel +//! routes via `crate::sys` and reads interface addressing through `libnet`. use crate::db::Db; use crate::discovery::{self, Version}; use ddm_api_types::db::{PeerStatus, RouterKind}; -use ddm_api_types::net::TunnelOrigin; +use ddm_api_types::net::{MulticastOrigin, TunnelOrigin}; use mg_common::lock; use oxnet::Ipv6Net; use slog::Logger; @@ -33,6 +33,17 @@ pub enum AdminEvent { /// Withdraw a set of IPv6 prefixes Withdraw(PrefixSet), + /// Announce a set of multicast origins to peers. + AnnounceMulticast(HashSet), + + /// Withdraw a set of multicast origins. Each state machine revalidates + /// remaining reachability against the database when it processes the + /// event, rather than acting on a snapshot captured at request time. + /// The modification lands before this event is enqueued, so the processing- + /// time read is guaranteed to observe the withdrawal, and it also + /// observes any later import that has since restored reachability. + WithdrawMulticast(HashSet), + /// Expire the peer at the specified address Expire(Ipv6Addr), @@ -48,7 +59,7 @@ pub enum PrefixSet { #[derive(Debug)] pub enum PeerEvent { - Push(ddm_protocol::v3::Update), + Push(Arc), } #[derive(Debug)] @@ -187,6 +198,11 @@ pub struct InterfaceState { pub fsm_state: Mutex, pub last_fsm_state_change: Mutex, pub peer_identity: Mutex>, + /// Orders route application with expiry and renumber cleanup for this + /// interface. Discovery must not take this lock: route programming can + /// perform network and kernel I/O, while discovery needs to keep updating + /// peer liveness independently. + pub route_update: Mutex<()>, } impl InterfaceState { @@ -218,6 +234,7 @@ impl Default for InterfaceState { fsm_state: Mutex::new(FsmState::Init), last_fsm_state_change: Mutex::new(Instant::now()), peer_identity: Mutex::new(None), + route_update: Mutex::new(()), } } } @@ -251,6 +268,11 @@ pub struct SmContext { pub hostname: String, pub iface: Arc, pub stats: Arc, + /// Notifies the `crate::mcast` sweep that an underlay group's imported + /// membership changed, by sending the group's address. The sweep wakes early + /// to reconcile the group's DPD members, so the control plane never touches + /// DPD directly. + pub mcast_notify: tokio::sync::mpsc::Sender, pub log: Logger, } diff --git a/ddm/src/sm/state.rs b/ddm/src/sm/state.rs index ecb8ffa68..718a0fe48 100644 --- a/ddm/src/sm/state.rs +++ b/ddm/src/sm/state.rs @@ -13,21 +13,34 @@ use super::{ }; use crate::{dbg, discovery, err, exchange, inf, wrn}; use ddm_api_types::db::RouterKind; -use ddm_protocol::v3::{PathVector, TunnelUpdate, UnderlayUpdate, Update}; +use ddm_api_types::net::TunnelOrigin; +use ddm_protocol::v3::{PathVector, TunnelUpdate, UnderlayUpdate}; +use ddm_protocol::v4::{MulticastPathHop, MulticastPathVector, Update}; use libnet::get_ipaddr_info; use slog::Logger; use std::collections::HashSet; use std::net::IpAddr; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::mpsc::Receiver; +use std::sync::mpsc::{Receiver, RecvTimeoutError}; use std::thread::{sleep, spawn}; -use std::time::Duration; +use std::time::{Duration, Instant}; use crate::discovery::Version; -use ddm_api_types::net::TunnelOrigin; use std::net::Ipv6Addr; +/// Cadence for the periodic pull in the [`Exchange`] state. The initial pull +/// is one-shot, so a neighbor that originates routes after we pull it, late +/// multicast group memberships for instance, would otherwise never be +/// imported absent a push from that neighbor. Pulling on this cadence +/// repairs that drift without operator intervention. It matches the +/// multicast sweep reconcile interval so both repair loops converge on the +/// same cadence. Pre-V4 peers are skipped, since their responses carry no +/// multicast section. The exchange loop checks a fixed deadline before +/// receiving another event, so a busy event queue cannot postpone the pull +/// indefinitely. +const EXCHANGE_RESYNC_INTERVAL: Duration = crate::mcast::RECONCILE_INTERVAL; + impl StateMachine { pub fn run(&mut self) -> Result<(), SmError> { let ctx = self.ctx.clone(); @@ -77,9 +90,8 @@ impl State for Init { wrn!( self.log, self.ctx.config.if_name, - "failed to get IPv6 address for interface {}: {}", + "failed to get IPv6 address for interface {}: {e}", &self.ctx.config.aobj_name, - e ); sleep(Duration::from_millis(self.ctx.config.ip_addr_wait)); continue; @@ -156,8 +168,7 @@ impl State for Solicit { err!( self.log, self.ctx.config.if_name, - "solicit event recv: {}", - e + "solicit event recv: {e}", ); continue; } @@ -169,6 +180,15 @@ impl State for Solicit { self.ctx.config.if_name, "transition solicit -> exchange" ); + + // The peer is now established on this link, so wake the + // multicast sweep for any of its groups whose import raced + // ahead of resolution. + crate::mcast::notify_peer_groups( + &self.ctx.db, + addr, + &self.ctx.mcast_notify, + ); return ( Box::new(Exchange::new( self.ctx.clone(), @@ -234,7 +254,20 @@ impl Exchange { } } - fn initial_pull(&self, stop: Arc) { + fn start_pull( + &self, + stop: Arc, + active: Arc, + mode: exchange::UpdateMode, + retry: bool, + ) { + if active + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + return; + } + let ctx = self.ctx.clone(); let peer = self.peer; let version = self.version; @@ -244,22 +277,58 @@ impl Exchange { let if_name = self.ctx.config.if_name.clone(); spawn(move || { - while let Err(e) = crate::exchange::pull( - ctx.clone(), - peer, - version, - rt.clone(), - log.clone(), - ) { - sleep(Duration::from_millis(interval)); - wrn!(log, if_name, "exchange pull: {}", e); + loop { if stop.load(Ordering::Relaxed) { break; } + + match crate::exchange::pull( + ctx.clone(), + peer, + version, + rt.clone(), + log.clone(), + mode, + Some(stop.as_ref()), + ) { + Ok(()) => break, + Err(e) => { + wrn!(log, if_name, "exchange pull: {e}"); + if !retry || stop.load(Ordering::Relaxed) { + break; + } + sleep(Duration::from_millis(interval)); + } + } } + active.store(false, Ordering::Release); }); } + fn initial_pull(&self, stop: Arc, active: Arc) { + self.start_pull(stop, active, exchange::UpdateMode::Redistribute, true); + } + + fn periodic_pull(&self, stop: Arc, active: Arc) { + // The resync exists to repair multicast drift, and multicast first + // appears on the wire in V4. An earlier peer's response has no + // multicast section, so a pull would only replay the full underlay and + // tunnel tables while holding the route_update lock. + if self.version < Version::V4 { + return; + } + self.start_pull(stop, active, exchange::UpdateMode::ImportOnly, false); + } + + fn sync_pull(&self, stop: Arc, active: Arc) { + self.start_pull( + stop, + active, + exchange::UpdateMode::Redistribute, + false, + ); + } + fn wait_for_exchange_server_to_start(&self) { inf!( self.log, @@ -268,11 +337,30 @@ impl Exchange { ); let interval = 250; // TODO as parameter loop { - match exchange::do_pull( - &self.ctx, - &self.ctx.config.addr, - &self.ctx.rt, - ) { + let res = match self.version { + Version::V2 => exchange::do_pull_v2( + &self.ctx, + &self.ctx.config.addr, + &self.ctx.rt, + ) + .map(|_| ()), + Version::V3 => exchange::do_pull_v3( + &self.ctx, + &self.ctx.config.addr, + &self.ctx.rt, + ) + .map(|_| ()), + // This probe only establishes that the server answers, so it + // reads the first page and stops. + Version::V4 => exchange::do_pull_v4( + &self.ctx, + &self.ctx.config.addr, + &self.ctx.rt, + None, + ) + .map(|_| ()), + }; + match res { Ok(_) => break, Err(e) => { wrn!( @@ -293,13 +381,35 @@ impl Exchange { fn expire_peer( &mut self, - exchange_thread: &tokio::task::JoinHandle<()>, + exchange_handle: &exchange::ExchangeHandle, pull_stop: &AtomicBool, ) { - exchange_thread.abort(); + exchange_handle.abort(); self.ctx.iface.clear_peer(); - let (to_remove, to_remove_tnl) = - self.ctx.db.remove_nexthop_routes(self.peer); + self.withdraw_peer_routes(self.peer); + pull_stop.store(true, Ordering::Relaxed); + } + + /// Remove all routes imported via `peer`, clean up the forwarding state + /// derived from them, and, on transit routers, propagate withdraws to the + /// remaining peers. + /// + /// Called on peer expiry and on renumber, where a re-advertisement + /// carries a new peer address and `peer` is the prior one. + fn withdraw_peer_routes(&self, peer: Ipv6Addr) { + // Exchange updates take the same lock. If an old-peer update is + // already running, cleanup follows it and removes its imports. If + // cleanup wins, the update subsequently observes the changed peer + // identity and is discarded. + let _route_update = mg_common::lock!(self.ctx.iface.route_update); + let removed = self.ctx.db.remove_nexthop_routes(peer); + self.redistribute_removed_routes(&removed); + let crate::db::RemovedNexthopRoutes { + underlay: to_remove, + tunnel: to_remove_tnl, + multicast: to_remove_mcast, + .. + } = removed; let mut routes: Vec = Vec::new(); for x in &to_remove { let mut r: crate::sys::Route = x.clone().into(); @@ -325,21 +435,34 @@ impl Exchange { to_remove_tnl ); } - // if we're a transit router propagate withdraws for the - // expired peer. + + // The peer's routes are gone from the imported set, so we notify the + // multicast sweep of each affected underlay group. The sweep drops the + // peer's replication membership from DPD. + crate::mcast::notify_affected_groups( + to_remove_mcast.iter(), + &self.ctx.mcast_notify, + ); + } + + fn redistribute_removed_routes( + &self, + removed: &crate::db::RemovedNexthopRoutes, + ) { + // If we're a transit router propagate withdraws for the + // removed routes. if self.ctx.config.kind == RouterKind::Transit { dbg!( self.log, self.ctx.config.if_name, - "redistributing expire to {} peers", + "redistributing withdraws to {} peers", self.ctx.event_channels.len() ); - let underlay = if to_remove.is_empty() { - None - } else { - Some(UnderlayUpdate::withdraw( - to_remove + let underlay = (!removed.underlay.is_empty()).then(|| { + UnderlayUpdate::withdraw( + removed + .underlay .iter() .map(|x| PathVector { destination: x.destination, @@ -350,23 +473,61 @@ impl Exchange { }, }) .collect(), - )) - }; + ) + }); + + let tunnel = (!removed.tunnel.is_empty()).then(|| { + TunnelUpdate::withdraw( + removed.tunnel.iter().cloned().map(Into::into).collect(), + ) + }); - let tunnel = if to_remove_tnl.is_empty() { + // Downstream peers collapse all paths through us into one route. + // For each removed origin, either withdraw the final path or + // refresh the peer with a remaining imported/local path. + let multicast = if removed.multicast.is_empty() { None } else { - Some(TunnelUpdate::withdraw( - to_remove_tnl.iter().cloned().map(Into::into).collect(), - )) + let withdrawals: HashSet<_> = removed + .multicast + .iter() + .map(|route| MulticastPathVector { + origin: (&route.origin).into(), + path: Vec::new(), + }) + .collect(); + let update = crate::exchange::reconcile_multicast_withdrawals( + &withdrawals, + &removed.mcast_reachability, + &MulticastPathHop::new( + self.ctx.hostname.clone(), + self.ctx.config.addr, + ), + ); + if update.announce.is_empty() && update.withdraw.is_empty() { + None + } else { + Some(update) + } }; - let push = Update { underlay, tunnel }; + let push = Arc::new(Update { + underlay, + tunnel, + multicast, + }); for ec in &self.ctx.event_channels { - ec.send(Event::Peer(PeerEvent::Push(push.clone()))).unwrap(); + if let Err(e) = + ec.send(Event::Peer(PeerEvent::Push(Arc::clone(&push)))) + { + err!( + self.log, + self.ctx.config.if_name, + "deliver redistributed withdraw: {e}", + ); + } } } - pull_stop.store(true, Ordering::Relaxed); } } @@ -376,7 +537,7 @@ impl State for Exchange { event: Receiver, ) -> (Box, Receiver) { self.ctx.iface.transition(FsmState::Exchange); - let exchange_thread = loop { + let exchange_handle = loop { match exchange::handler( self.ctx.clone(), self.ctx.config.addr, @@ -399,22 +560,31 @@ impl State for Exchange { self.wait_for_exchange_server_to_start(); - let pull_stop = Arc::new(AtomicBool::new(false)); + let mut pull_stop = Arc::new(AtomicBool::new(false)); + let mut pull_active = Arc::new(AtomicBool::new(false)); // Do an initial pull, in the event that exchange events are fired while // this pull is taking place, they will be queued and handled in the // loop below. - self.initial_pull(pull_stop.clone()); + self.initial_pull(pull_stop.clone(), pull_active.clone()); + let mut resync_deadline = Instant::now() + EXCHANGE_RESYNC_INTERVAL; loop { - let e = match event.recv() { + if Instant::now() >= resync_deadline { + self.periodic_pull(pull_stop.clone(), pull_active.clone()); + resync_deadline = Instant::now() + EXCHANGE_RESYNC_INTERVAL; + } + + let wait = + resync_deadline.saturating_duration_since(Instant::now()); + let e = match event.recv_timeout(wait) { Ok(e) => e, + Err(RecvTimeoutError::Timeout) => continue, Err(e) => { err!( self.log, self.ctx.config.if_name, - "exchange event recv: {}", - e + "exchange event recv: {e}", ); continue; } @@ -442,8 +612,7 @@ impl State for Exchange { err!( self.log, self.ctx.config.if_name, - "announce: {}", - e, + "announce: {e}", ); wrn!( self.log, @@ -451,7 +620,7 @@ impl State for Exchange { "expiring peer {} due to failed announce", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -477,8 +646,7 @@ impl State for Exchange { err!( self.log, self.ctx.config.if_name, - "announce tunnel: {}", - e, + "announce tunnel: {e}", ); wrn!( self.log, @@ -486,7 +654,7 @@ impl State for Exchange { "expiring peer {} due to failed tunnel announce", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -518,8 +686,7 @@ impl State for Exchange { err!( self.log, self.ctx.config.if_name, - "withdraw: {}", - e, + "withdraw: {e}", ); wrn!( self.log, @@ -527,7 +694,7 @@ impl State for Exchange { "expiring peer {} due to failed withdraw", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -553,8 +720,7 @@ impl State for Exchange { err!( self.log, self.ctx.config.if_name, - "withdraw tunnel: {}", - e, + "withdraw tunnel: {e}", ); wrn!( self.log, @@ -562,7 +728,7 @@ impl State for Exchange { "expiring peer {} due to failed tunnel withdraw", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -572,6 +738,151 @@ impl State for Exchange { ); } } + Event::Admin(AdminEvent::AnnounceMulticast(groups)) => { + // Build a `MulticastPathVector` for each origin, recording + // our hop in the path. + let hop = MulticastPathHop::new( + self.ctx.hostname.clone(), + self.ctx.config.addr, + ); + let path_vectors: HashSet<_> = groups + .iter() + .map(|origin| { + ddm_api_types::exchange::MulticastPathVector { + origin: origin.into(), + path: vec![hop.clone()], + } + }) + .collect(); + + if let Err(e) = crate::exchange::announce_multicast( + &self.ctx, + self.ctx.config.clone(), + path_vectors, + self.peer, + self.version, + self.ctx.rt.clone(), + self.log.clone(), + ) { + err!( + self.log, + self.ctx.config.if_name, + "announce multicast: {e}", + ); + if e.expires_peer() { + wrn!( + self.log, + self.ctx.config.if_name, + "expiring peer {} due to failed multicast announce", + self.peer, + ); + self.expire_peer(&exchange_handle, &pull_stop); + return ( + Box::new(Solicit::new( + self.ctx.clone(), + self.log.clone(), + )), + event, + ); + } + } + } + Event::Admin(AdminEvent::WithdrawMulticast(origins)) => { + // The persistent local origins were removed by the + // modification that preceded this event. The reachability + // snapshot is read here, at processing time, so an + // import that raced the admin request is observed and + // produces a replacement announcement rather than a + // stale final withdrawal. Whenever an imported path to + // an origin remains, replace the local announcement with + // that path. Otherwise, propagate the final withdrawal. + let hop = MulticastPathHop::new( + self.ctx.hostname.clone(), + self.ctx.config.addr, + ); + let withdrawals: HashSet<_> = origins + .iter() + .map(|origin| MulticastPathVector { + origin: origin.into(), + path: Vec::new(), + }) + .collect(); + let reachability = self.ctx.db.multicast_reachability(); + let update = + crate::exchange::reconcile_multicast_withdrawals( + &withdrawals, + &reachability, + &hop, + ); + + if !update.announce.is_empty() + && let Err(e) = crate::exchange::announce_multicast( + &self.ctx, + self.ctx.config.clone(), + update.announce, + self.peer, + self.version, + self.ctx.rt.clone(), + self.log.clone(), + ) + { + err!( + self.log, + self.ctx.config.if_name, + "replace withdrawn multicast path: {e}", + ); + if e.expires_peer() { + wrn!( + self.log, + self.ctx.config.if_name, + "expiring peer {} due to failed multicast replacement", + self.peer, + ); + self.expire_peer(&exchange_handle, &pull_stop); + return ( + Box::new(Solicit::new( + self.ctx.clone(), + self.log.clone(), + )), + event, + ); + } + } + + if !update.withdraw.is_empty() + && let Err(e) = crate::exchange::withdraw_multicast( + &self.ctx, + self.ctx.config.clone(), + update.withdraw, + self.peer, + self.version, + self.ctx.rt.clone(), + self.log.clone(), + ) + { + err!( + self.log, + self.ctx.config.if_name, + "withdraw multicast: {e}", + ); + if e.expires_peer() { + wrn!( + self.log, + self.ctx.config.if_name, + "expiring peer {} due to failed multicast withdraw", + self.peer, + ); + self.expire_peer(&exchange_handle, &pull_stop); + return ( + Box::new(Solicit::new( + self.ctx.clone(), + self.log.clone(), + )), + event, + ); + } + } + } Event::Admin(AdminEvent::Expire(peer)) => { if self.peer == peer { inf!( @@ -580,7 +891,7 @@ impl State for Exchange { "administratively expiring peer {}", peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -591,20 +902,7 @@ impl State for Exchange { } } Event::Admin(AdminEvent::Sync) => { - if let Err(e) = crate::exchange::pull( - self.ctx.clone(), - self.peer, - self.version, - self.ctx.rt.clone(), - self.log.clone(), - ) { - err!( - self.log, - self.ctx.config.if_name, - "exchange pull: {}", - e - ); - } + self.sync_pull(pull_stop.clone(), pull_active.clone()); } Event::Peer(PeerEvent::Push(update)) => { inf!( @@ -614,6 +912,8 @@ impl State for Exchange { self.peer, update, ); + let update = Arc::try_unwrap(update) + .unwrap_or_else(|arc| (*arc).clone()); if let Some(push) = update.underlay { if !push.announce.is_empty() && let Err(e) = crate::exchange::announce_underlay( @@ -629,8 +929,7 @@ impl State for Exchange { err!( self.log, self.ctx.config.if_name, - "announce: {}", - e, + "announce: {e}", ); wrn!( self.log, @@ -638,7 +937,7 @@ impl State for Exchange { "expiring peer {} due to failed announce", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -647,6 +946,7 @@ impl State for Exchange { event, ); } + if !push.withdraw.is_empty() && let Err(e) = crate::exchange::withdraw_underlay( &self.ctx, @@ -661,8 +961,7 @@ impl State for Exchange { err!( self.log, self.ctx.config.if_name, - "withdraw: {}", - e, + "withdraw: {e}", ); wrn!( self.log, @@ -670,7 +969,7 @@ impl State for Exchange { "expiring peer {} due to failed withdraw", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -680,6 +979,76 @@ impl State for Exchange { ); } } + + if let Some(push) = update.multicast { + if !push.announce.is_empty() + && let Err(e) = crate::exchange::announce_multicast( + &self.ctx, + self.ctx.config.clone(), + push.announce, + self.peer, + self.version, + self.ctx.rt.clone(), + self.log.clone(), + ) + { + err!( + self.log, + self.ctx.config.if_name, + "announce multicast: {e}", + ); + if e.expires_peer() { + wrn!( + self.log, + self.ctx.config.if_name, + "expiring peer {} due to failed multicast announce", + self.peer, + ); + self.expire_peer(&exchange_handle, &pull_stop); + return ( + Box::new(Solicit::new( + self.ctx.clone(), + self.log.clone(), + )), + event, + ); + } + } + + if !push.withdraw.is_empty() + && let Err(e) = crate::exchange::withdraw_multicast( + &self.ctx, + self.ctx.config.clone(), + push.withdraw, + self.peer, + self.version, + self.ctx.rt.clone(), + self.log.clone(), + ) + { + err!( + self.log, + self.ctx.config.if_name, + "withdraw multicast: {e}", + ); + if e.expires_peer() { + wrn!( + self.log, + self.ctx.config.if_name, + "expiring peer {} due to failed multicast withdraw", + self.peer, + ); + self.expire_peer(&exchange_handle, &pull_stop); + return ( + Box::new(Solicit::new( + self.ctx.clone(), + self.log.clone(), + )), + event, + ); + } + } + } } Event::Neighbor(NeighborEvent::Expire) => { wrn!( @@ -688,7 +1057,7 @@ impl State for Exchange { "expiring peer {} due to discovery event", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -704,15 +1073,51 @@ impl State for Exchange { "expiring peer {} due to failed solicit", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Init::new(self.ctx.clone(), self.log.clone())), event, ); } Event::Neighbor(NeighborEvent::Advertise((addr, version))) => { + let peer_changed = + addr != self.peer || version != self.version; + if peer_changed { + // Any worker still using the prior peer identity must + // stop before a new pull starts. + pull_stop.store(true, Ordering::Relaxed); + pull_stop = Arc::new(AtomicBool::new(false)); + pull_active = Arc::new(AtomicBool::new(false)); + } + if addr != self.peer { + // A re-advertisement carrying a new address renumbers + // the peer. Expiry removes routes keyed on the + // current address only, so routes under the prior + // address would otherwise persist indefinitely. + // Withdraw them as if that address expired. + inf!( + self.log, + self.ctx.config.if_name, + "peer renumbered from {} to {}", + self.peer, + addr, + ); + // Rebind the running exchange handler so that + // pushes arriving under the new address cannot + // recreate routes keyed by the prior peer after + // cleanup. + exchange_handle.renumber_peer(addr); + self.withdraw_peer_routes(self.peer); + } self.peer = addr; self.version = version; + // Wake the multicast sweep for the peer's groups under + // the advertised address. + crate::mcast::notify_peer_groups( + &self.ctx.db, + addr, + &self.ctx.mcast_notify, + ); } } } diff --git a/ddm/src/sys.rs b/ddm/src/sys.rs index c251ab600..ee9cdee49 100644 --- a/ddm/src/sys.rs +++ b/ddm/src/sys.rs @@ -8,6 +8,7 @@ use ddm_api_types::db::TunnelRoute; use dpd_client::Client; use dpd_client::ClientState; use dpd_client::types; +use mg_common::tfport::{TfportKind, parse_tfport_name, tfport_port_id}; use oxnet::{IpNet, Ipv4Net, Ipv6Net}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -23,7 +24,9 @@ use ::{ std::collections::HashMap, }; -const DDM_DPD_TAG: &str = "ddmd"; +/// Client identity tag carried in `dpd_client::ClientState` to identify +/// `ddmd` to DPD, distinct from any per-group authorization tag DPD owns. +pub(crate) const DDM_DPD_TAG: &str = "ddmd"; #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] pub struct Route { @@ -176,25 +179,25 @@ pub fn add_routes_dendrite( } }; - // TODO this is gross, use link type properties rather than futzing - // around with strings. - let Some(egress_port_num) = ifname - .strip_prefix("tfportrear") - .and_then(|x| x.strip_suffix("_0")) - .map(|x| x.trim()) - .and_then(|x| x.parse::().ok()) - else { - err!(log, ifname, "expected tfportrear"); - continue; + let tfport = match parse_tfport_name(ifname) { + Ok(tfport) => tfport, + Err(e) => { + err!(log, ifname, "{e}"); + continue; + } }; // TODO this assumes ddm only operates on rear ports, which will not be // true for multi-rack deployments. - let port_name = format!("rear{}", egress_port_num); - let port_id = match types::Rear::try_from(&port_name) { - Ok(rear) => PortId::Rear(rear), + if tfport.kind != TfportKind::Rear { + err!(log, ifname, "expected rear tfport, got {:?}", tfport.kind); + continue; + } + + let port_id = match tfport_port_id(tfport.kind, tfport.port) { + Ok(port_id) => port_id, Err(e) => { - err!(log, ifname, "bad port name ({port_name}): {e}"); + err!(log, ifname, "{e}"); continue; } }; @@ -206,11 +209,10 @@ pub fn add_routes_dendrite( r.dest, r.gw, port_id, - 0, + tfport.link, ); - // TODO breakout considerations - let link_id = types::LinkId(0); + let link_id = types::LinkId(tfport.link); let target = types::Ipv6Route { tag: DDM_DPD_TAG.into(), diff --git a/ddmadm/src/main.rs b/ddmadm/src/main.rs index 21d56fdf3..820fa0f79 100644 --- a/ddmadm/src/main.rs +++ b/ddmadm/src/main.rs @@ -6,7 +6,7 @@ use anyhow::Result; use clap::Parser; use colored::*; use ddm_admin_client::Client; -use ddm_api_types_versions::latest::db::PeerStatus; +use ddm_api_types_versions::latest::db::{PeerStatus, RouterKind}; use ddm_api_types_versions::latest::net as types; use mg_common::cli::oxide_cli_style; use mg_common::format_duration_human; @@ -63,6 +63,18 @@ enum SubCommand { /// Withdraw prefixes from a DDM router. TunnelWithdraw(TunnelEndpoint), + /// Get multicast groups imported from DDM peers. + MulticastImported, + + /// Get locally originated multicast groups. + MulticastOriginated, + + /// Advertise multicast groups from this router. + MulticastAdvertise(MulticastGroup), + + /// Withdraw multicast groups from this router. + MulticastWithdraw(MulticastGroup), + /// Sync prefix information from peers. Sync, } @@ -87,6 +99,29 @@ struct TunnelEndpoint { pub metric: u64, } +#[derive(Debug, Parser)] +struct MulticastGroup { + /// Overlay multicast group address (e.g. 233.252.0.1 or ff0e::1). + #[arg(short = 'g', long)] + pub overlay_group: IpAddr, + + /// Underlay multicast address (ff04::/64 admin-local scope). + #[arg(short = 'u', long)] + pub underlay_group: Ipv6Addr, + + /// Virtual Network Identifier. + #[arg(short, long)] + pub vni: u32, + + /// Path metric. + #[arg(short, long, default_value_t = 0)] + pub metric: u64, + + /// Source address for (S,G) routes (omit for (*,G)). + #[arg(short, long)] + pub source: Option, +} + #[derive(Debug, Parser)] struct Peer { addr: Ipv6Addr, @@ -144,8 +179,8 @@ async fn run() -> Result<()> { info.host, info.addr, match info.kind { - ddm_api_types_versions::latest::db::RouterKind::Server => "Server", - ddm_api_types_versions::latest::db::RouterKind::Transit => "Transit", + RouterKind::Server => "Server", + RouterKind::Transit => "Transit", }, state, format_duration_human(*duration), @@ -262,6 +297,107 @@ async fn run() -> Result<()> { }]) .await?; } + SubCommand::MulticastImported => { + let msg = client.get_multicast_groups().await?; + let mut routes: Vec<_> = msg.into_inner().into_iter().collect(); + routes.sort_by(|a, b| { + a.origin + .overlay_group + .cmp(&b.origin.overlay_group) + .then_with(|| a.origin.source.cmp(&b.origin.source)) + }); + let mut tw = TabWriter::new(stdout()); + writeln!( + &mut tw, + "{}\t{}\t{}\t{}\t{}\t{}", + "Overlay Group".dimmed(), + "Underlay Group".dimmed(), + "VNI".dimmed(), + "Metric".dimmed(), + "Source".dimmed(), + "Path".dimmed(), + )?; + for route in &routes { + let source = match &route.origin.source { + Some(s) => s.to_string(), + None => "(*,G)".to_string(), + }; + let path: Vec<_> = route + .path + .iter() + .rev() + .map(|h| h.router_id.clone()) + .collect(); + writeln!( + &mut tw, + "{}\t{}\t{}\t{}\t{}\t{}", + route.origin.overlay_group, + route.origin.underlay_group, + route.origin.vni, + route.origin.metric, + source, + path.join(" "), + )?; + } + tw.flush()?; + } + SubCommand::MulticastOriginated => { + let msg = client.get_originated_multicast_groups().await?; + let mut origins: Vec<_> = msg.into_inner().into_iter().collect(); + origins.sort_by(|a, b| { + a.overlay_group + .cmp(&b.overlay_group) + .then_with(|| a.source.cmp(&b.source)) + }); + let mut tw = TabWriter::new(stdout()); + writeln!( + &mut tw, + "{}\t{}\t{}\t{}\t{}", + "Overlay Group".dimmed(), + "Underlay Group".dimmed(), + "VNI".dimmed(), + "Metric".dimmed(), + "Source".dimmed(), + )?; + for origin in &origins { + let source = match &origin.source { + Some(s) => s.to_string(), + None => "(*,G)".to_string(), + }; + writeln!( + &mut tw, + "{}\t{}\t{}\t{}\t{}", + origin.overlay_group, + origin.underlay_group, + u32::from(origin.vni), + origin.metric, + source, + )?; + } + tw.flush()?; + } + SubCommand::MulticastAdvertise(mg) => { + client + .advertise_multicast_groups(&vec![types::MulticastOrigin { + overlay_group: mg.overlay_group.try_into()?, + underlay_group: mg.underlay_group.try_into()?, + vni: types::Vni::try_from(mg.vni)?, + metric: mg.metric, + source: mg.source, + }]) + .await?; + } + SubCommand::MulticastWithdraw(mg) => { + client + .withdraw_multicast_groups(&vec![types::MulticastOrigin { + overlay_group: mg.overlay_group.try_into()?, + underlay_group: mg.underlay_group.try_into()?, + vni: types::Vni::try_from(mg.vni)?, + metric: mg.metric, + source: mg.source, + }]) + .await?; + } SubCommand::Sync => { client.sync().await?; } diff --git a/ddmd/src/main.rs b/ddmd/src/main.rs index 671e5cdd1..e957ef1ef 100644 --- a/ddmd/src/main.rs +++ b/ddmd/src/main.rs @@ -108,9 +108,9 @@ struct Arg { sled_uuid: Option, /// Serve only the admin API. Skips the routing state machine - /// (discovery, exchange, route synchronization), allowing test fixtures - /// to obtain a real `ddmd` admin endpoint without the kernel-level - /// networking the state machine requires. + /// (discovery, exchange, route synchronization), allowing a real `ddmd` + /// admin endpoint without the kernel-level networking the state machine + /// requires. /// /// Analogous to `mgd --no-bgp-dispatcher`. #[arg(long, default_value_t = false, conflicts_with = "addr")] @@ -160,14 +160,34 @@ async fn run() { .to_string_lossy() .to_string(); + // Notify channel into the multicast membership sweep. Each state machine's + // context holds the sender and signals a group's address when its imported + // membership changes. The sweep started by start_mcast_sweep owns the + // receiver and wakes early to reconcile the full tracked set. + let (notify_tx, notify_rx) = + tokio::sync::mpsc::channel::(ddm::MCAST_NOTIFY_CHANNEL_DEPTH); + let (sms, event_channels) = - start_state_machines(&arg, &db, &dpd, &hostname, &rt, &log); + start_state_machines(&arg, &db, &dpd, &hostname, &rt, ¬ify_tx, &log); termination_handler(db.clone(), dpd.clone(), rt.clone(), log.clone()); let router_stats = Arc::new(RouterStats::default()); + // Per-interface state machine contexts shared between the multicast sweep + // and the admin context, seeded from the running state machines. + // + // Under --api-only there are no state machines, so the set is empty. let peers: Vec = sms.iter().map(|x| x.ctx.clone()).collect(); + start_mcast_sweep( + notify_rx, + dpd.clone(), + db.clone(), + peers.clone(), + rt.clone(), + log.clone(), + ); + let stats_handler = if arg.with_stats { if let (Some(rack_uuid), Some(sled_uuid)) = (arg.rack_uuid, arg.sled_uuid) @@ -233,6 +253,7 @@ fn start_state_machines( dpd: &Option, hostname: &str, rt: &Arc, + notify_tx: &tokio::sync::mpsc::Sender, log: &Logger, ) -> ( Vec, @@ -273,6 +294,7 @@ fn start_state_machines( rt: rt.clone(), iface: Arc::new(InterfaceState::default()), stats: Arc::new(ddm::sm::SessionStats::default()), + mcast_notify: notify_tx.clone(), }; let sm = StateMachine { ctx, rx: Some(rx) }; @@ -309,6 +331,7 @@ fn start_state_machines( _dpd: &Option, _hostname: &str, _rt: &Arc, + _notify_tx: &tokio::sync::mpsc::Sender, _log: &Logger, ) -> ( Vec, @@ -317,7 +340,51 @@ fn start_state_machines( (Vec::new(), Vec::new()) } -/// Install a Ctrl-C handler that withdraws ddmd's imported routes from the +/// Spawn the underlay multicast membership sweep, the multicast analog of the +/// unicast import-to-DPD path `ddmd` already performs in-process. The sweep +/// runs as a task on the daemon's runtime, reconciling every tracked group on +/// each pass, woken early by a trigger and otherwise self-ticking on a fixed +/// interval. +/// +/// Takes `notify_rx` by value so any path that does not start the sweep drops +/// it, closing the channel and making the state machines' notify sends fail +/// fast rather than accumulate as unread. +#[cfg(all(feature = "backend", target_os = "illumos"))] +fn start_mcast_sweep( + notify_rx: tokio::sync::mpsc::Receiver, + dpd: Option, + db: Db, + peers: Vec, + rt: Arc, + log: Logger, +) { + let Some(dpd) = dpd else { + // No backend: returning drops notify_rx and closes the channel. + return; + }; + // Under --api-only there are no state machines, so no peers or imports + // exist to derive membership from. A sweep would compute an empty + // desired set for every seeded group and drain its DPD members. + if peers.is_empty() { + return; + } + rt.spawn(ddm::mcast::run(db, peers, dpd, notify_rx, log)); +} + +/// Non-illumos variant: underlay multicast replicates only on a switch, so the +/// sweep never starts. Consuming `notify_rx` drops it, closing the channel. +#[cfg(not(all(feature = "backend", target_os = "illumos")))] +fn start_mcast_sweep( + _notify_rx: tokio::sync::mpsc::Receiver, + _dpd: Option, + _db: Db, + _peers: Vec, + _rt: Arc, + _log: Logger, +) { +} + +/// Install a Ctrl-C handler that withdraws `ddmd`'s imported routes from the /// kernel before exiting. On non-illumos builds there are no kernel routes /// to withdraw, so the handler just exits cleanly. fn termination_handler( diff --git a/mg-api-types/versions/src/impls/mrib.rs b/mg-api-types/versions/src/impls/mrib.rs index 1fc6d1b99..273752794 100644 --- a/mg-api-types/versions/src/impls/mrib.rs +++ b/mg-api-types/versions/src/impls/mrib.rs @@ -2,8 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -// Copyright 2026 Oxide Computer Company - //! Proptest `Arbitrary` impls and strategy helpers for the latest MRIB types. use std::net::{Ipv4Addr, Ipv6Addr}; diff --git a/mg-api-types/versions/src/multicast_support/mod.rs b/mg-api-types/versions/src/multicast_support/mod.rs index 5f0b1552b..76d666354 100644 --- a/mg-api-types/versions/src/multicast_support/mod.rs +++ b/mg-api-types/versions/src/multicast_support/mod.rs @@ -2,8 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -// Copyright 2026 Oxide Computer Company - //! Version `MULTICAST_SUPPORT` of the Maghemite Admin API. //! //! Adds MRIB (Multicast Routing Information Base) endpoints for static diff --git a/mg-api-types/versions/src/multicast_support/mrib.rs b/mg-api-types/versions/src/multicast_support/mrib.rs index 3f196894e..d1677dda0 100644 --- a/mg-api-types/versions/src/multicast_support/mrib.rs +++ b/mg-api-types/versions/src/multicast_support/mrib.rs @@ -14,7 +14,6 @@ use std::fmt::{self, Formatter}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; -use std::str::FromStr; use chrono::{DateTime, Utc}; use client_common::address::{ @@ -22,6 +21,8 @@ use client_common::address::{ IPV4_SSM_RESERVED_SUBNET, IPV6_MULTICAST_RANGE, UNDERLAY_MULTICAST_SUBNET, is_ssm_address, }; +pub use client_common::multicast::UnderlayMulticastIpv6; +pub use client_common::vni::{Vni, VniError}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -43,98 +44,6 @@ pub enum MulticastError { DbKey(String), } -/// Error raised while validating a [`Vni`]. -#[derive(thiserror::Error, Debug)] -pub enum VniError { - /// The value exceeds the 24-bit Geneve maximum. - #[error("VNI {value} exceeds the maximum 24-bit value {}", Vni::MAX_VNI)] - OutOfRange { value: u32 }, -} - -/// A validated Geneve Virtual Network Identifier. -/// -/// Wraps a 24-bit VNI, rejecting any value above [`Vni::MAX_VNI`] at -/// construction and deserialization so an out-of-range identifier is -/// unrepresentable. -#[derive( - Debug, - Copy, - Clone, - Eq, - PartialEq, - PartialOrd, - Ord, - Hash, - Serialize, - Deserialize, - JsonSchema, -)] -#[serde(try_from = "u32", into = "u32")] -#[schemars(transparent)] -pub struct Vni(u32); - -impl Vni { - /// Maximum Geneve VNI value. - /// - /// Virtual Network Identifiers are constrained to 24-bit values per the - /// Geneve specification (RFC 8926 Section 3.3). - pub const MAX_VNI: u32 = 0xFF_FFFF; - - /// Default VNI for fleet-wide multicast routing. - /// - /// A low-numbered VNI chosen to avoid colliding with user VNIs, though - /// it is not yet within the Oxide-reserved range. - pub const DEFAULT_MULTICAST: Self = Self(77); - - /// Create a validated VNI. - /// - /// # Errors - /// - /// Returns [`VniError::OutOfRange`] if `value` exceeds [`Vni::MAX_VNI`], - /// the largest 24-bit Geneve VNI. - /// - /// # Examples - /// - /// ``` - /// use mg_api_types_versions::latest::mrib::Vni; - /// - /// assert!(Vni::new(77).is_ok()); - /// assert!(Vni::new(Vni::MAX_VNI + 1).is_err()); - /// ``` - pub fn new(value: u32) -> Result { - if value > Self::MAX_VNI { - return Err(VniError::OutOfRange { value }); - } - Ok(Self(value)) - } - - /// Return the underlying 24-bit value. - #[inline] - pub const fn as_u32(self) -> u32 { - self.0 - } -} - -impl fmt::Display for Vni { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -impl TryFrom for Vni { - type Error = VniError; - - fn try_from(value: u32) -> Result { - Self::new(value) - } -} - -impl From for u32 { - fn from(vni: Vni) -> Self { - vni.0 - } -} - /// Input for adding static multicast routes. #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] pub struct StaticMulticastRouteInput { @@ -607,92 +516,6 @@ impl From for Ipv6Addr { } } -/// A validated underlay multicast IPv6 address within ff04::/64. -/// -/// The Oxide rack maps overlay multicast groups 1:1 to admin-local scoped -/// IPv6 multicast addresses in `UNDERLAY_MULTICAST_SUBNET` (ff04::/64). -/// This type enforces that invariant at construction time. -// TODO: Duplicates `dpd_types::mcast::UnderlayMulticastIpv6` in dendrite. -// Both should be consolidated into `oxnet`, the cycle-free leaf crate that -// maghemite, dendrite, and omicron already share. -#[derive( - Debug, - Copy, - Clone, - Eq, - PartialEq, - PartialOrd, - Ord, - Hash, - Serialize, - Deserialize, - JsonSchema, -)] -#[serde(try_from = "Ipv6Addr", into = "Ipv6Addr")] -#[schemars(transparent)] -pub struct UnderlayMulticastIpv6(Ipv6Addr); - -impl UnderlayMulticastIpv6 { - /// Create a new validated underlay multicast address. - /// - /// # Errors - /// - /// Returns an error if the address is not within `UNDERLAY_MULTICAST_SUBNET` - /// (ff04::/64). - pub fn new(value: Ipv6Addr) -> Result { - if !UNDERLAY_MULTICAST_SUBNET.contains(value) { - return Err(MulticastError::Validation(format!( - "underlay address {value} is not within \ - {UNDERLAY_MULTICAST_SUBNET}" - ))); - } - Ok(Self(value)) - } - - /// Returns the underlying IPv6 address. - #[inline] - pub const fn ip(&self) -> Ipv6Addr { - self.0 - } -} - -impl fmt::Display for UnderlayMulticastIpv6 { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -impl TryFrom for UnderlayMulticastIpv6 { - type Error = MulticastError; - - fn try_from(value: Ipv6Addr) -> Result { - Self::new(value) - } -} - -impl From for Ipv6Addr { - fn from(addr: UnderlayMulticastIpv6) -> Self { - addr.0 - } -} - -impl From for IpAddr { - fn from(addr: UnderlayMulticastIpv6) -> Self { - IpAddr::V6(addr.0) - } -} - -impl FromStr for UnderlayMulticastIpv6 { - type Err = MulticastError; - - fn from_str(s: &str) -> Result { - let addr: Ipv6Addr = s.parse().map_err(|_| { - MulticastError::Validation(format!("invalid IPv6 address: {s}")) - })?; - Self::new(addr) - } -} - /// A validated multicast group address (IPv4 or IPv6). /// /// This type guarantees that the contained address is a routable multicast @@ -1186,31 +1009,15 @@ pub enum MulticastSourceProtocol { #[cfg(test)] mod tests { - use omicron_common::api::external::Vni as CanonicalVni; - use super::*; - /// Assert the locally copied VNI literals equal their - /// `omicron_common::api::external::Vni` originals so they cannot drift. - /// - /// `omicron_common` is a dev-dependency only, so it does not appear in the - /// normal dependency tree the no-omicron CI check inspects. - #[test] - fn vni_constants_match_canonical_values() { - assert_eq!(Vni::MAX_VNI, CanonicalVni::MAX_VNI); - assert_eq!( - Vni::DEFAULT_MULTICAST.as_u32(), - CanonicalVni::DEFAULT_MULTICAST_VNI.as_u32() - ); - } - - /// The [`Vni`] newtype accepts in-range values and rejects values above - /// [`Vni::MAX_VNI`], enforcing the 24-bit invariant at construction. #[test] - fn vni_rejects_out_of_range() { - assert_eq!(Vni::new(0).unwrap().as_u32(), 0); - assert_eq!(Vni::new(Vni::MAX_VNI).unwrap().as_u32(), Vni::MAX_VNI); - assert!(Vni::new(Vni::MAX_VNI + 1).is_err()); - assert!(Vni::new(u32::MAX).is_err()); + fn unicast_rejects_non_routable_sources() { + // 0/8 "this network" (RFC 791). + assert!(UnicastAddrV4::new(Ipv4Addr::new(0, 1, 2, 3)).is_err()); + // Link-local, 169.254/16 (RFC 3927): not router-forwarded. + assert!(UnicastAddrV4::new(Ipv4Addr::new(169, 254, 0, 1)).is_err()); + // Link-local, fe80::/10 (RFC 4291): not forwarded. + assert!(UnicastAddrV6::new("fe80::1".parse().unwrap()).is_err()); } } diff --git a/mg-common/Cargo.toml b/mg-common/Cargo.toml index ad4218def..501e22f4e 100644 --- a/mg-common/Cargo.toml +++ b/mg-common/Cargo.toml @@ -7,6 +7,7 @@ edition = "2024" clap.workspace = true anyhow.workspace = true ddm-api-types.workspace = true +dpd-client.workspace = true anstyle.workspace = true serde.workspace = true schemars.workspace = true @@ -26,6 +27,10 @@ libc.workspace = true workspace = true optional = true +[dev-dependencies] +proptest.workspace = true +serde_json.workspace = true + [features] default = ["libnet"] libnet = ["dep:libnet"] diff --git a/mg-common/src/lib.rs b/mg-common/src/lib.rs index b2af8cd22..abb47e2fb 100644 --- a/mg-common/src/lib.rs +++ b/mg-common/src/lib.rs @@ -8,6 +8,7 @@ pub mod nexus; pub mod smf; pub mod stats; pub mod test; +pub mod tfport; pub mod thread; use std::time::Duration; diff --git a/mg-common/src/tfport.rs b/mg-common/src/tfport.rs new file mode 100644 index 000000000..b758b181f --- /dev/null +++ b/mg-common/src/tfport.rs @@ -0,0 +1,269 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Parsing of Tofino port (`tfport`) datalink names. +//! +//! Tofino switch ports are surfaced to the host as illumos datalinks named +//! `tfport_[.vlan]`; for example, `tfportqsfp10_0` (front +//! panel) or `tfportrear0_0.100` (backplane). This module parses that form +//! into its components so callers can map a datalink name back to a switch +//! port without each one hand-rolling its own string handling. + +/// Prefix shared by every `tfport` datalink name. +const TFPORT_DEVICE_PREFIX: &str = "tfport"; + +/// Switch-port device kind encoded in a `tfport` datalink name. +/// +/// Front-panel links typically appear as `qsfp`. Backplane links toward other +/// sleds (the multicast underlay path) typically appear as `rear`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TfportKind { + Qsfp, + Rear, +} + +impl TfportKind { + /// Device token as it appears both in a tfport name (`tfportrear0_0`) and + /// in a dpd port name (`rear0`). + pub fn token(self) -> &'static str { + match self { + TfportKind::Qsfp => "qsfp", + TfportKind::Rear => "rear", + } + } + + /// Parse a device kind from its datalink token (`qsfp`, `rear`). + pub fn from_token(s: &str) -> Option { + match s { + "qsfp" => Some(TfportKind::Qsfp), + "rear" => Some(TfportKind::Rear), + _ => None, + } + } +} + +/// Components parsed from a `tfport` datalink name. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TfportName { + /// Device kind (front-panel vs backplane). + pub kind: TfportKind, + /// Switch port number within the kind. + pub port: u8, + /// Link (lane) number within the port. + pub link: u8, + /// Optional VLAN tag appended after a `.`. + pub vlan: Option, +} + +/// Parse a `tfport` datalink name into its components. +/// +/// # Examples +/// +/// ``` +/// use mg_common::tfport::{parse_tfport_name, TfportKind, TfportName}; +/// assert_eq!( +/// parse_tfport_name("tfportqsfp10_0.100").unwrap(), +/// TfportName { kind: TfportKind::Qsfp, port: 10, link: 0, vlan: Some(100) }, +/// ); +/// assert_eq!( +/// parse_tfport_name("tfportrear0_0").unwrap(), +/// TfportName { kind: TfportKind::Rear, port: 0, link: 0, vlan: None }, +/// ); +/// ``` +/// +/// # Errors +/// +/// Returns an error if `name` lacks the `tfport` prefix, has an unrecognized +/// device kind, or has malformed port/link/vlan fields. +pub fn parse_tfport_name(name: &str) -> Result { + let body = name.strip_prefix(TFPORT_DEVICE_PREFIX).ok_or_else(|| { + format!("{name} missing expected prefix {TFPORT_DEVICE_PREFIX}") + })?; + + // The device kind is the leading alphabetic run (`qsfp`, `rear`), the + // remainder carries the port/link/vlan numbers. + let split = body + .find(|c: char| !c.is_ascii_alphabetic()) + .ok_or_else(|| format!("{name} has no port id"))?; + let (kind_str, rest) = body.split_at(split); + let kind = TfportKind::from_token(kind_str).ok_or_else(|| { + format!("{name} has unsupported device kind {kind_str}") + })?; + + let (port_link, vlan_str) = match rest.split_once('.') { + Some((port_link, vlan)) => (port_link, Some(vlan)), + None => (rest, None), + }; + + let (port, link) = port_link + .split_once('_') + .ok_or_else(|| format!("{name} has no link id"))?; + + let port = port + .parse::() + .map_err(|_| format!("{name} has invalid port {port}"))?; + + let link = link + .parse::() + .map_err(|_| format!("{name} has invalid link id {link}"))?; + + let vlan = match vlan_str { + None => None, + // A second `.` (e.g. `tfportqsfp10_0.100.200`) leaves a non-numeric + // remainder, so the parse below rejects it. + Some(vlan) => Some( + vlan.parse::() + .map_err(|_| format!("{name} has invalid vlan {vlan}"))?, + ), + }; + + Ok(TfportName { + kind, + port, + link, + vlan, + }) +} + +/// Build a dpd [`PortId`] from a parsed tfport kind and port number. +/// +/// # Errors +/// +/// Returns an error if the synthesized port name is not a valid dpd `qsfp` or +/// `rear` port identifier. +/// +/// [`PortId`]: dpd_client::types::PortId +pub fn tfport_port_id( + kind: TfportKind, + port: u8, +) -> Result { + use dpd_client::types; + + let port_name = format!("{}{port}", kind.token()); + match kind { + TfportKind::Qsfp => types::Qsfp::try_from(&port_name) + .map(types::PortId::Qsfp) + .map_err(|e| format!("bad qsfp port name {port_name}: {e}")), + TfportKind::Rear => types::Rear::try_from(&port_name) + .map(types::PortId::Rear) + .map_err(|e| format!("bad rear port name {port_name}: {e}")), + } +} + +/// Resolve a tfport datalink name (e.g. `tfportrear0_0`) to the dpd +/// `(PortId, LinkId)` pair that names the switch port and link. +/// +/// # Errors +/// +/// Returns an error if `ifname` is not a valid tfport datalink name or its +/// kind and port do not form a valid dpd port identifier. +/// +/// [`PortId`]: dpd_client::types::PortId +/// [`LinkId`]: dpd_client::types::LinkId +pub fn port_link_from_ifname( + ifname: &str, +) -> Result<(dpd_client::types::PortId, dpd_client::types::LinkId), String> { + let tfport = parse_tfport_name(ifname)?; + let port_id = tfport_port_id(tfport.kind, tfport.port)?; + // Breakout lanes surface as distinct datalinks (`qsfp0_0`, `qsfp0_1`, ...), + // sharing one `PortId`. The parsed link is the lane, which is the dpd + // `LinkId`. + let link_id = dpd_client::types::LinkId(tfport.link); + Ok((port_id, link_id)) +} + +#[cfg(test)] +mod tests { + use super::*; + use proptest::prelude::*; + + fn name( + kind: TfportKind, + port: u8, + link: u8, + vlan: Option, + ) -> TfportName { + TfportName { + kind, + port, + link, + vlan, + } + } + + #[test] + fn test_tfport_parser() { + // Valid qsfp (front-panel) names. + assert_eq!( + parse_tfport_name("tfportqsfp10_0").unwrap(), + name(TfportKind::Qsfp, 10, 0, None) + ); + assert_eq!( + parse_tfport_name("tfportqsfp10_0.100").unwrap(), + name(TfportKind::Qsfp, 10, 0, Some(100)) + ); + assert_eq!( + parse_tfport_name("tfportqsfp1_1").unwrap(), + name(TfportKind::Qsfp, 1, 1, None) + ); + + // Valid rear (backplane) names. + assert_eq!( + parse_tfport_name("tfportrear0_0").unwrap(), + name(TfportKind::Rear, 0, 0, None) + ); + assert_eq!( + parse_tfport_name("tfportrear31_0.200").unwrap(), + name(TfportKind::Rear, 31, 0, Some(200)) + ); + + // Malformed names. + assert!(parse_tfport_name("fportqsfp10_0").is_err()); + assert!(parse_tfport_name("10_0").is_err()); + assert!(parse_tfport_name("tfportqsfp10").is_err()); + assert!(parse_tfport_name("tfportqsfp_10").is_err()); + assert!(parse_tfport_name("tfportqsfp0_").is_err()); + assert!(parse_tfport_name("tfportqsfp10_10_10").is_err()); + assert!(parse_tfport_name("tfportqsfp10.100_0").is_err()); + + // Unsupported or missing device kind. + assert!(parse_tfport_name("tfportfoo0_0").is_err()); + assert!(parse_tfport_name("tfport0_0").is_err()); + + // Invalid numeric components. + assert!(parse_tfport_name("tfportqsfp1X_0.100").is_err()); + assert!(parse_tfport_name("tfportqsfp10_X.100").is_err()); + assert!(parse_tfport_name("tfportqsfp10_0.X").is_err()); + } + + proptest! { + /// Any well-formed name round-trips: formatting a `kind`, `port`, + /// `link`, `vlan` tuple and parsing it back yields the same components. + /// The parser is purely syntactic, so the full u8/u16 ranges are + /// exercised. + #[test] + fn prop_roundtrip( + is_rear in any::(), + port in any::(), + link in any::(), + vlan in proptest::option::of(any::()), + ) { + let kind = if is_rear { TfportKind::Rear } else { TfportKind::Qsfp }; + let mut ifname = format!("tfport{}{port}_{link}", kind.token()); + if let Some(vlan) = vlan { + ifname.push_str(&format!(".{vlan}")); + } + prop_assert_eq!( + parse_tfport_name(&ifname).unwrap(), + TfportName { kind, port, link, vlan }, + ); + } + + /// Parsing arbitrary input never panics; it always returns a `Result`. + #[test] + fn prop_never_panics(ifname in ".*") { + let _ = parse_tfport_name(&ifname); + } + } +} diff --git a/mg-lower/src/ddm.rs b/mg-lower/src/ddm.rs index 8e1744851..efa9e6d90 100644 --- a/mg-lower/src/ddm.rs +++ b/mg-lower/src/ddm.rs @@ -5,10 +5,13 @@ use crate::log::ddm_log; #[cfg(target_os = "illumos")] use ddm_admin_client::Client; -use ddm_api_types_versions::latest::net::TunnelOrigin; +use ddm_api_types_versions::latest::net::{MulticastOrigin, TunnelOrigin}; use oxnet::Ipv6Net; use slog::Logger; -use std::{net::Ipv6Addr, sync::Arc}; +use std::{ + net::{Ipv6Addr, SocketAddr}, + sync::Arc, +}; use crate::platform::Ddm; @@ -107,7 +110,71 @@ pub(crate) fn remove_tunnel_routes<'a, I: Iterator>( } } +/// Create a new DDM admin client. +/// +/// In production the lower half runs in the same zone as DDM, so `addr` is +/// `None` and the client targets the default `localhost:8000`. Tests pass an +/// explicit `addr` to reach a DDM listening elsewhere (for example a +/// dynamically assigned port in an integration harness). #[cfg(target_os = "illumos")] -pub fn new_ddm_client(log: &Logger) -> Client { - Client::new("http://localhost:8000", log.clone()) +pub fn new_ddm_client(log: &Logger, addr: Option) -> Client { + let host = match addr { + Some(addr) => format!("http://{addr}"), + None => "http://localhost:8000".to_string(), + }; + Client::new(&host, log.clone()) +} + +pub(crate) fn add_multicast_routes< + 'a, + I: Iterator, +>( + client: &impl Ddm, + routes: I, + rt: &Arc, + log: &Logger, +) { + let routes: Vec = routes.cloned().collect(); + if routes.is_empty() { + return; + } + let resp = + rt.block_on(async { client.advertise_multicast_groups(&routes).await }); + if let Err(e) = resp { + ddm_log!(log, + error, + "advertise multicast groups error: {e}"; + "error" => format!("{e}"), + "groups" => format!("{routes:#?}") + ); + } +} + +pub(crate) fn remove_multicast_routes< + 'a, + I: Iterator, +>( + client: &impl Ddm, + routes: I, + rt: &Arc, + log: &Logger, +) { + let routes: Vec = routes.cloned().collect(); + if routes.is_empty() { + return; + } + let resp = + rt.block_on(async { client.withdraw_multicast_groups(&routes).await }); + match resp { + Err(e) => ddm_log!(log, + error, + "withdraw multicast groups error: {e}"; + "groups" => format!("{routes:#?}") + ), + Ok(_) => ddm_log!(log, + debug, + "withdrew multicast groups"; + "groups" => format!("{routes:#?}") + ), + } } diff --git a/mg-lower/src/dendrite.rs b/mg-lower/src/dendrite.rs index 1de75f1da..d934f65fb 100644 --- a/mg-lower/src/dendrite.rs +++ b/mg-lower/src/dendrite.rs @@ -16,12 +16,11 @@ use slog::Logger; use std::{ collections::{BTreeSet, HashSet}, hash::Hash, - net::{IpAddr, Ipv4Addr, Ipv6Addr}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, sync::Arc, time::Duration, }; -const TFPORT_QSFP_DEVICE_PREFIX: &str = "tfportqsfp"; const UNIT_DPD: &str = "dpd"; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -329,68 +328,12 @@ where Ok(()) } -// Translate a tfport name into the underlying (port, link, vlan) tuple. -// tfportqsfp10_0 would translate to (10, 0, None) -// tfportqsfp10_0.100 would translate to (10, 0, Some(100)) -// TODO this is gross, use link type properties rather than futzing -// around with strings. -fn parse_tfport_name(name: &str) -> Result<(u8, u8, Option), Error> { - let body = - name.strip_prefix(TFPORT_QSFP_DEVICE_PREFIX) - .ok_or(Error::Tfport(format!( - "{} missing expected prefix {}", - name, TFPORT_QSFP_DEVICE_PREFIX - )))?; - let fields: Vec<&str> = body.split('.').collect(); - let (port, link) = fields[0] - .split_once('_') - .ok_or(Error::Tfport(format!("{} has no link id", name)))?; - - let port = port.parse::().map_err(|_| { - Error::Tfport(format!("{} has invalid port {}", name, port)) - })?; - - let link = link.parse::().map_err(|_| { - Error::Tfport(format!("{} has invalid link id {}", name, link)) - })?; - - let vlan = match fields.len() { - 1 => Ok(None), - 2 => fields[1].parse::().map(Some).map_err(|_| { - Error::Tfport(format!("{} has invalid vlan {}", name, fields[1])) - }), - _ => Err(Error::Tfport(format!( - "{} has multiple vlan deliminators", - name - ))), - }?; - - Ok((port, link, vlan)) -} - -#[test] -fn test_tfport_parser() { - // Test valid names - assert_eq!(parse_tfport_name("tfportqsfp10_0").unwrap(), (10, 0, None)); - assert_eq!( - parse_tfport_name("tfportqsfp10_0.100").unwrap(), - (10, 0, Some(100)) - ); - assert_eq!(parse_tfport_name("tfportqsfp1_1").unwrap(), (1, 1, None)); - - // test malformed names - assert!(parse_tfport_name("fportqsfp10_0").is_err()); - assert!(parse_tfport_name("10_0").is_err()); - assert!(parse_tfport_name("tfportqsfp10").is_err()); - assert!(parse_tfport_name("tfportqsfp_10").is_err()); - assert!(parse_tfport_name("tfportqsfp0_").is_err()); - assert!(parse_tfport_name("tfportqsfp10_10_10").is_err()); - assert!(parse_tfport_name("tfportqsfp10.100_0").is_err()); - - // test invalid components - assert!(parse_tfport_name("tfportqsfp1X_0.100").is_err()); - assert!(parse_tfport_name("tfportqsfp10_X.100").is_err()); - assert!(parse_tfport_name("tfportqsfp10_0.X").is_err()); +/// Resolve a tfport datalink name (e.g. `tfportrear0_0`) to the dpd +/// `(PortId, LinkId)` pair that names the switch port and link. +pub(crate) fn port_link_from_ifname( + ifname: &str, +) -> Result<(types::PortId, types::LinkId), Error> { + mg_common::tfport::port_link_from_ifname(ifname).map_err(Error::Tfport) } fn get_port_and_link( @@ -402,18 +345,7 @@ fn get_port_and_link( && nh6.is_unicast_link_local() && let Some(ref iface) = path.nexthop_interface { - let (port, link, _vlan) = parse_tfport_name(iface)?; - let port_name = format!("qsfp{port}"); - let port_id = types::Qsfp::try_from(&port_name) - .map(types::PortId::Qsfp) - .map_err(|e| { - Error::Tfport(format!( - "bad port name ifname: {iface} port name: {port_name}: {e}", - )) - })?; - // TODO breakout considerations - let link_id = types::LinkId(link); - return Ok((port_id, link_id)); + return port_link_from_ifname(iface); } // Standard nexthop resolution for numbered peers @@ -437,19 +369,7 @@ fn resolve_port_and_link( } }; - let (port, link, _vlan) = parse_tfport_name(&ifname)?; - let port_name = format!("qsfp{port}"); - let port_id = types::Qsfp::try_from(&port_name) - .map(types::PortId::Qsfp) - .map_err(|e| { - Error::Tfport(format!( - "bad port name ifname: {ifname} port name: {port_name}: {e}" - )) - })?; - - // TODO breakout considerations - let link_id = types::LinkId(link); - Ok((port_id, link_id)) + port_link_from_ifname(&ifname) } pub(crate) fn get_routes_for_prefix( @@ -560,16 +480,21 @@ pub(crate) fn get_routes_for_prefix( Ok(result.into_iter().collect()) } -/// Create a new Dendrite/dpd client. The lower half always runs on the same -/// host/zone as the underlying platform. +/// Create a new Dendrite (DPD) client. +/// +/// On the rack, the lower half runs in the same zone as DPD, so `addr` is +/// `None` and the client targets `localhost` on the default DPD port. Tests +/// pass an explicit `addr` to reach a DPD listening elsewhere (for example a +/// dynamically assigned port in an integration harness). #[cfg(target_os = "illumos")] -pub fn new_dpd_client(log: &Logger) -> DpdClient { +pub fn new_dpd_client(log: &Logger, addr: Option) -> DpdClient { let client_state = dpd_client::ClientState { tag: MG_LOWER_TAG.into(), log: log.clone(), }; - DpdClient::new( - &format!("http://localhost:{}", dpd_client::default_port()), - client_state, - ) + let host = match addr { + Some(addr) => format!("http://{addr}"), + None => format!("http://localhost:{}", dpd_client::default_port()), + }; + DpdClient::new(&host, client_state) } diff --git a/mg-lower/src/lib.rs b/mg-lower/src/lib.rs index 33096b5e1..042b4db16 100644 --- a/mg-lower/src/lib.rs +++ b/mg-lower/src/lib.rs @@ -40,6 +40,7 @@ mod ddm; mod dendrite; mod error; mod log; +pub mod mrib; mod platform; #[cfg(test)] diff --git a/mg-lower/src/mrib.rs b/mg-lower/src/mrib.rs new file mode 100644 index 000000000..0ecce505e --- /dev/null +++ b/mg-lower/src/mrib.rs @@ -0,0 +1,498 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! MRIB (Multicast Routing Information Base) lower-half synchronization. +//! +//! Advertises locally originated MRIB multicast groups to the DDM admin API, +//! which distributes them across the underlay to other sleds and racks. This is +//! the multicast analog of the unicast lower-half's tunnel-endpoint origination. +//! +//! Origination reads the local MRIB (`loc_mrib`) and is watch-driven, with the +//! periodic resync covering only missed notifications, the same shape as the +//! unicast lower-half's crate-level loop [`crate::run`]. +//! +//! The inbound membership half (resolving DDM-imported routes to switch +//! replication members in DPD) lives in ddmd (`ddm::mcast`), where both inputs, +//! the imported set and the peer table, are owned in-process. +//! +//! ## Data Flow +//! +//! ```text +//! Origination (MRIB -> DDM -> underlay) +//! MRIB (loc_mrib changes) +//! | [MribChangeNotification] +//! v [MulticastOrigin] (diff vs DDM originated) +//! DDM admin API --[DDM exchange]--> other sleds/racks +//! ``` +//! +//! See RFD 488 for the multicast architecture. + +use crate::ddm::{add_multicast_routes, remove_multicast_routes}; +use crate::platform::Ddm; +use ddm_api_types_versions::latest::net::{MulticastOrigin, OverlayMulticast}; +use rdb::Mrib; +use rdb::types::{MribChangeNotification, MulticastAddr, MulticastRoute}; +use slog::{Logger, debug, error, info}; +use std::collections::HashSet; +use std::sync::Arc; +use std::sync::mpsc::{RecvTimeoutError, channel}; +use std::thread::sleep; +use std::time::Duration; + +pub(crate) const MG_LOWER_MRIB_TAG: &str = "mg-lower-mrib"; + +/// Interval between periodic MRIB full syncs. +/// +/// Covers the case where an MRIB change notification is missed. Mirrors the +/// unicast lower-half's 1s resync cadence in `crate::run`. +const MRIB_PERIODIC_SYNC_INTERVAL: Duration = Duration::from_secs(1); + +/// Convert an MRIB `MulticastRoute` to a DDM `MulticastOrigin`. +fn ddm_origin(route: &MulticastRoute) -> MulticastOrigin { + MulticastOrigin { + // The MRIB key's group is a `MulticastAddr`, multicast by construction, + // so promotion to the validated overlay type cannot fail here. + overlay_group: OverlayMulticast::new(route.key.group().ip()) + .expect("MRIB group is multicast by construction"), + underlay_group: route.underlay_group, + vni: route.key.vni(), + metric: 0, + source: route.key.source(), + } +} + +/// Run the MRIB origination loop. +/// +/// This function loops forever, watching for MRIB changes and advertising +/// locally originated multicast groups to DDM. +/// +/// It runs on the calling thread, so callers are responsible for running it in +/// a separate thread if asynchronous execution is required. +pub fn run( + mrib: Mrib, + log: Logger, + rt: Arc, + ddm: &impl Ddm, +) { + loop { + let (tx, rx) = channel(); + + // Register as MRIB watcher + mrib.watch(MG_LOWER_MRIB_TAG.into(), tx); + + // Initial full sync + if let Err(e) = full_sync(&mrib, ddm, &log, &rt) { + error!(log, "MRIB full sync failed: {e}"); + info!(log, "restarting MRIB sync loop in one second"); + // Drop this iteration's watcher before retrying. The continue + // re-registers a fresh watcher, so without this the failed + // registration would accumulate in the MRIB watcher list on every + // retry. + mrib.unwatch(MG_LOWER_MRIB_TAG); + // Pause before retrying to keep a persistent failure from spinning. + // This is a backoff floor, kept independent of the resync cadence so + // tuning one does not silently change the other. + // + // Note: the unicast lower-half pauses in the same way + // (see `crate::run`). + sleep(Duration::from_secs(1)); + continue; + } + + // Handle incremental changes + loop { + match rx.recv_timeout(MRIB_PERIODIC_SYNC_INTERVAL) { + Ok(notification) => { + if let Err(e) = + handle_change(&mrib, notification, ddm, &log, &rt) + { + error!(log, "MRIB change handling failed: {e}"); + } + } + // if we've not received updates in the timeout interval, do a + // full sync in case something has changed out from under us. + Err(RecvTimeoutError::Timeout) => { + if let Err(e) = full_sync(&mrib, ddm, &log, &rt) { + error!(log, "MRIB periodic sync failed: {e}"); + info!(log, "restarting MRIB sync loop in one second"); + // Same backoff floor as the initial-sync failure, so a + // persistent failure retries at that pace rather than + // every resync interval. The unicast lower-half pauses + // in the same way (see `crate::run`). + sleep(Duration::from_secs(1)); + } + } + Err(RecvTimeoutError::Disconnected) => { + error!(log, "MRIB watcher disconnected"); + break; + } + } + } + } +} + +/// Perform a full synchronization of MRIB origination to DDM. +/// +/// Compares the current MRIB `loc_mrib` with what DDM has advertised and +/// reconciles any differences. +pub(crate) fn full_sync( + mrib: &Mrib, + ddm: &D, + log: &Logger, + rt: &Arc, +) -> Result<(), String> { + // Get current MRIB state (installed/selected routes) + let mrib_routes = mrib.loc_mrib(); + + // Convert to DDM MulticastOrigin set + let mrib_origins: HashSet = + mrib_routes.values().map(ddm_origin).collect(); + + // Get current DDM advertised state + let ddm_current: HashSet = rt + .block_on(async { ddm.get_originated_multicast_groups().await }) + .map_err(|e| format!("failed to get DDM multicast groups: {e}"))? + .into_inner() + .into_iter() + .collect(); + + // Compute diff + let to_add: Vec<_> = mrib_origins.difference(&ddm_current).collect(); + let to_remove: Vec<_> = ddm_current.difference(&mrib_origins).collect(); + + if !to_add.is_empty() { + info!( + log, + "MRIB sync: adding {} multicast groups to DDM", + to_add.len() + ); + add_multicast_routes(ddm, to_add.into_iter(), rt, log); + } + + if !to_remove.is_empty() { + info!( + log, + "MRIB sync: removing {} multicast groups from DDM", + to_remove.len() + ); + remove_multicast_routes(ddm, to_remove.into_iter(), rt, log); + } + + Ok(()) +} + +/// Handle an incremental MRIB change notification. +fn handle_change( + mrib: &Mrib, + notification: MribChangeNotification, + ddm: &D, + log: &Logger, + rt: &Arc, +) -> Result<(), String> { + // Get current DDM state for comparison + let ddm_current: HashSet = rt + .block_on(async { ddm.get_originated_multicast_groups().await }) + .map_err(|e| format!("failed to get DDM multicast groups: {e}"))? + .into_inner() + .into_iter() + .collect(); + + let mut to_add = Vec::new(); + let mut to_remove = Vec::new(); + + for key in notification.changed { + // Check if route exists in `loc_mrib` (installed) + if let Some(route) = mrib.get_selected_route(&key) { + let origin = ddm_origin(&route); + if !ddm_current.contains(&origin) { + to_add.push(origin); + } + } else { + // Route is not in `loc_mrib`, so we need to find matching DDM + // origin. We check all DDM origins to find any that match this key + for ddm_origin in &ddm_current { + // Reconstruct the key from the DDM origin to compare + if let Ok(overlay_group) = + MulticastAddr::try_from(ddm_origin.overlay_group.ip()) + && let Ok(ddm_key) = rdb::types::MulticastRouteKey::new( + ddm_origin.source, + overlay_group, + ddm_origin.vni, + ) + && ddm_key == key + { + to_remove.push(ddm_origin.clone()); + } + } + } + } + + if !to_add.is_empty() { + debug!(log, "MRIB change: adding {} multicast groups", to_add.len()); + add_multicast_routes(ddm, to_add.iter(), rt, log); + } + + if !to_remove.is_empty() { + debug!( + log, + "MRIB change: removing {} multicast groups", + to_remove.len() + ); + remove_multicast_routes(ddm, to_remove.iter(), rt, log); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::platform::test::TestDdm; + use rdb::test::get_test_db; + use rdb::types::{ + MulticastRouteKey, MulticastSourceProtocol, UnderlayMulticastIpv6, + }; + use std::net::Ipv6Addr; + + fn discard_logger() -> Logger { + Logger::root(slog::Discard, slog::o!()) + } + + /// Runtime handle for `full_sync`'s internal `block_on`. Tests run on a + /// plain thread (not a tokio worker), so blocking on this handle is safe. + fn runtime() -> (tokio::runtime::Runtime, Arc) { + let rt = tokio::runtime::Runtime::new().expect("build runtime"); + let handle = Arc::new(rt.handle().clone()); + (rt, handle) + } + + fn test_underlay() -> UnderlayMulticastIpv6 { + UnderlayMulticastIpv6::new(Ipv6Addr::new(0xff04, 0, 0, 0, 0, 0, 0, 1)) + .expect("valid underlay address") + } + + /// Build an any-source (*,G) static route for the given IPv4 group. + fn asm_route(a: u8, b: u8, c: u8, d: u8) -> MulticastRoute { + let group = MulticastAddr::new_v4(a, b, c, d).expect("valid group"); + let key = MulticastRouteKey::any_source(group); + MulticastRoute::new( + key, + test_underlay(), + MulticastSourceProtocol::Static, + ) + } + + /// Build an any-source (*,G) static route for an IPv6 group in + /// `ff0e::/16`, with a distinct underlay so the 1:1 overlay-to-underlay + /// mapping holds alongside `asm_route` entries. + fn asm_route_v6(last: u16) -> MulticastRoute { + let group = MulticastAddr::new_v6([0xff0e, 0, 0, 0, 0, 0, 0, last]) + .expect("valid group"); + let underlay = UnderlayMulticastIpv6::new(Ipv6Addr::new( + 0xff04, 0, 0, 0, 0, 0, 0x1, last, + )) + .expect("valid underlay address"); + let key = MulticastRouteKey::any_source(group); + MulticastRoute::new(key, underlay, MulticastSourceProtocol::Static) + } + + #[test] + fn full_sync_advertises_groups_missing_from_ddm() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_full_sync_add", log.clone()).expect("db"); + let route = asm_route(225, 1, 1, 1); + db.add_static_mcast_routes(std::slice::from_ref(&route)) + .expect("add route"); + + let ddm = TestDdm::default(); + full_sync(db.mrib(), &ddm, &log, &handle).expect("full sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(originated.len(), 1); + assert_eq!(originated[0], ddm_origin(&route)); + } + + #[test] + fn full_sync_advertises_v4_and_v6_overlay_groups() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_full_sync_v6", log.clone()).expect("db"); + let v4 = asm_route(225, 1, 1, 1); + let v6 = asm_route_v6(1); + db.add_static_mcast_routes(&[v4.clone(), v6.clone()]) + .expect("add routes"); + + let ddm = TestDdm::default(); + full_sync(db.mrib(), &ddm, &log, &handle).expect("full sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(originated.len(), 2); + assert!(originated.contains(&ddm_origin(&v4))); + assert!(originated.contains(&ddm_origin(&v6))); + } + + #[test] + fn full_sync_withdraws_groups_absent_from_mrib() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_full_sync_remove", log.clone()).expect("db"); + + let stale = asm_route(225, 9, 9, 9); + let ddm = TestDdm::default(); + ddm.multicast_originated + .lock() + .unwrap() + .push(ddm_origin(&stale)); + + full_sync(db.mrib(), &ddm, &log, &handle).expect("full sync"); + + assert!(ddm.multicast_originated.lock().unwrap().is_empty()); + } + + #[test] + fn full_sync_in_sync_makes_no_changes() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_full_sync_noop", log.clone()).expect("db"); + let route = asm_route(225, 5, 5, 5); + db.add_static_mcast_routes(std::slice::from_ref(&route)) + .expect("add route"); + + let ddm = TestDdm::default(); + ddm.multicast_originated + .lock() + .unwrap() + .push(ddm_origin(&route)); + + full_sync(db.mrib(), &ddm, &log, &handle).expect("full sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(originated.len(), 1, "in-sync group must not be re-added"); + assert_eq!(originated[0], ddm_origin(&route)); + } + + #[test] + fn full_sync_adds_and_withdraws_together() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_full_sync_mixed", log.clone()).expect("db"); + let keep = asm_route(225, 1, 1, 1); + db.add_static_mcast_routes(std::slice::from_ref(&keep)) + .expect("add route"); + + let ddm = TestDdm::default(); + let stale = asm_route(225, 2, 2, 2); + ddm.multicast_originated + .lock() + .unwrap() + .push(ddm_origin(&stale)); + + full_sync(db.mrib(), &ddm, &log, &handle).expect("full sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(originated.len(), 1); + assert_eq!(originated[0], ddm_origin(&keep)); + } + + #[test] + fn handle_change_advertises_newly_installed_group() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_change_add", log.clone()).expect("db"); + let route = asm_route(225, 1, 1, 1); + db.add_static_mcast_routes(std::slice::from_ref(&route)) + .expect("add route"); + + let ddm = TestDdm::default(); + let notification = MribChangeNotification::from(route.key); + handle_change(db.mrib(), notification, &ddm, &log, &handle) + .expect("handle change"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(originated.len(), 1); + assert_eq!(originated[0], ddm_origin(&route)); + } + + #[test] + fn handle_change_withdraws_removed_group() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_change_remove", log.clone()).expect("db"); + + // The group is advertised by DDM but never installed in the MRIB, + // modeling a route that was removed from `loc_mrib`. + let removed = asm_route(225, 9, 9, 9); + let ddm = TestDdm::default(); + ddm.multicast_originated + .lock() + .unwrap() + .push(ddm_origin(&removed)); + + let notification = MribChangeNotification::from(removed.key); + handle_change(db.mrib(), notification, &ddm, &log, &handle) + .expect("handle change"); + + assert!(ddm.multicast_originated.lock().unwrap().is_empty()); + } + + #[test] + fn handle_change_is_noop_when_already_advertised() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_change_noop", log.clone()).expect("db"); + let route = asm_route(225, 5, 5, 5); + db.add_static_mcast_routes(std::slice::from_ref(&route)) + .expect("add route"); + + let ddm = TestDdm::default(); + ddm.multicast_originated + .lock() + .unwrap() + .push(ddm_origin(&route)); + + let notification = MribChangeNotification::from(route.key); + handle_change(db.mrib(), notification, &ddm, &log, &handle) + .expect("handle change"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!( + originated.len(), + 1, + "already-advertised group must not be re-added" + ); + assert_eq!(originated[0], ddm_origin(&route)); + } + + #[test] + fn handle_change_withdraws_only_the_matching_group() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_change_selective", log.clone()).expect("db"); + + // DDM advertises two groups: only one is named in the change set and is + // absent from the MRIB, which means only that one must be withdrawn. + // This exercises the key-reconstruction match in the removal path. + let removed = asm_route(225, 1, 1, 1); + let other = asm_route(225, 2, 2, 2); + let ddm = TestDdm::default(); + { + let mut originated = ddm.multicast_originated.lock().unwrap(); + originated.push(ddm_origin(&removed)); + originated.push(ddm_origin(&other)); + } + + let notification = MribChangeNotification::from(removed.key); + handle_change(db.mrib(), notification, &ddm, &log, &handle) + .expect("handle change"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!( + originated.len(), + 1, + "unrelated group must remain advertised" + ); + assert_eq!(originated[0], ddm_origin(&other)); + } +} diff --git a/mg-lower/src/platform.rs b/mg-lower/src/platform.rs index 8350f245b..5115b457c 100644 --- a/mg-lower/src/platform.rs +++ b/mg-lower/src/platform.rs @@ -1,4 +1,4 @@ -//! This crate contains traits that decouple mg-lower from the underlying +//! This module contains traits that decouple mg-lower from the underlying //! platform. This is useful for testing mg-lower while not having to //! have a running dpd, ddmd, or switch zone. //! @@ -8,7 +8,7 @@ use std::net::IpAddr; use std::time::Duration; use ddm_admin_client::types::Error as DdmError; -use ddm_api_types_versions::latest::net::TunnelOrigin; +use ddm_api_types_versions::latest::net::{MulticastOrigin, TunnelOrigin}; use dpd_client::types::{Error as DpdError, *}; // Use the crate-specific re-exports of progenitor_client types to avoid // version conflicts: dpd_client re-exports from progenitor-client 0.11 (via @@ -217,6 +217,46 @@ pub trait Ddm { ddm_admin_client::ResponseValue<()>, ddm_admin_client::Error, >; + + /// Get multicast group subscriptions originated by this router. + /// + /// Each `MulticastOrigin` pairs an overlay group address with its + /// underlay mapping (ff04::/64) and optional source for (S,G) routes. + /// + /// Method names follow the DDM admin API convention + /// (`originated_multicast_groups`, not `originated_multicast_origins`). + async fn get_originated_multicast_groups( + &self, + ) -> Result< + ddm_admin_client::ResponseValue>, + ddm_admin_client::Error, + >; + + /// Advertise multicast group subscriptions to DDM peers. + /// + /// Each entry is a `MulticastOrigin` pairing an overlay group + /// with its ff04::/64 underlay mapping. + #[allow(clippy::ptr_arg)] + async fn advertise_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + >; + + /// Withdraw multicast group subscriptions from DDM peers. + /// + /// Each entry is a `MulticastOrigin` pairing an overlay group + /// with its ff04::/64 underlay mapping. + #[allow(clippy::ptr_arg)] + async fn withdraw_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + >; } /// This trait wraps the methods that have expectations about switch zone @@ -353,6 +393,7 @@ impl Dpd for ProductionDpd { /// Production ddm trait that simply passes through calls to a ddm client. #[cfg(target_os = "illumos")] +#[derive(Clone)] pub struct ProductionDdm { pub client: DdmClient, } @@ -406,6 +447,35 @@ impl Ddm for ProductionDdm { > { self.client.withdraw_tunnel_endpoints(body).await } + + async fn get_originated_multicast_groups( + &self, + ) -> Result< + ddm_admin_client::ResponseValue>, + ddm_admin_client::Error, + > { + self.client.get_originated_multicast_groups().await + } + + async fn advertise_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + self.client.advertise_multicast_groups(body).await + } + + async fn withdraw_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + self.client.withdraw_multicast_groups(body).await + } } /// Production switch zone that uses libnet for route lookups (illumos only). @@ -431,6 +501,7 @@ pub(crate) mod test { use crate::MG_LOWER_TAG; use super::*; + use mg_common::lock; use std::sync::Mutex; use std::{collections::HashMap, net::IpAddr}; @@ -485,7 +556,7 @@ pub(crate) mod test { link_id: &LinkId, ) -> Result, DpdClientError> { - let links = self.links.lock().unwrap(); + let links = lock!(self.links); let link = links .iter() .find(|x| &x.port_id == port_id && &x.link_id == link_id); @@ -503,10 +574,7 @@ pub(crate) mod test { dpd_client::ResponseValue>, DpdClientError, > { - let result = self - .v4_routes - .lock() - .unwrap() + let result = lock!(self.v4_routes) .get(cidr) .cloned() .unwrap_or(Vec::default()); @@ -520,10 +588,7 @@ pub(crate) mod test { dpd_client::ResponseValue>, DpdClientError, > { - let result = self - .v6_routes - .lock() - .unwrap() + let result = lock!(self.v6_routes) .get(cidr) .cloned() .unwrap_or(Vec::default()); @@ -535,7 +600,7 @@ pub(crate) mod test { addr: &Ipv6Entry, ) -> Result, DpdClientError> { - self.loopback.lock().unwrap().replace(addr.clone()); + lock!(self.loopback).replace(addr.clone()); Ok(dpd_response_ok!(())) } @@ -546,7 +611,7 @@ pub(crate) mod test { dpd_client::ResponseValue>, DpdClientError, > { - let links = self.links.lock().unwrap(); + let links = lock!(self.links); let result = links .iter() .filter(|x| match filter { @@ -613,7 +678,7 @@ pub(crate) mod test { RouteTarget::V4(v4) => Route::V4(v4.clone()), RouteTarget::V6(v6) => Route::V6(v6.clone()), }; - let mut routes = self.v4_routes.lock().unwrap(); + let mut routes = lock!(self.v4_routes); match routes.get_mut(&body.cidr) { Some(targets) => { targets.push(route); @@ -630,7 +695,7 @@ pub(crate) mod test { body: &'a Ipv6RouteUpdate, ) -> Result, DpdClientError> { - let mut routes = self.v6_routes.lock().unwrap(); + let mut routes = lock!(self.v6_routes); match routes.get_mut(&body.cidr) { Some(targets) => { targets.push(body.target.clone()); @@ -650,7 +715,7 @@ pub(crate) mod test { tgt_ip: &'a IpAddr, ) -> Result, DpdClientError> { - let mut routes = self.v4_routes.lock().unwrap(); + let mut routes = lock!(self.v4_routes); if let Some(targets) = routes.get_mut(cidr) { targets.retain(|x| match (x, tgt_ip) { (Route::V4(x), IpAddr::V4(ip)) => { @@ -678,7 +743,7 @@ pub(crate) mod test { tgt_ip: &'a std::net::Ipv6Addr, ) -> Result, DpdClientError> { - let mut routes = self.v6_routes.lock().unwrap(); + let mut routes = lock!(self.v6_routes); if let Some(targets) = routes.get_mut(cidr) { targets.retain(|x| { !(x.tgt_ip == *tgt_ip @@ -700,6 +765,7 @@ pub(crate) mod test { pub(crate) struct TestDdm { pub(crate) tunnel_originated: Mutex>, pub(crate) originated: Mutex>, + pub(crate) multicast_originated: Mutex>, } impl Default for TestDdm { @@ -707,6 +773,7 @@ pub(crate) mod test { Self { tunnel_originated: Mutex::new(Vec::default()), originated: Mutex::new(Vec::default()), + multicast_originated: Mutex::new(Vec::default()), } } } @@ -718,9 +785,7 @@ pub(crate) mod test { ddm_admin_client::ResponseValue>, ddm_admin_client::Error, > { - Ok(ddm_response_ok!( - self.tunnel_originated.lock().unwrap().clone() - )) + Ok(ddm_response_ok!(lock!(self.tunnel_originated).clone())) } async fn get_originated( @@ -729,7 +794,7 @@ pub(crate) mod test { ddm_admin_client::ResponseValue>, ddm_admin_client::Error, > { - Ok(ddm_response_ok!(self.originated.lock().unwrap().clone())) + Ok(ddm_response_ok!(lock!(self.originated).clone())) } async fn advertise_prefixes<'a>( @@ -739,7 +804,7 @@ pub(crate) mod test { ddm_admin_client::ResponseValue<()>, ddm_admin_client::Error, > { - self.originated.lock().unwrap().extend(body); + lock!(self.originated).extend(body); Ok(ddm_response_ok!(())) } @@ -750,7 +815,7 @@ pub(crate) mod test { ddm_admin_client::ResponseValue<()>, ddm_admin_client::Error, > { - self.tunnel_originated.lock().unwrap().extend(body.clone()); + lock!(self.tunnel_originated).extend(body.clone()); Ok(ddm_response_ok!(())) } @@ -761,10 +826,38 @@ pub(crate) mod test { ddm_admin_client::ResponseValue<()>, ddm_admin_client::Error, > { - self.tunnel_originated - .lock() - .unwrap() - .retain(|x| !body.contains(x)); + lock!(self.tunnel_originated).retain(|x| !body.contains(x)); + Ok(ddm_response_ok!(())) + } + + async fn get_originated_multicast_groups( + &self, + ) -> Result< + ddm_admin_client::ResponseValue>, + ddm_admin_client::Error, + > { + Ok(ddm_response_ok!(lock!(self.multicast_originated).clone())) + } + + async fn advertise_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + lock!(self.multicast_originated).extend(body.clone()); + Ok(ddm_response_ok!(())) + } + + async fn withdraw_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + lock!(self.multicast_originated).retain(|x| !body.contains(x)); Ok(ddm_response_ok!(())) } } diff --git a/mgd/src/main.rs b/mgd/src/main.rs index 571291c9d..39eec937b 100644 --- a/mgd/src/main.rs +++ b/mgd/src/main.rs @@ -114,6 +114,20 @@ struct RunArgs { /// SocketAddr for the BGP Dispatcher to listen on. #[arg(long, default_value = "[::]:179")] bgp_dispatcher_addr: SocketAddr, + + /// SocketAddr the Dendrite (DPD) API is listening on. When unset, the lower + /// half targets DPD at its default port on localhost (the co-located switch + /// zone). Set this to point the lower half at a DPD elsewhere, such as a + /// dynamically assigned port in an integration harness. + #[arg(long)] + dendrite_addr: Option, + + /// SocketAddr the DDM admin API is listening on. When unset, the lower half + /// targets DDM at its default localhost address (the co-located switch + /// zone). Set this to point the lower half at a DDM elsewhere, such as a + /// dynamically assigned port in an integration harness. + #[arg(long)] + ddm_addr: Option, } fn main() { @@ -167,24 +181,44 @@ async fn run(args: RunArgs) { #[cfg(all(feature = "mg-lower", target_os = "illumos"))] { - let rt = Arc::new(tokio::runtime::Handle::current()); - let ctx = context.clone(); - let log = log.clone(); - let db = ctx.db.clone(); - let stats = context.mg_lower_stats.clone(); let dpd = mg_lower::ProductionDpd { - client: mg_lower::new_dpd_client(&log), + client: mg_lower::new_dpd_client(&log, args.dendrite_addr), }; let ddm = mg_lower::ProductionDdm { - client: mg_lower::new_ddm_client(&log), + client: mg_lower::new_ddm_client(&log, args.ddm_addr), }; - let sw = mg_lower::ProductionSwitchZone {}; - Builder::new() - .name("mg-lower".to_string()) - .spawn(move || { - mg_lower::run(ctx.tep, db, log, stats, rt, &dpd, &ddm, &sw); - }) - .expect("failed to start mg-lower"); + + // Unicast lower-half: sync the unicast RIB to Dendrite and + // advertise tunnel endpoint routes to DDM. + { + let rt = Arc::new(tokio::runtime::Handle::current()); + let ctx = context.clone(); + let log = log.clone(); + let db = ctx.db.clone(); + let stats = context.mg_lower_stats.clone(); + let ddm = ddm.clone(); + let sw = mg_lower::ProductionSwitchZone {}; + Builder::new() + .name("mg-lower".to_string()) + .spawn(move || { + mg_lower::run(ctx.tep, db, log, stats, rt, &dpd, &ddm, &sw); + }) + .expect("failed to start mg-lower"); + } + + // Multicast lower-half: advertise locally originated MRIB groups to + // DDM. Underlay replication membership is reconciled in `ddmd`. + { + let rt = Arc::new(tokio::runtime::Handle::current()); + let log = log.clone(); + let mrib = context.db.mrib().clone(); + Builder::new() + .name("mg-lower-mrib".to_string()) + .spawn(move || { + mg_lower::mrib::run(mrib, log, rt, &ddm); + }) + .expect("failed to start mg-lower mrib sync"); + } } start_bgp_routers( diff --git a/openapi/ddm-admin/ddm-admin-2.0.0-45d40c.json.gitstub b/openapi/ddm-admin/ddm-admin-2.0.0-45d40c.json.gitstub new file mode 100644 index 000000000..b5ab91ce1 --- /dev/null +++ b/openapi/ddm-admin/ddm-admin-2.0.0-45d40c.json.gitstub @@ -0,0 +1 @@ +561931c41eafde867f931229e799d1b30df7a1d4:openapi/ddm-admin/ddm-admin-2.0.0-45d40c.json diff --git a/openapi/ddm-admin/ddm-admin-2.0.0-45d40c.json b/openapi/ddm-admin/ddm-admin-3.0.0-974538.json similarity index 68% rename from openapi/ddm-admin/ddm-admin-2.0.0-45d40c.json rename to openapi/ddm-admin/ddm-admin-3.0.0-974538.json index e891ca0e6..89d25b8db 100644 --- a/openapi/ddm-admin/ddm-admin-2.0.0-45d40c.json +++ b/openapi/ddm-admin/ddm-admin-3.0.0-974538.json @@ -6,7 +6,7 @@ "url": "https://oxide.computer", "email": "api@oxide.computer" }, - "version": "2.0.0" + "version": "3.0.0" }, "paths": { "/disable-stats": { @@ -51,6 +51,94 @@ } } }, + "/multicast_group": { + "put": { + "operationId": "advertise_multicast_groups", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Set_of_MulticastOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastOrigin" + }, + "uniqueItems": true + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "operationId": "withdraw_multicast_groups", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Set_of_MulticastOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastOrigin" + }, + "uniqueItems": true + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/multicast_groups": { + "get": { + "operationId": "get_multicast_groups", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Set_of_MulticastRoute", + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastRoute" + }, + "uniqueItems": true + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/originated": { "get": { "operationId": "get_originated", @@ -79,6 +167,34 @@ } } }, + "/originated_multicast_groups": { + "get": { + "operationId": "get_originated_multicast_groups", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Set_of_MulticastOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastOrigin" + }, + "uniqueItems": true + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/originated_tunnel_endpoints": { "get": { "operationId": "get_originated_tunnel_endpoints", @@ -463,6 +579,104 @@ "type": "string", "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$" }, + "MulticastOrigin": { + "description": "Origin information for a multicast group announcement.\n\nAnalogous to `TunnelOrigin` but for multicast groups. Represents a subscription to a multicast group that should be advertised via DDM. `overlay_group` is the application-visible multicast address (e.g., 233.252.0.1 or ff0e::1), while `underlay_group` is the mapped admin-local scoped IPv6 address (ff04::X) used in the underlay network.", + "type": "object", + "properties": { + "metric": { + "description": "Metric for path selection (lower is better).\n\nUsed for multi-rack replication optimization. Excluded from identity (Hash/Eq) so that metric changes update an existing entry rather than creating a duplicate.", + "default": 0, + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "overlay_group": { + "description": "The overlay multicast group address (IPv4 or IPv6). This is the group address visible to applications. Validated at construction to be a multicast address.", + "type": "string", + "format": "ip" + }, + "source": { + "nullable": true, + "description": "Optional source address for Source-Specific Multicast (S,G) routes. `None` for Any-Source Multicast (*,G) routes.", + "default": null, + "type": "string", + "format": "ip" + }, + "underlay_group": { + "description": "The underlay multicast group address (ff04::X). Validated at construction to be within ff04::/64.", + "type": "string", + "format": "ipv6" + }, + "vni": { + "description": "VNI for this multicast group (identifies the VPC/network context).", + "default": 77, + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "overlay_group", + "underlay_group" + ] + }, + "MulticastPathHop": { + "description": "A single hop in the multicast path, carrying metadata for replication optimization (RFD 488).", + "type": "object", + "properties": { + "downstream_subscriber_count": { + "description": "Number of downstream subscribers reachable via this hop.", + "default": 0, + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "router_id": { + "description": "Router identifier (hostname).", + "type": "string" + }, + "underlay_addr": { + "description": "The underlay address of this router (for replication targeting).", + "type": "string", + "format": "ipv6" + } + }, + "required": [ + "router_id", + "underlay_addr" + ] + }, + "MulticastRoute": { + "description": "A multicast route learned via DDM.\n\nCarries a `MulticastOrigin` (overlay group + ff04::/64 underlay mapping) and the path vector from the originating subscriber through intermediate transit routers.", + "type": "object", + "properties": { + "nexthop": { + "description": "Underlay nexthop address (DDM peer that advertised this route). Used to associate the route with a peer for expiration.", + "type": "string", + "format": "ipv6" + }, + "origin": { + "description": "The multicast group origin information.", + "allOf": [ + { + "$ref": "#/components/schemas/MulticastOrigin" + } + ] + }, + "path": { + "description": "Path vector from the originating subscriber outward. Each hop records the router that redistributed this subscription announcement. Used for loop detection on pull and for future replication optimization in multi-rack topologies.", + "default": [], + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastPathHop" + } + } + }, + "required": [ + "nexthop", + "origin" + ] + }, "PathVector": { "type": "object", "properties": { @@ -482,7 +696,7 @@ ] }, "PeerInfo": { - "description": "Information about a DDM peer.", + "description": "Peer information with an optional interface name.", "type": "object", "properties": { "addr": { @@ -492,6 +706,12 @@ "host": { "type": "string" }, + "if_name": { + "nullable": true, + "description": "Interface name the peer was discovered on (e.g., \"tfportrear0_0\").", + "default": null, + "type": "string" + }, "kind": { "$ref": "#/components/schemas/RouterKind" }, diff --git a/openapi/ddm-admin/ddm-admin-latest.json b/openapi/ddm-admin/ddm-admin-latest.json index 4eb6e8dbc..ca00e9746 120000 --- a/openapi/ddm-admin/ddm-admin-latest.json +++ b/openapi/ddm-admin/ddm-admin-latest.json @@ -1 +1 @@ -ddm-admin-2.0.0-45d40c.json \ No newline at end of file +ddm-admin-3.0.0-974538.json \ No newline at end of file diff --git a/smf/ddm/manifest.xml b/smf/ddm/manifest.xml index 5e5da1cf2..f9c7f7425 100644 --- a/smf/ddm/manifest.xml +++ b/smf/ddm/manifest.xml @@ -19,7 +19,7 @@ - + diff --git a/tests/src/ddm.rs b/tests/src/ddm.rs index 988ed973d..c5b13404f 100644 --- a/tests/src/ddm.rs +++ b/tests/src/ddm.rs @@ -5,7 +5,9 @@ use anyhow::{Result, anyhow}; use client_common::{eprintln_nopipe, println_nopipe}; use ddm_admin_client::Client; -use ddm_api_types_versions::latest::net::TunnelOrigin; +use ddm_api_types_versions::latest::net::{ + MulticastOrigin, TunnelOrigin, UnderlayMulticastIpv6, Vni, +}; use slog::{Drain, Logger}; use std::env; use std::net::Ipv6Addr; @@ -190,6 +192,32 @@ impl<'a> RouterZone<'a> { self.zone.zexec("pkill ddmd") } + /// Wait for an SMF service in this zone to come online, failing fast if + /// it lands in maintenance. A service that silently fails here otherwise + /// surfaces much later as an opaque peering assertion failure. + fn wait_for_service_online(&self, fmri: &str) -> Result<()> { + for _ in 0..30 { + let state = self + .zone + .zexec(&format!("svcs -Ho state {fmri}")) + .unwrap_or_default(); + match state.trim() { + "online" => return Ok(()), + "maintenance" => { + return Err(anyhow!( + "{fmri} entered maintenance in zone {}", + self.zone.name, + )); + } + _ => sleep(Duration::from_secs(1)), + } + } + Err(anyhow!( + "timed out waiting for {fmri} to come online in zone {}", + self.zone.name, + )) + } + fn start_router(&self, restart_dpd: bool) -> Result<()> { let addrs = self.ifx[1..] .iter() @@ -224,9 +252,8 @@ impl<'a> RouterZone<'a> { ))?; self.zone.zexec("svcadm refresh dendrite:default")?; self.zone.zexec("svcadm enable dendrite:default")?; - // wait for dendrite to come up - println_nopipe!("wait 10s for dendrite to come up ..."); - sleep(Duration::from_secs(10)); + println_nopipe!("waiting for dendrite to come online ..."); + self.wait_for_service_online("dendrite:default")?; self.zone.zexec( "svccfg -s tfport setprop config/pkt_source = none", )?; @@ -235,6 +262,8 @@ impl<'a> RouterZone<'a> { )?; self.zone.zexec("svcadm refresh tfport:default")?; self.zone.zexec("svcadm enable tfport")?; + println_nopipe!("waiting for tfport to come online ..."); + self.wait_for_service_online("tfport:default")?; } self.zone.zexec(&format!( "{} {ddm} --kind transit --dendrite {} {} &> /opt/ddmd.log &", @@ -648,6 +677,69 @@ async fn run_trio_tests( println_nopipe!("tunnel endpoint withdraw passed"); + // Multicast group advertise/withdraw across the trio. Mirrors how + // mg-lower in the switch zone publishes overlay→underlay multicast + // bindings: the transit router originates an advertisement, and the + // server routers learn it via DDM exchange. + wait_for_eq!(multicast_originated_count(&t1).await?, 0); + + // One origin per overlay address family: the overlay group is opaque to + // ddm (any multicast IpAddr), but both families must survive the + // advertise/exchange/withdraw round trip. + let mcast_origin_v4 = MulticastOrigin { + overlay_group: "233.252.0.1".parse().unwrap(), + underlay_group: UnderlayMulticastIpv6::new( + "ff04::100".parse().unwrap(), + ) + .unwrap(), + vni: Vni::try_from(77u32).unwrap(), + source: None, + metric: 0, + }; + let mcast_origin_v6 = MulticastOrigin { + overlay_group: "ff0e::1".parse().unwrap(), + underlay_group: UnderlayMulticastIpv6::new( + "ff04::101".parse().unwrap(), + ) + .unwrap(), + vni: Vni::try_from(77u32).unwrap(), + source: None, + metric: 0, + }; + + t1.advertise_multicast_groups(&vec![ + mcast_origin_v4.clone(), + mcast_origin_v6.clone(), + ]) + .await?; + + wait_for_eq!(multicast_originated_count(&t1).await?, 2); + wait_for_eq!(multicast_group_count(&t1).await?, 0); + wait_for_eq!(multicast_group_count(&s1).await?, 2); + wait_for_eq!(multicast_group_count(&s2).await?, 2); + + println_nopipe!("multicast group advertise passed"); + + // Server router restart: s1's view of the multicast group must + // converge again after ddmd restarts. wait_for_eq tolerates the + // restart window via unwrap_or sentinel. + zs1.stop_router()?; + zs1.start_router(false)?; + let s1 = Client::new("http://10.0.0.1:8000", log.clone()); + wait_for_eq!(multicast_group_count(&s1).await.unwrap_or(99), 2); + + println_nopipe!("multicast router restart passed"); + + t1.withdraw_multicast_groups(&vec![mcast_origin_v4, mcast_origin_v6]) + .await?; + + wait_for_eq!(multicast_originated_count(&t1).await?, 0); + wait_for_eq!(multicast_group_count(&t1).await?, 0); + wait_for_eq!(multicast_group_count(&s1).await?, 0); + wait_for_eq!(multicast_group_count(&s2).await?, 0); + + println_nopipe!("multicast group withdraw passed"); + Ok(()) } @@ -818,6 +910,14 @@ async fn tunnel_originated_endpoint_count(c: &Client) -> Result { Ok(c.get_originated_tunnel_endpoints().await?.len()) } +async fn multicast_group_count(c: &Client) -> Result { + Ok(c.get_multicast_groups().await?.len()) +} + +async fn multicast_originated_count(c: &Client) -> Result { + Ok(c.get_originated_multicast_groups().await?.len()) +} + fn init_logger() -> Logger { let decorator = slog_term::TermDecorator::new().build(); let drain = slog_term::FullFormat::new(decorator).build().fuse();