diff --git a/crates/opte-api/src/ip.rs b/crates/opte-api/src/ip.rs index 9db9bea1..db2ecc4f 100644 --- a/crates/opte-api/src/ip.rs +++ b/crates/opte-api/src/ip.rs @@ -835,14 +835,25 @@ impl Ipv6Addr { } } - /// Return `true` if this is a multicast IPv6 address with the ff04::/16 prefix - /// (admin-local scope with flags=0) as used by Omicron for underlay multicast. + /// Return `true` if this is a multicast IPv6 address with the ff04::/16 + /// prefix (admin-local scope with flags=0) as used by Omicron for underlay + /// multicast. /// /// This specifically checks for the ff04::/16 prefix where: /// - First byte: 0xFF (all multicast addresses) /// - Second byte: 0x04 (flags=0, scope=4 admin-local) /// - /// See [RFC 7346] for details on IPv6 multicast address scopes. + /// See [RFC 7346] for details on IPv6 multicast address scopes. Routers + /// must not forward a packet beyond the scope its destination names + /// ([RFC 4291 §2.7]). Admin-local keeps rack traffic in the rack, + /// link-local is dropped at the first hop, and global leaks past the + /// rack. + /// + /// Requiring flags=0 follows Omicron's allocation rather than the RFCs. + /// T=0 denotes an IANA-assigned well-known group ([RFC 4291 §2.7]), and + /// [RFC 3307 §4] requires dynamically allocated groups to set T=1. These + /// groups are operator-allocated, so ff14:: would be the conformant + /// choice. Transient groups are rejected here regardless. /// /// Omicron allocates multicast addresses from a /64 subnet within /// ff04::/16, and the narrower /64 constraint is enforced upstream @@ -851,6 +862,8 @@ impl Ipv6Addr { /// for correct packet handling at this layer. /// /// [RFC 7346]: https://www.rfc-editor.org/rfc/rfc7346.html + /// [RFC 4291 §2.7]: https://www.rfc-editor.org/rfc/rfc4291#section-2.7 + /// [RFC 3307 §4]: https://www.rfc-editor.org/rfc/rfc3307#section-4 pub const fn is_admin_scoped_multicast(&self) -> bool { if !self.is_multicast() { return false; @@ -1082,13 +1095,11 @@ impl MulticastUnderlay { /// Create a new `MulticastUnderlay` without validation. /// - /// Safety: The caller must ensure that `addr` is an admin-scoped IPv6 - /// multicast address (ff04::/16). Using this with an invalid address - /// violates the type's invariant and may lead to undefined behavior. + /// Callers of this fn must still uphold the type's invariant by supplying + /// an admin-scoped multicast address (ff04::/16). So, no validation here. /// - /// This is intended for cases where validation has already been performed - /// (e.g., after an explicit `is_admin_scoped_multicast()` check) to avoid - /// redundant validation overhead. + /// On the packet path, the address is read directly from the wire, and + /// the forwarding and subscription table lookups don't recheck it. #[inline] pub const fn new_unchecked(addr: Ipv6Addr) -> Self { Self(addr) @@ -1837,7 +1848,7 @@ mod test { assert!(to_ipv6("ff04::1").is_admin_scoped_multicast()); assert!(to_ipv6("ff04:1234:5678:9abc::1").is_admin_scoped_multicast()); - // Test other administrative scopes (NOT accepted) + // Test other administrative scopes (not accepted) assert!(!to_ipv6("ff05::1").is_admin_scoped_multicast()); // site-local assert!(!to_ipv6("ff08::1").is_admin_scoped_multicast()); // organization-local diff --git a/lib/oxide-vpc/src/api.rs b/lib/oxide-vpc/src/api.rs index 2d4c419e..f44ed2ae 100644 --- a/lib/oxide-vpc/src/api.rs +++ b/lib/oxide-vpc/src/api.rs @@ -94,14 +94,22 @@ pub const BOUNDARY_SERVICES_VNI: u32 = 99u32; /// Default VNI for rack-wide multicast groups (no VPC association). /// Must match Omicron's DEFAULT_MULTICAST_VNI. /// -/// This is the only VNI currently supported for multicast traffic. -/// All multicast groups (M2P mappings and forwarding entries) must use this VNI. -/// OPTE validates that multicast operations specify this VNI and rejects others. +/// This is the only VNI currently supported for multicast. OPTE rejects +/// multicast operations (M2P mappings and forwarding entries) that specify +/// any other VNI. /// -/// While M2P (Multicast-to-Physical) mappings are stored -/// per-VNI in the code, the enforcement of DEFAULT_MULTICAST_VNI means all -/// multicast traffic shares a single namespace across the rack, with no -/// VPC-level isolation (as multicast groups are fleet-wide) *as of now*. +/// M2P (Multicast-to-Physical) mappings are keyed by multicast group, not +/// VNI. All multicast traffic currently shares one rack-wide namespace; no +/// VPC isolation. +/// +/// On the inbound path, the overlay layer's `MulticastVniValidator` (see +/// [`overlay`]) accepts a multicast packet whose Geneve VNI is either this +/// value or the receiving port's own VPC VNI, and denies otherwise. +/// [`EncapAction`] stamps this VNI on every outbound multicast packet, so +/// nothing today actually hits the per-VPC branch. +/// +/// [`overlay`]: crate::engine::overlay +/// [`EncapAction`]: crate::engine::overlay::EncapAction pub const DEFAULT_MULTICAST_VNI: u32 = 77u32; /// Description of Boundary Services, the endpoint used to route traffic diff --git a/lib/oxide-vpc/src/engine/overlay.rs b/lib/oxide-vpc/src/engine/overlay.rs index a68d2a40..4daf1456 100644 --- a/lib/oxide-vpc/src/engine/overlay.rs +++ b/lib/oxide-vpc/src/engine/overlay.rs @@ -6,7 +6,90 @@ //! The Oxide Network VPC Overlay. //! -//! This implements the Oxide Network VPC Overlay. +//! # Multicast composition +//! +//! Multicast state is split across two stores on the sled and a third on the +//! switches. The split is the same at both layers: one store holds identity +//! ("what is this group") and the other membership ("which members are in it"). +//! +//! ```text +//! identity membership +//! "what is this group" "which members are in it" +//! ------------------------ ------------------------- +//! sled (OPTE) M2P subscription table +//! 233.252.0.1 ff04::e9fc:1 +//! -> ff04::e9fc:1 -> {port, source filter} +//! +//! switch (DPD) external entry replication list +//! 233.252.0.1 ff04::e9fc:1 +//! -> ff04::e9fc:1 -> {member sled rear ports} +//! ``` +//! +//! The examples above follow the control plane's allocation setup, +//! assigning underlay groups from a /64 inside `ff04::/16` and embedding the +//! IPv4 group in the low 32 bits, so that `233.252.0.1` maps to `ff04::e9fc:1`. +//! Maghemite and dendrite enforce the /64 as well. OPTE, on the other hand, +//! validates only the `ff04::/16` scope and treats the underlay group as opaque +//! otherwise. +//! +//! ## Identity: the M2P table +//! +//! [`Mcast2Phys`] holds the group's identity translation, mapping an +//! external group address to the admin-scoped underlay group that carries +//! it (1:1). There is one entry per group per sled. The table knows nothing +//! about members or ports. +//! +//! V2P resolves to one sled's underlay address. M2P resolves to an underlay +//! group that the switches fan out to many sleds. `Phys` denotes the underlay +//! in both cases, hence the naming. Same layer, but different cardinality. +//! +//! ## Membership: the subscription table +//! +//! Per-port membership lives in a separate subscription table, populated by +//! `McastSubscribeReq { port_name, group, filter }`. Each member carries its +//! own source filter, where [`SourceFilter`] defaults to excluding nothing +//! and therefore accepts any source. `Include` sources get validated as +//! usable (S,G) sources. `Exclude` sets aren't checked due to the possibility +//! of an operator wanting to block an address that could never be a real +//! source anyway. The Rx delivery set is computed from these subscriptions at +//! delivery time, and is not stored as a third, separate table. +//! +//! ## Underlay replication lists +//! +//! The switches hold the remaining store, i.e., the underlay +//! replication lists that map an underlay group to the rear ports of the member +//! sleds. Those lists are driven by DDM exchange, and OPTE never sees them. +//! +//! ## Where does the state come from? +//! +//! Nexus owns the intent. The sled agent programs it over the ioctl surface and +//! attempts to reconcile state when the sled drifts. +//! +//! ## Ingress and egress +//! +//! The two directions rely on different state. Guest-originated (Tx) traffic +//! reaches [`EncapAction`], which resolves the destination through M2P per +//! flow and denies the send when no such mapping exists. The deny binds at +//! flow establishment. Removing a mapping stops new flows from establishing, +//! while established ones keep encapsulating until they age out. +//! +//! External-ingress delivery never reads the M2P store per packet: XDE +//! decaps the packet and fans a copy out to each subscribed port, applying +//! that member's source filter. +//! +//! Ingress still uses M2P, just earlier: the subscribe ioctl translates the +//! overlay group a port joins into the underlay key it listens on. The table +//! serves egress in the data path and ingress at subscribe time. +//! +//! ```text +//! Tx guest -> gateway -> EncapAction --M2P--> ff04::e9fc:1 -> switch (PRE) +//! deny when unmapped +//! +//! Rx ff04::e9fc:1 -> XDE decap -> subscriptions + source filter -> ports +//! M2P used at subscribe time (not here) +//! ``` +//! +//! [`SourceFilter`]: crate::api::SourceFilter use super::geneve::OxideOptions; use super::router::RouterTargetInternal; use crate::api::DEFAULT_MULTICAST_VNI; @@ -245,6 +328,13 @@ impl StaticAction for EncapAction { // destination IP is a multicast address. Multicast operates at the fleet // level (cross-VPC) and doesn't go through VPC routing, so router // metadata is not required in that case. + // + // This is the guest-egress decision point for multicast. Encapsulation + // runs only on outbound (Tx) traffic originated by a guest, so the M2P + // lookup below decides which overlay groups a guest may send to. + // External-ingress delivery takes the opposite path: decapsulated + // copies of an underlay group are fanned out to subscribed ports by + // XDE, and never touch this mapping. let is_mcast_addr = dst_ip.is_multicast(); let (is_internal, phys_target, is_mcast) = if is_mcast_addr { @@ -262,7 +352,8 @@ impl StaticAction for EncapAction { true, ), None => { - // No M2P mapping configured for this multicast group; deny. + // No M2P mapping configured for this multicast group, so + // deny the guest-originated send. return Ok(AllowOrDeny::Deny); } } @@ -624,8 +715,9 @@ impl StaticAction for DecapAction { /// ## Validation Policy on Rx Path /// This validator accepts multicast packets with either of two VNI values: /// - **VNI 77 (DEFAULT_MULTICAST_VNI)**: Fleet-wide multicast, accepted by all -/// ports regardless of VPC. This enables rack-wide multicast delivery. -/// - **Guest's VPC VNI**: Enables per-VPC multicast isolation **in the future**. +/// ports regardless of VPC, giving rack-wide delivery. +/// - **Guest's VPC VNI**: reserved for per-VPC multicast isolation, which no +/// current encapsulation exercises. /// /// The validator enforces VPC isolation by rejecting multicast packets with /// VNI values that don't match either the fleet-wide VNI or this port's VPC. @@ -1029,7 +1121,7 @@ impl MappingResource for Virt2Phys { } impl Mcast2Phys { - /// Create a new empty multicast-to-physical mapping table. + /// Create a new empty multicast-to-underlay mapping table. pub fn new() -> Self { Self { ip4: KMutex::new(BTreeMap::new()), diff --git a/xde/src/dev_map.rs b/xde/src/dev_map.rs index e9bfcccc..be3bb102 100644 --- a/xde/src/dev_map.rs +++ b/xde/src/dev_map.rs @@ -64,12 +64,15 @@ pub struct DevMap { names: BTreeMap, /// Subscriptions keyed by underlay IPv6 multicast group (admin-scoped ff04::/16). /// Each port has its own source filter for per-member filtering. - /// This table is sled-local and independent of any per-VPC VNI. VNI validation - /// and VPC isolation are enforced during inbound overlay decapsulation on the - /// destination port, not here. /// - /// Rationale: multicast groups are fleet-wide; ports opt-in to receive a given - /// underlay group, and the overlay layer subsequently filters by VNI as appropriate. + /// The outer group key carries no VNI, as all multicast traffic currently + /// uses `oxide_vpc::api::DEFAULT_MULTICAST_VNI`, so a VNI component would + /// be constant. The per-member `VniMac` keys still carry each port's VPC + /// VNI. Both outbound encapsulation and the forwarding ioctls require + /// that value. Delivery is gated by the subscription itself and by the + /// per-member source filter. There is no VPC-level isolation for + /// multicast right now, as the inbound overlay layer's VNI validator + /// accepts either the fleet VNI or the port's own VPC VNI. mcast_groups: BTreeMap>, } @@ -221,7 +224,7 @@ impl DevMap { /// /// Safety: Callers must hold a read lock on this `DevMap` for the duration /// of delivery. This prevents port removal from tearing down DLS/MAC - /// resources while delivery is in progress—management operations attempting + /// resources while delivery is in progress. Management operations attempting /// to remove a port will block when trying to acquire the write lock to /// update the map. #[inline] diff --git a/xde/src/xde.rs b/xde/src/xde.rs index 3b51af9a..1e14cfd7 100644 --- a/xde/src/xde.rs +++ b/xde/src/xde.rs @@ -126,6 +126,17 @@ //! is kept per-port, updated by `refresh_maps()` whenever the canonical //! forwarding table changes. //! +//! ### Multicast directions +//! The multicast state stores, and the identity/membership split between +//! them, are described in detail in the [`overlay`] module docs. XDE implements +//! both directions over that state. Rx delivery, described above, is the +//! external-ingress half: a decapsulated packet addressed to an admin-scoped +//! underlay group is fanned out to the ports on this sled that subscribed to +//! it, and each copy goes through that member's source filter. For the other +//! direction, the per-port `mcast_fwd` state and Tx fan-out serve +//! guest-originated sends, which resolve the group through the M2P table +//! during encapsulation, before XDE's multicast forwarding stage. +//! //! ### [`TokenLock`] and [`DevMap`] updates //! The `TokenLock` primitive provides us with logical mutual exclusion around //! the underlay and the ability to modify the canonical [`DevMap`] -- without @@ -148,6 +159,8 @@ //! the final `refresh_maps()` calls during port deletion). The management lock //! ensures no concurrent modifications, allowing underlay port Arcs to be //! safely unwrapped. +//! +//! [`overlay`]: oxide_vpc::engine::overlay use crate::dev_map::DevMap; use crate::dev_map::ReadOnlyDevMap; @@ -2451,7 +2464,7 @@ fn select_nexthops( /// replication based on the XDE-wide multicast forwarding table. /// /// Always delivers to local same-sled subscribers regardless of replication mode. -/// Routes to next hop unicast addresses for ALL replication modes to determine +/// Routes to next hop unicast addresses for every replication mode to determine /// reachability and underlay port/MAC. Packet destination is always the multicast /// address with multicast MAC. The [`Replication`] type is a Tx-only instruction /// telling the switch which port groups to replicate to: External (front panel), @@ -2477,11 +2490,18 @@ fn handle_mcast_tx<'a>( + usize::from(ctx.tun_meoi.meoi_l3hlen) + usize::from(ctx.tun_meoi.meoi_l4hlen); - // Local same-sled delivery: always deliver to subscribers on this sled, - // independent of the Tx-only Replication instruction (not an access control mechanism). + // Local same-sled delivery on the guest-egress path: the sending guest + // has already gotten past the M2P lookup in the overlay layer, so this + // loop only decides which other local ports can receive a copy. + // + // The external-ingress fan-out is separate; `handle_mcast_rx` handles + // copies arriving from the underlay. + // + // We always deliver to subscribers on this sled, independent of the Tx-only + // Replication instruction (not an access control mechanism). // // The Replication type only affects how switches handle the packet on Tx. - // Subscription is keyed by underlay (outer) IPv6 multicast address. + // Subscription is keyed by the underlay (outer) IPv6 multicast address. let underlay_addr = oxide_vpc::api::Ipv6Addr::from(ctx.underlay_dst.bytes()); let group_key = MulticastUnderlay::new_unchecked(underlay_addr); @@ -2755,12 +2775,23 @@ fn handle_mcast_tx<'a>( /// Handle multicast packet reception from the underlay. /// -/// OPTE is always a leaf node in the multicast replication tree. -/// This function only delivers packets to local subscribers. +/// This is the external-ingress half of multicast. A packet addressed to an +/// admin-scoped underlay group arrives already decapsulated by the caller, +/// and we fan it out to the ports on this sled that are subscribed to that +/// group. Each copy must go through that member's source filter. +/// +/// Note: OPTE is always a leaf node in the multicast replication tree; no +/// further replication toward the underlay (or across transit, or anything +/// similar) can happen here. +/// +/// There is no per-packet M2P lookup here. Subscriptions are keyed +/// by the underlay group, and (by this point) the subscribe ioctl has already +/// translated the overlay group through M2P. The guest-egress half, where M2P +/// decides which overlay groups a guest may send to, lives in the overlay +/// layer's encap action. /// /// The Replication type is Tx-only (instructions to the switch), so the -/// replication field is ignored on Rx. Local delivery is based purely on -/// subscriptions. +/// replication field is ignored on Rx. fn handle_mcast_rx( ctx: MulticastRxContext, stream: &DlsStream,