falcon-lab: interact with nodes concurrently

falcon-lab/README.md

@@ -153,13 +153,20 @@ base-image dataset destroyed/replaced so the new image is used.
 
 ## Running with diagnostics disabled
 
-Failure diagnostics are enabled by default. For faster local iteration while
-preserving the failed topology, use:
+Failure diagnostics are enabled by default. Before collecting them, falcon-lab
+attempts to start FRR and unpause cEOS and cRPD so their normal CLI and API
+paths are available. `--no-cleanup` preserves the topology after the run, but
+does not prevent this diagnostic recovery.
+
+To preserve the exact failure state for manual inspection, including peers
+that the scenario left paused or stopped, disable diagnostics as well as
+cleanup:
 
 ```sh
 pfexec target/release/falcon-lab run interop bfd-static-routing \
   --no-cleanup --no-diag-on-fail
 ```
 
-Even with `--no-diag-on-fail`, interop scenarios make a best-effort attempt to
-restart FRR and unpause cEOS/cRPD before returning the failure.
+Use this combination when automated recovery would disturb the state being
+investigated, such as when inspecting one node while its peer remains paused.
+Without `--no-cleanup`, disabling diagnostics does not preserve the topology.

falcon-lab/src/eos.rs

@@ -128,35 +128,36 @@ impl EosNode {
         Ok(())
     }
 
-    /// Query ceos for the local status of a BFD session to `peer`. Returns
-    /// `true` iff EOS reports any per-interface peerStats entry under this
-    /// peer with status `up`. The nested shape is:
+    /// Query ceos for the local status of multiple BFD sessions. Returns
+    /// `true` iff EOS reports every requested session as `up`. The nested
+    /// shape is:
     ///   vrfs.<vrf>.ipv4Neighbors.<peer>.peers.<iface>.types.normal.peerStats.<local>.status
-    pub async fn bfd_peer_up(&self, d: &Runner, peer: IpAddr) -> Result<bool> {
+    pub async fn bfd_peers_up(
+        &self,
+        d: &Runner,
+        wanted: &[IpAddr],
+    ) -> Result<bool> {
         let output = self.shell(d, "show bfd peers | json").await?;
         let resp: EosBfdResponse = serde_json::from_str(&output)
             .context("parse eos bfd peers json")?;
-        let key = peer.to_string();
-        for vrf in resp.vrfs.values() {
-            let neighbors = match peer {
-                IpAddr::V4(_) => &vrf.ipv4_neighbors,
-                IpAddr::V6(_) => &vrf.ipv6_neighbors,
-            };
-            let Some(neighbor) = neighbors.get(&key) else {
-                continue;
-            };
-            for if_peer in neighbor.peers.values() {
-                let Some(normal) = if_peer.types.normal.as_ref() else {
-                    continue;
+        Ok(wanted.iter().all(|wanted| {
+            let key = wanted.to_string();
+            resp.vrfs.values().any(|vrf| {
+                let neighbors = match wanted {
+                    IpAddr::V4(_) => &vrf.ipv4_neighbors,
+                    IpAddr::V6(_) => &vrf.ipv6_neighbors,
                 };
-                for stats in normal.peer_stats.values() {
-                    if stats.status.eq_ignore_ascii_case("up") {
-                        return Ok(true);
-                    }
-                }
-            }
-        }
-        Ok(false)
+                neighbors.get(&key).is_some_and(|neighbor| {
+                    neighbor.peers.values().any(|peer| {
+                        peer.types.normal.as_ref().is_some_and(|normal| {
+                            normal.peer_stats.values().any(|stats| {
+                                stats.status.eq_ignore_ascii_case("up")
+                            })
+                        })
+                    })
+                })
+            })
+        }))
     }
 
     /// Get BGP IPv4 imported prefixes from EOS.

falcon-lab/src/frr.rs

@@ -64,7 +64,10 @@ impl FrrNode {
 
     pub async fn start_frr(&self, d: &Runner) -> Result<()> {
         info!(d.log, "{}: starting frr", self.name(d));
-        d.exec(self.0, "systemctl start frr").await?;
+        // Scenarios intentionally cycle FRR often enough to exhaust systemd's
+        // start limit before its interval resets.
+        d.exec(self.0, "systemctl reset-failed frr && systemctl start frr")
+            .await?;
         // XXX do better than arbitrary wait
         sleep(Duration::from_secs(5)).await;
         Ok(())
@@ -101,15 +104,21 @@ impl FrrNode {
         Ok(())
     }
 
-    /// Query FRR for the local status of a BFD session to `peer`. Returns
-    /// `true` iff bfdd reports the session as `up`.
-    pub async fn bfd_peer_up(&self, d: &Runner, peer: IpAddr) -> Result<bool> {
+    /// Query FRR for the local status of multiple BFD sessions. Returns `true`
+    /// iff bfdd reports every requested session as `up`.
+    pub async fn bfd_peers_up(
+        &self,
+        d: &Runner,
+        wanted: &[IpAddr],
+    ) -> Result<bool> {
         let output = self.shell(d, "show bfd peers json").await?;
         let peers: Vec<FrrBfdPeer> = serde_json::from_str(&output)
             .context("parse frr bfd peers json")?;
-        Ok(peers
-            .iter()
-            .any(|p| p.peer == peer && p.status.eq_ignore_ascii_case("up")))
+        Ok(wanted.iter().all(|wanted| {
+            peers.iter().any(|peer| {
+                peer.peer == *wanted && peer.status.eq_ignore_ascii_case("up")
+            })
+        }))
     }
 
     /// Capture protocol-specific FRR state via vtysh, plus Linux network state.

falcon-lab/src/juniper.rs

@@ -231,18 +231,25 @@ impl JuniperNode {
         Ok(output.contains(prefix) && output.contains("BGP"))
     }
 
-    /// Query cRPD for the local status of a BFD session to `peer`. Returns
-    /// true iff Junos reports the session as `Up`.
-    pub async fn bfd_peer_up(&self, d: &Runner, peer: IpAddr) -> Result<bool> {
+    /// Query cRPD for the local status of multiple BFD sessions. Returns true
+    /// iff Junos reports every requested session as `Up`.
+    pub async fn bfd_peers_up(
+        &self,
+        d: &Runner,
+        wanted: &[IpAddr],
+    ) -> Result<bool> {
         let output = self.shell(d, "show bfd session | display json").await?;
         let resp: JunosBfdResponse = serde_json::from_str(&output)
             .context("parse juniper bfd session json")?;
-        let peer = peer.to_string();
-        Ok(resp
+        let sessions = resp
             .bfd_session_information
-            .into_iter()
-            .flat_map(|info| info.bfd_session)
-            .any(|session| session.is_up_for(&peer)))
+            .iter()
+            .flat_map(|info| &info.bfd_session)
+            .collect::<Vec<_>>();
+        Ok(wanted.iter().all(|wanted| {
+            let wanted = wanted.to_string();
+            sessions.iter().any(|session| session.is_up_for(&wanted))
+        }))
     }
 
     /// Capture non-secret Juniper diagnostics. Do not add `show configuration`