fix(vmm): expose failed starts through boot_error

dstack/vmm/src/app.rs

@@ -492,7 +492,18 @@ impl App {
     }
 
     pub async fn start_vm(&self, id: &str) -> Result<()> {
-        self.start_vm_with_restart_policy(id, true).await
+        let result = self.start_vm_with_restart_policy(id, true).await;
+        if let Err(err) = &result {
+            self.set_boot_error(id, err);
+        }
+        result
+    }
+
+    fn set_boot_error(&self, id: &str, err: &anyhow::Error) {
+        if let Some(vm) = self.lock().get_mut(id) {
+            vm.state.boot_error = ra_rpc::log_text(&format!("{err:#}"));
+            vm.state.boot_progress = "failed".into();
+        }
     }
 
     async fn start_vm_with_restart_policy(
@@ -767,8 +778,8 @@ impl App {
             let queues = network.queue_pairs();
             let filtered = filters_bridge_traffic(network, &self.config.cvm);
             let netd = netd::client(&self.config.netd.socket);
-            let result = match network.nic.mode {
-                NetworkingMode::Bridge => {
+            let (result, method) = match network.nic.mode {
+                NetworkingMode::Bridge => (
                     netd.prepare_bridge(PrepareBridgeRequest {
                         identity: Some(identity.clone()),
                         bridge: network.nic.bridge.clone(),
@@ -782,9 +793,10 @@ impl App {
                         queues,
                         workdir: workdir.clone(),
                     })
-                    .await
-                }
-                NetworkingMode::Macvtap => {
+                    .await,
+                    "PrepareBridge",
+                ),
+                NetworkingMode::Macvtap => (
                     netd.prepare_macvtap(PrepareMacvtapRequest {
                         identity: Some(identity.clone()),
                         parent: network.nic.parent.clone(),
@@ -794,8 +806,9 @@ impl App {
                         queues,
                         workdir: workdir.clone(),
                     })
-                    .await
-                }
+                    .await,
+                    "PrepareMacvtap",
+                ),
                 NetworkingMode::User | NetworkingMode::Custom | NetworkingMode::Passt => continue,
             };
             let response = match result {
@@ -824,18 +837,22 @@ impl App {
                     // neither the NIC that asked nor what a node has to install
                     // to satisfy it appears anywhere in the failure.
                     let mode = network.nic.mode.as_str();
-                    let error = Err(error).context("failed to prepare netd-managed networking");
+                    let socket = self.config.netd.socket.display();
+                    let secs = netd::REQUEST_TIMEOUT.as_secs();
+                    let context = if error.chain().any(|e| e.is::<tokio::time::error::Elapsed>()) {
+                        format!("netd {method} on {socket} did not answer within {secs}s")
+                    } else {
+                        format!("netd {method} on {socket} failed")
+                    };
+                    let error = Err(error.context(context))
+                        .context("failed to prepare netd-managed networking");
                     return if unreachable {
                         error.with_context(|| {
                             format!(
                                 "interface {nic_index} is {mode}, whose host interface only netd \
                                  can build; run dstack-vmm netd on this host"
                             )
                         })
-                    } else if queues > 1 {
-                        error.with_context(|| {
-                            format!("interface {nic_index} asked for {queues} queue pairs")
-                        })
                     } else {
                         error.with_context(|| format!("interface {nic_index} is {mode}"))
                     };
@@ -1908,6 +1925,7 @@ impl App {
                 continue;
             }
             if let Err(error) = self.start_vm_with_restart_policy(&id, false).await {
+                self.set_boot_error(&id, &error);
                 warn!(id, %error, "automatic restart attempt failed");
             }
         }
@@ -2571,6 +2589,80 @@ mod tests {
         assert_eq!(netd.methods(), vec!["PrepareBridge", "RemoveInterface"]);
     }
 
+    /// Regression: an immediately rejected network preparation must surface
+    /// the complete StartVm error chain in the existing boot diagnostics.
+    #[tokio::test]
+    async fn failed_start_reports_the_netd_error_chain_in_boot_error() {
+        let dir = tempfile::tempdir().unwrap();
+        let netd = netd::testing::FakeNetd::spawn(&["RemoveInterface"]);
+        let (supervisor, server) = stopped_supervisor().await;
+        let mut config = test_config(netd.socket(), dir.path());
+        config.image.path = dir.path().join("images");
+        config.cvm.qemu_version = Some("10.1.0".into());
+        let image = config.image.path.join("dstack-test");
+        fs::create_dir_all(&image).unwrap();
+        fs::write(image.join("kernel"), b"kernel").unwrap();
+        fs::write(image.join("initrd"), b"initrd").unwrap();
+        fs::write(
+            image.join("metadata.json"),
+            serde_json::to_vec(&json!({
+                "kernel": "kernel", "initrd": "initrd", "hda": null,
+                "rootfs": null, "bios": null, "cmdline": null,
+                "version": "0.5.7", "shared_ro": true,
+            }))
+            .unwrap(),
+        )
+        .unwrap();
+        let app = App::new(config, supervisor);
+        let (mut vm, _) = bridge_vm(&app, "vm-1");
+        vm.manifest.no_tee = true;
+        vm.manifest.networks[0].queues = Some(2);
+        let workdir = app.work_dir("vm-1").unwrap();
+        fs::create_dir_all(workdir.shared_dir()).unwrap();
+        workdir.put_manifest(&vm.manifest).unwrap();
+        fs::write(workdir.app_compose_path(), r#"{"manifest_version":2,"name":"test","runner":"docker-compose","docker_compose_file":"services: {}","gateway_enabled":false,"storage_fs":"zfs"}"#).unwrap();
+        app.lock().add(VmState::new(vm));
+
+        let error = app.start_vm("vm-1").await.unwrap_err();
+        let chain = format!("{error:#}");
+        assert!(
+            chain.contains("failed to prepare netd-managed networking"),
+            "{chain}"
+        );
+        assert!(chain.contains("fake netd refuses PrepareBridge"), "{chain}");
+        let info = app.vm_info("vm-1").await.unwrap().unwrap();
+        assert!(
+            info.boot_error
+                .contains("failed to prepare netd-managed networking"),
+            "boot_error lost preparation context: {:?}",
+            info.boot_error
+        );
+        assert!(
+            info.boot_error.contains("fake netd refuses PrepareBridge"),
+            "boot_error lost netd reason: {:?}",
+            info.boot_error
+        );
+        assert!(
+            info.boot_error.contains("netd PrepareBridge on"),
+            "{}",
+            info.boot_error
+        );
+        assert!(
+            info.boot_error
+                .contains(&netd.socket().display().to_string()),
+            "{}",
+            info.boot_error
+        );
+        assert!(
+            !info.boot_error.contains("queue pairs"),
+            "{}",
+            info.boot_error
+        );
+        assert_eq!(info.boot_progress, "failed");
+        assert_eq!(netd.methods(), ["PrepareBridge", "RemoveInterface"]);
+        server.abort();
+    }
+
     /// A netd outage must not become a fleet that cannot be stopped.
     #[tokio::test]
     async fn a_stop_survives_a_netd_that_is_not_there() {

dstack/vmm/src/netd.rs

@@ -47,7 +47,7 @@ use crate::config::{NetdConfig, NetworkFilterConfig};
 
 /// Bounds a whole call from the VMM, including waiting for netd to finish the
 /// operations queued ahead of it.
-const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
+pub(crate) const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
 const COMMAND_TIMEOUT: Duration = Duration::from_secs(30);
 /// Bound on listing nwfilter bindings. See [`existing_bindings`].
 const LISTING_TIMEOUT: Duration = Duration::from_secs(10);