diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 5baf6aeae..44feaf57f 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -492,7 +492,18 @@ impl App { } pub async fn start_vm(&self, id: &str) -> Result<()> { - self.start_vm_with_restart_policy(id, true).await + let result = self.start_vm_with_restart_policy(id, true).await; + if let Err(err) = &result { + self.set_boot_error(id, err); + } + result + } + + fn set_boot_error(&self, id: &str, err: &anyhow::Error) { + if let Some(vm) = self.lock().get_mut(id) { + vm.state.boot_error = ra_rpc::log_text(&format!("{err:#}")); + vm.state.boot_progress = "failed".into(); + } } async fn start_vm_with_restart_policy( @@ -767,8 +778,8 @@ impl App { let queues = network.queue_pairs(); let filtered = filters_bridge_traffic(network, &self.config.cvm); let netd = netd::client(&self.config.netd.socket); - let result = match network.nic.mode { - NetworkingMode::Bridge => { + let (result, method) = match network.nic.mode { + NetworkingMode::Bridge => ( netd.prepare_bridge(PrepareBridgeRequest { identity: Some(identity.clone()), bridge: network.nic.bridge.clone(), @@ -782,9 +793,10 @@ impl App { queues, workdir: workdir.clone(), }) - .await - } - NetworkingMode::Macvtap => { + .await, + "PrepareBridge", + ), + NetworkingMode::Macvtap => ( netd.prepare_macvtap(PrepareMacvtapRequest { identity: Some(identity.clone()), parent: network.nic.parent.clone(), @@ -794,8 +806,9 @@ impl App { queues, workdir: workdir.clone(), }) - .await - } + .await, + "PrepareMacvtap", + ), NetworkingMode::User | NetworkingMode::Custom | NetworkingMode::Passt => continue, }; let response = match result { @@ -824,7 +837,15 @@ impl App { // neither the NIC that asked nor what a node has to install // to satisfy it appears anywhere in the failure. let mode = network.nic.mode.as_str(); - let error = Err(error).context("failed to prepare netd-managed networking"); + let socket = self.config.netd.socket.display(); + let secs = netd::REQUEST_TIMEOUT.as_secs(); + let context = if error.chain().any(|e| e.is::()) { + format!("netd {method} on {socket} did not answer within {secs}s") + } else { + format!("netd {method} on {socket} failed") + }; + let error = Err(error.context(context)) + .context("failed to prepare netd-managed networking"); return if unreachable { error.with_context(|| { format!( @@ -832,10 +853,6 @@ impl App { can build; run dstack-vmm netd on this host" ) }) - } else if queues > 1 { - error.with_context(|| { - format!("interface {nic_index} asked for {queues} queue pairs") - }) } else { error.with_context(|| format!("interface {nic_index} is {mode}")) }; @@ -1908,6 +1925,7 @@ impl App { continue; } if let Err(error) = self.start_vm_with_restart_policy(&id, false).await { + self.set_boot_error(&id, &error); warn!(id, %error, "automatic restart attempt failed"); } } @@ -2571,6 +2589,80 @@ mod tests { assert_eq!(netd.methods(), vec!["PrepareBridge", "RemoveInterface"]); } + /// Regression: an immediately rejected network preparation must surface + /// the complete StartVm error chain in the existing boot diagnostics. + #[tokio::test] + async fn failed_start_reports_the_netd_error_chain_in_boot_error() { + let dir = tempfile::tempdir().unwrap(); + let netd = netd::testing::FakeNetd::spawn(&["RemoveInterface"]); + let (supervisor, server) = stopped_supervisor().await; + let mut config = test_config(netd.socket(), dir.path()); + config.image.path = dir.path().join("images"); + config.cvm.qemu_version = Some("10.1.0".into()); + let image = config.image.path.join("dstack-test"); + fs::create_dir_all(&image).unwrap(); + fs::write(image.join("kernel"), b"kernel").unwrap(); + fs::write(image.join("initrd"), b"initrd").unwrap(); + fs::write( + image.join("metadata.json"), + serde_json::to_vec(&json!({ + "kernel": "kernel", "initrd": "initrd", "hda": null, + "rootfs": null, "bios": null, "cmdline": null, + "version": "0.5.7", "shared_ro": true, + })) + .unwrap(), + ) + .unwrap(); + let app = App::new(config, supervisor); + let (mut vm, _) = bridge_vm(&app, "vm-1"); + vm.manifest.no_tee = true; + vm.manifest.networks[0].queues = Some(2); + let workdir = app.work_dir("vm-1").unwrap(); + fs::create_dir_all(workdir.shared_dir()).unwrap(); + workdir.put_manifest(&vm.manifest).unwrap(); + fs::write(workdir.app_compose_path(), r#"{"manifest_version":2,"name":"test","runner":"docker-compose","docker_compose_file":"services: {}","gateway_enabled":false,"storage_fs":"zfs"}"#).unwrap(); + app.lock().add(VmState::new(vm)); + + let error = app.start_vm("vm-1").await.unwrap_err(); + let chain = format!("{error:#}"); + assert!( + chain.contains("failed to prepare netd-managed networking"), + "{chain}" + ); + assert!(chain.contains("fake netd refuses PrepareBridge"), "{chain}"); + let info = app.vm_info("vm-1").await.unwrap().unwrap(); + assert!( + info.boot_error + .contains("failed to prepare netd-managed networking"), + "boot_error lost preparation context: {:?}", + info.boot_error + ); + assert!( + info.boot_error.contains("fake netd refuses PrepareBridge"), + "boot_error lost netd reason: {:?}", + info.boot_error + ); + assert!( + info.boot_error.contains("netd PrepareBridge on"), + "{}", + info.boot_error + ); + assert!( + info.boot_error + .contains(&netd.socket().display().to_string()), + "{}", + info.boot_error + ); + assert!( + !info.boot_error.contains("queue pairs"), + "{}", + info.boot_error + ); + assert_eq!(info.boot_progress, "failed"); + assert_eq!(netd.methods(), ["PrepareBridge", "RemoveInterface"]); + server.abort(); + } + /// A netd outage must not become a fleet that cannot be stopped. #[tokio::test] async fn a_stop_survives_a_netd_that_is_not_there() { diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index f132f5bda..61d51bae7 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -47,7 +47,7 @@ use crate::config::{NetdConfig, NetworkFilterConfig}; /// Bounds a whole call from the VMM, including waiting for netd to finish the /// operations queued ahead of it. -const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +pub(crate) const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); const COMMAND_TIMEOUT: Duration = Duration::from_secs(30); /// Bound on listing nwfilter bindings. See [`existing_bindings`]. const LISTING_TIMEOUT: Duration = Duration::from_secs(10);