Skip to content

Commit d773d26

Browse files
committed
fix(vmm): expose failed starts through boot_error
Signed-off-by: Leechael Yim <yanleech@gmail.com>
1 parent 3c87784 commit d773d26

1 file changed

Lines changed: 70 additions & 1 deletion

File tree

‎dstack/vmm/src/app.rs‎

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -492,7 +492,18 @@ impl App {
492492
}
493493

494494
pub async fn start_vm(&self, id: &str) -> Result<()> {
495-
self.start_vm_with_restart_policy(id, true).await
495+
let result = self.start_vm_with_restart_policy(id, true).await;
496+
if let Err(err) = &result {
497+
self.set_boot_error(id, err);
498+
}
499+
result
500+
}
501+
502+
fn set_boot_error(&self, id: &str, err: &anyhow::Error) {
503+
if let Some(vm) = self.lock().get_mut(id) {
504+
vm.state.boot_error = ra_rpc::log_text(&format!("{err:#}"));
505+
vm.state.boot_progress = "failed".into();
506+
}
496507
}
497508

498509
async fn start_vm_with_restart_policy(
@@ -1908,6 +1919,7 @@ impl App {
19081919
continue;
19091920
}
19101921
if let Err(error) = self.start_vm_with_restart_policy(&id, false).await {
1922+
self.set_boot_error(&id, &error);
19111923
warn!(id, %error, "automatic restart attempt failed");
19121924
}
19131925
}
@@ -2571,6 +2583,63 @@ mod tests {
25712583
assert_eq!(netd.methods(), vec!["PrepareBridge", "RemoveInterface"]);
25722584
}
25732585

2586+
/// Regression: an immediately rejected network preparation must surface
2587+
/// the complete StartVm error chain in the existing boot diagnostics.
2588+
#[tokio::test]
2589+
async fn failed_start_reports_the_netd_error_chain_in_boot_error() {
2590+
let dir = tempfile::tempdir().unwrap();
2591+
let netd = netd::testing::FakeNetd::spawn(&["RemoveInterface"]);
2592+
let (supervisor, server) = stopped_supervisor().await;
2593+
let mut config = test_config(netd.socket(), dir.path());
2594+
config.image.path = dir.path().join("images");
2595+
config.cvm.qemu_version = Some("10.1.0".into());
2596+
let image = config.image.path.join("dstack-test");
2597+
fs::create_dir_all(&image).unwrap();
2598+
fs::write(image.join("kernel"), b"kernel").unwrap();
2599+
fs::write(image.join("initrd"), b"initrd").unwrap();
2600+
fs::write(
2601+
image.join("metadata.json"),
2602+
serde_json::to_vec(&json!({
2603+
"kernel": "kernel", "initrd": "initrd", "hda": null,
2604+
"rootfs": null, "bios": null, "cmdline": null,
2605+
"version": "0.5.7", "shared_ro": true,
2606+
}))
2607+
.unwrap(),
2608+
)
2609+
.unwrap();
2610+
let app = App::new(config, supervisor);
2611+
let (mut vm, _) = bridge_vm(&app, "vm-1");
2612+
vm.manifest.no_tee = true;
2613+
let workdir = app.work_dir("vm-1").unwrap();
2614+
fs::create_dir_all(workdir.shared_dir()).unwrap();
2615+
workdir.put_manifest(&vm.manifest).unwrap();
2616+
fs::write(workdir.app_compose_path(), r#"{"manifest_version":2,"name":"test","runner":"docker-compose","docker_compose_file":"services: {}","gateway_enabled":false,"storage_fs":"zfs"}"#).unwrap();
2617+
app.lock().add(VmState::new(vm));
2618+
2619+
let error = app.start_vm("vm-1").await.unwrap_err();
2620+
let chain = format!("{error:#}");
2621+
assert!(
2622+
chain.contains("failed to prepare netd-managed networking"),
2623+
"{chain}"
2624+
);
2625+
assert!(chain.contains("fake netd refuses PrepareBridge"), "{chain}");
2626+
let info = app.vm_info("vm-1").await.unwrap().unwrap();
2627+
assert!(
2628+
info.boot_error
2629+
.contains("failed to prepare netd-managed networking"),
2630+
"boot_error lost preparation context: {:?}",
2631+
info.boot_error
2632+
);
2633+
assert!(
2634+
info.boot_error.contains("fake netd refuses PrepareBridge"),
2635+
"boot_error lost netd reason: {:?}",
2636+
info.boot_error
2637+
);
2638+
assert_eq!(info.boot_progress, "failed");
2639+
assert_eq!(netd.methods(), ["PrepareBridge", "RemoveInterface"]);
2640+
server.abort();
2641+
}
2642+
25742643
/// A netd outage must not become a fleet that cannot be stopped.
25752644
#[tokio::test]
25762645
async fn a_stop_survives_a_netd_that_is_not_there() {

0 commit comments

Comments
 (0)