tests: run the qt widget tests on the process main thread

tests/backends/Cargo.toml

@@ -1,7 +1,7 @@
 # Copyright © SixtyFPS GmbH <info@slint.dev>
 # SPDX-License-Identifier: GPL-3.0-only OR LicenseRef-Slint-Royalty-free-2.0 OR LicenseRef-Slint-Software-3.0
 
-# cSpell: ignore libtest linkme
+# cSpell: ignore linkme
 [package]
 name = "test-backends"
 description = "Tests that are shared between the different backends"
@@ -25,11 +25,10 @@ i-slint-core = { workspace = true }
 # TODO: Add additional test for LinuxKMS, but that requires root permissions.
 # TODO: Add additional test for Android Activity, but that requires an Android device and the Android SDK setup.
 
-# Unfortunately we need to provide our own test harness, so that we can run each test in a separate process and on the main thread.
-# But with satchel + libtest-mimic, this is reasonably easy to do.
+# Each test runs in a separate process on the main thread, see test_driver_lib::fork_harness.
 satchel = "0.3"
-libtest-mimic = "0.8"
 linkme = "0.3"
+test_driver_lib = { path = "../driver/driverlib", features = ["fork-harness"] }
 
 [[test]]
 name = "qt"

tests/backends/tests/cases/mod.rs

@@ -2,7 +2,6 @@
 // SPDX-License-Identifier: GPL-3.0-only OR LicenseRef-Slint-Royalty-free-2.0 OR LicenseRef-Slint-Software-3.0
 
 pub mod context_menu;
-pub mod harness;
 pub mod menubar;
 pub mod platform_uses_mock_data;
 pub mod set_size_before_show;

tests/backends/tests/qt.rs

@@ -9,5 +9,5 @@ fn init() {
 }
 
 fn main() {
-    cases::harness::test_main();
+    test_driver_lib::fork_harness::test_main(satchel::get_tests!(), init);
 }

tests/backends/tests/winit.rs

@@ -11,5 +11,5 @@ fn init() {
 }
 
 fn main() {
-    cases::harness::test_main();
+    test_driver_lib::fork_harness::test_main(satchel::get_tests!(), init);
 }

tests/driver/driverlib/Cargo.toml

@@ -16,6 +16,14 @@ publish = false
 [lib]
 path = "lib.rs"
 
+[features]
+# The fork harness for test binaries whose tests must run on the process main thread.
+fork-harness = ["dep:libtest-mimic", "dep:satchel"]
+
 [dependencies]
 regex = "1.4"
 walkdir = "2"
+
+# cSpell: ignore libtest
+libtest-mimic = { version = "0.8", optional = true }
+satchel = { version = "0.3", optional = true }

tests/driver/driverlib/fork_harness.rs

@@ -0,0 +1,68 @@
+// Copyright © SixtyFPS GmbH <info@slint.dev>
+// SPDX-License-Identifier: GPL-3.0-only OR LicenseRef-Slint-Royalty-free-2.0 OR LicenseRef-Slint-Software-3.0
+
+//! A test harness that runs every test in its own process, on that process's main thread.
+//!
+//! libtest runs each test on a worker thread, but Qt widgets and AppKit controls only work
+//! on the process main thread.
+//! A test binary with `harness = false` registers its tests with `#[satchel::test]`
+//! and calls [`test_main`] from its `main`.
+//! The parent process forks one child per test with `--exact <name>`,
+//! the convention cargo nextest uses, and the child runs that single test on its main thread.
+
+// cSpell: ignore nextest
+
+use libtest_mimic::{Arguments, Failed, Trial};
+use satchel::TestCase;
+use std::process::{Command, Stdio};
+
+/// Runs `tests`, forking a process per test, and exits with the result.
+/// The child calls `init` before its test.
+pub fn test_main(tests: impl Iterator<Item = &'static TestCase>, init: fn()) {
+    let args = Arguments::from_args();
+    let tests: Vec<_> = tests.collect();
+
+    // cargo forwards `--exact <name>` to every test binary of a package, so a name
+    // that belongs to another binary falls through and reports zero matching tests.
+    if args.exact && !args.list {
+        let single = args
+            .filter
+            .as_deref()
+            .and_then(|name| tests.iter().find(|test| qualified_name(test) == name));
+        if let Some(test) = single {
+            init();
+            (test.test_fn)();
+            return;
+        }
+    }
+
+    let trials = tests
+        .into_iter()
+        .map(|test| {
+            let name = qualified_name(test);
+            Trial::test(name.clone(), move || run_forked(&name))
+                .with_ignored_flag(test.ignore.is_some())
+        })
+        .collect();
+    libtest_mimic::run(&args, trials).exit();
+}
+
+fn qualified_name(test: &TestCase) -> String {
+    format!("{}::{}", test.module_path, test.name)
+}
+
+fn run_forked(name: &str) -> Result<(), Failed> {
+    println!("### FORKING TEST: {name}");
+    let status = Command::new(std::env::current_exe()?)
+        .args(["--exact", name])
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit())
+        .status()?;
+    println!("### SUBPROCESS STATUS: {status}");
+
+    if status.success() {
+        Ok(())
+    } else {
+        Err(Failed::from(format!("Test {name} failed in subprocess")))
+    }
+}

tests/driver/driverlib/lib.rs

@@ -3,6 +3,9 @@
 
 use std::sync::LazyLock;
 
+#[cfg(feature = "fork-harness")]
+pub mod fork_harness;
+
 use regex::Regex;
 
 pub struct TestCase {

tests/driver/rust/Cargo.toml

@@ -18,6 +18,12 @@ publish = false
 path = "main.rs"
 name = "test-driver-rust"
 
+# The qt widget tests need the main thread, so they run through
+# test_driver_lib::fork_harness; see tests/widgets-qt.rs.
+[[test]]
+name = "widgets-qt"
+harness = false
+
 [features]
 default = ["backend-qt"]
 
@@ -36,6 +42,12 @@ spin_on = { workspace = true }
 [target.'cfg(not(target_os = "wasm"))'.dependencies]
 i-slint-backend-qt = { workspace = true }
 
+# cSpell: ignore linkme
+[dev-dependencies]
+linkme = "0.3"
+satchel = "0.3"
+test_driver_lib = { path = "../driverlib", features = ["fork-harness"] }
+
 [build-dependencies]
 i-slint-compiler = { workspace = true, features = ["default", "rust", "display-diagnostics", "bundle-translations", "renderer-software"], optional = true }
 

tests/driver/rust/build.rs

@@ -45,7 +45,10 @@ fn make_generator_files() -> std::io::Result<HashMap<OsString, BufWriter<File>>>
     for file in std::fs::read_dir(tests_folder)? {
         let file = file?.path();
         let base = file.file_stem().expect("Missing file name!");
-        validate_test_file(base, &file)?;
+        // widgets-qt.rs carries its own main, see the file.
+        if base != "widgets-qt" {
+            validate_test_file(base, &file)?;
+        }
 
         let generated_path =
             PathBuf::from(&std::env::var_os("OUT_DIR").unwrap()).join(file.file_name().unwrap());
@@ -166,14 +169,18 @@ fn process_case(
         ""
     };
 
+    // The widgets-qt target runs its tests through test_driver_lib::fork_harness,
+    // which collects them with satchel instead of libtest.
+    let main_thread = testcase.requested_style == Some("qt");
+
     let mut output = BufWriter::new(File::create(
         Path::new(&std::env::var_os("OUT_DIR").unwrap()).join(format!("{module_name}.rs")),
     )?);
 
     output.write_all(b"#![deny(warnings)]\n#![deny(rust_2018_idioms)]\n#![deny(unsafe_code)]\n")?;
 
     #[cfg(not(feature = "build-time"))]
-    if !generate_macro(&source, &mut output, testcase)? {
+    if !generate_macro(&source, &mut output, testcase, main_thread)? {
         output.flush()?;
         return Ok(module_line);
     }
@@ -184,9 +191,29 @@ fn process_case(
         .filter(|x| x.language_id == "rust")
         .enumerate()
     {
-        write!(
-            output,
-            r"
+        if main_thread {
+            write!(
+                output,
+                r"
+#[rust_analyzer::skip]
+#[satchel::test] {} fn t_{}() {{
+    (|| -> ::std::result::Result<(), ::std::boxed::Box<dyn ::std::error::Error>> {{
+        use i_slint_backend_testing as slint_testing;
+        slint_testing::init_no_event_loop();
+        slint_testing::configure_test_fonts();
+        {}
+        Ok(())
+    }})()
+    .unwrap()
+}}",
+                ignored,
+                i,
+                x.source.replace('\n', "\n        ")
+            )?;
+        } else {
+            write!(
+                output,
+                r"
 #[rust_analyzer::skip]
 #[test] {} fn t_{}() -> ::std::result::Result<(), ::std::boxed::Box<dyn ::std::error::Error>> {{
     use i_slint_backend_testing as slint_testing;
@@ -195,10 +222,11 @@ fn process_case(
     {}
     Ok(())
 }}",
-            ignored,
-            i,
-            x.source.replace('\n', "\n    ")
-        )?;
+                ignored,
+                i,
+                x.source.replace('\n', "\n    ")
+            )?;
+        }
     }
 
     output.flush()?;
@@ -210,14 +238,16 @@ fn generate_macro(
     source: &str,
     output: &mut dyn Write,
     testcase: &test_driver_lib::TestCase,
+    main_thread: bool,
 ) -> Result<bool, std::io::Error> {
+    let test_attribute = if main_thread { "#[satchel::test]" } else { "#[test]" };
     if source.contains("\\{") {
         // Unfortunately, \{ is not valid in a rust string so it cannot be used in a slint! macro
-        output.write_all(b"#[test] #[ignore = \"string template don't work in macros\"] fn ignored_because_string_template() {{}}")?;
+        writeln!(output, "{test_attribute} #[ignore = \"string template don't work in macros\"] fn ignored_because_string_template() {{}}")?;
         return Ok(false);
     }
     if testcase.is_ignored("rust-macro") {
-        output.write_all(b"#[test] #[ignore = \"testcase ignored for the slint! macro\"] fn ignored_for_macro() {}")?;
+        writeln!(output, "{test_attribute} #[ignore = \"testcase ignored for the slint! macro\"] fn ignored_for_macro() {{}}")?;
         return Ok(false);
     }
     // to silence all the warnings in .slint files that would be turned into errors