diff --git a/tests/Cargo.lock b/tests/Cargo.lock index 35bf9410263..c5ddb45d73a 100644 --- a/tests/Cargo.lock +++ b/tests/Cargo.lock @@ -5099,10 +5099,10 @@ dependencies = [ "i-slint-backend-qt", "i-slint-backend-winit", "i-slint-core", - "libtest-mimic", "linkme", "satchel", "slint", + "test_driver_lib", ] [[package]] @@ -5158,7 +5158,9 @@ dependencies = [ "i-slint-backend-testing", "i-slint-compiler", "i-slint-core", + "linkme", "rayon", + "satchel", "slint", "slint-interpreter", "spin_on", @@ -5203,7 +5205,9 @@ dependencies = [ name = "test_driver_lib" version = "1.19.0" dependencies = [ + "libtest-mimic", "regex", + "satchel", "walkdir", ] diff --git a/tests/backends/Cargo.toml b/tests/backends/Cargo.toml index 838bbd86e61..03938a2dd37 100644 --- a/tests/backends/Cargo.toml +++ b/tests/backends/Cargo.toml @@ -1,7 +1,7 @@ # Copyright © SixtyFPS GmbH # SPDX-License-Identifier: GPL-3.0-only OR LicenseRef-Slint-Royalty-free-2.0 OR LicenseRef-Slint-Software-3.0 -# cSpell: ignore libtest linkme +# cSpell: ignore linkme [package] name = "test-backends" description = "Tests that are shared between the different backends" @@ -25,11 +25,10 @@ i-slint-core = { workspace = true } # TODO: Add additional test for LinuxKMS, but that requires root permissions. # TODO: Add additional test for Android Activity, but that requires an Android device and the Android SDK setup. -# Unfortunately we need to provide our own test harness, so that we can run each test in a separate process and on the main thread. -# But with satchel + libtest-mimic, this is reasonably easy to do. +# Each test runs in a separate process on the main thread, see test_driver_lib::fork_harness. satchel = "0.3" -libtest-mimic = "0.8" linkme = "0.3" +test_driver_lib = { path = "../driver/driverlib", features = ["fork-harness"] } [[test]] name = "qt" diff --git a/tests/backends/tests/cases/harness.rs b/tests/backends/tests/cases/harness.rs deleted file mode 100644 index b56e40cab57..00000000000 --- a/tests/backends/tests/cases/harness.rs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright © SixtyFPS GmbH -// SPDX-License-Identifier: GPL-3.0-only OR LicenseRef-Slint-Royalty-free-2.0 OR LicenseRef-Slint-Software-3.0 - -// cSpell: ignore nextest -use libtest_mimic::{Arguments, Failed}; -use satchel::TestCase; -use std::process::{Command, Stdio}; - -fn extract_single_test(args: &Arguments) -> Option { - // Note: Cargo nextest also uses `--exact` to run a single test, - // so we can use that to detect if we're already in a subprocess. - if args.exact { args.filter.clone() } else { None } -} - -pub(crate) fn run_forked(name: &str) -> Result<(), Failed> { - println!("### FORKING TEST: {name}"); - let status = Command::new(std::env::current_exe().unwrap()) - .args(["--exact", &name]) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .status()?; - - println!("### SUBPROCESS STATUS: {}", status); - - if !status.success() { - Err(Failed::from(format!("Test {name} failed in subprocess"))) - } else { - Ok(()) - } -} - -// single-test mode: run only that test on the main process, and don't fork a subprocess. -pub fn run_test(test_name: &str) { - let tests: Vec<_> = satchel::get_tests!().collect(); - - let test = tests - .iter() - .find(|test| qualified_name(test) == test_name) - .unwrap_or_else(|| panic!("Test {test_name} not found")); - crate::init(); - (test.test_fn)(); -} - -fn qualified_name(test: &TestCase) -> String { - format!("{}::{}", test.module_path, test.name) -} - -// Run all tests, but fork a subprocess for each test. -pub fn fork_tests(args: Arguments) { - let tests = satchel::get_tests!() - .map(|test| { - let name = qualified_name(&test); - let test_fn = { - let name = name.clone(); - move || run_forked(&name) - }; - - libtest_mimic::Trial::test(name, test_fn) - }) - .collect(); - libtest_mimic::run(&args, tests).exit(); -} - -pub fn test_main() { - let args = libtest_mimic::Arguments::from_args(); - - if let Some(test_filter) = extract_single_test(&args) { - run_test(&test_filter); - } else { - fork_tests(args); - } -} diff --git a/tests/backends/tests/cases/mod.rs b/tests/backends/tests/cases/mod.rs index 25889a7a210..4ad8bbec24d 100644 --- a/tests/backends/tests/cases/mod.rs +++ b/tests/backends/tests/cases/mod.rs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: GPL-3.0-only OR LicenseRef-Slint-Royalty-free-2.0 OR LicenseRef-Slint-Software-3.0 pub mod context_menu; -pub mod harness; pub mod menubar; pub mod platform_uses_mock_data; pub mod set_size_before_show; diff --git a/tests/backends/tests/qt.rs b/tests/backends/tests/qt.rs index 896c919ed7b..61defbf8d58 100644 --- a/tests/backends/tests/qt.rs +++ b/tests/backends/tests/qt.rs @@ -9,5 +9,5 @@ fn init() { } fn main() { - cases::harness::test_main(); + test_driver_lib::fork_harness::test_main(satchel::get_tests!(), init); } diff --git a/tests/backends/tests/winit.rs b/tests/backends/tests/winit.rs index ddf9744292a..f9c49160433 100644 --- a/tests/backends/tests/winit.rs +++ b/tests/backends/tests/winit.rs @@ -11,5 +11,5 @@ fn init() { } fn main() { - cases::harness::test_main(); + test_driver_lib::fork_harness::test_main(satchel::get_tests!(), init); } diff --git a/tests/driver/driverlib/Cargo.toml b/tests/driver/driverlib/Cargo.toml index 164b863c4a0..c7a4d77e822 100644 --- a/tests/driver/driverlib/Cargo.toml +++ b/tests/driver/driverlib/Cargo.toml @@ -16,6 +16,14 @@ publish = false [lib] path = "lib.rs" +[features] +# The fork harness for test binaries whose tests must run on the process main thread. +fork-harness = ["dep:libtest-mimic", "dep:satchel"] + [dependencies] regex = "1.4" walkdir = "2" + +# cSpell: ignore libtest +libtest-mimic = { version = "0.8", optional = true } +satchel = { version = "0.3", optional = true } diff --git a/tests/driver/driverlib/fork_harness.rs b/tests/driver/driverlib/fork_harness.rs new file mode 100644 index 00000000000..02e40e5326b --- /dev/null +++ b/tests/driver/driverlib/fork_harness.rs @@ -0,0 +1,68 @@ +// Copyright © SixtyFPS GmbH +// SPDX-License-Identifier: GPL-3.0-only OR LicenseRef-Slint-Royalty-free-2.0 OR LicenseRef-Slint-Software-3.0 + +//! A test harness that runs every test in its own process, on that process's main thread. +//! +//! libtest runs each test on a worker thread, but Qt widgets and AppKit controls only work +//! on the process main thread. +//! A test binary with `harness = false` registers its tests with `#[satchel::test]` +//! and calls [`test_main`] from its `main`. +//! The parent process forks one child per test with `--exact `, +//! the convention cargo nextest uses, and the child runs that single test on its main thread. + +// cSpell: ignore nextest + +use libtest_mimic::{Arguments, Failed, Trial}; +use satchel::TestCase; +use std::process::{Command, Stdio}; + +/// Runs `tests`, forking a process per test, and exits with the result. +/// The child calls `init` before its test. +pub fn test_main(tests: impl Iterator, init: fn()) { + let args = Arguments::from_args(); + let tests: Vec<_> = tests.collect(); + + // cargo forwards `--exact ` to every test binary of a package, so a name + // that belongs to another binary falls through and reports zero matching tests. + if args.exact && !args.list { + let single = args + .filter + .as_deref() + .and_then(|name| tests.iter().find(|test| qualified_name(test) == name)); + if let Some(test) = single { + init(); + (test.test_fn)(); + return; + } + } + + let trials = tests + .into_iter() + .map(|test| { + let name = qualified_name(test); + Trial::test(name.clone(), move || run_forked(&name)) + .with_ignored_flag(test.ignore.is_some()) + }) + .collect(); + libtest_mimic::run(&args, trials).exit(); +} + +fn qualified_name(test: &TestCase) -> String { + format!("{}::{}", test.module_path, test.name) +} + +fn run_forked(name: &str) -> Result<(), Failed> { + println!("### FORKING TEST: {name}"); + let status = Command::new(std::env::current_exe()?) + .args(["--exact", name]) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .status()?; + println!("### SUBPROCESS STATUS: {status}"); + + if status.success() { + Ok(()) + } else { + Err(Failed::from(format!("Test {name} failed in subprocess"))) + } +} diff --git a/tests/driver/driverlib/lib.rs b/tests/driver/driverlib/lib.rs index 9eeb221da25..8dfa8254904 100644 --- a/tests/driver/driverlib/lib.rs +++ b/tests/driver/driverlib/lib.rs @@ -3,6 +3,9 @@ use std::sync::LazyLock; +#[cfg(feature = "fork-harness")] +pub mod fork_harness; + use regex::Regex; pub struct TestCase { diff --git a/tests/driver/rust/Cargo.toml b/tests/driver/rust/Cargo.toml index fdd3faaca07..3d60c4ecb14 100644 --- a/tests/driver/rust/Cargo.toml +++ b/tests/driver/rust/Cargo.toml @@ -18,6 +18,12 @@ publish = false path = "main.rs" name = "test-driver-rust" +# The qt widget tests need the main thread, so they run through +# test_driver_lib::fork_harness; see tests/widgets-qt.rs. +[[test]] +name = "widgets-qt" +harness = false + [features] default = ["backend-qt"] @@ -36,6 +42,12 @@ spin_on = { workspace = true } [target.'cfg(not(target_os = "wasm"))'.dependencies] i-slint-backend-qt = { workspace = true } +# cSpell: ignore linkme +[dev-dependencies] +linkme = "0.3" +satchel = "0.3" +test_driver_lib = { path = "../driverlib", features = ["fork-harness"] } + [build-dependencies] i-slint-compiler = { workspace = true, features = ["default", "rust", "display-diagnostics", "bundle-translations", "renderer-software"], optional = true } diff --git a/tests/driver/rust/build.rs b/tests/driver/rust/build.rs index 3d47e2af468..1a4d4b9339f 100644 --- a/tests/driver/rust/build.rs +++ b/tests/driver/rust/build.rs @@ -45,7 +45,10 @@ fn make_generator_files() -> std::io::Result>> for file in std::fs::read_dir(tests_folder)? { let file = file?.path(); let base = file.file_stem().expect("Missing file name!"); - validate_test_file(base, &file)?; + // widgets-qt.rs carries its own main, see the file. + if base != "widgets-qt" { + validate_test_file(base, &file)?; + } let generated_path = PathBuf::from(&std::env::var_os("OUT_DIR").unwrap()).join(file.file_name().unwrap()); @@ -166,6 +169,10 @@ fn process_case( "" }; + // The widgets-qt target runs its tests through test_driver_lib::fork_harness, + // which collects them with satchel instead of libtest. + let main_thread = testcase.requested_style == Some("qt"); + let mut output = BufWriter::new(File::create( Path::new(&std::env::var_os("OUT_DIR").unwrap()).join(format!("{module_name}.rs")), )?); @@ -173,7 +180,7 @@ fn process_case( output.write_all(b"#![deny(warnings)]\n#![deny(rust_2018_idioms)]\n#![deny(unsafe_code)]\n")?; #[cfg(not(feature = "build-time"))] - if !generate_macro(&source, &mut output, testcase)? { + if !generate_macro(&source, &mut output, testcase, main_thread)? { output.flush()?; return Ok(module_line); } @@ -184,9 +191,29 @@ fn process_case( .filter(|x| x.language_id == "rust") .enumerate() { - write!( - output, - r" + if main_thread { + write!( + output, + r" +#[rust_analyzer::skip] +#[satchel::test] {} fn t_{}() {{ + (|| -> ::std::result::Result<(), ::std::boxed::Box> {{ + use i_slint_backend_testing as slint_testing; + slint_testing::init_no_event_loop(); + slint_testing::configure_test_fonts(); + {} + Ok(()) + }})() + .unwrap() +}}", + ignored, + i, + x.source.replace('\n', "\n ") + )?; + } else { + write!( + output, + r" #[rust_analyzer::skip] #[test] {} fn t_{}() -> ::std::result::Result<(), ::std::boxed::Box> {{ use i_slint_backend_testing as slint_testing; @@ -195,10 +222,11 @@ fn process_case( {} Ok(()) }}", - ignored, - i, - x.source.replace('\n', "\n ") - )?; + ignored, + i, + x.source.replace('\n', "\n ") + )?; + } } output.flush()?; @@ -210,14 +238,16 @@ fn generate_macro( source: &str, output: &mut dyn Write, testcase: &test_driver_lib::TestCase, + main_thread: bool, ) -> Result { + let test_attribute = if main_thread { "#[satchel::test]" } else { "#[test]" }; if source.contains("\\{") { // Unfortunately, \{ is not valid in a rust string so it cannot be used in a slint! macro - output.write_all(b"#[test] #[ignore = \"string template don't work in macros\"] fn ignored_because_string_template() {{}}")?; + writeln!(output, "{test_attribute} #[ignore = \"string template don't work in macros\"] fn ignored_because_string_template() {{}}")?; return Ok(false); } if testcase.is_ignored("rust-macro") { - output.write_all(b"#[test] #[ignore = \"testcase ignored for the slint! macro\"] fn ignored_for_macro() {}")?; + writeln!(output, "{test_attribute} #[ignore = \"testcase ignored for the slint! macro\"] fn ignored_for_macro() {{}}")?; return Ok(false); } // to silence all the warnings in .slint files that would be turned into errors diff --git a/tests/driver/rust/tests/widgets-qt.rs b/tests/driver/rust/tests/widgets-qt.rs index 8846043fae0..4fdf515f599 100644 --- a/tests/driver/rust/tests/widgets-qt.rs +++ b/tests/driver/rust/tests/widgets-qt.rs @@ -1,10 +1,14 @@ // Copyright © SixtyFPS GmbH // SPDX-License-Identifier: GPL-3.0-only OR LicenseRef-Slint-Royalty-free-2.0 OR LicenseRef-Slint-Software-3.0 -// NOTE: The files under tests/driver/rust/tests/ are generated and validated against this template. -// Do not edit them directly, instead edit the template.rs file! +// Unlike the other files in this directory, this one is not generated from template.rs: +// the qt style needs the process main thread, so the tests run through the fork harness. // needs to be crate-level #![deny(rust_2024_compatibility)] include!(concat!(env!("OUT_DIR"), "/widgets-qt.rs")); + +fn main() { + test_driver_lib::fork_harness::test_main(satchel::get_tests!(), || {}); +}