From 18245b52ce6e38f87ede8d1f01de06d773460088 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Thu, 3 Sep 2026 19:18:34 -0400 Subject: [PATCH 1/2] ci: print pytest names and keep the slow suite off main Quiet mode hid the current test. Verbose mode plus durations prints it. PRs and exact-main pushes keep the required gate: unit tests minus the qualification campaign, golden e2e, and record/compile/replay. The campaign and extra browser files stay on schedule and dispatch. Opened by an agent session, not the founder. --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/workflows/ci.yml | 64 +++++++++++++-------------- .github/workflows/validate-claims.yml | 4 +- public-artifacts.json | 20 ++++++++- tests/test_ci_workflow_contract.py | 53 +++++++++++++--------- 5 files changed, 85 insertions(+), 58 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index cdb6cfd1..31dd2d85 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -16,7 +16,7 @@ - [ ] PR title uses Conventional Commit format - [ ] `ruff check openadapt_flow` and `ruff format --check openadapt_flow` pass - [ ] `mypy` passes -- [ ] `pytest -q` passes locally +- [ ] `pytest -vv` passes locally - [ ] Tests added/updated for behavior changes - [ ] Docs updated (README/DESIGN/docs) if behavior or contracts changed - [ ] If this touches the identity gate / resolution ladder / halt logic, I diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4a5bc23..67b3a25f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,12 +18,12 @@ name: CI # windows-mock (non-injecting Win32 ABI + fake behavior contract). # The separate Validate claims workflow contributes its required CheckRun # context named `gate`, bringing the app-pinned routine total to 13. -# - push to main: run the complete unit and browser suites, plus the same -# required safety, packaging, and platform-contract gates, against the -# exact merge commit. +# - push to main: the same required gate as a PR, against the exact merge +# commit. This does not re-run the qualification campaign or the extra +# browser scenarios; those stay on schedule and dispatch. # - schedule: run those gates plus the complete Python 3.10-3.12 / macOS -# matrix weekly. Release candidates can dispatch the same matrix on demand; -# routine PRs and exact-main pushes already exercise the required gates. +# matrix weekly, including the qualification campaign and extra e2e files. +# Release candidates can dispatch the same matrix on demand. # - workflow_dispatch: run the same complete matrix on an exact ref as the # explicit release-qualification lane. # There is intentionally NO bare `push:` -- a bare push trigger fires a SECOND @@ -185,7 +185,7 @@ jobs: if: needs.scope.outputs.code_changed == 'true' run: | python scripts/check_consistency.py - pytest -q tests/test_consistency.py + pytest -vv --durations=20 tests/test_consistency.py # --- STRICT type check on the safety-critical path (required on PRs) ------ # A second, STRICTER mypy pass scoped to the safety modules ONLY: the @@ -368,7 +368,7 @@ jobs: - name: Test openadapt-types boundary (no skips) if: needs.scope.outputs.code_changed == 'true' - run: pytest -q tests/test_interop_types.py tests/test_runtime_control_overlay.py + run: pytest -vv --durations=20 tests/test_interop_types.py tests/test_runtime_control_overlay.py - name: Type-check boundary against released schema if: needs.scope.outputs.code_changed == 'true' @@ -404,11 +404,11 @@ jobs: # It MUST stay a single, non-matrix job named exactly `test` so its reported # context is exactly `test` -- a matrix would report `test (3.12)` etc. and # the required `test` check would never report, permanently blocking merges. - # PRs exclude only the long deterministic qualification campaign. The exact - # merge commit, scheduled runs, and explicit dispatches run it here. Every - # lane excludes the browser/OCR record->compile->replay suite (tests/e2e), - # which has its own required `e2e-browser` check. This keeps PR feedback - # bounded while the exact main commit retains complete-suite evidence. + # PRs and exact-main pushes exclude the long deterministic qualification + # campaign. Scheduled runs and explicit dispatches run it here. Every lane + # excludes the browser/OCR record->compile->replay suite (tests/e2e), which + # has its own required `e2e-browser` check. This keeps merge feedback + # bounded. Complete-suite evidence stays on the weekly/dispatch matrix. test: needs: [scope] runs-on: ubuntu-latest @@ -447,7 +447,7 @@ jobs: # on GitHub Linux and does not depend on an apt mirror. - name: Linux retained installer process group (non-injecting) if: needs.scope.outputs.code_changed == 'true' - run: pytest -q tests/test_install_playwright_browser.py + run: pytest -vv --durations=20 tests/test_install_playwright_browser.py # Cache the Playwright browser binaries (chromium) so the lightweight # browser-backed unit tests don't re-download ~150MB every run. Keyed on @@ -489,11 +489,11 @@ jobs: if: needs.scope.outputs.code_changed == 'true' run: | mkdir -p runs - extra_args=() - if [ "$GITHUB_EVENT_NAME" = pull_request ]; then - extra_args+=(--ignore=tests/test_qualification_gate_campaign.py) + extra_args=(--ignore=tests/test_qualification_gate_campaign.py) + if [ "$GITHUB_EVENT_NAME" = schedule ] || [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then + extra_args=() fi - pytest -q --ignore=tests/e2e \ + pytest -vv --durations=20 --ignore=tests/e2e \ --ignore=tests/test_install_playwright_browser.py \ "${extra_args[@]}" \ --basetemp=runs/ci \ @@ -543,10 +543,10 @@ jobs: # HEADLESS Playwright browser against the bundled MockMed app, so it needs no # OS-level input permissions and runs deterministically on a Linux runner -- # which is exactly why it is a REQUIRED PR check here, not a weekly-only one. - # PRs run the golden task and the canonical record/compile/replay contract. - # The exact merge commit, schedules, and explicit dispatches run all other - # browser scenarios. Desktop/Citrix/Parallels tests self-skip when their - # macOS/VM backends are absent on this Linux runner. + # PRs and exact-main pushes run the golden task and the canonical + # record/compile/replay contract. Schedules and explicit dispatches run all + # other browser scenarios. Desktop/Citrix/Parallels tests self-skip when + # their macOS/VM backends are absent on this Linux runner. # The GOLDEN TASK (tests/e2e/test_free_path_e2e.py) runs as the job's FIRST # step and is excluded from the later complete-suite run, so the core loop # answers first without costing a second run of it. @@ -609,17 +609,17 @@ jobs: if: needs.scope.outputs.code_changed == 'true' run: | mkdir -p runs - pytest -q tests/e2e/test_free_path_e2e.py --basetemp=runs/ci-golden + pytest -vv --durations=20 tests/e2e/test_free_path_e2e.py --basetemp=runs/ci-golden - name: E2E (PR gate or complete post-merge suite) if: needs.scope.outputs.code_changed == 'true' run: | mkdir -p runs - test_paths=(tests/e2e --ignore=tests/e2e/test_free_path_e2e.py) - if [ "$GITHUB_EVENT_NAME" = pull_request ]; then - test_paths=(tests/e2e/test_record_compile_replay.py) + test_paths=(tests/e2e/test_record_compile_replay.py) + if [ "$GITHUB_EVENT_NAME" = schedule ] || [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then + test_paths=(tests/e2e --ignore=tests/e2e/test_free_path_e2e.py) fi - pytest -q "${test_paths[@]}" \ + pytest -vv --durations=20 "${test_paths[@]}" \ --junitxml=runs/e2e-claims-junit.xml \ --basetemp=runs/ci @@ -805,7 +805,7 @@ jobs: OPENADAPT_IDENTITY_LADDER_EXHAUSTIVE: "1" run: | mkdir -p runs - pytest -q --basetemp=runs/ci + pytest -vv --durations=20 --basetemp=runs/ci # The gate-standard qualification campaign is deselected on macOS for the # same reason as the identity harness above: it is platform-neutral by @@ -821,7 +821,7 @@ jobs: if: runner.os == 'macOS' run: | mkdir -p runs - pytest -q --basetemp=runs/ci \ + pytest -vv --durations=20 --basetemp=runs/ci \ --deselect=tests/test_identity_ladder.py::test_harness_zero_false_accept_all_configs \ --deselect=tests/test_qualification_gate_campaign.py @@ -866,12 +866,12 @@ jobs: - name: Windows live execution-context probe (read-only) if: needs.scope.outputs.code_changed == 'true' run: | - pytest -q tests/test_windows_context_identity_native.py + pytest -vv --durations=20 tests/test_windows_context_identity_native.py - name: Windows retained installer Job Object (non-injecting) if: needs.scope.outputs.code_changed == 'true' run: | - pytest -q tests/test_install_playwright_browser.py + pytest -vv --durations=20 tests/test_install_playwright_browser.py # The remaining WindowsBackend + RDP backend suites mock the WAA/RDP # servers with stdlib HTTP + fakes (no live VM, no Playwright browser), @@ -882,8 +882,8 @@ jobs: - name: Windows ABI and mock backend tests (non-injecting) if: needs.scope.outputs.code_changed == 'true' run: | - pytest -q tests/test_windows_backend.py tests/test_rdp_backend.py tests/test_win32_window_client.py tests/test_remote_display_backend.py - pytest -q tests/test_hosted.py -k client_run_id + pytest -vv --durations=20 tests/test_windows_backend.py tests/test_rdp_backend.py tests/test_win32_window_client.py tests/test_remote_display_backend.py + pytest -vv --durations=20 tests/test_hosted.py -k client_run_id # --- Clean-wheel install + CLI smoke (REQUIRED on PRs) ------------------- # Builds the wheel, installs it in a FRESH venv (no source tree), and proves diff --git a/.github/workflows/validate-claims.yml b/.github/workflows/validate-claims.yml index 48be76cc..633c0f6d 100644 --- a/.github/workflows/validate-claims.yml +++ b/.github/workflows/validate-claims.yml @@ -86,7 +86,7 @@ jobs: - name: Also run the validator's own tests run: | pip install "pytest>=8" - pytest -q tests/test_validate_claims.py + pytest -vv --durations=20 tests/test_validate_claims.py # --- Infra-gated: scoped validating evidence on its exact substrate ------- # These selected tests require macOS, local Parallels, a running configured @@ -171,7 +171,7 @@ jobs: OAFLOW_CITRIX_PIXEL_E2E: "1" run: | mkdir -p runs - python -m pytest -q \ + python -m pytest -vv --durations=20 \ tests/e2e/test_citrix_pixel_e2e.py \ tests/e2e/test_parallels_desktop_e2e.py \ --junitxml=runs/validating-junit.xml --basetemp=runs/ci diff --git a/public-artifacts.json b/public-artifacts.json index 477071b3..e5f7b650 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -93,7 +93,7 @@ }, { "path": ".github/workflows/ci.yml", - "sha256": "a2a361f21e93a7fab511f539efd74ddb84fbeea99cf9cc21efa76e341a6cc231" + "sha256": "f48bbee282d3ef293c81af185e2827fcf440ff4719dfc9f91a17d344c49c3108" }, { "path": ".github/workflows/citrix-workspace-standin.yml", @@ -125,7 +125,7 @@ }, { "path": ".github/workflows/validate-claims.yml", - "sha256": "5cfeab8e231b79bbeacb106f069a331ee46f007631b9d28f9aca4239553afdbc" + "sha256": "8739e95a294523fe5696e9c01602aaf52ee4941210db25ce625b0cf844d2a950" }, { "path": ".pre-commit-config.yaml", @@ -5835,6 +5835,22 @@ "path": "tests/fixtures/theme-region-stable-overhalt-v1.16.1.json", "sha256": "4bf055e6c11f837ed6daa4a5b89ab28dd431a7eccd2ce3044758ba260eed52cd" }, + { + "path": "tests/fixtures/v4-qualification/decision-receipt.json", + "sha256": "6ffe691f8e40e06b73e24391d180424ed86bb289ddae001cf7b49de2e745312a" + }, + { + "path": "tests/fixtures/v4-qualification/qualification-admission.json", + "sha256": "dcdb32a762aca87fbb1a7c9df5d346403b167ec5850d35fe47fd64d942684a04" + }, + { + "path": "tests/fixtures/v4-qualification/revocation-state.json", + "sha256": "18633b8cc243f686706606162bfa249a29733811563f63b9e68cc7c4a3507676" + }, + { + "path": "tests/fixtures/v4-qualification/signer-registry.json", + "sha256": "7a81bf3d213c74673f3c6b5fa179234cbee534c9432aaea6ae09e455562f96b6" + }, { "path": "tests/golden/tutorial_run.json", "sha256": "14242973a8fa8355e39633d23a51f5615f3f7bf9e956bc5ac6909d57ccfac3de" diff --git a/tests/test_ci_workflow_contract.py b/tests/test_ci_workflow_contract.py index ff074af4..fb4f745c 100644 --- a/tests/test_ci_workflow_contract.py +++ b/tests/test_ci_workflow_contract.py @@ -90,14 +90,14 @@ def test_playwright_installs_and_enclosing_jobs_are_bounded() -> None: # The privileged cleanup proof runs before browser delivery and does not # depend on an apt mirror. The full suite excludes only that duplicate. assert "Linux retained installer process group (non-injecting)" in test_job - assert "pytest -q tests/test_install_playwright_browser.py" in test_job + assert "pytest -vv --durations=20 tests/test_install_playwright_browser.py" in test_job assert "--ignore=tests/test_install_playwright_browser.py" in test_job assert "coverage report --fail-under=85" in test_job - assert "pytest -q tests/e2e/test_free_path_e2e.py" in e2e_job + assert "pytest -vv --durations=20 tests/e2e/test_free_path_e2e.py" in e2e_job assert "test_paths=(tests/e2e/test_record_compile_replay.py)" in e2e_job - assert 'pytest -q "${test_paths[@]}"' in e2e_job + assert 'pytest -vv --durations=20 "${test_paths[@]}"' in e2e_job assert "--ignore=tests/e2e/test_free_path_e2e.py" in e2e_job - assert "pytest -q --basetemp=runs/ci" in matrix_job + assert "pytest -vv --durations=20 --basetemp=runs/ci" in matrix_job def test_standard_browser_step_covers_cleanup_and_launch_worst_cases() -> None: @@ -183,7 +183,10 @@ def test_windows_required_job_proves_retained_installer_job_object() -> None: windows_job = workflow[windows_start:wheel_start] assert "Windows retained installer Job Object (non-injecting)" in windows_job - assert "pytest -q tests/test_install_playwright_browser.py" in windows_job + assert ( + "pytest -vv --durations=20 tests/test_install_playwright_browser.py" + in windows_job + ) def test_required_linux_atspi_qualification_is_bounded() -> None: @@ -350,7 +353,7 @@ def test_macos_deselects_only_redundant_platform_neutral_heavy_nodes() -> None: assert "if: runner.os == 'Linux'" in linux_step assert "--deselect" not in linux_step - assert "pytest -q --basetemp=runs/ci" in linux_step + assert "pytest -vv --durations=20 --basetemp=runs/ci" in linux_step assert "if: runner.os == 'macOS'" in macos_step for node in nodes: assert macos_step.count(f"--deselect={node}") == 1 @@ -406,7 +409,7 @@ def test_supported_claims_consume_their_required_jobs_real_junit() -> None: def test_pr_and_complete_test_tiers_are_event_bound_and_fail_closed() -> None: - """Only pull requests get the focused tier; every other event gets full tests.""" + """PR and main-push stay focused. Only schedule and dispatch get the rest.""" workflow = CI.read_text(encoding="utf-8") unit_start = workflow.index("\n test:") @@ -415,18 +418,25 @@ def test_pr_and_complete_test_tiers_are_event_bound_and_fail_closed() -> None: unit = workflow[unit_start:browser_start] browser = workflow[browser_start:linux_start] - pr_condition = 'if [ "$GITHUB_EVENT_NAME" = pull_request ]; then' + complete_condition = ( + 'if [ "$GITHUB_EVENT_NAME" = schedule ] || ' + '[ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then' + ) campaign = "--ignore=tests/test_qualification_gate_campaign.py" assert unit.count("- name: Test (PR gate or complete post-merge suite)") == 1 - assert unit.count(pr_condition) == 1 + assert unit.count(complete_condition) == 1 unit_step = unit[ unit.index("- name: Test (PR gate or complete post-merge suite)") : unit.index( "- name: Validate passing unit claim evidence" ) ] + assert f"extra_args=({campaign})" in unit_step assert "extra_args=()" in unit_step - assert f"extra_args+=({campaign})" in unit_step + assert unit_step.index(f"extra_args=({campaign})") < unit_step.index( + complete_condition + ) + assert unit_step.index(complete_condition) < unit_step.index("extra_args=()") assert '"${extra_args[@]}"' in unit_step assert "--ignore=tests/e2e" in unit_step assert "--ignore=tests/test_install_playwright_browser.py" in unit_step @@ -434,24 +444,25 @@ def test_pr_and_complete_test_tiers_are_event_bound_and_fail_closed() -> None: assert "--cov=openadapt_flow --cov-report=" in unit_step assert browser.count("- name: E2E (PR gate or complete post-merge suite)") == 1 - assert browser.count(pr_condition) == 1 + assert browser.count(complete_condition) == 1 browser_step = browser[ browser.index( "- name: E2E (PR gate or complete post-merge suite)" ) : browser.index("- name: Validate passing browser claim evidence") ] - assert ( - "test_paths=(tests/e2e --ignore=tests/e2e/test_free_path_e2e.py)" - in browser_step - ) - assert "test_paths=(tests/e2e/test_record_compile_replay.py)" in browser_step - assert 'pytest -q "${test_paths[@]}"' in browser_step + focused = "test_paths=(tests/e2e/test_record_compile_replay.py)" + complete = "test_paths=(tests/e2e --ignore=tests/e2e/test_free_path_e2e.py)" + assert focused in browser_step + assert complete in browser_step + assert browser_step.index(focused) < browser_step.index(complete_condition) + assert browser_step.index(complete_condition) < browser_step.index(complete) + assert 'pytest -vv --durations=20 "${test_paths[@]}"' in browser_step assert "--junitxml=runs/e2e-claims-junit.xml" in browser_step - # The arrays start with the complete selections. Only the exact PR branch - # replaces them. A new event cannot silently inherit the reduced selection. - for event in ("push", "schedule", "workflow_dispatch"): - assert f'GITHUB_EVENT_NAME" = {event}' not in unit + browser + # The arrays start focused. Only named slow events expand them. A new + # event cannot silently inherit the complete selection. + assert 'GITHUB_EVENT_NAME" = pull_request' not in unit + browser + assert 'GITHUB_EVENT_NAME" = push' not in unit + browser def test_validating_refresh_uses_exact_macos_parallels_substrate_and_scope() -> None: From 4a1db00bc76856489b841c7880b110c9a2bb8e91 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Thu, 3 Sep 2026 19:26:33 -0400 Subject: [PATCH 2/2] style: wrap the CI contract pytest assertion for ruff --- tests/test_ci_workflow_contract.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_ci_workflow_contract.py b/tests/test_ci_workflow_contract.py index fb4f745c..862c5615 100644 --- a/tests/test_ci_workflow_contract.py +++ b/tests/test_ci_workflow_contract.py @@ -90,7 +90,9 @@ def test_playwright_installs_and_enclosing_jobs_are_bounded() -> None: # The privileged cleanup proof runs before browser delivery and does not # depend on an apt mirror. The full suite excludes only that duplicate. assert "Linux retained installer process group (non-injecting)" in test_job - assert "pytest -vv --durations=20 tests/test_install_playwright_browser.py" in test_job + assert ( + "pytest -vv --durations=20 tests/test_install_playwright_browser.py" in test_job + ) assert "--ignore=tests/test_install_playwright_browser.py" in test_job assert "coverage report --fail-under=85" in test_job assert "pytest -vv --durations=20 tests/e2e/test_free_path_e2e.py" in e2e_job