Merge pull request #2 from agentic-datasets/release-0.2.0 #47
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: conformance | |
| on: | |
| push: | |
| pull_request: | |
| workflow_dispatch: | |
| jobs: | |
| harness-standalone: | |
| # The portability claim rests on the conformance distribution needing no | |
| # implementation at all. This job installs *only* that package -- not the | |
| # reference implementation, not this repository -- and runs the suite. | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - run: pip install ./packages/agentic-dataset-conformance | |
| - name: the reference implementation is not installed | |
| run: | | |
| python - <<'PY' | |
| import importlib.util as u | |
| # find_spec on a dotted name imports the parent package first, so | |
| # find_spec("google.adk") raises ModuleNotFoundError when `google` | |
| # itself is absent -- which is the very state this job asserts. The | |
| # exception and a None spec mean the same thing here: not installed. | |
| for m in ("agentic_dataset", "langgraph", "llama_index", "google.adk", "mcp"): | |
| try: | |
| spec = u.find_spec(m) | |
| except ModuleNotFoundError: | |
| spec = None | |
| assert spec is None, f"{m} should not be installed in this job" | |
| PY | |
| - name: the suite runs against its own subject, from an unrelated directory | |
| run: cd /tmp && agentic-dataset-conformance run --matrix | |
| - name: the vectors can be exported for use elsewhere | |
| run: cd /tmp && agentic-dataset-conformance vectors --export ./exported && ls ./exported/vectors | wc -l | |
| packages-build: | |
| # Both distributions must build and install standalone, wheel and sdist. | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| - package: agentic-dataset-conformance | |
| module: agentic_dataset_conformance | |
| - package: authorized-recall | |
| module: authorized_recall | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - run: pip install build | |
| - run: python -m build --outdir dist ./packages/${{ matrix.package }} | |
| - name: wheel installs standalone and imports | |
| run: | | |
| python -m venv /tmp/w | |
| /tmp/w/bin/pip install dist/*.whl | |
| cd /tmp && /tmp/w/bin/python -c "import ${{ matrix.module }} as m; print(m.__name__, m.__version__)" | |
| - name: sdist installs standalone and imports | |
| run: | | |
| python -m venv /tmp/s | |
| /tmp/s/bin/pip install dist/*.tar.gz | |
| cd /tmp && /tmp/s/bin/python -c "import ${{ matrix.module }} as m; print(m.__name__, m.__version__)" | |
| all-subjects: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.11", "3.12", "3.13"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - run: pip install -e ./packages/authorized-recall -e ./packages/agentic-dataset-conformance -e ".[all]" | |
| - name: portable suite, every subject, plus the mutation matrix | |
| run: agentic-dataset-conformance run --subject conformance.subjects:subjects --matrix | |
| - name: white-box suite, 8 configurations | |
| run: python -m agentic_dataset.reference_suite | |
| - run: pytest -q | |
| - name: evaluation and the M6 measurement | |
| run: | | |
| python evals/evaluate.py | |
| python -m authorized_recall | |
| - name: the committed vectors match the generator | |
| run: | | |
| python conformance/generate.py | |
| git diff --exit-code conformance/ |