diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d9f30174..9d86cf9a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,7 +56,7 @@ jobs: - name: run tests run: | uv sync --all-extras - uvx --python ${{ matrix.python-version }} tox -c ${TOXCFG} -e ${TOXENV} + uvx --python ${{ matrix.python-version }} tox -c ${TOXCFG} -e ${TOXENV} -- -n 3 integration-tests: runs-on: ubuntu-24.04 diff --git a/README.md b/README.md index ea7f51d4..55351cd7 100644 --- a/README.md +++ b/README.md @@ -165,8 +165,11 @@ environment ([this page](http://docs.python-guide.org/en/latest/dev/virtualenvs/) has a good primer for working with development environments with Python). After you've created your development environment, just -`pip install tox` and run `tox` to run the unit tests. If you want -to run the integration tests, make sure you have +`pip install tox` and run `tox` to run the unit tests — this also runs +the HTTP integration tests in `tests/integration/`, which start their own +server in a background thread and need no extra setup. If you want +to run the full example suites (mqtt/grpc/graphql/etc.) and the +`tavern-ci`/library entrypoint checks, make sure you have [docker](https://www.docker.com/) installed and run `tox -c tox-integration.ini` (bear in mind this might take a while.) It's that simple! diff --git a/docs/source/core_concepts/external_code.md b/docs/source/core_concepts/external_code.md index 6c407cfb..6e1e9c93 100644 --- a/docs/source/core_concepts/external_code.md +++ b/docs/source/core_concepts/external_code.md @@ -52,8 +52,7 @@ response (HTTP or MQTT): ```python # testing_utils.py def message_says_hello(response): - """Make sure that the response was friendly - """ + """Make sure that the response was friendly""" assert response.json().get("message") == "hello world" ``` @@ -233,9 +232,7 @@ from box import Box def generate_bearer_token(): token = sign_a_jwt() - auth_header = { - "Authorization": "Bearer {}".format(token) - } + auth_header = {"Authorization": "Bearer {}".format(token)} return Box(auth_header) ``` @@ -262,6 +259,7 @@ The `--tavern-merge-ext-function-values` flag has been removed because this is n ```python # ext_functions.py + def return_hello(): return {"hello": "there"} ``` diff --git a/docs/source/core_concepts/marks.md b/docs/source/core_concepts/marks.md index fc2db1f0..83f9d468 100644 --- a/docs/source/core_concepts/marks.md +++ b/docs/source/core_concepts/marks.md @@ -500,6 +500,7 @@ In the `read_uuid` fixture: import pytest import json + @pytest.fixture def read_uuid(request): # 'request' is a built in pytest fixture marks = request.node.own_markers diff --git a/docs/source/http.md b/docs/source/http.md index c3b4d292..2b985639 100644 --- a/docs/source/http.md +++ b/docs/source/http.md @@ -269,6 +269,7 @@ Your external module would return an `AuthBase` instance: ```python from requests.auth import HTTPDigestAuth + def get_digest_auth(): return HTTPDigestAuth("myuser", "mypassword") ``` @@ -278,6 +279,7 @@ You can also use custom auth classes for non-standard schemes: ```python from requests.auth import AuthBase + class TokenAuth(AuthBase): def __init__(self, token): self.token = token @@ -286,6 +288,7 @@ class TokenAuth(AuthBase): r.headers["X-api-token"] = f"Token {self.token}" return r + def get_token_auth(): return TokenAuth("abc123") ``` diff --git a/docs/source/plugins.md b/docs/source/plugins.md index fe9c4427..5f5f6e17 100644 --- a/docs/source/plugins.md +++ b/docs/source/plugins.md @@ -111,7 +111,6 @@ requirements: ```python class MySession: - def __enter__(self): pass diff --git a/example/allure/docker-compose.yaml b/example/allure/docker-compose.yaml index 416d55c1..1bb317a3 100644 --- a/example/allure/docker-compose.yaml +++ b/example/allure/docker-compose.yaml @@ -5,7 +5,7 @@ services: context: ../http dockerfile: Dockerfile ports: - - "5000:5000" + - "5004:5000" # Only used to turn the results written by allure-pytest into a report - not # started by 'docker compose up', run it with 'docker compose run --rm allure' diff --git a/example/allure/tests/common.yaml b/example/allure/tests/common.yaml index b12788df..2a667dff 100644 --- a/example/allure/tests/common.yaml +++ b/example/allure/tests/common.yaml @@ -3,7 +3,7 @@ name: Allure example includes description: variables and the login stage used by the tests in this folder variables: - service: http://localhost:5000 + service: http://localhost:5004 user: user: test-user pass: correct-password diff --git a/pyproject.toml b/pyproject.toml index 5e6c7070..ca9636fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -148,6 +148,11 @@ exclude = [ [tool.coverage.run] branch = false +# Measure python subprocesses too (e.g. the pytester-spawned pytest runs in +# tests/integration/test_run_integration_suite.py); parallel data files are +# combined by pytest-cov / 'coverage combine' +parallel = true +patch = ["subprocess"] omit = [ "tests/*", ".eggs/*", @@ -165,9 +170,12 @@ tavern = [ ] [tool.pytest.ini_options] -testpaths = ["tavern", "tests/unit"] +testpaths = ["tavern", "tests/unit", "tests/integration"] addopts = [ "--doctest-modules", + # Stop --doctest-modules importing every conftest.py as a module: with + # multiple non-package conftest.py files the basenames clash in sys.modules + "--ignore-glob=*/conftest.py", "-r", "xs", "-vv", "--strict-markers", @@ -176,12 +184,18 @@ addopts = [ "-m", "not do_not_run" ] norecursedirs = [ - ".git", - ".tox", - ".venv", - "example", - "node_modules", + # pytest's defaults - setting this key replaces rather than extends them + "*.egg", + ".*", + "_darcs", + "build", + "CVS", "dist", + "node_modules", + "venv", + "{arch}", + # ours + "example", "docs", ] markers = [ diff --git a/scripts/coverage.sh b/scripts/coverage.sh index eefabb12..a4f4baa6 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -2,9 +2,6 @@ set -ex -tox -c tox-integration.ini -e py311-generic -tox -c tox-integration.ini -e py311-mqtt -tox -e py311 - -coverage combine --append .coverage tests/integration/.coverage example/mqtt/.coverage -coverage report -m +# Runs the unit tests and the tests/integration suite in one top-level pytest +# run and reports combined coverage for tavern. +uv run pytest --cov tavern --cov-report=term-missing "$@" diff --git a/tests/integration/README.md b/tests/integration/README.md index 49b9c499..101bc841 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -6,5 +6,15 @@ matching that don't nicely slot into the examples. Essentially, tests in this folder will typically consist of one stage (unless multi-stage functionality is being tested), and will not require logging in. -For the time being, all the random tests are just being put into the same -server.py and will be run with docker. +All the tests run against the flask app in `server.py`, in one of two modes: + +- **Plain `pytest`** (the default, also part of the top-level test run): the + server is started in a background thread on a random port (one per + pytest-xdist worker, if used) and each `test_*.tavern.yaml` file is run in a + pytest subprocess via the `pytester` plugin — see + `test_run_integration_suite.py`. No docker required. +- **`tox -c tox-integration.ini -e py3-generic`** (or `py3-noextra`): the server + runs under docker compose and the yaml files are collected directly, which + also exercises the `tavern-ci` CLI and `tavern.core.run` library entrypoints. + This mode sets `TAVERN_INTEGRATION_ALLOW_COLLECT=1` (see `conftest.py`) to + collect the yaml files in-process instead of via the pytester runner. diff --git a/tests/integration/common.yaml b/tests/integration/common.yaml index 1dd43898..f6283ea5 100644 --- a/tests/integration/common.yaml +++ b/tests/integration/common.yaml @@ -3,7 +3,7 @@ name: test includes description: used for testing against local server variables: - host: http://localhost:5003 + host: "{tavern.env_vars.TEST_HOST}" first_part: "nested" second_part: "{tavern.env_vars.SECOND_URL_PART}" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 19074bae..2786abf9 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -4,9 +4,29 @@ import pytest from box import Box +from server import run_in_background from tavern._core import exceptions +if not os.environ.get("TAVERN_INTEGRATION_ALLOW_COLLECT"): + # Plain top-level pytest: run the yaml suite only via the pytester runner + # (subprocess), never in-process — tavern patches the yaml parser globally + # and the suite needs --tavern-global-cfg and a running server. + collect_ignore_glob = ["test_*.tavern.yaml"] +else: + # tox-integration generic/noextra (docker server on port 5003) or the + # pytester subprocess: yaml files run directly, so the runner must not also + # run (it would run the whole suite a second time). + collect_ignore = ["test_run_integration_suite.py"] + + +@pytest.fixture(scope="session") +def integration_server(): + """Base URL of the flask server from server.py, running in a background + thread on a random port (one server per pytest-xdist worker).""" + with run_in_background() as port: + yield f"http://localhost:{port}" + @pytest.fixture def str_fixture(): @@ -32,7 +52,7 @@ def autouse_thing(): @pytest.fixture(scope="session", autouse=True) def fixture_echo_url(): - return "http://localhost:5003/echo" + return os.environ.get("TEST_HOST", "http://localhost:5003") + "/echo" @pytest.fixture(scope="session", autouse=True, name="autouse_thing_named") diff --git a/tests/integration/extra.yaml b/tests/integration/extra.yaml index 98c1bd21..d628e99f 100644 --- a/tests/integration/extra.yaml +++ b/tests/integration/extra.yaml @@ -3,6 +3,6 @@ name: global includes description: Used for testing global config tests variables: - global_host: http://localhost:5003 + global_host: "{tavern.env_vars.TEST_HOST}" test_string_1: "hello" test_string_2: "{tavern.env_vars.SECOND_URL_PART}" diff --git a/tests/integration/global_cfg.yaml b/tests/integration/global_cfg.yaml index 401d5211..e8f0574a 100644 --- a/tests/integration/global_cfg.yaml +++ b/tests/integration/global_cfg.yaml @@ -3,7 +3,7 @@ name: test global includes description: used for testing against local server variables: - global_host: http://localhost:5003 + global_host: "{tavern.env_vars.TEST_HOST}" global_test_string_1: abc global_test_string_2: "{tavern.env_vars.SECOND_URL_PART}" diff --git a/tests/integration/server.py b/tests/integration/server.py index af210c78..0f5b0a8c 100644 --- a/tests/integration/server.py +++ b/tests/integration/server.py @@ -1,17 +1,22 @@ import base64 +import contextlib import gzip import itertools import json import math import mimetypes import os +import threading import time +import urllib.error +import urllib.request import uuid from datetime import datetime, timedelta from hashlib import sha512 from urllib.parse import unquote_plus, urlencode import jwt +import werkzeug.serving from box import Box from flask import Flask, Response, jsonify, make_response, redirect, request, session from flask_httpauth import HTTPDigestAuth @@ -33,7 +38,7 @@ def get_digest_password(username): @app.route("/token", methods=["GET"]) def token(): return ( - '
Link
', + f'
Link
', 200, ) @@ -575,3 +580,33 @@ def ascii_table(): def expected_text(): """Echoes back plain text from request body""" return Response(request.get_data(as_text=True), content_type="text/plain") + + +def _wait_until_ready(url: str, timeout: float = 10.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + urllib.request.urlopen(url, timeout=0.5) + except (urllib.error.URLError, OSError): + time.sleep(0.05) + else: + return + raise RuntimeError(f"server not ready at {url} within {timeout}s") + + +@contextlib.contextmanager +def run_in_background(): + """Run the app in a daemon thread on an OS-assigned free port for the + duration of the with block, yielding the port. Using a random port means + concurrent runs (e.g. pytest-xdist workers) never clash.""" + httpd = werkzeug.serving.make_server("127.0.0.1", 0, app, threaded=True) + port = httpd.server_port + + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + try: + _wait_until_ready(f"http://localhost:{port}/headers") + yield port + finally: + httpd.shutdown() + thread.join(timeout=5) diff --git a/tests/integration/test_parametrize.tavern.yaml b/tests/integration/test_parametrize.tavern.yaml index 6d825773..22a1808c 100644 --- a/tests/integration/test_parametrize.tavern.yaml +++ b/tests/integration/test_parametrize.tavern.yaml @@ -268,7 +268,7 @@ marks: stages: - name: Echo back a unicode value and make sure it matches request: - url: "http://localhost:5003/echo" + url: "{tavern.env_vars.TEST_HOST}/echo" method: POST json: value: "{to_send}" diff --git a/tests/integration/test_run_integration_suite.py b/tests/integration/test_run_integration_suite.py new file mode 100644 index 00000000..bc776254 --- /dev/null +++ b/tests/integration/test_run_integration_suite.py @@ -0,0 +1,70 @@ +"""Runs the test_*.tavern.yaml files in this directory against the Flask server +from server.py, started in a background thread — no docker compose needed. + +Each yaml file runs in a pytest subprocess because tavern patches the yaml +parser globally when it runs (see https://github.com/taverntesting/tavern/issues/825, +and the same pattern in tests/unit/test_allure.py). + +tox-integration.ini's generic/noextra envs instead run the yaml files directly +against the docker compose server (exercising the tavern-ci and tavern.core.run +entrypoints); in that mode this module is collect-ignored by conftest.py. +""" + +from pathlib import Path + +import pytest + +pytest_plugins = ["pytester"] + +INTEGRATION_DIR = Path(__file__).parent +GLOBAL_CFG = INTEGRATION_DIR / "global_cfg.yaml" +YAML_FILES = sorted(INTEGRATION_DIR.glob("test_*.tavern.yaml")) +assert YAML_FILES, f"no test_*.tavern.yaml files found in {INTEGRATION_DIR}" + + +@pytest.fixture(autouse=True) +def _subprocess_env(monkeypatch, integration_server): + monkeypatch.setenv("TEST_HOST", integration_server) + monkeypatch.setenv("SECOND_URL_PART", "again") + # So `external_functions: ext_functions:...` resolves in the subprocess + monkeypatch.setenv("PYTHONPATH", str(INTEGRATION_DIR)) + # Lift conftest.py's collection guard for the subprocess + monkeypatch.setenv("TAVERN_INTEGRATION_ALLOW_COLLECT", "1") + + +@pytest.mark.parametrize("yaml_file", YAML_FILES, ids=lambda p: p.name) +def test_integration_yaml_file(pytester, yaml_file, monkeypatch): + # Schema validation resolves file upload paths relative to cwd (see + # extensions.py::validate_file_spec), so run from this directory — after the + # pytester fixture has already chdir'd into its tmpdir + monkeypatch.chdir(INTEGRATION_DIR) + + result = pytester.runpytest_subprocess( + str(yaml_file), + "--tavern-global-cfg", + str(GLOBAL_CFG), + "-m", + "not do_not_run", + # Plugin loading dominates the subprocess startup time, so disable + # everything the yaml tests don't use (cov is kept for coverage) + "-p", + "no:hypothesispytest", + "-p", + "no:xdist", + "-p", + "no:xdist.looponfail", + "-p", + "no:allure_pytest", + "-p", + "no:asyncio", + "-p", + "no:anyio", + "-p", + "no:faker", + "-p", + "no:random_order", + ) + + assert result.ret in (pytest.ExitCode.OK,), "\n".join( + result.outlines[-200:] + result.errlines[-200:] + ) diff --git a/tests/unit/test_schema.py b/tests/unit/test_schema.py index 3cedd9e5..06bb3316 100644 --- a/tests/unit/test_schema.py +++ b/tests/unit/test_schema.py @@ -7,7 +7,7 @@ import yaml from tavern._core.exceptions import BadSchemaError -from tavern._core.loader import load_single_document_yaml +from tavern._core.loader import error_on_empty_scalar, load_single_document_yaml from tavern._core.schema.files import verify_tests @@ -136,6 +136,16 @@ def test_verify_with_incorrect_value(self, test_dict, incorrect_value): class TestBadSchemaAtCollect: """Some errors happen at collection time - harder to test""" + @pytest.fixture(autouse=True) + def patch_yaml_parser(self, monkeypatch): + """Empty values only raise BadSchemaError once YamlItem has patched the + yaml parser globally (see item.py); apply the patch deterministically + instead of relying on an earlier test having constructed a YamlItem, + which is not guaranteed when the tests are distributed with xdist""" + monkeypatch.setattr( + yaml.parser.Parser, "process_empty_scalar", error_on_empty_scalar + ) + @staticmethod @contextlib.contextmanager def wrapfile_nondict(to_wrap): diff --git a/tox-integration.ini b/tox-integration.ini index ab079289..369736ce 100644 --- a/tox-integration.ini +++ b/tox-integration.ini @@ -12,6 +12,9 @@ setenv = SECOND_URL_PART = again PYTHONPATH = . COMPOSE_PROJECT_NAME = tavern-{envname} + ; Collect the yaml files directly (they run against the docker compose + ; server) instead of via tests/integration/test_run_integration_suite.py + generic,noextra: TAVERN_INTEGRATION_ALLOW_COLLECT = 1 changedir = grpc: example/grpc mqtt: example/mqtt