test: run tests/integration via pytester without docker compose

.github/workflows/main.yml

@@ -56,7 +56,7 @@ jobs:
       - name: run tests
         run: |
           uv sync --all-extras
-          uvx --python ${{ matrix.python-version }} tox -c ${TOXCFG} -e ${TOXENV}
+          uvx --python ${{ matrix.python-version }} tox -c ${TOXCFG} -e ${TOXENV} -- -n 3
 
   integration-tests:
     runs-on: ubuntu-24.04

README.md

@@ -165,8 +165,11 @@ environment ([this
 page](http://docs.python-guide.org/en/latest/dev/virtualenvs/) has a
 good primer for working with development environments with Python).
 After you've created your development environment, just
-`pip install tox` and run `tox` to run the unit tests. If you want
-to run the integration tests, make sure you have
+`pip install tox` and run `tox` to run the unit tests — this also runs
+the HTTP integration tests in `tests/integration/`, which start their own
+server in a background thread and need no extra setup. If you want
+to run the full example suites (mqtt/grpc/graphql/etc.) and the
+`tavern-ci`/library entrypoint checks, make sure you have
 [docker](https://www.docker.com/) installed and run
 `tox -c tox-integration.ini` (bear in mind this might take a while.)
 It's that simple!

docs/source/core_concepts/external_code.md

@@ -52,8 +52,7 @@ response (HTTP or MQTT):
 ```python
 # testing_utils.py
 def message_says_hello(response):
-    """Make sure that the response was friendly
-    """
+    """Make sure that the response was friendly"""
     assert response.json().get("message") == "hello world"
 ```
 
@@ -233,9 +232,7 @@ from box import Box
 
 def generate_bearer_token():
     token = sign_a_jwt()
-    auth_header = {
-        "Authorization": "Bearer {}".format(token)
-    }
+    auth_header = {"Authorization": "Bearer {}".format(token)}
     return Box(auth_header)
 ```
 
@@ -262,6 +259,7 @@ The `--tavern-merge-ext-function-values` flag has been removed because this is n
 ```python
 # ext_functions.py
 
+
 def return_hello():
     return {"hello": "there"}
 ```

docs/source/core_concepts/marks.md

@@ -500,6 +500,7 @@ In the `read_uuid` fixture:
 import pytest
 import json
 
+
 @pytest.fixture
 def read_uuid(request):  # 'request' is a built in pytest fixture
     marks = request.node.own_markers

docs/source/http.md

@@ -269,6 +269,7 @@ Your external module would return an `AuthBase` instance:
 ```python
 from requests.auth import HTTPDigestAuth
 
+
 def get_digest_auth():
     return HTTPDigestAuth("myuser", "mypassword")
 ```
@@ -278,6 +279,7 @@ You can also use custom auth classes for non-standard schemes:
 ```python
 from requests.auth import AuthBase
 
+
 class TokenAuth(AuthBase):
     def __init__(self, token):
         self.token = token
@@ -286,6 +288,7 @@ class TokenAuth(AuthBase):
         r.headers["X-api-token"] = f"Token {self.token}"
         return r
 
+
 def get_token_auth():
     return TokenAuth("abc123")
 ```

docs/source/plugins.md

@@ -111,7 +111,6 @@ requirements:
 
 ```python
 class MySession:
-
     def __enter__(self):
         pass
 

example/allure/docker-compose.yaml

@@ -5,7 +5,7 @@ services:
       context: ../http
       dockerfile: Dockerfile
     ports:
-      - "5000:5000"
+      - "5004:5000"
 
   # Only used to turn the results written by allure-pytest into a report - not
   # started by 'docker compose up', run it with 'docker compose run --rm allure'

example/allure/tests/common.yaml

@@ -3,7 +3,7 @@ name: Allure example includes
 description: variables and the login stage used by the tests in this folder
 
 variables:
-  service: http://localhost:5000
+  service: http://localhost:5004
   user:
     user: test-user
     pass: correct-password

pyproject.toml

@@ -148,6 +148,11 @@ exclude = [
 
 [tool.coverage.run]
 branch = false
+# Measure python subprocesses too (e.g. the pytester-spawned pytest runs in
+# tests/integration/test_run_integration_suite.py); parallel data files are
+# combined by pytest-cov / 'coverage combine'
+parallel = true
+patch = ["subprocess"]
 omit = [
     "tests/*",
     ".eggs/*",
@@ -165,9 +170,12 @@ tavern = [
 ]
 
 [tool.pytest.ini_options]
-testpaths = ["tavern", "tests/unit"]
+testpaths = ["tavern", "tests/unit", "tests/integration"]
 addopts = [
     "--doctest-modules",
+    # Stop --doctest-modules importing every conftest.py as a module: with
+    # multiple non-package conftest.py files the basenames clash in sys.modules
+    "--ignore-glob=*/conftest.py",
     "-r", "xs",
     "-vv",
     "--strict-markers",
@@ -176,12 +184,18 @@ addopts = [
     "-m", "not do_not_run"
 ]
 norecursedirs = [
-    ".git",
-    ".tox",
-    ".venv",
-    "example",
-    "node_modules",
+    # pytest's defaults - setting this key replaces rather than extends them
+    "*.egg",
+    ".*",
+    "_darcs",
+    "build",
+    "CVS",
     "dist",
+    "node_modules",
+    "venv",
+    "{arch}",
+    # ours
+    "example",
     "docs",
 ]
 markers = [

scripts/coverage.sh

@@ -2,9 +2,6 @@
 
 set -ex
 
-tox -c tox-integration.ini -e py311-generic
-tox -c tox-integration.ini -e py311-mqtt
-tox -e py311
-
-coverage combine --append .coverage tests/integration/.coverage example/mqtt/.coverage
-coverage report -m
+# Runs the unit tests and the tests/integration suite in one top-level pytest
+# run and reports combined coverage for tavern.
+uv run pytest --cov tavern --cov-report=term-missing "$@"

tests/integration/README.md

@@ -6,5 +6,15 @@ matching that don't nicely slot into the examples. Essentially, tests in this
 folder will typically consist of one stage (unless multi-stage functionality is
 being tested), and will not require logging in.
 
-For the time being, all the random tests are just being put into the same
-server.py and will be run with docker.
+All the tests run against the flask app in `server.py`, in one of two modes:
+
+- **Plain `pytest`** (the default, also part of the top-level test run): the
+  server is started in a background thread on a random port (one per
+  pytest-xdist worker, if used) and each `test_*.tavern.yaml` file is run in a
+  pytest subprocess via the `pytester` plugin — see
+  `test_run_integration_suite.py`. No docker required.
+- **`tox -c tox-integration.ini -e py3-generic`** (or `py3-noextra`): the server
+  runs under docker compose and the yaml files are collected directly, which
+  also exercises the `tavern-ci` CLI and `tavern.core.run` library entrypoints.
+  This mode sets `TAVERN_INTEGRATION_ALLOW_COLLECT=1` (see `conftest.py`) to
+  collect the yaml files in-process instead of via the pytester runner.

tests/integration/common.yaml

@@ -3,7 +3,7 @@ name: test includes
 description: used for testing against local server
 
 variables:
-  host: http://localhost:5003
+  host: "{tavern.env_vars.TEST_HOST}"
   first_part: "nested"
   second_part: "{tavern.env_vars.SECOND_URL_PART}"
 

tests/integration/conftest.py

@@ -4,9 +4,29 @@
 
 import pytest
 from box import Box
+from server import run_in_background
 
 from tavern._core import exceptions
 
+if not os.environ.get("TAVERN_INTEGRATION_ALLOW_COLLECT"):
+    # Plain top-level pytest: run the yaml suite only via the pytester runner
+    # (subprocess), never in-process — tavern patches the yaml parser globally
+    # and the suite needs --tavern-global-cfg and a running server.
+    collect_ignore_glob = ["test_*.tavern.yaml"]
+else:
+    # tox-integration generic/noextra (docker server on port 5003) or the
+    # pytester subprocess: yaml files run directly, so the runner must not also
+    # run (it would run the whole suite a second time).
+    collect_ignore = ["test_run_integration_suite.py"]
+
+
+@pytest.fixture(scope="session")
+def integration_server():
+    """Base URL of the flask server from server.py, running in a background
+    thread on a random port (one server per pytest-xdist worker)."""
+    with run_in_background() as port:
+        yield f"http://localhost:{port}"
+
 
 @pytest.fixture
 def str_fixture():
@@ -32,7 +52,7 @@ def autouse_thing():
 
 @pytest.fixture(scope="session", autouse=True)
 def fixture_echo_url():
-    return "http://localhost:5003/echo"
+    return os.environ.get("TEST_HOST", "http://localhost:5003") + "/echo"
 
 
 @pytest.fixture(scope="session", autouse=True, name="autouse_thing_named")

tests/integration/extra.yaml

@@ -3,6 +3,6 @@ name: global includes
 description: Used for testing global config tests
 
 variables:
-  global_host: http://localhost:5003
+  global_host: "{tavern.env_vars.TEST_HOST}"
   test_string_1: "hello"
   test_string_2: "{tavern.env_vars.SECOND_URL_PART}"

tests/integration/global_cfg.yaml

@@ -3,7 +3,7 @@ name: test global includes
 description: used for testing against local server
 
 variables:
-  global_host: http://localhost:5003
+  global_host: "{tavern.env_vars.TEST_HOST}"
   global_test_string_1: abc
   global_test_string_2: "{tavern.env_vars.SECOND_URL_PART}"
 

tests/integration/server.py

@@ -1,17 +1,22 @@
 import base64
+import contextlib
 import gzip
 import itertools
 import json
 import math
 import mimetypes
 import os
+import threading
 import time
+import urllib.error
+import urllib.request
 import uuid
 from datetime import datetime, timedelta
 from hashlib import sha512
 from urllib.parse import unquote_plus, urlencode
 
 import jwt
+import werkzeug.serving
 from box import Box
 from flask import Flask, Response, jsonify, make_response, redirect, request, session
 from flask_httpauth import HTTPDigestAuth
@@ -33,7 +38,7 @@ def get_digest_password(username):
 @app.route("/token", methods=["GET"])
 def token():
     return (
-        '<div><a src="http://127.0.0.1:5003/verify?token=c9bb34ba-131b-11e8-b642-0ed5f89f718b">Link</a></div>',
+        f'<div><a src="{request.host_url}verify?token=c9bb34ba-131b-11e8-b642-0ed5f89f718b">Link</a></div>',
         200,
     )
 
@@ -575,3 +580,33 @@ def ascii_table():
 def expected_text():
     """Echoes back plain text from request body"""
     return Response(request.get_data(as_text=True), content_type="text/plain")
+
+
+def _wait_until_ready(url: str, timeout: float = 10.0) -> None:
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        try:
+            urllib.request.urlopen(url, timeout=0.5)
+        except (urllib.error.URLError, OSError):
+            time.sleep(0.05)
+        else:
+            return
+    raise RuntimeError(f"server not ready at {url} within {timeout}s")
+
+
+@contextlib.contextmanager
+def run_in_background():
+    """Run the app in a daemon thread on an OS-assigned free port for the
+    duration of the with block, yielding the port. Using a random port means
+    concurrent runs (e.g. pytest-xdist workers) never clash."""
+    httpd = werkzeug.serving.make_server("127.0.0.1", 0, app, threaded=True)
+    port = httpd.server_port
+
+    thread = threading.Thread(target=httpd.serve_forever, daemon=True)
+    thread.start()
+    try:
+        _wait_until_ready(f"http://localhost:{port}/headers")
+        yield port
+    finally:
+        httpd.shutdown()
+        thread.join(timeout=5)

tests/integration/test_parametrize.tavern.yaml

@@ -268,7 +268,7 @@ marks:
 stages:
   - name: Echo back a unicode value and make sure it matches
     request:
-      url: "http://localhost:5003/echo"
+      url: "{tavern.env_vars.TEST_HOST}/echo"
       method: POST
       json:
         value: "{to_send}"