scrapy/conftest.py at 2.11-dev · lighthouse-intelligence/scrapy · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
import platform
import sys
from pathlib import Path

import pytest
from twisted import version as twisted_version
from twisted.python.versions import Version
from twisted.web.http import H2_ENABLED

from scrapy.utils.reactor import install_reactor
from tests.keys import generate_keys


def _py_files(folder):
    return (str(p) for p in Path(folder).rglob("*.py"))


collect_ignore = [
    # not a test, but looks like a test
    "scrapy/utils/testsite.py",
    "tests/ftpserver.py",
    "tests/mockserver.py",
    "tests/pipelines.py",
    "tests/spiders.py",
    # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
    *_py_files("tests/CrawlerProcess"),
    # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
    *_py_files("tests/CrawlerRunner"),
]

with Path("tests/ignores.txt").open(encoding="utf-8") as reader:
    for line in reader:
        file_path = line.strip()
        if file_path and file_path[0] != "#":
            collect_ignore.append(file_path)

if not H2_ENABLED:
    collect_ignore.extend(
        (
            "scrapy/core/downloader/handlers/http2.py",
            *_py_files("scrapy/core/http2"),
        )
    )


@pytest.fixture()
def chdir(tmpdir):
    """Change to pytest-provided temporary directory"""
    tmpdir.chdir()


def pytest_addoption(parser):
    parser.addoption(
        "--reactor",
        default="default",
        choices=["default", "asyncio"],
    )


@pytest.fixture(scope="class")
def reactor_pytest(request):
    if not request.cls:
        # doctests
        return
    request.cls.reactor_pytest = request.config.getoption("--reactor")
    return request.cls.reactor_pytest


@pytest.fixture(autouse=True)
def only_asyncio(request, reactor_pytest):
    if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio":
        pytest.skip("This test is only run with --reactor=asyncio")


@pytest.fixture(autouse=True)
def only_not_asyncio(request, reactor_pytest):
    if (
        request.node.get_closest_marker("only_not_asyncio")
        and reactor_pytest == "asyncio"
    ):
        pytest.skip("This test is only run without --reactor=asyncio")


@pytest.fixture(autouse=True)
def requires_uvloop(request):
    if not request.node.get_closest_marker("requires_uvloop"):
        return
    if sys.implementation.name == "pypy":
        pytest.skip("uvloop does not support pypy properly")
    if platform.system() == "Windows":
        pytest.skip("uvloop does not support Windows")
    if twisted_version == Version("twisted", 21, 2, 0):
        pytest.skip("https://twistedmatrix.com/trac/ticket/10106")
    if sys.version_info >= (3, 12):
        pytest.skip("uvloop doesn't support Python 3.12 yet")


def pytest_configure(config):
    if config.getoption("--reactor") == "asyncio":
        install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")


# Generate localhost certificate files, needed by some tests
generate_keys()