diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 323819b55..b64ba7faf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -113,11 +113,15 @@ jobs: - tests/mock_vws/test_update_target.py::TestWidth - tests/mock_vws/test_update_target.py::TestInactiveProject - tests/mock_vws/test_requests_mock_usage.py + - tests/mock_vws/test_respx_mock_usage.py + - tests/mock_vws/test_target_validators.py - tests/mock_vws/test_flask_app_usage.py - tests/mock_vws/test_vumark_generation_api.py - tests/mock_vws/test_docker.py - README.rst - docs/source/basic-example.rst + - docs/source/httpx-example.rst + - ci/ steps: - uses: actions/checkout@v6 diff --git a/ci/test_custom_linters.py b/ci/test_custom_linters.py index c02fcacc6..0817a0896 100644 --- a/ci/test_custom_linters.py +++ b/ci/test_custom_linters.py @@ -1,15 +1,11 @@ """Custom lint tests.""" from pathlib import Path -from typing import TYPE_CHECKING import pytest import yaml from beartype import beartype -if TYPE_CHECKING: - from collections.abc import Iterable - @beartype def _ci_patterns(*, repository_root: Path) -> set[str]: @@ -23,32 +19,55 @@ def _ci_patterns(*, repository_root: Path) -> set[str]: return ci_patterns +class _CollectPlugin: + """Pytest plugin that records collected node IDs.""" + + def __init__(self) -> None: + """Initialize an empty set of collected node IDs.""" + self.nodeids: set[str] = set() + + def pytest_collection_modifyitems( + self, + items: list[pytest.Item], + ) -> None: + """Record the node IDs of all collected items.""" + self.nodeids.update(item.nodeid for item in items) + + @beartype -def _tests_from_pattern( - *, - ci_pattern: str, - capsys: pytest.CaptureFixture[str], -) -> set[str]: - """From a CI pattern, get all tests ``pytest`` would collect.""" - # Clear the captured output. - capsys.readouterr() - tests: Iterable[str] = set() - pytest.main( +def _tests_from_pattern(*, ci_pattern: str) -> set[str]: + """From a CI pattern, get all tests ``pytest`` would collect. + + Uses a collection-hook plugin instead of parsing stdout: an in-process + ``pytest.main()`` installs its own output capture, so reading from + ``capsys`` would see an empty string and the test would pass vacuously. + """ + plugin = _CollectPlugin() + exit_code = pytest.main( args=[ - "-q", "--collect-only", - # If there are any warnings, these obscure the output. + # Disable pytest-retry to avoid: + # ``` + # ValueError: no option named 'filtered_exceptions' + # ``` + "-p", + "no:pytest-retry", + # Disable warnings to avoid many instances of: + # ``` + # Unknown config option: retry_delay + # ``` "--disable-warnings", ci_pattern, ], + plugins=[plugin], + ) + # Fail loudly on collection errors (import failures, syntax errors, etc.) + # rather than silently using whatever items were captured before the + # crash. + assert exit_code == pytest.ExitCode.OK, ( + f"Collection for {ci_pattern!r} failed with exit code {exit_code}." ) - data = capsys.readouterr().out - for line in data.splitlines(): - # We filter empty lines and lines which look like - # "9 tests collected in 0.01s". - if line and "collected in" not in line: - tests = {*tests, line} - return set(tests) + return plugin.nodeids def test_ci_patterns_valid(request: pytest.FixtureRequest) -> None: @@ -60,31 +79,13 @@ def test_ci_patterns_valid(request: pytest.FixtureRequest) -> None: ci_patterns = _ci_patterns(repository_root=request.config.rootpath) for ci_pattern in ci_patterns: - collect_only_result = pytest.main( - args=[ - "--collect-only", - ci_pattern, - # Disable pytest-retry to avoid: - # ``` - # ValueError: no option named 'filtered_exceptions' - # ```` - "-p", - "no:pytest-retry", - # Disable warnings to avoid many instances of: - # ``` - # Unknown config option: retry_delay - # ``` - "--disable-warnings", - ], - ) - + tests = _tests_from_pattern(ci_pattern=ci_pattern) message = f'"{ci_pattern}" does not match any tests.' - assert collect_only_result == 0, message + assert tests, message def test_tests_collected_once( *, - capsys: pytest.CaptureFixture[str], request: pytest.FixtureRequest, ) -> None: """Each test in the test suite is collected exactly once. @@ -95,12 +96,9 @@ def test_tests_collected_once( tests_to_patterns: dict[str, set[str]] = {} for pattern in ci_patterns: - tests = _tests_from_pattern(ci_pattern=pattern, capsys=capsys) + tests = _tests_from_pattern(ci_pattern=pattern) for test in tests: - if test in tests_to_patterns: - tests_to_patterns[test].add(pattern) - else: - tests_to_patterns[test] = {pattern} + tests_to_patterns.setdefault(test, set()).add(pattern) for test_name, patterns in tests_to_patterns.items(): message = ( @@ -110,6 +108,6 @@ def test_tests_collected_once( ) assert len(patterns) == 1, message - all_tests = _tests_from_pattern(ci_pattern=".", capsys=capsys) + all_tests = _tests_from_pattern(ci_pattern=".") assert tests_to_patterns.keys() - all_tests == set() assert all_tests - tests_to_patterns.keys() == set()