From b6ccadb7dcfcc78b6b497e1d7b60e6dd2dd517a4 Mon Sep 17 00:00:00 2001 From: Arthur Pastel Date: Wed, 9 Nov 2022 16:08:30 +0100 Subject: [PATCH] feat: allow running along with pytest-benchmarks --- pyproject.toml | 1 + src/pytest_codspeed/plugin.py | 82 ++++++++++++++++++++++++++--------- tests/conftest.py | 7 +++ tests/test_pytest_plugin.py | 27 +++++++----- 4 files changed, 86 insertions(+), 31 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8282f3d..5b729dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ dev = [ "pytest-cov ~= 4.0.0", "ruff ~= 0.0.100", ] +compatibility = ["pytest-benchmarks ~= 3.4.1"] [project.entry-points] pytest11 = { codspeed = "pytest_codspeed.plugin" } diff --git a/src/pytest_codspeed/plugin.py b/src/pytest_codspeed/plugin.py index 92995cc..f49f4f3 100644 --- a/src/pytest_codspeed/plugin.py +++ b/src/pytest_codspeed/plugin.py @@ -1,8 +1,10 @@ import os -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Callable, List, Optional +import pkgutil +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Callable, List, Optional, Tuple import pytest +from _pytest.fixtures import FixtureManager from . import __version__ from ._wrapper import get_lib @@ -10,6 +12,8 @@ if TYPE_CHECKING: from ._wrapper import LibType +IS_PYTEST_BENCHMARK_INSTALLED = pkgutil.find_loader("pytest_benchmark") is not None + @pytest.hookimpl(trylast=True) def pytest_addoption(parser: "pytest.Parser"): @@ -26,8 +30,9 @@ def pytest_addoption(parser: "pytest.Parser"): class CodSpeedPlugin: is_codspeed_enabled: bool should_measure: bool - lib: Optional["LibType"] = None - benchmark_count: int = 0 + lib: Optional["LibType"] + disabled_plugins: Tuple[str, ...] + benchmark_count: int = field(default=0, hash=False, compare=False) PLUGIN_NAME = "codspeed_plugin" @@ -37,7 +42,7 @@ def get_plugin(config: "pytest.Config") -> "CodSpeedPlugin": return config.pluginmanager.get_plugin(PLUGIN_NAME) -@pytest.hookimpl() +@pytest.hookimpl(tryfirst=True) def pytest_configure(config: "pytest.Config"): config.addinivalue_line( "markers", "codspeed_benchmark: mark an entire test for codspeed benchmarking" @@ -45,27 +50,61 @@ def pytest_configure(config: "pytest.Config"): config.addinivalue_line( "markers", "benchmark: mark an entire test for codspeed benchmarking" ) + is_codspeed_enabled = ( + config.getoption("--codspeed") or os.environ.get("CODSPEED_ENV") is not None + ) + should_measure = os.environ.get("CODSPEED_ENV") is not None + lib = get_lib() if should_measure else None + disabled_plugins: List[str] = [] + # Disable pytest-benchmark if codspeed is enabled + if is_codspeed_enabled and IS_PYTEST_BENCHMARK_INSTALLED: + object.__setattr__(config.option, "benchmark_disable", True) + config.pluginmanager.set_blocked("pytest-benchmark") + disabled_plugins.append("pytest-benchmark") + plugin = CodSpeedPlugin( - is_codspeed_enabled=config.getoption("--codspeed") - or os.environ.get("CODSPEED_ENV") is not None, - should_measure=os.environ.get("CODSPEED_ENV") is not None, + is_codspeed_enabled=is_codspeed_enabled, + should_measure=should_measure, + lib=lib, + disabled_plugins=tuple(disabled_plugins), ) - if plugin.should_measure: - plugin.lib = get_lib() config.pluginmanager.register(plugin, PLUGIN_NAME) +def pytest_plugin_registered(plugin, manager: "pytest.PytestPluginManager"): + """Patch the benchmark fixture to use the codspeed one if codspeed is enabled""" + if IS_PYTEST_BENCHMARK_INSTALLED and isinstance(plugin, FixtureManager): + fixture_manager = plugin + codspeed_plugin: CodSpeedPlugin = manager.get_plugin(PLUGIN_NAME) + if codspeed_plugin.is_codspeed_enabled: + codspeed_benchmark_fixtures = plugin.getfixturedefs( + "codspeed_benchmark", "" + ) + assert codspeed_benchmark_fixtures is not None + fixture_manager._arg2fixturedefs[ + "__benchmark" + ] = fixture_manager._arg2fixturedefs["benchmark"] + fixture_manager._arg2fixturedefs["benchmark"] = list( + codspeed_benchmark_fixtures + ) + + @pytest.hookimpl(trylast=True) def pytest_report_header(config: "pytest.Config"): out = [f"codspeed: {__version__}"] plugin = get_plugin(config) if plugin.is_codspeed_enabled and not plugin.should_measure: out.append( - "\033[93m" + "\033[1m" "NOTICE: codspeed is enabled, but no performance measurement" " will be made since it's running in an unknown environment." "\033[0m" ) + if len(plugin.disabled_plugins) > 0: + out.append( + "\033[93mCodSpeed had to disable the following plugins: " + f"{', '.join(plugin.disabled_plugins)}\033[0m" + ) return "\n".join(out) @@ -101,11 +140,12 @@ def pytest_collection_modifyitems( items[:] = selected -@pytest.hookimpl() +@pytest.hookimpl(trylast=True) def pytest_runtest_call(item: "pytest.Item"): plugin = get_plugin(item.config) + if not plugin.is_codspeed_enabled or not should_benchmark_item(item): - item.runtest() + return # Avoid running the test multiple times when codspeed is disabled else: plugin.benchmark_count += 1 if "benchmark" in getattr(item, "fixturenames", []): @@ -134,7 +174,7 @@ def pytest_sessionfinish(session: "pytest.Session", exitstatus): ) -@pytest.fixture +@pytest.fixture(scope="function") def codspeed_benchmark(request: "pytest.FixtureRequest") -> Callable: plugin = get_plugin(request.config) @@ -152,9 +192,11 @@ def run(func: Callable[..., Any], *args: Any): return run -@pytest.fixture -def benchmark(codspeed_benchmark): - """ - Compatibility with pytest-benchmark - """ - return codspeed_benchmark +if not IS_PYTEST_BENCHMARK_INSTALLED: + + @pytest.fixture(scope="function") + def benchmark(codspeed_benchmark, request: "pytest.FixtureRequest"): + """ + Compatibility with pytest-benchmark + """ + return codspeed_benchmark diff --git a/tests/conftest.py b/tests/conftest.py index c6481d5..6bc3651 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1 +1,8 @@ +import pkgutil +import sys + pytest_plugins = ["pytester"] + +if pkgutil.find_loader("pytest_benchmark") is not None: + pytest_plugins.append("pytest_benchmark") + print("NOTICE: Testing with pytest-benchmark compatibility", file=sys.stderr) diff --git a/tests/test_pytest_plugin.py b/tests/test_pytest_plugin.py index 717438a..41aea7f 100644 --- a/tests/test_pytest_plugin.py +++ b/tests/test_pytest_plugin.py @@ -1,4 +1,3 @@ -import os import shutil from contextlib import contextmanager @@ -10,13 +9,17 @@ ) -@contextmanager -def codspeed_env(): - os.environ["CODSPEED_ENV"] = "1" - try: - yield - finally: - del os.environ["CODSPEED_ENV"] +@pytest.fixture(scope="function") +def codspeed_env(monkeypatch): + @contextmanager + def ctx_manager(): + monkeypatch.setenv("CODSPEED_ENV", "1") + try: + yield + finally: + monkeypatch.delenv("CODSPEED_ENV", raising=False) + + return ctx_manager def test_plugin_enabled_without_env(pytester: pytest.Pytester) -> None: @@ -42,7 +45,7 @@ def _(): @skip_without_valgrind -def test_plugin_enabled_by_env(pytester: pytest.Pytester) -> None: +def test_plugin_enabled_by_env(pytester: pytest.Pytester, codspeed_env) -> None: pytester.makepyfile( """ def test_some_addition_performance(benchmark): @@ -57,7 +60,7 @@ def _(): @skip_without_valgrind -def test_plugin_enabled_and_env(pytester: pytest.Pytester) -> None: +def test_plugin_enabled_and_env(pytester: pytest.Pytester, codspeed_env) -> None: pytester.makepyfile( """ def test_some_addition_performance(benchmark): @@ -85,7 +88,9 @@ def _(): @skip_without_valgrind -def test_plugin_enabled_nothing_to_benchmark(pytester: pytest.Pytester) -> None: +def test_plugin_enabled_nothing_to_benchmark( + pytester: pytest.Pytester, codspeed_env +) -> None: pytester.makepyfile( """ def test_some_addition_performance():