From 2a40279691d67250c15018e42cda65a870e1a039 Mon Sep 17 00:00:00 2001 From: fjetter Date: Tue, 8 Jun 2021 19:50:34 +0200 Subject: [PATCH 1/9] Add timeout options --- pytest_asyncio/plugin.py | 45 ++++++++++++++++++++++++++++++++++++++-- tests/test_simple.py | 12 +++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/pytest_asyncio/plugin.py b/pytest_asyncio/plugin.py index 7665ff4d..dd8ee5be 100644 --- a/pytest_asyncio/plugin.py +++ b/pytest_asyncio/plugin.py @@ -34,6 +34,13 @@ def pytest_configure(config): ) +def pytest_addoption(parser): + group = parser.getgroup("asyncio") + help_ = "Timeout in seconds after which the test coroutine shall be cancelled and marked as failed" + group.addoption("--asyncio-timeout", dest="asyncio_timeout", type=float, help=help_) + parser.addini("asyncio_timeout", help=help_) + + @pytest.mark.tryfirst def pytest_pycollect_makeitem(collector, name, obj): """A pytest hook to collect asyncio coroutines.""" @@ -163,6 +170,33 @@ async def setup(): yield +def get_timeout(obj): + """ + Get the timeout for the provided test function. + + Priority: + + * Marker keyword arguments `asyncio_timeout` and `timeout` + * CLI + * INI file + """ + marker = obj.get_closest_marker("asyncio") + timeout = marker.kwargs.get("asyncio_timeout", marker.kwargs.get("timeout")) + + if not timeout: + timeout = obj._request.config.getvalue("asyncio_timeout") + if not timeout: + timeout = obj._request.config.getini("asyncio_timeout") + + if timeout: + try: + return float(timeout) + except: + raise ValueError( + f"Invalid timeout (asyncio_timeout) provided. Got {timeout} but expected a float-like." + ) + + @pytest.hookimpl(tryfirst=True, hookwrapper=True) def pytest_pyfunc_call(pyfuncitem): """ @@ -170,19 +204,23 @@ def pytest_pyfunc_call(pyfuncitem): function call. """ if "asyncio" in pyfuncitem.keywords: + timeout = get_timeout(pyfuncitem) if getattr(pyfuncitem.obj, "is_hypothesis_test", False): pyfuncitem.obj.hypothesis.inner_test = wrap_in_sync( pyfuncitem.obj.hypothesis.inner_test, _loop=pyfuncitem.funcargs["event_loop"], + timeout=timeout, ) else: pyfuncitem.obj = wrap_in_sync( - pyfuncitem.obj, _loop=pyfuncitem.funcargs["event_loop"] + pyfuncitem.obj, + _loop=pyfuncitem.funcargs["event_loop"], + timeout=timeout, ) yield -def wrap_in_sync(func, _loop): +def wrap_in_sync(func, _loop, timeout): """Return a sync wrapper around an async function executing it in the current event loop.""" @@ -190,7 +228,10 @@ def wrap_in_sync(func, _loop): def inner(**kwargs): coro = func(**kwargs) if coro is not None: + if timeout: + coro = asyncio.wait_for(coro, timeout=timeout) task = asyncio.ensure_future(coro, loop=_loop) + try: _loop.run_until_complete(task) except BaseException: diff --git a/tests/test_simple.py b/tests/test_simple.py index 854faaf3..ba2c2c41 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -145,3 +145,15 @@ async def test_no_warning_on_skip(): def test_async_close_loop(event_loop): event_loop.close() return "ok" + + +@pytest.mark.asyncio(timeout=0.1) +@pytest.mark.xfail(strict=True) +async def test_timeout(): + await asyncio.sleep(1) + + +@pytest.mark.asyncio(asyncio_timeout=0.1) +@pytest.mark.xfail(strict=True) +async def test_timeout_2(): + await asyncio.sleep(1) From c68680df6bc9d505b4834c37bdc24c4bd9f6f0bc Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 9 Jan 2022 13:27:19 +0200 Subject: [PATCH 2/9] Update test_simple.py --- tests/test_simple.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index ba2c2c41..4e77c5f2 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -148,12 +148,6 @@ def test_async_close_loop(event_loop): @pytest.mark.asyncio(timeout=0.1) -@pytest.mark.xfail(strict=True) +@pytest.mark.xfail(strict=True, raises=asyncio.TimeoutError) async def test_timeout(): await asyncio.sleep(1) - - -@pytest.mark.asyncio(asyncio_timeout=0.1) -@pytest.mark.xfail(strict=True) -async def test_timeout_2(): - await asyncio.sleep(1) From e534a2158a45751a8fbcc431f12bde0247dcaea9 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 9 Jan 2022 13:43:31 +0200 Subject: [PATCH 3/9] Fix tests --- pytest_asyncio/plugin.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/pytest_asyncio/plugin.py b/pytest_asyncio/plugin.py index cb03b428..48f08d2a 100644 --- a/pytest_asyncio/plugin.py +++ b/pytest_asyncio/plugin.py @@ -41,6 +41,11 @@ class Mode(str, enum.Enum): auto-handling is disabled but pytest_asyncio.fixture usage is not enforced """ +ASYNCIO_TIMEOUT_HELP = """\ +Timeout in seconds after which the test coroutine \ +shall be cancelled and marked as failed, 0 for no-timeout +""" + def pytest_addoption(parser, pluginmanager): group = parser.getgroup("asyncio") @@ -57,6 +62,10 @@ def pytest_addoption(parser, pluginmanager): type="string", default="legacy", ) + group.addoption("--asyncio-timeout", dest="asyncio_timeout", type=float, help=ASYNCIO_TIMEOUT_HELP, default=None) + parser.addini("asyncio_timeout", type="string", help="default value for --asyncio-timeout",default=0) + + def fixture(fixture_function=None, **kwargs): @@ -127,13 +136,6 @@ def _issue_warning_captured(warning, hook, *, stacklevel=1): ) -def pytest_addoption(parser): - group = parser.getgroup("asyncio") - help_ = "Timeout in seconds after which the test coroutine shall be cancelled and marked as failed" - group.addoption("--asyncio-timeout", dest="asyncio_timeout", type=float, help=help_) - parser.addini("asyncio_timeout", help=help_) - - @pytest.mark.tryfirst def pytest_pycollect_makeitem(collector, name, obj): """A pytest hook to collect asyncio coroutines.""" @@ -317,13 +319,16 @@ def get_timeout(obj): if not timeout: timeout = obj._request.config.getini("asyncio_timeout") - if timeout: - try: - return float(timeout) - except: - raise ValueError( - f"Invalid timeout (asyncio_timeout) provided. Got {timeout} but expected a float-like." - ) + if not timeout: + return None + + try: + return float(timeout) + except (TypeError, ValueError): + raise ValueError( + f"Invalid asyncio timeout {timeout!r} provided, " + "a float-like value is expected." + ) from None @pytest.hookimpl(tryfirst=True, hookwrapper=True) From 092396e75249b287fbc072f13ac4b97c06a2d10d Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 9 Jan 2022 14:30:00 +0200 Subject: [PATCH 4/9] Reformat --- pytest_asyncio/plugin.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pytest_asyncio/plugin.py b/pytest_asyncio/plugin.py index 48f08d2a..ba2b0b92 100644 --- a/pytest_asyncio/plugin.py +++ b/pytest_asyncio/plugin.py @@ -62,10 +62,19 @@ def pytest_addoption(parser, pluginmanager): type="string", default="legacy", ) - group.addoption("--asyncio-timeout", dest="asyncio_timeout", type=float, help=ASYNCIO_TIMEOUT_HELP, default=None) - parser.addini("asyncio_timeout", type="string", help="default value for --asyncio-timeout",default=0) - - + group.addoption( + "--asyncio-timeout", + dest="asyncio_timeout", + type=float, + help=ASYNCIO_TIMEOUT_HELP, + default=None, + ) + parser.addini( + "asyncio_timeout", + type="string", + help="default value for --asyncio-timeout", + default=0, + ) def fixture(fixture_function=None, **kwargs): From 83d5ad9fcc64986d757556077ef8c6e8aefbe0e2 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 9 Jan 2022 14:30:57 +0200 Subject: [PATCH 5/9] rename --- pytest_asyncio/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytest_asyncio/plugin.py b/pytest_asyncio/plugin.py index ba2b0b92..4ee6fa1f 100644 --- a/pytest_asyncio/plugin.py +++ b/pytest_asyncio/plugin.py @@ -310,7 +310,7 @@ async def setup(): yield -def get_timeout(obj): +def _get_timeout(obj): """ Get the timeout for the provided test function. @@ -348,7 +348,7 @@ def pytest_pyfunc_call(pyfuncitem): Wraps marked tests in a synchronous function where the wrapped test coroutine is executed in an event loop. """ if "asyncio" in pyfuncitem.keywords: - timeout = get_timeout(pyfuncitem) + timeout = _get_timeout(pyfuncitem) if getattr(pyfuncitem.obj, "is_hypothesis_test", False): pyfuncitem.obj.hypothesis.inner_test = wrap_in_sync( pyfuncitem.obj.hypothesis.inner_test, From 5bb0f6ae3f638099513b7620ef95d6251f3da896 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 9 Jan 2022 15:07:59 +0200 Subject: [PATCH 6/9] Add a test --- tests/test_simple.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_simple.py b/tests/test_simple.py index a08be690..dae3392b 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -246,3 +246,9 @@ def test_async_close_loop(event_loop): @pytest.mark.xfail(strict=True, raises=asyncio.TimeoutError) async def test_timeout(): await asyncio.sleep(1) + + +@pytest.mark.asyncio(timeout="abc") +@pytest.mark.xfail(strict=True, raises=ValueError) +async def test_timeout_not_numeric(): + await asyncio.sleep(1) From 1009cc047e6b58f77cc6156d48356536a48afbb5 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 9 Jan 2022 15:24:53 +0200 Subject: [PATCH 7/9] Add tests --- tests/test_simple.py | 12 -------- tests/test_timeout.py | 64 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 12 deletions(-) create mode 100644 tests/test_timeout.py diff --git a/tests/test_simple.py b/tests/test_simple.py index dae3392b..31204b6c 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -240,15 +240,3 @@ async def test_no_warning_on_skip(): def test_async_close_loop(event_loop): event_loop.close() return "ok" - - -@pytest.mark.asyncio(timeout=0.1) -@pytest.mark.xfail(strict=True, raises=asyncio.TimeoutError) -async def test_timeout(): - await asyncio.sleep(1) - - -@pytest.mark.asyncio(timeout="abc") -@pytest.mark.xfail(strict=True, raises=ValueError) -async def test_timeout_not_numeric(): - await asyncio.sleep(1) diff --git a/tests/test_timeout.py b/tests/test_timeout.py new file mode 100644 index 00000000..b4d285b1 --- /dev/null +++ b/tests/test_timeout.py @@ -0,0 +1,64 @@ +import asyncio +from textwrap import dedent + +import pytest + +pytest_plugins = "pytester" + + +@pytest.mark.asyncio(timeout=0.01) +@pytest.mark.xfail(strict=True, raises=asyncio.TimeoutError) +async def test_timeout(): + await asyncio.sleep(1) + + +@pytest.mark.asyncio(timeout=0) +async def test_timeout_disabled(): + await asyncio.sleep(0.01) + + +@pytest.mark.asyncio(timeout="abc") +@pytest.mark.xfail(strict=True, raises=ValueError) +async def test_timeout_not_numeric(): + await asyncio.sleep(1) + + +def test_timeout_cmdline(pytester): + pytester.makepyfile( + dedent( + """\ + import asyncio + import pytest + + pytest_plugins = 'pytest_asyncio' + + @pytest.mark.asyncio + @pytest.mark.xfail(strict=True, raises=asyncio.TimeoutError) + async def test_a(): + await asyncio.sleep(1) + """ + ) + ) + result = pytester.runpytest("--asyncio-timeout=0.01", "--asyncio-mode=strict") + result.assert_outcomes(xfailed=1) + + +def test_timeout_cfg(pytester): + pytester.makepyfile( + dedent( + """\ + import asyncio + import pytest + + pytest_plugins = 'pytest_asyncio' + + @pytest.mark.asyncio + @pytest.mark.xfail(strict=True, raises=asyncio.TimeoutError) + async def test_a(): + await asyncio.sleep(1) + """ + ) + ) + pytester.makefile(".ini", pytest="[pytest]\nasyncio_timeout = 0.01\n") + result = pytester.runpytest("--asyncio-mode=strict") + result.assert_outcomes(xfailed=1) From ba587acd8b6d159d13b040aef08e831c8935033a Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 10 Jan 2022 20:25:03 +0200 Subject: [PATCH 8/9] Work on --- README.rst | 6 ++++ pytest_asyncio/plugin.py | 30 +++++++++++++----- tests/test_timeout.py | 67 +++++++++++++++++++++++++++++++--------- 3 files changed, 82 insertions(+), 21 deletions(-) diff --git a/README.rst b/README.rst index 0b35000b..d0152d93 100644 --- a/README.rst +++ b/README.rst @@ -247,6 +247,12 @@ automatically to *async* test functions. .. |pytestmark| replace:: ``pytestmark`` .. _pytestmark: http://doc.pytest.org/en/latest/example/markers.html#marking-whole-classes-or-modules +Timeout protection +------------------ + +Sometime tests can work much slowly than expected or even hang. + + Note about unittest ------------------- diff --git a/pytest_asyncio/plugin.py b/pytest_asyncio/plugin.py index 24c862b8..9d94ead3 100644 --- a/pytest_asyncio/plugin.py +++ b/pytest_asyncio/plugin.py @@ -207,7 +207,8 @@ def pytest_fixture_setup(fixturedef, request): yield return - config = request.node.config + node = request.node + config = node.config asyncio_mode = _get_asyncio_mode(config) if not _has_explicit_asyncio_mark(func): @@ -253,13 +254,18 @@ def wrapper(*args, **kwargs): gen_obj = generator(*args, **kwargs) async def setup(): - res = await gen_obj.__anext__() - return res + node._asyncio_task = asyncio.get_running_loop() + try: + res = await gen_obj.__anext__() + return res + finally: + node._asyncio_task = None def finalizer(): """Yield again, to finalize.""" async def async_finalizer(): + node._asyncio_task = asyncio.get_running_loop() try: await gen_obj.__anext__() except StopAsyncIteration: @@ -268,6 +274,8 @@ async def async_finalizer(): msg = "Async generator fixture didn't stop." msg += "Yield only once." raise ValueError(msg) + finally: + node._asyncio_task = None loop.run_until_complete(async_finalizer()) @@ -287,8 +295,12 @@ def wrapper(*args, **kwargs): ) async def setup(): - res = await coro(*args, **kwargs) - return res + node._asyncio_task = asyncio.get_running_loop() + try: + res = await coro(*args, **kwargs) + return res + finally: + node._asyncio_task = None return loop.run_until_complete(setup()) @@ -338,12 +350,14 @@ def pytest_pyfunc_call(pyfuncitem): timeout = _get_timeout(pyfuncitem) if getattr(pyfuncitem.obj, "is_hypothesis_test", False): pyfuncitem.obj.hypothesis.inner_test = wrap_in_sync( + pyfuncitem, pyfuncitem.obj.hypothesis.inner_test, _loop=pyfuncitem.funcargs["event_loop"], timeout=timeout, ) else: pyfuncitem.obj = wrap_in_sync( + pyfuncitem, pyfuncitem.obj, _loop=pyfuncitem.funcargs["event_loop"], timeout=timeout, @@ -351,7 +365,7 @@ def pytest_pyfunc_call(pyfuncitem): yield -def wrap_in_sync(func, _loop, timeout): +def wrap_in_sync(node, func, _loop, timeout): """Return a sync wrapper around an async function executing it in the current event loop.""" @@ -367,7 +381,7 @@ def inner(**kwargs): if coro is not None: if timeout: coro = asyncio.wait_for(coro, timeout=timeout) - task = asyncio.ensure_future(coro, loop=_loop) + node._asyncio_task = task = asyncio.ensure_future(coro, loop=_loop) try: _loop.run_until_complete(task) @@ -378,6 +392,8 @@ def inner(**kwargs): if task.done() and not task.cancelled(): task.exception() raise + finally: + node._asyncio_task = None inner._raw_test_func = func return inner diff --git a/tests/test_timeout.py b/tests/test_timeout.py index b4d285b1..e40c5946 100644 --- a/tests/test_timeout.py +++ b/tests/test_timeout.py @@ -1,26 +1,65 @@ -import asyncio from textwrap import dedent -import pytest - pytest_plugins = "pytester" -@pytest.mark.asyncio(timeout=0.01) -@pytest.mark.xfail(strict=True, raises=asyncio.TimeoutError) -async def test_timeout(): - await asyncio.sleep(1) +def test_timeout_ok(pytester): + pytester.makepyfile( + dedent( + """\ + import asyncio + import pytest + + pytest_plugins = 'pytest_asyncio' + + @pytest.mark.asyncio(timeout=0.01) + @pytest.mark.xfail(strict=True, raises=asyncio.TimeoutError) + async def test_a(): + await asyncio.sleep(1) + """ + ) + ) + result = pytester.runpytest("--asyncio-mode=strict") + result.assert_outcomes(xfailed=1) + + +def test_timeout_disabled(pytester): + pytester.makepyfile( + dedent( + """\ + import asyncio + import pytest + + pytest_plugins = 'pytest_asyncio' + + @pytest.mark.asyncio(timeout=0) + async def test_a(): + await asyncio.sleep(0.01) + """ + ) + ) + result = pytester.runpytest("--asyncio-mode=strict") + result.assert_outcomes(passed=1) -@pytest.mark.asyncio(timeout=0) -async def test_timeout_disabled(): - await asyncio.sleep(0.01) +def test_timeout_not_numeric(pytester): + pytester.makepyfile( + dedent( + """\ + import asyncio + import pytest + pytest_plugins = 'pytest_asyncio' -@pytest.mark.asyncio(timeout="abc") -@pytest.mark.xfail(strict=True, raises=ValueError) -async def test_timeout_not_numeric(): - await asyncio.sleep(1) + @pytest.mark.asyncio(timeout="abc") + @pytest.mark.xfail(strict=True, raises=ValueError) + async def test_a(): + await asyncio.sleep(0.01) + """ + ) + ) + result = pytester.runpytest("--asyncio-mode=strict") + result.assert_outcomes(xfailed=1) def test_timeout_cmdline(pytester): From 229d3ba1226f1002f8ab15bb1f1df6d13b37f0c9 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 21 Jan 2022 13:37:49 +0200 Subject: [PATCH 9/9] Work on --- pytest_asyncio/_runner.py | 7 ++--- pytest_asyncio/plugin.py | 1 + tests/test_timeout.py | 55 ++++++--------------------------------- 3 files changed, 13 insertions(+), 50 deletions(-) diff --git a/pytest_asyncio/_runner.py b/pytest_asyncio/_runner.py index c65f7778..114ca8a0 100644 --- a/pytest_asyncio/_runner.py +++ b/pytest_asyncio/_runner.py @@ -29,13 +29,14 @@ def run_test(self, coro: Awaitable[None]) -> None: raise def set_timer(self, timeout: Union[int, float]) -> None: - assert self._timeout_hande is None + if self._timeout_hande is not None: + self._timeout_hande.cancel() self._timeout_reached = False self._timeout_hande = self._loop.call_later(timeout, self._on_timeout) def cancel_timer(self) -> None: - assert self._timeout_hande is not None - self._timeout_hande.cancel() + if self._timeout_hande is not None: + self._timeout_hande.cancel() self._timeout_reached = False self._timeout_hande = None diff --git a/pytest_asyncio/plugin.py b/pytest_asyncio/plugin.py index c12a4cbc..6639c515 100644 --- a/pytest_asyncio/plugin.py +++ b/pytest_asyncio/plugin.py @@ -294,6 +294,7 @@ def pytest_fixture_setup( if fixturedef.argname == "event_loop": outcome = yield loop = outcome.get_result() + print("\ninstall runner", request.node, id(request.node), id(loop)) _install_runner(request.node, loop) policy = asyncio.get_event_loop_policy() try: diff --git a/tests/test_timeout.py b/tests/test_timeout.py index e40c5946..9754cda7 100644 --- a/tests/test_timeout.py +++ b/tests/test_timeout.py @@ -10,10 +10,11 @@ def test_timeout_ok(pytester): import asyncio import pytest - pytest_plugins = 'pytest_asyncio' + pytest_plugins = ['pytest_asyncio'] - @pytest.mark.asyncio(timeout=0.01) @pytest.mark.xfail(strict=True, raises=asyncio.TimeoutError) + @pytest.mark.timeout(0.01) + @pytest.mark.asyncio async def test_a(): await asyncio.sleep(1) """ @@ -30,9 +31,10 @@ def test_timeout_disabled(pytester): import asyncio import pytest - pytest_plugins = 'pytest_asyncio' + pytest_plugins = ['pytest_asyncio'] - @pytest.mark.asyncio(timeout=0) + @pytest.mark.timeout(0) + @pytest.mark.asyncio async def test_a(): await asyncio.sleep(0.01) """ @@ -42,26 +44,6 @@ async def test_a(): result.assert_outcomes(passed=1) -def test_timeout_not_numeric(pytester): - pytester.makepyfile( - dedent( - """\ - import asyncio - import pytest - - pytest_plugins = 'pytest_asyncio' - - @pytest.mark.asyncio(timeout="abc") - @pytest.mark.xfail(strict=True, raises=ValueError) - async def test_a(): - await asyncio.sleep(0.01) - """ - ) - ) - result = pytester.runpytest("--asyncio-mode=strict") - result.assert_outcomes(xfailed=1) - - def test_timeout_cmdline(pytester): pytester.makepyfile( dedent( @@ -69,7 +51,7 @@ def test_timeout_cmdline(pytester): import asyncio import pytest - pytest_plugins = 'pytest_asyncio' + pytest_plugins = ['pytest_asyncio'] @pytest.mark.asyncio @pytest.mark.xfail(strict=True, raises=asyncio.TimeoutError) @@ -78,26 +60,5 @@ async def test_a(): """ ) ) - result = pytester.runpytest("--asyncio-timeout=0.01", "--asyncio-mode=strict") - result.assert_outcomes(xfailed=1) - - -def test_timeout_cfg(pytester): - pytester.makepyfile( - dedent( - """\ - import asyncio - import pytest - - pytest_plugins = 'pytest_asyncio' - - @pytest.mark.asyncio - @pytest.mark.xfail(strict=True, raises=asyncio.TimeoutError) - async def test_a(): - await asyncio.sleep(1) - """ - ) - ) - pytester.makefile(".ini", pytest="[pytest]\nasyncio_timeout = 0.01\n") - result = pytester.runpytest("--asyncio-mode=strict") + result = pytester.runpytest("--timeout=0.01", "--asyncio-mode=strict") result.assert_outcomes(xfailed=1)