From 6fc4e008f60542e5577d63d2570dfb02261cd987 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 15 Nov 2026 14:32:09 -0700 Subject: [PATCH] Bounded each test and its fixtures with a hang watchdog. pytest-timeout covers only the test call and can't stop a main thread blocked outside Python, so one hang stalled a CI step for 20 minutes. The new timer dumps every stack and ends the process, and xdist then names the test that hung. --- tests/conftest.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index b96e58f66..cda0c0e1b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ import asyncio +import faulthandler import gc import multiprocessing.context import os @@ -57,6 +58,28 @@ def pytest_addoption(parser): # type: ignore[reportMissingParameterType] default="local", help="Which workflow environment to use ('local', 'time-skipping-v1', 'time-skipping-v2', 'envconfig', or ip:port for existing server)", ) + parser.addini( + "test_hang_limit", + "Seconds one test may take with its fixtures before the process dumps all stacks and exits", + default="300", + ) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_protocol(item: pytest.Item): + # pytest-timeout only covers the test call, and its signal can't stop a main thread that is + # blocked outside Python. A hang in a fixture or in native code then stalls the whole run. + # This timer runs on a C thread, so it also fires there. It prints every stack and ends the + # process, and xdist then reports the test as crashed. + faulthandler.dump_traceback_later( + float(item.config.getini("test_hang_limit")), + exit=True, + file=sys.__stderr__ or sys.stderr, + ) + try: + yield + finally: + faulthandler.cancel_dump_traceback_later() def _uses_envconfig_server(env_type: str | None) -> bool: