Make the timing tests independent of sleep accuracy

The timeout tests counted items against real sleeps and left 10 to 40 ms
of slack. A sleep only promises to take at least as long as requested, so
the counts changed on a busy machine and on a coarse clock:

- Blocking sleeps that overshoot by 40 ms or more made timeout_generator
  yield one item fewer, in five test cases and in its doctest.
- A 15.6 ms event loop clock resolution, the Windows default, let the
  0.05 s timeout fire together with a 0.04 s sleep, so the detector tests
  stopped at 3 instead of 4.

The sync tests and the doctest now run on a fake clock that only moves
when it is slept on, and they check the requested sleeps as well. The
total timeout tests advance the same clock. The per-item timeout tests
yield without waiting and then stall for 10 s against a 0.05 s timeout.
One test stays on the real clock and only checks what holds for any sleep
accuracy.

The fixtures are loaded from a conftest.py in the repository root so the
doctests can use them, and the sdist ships that file.

Author: wolph

_python_utils_tests/clock.py

@@ -0,0 +1,56 @@
+"""A fake clock for the tests and doctests of ``python_utils.time``."""
+
+import typing
+
+import pytest
+
+import python_utils.time
+
+#: The doctest that sleeps on the clock, by the name pytest gives it.
+TIMEOUT_GENERATOR_DOCTEST: str = 'python_utils.time.timeout_generator'
+
+
+class FakeClock:
+    """
+    A clock that only moves when something sleeps on it.
+
+    ``time.sleep`` promises to sleep at least as long as requested. A busy
+    machine sleeps tens of milliseconds longer, and that changes how many
+    items ``timeout_generator`` yields before its timeout. On this clock the
+    number of items depends on the arguments alone.
+
+    Attributes:
+        now (float): The current time in seconds.
+        sleeps (list[float]): Every requested sleep, in order.
+    """
+
+    def __init__(self) -> None:
+        """Start at zero without any recorded sleeps."""
+        self.now: float = 0.0
+        self.sleeps: list[float] = []
+
+    def perf_counter(self) -> float:
+        """Return the current time, like ``time.perf_counter``."""
+        return self.now
+
+    def sleep(self, seconds: float) -> None:
+        """Record the sleep and move the clock forward, without waiting."""
+        self.sleeps.append(seconds)
+        self.now += seconds
+
+
+@pytest.fixture
+def fake_clock(monkeypatch: pytest.MonkeyPatch) -> FakeClock:
+    """Replace the ``time`` module inside ``python_utils.time``."""
+    clock: FakeClock = FakeClock()
+    monkeypatch.setattr(python_utils.time, 'time', clock)
+    return clock
+
+
+@pytest.fixture(autouse=True)
+def fake_clock_in_doctest(request: pytest.FixtureRequest) -> None:
+    """Run the ``timeout_generator`` doctest on the fake clock."""
+    # pytest leaves `FixtureRequest.node` without a type.
+    node: pytest.Item = typing.cast(pytest.Item, request.node)
+    if node.name == TIMEOUT_GENERATOR_DOCTEST:
+        request.getfixturevalue('fake_clock')

_python_utils_tests/test_time.py

@@ -3,12 +3,18 @@
 import asyncio
 import datetime
 import itertools
+import time
 
 import pytest
 
 import python_utils
+from _python_utils_tests import clock
 from python_utils import types
 
+#: Far longer than every timeout in this module, so a generator that sleeps
+#: this long is always interrupted first.
+STALL: float = 10.0
+
 
 @pytest.mark.parametrize(
     'timeout,interval,interval_multiplier,maximum_interval,iterable,result',
@@ -46,23 +52,26 @@ async def test_aio_timeout_generator(
 
 
 @pytest.mark.parametrize(
-    'timeout,interval,interval_multiplier,maximum_interval,iterable,result',
+    'timeout,interval,interval_multiplier,maximum_interval,iterable,result,'
+    'sleeps',
     [
-        (0.1, 0.06, 0.5, 0.1, 'abc', 'c'),
-        (0.1, 0.07, 0.5, 0.1, itertools.count, 2),
-        (0.1, 0.07, 0.5, 0.1, itertools.count(), 2),
-        (0.1, 0.06, 1.0, None, 'abc', 'c'),
+        (0.1, 0.06, 0.5, 0.1, 'abc', 'c', [0.06, 0.03, 0.015]),
+        (0.1, 0.07, 0.5, 0.1, itertools.count, 2, [0.07, 0.035]),
+        (0.1, 0.07, 0.5, 0.1, itertools.count(), 2, [0.07, 0.035]),
+        (0.1, 0.06, 1.0, None, 'abc', 'c', [0.06, 0.06]),
         (
             datetime.timedelta(seconds=0.1),
             datetime.timedelta(seconds=0.06),
             2.0,
             datetime.timedelta(seconds=0.1),
             itertools.count,
             2,
+            [0.06, 0.1],
         ),
     ],
 )
 def test_timeout_generator(
+    fake_clock: clock.FakeClock,
     timeout: float,
     interval: float,
     interval_multiplier: float,
@@ -73,8 +82,9 @@ def test_timeout_generator(
         types.Callable[..., types.Iterable[types.Any]],
     ],
     result: int,
+    sleeps: types.List[float],
 ) -> None:
-    """Stop the sync generator near the configured timeout."""
+    """Stop the sync generator at the timeout and scale the interval."""
     i = None
     for i in python_utils.timeout_generator(
         timeout=timeout,
@@ -86,59 +96,97 @@ def test_timeout_generator(
         assert i is not None
 
     assert i == result
+    assert fake_clock.sleeps == pytest.approx(sleeps)
 
 
-@pytest.mark.asyncio
-async def test_aio_generator_timeout_detector() -> None:
-    """Raise or exit on per-item and total timeouts."""
-    # Make pyright happy
-    i = None
+def test_timeout_generator_real_clock() -> None:
+    """Keep yielding on the real clock until the timeout has passed."""
+    timeout: float = 0.05
+    interval: float = 0.01
+    start: float = time.perf_counter()
+    items: types.List[int] = list(
+        python_utils.timeout_generator(timeout, interval, itertools.count())
+    )
+    elapsed: float = time.perf_counter() - start
+
+    # A sleep can take longer than requested but never shorter, so these
+    # hold on any machine. The exact number of items does not.
+    assert items == list(range(len(items)))
+    assert len(items) <= timeout / interval + 2
+    assert elapsed >= timeout
+
+
+async def stalling_generator() -> types.AsyncGenerator[int, None]:
+    """Yield 0-4 without waiting, then stall before the next item."""
+    for i in range(10):
+        if i == 5:
+            await asyncio.sleep(STALL)
+        yield i
+
+
+def ticking_generator(
+    fake_clock: clock.FakeClock,
+) -> types.AsyncGenerator[int, None]:
+    """Yield 0-9 and let 0.1 seconds pass on the fake clock for each item."""
 
     async def generator() -> types.AsyncGenerator[int, None]:
-        """Yield 0-9 with increasing sleeps between items."""
+        """Advance the fake clock before every item."""
         for i in range(10):
-            await asyncio.sleep(i / 20.0)
+            fake_clock.sleep(0.1)
             yield i
 
+    return generator()
+
+
+@pytest.mark.asyncio
+async def test_aio_generator_timeout_detector(
+    fake_clock: clock.FakeClock,
+) -> None:
+    """Raise or exit on per-item and total timeouts."""
+    # Make pyright happy
+    i = None
+
     detector = python_utils.aio_generator_timeout_detector
     # Test regular timeout with reraise
     with pytest.raises(asyncio.TimeoutError):
-        async for i in detector(generator(), 0.25):
+        async for i in detector(stalling_generator(), 0.05):
             pass
 
     # Test regular timeout with clean exit
-    async for i in detector(generator(), 0.25, on_timeout=None):
+    async for i in detector(stalling_generator(), 0.05, on_timeout=None):
         pass
 
     assert i == 4
 
     # Test total timeout with reraise
     with pytest.raises(asyncio.TimeoutError):
-        async for i in detector(generator(), total_timeout=0.5):
+        async for i in detector(
+            ticking_generator(fake_clock), total_timeout=0.45
+        ):
             pass
 
     # Test total timeout with clean exit
-    async for i in detector(generator(), total_timeout=0.5, on_timeout=None):
+    async for i in detector(
+        ticking_generator(fake_clock), total_timeout=0.45, on_timeout=None
+    ):
         pass
 
     assert i == 4
 
     # Test stop iteration
-    async for i in detector(generator(), on_timeout=None):
+    async for i in detector(ticking_generator(fake_clock), on_timeout=None):
         pass
 
+    assert i == 9
+
 
 @pytest.mark.asyncio
 async def test_aio_generator_timeout_detector_decorator_reraise() -> None:
     """Reraise ``TimeoutError`` on a per-item timeout."""
-
     # Test regular timeout with reraise
-    @python_utils.aio_generator_timeout_detector_decorator(timeout=0.05)
-    async def generator_timeout() -> types.AsyncGenerator[int, None]:
-        """Yield with increasing delays to trip the timeout."""
-        for i in range(10):
-            await asyncio.sleep(i / 100.0)
-            yield i
+    generator_timeout = python_utils.aio_generator_timeout_detector_decorator(
+        timeout=0.05
+    )(stalling_generator)
 
     with pytest.raises(asyncio.TimeoutError):
         async for _ in generator_timeout():
@@ -152,14 +200,9 @@ async def test_aio_generator_timeout_detector_decorator_clean_exit() -> None:
     i = None
 
     # Test regular timeout with clean exit
-    @python_utils.aio_generator_timeout_detector_decorator(
+    generator_clean = python_utils.aio_generator_timeout_detector_decorator(
         timeout=0.05, on_timeout=None
-    )
-    async def generator_clean() -> types.AsyncGenerator[int, None]:
-        """Yield with increasing delays to trip the timeout."""
-        for i in range(10):
-            await asyncio.sleep(i / 100.0)
-            yield i
+    )(stalling_generator)
 
     async for i in generator_clean():
         pass
@@ -168,17 +211,16 @@ async def generator_clean() -> types.AsyncGenerator[int, None]:
 
 
 @pytest.mark.asyncio
-async def test_aio_generator_timeout_detector_decorator_reraise_total() -> (
-    None
-):
+async def test_aio_generator_timeout_detector_decorator_reraise_total(
+    fake_clock: clock.FakeClock,
+) -> None:
     """Reraise ``TimeoutError`` on a total timeout."""
 
     # Test total timeout with reraise
-    @python_utils.aio_generator_timeout_detector_decorator(total_timeout=0.1)
+    @python_utils.aio_generator_timeout_detector_decorator(total_timeout=0.45)
     async def generator_reraise() -> types.AsyncGenerator[int, None]:
-        """Yield with increasing delays to trip the timeout."""
-        for i in range(10):
-            await asyncio.sleep(i / 100.0)
+        """Let the fake clock pass the total timeout while yielding."""
+        async for i in ticking_generator(fake_clock):
             yield i
 
     with pytest.raises(asyncio.TimeoutError):
@@ -187,19 +229,20 @@ async def generator_reraise() -> types.AsyncGenerator[int, None]:
 
 
 @pytest.mark.asyncio
-async def test_aio_generator_timeout_detector_decorator_clean_total() -> None:
+async def test_aio_generator_timeout_detector_decorator_clean_total(
+    fake_clock: clock.FakeClock,
+) -> None:
     """Exit cleanly on total timeout when ``on_timeout`` is ``None``."""
     # Make pyright happy
     i = None
 
     # Test total timeout with clean exit
     @python_utils.aio_generator_timeout_detector_decorator(
-        total_timeout=0.1, on_timeout=None
+        total_timeout=0.45, on_timeout=None
     )
     async def generator_clean_total() -> types.AsyncGenerator[int, None]:
-        """Yield with increasing delays to trip the timeout."""
-        for i in range(10):
-            await asyncio.sleep(i / 100.0)
+        """Let the fake clock pass the total timeout while yielding."""
+        async for i in ticking_generator(fake_clock):
             yield i
 
     async for i in generator_clean_total():

conftest.py

@@ -0,0 +1,8 @@
+"""
+Load the shared fixtures for the tests and the doctests.
+
+The fixtures live in the tests package. They are loaded from the repository
+root because the doctests in ``python_utils`` need them as well.
+"""
+
+pytest_plugins: tuple[str, ...] = ('_python_utils_tests.clock',)

pyproject.toml

@@ -8,7 +8,7 @@ module-root = ''
 module-name = 'python_utils'
 # Keep the tests and tox config in the sdist (parity with the old MANIFEST.in)
 # so downstream packagers can build and test from the source distribution.
-source-include = ['_python_utils_tests/**/*.py', 'tox.ini']
+source-include = ['_python_utils_tests/**/*.py', 'conftest.py', 'tox.ini']
 
 [project]
 name = 'python-utils'