diff --git a/docs/reference/collections.md b/docs/reference/collections.md index 4101281..86cbe0e 100644 --- a/docs/reference/collections.md +++ b/docs/reference/collections.md @@ -20,6 +20,80 @@ return annotations include this empty-input case. `sum_by` requires summable projection results, not summable source elements; for example, `Block.of("one", "four").sum_by(len)` returns `7`. +## Async callbacks + +Use `asyncseq.amap` when the mapper returns an awaitable. It accepts ordinary +iterables and async iterables, and returns a lazy async iterable: + +```python +from expression import pipe +from expression.collections import asyncseq + + +async def enrich_inputs(value: int) -> int: + return value * 2 + + +async def example() -> list[int]: + stream = pipe( + [1, 2, 3], + asyncseq.amap(enrich_inputs), + asyncseq.filter(lambda value: value > 2), + ) + return [value async for value in stream] # [4, 6] +``` + +The fluent equivalent is `AsyncSeq.range(3).amap(enrich_inputs)`. Constructing +either transformation requires no `await`; iteration awaits each mapper call. +Ordinary `pipe` is sufficient because the stages return async iterables, rather +than coroutines that must be resolved before passing to the next stage. + +`asyncseq.map` applies a synchronous mapper; `asyncseq.amap` awaits its mapper. +Both preserve one output per input: an awaited list remains one list element, +unlike `collect`, which flattens inner sequences. Mapping is sequential and +preserves source order, with no task scheduling or prefetching. Mapper and source +exceptions, including cancellation, propagate to the consumer. Sources supporting +both iteration protocols are consumed asynchronously. + +The same convention applies to the other callback-taking operations. Each has a +module-level function for pipelines and an `AsyncSeq` method: + +| Operation | Awaited callback result | Output | +| --- | --- | --- | +| `amap(mapper)` | Mapped value | One item per source item | +| `afilter(predicate)` | Boolean | Items satisfying the predicate | +| `achoose(chooser)` | `Option` | Values from `Some`, skipping `Nothing` | +| `acollect(mapping)` | Ordinary or async iterable | Items from each inner iterable | +| `ascan(scanner, state)` | Updated state | Initial state followed by each update | +| `afold(folder, state)` | Updated state | Coroutine returning the final state | +| `afold_back(folder, state)` | Updated state | Coroutine returning the reverse fold | + +All accept ordinary or async sources and await callbacks sequentially. Stream +transformations are constructed synchronously and return lazy async iterables; +the fluent forms return `AsyncSeq`, allowing chaining. `acollect` awaits the +callback and fully consumes its inner iterable before requesting another source +item. An async generator function already returns an async iterable directly, so +use the existing `collect` for that kind of callback. + +`ascan` yields the initial state even for empty input, without advancing the source. +Both folds return the initial state for empty input. `afold` and `ascan` call their +callback with `(state, item)`; `afold_back` calls it with `(item, state)` in reverse +source order. Reverse folding buffers the entire finite source and therefore uses +memory proportional to its length. + +```python +async def add(state: int, value: int) -> int: + return state + value + + +async def total() -> int: + return await pipe([1, 2, 3], asyncseq.afold(add, 0)) # 6 +``` + +When a fold's resolved result must feed another pipeline stage, use `apipe` to +await it between stages. Operations without callbacks, such as `take`, `skip`, +and `length`, do not need another variant. + ## Additional modules ```{eval-rst} diff --git a/expression/collections/asyncseq.py b/expression/collections/asyncseq.py index c6917af..2a2d963 100644 --- a/expression/collections/asyncseq.py +++ b/expression/collections/asyncseq.py @@ -1,6 +1,6 @@ import builtins import itertools -from collections.abc import AsyncIterable, AsyncIterator, Callable +from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Coroutine, Iterable from typing import Any, cast, overload from expression.core import Option, pipe @@ -16,6 +16,42 @@ async def map[TResult](self, mapper: Callable[[TSource], TResult]) -> "AsyncSeq[ # underlying async iterable and wrap the result back into an AsyncSeq. return AsyncSeq(pipe(self._ai, map(mapper))) + def amap[TResult](self, mapper: Callable[[TSource], Awaitable[TResult]]) -> "AsyncSeq[TResult]": + """Lazily await the mapper for each item, sequentially and in source order. + + Constructing the sequence requires no await. Exceptions and cancellation + propagate during iteration; returned collections remain individual items. + """ + return AsyncSeq(amap(mapper)(self)) + + def afilter(self, predicate: Callable[[TSource], Awaitable[bool]]) -> "AsyncSeq[TSource]": + """Lazily keep items whose awaited predicate is true.""" + return AsyncSeq(afilter(predicate)(self)) + + def achoose[TResult](self, chooser: Callable[[TSource], Awaitable[Option[TResult]]]) -> "AsyncSeq[TResult]": + """Lazily await the chooser and yield the values of present options.""" + return AsyncSeq(achoose(chooser)(self)) + + def acollect[TResult]( + self, mapping: Callable[[TSource], Awaitable[Iterable[TResult] | AsyncIterable[TResult]]] + ) -> "AsyncSeq[TResult]": + """Lazily await each mapping and flatten its ordinary or async iterable.""" + return AsyncSeq(acollect(mapping)(self)) + + async def afold[TState](self, folder: Callable[[TState, TSource], Awaitable[TState]], state: TState) -> TState: + """Consume the sequence, awaiting each accumulator update in order.""" + return await afold(folder, state)(self) + + async def afold_back[TState](self, folder: Callable[[TSource, TState], Awaitable[TState]], state: TState) -> TState: + """Buffer the sequence, then await accumulator updates in reverse order.""" + return await afold_back(folder, state)(self) + + def ascan[TState]( + self, scanner: Callable[[TState, TSource], Awaitable[TState]], state: TState + ) -> "AsyncSeq[TState]": + """Lazily yield the initial state and each awaited accumulator update.""" + return AsyncSeq(ascan(scanner, state)(self)) + @classmethod def empty(cls) -> "AsyncSeq[Any]": return AsyncSeq(empty()) @@ -289,6 +325,25 @@ async def _map(source: AsyncIterable[TSource]) -> AsyncIterable[TResult]: return _map +def amap[TSource, TResult]( + mapper: Callable[[TSource], Awaitable[TResult]], +) -> Callable[[Iterable[TSource] | AsyncIterable[TSource]], AsyncIterable[TResult]]: + """Map an ordinary or async iterable with an awaitable-returning callback. + + The returned async iterable is lazy: each requested item awaits one mapper + call before advancing the source. Results preserve source order and are not + flattened. No tasks are scheduled, and exceptions and cancellation propagate. + Use ordinary ``pipe`` to compose this transformation; consume it with + ``async for``. Sources implementing both protocols use async iteration. + """ + + async def _amap(source: Iterable[TSource] | AsyncIterable[TSource]) -> AsyncIterator[TResult]: + async for value in _iterate(source): + yield await mapper(value) + + return _amap + + def choose[TSource, TResult]( chooser: Callable[[TSource], Option[TResult]], ) -> Callable[[AsyncIterable[TSource]], AsyncIterable[TResult]]: @@ -314,3 +369,122 @@ async def _collect(source: AsyncIterable[TSource]) -> AsyncIterable[TResult]: yield item return _collect + + +async def _iterate[TSource](source: Iterable[TSource] | AsyncIterable[TSource]) -> AsyncIterator[TSource]: + if isinstance(source, AsyncIterable): + async for value in source: + yield value + else: + for value in source: + yield value + + +def afilter[TSource]( + predicate: Callable[[TSource], Awaitable[bool]], +) -> Callable[[Iterable[TSource] | AsyncIterable[TSource]], AsyncIterable[TSource]]: + """Lazily await a predicate for each item, yielding only matching items. + + Accept ordinary or async iterables. Callbacks run sequentially in source + order; exceptions and cancellation propagate during iteration. + """ + + async def _afilter(source: Iterable[TSource] | AsyncIterable[TSource]) -> AsyncIterator[TSource]: + async for value in _iterate(source): + if await predicate(value): + yield value + + return _afilter + + +def achoose[TSource, TResult]( + chooser: Callable[[TSource], Awaitable[Option[TResult]]], +) -> Callable[[Iterable[TSource] | AsyncIterable[TSource]], AsyncIterable[TResult]]: + """Lazily await a chooser and yield present values, skipping Nothing. + + Accept ordinary or async iterables. Callbacks run sequentially in source + order; exceptions and cancellation propagate during iteration. + """ + + async def _achoose(source: Iterable[TSource] | AsyncIterable[TSource]) -> AsyncIterator[TResult]: + async for value in _iterate(source): + result = await chooser(value) + if not result.is_none(): + yield result.value + + return _achoose + + +def acollect[TSource, TResult]( + mapping: Callable[[TSource], Awaitable[Iterable[TResult] | AsyncIterable[TResult]]], +) -> Callable[[Iterable[TSource] | AsyncIterable[TSource]], AsyncIterable[TResult]]: + """Lazily await each mapping and flatten its ordinary or async iterable. + + Accept ordinary or async sources. Consume each inner iterable before advancing + the source. Exceptions and cancellation propagate. For callbacks returning + async iterables directly, without an awaitable wrapper, use ``collect``. + """ + + async def _acollect(source: Iterable[TSource] | AsyncIterable[TSource]) -> AsyncIterator[TResult]: + async for value in _iterate(source): + async for item in _iterate(await mapping(value)): + yield item + + return _acollect + + +def afold[TSource, TState]( + folder: Callable[[TState, TSource], Awaitable[TState]], state: TState +) -> Callable[[Iterable[TSource] | AsyncIterable[TSource]], Coroutine[Any, Any, TState]]: + """Consume an ordinary or async iterable, awaiting each accumulator update. + + Return the initial state for empty input. Run callbacks sequentially in source + order, propagating exceptions and cancellation. Await the resulting coroutine. + """ + + async def _afold(source: Iterable[TSource] | AsyncIterable[TSource]) -> TState: + result = state + async for value in _iterate(source): + result = await folder(result, value) + return result + + return _afold + + +def afold_back[TSource, TState]( + folder: Callable[[TSource, TState], Awaitable[TState]], state: TState +) -> Callable[[Iterable[TSource] | AsyncIterable[TSource]], Coroutine[Any, Any, TState]]: + """Buffer a finite ordinary or async iterable, then fold in reverse order. + + Await callbacks sequentially with item before state. Empty input returns the + initial state. Exceptions and cancellation propagate. Memory use is linear in + source length; await the resulting coroutine. + """ + + async def _afold_back(source: Iterable[TSource] | AsyncIterable[TSource]) -> TState: + items = [value async for value in _iterate(source)] + result = state + for value in reversed(items): + result = await folder(value, result) + return result + + return _afold_back + + +def ascan[TSource, TState]( + scanner: Callable[[TState, TSource], Awaitable[TState]], state: TState +) -> Callable[[Iterable[TSource] | AsyncIterable[TSource]], AsyncIterable[TState]]: + """Lazily yield the initial state and each awaited accumulator update. + + Accept ordinary or async iterables. Yield the initial state without touching + the source. Callbacks run sequentially; exceptions and cancellation propagate. + """ + + async def _ascan(source: Iterable[TSource] | AsyncIterable[TSource]) -> AsyncIterator[TState]: + result = state + yield result + async for value in _iterate(source): + result = await scanner(result, value) + yield result + + return _ascan diff --git a/tests/test_asyncseq.py b/tests/test_asyncseq.py index cf6ee0f..139d10a 100644 --- a/tests/test_asyncseq.py +++ b/tests/test_asyncseq.py @@ -1,9 +1,13 @@ import asyncio +from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Iterable +from typing import assert_type import pytest from hypothesis import given # type: ignore from hypothesis import strategies as st +from expression import NOTHING, Option, Some, pipe +from expression.collections import asyncseq from expression.collections.asyncseq import AsyncSeq @@ -62,3 +66,337 @@ async def source(): raise AssertionError("Requested an item beyond the take limit") assert [value async for value in await AsyncSeq(source()).take(count)] == list(range(count)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_source", [False, True]) +@pytest.mark.parametrize("count", [0, 3]) +async def test_amap_pipe_preserves_nested_results(async_source: bool, count: int): + async def enrich(value: int) -> list[str]: + await asyncio.sleep(0) + return [str(value), str(value * 2)] + + source = asyncseq.range(count) if async_source else range(count) + stream = pipe(source, asyncseq.amap(enrich), asyncseq.map(tuple)) + assert [item async for item in stream] == [(str(value), str(value * 2)) for value in range(count)] + + +@pytest.mark.asyncio +async def test_amap_fluent_and_awaitable_callback(): + def enrich(value: int) -> Awaitable[str]: + future: asyncio.Future[str] = asyncio.get_running_loop().create_future() + future.set_result(str(value)) + return future + + stream = AsyncSeq.range(3).amap(enrich) + assert_type(stream, AsyncSeq[str]) + assert_type(asyncseq.amap(enrich)([1, 2]), AsyncIterable[str]) + assert_type(pipe(asyncseq.range(3), asyncseq.amap(enrich)), AsyncIterable[str]) + assert [item async for item in stream] == ["0", "1", "2"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_source", [False, True]) +async def test_amap_is_lazy_and_sequential(async_source: bool): + events: list[tuple[str, int]] = [] + started = asyncio.Event() + release = asyncio.Event() + + def source() -> Iterable[int]: + for value in range(3): + events.append(("source", value)) + yield value + + async def asource() -> AsyncIterator[int]: + for value in source(): + yield value + + async def enrich(value: int) -> int: + events.append(("start", value)) + started.set() + await release.wait() + events.append(("end", value)) + return value * 2 + + stream = asyncseq.amap(enrich)(asource() if async_source else source()) + assert events == [] + iterator = aiter(stream) + + async def next_item() -> int: + return await anext(iterator) + + task = asyncio.create_task(next_item()) + try: + await asyncio.wait_for(started.wait(), timeout=5) + assert events == [("source", 0), ("start", 0)] + release.set() + assert await asyncio.wait_for(task, timeout=5) == 0 + assert events == [("source", 0), ("start", 0), ("end", 0)] + assert [item async for item in iterator] == [2, 4] + assert events == [(event, value) for value in range(3) for event in ("source", "start", "end")] + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_source", [False, True]) +async def test_amap_propagates_mapper_failure(async_source: bool): + visited: list[int] = [] + failure = ValueError("mapper failed") + + async def enrich(value: int) -> int: + visited.append(value) + if value == 1: + raise failure + return value + + source = asyncseq.range(3) if async_source else range(3) + iterator = aiter(asyncseq.amap(enrich)(source)) + assert await anext(iterator) == 0 + with pytest.raises(ValueError) as error: + await anext(iterator) + assert error.value is failure + assert visited == [0, 1] + with pytest.raises(StopAsyncIteration): + await anext(iterator) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_source", [False, True]) +async def test_amap_propagates_source_failure(async_source: bool): + def source() -> Iterable[int]: + yield 1 + raise ValueError("source failed") + + async def asource() -> AsyncIterator[int]: + for value in source(): + yield value + + async def enrich(value: int) -> int: + return value * 2 + + iterator = aiter(asyncseq.amap(enrich)(asource() if async_source else source())) + assert await anext(iterator) == 2 + with pytest.raises(ValueError, match="source failed"): + await anext(iterator) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_source", [False, True]) +async def test_amap_cancellation_stops_mapping(async_source: bool): + started = asyncio.Event() + cancelled = asyncio.Event() + visited: list[int] = [] + + async def enrich(value: int) -> int: + visited.append(value) + started.set() + try: + await asyncio.Event().wait() + finally: + cancelled.set() + return value + + source = asyncseq.range(3) if async_source else range(3) + iterator = aiter(asyncseq.amap(enrich)(source)) + + async def next_item() -> int: + return await anext(iterator) + + task = asyncio.create_task(next_item()) + try: + await asyncio.wait_for(started.wait(), timeout=5) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert cancelled.is_set() + assert visited == [0] + with pytest.raises(StopAsyncIteration): + await anext(iterator) + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_source", [False, True]) +@pytest.mark.parametrize("count", [0, 4]) +async def test_async_callback_operations(async_source: bool, count: int): + def source() -> Iterable[int] | AsyncIterable[int]: + return asyncseq.range(count) if async_source else range(count) + + async def predicate(value: int) -> bool: + await asyncio.sleep(0) + return value % 2 == 0 + + async def chooser(value: int) -> Option[str]: + return Some(str(value)) if value % 2 == 0 else NOTHING + + async def folder(state: str, value: int) -> str: + await asyncio.sleep(0) + return state + str(value) + + async def back_folder(value: int, state: str) -> str: + return state + str(value) + + expected = list(range(0, count, 2)) + assert [x async for x in pipe(source(), asyncseq.afilter(predicate))] == expected + assert [x async for x in AsyncSeq.range(count).afilter(predicate)] == expected + assert [x async for x in pipe(source(), asyncseq.achoose(chooser))] == list(map(str, expected)) + assert [x async for x in AsyncSeq.range(count).achoose(chooser)] == list(map(str, expected)) + state = "".join(map(str, range(count))) + assert await pipe(source(), asyncseq.afold(folder, "")) == state + assert await AsyncSeq.range(count).afold(folder, "") == state + assert await pipe(source(), asyncseq.afold_back(back_folder, "")) == state[::-1] + assert await AsyncSeq.range(count).afold_back(back_folder, "") == state[::-1] + states = [state[:i] for i in range(count + 1)] + assert [x async for x in pipe(source(), asyncseq.ascan(folder, ""))] == states + assert [x async for x in AsyncSeq.range(count).ascan(folder, "")] == states + assert_type(AsyncSeq.range(count).achoose(chooser), AsyncSeq[str]) + assert_type(AsyncSeq.range(count).afilter(predicate), AsyncSeq[int]) + assert_type(AsyncSeq.range(count).ascan(folder, ""), AsyncSeq[str]) + assert_type(await AsyncSeq.range(count).afold(folder, ""), str) + assert_type(await AsyncSeq.range(count).afold_back(back_folder, ""), str) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_source", [False, True]) +@pytest.mark.parametrize("async_inner", [False, True]) +async def test_acollect_flattens_in_order(async_source: bool, async_inner: bool): + visited: list[int] = [] + + async def mapping(value: int) -> Iterable[str] | AsyncIterable[str]: + visited.append(value) + + async def inner() -> AsyncIterator[str]: + for i in range(value): + yield str(i) + + return inner() if async_inner else [str(i) for i in range(value)] + + source = asyncseq.range(4) if async_source else range(4) + stream = pipe(source, asyncseq.acollect(mapping)) + assert_type(stream, AsyncIterable[str]) + assert visited == [] + iterator = aiter(stream) + assert await anext(iterator) == "0" + assert visited == [0, 1] + assert [x async for x in iterator] == ["0", "1", "0", "1", "2"] + assert visited == [0, 1, 2, 3] + fluent = AsyncSeq.range(4).acollect(mapping) + assert_type(fluent, AsyncSeq[str]) + assert [x async for x in fluent] == ["0", "0", "1", "0", "1", "2"] + + +@pytest.mark.asyncio +async def test_ascan_initial_state_does_not_touch_source(): + async def source() -> AsyncIterator[int]: + raise AssertionError("Source should not be advanced") + yield 0 + + async def scanner(state: int, value: int) -> int: + raise AssertionError("Scanner should not be called") + + iterator = aiter(asyncseq.ascan(scanner, 42)(source())) + assert await anext(iterator) == 42 + + +async def _consume_callback_operation( + operation: str, source: Iterable[int] | AsyncIterable[int], callback: Callable[[int], Awaitable[int]] +) -> list[int]: + async def predicate(value: int) -> bool: + return bool(await callback(value)) + + async def chooser(value: int) -> Option[int]: + return Some(await callback(value)) + + async def mapping(value: int) -> Iterable[int]: + return [await callback(value)] + + async def folder(state: int, value: int) -> int: + return state + await callback(value) + + async def back_folder(value: int, state: int) -> int: + return state + await callback(value) + + if operation == "afold": + return [await asyncseq.afold(folder, 0)(source)] + if operation == "afold_back": + return [await asyncseq.afold_back(back_folder, 0)(source)] + if operation == "afilter": + stream = asyncseq.afilter(predicate)(source) + elif operation == "achoose": + stream = asyncseq.achoose(chooser)(source) + elif operation == "acollect": + stream = asyncseq.acollect(mapping)(source) + else: + stream = asyncseq.ascan(folder, 0)(source) + return [value async for value in stream] + + +_CALLBACK_OPERATIONS = ["afilter", "achoose", "acollect", "afold", "afold_back", "ascan"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("operation", _CALLBACK_OPERATIONS) +@pytest.mark.parametrize("async_source", [False, True]) +async def test_async_operations_propagate_callback_failures(operation: str, async_source: bool): + visited: list[int] = [] + failure = ValueError("callback failed") + + async def callback(value: int) -> int: + visited.append(value) + raise failure + + source = asyncseq.range(3) if async_source else range(3) + with pytest.raises(ValueError) as error: + await _consume_callback_operation(operation, source, callback) + assert error.value is failure + assert visited == ([2] if operation == "afold_back" else [0]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("operation", _CALLBACK_OPERATIONS) +@pytest.mark.parametrize("async_source", [False, True]) +async def test_async_operations_cancel_pending_callback(operation: str, async_source: bool): + started = asyncio.Event() + finished = asyncio.Event() + visited: list[int] = [] + + async def callback(value: int) -> int: + visited.append(value) + started.set() + try: + await asyncio.Event().wait() + finally: + finished.set() + return value + + source = asyncseq.range(3) if async_source else range(3) + task = asyncio.create_task(_consume_callback_operation(operation, source, callback)) + try: + await asyncio.wait_for(started.wait(), timeout=5) + assert visited == ([2] if operation == "afold_back" else [0]) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert finished.is_set() + assert len(visited) == 1 + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("operation", _CALLBACK_OPERATIONS) +async def test_async_operations_propagate_source_failure(operation: str): + async def source() -> AsyncIterator[int]: + raise ValueError("source failed") + yield 0 + + async def callback(value: int) -> int: + raise AssertionError("Callback should not run") + + with pytest.raises(ValueError, match="source failed"): + await _consume_callback_operation(operation, source(), callback)