Skip to content

Commit 1e5bf2f

Browse files
andystaplesCopilot
andauthored
Add asyncio composition and durable timers for coroutine orchestrators
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
1 parent 1c76780 commit 1e5bf2f

3 files changed

Lines changed: 747 additions & 13 deletions

File tree

‎durabletask/internal/asyncio_loop.py‎

Lines changed: 155 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,12 @@
77

88
import asyncio
99
import contextvars
10+
import math
1011
import logging
1112
from collections import deque
1213
from collections.abc import Callable, Coroutine, Generator
13-
from datetime import datetime
14-
from typing import Any, TypeVar
14+
from datetime import datetime, timedelta
15+
from typing import Any, TypeVar, cast
1516

1617
from durabletask import task
1718

@@ -33,11 +34,67 @@ def detach(self) -> None:
3334
self._notify_parent()
3435

3536

37+
class _DurableAsyncTask(asyncio.Task[T]):
38+
def __await__(self) -> Generator[Any, None, T]:
39+
try:
40+
return (yield from super().__await__())
41+
finally:
42+
self._retire_if_observed()
43+
44+
__iter__ = __await__
45+
46+
def result(self) -> T:
47+
try:
48+
return super().result()
49+
finally:
50+
self._retire_if_observed()
51+
52+
def exception(self) -> BaseException | None:
53+
try:
54+
return super().exception()
55+
finally:
56+
self._retire_if_observed()
57+
58+
def _retire_if_observed(self) -> None:
59+
loop = self.get_loop()
60+
assert isinstance(loop, DurableEventLoop)
61+
loop.retire_task(self)
62+
63+
def cancel(self, msg: Any = None) -> bool:
64+
loop = self.get_loop()
65+
assert isinstance(loop, DurableEventLoop)
66+
previous = loop.cancelling_task
67+
loop.cancelling_task = self
68+
try:
69+
return super().cancel(msg)
70+
finally:
71+
loop.cancelling_task = previous
72+
73+
74+
class _DurableTimerHandle(asyncio.TimerHandle):
75+
def __eq__(self, other: object) -> bool:
76+
return self is other
77+
78+
__hash__ = object.__hash__
79+
80+
def __init__(
81+
self, when: float, callback: Callable[..., Any], args: tuple[Any, ...],
82+
loop: DurableEventLoop, context: contextvars.Context | None,
83+
) -> None:
84+
super().__init__(when, callback, args, loop, context=context)
85+
self.durable_timer: task.TimerTask | None = None
86+
87+
def cancel(self) -> None:
88+
super().cancel()
89+
if self.durable_timer is not None:
90+
self.durable_timer.cancel()
91+
92+
3693
class DurableEventLoop(asyncio.AbstractEventLoop):
3794
"""A FIFO microtask loop; it never polls a selector or advances wall time.
3895
39-
Only history events can complete durable waits. asyncio owns coroutine
40-
stepping; this loop owns the ordering of ready callbacks.
96+
Only history events can complete durable waits/timers. asyncio owns coroutine
97+
stepping, gather and structured cancellation; this loop owns their ordering.
4198
"""
4299

43100
def __init__(
@@ -48,10 +105,14 @@ def __init__(
48105
self._is_complete = is_complete
49106
self._logger = logger or logging.getLogger(__name__)
50107
self._ready: deque[asyncio.Handle] = deque()
51-
self._tasks: list[asyncio.Task[Any]] = []
108+
self._cancel_ready: list[tuple[int, int, asyncio.Handle]] = []
109+
self._tasks: dict[asyncio.Task[Any], int] = {}
110+
self._task_sequence = 0
111+
self.cancelling_task: asyncio.Task[Any] | None = None
52112
self._waits: dict[int, tuple[task.Task[Any], list[asyncio.Future[Any]]]] = {}
53113
self._observers: dict[int, _CompletionObserver] = {}
54114
self._completions: deque[Callable[[], None]] = deque()
115+
self._timers: dict[_DurableTimerHandle, _CompletionObserver] = {}
55116
self._closed = False
56117
self._closing = False
57118
self._fatal: BaseException | None = None
@@ -86,18 +147,75 @@ def create_task(
86147
if kwargs:
87148
coro.close()
88149
raise TypeError("Custom/eager task execution is not supported in durable orchestrators.")
89-
future = asyncio.Task(coro, loop=self, name=name, context=context)
90-
self._tasks.append(future)
150+
future = _DurableAsyncTask(coro, loop=self, name=name, context=context)
151+
self._tasks[future] = self._task_sequence
152+
self._task_sequence += 1
153+
future.add_done_callback(self.retire_task)
91154
return future
92155

156+
def retire_task(self, future: asyncio.Task[Any]) -> None:
157+
# A failed task may be observed after its done callbacks have run.
158+
# Inspect without retrieving exceptions so unobserved failures still log.
159+
if not self._closing and future.done() and not getattr(future, "_log_traceback", False):
160+
self._tasks.pop(future, None)
161+
93162
def call_soon(
94163
self, callback: Callable[..., Any], *args: Any,
95164
context: contextvars.Context | None = None,
96165
) -> asyncio.Handle:
97166
handle = asyncio.Handle(callback, args, self, context=context)
98-
self._ready.append(handle)
167+
if self.cancelling_task is not None:
168+
# TaskGroup cancels a set of tasks. Do not let object hashes
169+
# determine the order in which their finally blocks execute.
170+
owner = getattr(callback, "__self__", None)
171+
index = self._tasks[self.cancelling_task]
172+
if isinstance(owner, asyncio.Task):
173+
index = self._tasks.get(cast(asyncio.Task[Any], owner), index)
174+
self._cancel_ready.append((index, len(self._cancel_ready), handle))
175+
else:
176+
self._ready.append(handle)
99177
return handle
100178

179+
def call_later(
180+
self, delay: float, callback: Callable[..., Any], *args: Any,
181+
context: contextvars.Context | None = None,
182+
) -> asyncio.TimerHandle:
183+
if not math.isfinite(delay):
184+
raise ValueError("Durable delays must be finite.")
185+
return self.call_at(self.time() + max(0, delay), callback, *args, context=context)
186+
187+
def call_at(
188+
self, when: float, callback: Callable[..., Any], *args: Any,
189+
context: contextvars.Context | None = None,
190+
) -> asyncio.TimerHandle:
191+
if not math.isfinite(when):
192+
raise ValueError("Durable deadlines must be finite.")
193+
handle = _DurableTimerHandle(when, callback, args, self, context)
194+
if when <= self.time():
195+
self._ready.append(handle)
196+
else:
197+
if self._closing:
198+
raise RuntimeError("Dispatch cleanup cannot schedule durable timers.")
199+
deadline = datetime(1970, 1, 1) + timedelta(seconds=when)
200+
if (deadline - datetime(1970, 1, 1)).total_seconds() < when:
201+
deadline += timedelta(microseconds=1)
202+
handle.durable_timer = self.ctx.create_timer(deadline)
203+
self._timers[handle] = _CompletionObserver(
204+
handle.durable_timer,
205+
lambda: self._completions.append(lambda: self._complete_timer(handle)))
206+
return handle
207+
208+
def _timer_handle_cancelled(self, handle: asyncio.TimerHandle) -> None:
209+
if isinstance(handle, _DurableTimerHandle):
210+
observer = self._timers.pop(handle, None)
211+
if observer is not None:
212+
observer.detach()
213+
214+
def _complete_timer(self, handle: _DurableTimerHandle) -> None:
215+
self._timers.pop(handle, None)
216+
if not handle.cancelled():
217+
self._ready.append(handle)
218+
101219
async def wait_durable(self, durable: task.Task[T]) -> T:
102220
if durable.is_complete:
103221
return durable.get_result()
@@ -154,7 +272,7 @@ def resume(self) -> None:
154272
"Await durable activities, events or timers; external I/O is unsupported.")
155273

156274
def _has_durable_work(self) -> bool:
157-
return any(not durable.is_complete for durable, _ in self._waits.values())
275+
return any(not durable.is_complete for durable, _ in self._waits.values()) or bool(self._timers)
158276

159277
def _drain(self) -> None:
160278
previous_loop = asyncio._get_running_loop()
@@ -163,7 +281,9 @@ def _drain(self) -> None:
163281
asyncio.tasks._leave_task(previous_loop, previous_task)
164282
asyncio._set_running_loop(self)
165283
try:
166-
while self._ready:
284+
while self._ready or self._cancel_ready:
285+
self._ready.extend(handle for _, _, handle in sorted(self._cancel_ready))
286+
self._cancel_ready.clear()
167287
handle = self._ready.popleft()
168288
if not handle.cancelled():
169289
handle._run()
@@ -217,9 +337,33 @@ def close(self) -> None:
217337
finally:
218338
self._closed = True
219339
self._ready.clear()
220-
for observer in self._observers.values():
340+
self._cancel_ready.clear()
341+
for observer in (*self._observers.values(), *self._timers.values()):
221342
observer.detach()
222343
self._completions.clear()
223344
self._observers.clear()
224345
self._waits.clear()
346+
self._timers.clear()
225347
self._tasks.clear()
348+
349+
def run_in_executor(self, executor: Any, func: Any, *args: Any) -> asyncio.Future[Any]:
350+
raise RuntimeError("Threads/executors are unsupported in orchestrators; use an activity.")
351+
352+
def call_soon_threadsafe(self, callback: Any, *args: Any, context: Any = None) -> asyncio.Handle:
353+
raise RuntimeError("Cross-thread scheduling is unsupported in orchestrators; use an activity.")
354+
355+
async def _unsupported_io(self, *args: Any, **kwargs: Any) -> Any:
356+
raise RuntimeError("External I/O is unsupported in orchestrators; move it to an ordinary activity.")
357+
358+
create_connection = _unsupported_io
359+
create_server = _unsupported_io
360+
create_datagram_endpoint = _unsupported_io
361+
connect_read_pipe = _unsupported_io
362+
connect_write_pipe = _unsupported_io
363+
getaddrinfo = _unsupported_io
364+
getnameinfo = _unsupported_io
365+
sock_connect = _unsupported_io
366+
sock_recv = _unsupported_io
367+
sock_sendall = _unsupported_io
368+
subprocess_exec = _unsupported_io
369+
subprocess_shell = _unsupported_io

‎durabletask/task.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -633,8 +633,10 @@ def __await__(self) -> Generator[Task[T], Any, T]:
633633
async def wait(self) -> T:
634634
"""Borrow this operation's result through a fresh native coroutine.
635635
636-
Calling this method does not schedule another durable operation. Each
637-
call creates an independent waiter.
636+
Suitable for ``asyncio.create_task`` and task groups. Calling this
637+
method does not schedule another durable operation. Each call creates
638+
an independent waiter; cancelling it does not cancel the underlying
639+
operation, timer, or event subscription, or any other waiter.
638640
"""
639641
return await self
640642

0 commit comments

Comments
 (0)