3939from typing import Any , TypeVar , cast
4040
4141from durabletask import task
42- from durabletask .internal .exceptions import is_control_flow_exception
4342
4443T = TypeVar ("T" )
4544
@@ -169,6 +168,7 @@ def __init__(
169168 self ._closing = False
170169 self ._fatal : BaseException | None = None
171170 self ._cleanup_failure : BaseException | None = None
171+ self ._cleanup_truncated = False
172172 self ._root : asyncio .Task [Any ] | None = None
173173
174174 def _now (self ) -> datetime :
@@ -343,7 +343,7 @@ def resume(self) -> None:
343343 def _has_durable_work (self ) -> bool :
344344 return any (not durable .is_complete for durable , _ in self ._waits .values ()) or bool (self ._timers )
345345
346- def _drain (self ) -> int :
346+ def _drain (self , cleanup_limit : int | None = None ) -> int :
347347 previous_loop = asyncio ._get_running_loop ()
348348 previous_task = asyncio .current_task (previous_loop ) if previous_loop is not None else None
349349 previous_hooks = sys .get_asyncgen_hooks ()
@@ -353,7 +353,8 @@ def _drain(self) -> int:
353353 sys .set_asyncgen_hooks (firstiter = self ._asyncgen_firstiter , finalizer = self ._asyncgen_finalizer )
354354 try :
355355 steps = 0
356- cleanup_limit = max (10000 , 4 * (len (self ._tasks ) + len (self ._asyncgens )))
356+ if cleanup_limit is None :
357+ cleanup_limit = self ._cleanup_budget ()
357358 while self ._ready or self ._cancel_ready :
358359 self ._ready .extend (handle for _ , _ , handle in sorted (self ._cancel_ready ))
359360 self ._cancel_ready .clear ()
@@ -379,10 +380,7 @@ def _drain(self) -> int:
379380 self ._deliver_completions ()
380381 steps += 1
381382 if self ._closing and steps >= cleanup_limit :
382- self ._record_cleanup_error (RuntimeError (
383- "Async cleanup did not quiesce; cleanup must cooperate with cancellation." ))
384- self ._ready .clear ()
385- self ._cancel_ready .clear ()
383+ self ._report_unquiesced_cleanup ()
386384 break
387385 return steps
388386 finally :
@@ -411,35 +409,50 @@ def call_exception_handler(self, context: dict[str, Any]) -> None:
411409 error = failure
412410 self ._fatal = error if isinstance (error , BaseException ) else RuntimeError (context ["message" ])
413411
412+ def _cleanup_budget (self ) -> int :
413+ # Cooperative cleanup is capped in scheduling steps, not wall-clock time.
414+ return max (10000 , 4 * (len (self ._tasks ) + len (self ._asyncgens )))
415+
416+ def _report_unquiesced_cleanup (self ) -> None :
417+ self ._ready .clear ()
418+ self ._cancel_ready .clear ()
419+ if not self ._cleanup_truncated :
420+ self ._cleanup_truncated = True
421+ self ._record_cleanup_error (RuntimeError (
422+ "Async cleanup did not quiesce; cleanup must cooperate with cancellation." ))
423+
414424 def _record_cleanup_error (self , error : BaseException , source : str = "cleanup callback" ) -> None :
415425 self ._logger .warning ("%s: Async %s failed." , self .ctx .instance_id , source , exc_info = error )
416426 if not isinstance (error , (Exception , asyncio .CancelledError )):
417427 if _is_process_control (self ._cleanup_failure ):
418428 return
419- if self ._cleanup_failure is None or is_control_flow_exception (error ):
429+ # Keep the first failure unless a process-control exit supersedes it.
430+ if self ._cleanup_failure is None or _is_process_control (error ):
420431 self ._cleanup_failure = error
421432
422433 def _close_asyncgens (self , closed : dict [int , Any ]) -> None :
423434 # Closing one generator may first-iterate another; sweep until no new generators appear.
424435 # Closed generators stay referenced so their ids cannot be reused during cleanup.
425436 steps = 0
426437 # Cleanup-created tasks and generators must not replenish the sweep's budget.
427- limit = max ( 10000 , 4 * ( len ( self ._tasks ) + len ( self . _asyncgens )) )
438+ limit = self ._cleanup_budget ( )
428439 while True :
429440 pending = [
430441 generator for key , generator in self ._asyncgens .items () if closed .get (key ) is not generator ]
431442 if not pending :
432443 return
433444 if steps >= limit :
434- self ._record_cleanup_error (RuntimeError (
435- "Async cleanup did not quiesce; cleanup must cooperate with cancellation." ))
445+ self ._report_unquiesced_cleanup ()
436446 return
437447 for generator in pending :
438448 closed [id (generator )] = generator
439449 self .create_task (generator .aclose ())
440- steps += max (1 , self ._drain ())
450+ # Each drain spends from this sweep's remaining budget.
451+ steps += max (1 , self ._drain (limit - steps ))
441452
442453 def close (self ) -> None :
454+ if self ._closed :
455+ return
443456 self ._closing = True
444457 try :
445458 completed_at_close = {future for future in self ._tasks if future .done ()}
0 commit comments