Add deterministic async/await orchestrators across all three Python packages

.github/workflows/codeql.yml

@@ -67,7 +67,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
+      uses: github/codeql-action/init@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0
       with:
         languages: ${{ matrix.language }}
         build-mode: ${{ matrix.build-mode }}
@@ -96,6 +96,6 @@ jobs:
         exit 1
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
+      uses: github/codeql-action/analyze@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/durabletask-azurefunctions.yml

@@ -57,7 +57,7 @@ jobs:
     needs: run-tests
     runs-on: ubuntu-latest
     # The end-to-end suite launches a real Azure Functions host, which requires
-    # ``azure-functions>=2.3.0b2``. That build is published to PyPI and pulled
+    # ``azure-functions>=2.4.0b1``. That build is published to PyPI and pulled
     # in as a declared dependency of ``azure-functions-durable``.
     # Run the suite in both payload-typing modes: the default (loose) codec and
     # strict typing (``AZURE_FUNCTIONS_DURABLE_STRICT_TYPING=1``), which requires
@@ -78,7 +78,7 @@ jobs:
           python-version: "3.13"
 
       - name: Set up Node.js (needed for Azurite and Functions Core Tools)
-        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
         with:
           node-version: "22.x"
 
@@ -106,7 +106,7 @@ jobs:
       # pytest summary does not surface, so upload it for diagnosis.
       - name: Upload Functions host logs
         if: failure()
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
           name: func-host-logs-strict-${{ matrix.strict-typing }}
           path: tests/azure-functions-durable/e2e/apps/*/_func_host.log

.github/workflows/durabletask.yml

@@ -14,6 +14,29 @@ permissions:
   contents: read
 
 jobs:
+  async-runtime-windows:
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install runtime test dependencies
+        run: python -m pip install -r requirements.txt -e .
+      - name: Check asyncio, AnyIO, replay, and dispatch cleanup
+        run: >-
+          python -m pytest
+          tests\durabletask\test_asyncio_orchestrators.py
+          tests\durabletask\test_coroutine_orchestrators.py
+          tests\durabletask\test_pydantic_ai_orchestrator.py
+          tests\durabletask\test_orchestration_generator_cleanup.py
+          tests\durabletask\test_orchestration_dispatch_recovery.py
+          -q
+
   lint-and-unit-tests:
     runs-on: ubuntu-latest
     steps:
@@ -46,7 +69,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Set up Node.js (needed for Azurite)
-        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
         with:
           node-version: '22.x'
 

CHANGELOG.md

@@ -7,6 +7,42 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## Unreleased
 
+ADDED
+
+- Added `OrchestrationQuery.task_hub_names` to filter orchestration instances
+by one or more task hubs.
+- Added native `async def` orchestrators and nested async helpers alongside
+existing generator orchestrators. Await durable operations and compose them
+with `asyncio.gather`, local tasks, and structured task groups. Async sleeps and
+timeouts use durable timers and the orchestration clock; `sleep(0)` yields
+locally without a timer. See the [supported contract](docs/async-orchestrators.md)
+for cancellation, per-dispatch cleanup, replay/migration rules, and unsupported
+APIs. External I/O still belongs in ordinary activities.
+
+FIXED
+
+- Implemented `__hash__` and `__repr__` for `EntityInstanceId`, making identity attributes immutable to guarantee hash stability when used in sets or dict keys.
+- Large-payload externalization now checks every eligible payload's maximum
+size before uploading or replacing any protobuf values, including carryover
+events and entity payloads, for both synchronous and asynchronous calls.
+Oversized messages remain unchanged and cause no uploads; their size error
+now takes precedence over hypothetical earlier storage failures.
+- Orchestrations that exceed the configured external payload storage limit now
+fail without retrying and release their entity locks, allowing other callers to
+progress. Transient payload-store failures remain retryable.
+- Previously unhandled non-cancellation `BaseException` exits, including result
+serialization and generator teardown failures, now fail with a clear reason instead of leaving an accepted dispatch
+unfinished. Cancellation and process-control signals retain their semantics.
+The gRPC worker abandons work that exits before an outcome is sent and resets
+the originating work-item stream after an uncertain completion or abandonment
+RPC, allowing scheduler ownership recovery without retrying an uncertain outcome.
+- Orchestrator generators are now closed at the end of each dispatch, including
+while waiting for durable work, releasing resources held by async bridges.
+`finally` blocks and context-manager exits can run before orchestration completion
+and repeat across dispatches; see [cleanup semantics](docs/supported-patterns.md#cleanup-in-orchestrator-finally-blocks).
+- Async cancellation raised by generator teardown no longer replaces dispatch
+responses or version-rejection abandonment.
+
 ## v1.12.0
 
 ADDED

README.md

@@ -27,6 +27,46 @@ and [Azure Durable Functions](https://learn.microsoft.com/azure/azure-functions/
 
 ## Optional Features
 
+### Async Orchestrators
+
+All three packages support `async def` orchestrators and nested async helpers.
+Existing generator orchestrators continue to work.
+
+```python
+from durabletask import task
+
+
+async def sequence(ctx: task.OrchestrationContext, value: int) -> list[int]:
+    first = await ctx.call_activity("plus_one", input=value, return_type=int)
+    pending: list[task.Task[int]] = [
+        ctx.call_activity("plus_one", input=first, return_type=int),
+        ctx.call_activity("plus_one", input=value, return_type=int),
+    ]
+    return await task.when_all(pending)
+```
+
+Register the function with `TaskHubGrpcWorker`, `DurableTaskSchedulerWorker`,
+or the Azure Functions `orchestration_trigger` decorator as usual. Functions
+also supports single-argument `async def` orchestrators using the compatibility
+context and `await context.task_all(...)`.
+
+> [!WARNING]
+> This is replay-driven await syntax, not general-purpose asyncio execution.
+> Compose durable work with `asyncio.gather`, local tasks/task groups, and
+> durable `asyncio.sleep` / timeouts within the
+> [supported contract](docs/async-orchestrators.md).
+> Perform network, storage, and other nondeterministic I/O in activities.
+> Completed foreign awaitables and synchronous side effects cannot reliably be
+> detected by the replay engine.
+
+An async helper is not a durable sub-orchestration: use
+`await ctx.call_sub_orchestrator(...)` for a separate orchestration instance.
+An `async def` function containing `yield` is an async generator, not a supported
+orchestrator. Converting an existing orchestrator must preserve durable action
+order and payloads for replay compatibility.
+See the contract for cancellation, per-dispatch async cleanup, unsupported
+ordering-sensitive APIs, framework adapters, and coordinated release rules.
+
 ### Large Payload Externalization
 
 Install the `azure-blob-payloads` extra to automatically offload oversized orchestration payloads to

azure-functions-durable/CHANGELOG.md

@@ -7,6 +7,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+ADDED
+
+- With the corresponding core SDK, native and v1-style orchestrators can use
+`async def`, await durable tasks, and use deterministic asyncio composition and
+durable sleeps/timeouts. See the [async contract](../docs/async-orchestrators.md).
+The `orchestrator_generator_wrapper` helper accepts direct-await coroutines;
+asyncio scheduling requires replay/backend tests.
+Requires a coordinated release after the core SDK is published; dependency
+minimums are intentionally unchanged in this feature PR.
+- Added `DFApp.configure_serialization(ensure_ascii=False)` to opt in to compact
+Unicode JSON across durable clients, orchestration and entity workers, activity
+bindings, and registered blueprints. The default remains ASCII-escaped JSON.
+The opt-in uses the shared codec's `ensure_ascii` option available in
+`azure-functions` v2.4.0b1; unsupported SDKs fail explicitly at configuration time.
+
+CHANGED
+
+- Updated the minimum `azure-functions` dependency to v2.4.0b1.
+
+FIXED
+
+- With the corresponding core `durabletask` SDK fix, oversized messages fail
+large-payload validation before any uploads or payload replacements. Size
+errors now take precedence over hypothetical earlier storage failures.
+- With the corresponding core `durabletask` SDK fix, previously unhandled non-cancellation
+`BaseException` exits from orchestration execution, result serialization, or teardown now produce a failed response
+with the exception's type, message, and stack trace instead of losing the response.
+- With the corresponding core `durabletask` SDK fix, orchestrator generators
+are closed at the end of each dispatch, including while waiting for durable work.
+See [cleanup semantics](../docs/supported-patterns.md#cleanup-in-orchestrator-finally-blocks)
+for repeated finalizers, discarded teardown actions/status, and yielding restrictions.
+- With the corresponding core `durabletask` SDK fix, async cancellation raised by
+generator teardown no longer replaces dispatch responses or version-rejection abandonment.
+
 ## v2.0.0rc3
 
 ADDED

azure-functions-durable/MIGRATION_GUIDE.md

@@ -66,7 +66,7 @@ later.
 Replace the 1.x package constraint in `requirements.txt`:
 
 ```text
-azure-functions>=2.3.0b2
+azure-functions>=2.4.0b1
 azure-functions-durable>=2.0.0b1,<3
 ```
 

azure-functions-durable/README.md

@@ -32,6 +32,70 @@ Key capabilities include durable orchestrations and sub-orchestrations, durable
 timers, external events, durable entities, retries, versioning, durable HTTP
 calls (`context.call_http(...)`), recurring scheduled tasks, and history export.
 
+## Async orchestrators
+
+Both authoring styles accept `async def` and `await` without an adapter loop.
+Await durable operations and compose branches with `asyncio.gather` or task
+groups; `asyncio.sleep` and timeouts use durable timers. Synchronous generator
+orchestrators remain supported, including mixed-style sub-orchestrations.
+
+The [shared async contract](../docs/async-orchestrators.md) defines the supported
+API subset, cancellation, per-dispatch `finally`/async-generator cleanup,
+replay/migration rules, and framework adapter responsibilities. Arbitrary I/O,
+threads, subprocesses, `asyncio.wait`/`as_completed`, and automatic recording of
+ordinary async helpers are not supported. Put nondeterministic work in ordinary
+activities, not orchestrators.
+
+## Unicode JSON serialization
+
+By default, durable payloads preserve the shared `azure-functions` codec's
+ASCII-escaped JSON representation. With `azure-functions>=2.4.0b1`, opt in to
+literal Unicode JSON on the root `DFApp` at startup:
+
+```python
+import azure.durable_functions as df
+
+app = df.DFApp()
+app.configure_serialization(ensure_ascii=False)
+```
+
+The shared `df_dumps(..., ensure_ascii=...)` API was released in
+[`azure-functions` 2.4.0b1](https://github.com/Azure/azure-functions-python-library/releases/tag/2.4.0b1),
+which is the minimum required shared SDK version. Dependency installation
+selects a compatible release automatically. If an older SDK is manually
+installed despite that requirement, configuring the opt-in raises `RuntimeError`
+rather than silently ignoring it. Leaving the option unset, or explicitly
+choosing `ensure_ascii=True`, preserves the existing JSON encoding.
+
+There is one encoding policy per Python worker process. It applies to both
+synchronous and asynchronous durable clients, orchestration and entity workers,
+activity binding outputs, and all registered blueprints, including handlers
+created before configuration. Configure every scaled-out worker consistently
+before invocations. Repeating the same setting is allowed; a conflicting setting
+or changing the policy after the first durable JSON serialization raises
+`ValueError`.
+
+Literal Unicode reduces escaping overhead for non-Latin text. For example,
+a JSON object containing 100,000 Cyrillic characters occupies approximately
+200,012 UTF-8 bytes rather than 600,012 ASCII-escaped bytes. Actual reduction
+depends on the payload. The shared codec preserves surrogate-code-unit and
+ASCII DEL escapes, custom-object envelopes, and strict/loose typing semantics.
+This option does not normalize Unicode or change the decoder; existing escaped
+payloads remain readable.
+
+> [!WARNING]
+> Enabling or disabling this option changes serialized bytes for non-ASCII
+> text. Review applications deriving hashes, signatures, IDs, or replay
+> decisions from serialized JSON before rolling out or rolling back a change.
+> Do not change the setting inside orchestrator code or during invocations.
+
+The policy applies before large-payload externalization, so uploads use the
+selected JSON encoding and size thresholds use its serialized UTF-8 bytes.
+Reduced escaping does not eliminate aggregate request limits or replace
+externalization. Protocol envelopes, reference tokens, management responses, and
+JSON serialized explicitly by application code are not reformatted by this
+setting.
+
 ## Large payloads
 
 Configure a `durabletask.payload.PayloadStore` once at app startup to store large

azure-functions-durable/azure/durable_functions/decorators/durable_app.py

@@ -493,6 +493,23 @@ class DFApp(Blueprint, FunctionRegister):
     Exports the decorators required to declare and index DF Function-types.
     """
 
+    def configure_serialization(self, *, ensure_ascii: bool = True) -> None:
+        """Configure JSON encoding for this app and its registered blueprints.
+
+        The default preserves ASCII-escaped JSON. Set ``ensure_ascii=False``
+        to emit ordinary Unicode literally with ``azure-functions>=2.4.0b1``.
+        An older SDK raises RuntimeError when this option is requested.
+
+        Call at startup in every worker process, before durable payloads are
+        serialized. Clients, orchestrations, entities, and activity bindings
+        share one policy per process, including handlers created earlier.
+        Repeating the same setting is allowed. A conflicting setting, or
+        changing the policy after serialization starts, raises ValueError.
+        """
+        from ..internal.serialization import configure_serialization
+
+        configure_serialization(ensure_ascii=ensure_ascii)
+
     def configure_large_payloads(self, *, payload_store: PayloadStore) -> None:
         """Enable payload externalization for this app and its blueprints.
 

azure-functions-durable/azure/durable_functions/internal/converters.py

@@ -32,7 +32,6 @@
 from azure.functions._durable_functions import (
     EntityContext,
     OrchestrationContext,
-    df_dumps,
     df_loads,
 )
 
@@ -43,6 +42,7 @@
     ORCHESTRATION_TRIGGER,
 )
 from .payloads import ActivityPayload, get_transport_payload_store
+from .serialization import serialize_json
 
 _TriggerMetadata = Optional[Mapping[str, meta.Datum]]
 
@@ -162,7 +162,7 @@ def encode(cls, obj: Any, *,
         if isinstance(obj, ActivityPayload):
             return meta.Datum(type='json', value=obj.value)
         try:
-            result = df_dumps(obj)
+            result = serialize_json(obj)
         except TypeError as e:
             raise ValueError(
                 f'activity trigger output must be json serializable ({obj})') from e