diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38c9c7a..80582b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,19 @@ on: push: branches: [main] workflow_dispatch: + inputs: + cooperative_qualification: + description: Run cooperative protocol 1.20 cases against an isolated Server candidate + type: boolean + default: false + server_commit: + description: Exact 40-character public Server candidate SHA + type: string + default: "" + native_commit: + description: Optional exact public Native candidate SHA for readonly source qualification + type: string + default: "" permissions: contents: read @@ -181,10 +194,107 @@ jobs: python scripts/ci/test-release-package.py python scripts/ci/test-example-base-urls.py + cooperative-server: + name: Cooperative Server source qualification + if: ${{ github.event_name == 'workflow_dispatch' && inputs.cooperative_qualification }} + needs: [action-policy, test] + runs-on: ubuntu-latest + timeout-minutes: 25 + env: + COMPOSE_PROJECT_NAME: rust-cooperative-${{ github.run_id }}-${{ github.run_attempt }} + SERVER_CANDIDATE_SHA: ${{ inputs.server_commit }} + NATIVE_CANDIDATE_SHA: ${{ inputs.native_commit }} + DURABLE_WORKFLOW_SERVER_SOURCE: ${{ github.workspace }}/.cooperative-server-source + DURABLE_WORKFLOW_NATIVE_SOURCE: ${{ github.workspace }}/.cooperative-native-source + DURABLE_WORKFLOW_AUTH_TOKEN: test-token + COMPOSE_FILE: docker-compose.cooperative.yml + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Require an exact Server candidate + run: '[[ "$SERVER_CANDIDATE_SHA" =~ ^[0-9a-f]{40}$ ]]' + - name: Check out the public Server candidate + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: durable-workflow/server + ref: ${{ inputs.server_commit }} + path: .cooperative-server-source + persist-credentials: false + - name: Verify checked-out candidate identity + run: test "$(git -C .cooperative-server-source rev-parse HEAD)" = "$SERVER_CANDIDATE_SHA" + - name: Require an exact optional Native candidate + if: ${{ inputs.native_commit != '' }} + run: '[[ "$NATIVE_CANDIDATE_SHA" =~ ^[0-9a-f]{40}$ ]]' + - name: Check out the public Native candidate + if: ${{ inputs.native_commit != '' }} + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: durable-workflow/workflow + ref: ${{ inputs.native_commit }} + path: .cooperative-native-source + persist-credentials: false + - name: Verify Native identity and select its readonly source overlay + if: ${{ inputs.native_commit != '' }} + run: | + test "$(git -C .cooperative-native-source rev-parse HEAD)" = "$NATIVE_CANDIDATE_SHA" + printf 'COMPOSE_FILE=docker-compose.cooperative.yml:docker-compose.native-cancellation.yml\n' >> "$GITHUB_ENV" + printf 'DURABLE_WORKFLOW_NATIVE_SOURCE_QUALIFICATION=1\n' >> "$GITHUB_ENV" + - name: Install Rust 1.86 + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable + with: + toolchain: 1.86.0 + - name: Start isolated Server stack + run: docker compose up -d --build --wait --wait-timeout 300 + - name: Retain runtime identity and select endpoint + run: | + docker compose exec -T server cat /app/.package-provenance > cooperative-package-provenance.txt + docker compose images --format json > cooperative-images.json + jq --null-input --arg sdk "$GITHUB_SHA" --arg server "$SERVER_CANDIDATE_SHA" --arg native "$NATIVE_CANDIDATE_SHA" \ + '{qualification:"source",sdk_commit:$sdk,server_commit:$server,native_source_overlay:($native | if length > 0 then . else null end)}' \ + > cooperative-source-provenance.json + endpoint="$(docker compose port server 8080)" + echo "DURABLE_WORKFLOW_SERVER_URL=http://$endpoint" >> "$GITHUB_ENV" + - name: Run actual Worker cooperative scenarios + shell: bash + run: | + arguments=(--ignored --nocapture --test-threads=1) + if [ -z "$NATIVE_CANDIDATE_SHA" ]; then + arguments+=(--skip server_scope_opening_proves_real_native_tree_on_original_claim) + arguments+=(--skip server_scope_authoring_replays_nested_tree_and_deferred_membership_on_replacement) + arguments+=(--skip server_scope_cleanup_preserves_original_delivery_and_budget_after_replacement) + arguments+=(--skip server_scope_group_cleanup_preserves_original_delivery_and_budget_after_replacement) + arguments+=(--skip server_sigkill_scoped_cleanup_replays_before_root_with_original_30_second_deadline) + arguments+=(--skip server_scope_try_stops_actual_remote_callback_without_application_heartbeats) + arguments+=(--skip server_scope_wait_stops_actual_remote_callback_before_cleanup) + arguments+=(--skip server_scope_group_wait_stops_actual_remote_callback_before_cleanup) + fi + cargo test --test cooperative_server -- "${arguments[@]}" 2>&1 | tee cooperative-results.log + - name: Retain bounded scenario evidence + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: cooperative-server-${{ github.sha }} + path: | + cooperative-results.log + cooperative-package-provenance.txt + cooperative-images.json + cooperative-source-provenance.json + retention-days: 90 + if-no-files-found: warn + - name: Emit failure diagnostics + if: ${{ failure() }} + run: docker compose logs --no-color --tail 100 bootstrap server worker repair + - name: Remove task runtime resources + if: ${{ always() }} + run: | + docker compose down -v + docker image rm "dw-rust-cooperative-server:$SERVER_CANDIDATE_SHA" + target-branch-qualification: name: Target branch qualification if: ${{ always() }} - needs: [action-policy, regression-corpus, test, protocol, docs, package] + needs: [action-policy, regression-corpus, test, protocol, docs, package, cooperative-server] runs-on: ubuntu-latest timeout-minutes: 2 steps: @@ -200,9 +310,14 @@ jobs: PACKAGE_RESULT: ${{ needs.package.result }} PROTOCOL_RESULT: ${{ needs.protocol.result }} TEST_RESULT: ${{ needs.test.result }} + COOPERATIVE_REQUESTED: ${{ inputs.cooperative_qualification }} + COOPERATIVE_RESULT: ${{ needs.cooperative-server.result }} run: | test "$CORPUS_RESULT" = success test "$DOCS_RESULT" = success test "$PACKAGE_RESULT" = success test "$PROTOCOL_RESULT" = success test "$TEST_RESULT" = success + if [ "$COOPERATIVE_REQUESTED" = true ]; then + test "$COOPERATIVE_RESULT" = success + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a09945..f0078d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## 3.0.0-rc.1 + +- Add cooperative whole-run cancellation with immutable request metadata, + lineage, deterministic cleanup time helpers and the original bounded deadline. +- Add explicit Activity and Child TryCancel, WaitCancellationCompleted and + Abandon policies, and cooperative RequestCancellation on parent closure. +- Supervise opted-in async Activity callbacks independently of application + heartbeats, acknowledge their stop and reject stale publication. Resume + shielded durable cleanup from the original delivery boundary after worker loss. +- Keep cooperation opt-in and ordinary worker protocol 1.19. Independently + cancellable scopes remain a disabled source preview. Portable local activities, + worker sessions and sticky execution remain unsupported by this Rust profile. +- This major release adds public cancellation fields and enum variants. Follow + the [v3 migration guide](https://github.com/durable-workflow/sdk-rust/blob/main/docs/migrating-to-v3.md) + when updating struct literals and exhaustive matches. The wire protocol remains + compatible with older same-major workers on a supporting Server. + ## 2.1.5 - Allow the ordinary Worker to append and close workflow streams when Server diff --git a/Cargo.toml b/Cargo.toml index f9e633c..1a62985 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "durable-workflow" -version = "2.1.5" +version = "3.0.0-rc.1" edition = "2021" description = "Rust client and worker SDK for Durable Workflow Cloud and self-hosted Server" license = "MIT" @@ -14,10 +14,10 @@ categories = ["api-bindings", "asynchronous"] include = ["/src/**", "/schema/**", "/examples/**", "/Cargo.toml", "/README.md", "/CHANGELOG.md", "/LICENSE"] [package.metadata.durable-workflow] -product-train = "2.1.5" +product-train = "3.0.0-rc.1" compatibility-authority = "protocol-manifests" supported-server-versions = "2.0.0" -qualified-server-version = "2.4.0" +qualified-server-version = "2.5.0-rc.1" worker-protocol-version = "1.19" server-worker-protocol-versions = ">=1.19,<2.0" portable-worker-affinity-minimum-worker-protocol-version = "1.18" @@ -110,7 +110,7 @@ apache-avro = { version = "0.21", default-features = false } base64 = "0.22" # Keep apache-avro's dependency resolution compatible with the Rust 1.86 MSRV. bon = { version = "=3.8.1", default-features = false } -chrono = { version = "0.4", default-features = false } +chrono = { version = "0.4", default-features = false, features = ["alloc"] } futures-util = "0.3" # Keep reqwest's IDNA dependency resolution compatible with the Rust 1.86 MSRV. icu_collections = { version = "=2.2.0", default-features = false } diff --git a/README.md b/README.md index 49820ce..58c94fe 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,14 @@ payload fetches never follow redirects. ## Compatibility +The cancellation release candidate adds cooperative requests and bounded, +replayable cleanup. Enable `Worker::cooperative_cancellation(true)` against a +Server that advertises protocol 1.20 and the required capabilities. This profile +supervises async Activity futures independently of application heartbeats. +Independently cancellable scopes remain disabled. See the +[cancellation guide](https://github.com/durable-workflow/sdk-rust/blob/main/docs/cooperative-cancellation-design.md) +and [v3 migration guide](https://github.com/durable-workflow/sdk-rust/blob/main/docs/migrating-to-v3.md). + The crate publishes its supported Server and worker-protocol ranges in `[package.metadata.durable-workflow]` in [`Cargo.toml`](https://github.com/durable-workflow/sdk-rust/blob/main/Cargo.toml). Runtime capability manifests, not matching package version strings, determine protocol diff --git a/docker-compose.cooperative.yml b/docker-compose.cooperative.yml new file mode 100644 index 0000000..880b80a --- /dev/null +++ b/docker-compose.cooperative.yml @@ -0,0 +1,79 @@ +services: + bootstrap: + image: dw-rust-cooperative-server:${SERVER_CANDIDATE_SHA:-candidate} + build: + context: ${DURABLE_WORKFLOW_SERVER_SOURCE:-../server} + command: ["server-bootstrap"] + environment: &runtime-env + APP_ENV: testing + APP_KEY: "base64:dGVzdGluZy1rZXktMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMg==" + APP_DEBUG: "false" + DB_CONNECTION: mysql + DB_HOST: mysql + DB_DATABASE: durable_workflow + DB_USERNAME: workflow + DB_PASSWORD: workflow + REDIS_HOST: redis + QUEUE_CONNECTION: redis + CACHE_STORE: redis + WORKFLOW_SERVER_AUTH_DRIVER: token + WORKFLOW_SERVER_AUTH_TOKEN: test-token + DW_WORKFLOW_TASK_TIMEOUT: 10 + DW_WORKER_PROTOCOL_VERSION: "1.20" + depends_on: + mysql: + condition: service_healthy + redis: + condition: service_healthy + server: + image: dw-rust-cooperative-server:${SERVER_CANDIDATE_SHA:-candidate} + build: + context: ${DURABLE_WORKFLOW_SERVER_SOURCE:-../server} + environment: *runtime-env + ports: + - "127.0.0.1::8080" + depends_on: + bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/api/health"] + interval: 5s + timeout: 3s + retries: 12 + worker: + image: dw-rust-cooperative-server:${SERVER_CANDIDATE_SHA:-candidate} + build: + context: ${DURABLE_WORKFLOW_SERVER_SOURCE:-../server} + command: ["php", "artisan", "queue:work", "--sleep=1", "--tries=3", "--max-time=1800"] + environment: *runtime-env + depends_on: + server: + condition: service_healthy + repair: + image: dw-rust-cooperative-server:${SERVER_CANDIDATE_SHA:-candidate} + # Match the production scheduler's unscoped, unthrottled repair cadence. + command: sh -c 'while true; do php artisan workflow:v2:repair-pass --json; sleep 10; done' + environment: *runtime-env + depends_on: + server: + condition: service_healthy + mysql: + image: mysql:8.0 + environment: + MYSQL_DATABASE: durable_workflow + MYSQL_USER: workflow + MYSQL_PASSWORD: workflow + MYSQL_ROOT_PASSWORD: root + tmpfs: ["/var/lib/mysql"] + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] + interval: 3s + timeout: 2s + retries: 20 + redis: + image: redis:7-alpine + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 2s + retries: 10 diff --git a/docker-compose.native-cancellation.yml b/docker-compose.native-cancellation.yml new file mode 100644 index 0000000..8ca3cdd --- /dev/null +++ b/docker-compose.native-cancellation.yml @@ -0,0 +1,15 @@ +# Source qualification only. Published Composer authority is retained in the image. +services: + bootstrap: + volumes: + - ${DURABLE_WORKFLOW_NATIVE_SOURCE:?exact Native checkout}:/app/vendor/durable-workflow/workflow:ro + server: + volumes: + - ${DURABLE_WORKFLOW_NATIVE_SOURCE:?exact Native checkout}:/app/vendor/durable-workflow/workflow:ro + - ./tests/fixtures/source/native-scope-request.php:/app/sdk-source-fixtures/native-scope-request.php:ro + worker: + volumes: + - ${DURABLE_WORKFLOW_NATIVE_SOURCE:?exact Native checkout}:/app/vendor/durable-workflow/workflow:ro + repair: + volumes: + - ${DURABLE_WORKFLOW_NATIVE_SOURCE:?exact Native checkout}:/app/vendor/durable-workflow/workflow:ro diff --git a/docs/cooperative-cancellation-design.md b/docs/cooperative-cancellation-design.md new file mode 100644 index 0000000..a414565 --- /dev/null +++ b/docs/cooperative-cancellation-design.md @@ -0,0 +1,199 @@ +# Cooperative cancellation source design + +This document describes the unfinished source candidate for shared cancellation +issue 136. Default worker protocol remains 1.19. Cooperation requires explicit +protocol 1.20 opt-in and a compatible Server and Native backend. The shared +specification and published mixed-language qualification remain pending. + +## Recorded context + +`WorkflowContext::cancellation_context()` returns the original immutable metadata +after committed authored cancellation delivery. Earlier workflow code receives +`None`. The existing `Error::CooperativeCancellationRequested` carries the same +snapshot in `request.context`. Poll and heartbeat observations do not supply +workflow-facing context, even when their transport object contains extra fields. + +`CancellationContext::remaining()` returns a `Result` from the accepted +cleanup deadline, bounded by the original global deadline, and the recorded +blocking boundary consumed by this workflow replay. A scoped child can have an +earlier authority ceiling. +Committed delivery starts the clock. Activity/child results, timer fires, signal +and condition resolutions advance it. Parallel groups use their consumed +members and exclude results later than the failure returned to workflow code. +Selection uses its committed winner marker, then the result or first cancellation +receipt of a handle that workflow code actually awaits. Synchronous side effects, +version markers, memo and search attributes preserve the same budget before and +after persistence. Recorded clock skew cannot increase the budget, microseconds +are preserved, and expiry returns zero. + +The helper reads no host clock and cannot renew a deadline. Contexts restored +from portable metadata, ended or unrelated workflow replays, and missing or +invalid committed timestamps return an explicit error. A weak replay binding +does not keep workflow history alive. The Server independently enforces the +original deadline while a worker is suspended or absent. + +`CancellationContext` and `CancellationLineage` expose read-only accessors. +Metadata includes local and root request IDs, root workflow instance and run IDs, +the immediate parent request ID, original reason, requester/source, root request +time, original deadline and ordered lineage. Requester fields are limited to +caller type, ID and label. Timestamp helpers return immutable UTC dates. +`to_value()` produces a detached portable snapshot. + +A scoped child context preserves its complete immutable `scope_origin` alongside +the local run lineage. `ScopedCancellationContext::root_context()` retains the +original root deadline. Its scoped lineage records each run, instance, scope, +request identity and accepted authority ceiling, including intermediate scopes +in the same run. The child deadline can narrow that ceiling and never extend it. +Legacy version 1 snapshots remain readable. Version 2 parsing rejects altered +root metadata, substituted ancestry and widened budgets. This metadata support +does not advertise scoped execution while that source contract is unfinished. + +A child keeps the root request time and cleanup budget even if its local request +is accepted later. Cold replay restores the object from canonical request history. +Explicit checks and shielded cleanup retain it. The parser validates local +request/run binding, cycles, budget and delivery snapshot equality. Older +cancellation histories without rich context keep their existing delivery behavior +with `context == None`. + +## Child policies + +`ChildWorkflowOptions` accepts `CancellationPolicy` through its builder for +ordinary, parallel and selected child calls. + +```rust +ChildWorkflowOptions::new("python-workers") + .cancellation_policy(CancellationPolicy::WaitCancellationCompleted) + .parent_close_policy(ParentClosePolicy::RequestCancellation) +``` + +`TryCancel` requests child cleanup and delivers parent cancellation without +waiting. `WaitCancellationCompleted` parks the parent until the child reaches a +recorded terminal outcome, releasing its task claim for other work. `Abandon` +leaves the child independent and preserves the historical default. Parent +closure remains separate. `RequestCancellation` requests genuine cooperative +cleanup with the original lineage and budget. `RequestCancel` retains its +legacy terminal behavior. + +Cold replay compares both policies, including cancellation delivery and groups. +Omitted historical fields preserve the `Abandon` defaults. Later events with +missing fields retain the scheduled snapshot. Invalid or conflicting policy +history fails explicitly. A worker without the cooperation opt-in refuses the +commands before completion with its identity and required protocol. Server also +checks the immutable task claim and installed backend. + +## Remote Activity policies + +`ActivityOptions::cancellation_policy()` accepts the same `CancellationPolicy` +enum for ordinary, parallel and selection Activity calls. Omission retains the +historical Try behavior and wire shape. + +```rust +ActivityOptions::new() + .cancellation_policy(CancellationPolicy::WaitCancellationCompleted) +``` + +Try requests cancellation and continues without waiting for the stop receipt. +Wait delays workflow delivery until the original remote attempt's callback drop +is durably acknowledged. Abandon leaves work independent after parent +cancellation. It requires a finite positive `schedule_to_close_timeout` and +keeps that original deadline as its total lifetime. It does not extend the +parent's cleanup budget. + +The opted-in managed worker runs at most `max_concurrent_activity_tasks` +callbacks concurrently and joins all slots before deregistration. Reserve an +available activity slot for remote cleanup when independent Abandon work uses +the same worker. Workflow polling continues independently. Default protocol +1.19 workers retain their existing serial activity loop. + +Explicit policies require cooperative worker opt-in, protocol 1.20 and a +compatible installed backend. Unsupported workers identify their operation, +identity and required protocol before submission. Server also checks the +original immutable claim. Replay compares the policy against canonical history +before ordinary/group/selection settlement or cancellation delivery. Later +events with missing fields retain the original policy. Invalid, conflicting +and changed history fails explicitly. Historical omission retains Try. + +Connected Source cases include explicit Try/Wait callback drop without app +heartbeats, receipt ordering for Wait and stale-result refusal. Bounded Abandon +checks that the callback future survives parent closure, commits independent +completion under its original total timeout and cannot publish a second outcome +or reopen the parent. These tests supervise async callback futures. Detached +threads, processes and downstream effects need their own cooperation and fencing. + +## Remote callback-stop receipts + +The managed worker drops its activity callback future before reporting a stop. +Cloned `ActivityContext` values remain fenced. Only a canonical cancellation +observation on the original task, attempt and owner supplies the local request ID +for `Client::acknowledge_activity_cancellation()`. A lost lease, transport error, +shutdown or malformed observation does not supply a cooperative stop receipt. +Callbacks that never started are not reported as dropped. + +The explicit protocol 1.20 receipt request has a five-second budget. It validates +the Server's original claim, request and history receipt. A failed acknowledgment +remains an error and cannot become a completion or a new cleanup budget. A +duplicate returns the original event. The report covers the managed callback +future. Detached threads, processes and downstream effects need their own +cooperating cancellation and fencing. + +Connected source qualification accepts an optional exact `native_commit` in +addition to `server_commit`. The Native checkout is mounted read-only while the +image retains its published Composer authority. The source lane verifies the +durable receipt for callbacks with and without application heartbeats, +duplicates, the original deadline and stale result refusal. Actions retains all +three source identities and scenario history. This does not qualify a published +Native image or package. + +## Rust API release boundary + +The candidate adds a variant to the existing exhaustive `ParentClosePolicy` +enum and fields to the public `ChildWorkflowOptions` and `ActivityOptions` +structs. It also adds a total-lifetime error category to the exhaustive public +`ActivityOptionsErrorKind` enum. Existing struct +literals and exhaustive matches need updates. The cooperative error variants +also extend an existing exhaustive public enum. The maintainer approved Rust +SDK 3.0.0 in the [October 3 release decision](https://github.com/durable-workflow/sdk-rust/pull/55#issuecomment-5966048960). +The [migration and release proposal](migrating-to-v3.md) covers these source +changes, historical defaults, upgrade order and rollback. Approval resolves the +major-version decision. Source qualification, protocol freeze and the published +mixed-language cancellation acceptance scenario remain release gates. The +candidate does not change a published artifact. + +## Candidate scoped delivery + +The private scope opt-ins consume scalar calls and flat or nested `parallel` +groups of remote activities, timers, children and conditions. The complete group +is checked against its original definitions, policies, member addresses and paths +before workflow code receives `CancellationScopeRequested`. Earlier completed +members do not hide pending siblings. A committed condition is interrupted +without evaluating its predicate. Replacement replay retains the original group +span, cleanup sequence, context and deadline. + +Shielded cleanup timers carry only the original scope, request and delivery +references. Server derives their immutable authority and rejects a timer that +would reach its ceiling. Unsupported local, signal, selection, incomplete or +mixed-scope groups are refused before the workflow factory runs. An ancestor +delivery restores each included descendant's original accepted context as code +unwinds. Shielded branches remain unaffected. Cleanup timers in each included +scope retain its own request, the ancestor receipt and its narrower authority +ceiling. Replacement replay preserves the recorded clock and immutable metadata. +Pending ancestor requests keep the original authored boundary without entering +cleanup. Competing roots, overlapping deliveries, subtree local callbacks and +root/scope composition remain gated. These opt-ins remain disabled by default +and do not advertise general scoped execution. Exact connected qualification +and the published acceptance scenario remain separate gates. + +## Remaining qualification + +Complete the remaining scoped consumers, competitive qualification and exact +published artifacts. Do not subtract the host clock from the deadline in workflow code. +The runtime continues enforcing the original deadline and fencing task and +activity ownership. Rust local activities and worker affinity remain unsupported. + +Connected qualification must cover the PHP parent, Python child, Rust remote +activity and PHP local activity together. Both callbacks must stop without +application heartbeats. A replacement after SIGKILL during cleanup must replay +the same boundary and finish before the original 30-second deadline. Record +supported workflow lease, heartbeat and repair settings with that scenario. +Exact published artifacts and one cascade inspection view remain required before +release claims. diff --git a/docs/migrating-to-v3.md b/docs/migrating-to-v3.md new file mode 100644 index 0000000..28f9780 --- /dev/null +++ b/docs/migrating-to-v3.md @@ -0,0 +1,122 @@ +# Rust SDK 3.0 migration and release proposal + +Rust SDK 3.0.0 is the approved major version for cooperative cancellation. This +document describes the source candidate. Publish it after the reviewed portable +contract is frozen and the release qualifications below pass. + +## Source changes + +The minimum supported Rust version remains 1.86. Existing workflow payloads use +the same Avro encoding. Protocol 1.19 remains the default, and cooperative +execution requires explicit protocol 1.20 support on the Server and worker. + +### Activity options + +`ActivityOptions` adds `cancellation_policy: Option`. +Applications using a complete struct literal must add `cancellation_policy: +None` or use `ActivityOptions::new()` and the existing builders. + +`None` preserves historical TryCancel behavior and the old command shape. +Explicit TryCancel or WaitCancellationCompleted requires cooperative support. +Abandon also requires a finite positive `schedule_to_close_timeout`. + +```rust +use durable_workflow::{ActivityOptions, CancellationPolicy}; +use std::time::Duration; + +let historical = ActivityOptions::new(); +let wait_for_stop = ActivityOptions::new() + .cancellation_policy(CancellationPolicy::WaitCancellationCompleted); +let independently_bounded = ActivityOptions::new() + .schedule_to_close_timeout(Duration::from_secs(60)) + .cancellation_policy(CancellationPolicy::Abandon); +``` + +### Child options and parent closure + +`ChildWorkflowOptions` adds `cancellation_policy: CancellationPolicy`. Complete +struct literals must add `CancellationPolicy::Abandon`, the historical default, +or use `ChildWorkflowOptions::new(task_queue)` and its builders. + +`ParentClosePolicy` adds `RequestCancellation`. Update exhaustive matches to +handle it. `RequestCancel` keeps its existing terminal semantics. + +```rust +use durable_workflow::{CancellationPolicy, ChildWorkflowOptions, ParentClosePolicy}; + +let historical = ChildWorkflowOptions::new("child-workers"); +let cooperative = ChildWorkflowOptions::new("child-workers") + .cancellation_policy(CancellationPolicy::WaitCancellationCompleted) + .parent_close_policy(ParentClosePolicy::RequestCancellation); +``` + +Operation cancellation and parent closure are separate decisions. A child +using WaitCancellationCompleted participates in cooperative cancellation at the +awaiting operation. RequestCancellation propagates a cooperative request when +the parent closes. Both retain the original root lineage and budget. + +### Errors and request APIs + +Update exhaustive matches on `ActivityOptionsErrorKind` for +`MissingTotalTimeout`. Update exhaustive matches on `Error` for +`CooperativeCancellationRequested`, `CooperativeCancellationUnavailable`, +`InvalidCooperativeCancellation` and `ActivityExecutionAbandoned`. + +Propagate cooperative workflow cancellation through the SDK's authored cleanup +path. Do not turn it into an ordinary workflow failure or publish an abandoned +Activity result. Capability and malformed-authority errors remain errors. + +Use `WorkflowHandle::request_cancellation()` for bounded cooperative cleanup, or +`request_selected_run_cancellation()` for an explicit run. The corresponding +Client methods are `request_workflow_cancellation()` and +`request_workflow_run_cancellation()`. Existing `cancel()` and +`cancel_selected_run()` remain terminal cancellation. + +`Worker::cooperative_cancellation(true)` opts into managed cooperation. Runtime +discovery and the immutable issued claim must support it. Unsupported capability +is an explicit diagnostic, never a silent conversion to terminal cancellation. + +## Existing runs and deployment order + +Omitted historical policies retain Activity TryCancel and child Abandon. Keep +those authored defaults while replaying existing runs. Changing a policy on an +already recorded operation is a replay mismatch. Introduce changed policies +through a new workflow type or the product's documented workflow versioning. + +1. Retain the old worker artifact and a tested recovery path. Qualify the exact + Server/Native upgrade with retained ordinary histories before deployment. +2. Upgrade workers without changing their authored historical defaults. Verify + ordinary published workflow execution and replay on the selected tuple. +3. Enable cooperation only after Server/Native discovery, protocol 1.20 and the + installed capability checks agree. Route new cooperative workflows to workers + that can replay their cancellation and operation-policy histories. +4. Inspect the cascade API/UI and durable cleanup outcomes before expanding use. + +Downgrading to 2.x is safe only for histories it can replay. Older workers cannot +take over runs requiring the new cooperative contract. Keep a compatible worker +available to finish those runs or use the documented recovery procedure before +rolling back a deployment. Do not delete history to make a downgrade appear safe. + +## Qualification and publication + +Release 3.0.0 only after current-source Rust 1.86/stable, replay corpus, official +Avro, package and documentation gates pass, nested-scope work is complete and the +portable contract is frozen. Qualify supported/default lease and repair settings. + +Publish an exact compatible tuple, then run the complete PHP parent, Python child, +Rust remote Activity and PHP local Activity scenario. Verify one root identity, +the original +30-second deadline, cooperative child delivery, both managed +callbacks stopped without application heartbeats, stale publication fenced, +actual workflow-process SIGKILL during cleanup, replacement replay of the same +boundary and consumed budget, duplicate identity/deadline, both runs Cancelled +before the original deadline and one coherent API/UI cascade. + +Verify crates.io availability, source provenance, archive checksum and a fresh +Rust 1.86 consumer. Update the published examples and documentation to that exact +qualified artifact. Shared cancellation issue 136 stays open until its complete +customer outcome and competitive qualification are demonstrated. + +Managed async callback supervision does not undo external effects or stop +detached threads and processes. Cooperating downstream systems still need +idempotency and fencing. Rust local Activities and worker affinity remain outside +this release's supported cancellation surfaces. diff --git a/scripts/ci/publish-rust-sdk.sh b/scripts/ci/publish-rust-sdk.sh index b8550d7..61e1a0a 100755 --- a/scripts/ci/publish-rust-sdk.sh +++ b/scripts/ci/publish-rust-sdk.sh @@ -116,7 +116,7 @@ if [[ "$package_version" != "$product_train" ]]; then printf 'Rust SDK package version must match its product train metadata: %s, %s\n' "$package_version" "$product_train" >&2 exit 1 fi -if [[ "$compatibility_authority" != "protocol-manifests" || "$server_compatibility" != "2.0.0" || "$qualified_server_version" != "2.4.0" || "$server_worker_protocols" != ">=1.19,<2.0" ]]; then +if [[ "$compatibility_authority" != "protocol-manifests" || "$server_compatibility" != "2.0.0" || ! "$qualified_server_version" =~ ^2\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[1-9][0-9]*)?$ || "$server_worker_protocols" != ">=1.19,<2.0" ]]; then printf 'unexpected Rust SDK supported Server contract: %s, %s, %s, %s\n' "$compatibility_authority" "$server_compatibility" "$qualified_server_version" "$server_worker_protocols" >&2 exit 1 fi diff --git a/scripts/ci/test-publish-rust-sdk.py b/scripts/ci/test-publish-rust-sdk.py index 8f0f832..25d864a 100644 --- a/scripts/ci/test-publish-rust-sdk.py +++ b/scripts/ci/test-publish-rust-sdk.py @@ -18,10 +18,11 @@ PUBLISH = ROOT / "scripts" / "ci" / "publish-rust-sdk.sh" RELEASE_WORKFLOW = ROOT / ".github" / "workflows" / "release.yml" RELEASE_TOOLING_INSTALLER = ROOT / "scripts" / "ci" / "install-release-tooling.sh" -PACKAGE_VERSION = "2.1.5" +SOURCE_PACKAGE = tomllib.loads(MANIFEST.read_text(encoding="utf-8"))["package"] +PACKAGE_VERSION = SOURCE_PACKAGE["version"] PRODUCT_TRAIN = PACKAGE_VERSION SERVER_VERSIONS = "2.0.0" -QUALIFIED_SERVER_VERSION = "2.4.0" +QUALIFIED_SERVER_VERSION = SOURCE_PACKAGE["metadata"]["durable-workflow"]["qualified-server-version"] WORKER_PROTOCOL_VERSION = "1.19" SERVER_WORKER_PROTOCOLS = ">=1.19,<2.0" RELEASE_COMMIT = "0123456789abcdef0123456789abcdef01234567" @@ -429,6 +430,17 @@ def test_release_path_rejects_missing_condition_capability(self) -> None: result = self._publish(manifest) self.assertNotEqual(0, result.returncode) + def test_release_path_rejects_an_invalid_qualified_server_identity(self) -> None: + for version in ("3.0.0", "2.5", "2.5.0-rc.0", "2.5.0-preview.1"): + with self.subTest(version=version): + manifest = self._manifest_with( + f'qualified-server-version = "{QUALIFIED_SERVER_VERSION}"', + f'qualified-server-version = "{version}"', + ) + result = self._publish(manifest) + self.assertNotEqual(0, result.returncode) + self.assertIn("supported Server contract", result.stderr) + def test_release_path_rejects_package_without_current_release_notes(self) -> None: changelog = self.temp / "CHANGELOG.md" changelog.write_text( diff --git a/src/cancellation_context.rs b/src/cancellation_context.rs new file mode 100644 index 0000000..1917bce --- /dev/null +++ b/src/cancellation_context.rs @@ -0,0 +1,618 @@ +use super::*; +use chrono::{SecondsFormat, Utc}; +use std::sync::Weak; + +/// One immutable local request in the ordered cancellation lineage. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CancellationLineage { + request_id: String, + workflow_instance_id: String, + workflow_run_id: String, +} + +impl CancellationLineage { + pub fn request_id(&self) -> &str { + &self.request_id + } + + pub fn workflow_instance_id(&self) -> &str { + &self.workflow_instance_id + } + + pub fn workflow_run_id(&self) -> &str { + &self.workflow_run_id + } + + fn to_value(&self) -> Value { + json!({ + "request_id": self.request_id, + "workflow_instance_id": self.workflow_instance_id, + "workflow_run_id": self.workflow_run_id, + }) + } +} + +/// Original cancellation metadata restored from canonical request history. +/// +/// Fields and nested metadata have read-only accessors. Descendants retain +/// the original root identity, request time and cleanup budget. +#[derive(Clone, Debug)] +pub struct CancellationContext { + request_id: String, + root_request_id: String, + root_workflow_instance_id: String, + root_workflow_run_id: String, + parent_request_id: Option, + reason: Option, + requester: BTreeMap, + source: String, + requested_at: DateTime, + cleanup_deadline_at: DateTime, + lineage: Vec, + scope_origin: Option>, + replay: Option>>, +} + +impl PartialEq for CancellationContext { + fn eq(&self, other: &Self) -> bool { + self.request_id == other.request_id + && self.root_request_id == other.root_request_id + && self.root_workflow_instance_id == other.root_workflow_instance_id + && self.root_workflow_run_id == other.root_workflow_run_id + && self.parent_request_id == other.parent_request_id + && self.reason == other.reason + && self.requester == other.requester + && self.source == other.source + && self.requested_at == other.requested_at + && self.cleanup_deadline_at == other.cleanup_deadline_at + && self.lineage == other.lineage + && self.scope_origin == other.scope_origin + } +} + +impl Eq for CancellationContext {} + +fn invalid_context(message: &str) -> Error { + Error::InvalidCooperativeCancellation(message.to_owned()) +} + +fn context_text<'a>(value: &'a Value, key: &str) -> Result<&'a str> { + value[key] + .as_str() + .filter(|text| !text.trim().is_empty()) + .ok_or_else(|| invalid_context("cancellation context identity must be a non-empty string")) +} + +fn nullable_text(value: &Value, key: &str, allow_empty: bool) -> Result> { + match value.get(key) { + None | Some(Value::Null) => Ok(None), + Some(Value::String(text)) if allow_empty || !text.is_empty() => Ok(Some(text.clone())), + _ => Err(invalid_context( + "cancellation parent identity or reason is invalid", + )), + } +} + +impl CancellationContext { + /// Read legacy v1 and candidate scoped v2 without granting a new cleanup budget. + pub fn from_value(value: &Value) -> Result { + let scope_origin = match value["schema"].as_str() { + Some("durable-workflow.cancellation-context/v2") => Some(Box::new( + ScopedCancellationContext::from_value(&value["scope_origin"])?, + )), + Some("durable-workflow.cancellation-context/v1") => { + if value.get("scope_origin").is_some() + || value.get("scope_authority_deadline_at").is_some() + { + return Err(invalid_context( + "legacy cancellation context cannot discard a scoped origin", + )); + } + None + } + _ => return Err(invalid_context("unsupported cancellation context schema")), + }; + let requester = value["requester"] + .as_object() + .filter(|requester| !requester.is_empty()) + .ok_or_else(|| invalid_context("cancellation requester must identify its caller"))?; + let mut normalized_requester = BTreeMap::new(); + for (key, value) in requester { + let text = value + .as_str() + .filter(|text| !text.is_empty()) + .ok_or_else(|| { + invalid_context("cancellation requester contains unsupported metadata") + })?; + if !matches!(key.as_str(), "type" | "id" | "label") { + return Err(invalid_context( + "cancellation requester contains unsupported metadata", + )); + } + normalized_requester.insert(key.clone(), text.to_owned()); + } + let lineage = value["lineage"] + .as_array() + .filter(|lineage| !lineage.is_empty()) + .ok_or_else(|| invalid_context("cancellation lineage must contain the root request"))?; + let mut normalized = Vec::new(); + for entry in lineage { + normalized.push(CancellationLineage { + request_id: context_text(entry, "request_id")?.to_owned(), + workflow_instance_id: context_text(entry, "workflow_instance_id")?.to_owned(), + workflow_run_id: context_text(entry, "workflow_run_id")?.to_owned(), + }); + } + let request_id = context_text(value, "request_id")?.to_owned(); + let root_request_id = context_text(value, "root_request_id")?.to_owned(); + let root_workflow_instance_id = + context_text(value, "root_workflow_instance_id")?.to_owned(); + let root_workflow_run_id = context_text(value, "root_workflow_run_id")?.to_owned(); + let parent_request_id = nullable_text(value, "parent_request_id", false)?; + let reason = nullable_text(value, "reason", true)?; + let requests: BTreeSet<_> = normalized.iter().map(|entry| &entry.request_id).collect(); + let runs: BTreeSet<_> = normalized + .iter() + .map(|entry| &entry.workflow_run_id) + .collect(); + if requests.len() != normalized.len() || runs.len() != normalized.len() { + return Err(invalid_context( + "cancellation lineage cannot contain a cycle", + )); + } + let expected_parent = normalized + .iter() + .rev() + .nth(1) + .map(|entry| &entry.request_id); + if normalized[0].request_id != root_request_id + || normalized[0].workflow_instance_id != root_workflow_instance_id + || normalized[0].workflow_run_id != root_workflow_run_id + || normalized.last().unwrap().request_id != request_id + || (scope_origin.is_none() && parent_request_id.as_ref() != expected_parent) + { + return Err(invalid_context( + "cancellation lineage does not match its request identities", + )); + } + let requested_at = DateTime::parse_from_rfc3339(context_text(value, "requested_at")?) + .map_err(|_| invalid_context("cancellation request timestamp is invalid"))? + .with_timezone(&Utc); + let cleanup_deadline_at = + DateTime::parse_from_rfc3339(context_text(value, "cleanup_deadline_at")?) + .map_err(|_| invalid_context("cancellation deadline timestamp is invalid"))? + .with_timezone(&Utc); + if cleanup_deadline_at <= requested_at { + return Err(invalid_context( + "cancellation deadline must follow the original request", + )); + } + let context = Self { + request_id, + root_request_id, + root_workflow_instance_id, + root_workflow_run_id, + parent_request_id, + reason, + requester: normalized_requester, + source: context_text(value, "source")?.to_owned(), + requested_at, + cleanup_deadline_at, + lineage: normalized, + scope_origin, + replay: None, + }; + context.assert_scope_origin(value)?; + Ok(context) + } + + fn assert_scope_origin(&self, value: &Value) -> Result<()> { + let Some(origin) = self.scope_origin.as_deref() else { + return Ok(()); + }; + let root = &origin.root_context; + let last = self.lineage.last().unwrap(); + let authority = + DateTime::parse_from_rfc3339(context_text(value, "scope_authority_deadline_at")?) + .map_err(|_| invalid_context("scoped cancellation authority timestamp is invalid"))? + .with_timezone(&Utc); + if self.parent_request_id.as_deref() != Some(origin.request_id()) + || self.root_request_id != root.root_request_id + || self.root_workflow_instance_id != root.root_workflow_instance_id + || self.root_workflow_run_id != root.root_workflow_run_id + || self.reason != root.reason + || self.requester != root.requester + || self.source != root.source + || self.requested_at != root.requested_at + || self.cleanup_deadline_at != authority + || self.cleanup_deadline_at > origin.deadline() + || origin.lineage.iter().any(|entry| { + entry.request_id == last.request_id || entry.workflow_run_id == last.workflow_run_id + }) + { + return Err(invalid_context( + "run cancellation does not preserve its original scope context", + )); + } + let mut expected = vec![root.lineage[0].clone()]; + for entry in &origin.lineage { + if entry.workflow_run_id == root.root_workflow_run_id { + continue; + } + let address = CancellationLineage { + request_id: entry.request_id.clone(), + workflow_instance_id: entry.workflow_instance_id.clone(), + workflow_run_id: entry.workflow_run_id.clone(), + }; + if expected.last().unwrap().workflow_run_id == entry.workflow_run_id { + *expected.last_mut().unwrap() = address; + } else { + expected.push(address); + } + } + expected.push(last.clone()); + if self.lineage != expected { + return Err(invalid_context( + "run cancellation lineage discards or replaces its original scope ancestry", + )); + } + Ok(()) + } + + pub fn request_id(&self) -> &str { + &self.request_id + } + pub fn root_request_id(&self) -> &str { + &self.root_request_id + } + pub fn root_workflow_instance_id(&self) -> &str { + &self.root_workflow_instance_id + } + pub fn root_workflow_run_id(&self) -> &str { + &self.root_workflow_run_id + } + pub fn parent_request_id(&self) -> Option<&str> { + self.parent_request_id.as_deref() + } + pub fn reason(&self) -> Option<&str> { + self.reason.as_deref() + } + pub fn requester(&self) -> &BTreeMap { + &self.requester + } + pub fn source(&self) -> &str { + &self.source + } + pub fn requested_at(&self) -> DateTime { + self.requested_at + } + pub fn deadline(&self) -> DateTime { + self.cleanup_deadline_at + } + pub fn scope_origin(&self) -> Option<&ScopedCancellationContext> { + self.scope_origin.as_deref() + } + + /// Remaining cleanup budget at the last blocking result consumed by this replay. + /// + /// Never reads host time. Detached metadata, an ended replay, and a missing or + /// invalid committed timestamp return an explicit error. Expiry returns zero. + pub fn remaining(&self) -> Result { + let state = self + .replay + .as_ref() + .and_then(Weak::upgrade) + .filter(cancellation_replay_clock::is_active) + .ok_or_else(|| { + invalid_context("remaining() is available only in its active workflow replay") + })?; + let time = state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)? + .cancellation_time()?; + Ok((self.cleanup_deadline_at - time) + .to_std() + .unwrap_or(Duration::ZERO)) + } + + pub(super) fn with_replay(mut self, replay: Option>>) -> Self { + self.replay = replay; + self + } + pub fn lineage(&self) -> &[CancellationLineage] { + &self.lineage + } + + /// Detached metadata in the portable context schema. + pub fn to_value(&self) -> Value { + let mut value = json!({ + "schema": if self.scope_origin.is_none() { "durable-workflow.cancellation-context/v1" } + else { "durable-workflow.cancellation-context/v2" }, + "request_id": self.request_id, "root_request_id": self.root_request_id, + "root_workflow_instance_id": self.root_workflow_instance_id, + "root_workflow_run_id": self.root_workflow_run_id, + "parent_request_id": self.parent_request_id, "reason": self.reason, + "requester": self.requester, "source": self.source, + "requested_at": self.requested_at.to_rfc3339_opts(SecondsFormat::Micros, true), + "cleanup_deadline_at": self.cleanup_deadline_at.to_rfc3339_opts(SecondsFormat::Micros, true), + "lineage": self.lineage.iter().map(CancellationLineage::to_value).collect::>(), + }); + if let Some(origin) = &self.scope_origin { + value["scope_origin"] = origin.to_value(); + value["scope_authority_deadline_at"] = value["cleanup_deadline_at"].clone(); + } + value + } +} + +/// One immutable scope address and its bounded cleanup deadline. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScopedCancellationLineage { + request_id: String, + workflow_instance_id: String, + workflow_run_id: String, + scope_id: String, + cleanup_deadline_at: DateTime, +} + +impl ScopedCancellationLineage { + pub fn request_id(&self) -> &str { + &self.request_id + } + pub fn workflow_instance_id(&self) -> &str { + &self.workflow_instance_id + } + pub fn workflow_run_id(&self) -> &str { + &self.workflow_run_id + } + pub fn scope_id(&self) -> &str { + &self.scope_id + } + pub fn deadline(&self) -> DateTime { + self.cleanup_deadline_at + } + + fn to_value(&self) -> Value { + json!({ + "request_id": self.request_id, "workflow_instance_id": self.workflow_instance_id, + "workflow_run_id": self.workflow_run_id, "scope_id": self.scope_id, + "cleanup_deadline_at": self.cleanup_deadline_at.to_rfc3339_opts(SecondsFormat::Micros, true), + }) + } +} + +/// Original scope ancestry carried by a candidate cooperative child request. +/// Reading this immutable metadata does not authorize entering a scope body. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScopedCancellationContext { + root_context: CancellationContext, + lineage: Vec, +} + +fn assert_context_keys(value: &Value, keys: &[&str]) -> Result<()> { + let object = value + .as_object() + .ok_or_else(|| invalid_context("scoped context must be an object"))?; + if object.len() != keys.len() || keys.iter().any(|key| !object.contains_key(*key)) { + return Err(invalid_context( + "scoped cancellation context has missing or unsupported fields", + )); + } + Ok(()) +} + +impl ScopedCancellationContext { + /// Remaining original scope authority at the last consumed durable result. + #[doc(hidden)] + pub fn remaining(&self) -> Result { + let state = self + .root_context + .replay + .as_ref() + .and_then(Weak::upgrade) + .filter(cancellation_replay_clock::is_active) + .ok_or_else(|| { + invalid_context("remaining() is available only in its active scope replay") + })?; + let state = state.lock().map_err(|_| Error::WorkflowStatePoisoned)?; + let (_, ceiling) = state + .scope_delivery + .as_ref() + .and_then(|replay| replay.contexts.get(self.scope_id())) + .filter(|(original, _)| original == self) + .ok_or_else(|| { + invalid_context("scope remaining() requires its original consumed delivery") + })?; + Ok((self.deadline().min(*ceiling) - state.cancellation_time()?) + .to_std() + .unwrap_or(Duration::ZERO)) + } + + pub(super) fn with_replay(mut self, replay: Option>>) -> Self { + self.root_context = self.root_context.with_replay(replay); + self + } + + pub(super) fn from_run_context(context: &CancellationContext) -> Result { + let mut root = context.to_value(); + let mut lineage = Vec::new(); + if let Some(origin) = context.scope_origin() { + root = origin.root_context().to_value(); + lineage.extend( + origin + .lineage() + .iter() + .map(ScopedCancellationLineage::to_value), + ); + let local = context.lineage().last().unwrap(); + lineage.push(json!({"request_id":context.request_id(), + "workflow_instance_id":local.workflow_instance_id(), + "workflow_run_id":local.workflow_run_id(), "scope_id":"root", + "cleanup_deadline_at":context.deadline().to_rfc3339_opts(SecondsFormat::Micros, true)})); + } else { + root["request_id"] = json!(context.root_request_id()); + root["parent_request_id"] = Value::Null; + root["lineage"] = json!([context.lineage()[0].to_value()]); + lineage.extend(context.lineage().iter().map(|entry| json!({ + "request_id":entry.request_id(), "workflow_instance_id":entry.workflow_instance_id(), + "workflow_run_id":entry.workflow_run_id(), "scope_id":"root", + "cleanup_deadline_at":context.deadline().to_rfc3339_opts(SecondsFormat::Micros, true)}))); + } + Self::from_value( + &json!({"schema":"durable-workflow.scoped-cancellation-context/v1", + "root_context":root, "lineage":lineage}), + ) + } + + pub fn from_value(value: &Value) -> Result { + assert_context_keys(value, &["schema", "root_context", "lineage"])?; + if value["schema"] != "durable-workflow.scoped-cancellation-context/v1" + || value["root_context"]["schema"] != "durable-workflow.cancellation-context/v1" + { + return Err(invalid_context( + "scoped cancellation requires its original root context", + )); + } + let root = CancellationContext::from_value(&value["root_context"])?; + let root_value = root.to_value(); + let root_keys: Vec<_> = root_value + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + assert_context_keys(&value["root_context"], &root_keys)?; + if root.request_id != root.root_request_id + || root.parent_request_id.is_some() + || root.lineage.len() != 1 + { + return Err(invalid_context( + "scoped cancellation root must contain the original root request", + )); + } + let lineage = value["lineage"] + .as_array() + .filter(|rows| !rows.is_empty()) + .ok_or_else(|| { + invalid_context("scoped cancellation lineage must contain its root address") + })?; + let mut normalized = Vec::new(); + let mut requests = BTreeSet::new(); + let mut addresses = BTreeSet::new(); + let mut instances_by_run = BTreeMap::new(); + let mut last_run = String::new(); + let mut deadline = root.deadline(); + for (index, entry) in lineage.iter().enumerate() { + assert_context_keys( + entry, + &[ + "request_id", + "workflow_instance_id", + "workflow_run_id", + "scope_id", + "cleanup_deadline_at", + ], + )?; + let address = ScopedCancellationLineage { + request_id: context_text(entry, "request_id")?.to_owned(), + workflow_instance_id: context_text(entry, "workflow_instance_id")?.to_owned(), + workflow_run_id: context_text(entry, "workflow_run_id")?.to_owned(), + scope_id: context_text(entry, "scope_id")?.to_owned(), + cleanup_deadline_at: DateTime::parse_from_rfc3339(context_text( + entry, + "cleanup_deadline_at", + )?) + .map_err(|_| invalid_context("scoped cancellation deadline timestamp is invalid"))? + .with_timezone(&Utc), + }; + if index == 0 + && (address.request_id != root.request_id + || address.workflow_instance_id != root.root_workflow_instance_id + || address.workflow_run_id != root.root_workflow_run_id + || address.cleanup_deadline_at != root.deadline()) + { + return Err(invalid_context( + "scoped cancellation root address does not match its request", + )); + } + if !requests.insert(address.request_id.clone()) + || !addresses.insert((address.workflow_run_id.clone(), address.scope_id.clone())) + { + return Err(invalid_context( + "scoped cancellation cannot repeat a request or address", + )); + } + if let Some(instance) = instances_by_run.get(&address.workflow_run_id) { + if instance != &address.workflow_instance_id || last_run != address.workflow_run_id + { + return Err(invalid_context( + "scoped cancellation cannot reenter or reassign an earlier run", + )); + } + } + if address.cleanup_deadline_at <= root.requested_at + || address.cleanup_deadline_at > deadline + { + return Err(invalid_context( + "scoped cancellation cannot extend a descendant budget", + )); + } + instances_by_run.insert( + address.workflow_run_id.clone(), + address.workflow_instance_id.clone(), + ); + last_run = address.workflow_run_id.clone(); + deadline = address.cleanup_deadline_at; + normalized.push(address); + } + Ok(Self { + root_context: root, + lineage: normalized, + }) + } + + pub fn root_context(&self) -> &CancellationContext { + &self.root_context + } + pub fn lineage(&self) -> &[ScopedCancellationLineage] { + &self.lineage + } + pub fn request_id(&self) -> &str { + &self.lineage.last().unwrap().request_id + } + pub fn parent_request_id(&self) -> Option<&str> { + self.lineage + .iter() + .rev() + .nth(1) + .map(|entry| entry.request_id.as_str()) + } + pub fn workflow_instance_id(&self) -> &str { + &self.lineage.last().unwrap().workflow_instance_id + } + pub fn workflow_run_id(&self) -> &str { + &self.lineage.last().unwrap().workflow_run_id + } + pub fn scope_id(&self) -> &str { + &self.lineage.last().unwrap().scope_id + } + pub fn root_scope_id(&self) -> &str { + &self.lineage[0].scope_id + } + pub fn requested_at(&self) -> DateTime { + self.root_context.requested_at() + } + pub fn root_deadline(&self) -> DateTime { + self.root_context.deadline() + } + pub fn deadline(&self) -> DateTime { + self.lineage.last().unwrap().cleanup_deadline_at + } + pub fn to_value(&self) -> Value { + json!({ + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": self.root_context.to_value(), + "lineage": self.lineage.iter().map(ScopedCancellationLineage::to_value).collect::>(), + }) + } +} diff --git a/src/cancellation_replay_clock.rs b/src/cancellation_replay_clock.rs new file mode 100644 index 0000000..dd9e9ea --- /dev/null +++ b/src/cancellation_replay_clock.rs @@ -0,0 +1,301 @@ +use super::*; +use chrono::Utc; +use std::{cell::RefCell, sync::Weak}; + +thread_local! { + static ACTIVE_REPLAY: RefCell>>> = const { RefCell::new(None) }; +} + +/// A binding is useful only while its original workflow future is being polled. +pub(super) struct ReplayGuard { + previous: Option>>, +} + +impl ReplayGuard { + pub(super) fn enter(ctx: &WorkflowContext) -> Result> { + let bound = ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)? + .cancellation_clock + .is_some(); + if !bound && ACTIVE_REPLAY.with(|active| active.borrow().is_none()) { + return Ok(None); + } + let previous = + ACTIVE_REPLAY.with(|active| active.replace(bound.then(|| Arc::downgrade(&ctx.state)))); + Ok(Some(Self { previous })) + } +} + +impl Drop for ReplayGuard { + fn drop(&mut self) { + ACTIVE_REPLAY.with(|active| active.replace(self.previous.take())); + } +} + +pub(super) fn active_binding() -> Option>> { + ACTIVE_REPLAY.with(|active| active.borrow().clone()) +} + +pub(super) fn is_active(state: &Arc>) -> bool { + ACTIVE_REPLAY.with(|active| { + active + .borrow() + .as_ref() + .and_then(Weak::upgrade) + .is_some_and(|current| Arc::ptr_eq(¤t, state)) + }) +} + +/// Index only committed blocking results. Synchronous metadata never enters this clock. +#[derive(Debug)] +pub(super) struct ReplayClock { + boundaries: HashMap, + failures: BTreeSet, + selections: HashMap, + cancellations: HashMap<(String, u64), usize>, + delivery_index: Option, + time: Option>, + available: bool, + started: bool, +} + +impl ReplayClock { + pub(super) fn new( + history: &[HistoryEvent], + commands: &[RecordedCommand], + delivery_index: Option, + ) -> Self { + let mut clock = Self { + boundaries: HashMap::new(), + failures: BTreeSet::new(), + selections: HashMap::new(), + cancellations: HashMap::new(), + delivery_index, + time: None, + available: false, + started: false, + }; + for (index, event) in history.iter().enumerate() { + if matches!( + event.event_type.as_str(), + "ActivityCompleted" + | "ActivityFailed" + | "ActivityCancelled" + | "ActivityTimedOut" + | "ChildRunCompleted" + | "ChildRunFailed" + | "ChildRunCancelled" + | "ChildRunTerminated" + | "TimerFired" + | "SignalApplied" + | "ConditionWaitSatisfied" + | "ConditionWaitTimedOut" + ) { + if let Some(sequence) = durable_event_sequence(event) { + if let std::collections::hash_map::Entry::Vacant(entry) = + clock.boundaries.entry(sequence) + { + entry.insert(index); + if matches!( + event.event_type.as_str(), + "ActivityFailed" + | "ActivityCancelled" + | "ActivityTimedOut" + | "ChildRunFailed" + | "ChildRunCancelled" + | "ChildRunTerminated" + ) { + clock.failures.insert(sequence); + } + } + } + } else if event.event_type == "SelectionResolved" { + if let Some(group) = event.payload["selection_group_id"].as_str() { + clock.selections.entry(group.to_owned()).or_insert(index); + } + } else if event.event_type == "SelectionOperationCancelled" { + if let (Some(group), Some(sequence)) = ( + event.payload["selection_group_id"].as_str(), + event.payload["member_base_sequence"].as_u64(), + ) { + clock + .cancellations + .entry((group.to_owned(), sequence)) + .or_insert(index); + } + } + } + // One logical grouped condition may have several physical waits. The + // group consumes its final resolved wait, as the operation replayer does. + let mut conditions = HashMap::new(); + for command in commands { + if let RecordedCommand::ConditionWait { + sequence, + occurrence_id, + parallel_group_path: Some(_), + .. + } = command + { + let original = *conditions.entry(occurrence_id).or_insert(*sequence); + if original != *sequence { + if let Some(index) = clock.boundaries.get(sequence).copied() { + clock.boundaries.insert(original, index); + } else { + clock.boundaries.remove(&original); + } + } + } + } + clock + } + + fn observe(&mut self, event: Option<&HistoryEvent>) { + let time = event.and_then(|event| { + event + .raw + .get("timestamp") + .filter(|value| !value.is_null()) + .or_else(|| event.raw.get("recorded_at")) + .and_then(Value::as_str) + .and_then(|time| DateTime::parse_from_rfc3339(time).ok()) + .map(|time| time.with_timezone(&Utc)) + }); + self.available = time.is_some(); + if let Some(time) = time { + self.time = Some(self.time.map_or(time, |previous| previous.max(time))); + } + } +} + +impl WorkflowState { + pub(super) fn observe_scope_cancellation_delivery(&mut self, event: &HistoryEvent) { + if let Some(clock) = &mut self.cancellation_clock { + clock.started = true; + clock.observe(Some(event)); + } + } + + pub(super) fn start_cancellation_clock(&mut self) { + if let Some(clock) = &mut self.cancellation_clock { + clock.started = true; + clock.observe( + clock + .delivery_index + .and_then(|index| self.history_events.get(index)), + ); + } + } + + pub(super) fn cancellation_time(&self) -> Result> { + self.cancellation_clock + .as_ref() + .filter(|clock| clock.started && clock.available) + .and_then(|clock| clock.time) + .ok_or_else(|| { + Error::InvalidCooperativeCancellation( + "remaining() requires a recorded consumed blocking timestamp".to_owned(), + ) + }) + } + + fn advance_cancellation_indices(&mut self, mut indices: Vec>) { + let Some(clock) = &mut self.cancellation_clock else { + return; + }; + if !clock.started { + return; + } + // A missing timestamp/result cannot borrow a different operation's clock. + indices.sort_unstable(); + indices.dedup(); + let mut missing = false; + for index in indices { + clock.observe(index.and_then(|index| self.history_events.get(index))); + missing |= !clock.available; + } + if missing { + clock.available = false; + } + } + + pub(super) fn advance_cancellation_sequence( + &mut self, + sequence: u64, + path: &[ParallelGroupMetadata], + ) { + if path.is_empty() { + self.advance_cancellation_sequences(&[sequence], None); + } + } + + pub(super) fn unavailable_cancellation_boundary(&mut self, path: &[ParallelGroupMetadata]) { + if path.is_empty() { + self.advance_cancellation_indices(vec![None]); + } + } + + pub(super) fn advance_cancellation_sequences( + &mut self, + sequences: &[u64], + failed_sequence: Option, + ) { + let Some(clock) = &self.cancellation_clock else { + return; + }; + let cutoff = failed_sequence.and_then(|sequence| clock.boundaries.get(&sequence).copied()); + let indices = sequences + .iter() + .filter_map(|sequence| { + let index = clock.boundaries.get(sequence).copied(); + if cutoff.is_some_and(|cutoff| index.is_some_and(|index| index > cutoff)) { + None + } else { + Some(index) + } + }) + .collect(); + self.advance_cancellation_indices(indices); + } + + pub(super) fn advance_cancellation_selection(&mut self, group: &str) { + let index = self + .cancellation_clock + .as_ref() + .and_then(|clock| clock.selections.get(group).copied()); + self.advance_cancellation_indices(vec![index]); + } + + pub(super) fn advance_cancellation_receipt(&mut self, group: &str, sequence: u64) { + let index = self.cancellation_clock.as_ref().and_then(|clock| { + clock + .cancellations + .get(&(group.to_owned(), sequence)) + .copied() + }); + self.advance_cancellation_indices(vec![index]); + } + + pub(super) fn advance_cancellation_handle( + &mut self, + handle: &DurableOperationHandle, + failed: bool, + ) { + let sequences = (handle.base_sequence + ..handle.base_sequence.saturating_add(handle.size as u64)) + .collect::>(); + let failure = self + .cancellation_clock + .as_ref() + .filter(|_| failed) + .and_then(|clock| { + sequences + .iter() + .filter(|sequence| clock.failures.contains(sequence)) + .min_by_key(|sequence| clock.boundaries.get(sequence)) + .copied() + }); + self.advance_cancellation_sequences(&sequences, failure); + } +} diff --git a/src/cancellation_scope.rs b/src/cancellation_scope.rs new file mode 100644 index 0000000..28600ce --- /dev/null +++ b/src/cancellation_scope.rs @@ -0,0 +1,599 @@ +use super::*; + +const OPEN_BUDGET: Duration = Duration::from_secs(5); +const MAX_SCOPE_SEQUENCE: u64 = i64::MAX as u64; +const MAX_HISTORY_PAGES: usize = 128; + +fn invalid() -> Error { + Error::InvalidCooperativeCancellation("invalid canonical cancellation scope opening".into()) +} + +fn identity(value: &str) -> bool { + !value.trim().is_empty() && value.len() <= 255 +} + +fn text<'a>(value: &'a Value, field: &str) -> Result<&'a str> { + value[field] + .as_str() + .filter(|value| identity(value)) + .ok_or_else(invalid) +} + +#[derive(Clone, Debug)] +pub(super) struct CanonicalScopeOpening { + pub scope_id: String, + pub parent_scope_id: String, + pub shield_parent: bool, +} + +#[derive(Clone, Default, Debug)] +pub(super) struct CancellationScopeHistory { + pub openings: BTreeMap, + pub memberships: BTreeMap, +} + +fn invalid_history(detail: &str) -> Error { + invalid_recorded_history( + "invalid_cancellation_scope_history", + 0, + "canonical scope tree and original membership", + "invalid history", + detail, + ) +} + +fn starts_with_workflow_start(events: &[HistoryEvent]) -> bool { + events + .first() + .is_some_and(|event| event.event_type == "WorkflowStarted") + || (events + .first() + .is_some_and(|event| event.event_type == "StartAccepted") + && events + .get(1) + .is_some_and(|event| event.event_type == "WorkflowStarted")) +} + +impl CancellationScopeHistory { + pub fn read(events: &[HistoryEvent], run_id: &str) -> Result { + let has_scopes = events + .iter() + .any(|event| event.event_type == "CancellationScopeOpened"); + if has_scopes && !starts_with_workflow_start(events) { + return Err(invalid_history( + "scope history lacks its original workflow start", + )); + } + let mut history = Self::default(); + let mut event_ids = BTreeSet::new(); + let mut scopes = BTreeMap::new(); + let mut namespace: Option<&str> = None; + let mut last_event_sequence = 0; + let mut last_opening = 0; + for event in events { + if has_scopes { + let event_id = event + .raw + .get("id") + .and_then(Value::as_str) + .filter(|value| identity(value)) + .ok_or_else(|| invalid_history("missing canonical event identity"))?; + let sequence = event + .raw + .get("sequence") + .and_then(Value::as_u64) + .filter(|value| *value > last_event_sequence && *value <= MAX_SCOPE_SEQUENCE) + .ok_or_else(|| invalid_history("canonical event order changed"))?; + let incoming_namespace = event + .raw + .get("namespace") + .and_then(Value::as_str) + .filter(|value| identity(value)) + .ok_or_else(|| invalid_history("missing canonical namespace"))?; + if !event_ids.insert(event_id) + || namespace.is_some_and(|previous| previous != incoming_namespace) + || !event.payload.is_object() + || event.event_type.trim().is_empty() + { + return Err(invalid_history( + "canonical event identity, namespace or payload changed", + )); + } + namespace = Some(incoming_namespace); + last_event_sequence = sequence; + } + let payload = &event.payload; + let sequence = payload["sequence"] + .as_u64() + .filter(|value| *value > 0 && *value <= MAX_SCOPE_SEQUENCE); + if event.event_type == "CancellationScopeOpened" { + let scope_id = text(payload, "scope_id") + .map_err(|_| invalid_history("invalid scope identity"))?; + let parent = text(payload, "parent_scope_id") + .map_err(|_| invalid_history("invalid parent identity"))?; + let sequence = + sequence.ok_or_else(|| invalid_history("invalid authored opening sequence"))?; + let shield = payload["shield_parent"] + .as_bool() + .ok_or_else(|| invalid_history("invalid shielding"))?; + if payload["schema"] != "durable-workflow.cancellation-scope/v1" + || run_id.is_empty() + || payload["workflow_run_id"].as_str() != Some(run_id) + || scope_id == "root" + || scopes.contains_key(scope_id) + || (parent != "root" && !scopes.contains_key(parent)) + || sequence <= last_opening + || history.memberships.contains_key(&sequence) + { + return Err(invalid_history("invalid canonical opening tree")); + } + scopes.insert(scope_id, sequence); + last_opening = sequence; + history.openings.insert( + sequence, + CanonicalScopeOpening { + scope_id: scope_id.into(), + parent_scope_id: parent.into(), + shield_parent: shield, + }, + ); + continue; + } + let admission = matches!( + event.event_type.as_str(), + "ActivityScheduled" + | "TimerScheduled" + | "ChildWorkflowScheduled" + | "ConditionWaitOpened" + | "SignalWaitOpened" + ); + let operation = admission + || matches!( + event.event_type.as_str(), + "ActivityStarted" + | "ActivityCompleted" + | "ActivityFailed" + | "ActivityTimedOut" + | "ActivityCancelled" + | "ActivityRetryScheduled" + | "TimerFired" + | "TimerCancelled" + | "ChildRunStarted" + | "ChildRunCompleted" + | "ChildRunFailed" + | "ChildRunCancelled" + | "ChildRunTerminated" + | "ConditionWaitSatisfied" + | "ConditionWaitTimedOut" + | "ConditionWaitCancelled" + | "SignalWaitReceived" + | "SignalWaitTimedOut" + | "SignalWaitCancelled" + ); + if !operation { + continue; + } + let mut membership: Option<&str> = None; + for snapshot in std::iter::once(payload).chain( + ["activity", "timer", "child_workflow"] + .iter() + .filter_map(|name| payload.get(name)), + ) { + let Some(value) = snapshot.get("cancellation_scope_id") else { + continue; + }; + let incoming = value + .as_str() + .filter(|value| identity(value)) + .ok_or_else(|| invalid_history("invalid operation scope membership"))?; + if membership.is_some_and(|previous| previous != incoming) { + return Err(invalid_history("contradictory operation scope membership")); + } + membership = Some(incoming); + } + if membership.is_none() && !admission { + continue; + } + let membership = membership.unwrap_or("root"); + if membership != "root" + && !sequence.is_some_and(|sequence| { + scopes + .get(membership) + .is_some_and(|opening| *opening < sequence) + }) + { + return Err(invalid_history( + "operation scope was not opened before original admission", + )); + } + let Some(sequence) = sequence else { + continue; + }; + if history.openings.contains_key(&sequence) + || history + .memberships + .get(&sequence) + .is_some_and(|previous| previous != membership) + { + return Err(invalid_history( + "operation changed its original scope membership", + )); + } + history.memberships.insert(sequence, membership.into()); + } + Ok(history) + } +} + +#[derive(Clone, Debug)] +pub(super) struct CancellationScopeOpening { + pub sequence: u64, + pub parent_scope_id: String, + pub shield_parent: bool, + pub command_count: usize, +} + +impl WorkflowContext { + pub(super) fn validate_scope_membership( + &self, + state: &mut WorkflowState, + cursor: usize, + ) -> Result<()> { + if !state.allow_cancellation_scope_authoring { + return Ok(()); + } + if let Some(replay) = &mut state.scope_delivery { + replay.active_scope = self.cancellation_scope_id.clone(); + } + if let Some(recorded) = state.recorded_commands.get(cursor) { + let sequence = recorded.sequence(); + let original = state + .cancellation_scope_memberships + .get(&sequence) + .map(String::as_str) + .or_else(|| { + state + .scope_delivery + .as_ref() + .and_then(|replay| replay.canonical.deliveries.get(&sequence)) + .map(|delivered| delivered.context.scope_id()) + }) + .unwrap_or("root"); + if original != self.cancellation_scope_id { + return Err(invalid_recorded_history( + "cancellation_scope_membership_changed", + sequence, + original, + &self.cancellation_scope_id, + "operation changed the scope where it was created", + )); + } + } + Ok(()) + } + + pub(super) fn apply_scope_membership(&self, command: &mut serde_json::Map) { + if self.cancellation_scope_id != "root" { + command.insert( + "cancellation_scope_id".into(), + json!(self.cancellation_scope_id), + ); + } + } + + /// Await the original durable opening, then run a body with its own context. + /// Operations created from that context retain their scope when awaited later. + /// Candidate authoring remains disabled on ordinary workers. + pub async fn cancellation_scope(&self, shield_parent: bool, body: F) -> Result + where + F: FnOnce(WorkflowContext) -> Fut, + Fut: Future>, + { + let scope_id = ScopeOpeningCall { + ctx: self.clone(), + shield_parent, + } + .await?; + let mut scoped = self.clone(); + scoped.cancellation_scope_id = scope_id; + body(scoped).await + } +} + +struct ScopeOpeningCall { + ctx: WorkflowContext, + shield_parent: bool, +} + +impl Future for ScopeOpeningCall { + type Output = Result; + + fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll { + let mut state = match self.ctx.state.lock() { + Ok(state) => state, + Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)), + }; + if !state.allow_cancellation_scope_authoring { + return Poll::Ready(Err(Error::CancellationScopeExecutionUnavailable)); + } + if state.cancellation_consumed { + return Poll::Ready(Err(Error::CancellationScopeExecutionUnavailable)); + } + if state + .cancellation_scope_opening + .as_ref() + .is_some_and(|opening| opening.command_count != state.commands.len()) + { + return Poll::Ready(Err(invalid_history( + "workflow authored commands after an uncommitted opening", + ))); + } + let sequence = (state.command_cursor as u64) + .saturating_add(state.commands.len() as u64) + .saturating_add(1); + if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() { + return match recorded { + RecordedCommand::CancellationScope { + sequence: original, + scope_id, + parent_scope_id, + shield_parent, + } if original == sequence + && parent_scope_id == self.ctx.cancellation_scope_id + && shield_parent == self.shield_parent => + { + state.command_cursor += 1; + Poll::Ready(Ok(scope_id)) + } + recorded => Poll::Ready(Err(invalid_recorded_history( + "cancellation_scope_opening_changed", + sequence, + "original scope opening, parent and shielding", + recorded.shape(), + "authored cancellation scope differs from committed history", + ))), + }; + } + if state.history_events.iter().any(|event| { + matches!( + event.event_type.as_str(), + "WorkflowCompleted" + | "WorkflowFailed" + | "WorkflowCancelled" + | "WorkflowTerminated" + | "WorkflowContinuedAsNew" + ) + }) { + return Poll::Ready(Err(invalid_recorded_history( + "cancellation_scope_opening_changed", + sequence, + "original scope opening", + "closed history", + "closed history cannot admit an unrecorded scope", + ))); + } + state.cancellation_scope_opening = Some(CancellationScopeOpening { + sequence, + parent_scope_id: self.ctx.cancellation_scope_id.clone(), + shield_parent: self.shield_parent, + command_count: state.commands.len(), + }); + Poll::Pending + } +} + +/// A scope opening proved against complete history on its original claim. +/// This proof does not advertise scoped workflow execution support. +#[derive(Clone, Debug)] +pub struct CancellationScopeOpenReceipt { + scope_id: String, + history_event_id: String, + sequence: u64, + parent_scope_id: String, + shield_parent: bool, + duplicate: bool, + history: Vec, +} + +impl CancellationScopeOpenReceipt { + pub fn scope_id(&self) -> &str { + &self.scope_id + } + pub fn history_event_id(&self) -> &str { + &self.history_event_id + } + pub fn sequence(&self) -> u64 { + self.sequence + } + pub fn parent_scope_id(&self) -> &str { + &self.parent_scope_id + } + pub fn shield_parent(&self) -> bool { + self.shield_parent + } + pub fn duplicate(&self) -> bool { + self.duplicate + } + pub fn history(&self) -> &[HistoryEvent] { + &self.history + } + + fn acknowledge<'a>(receipt: &'a Value, expected: &Value) -> Result<&'a str> { + if expected + .as_object() + .ok_or_else(invalid)? + .iter() + .any(|(key, value)| key != "namespace" && receipt.get(key) != Some(value)) + || receipt["opened"].as_bool() != Some(true) + || receipt["duplicate"].as_bool().is_none() + || receipt["claim_released"].as_bool() != Some(false) + || receipt.get("created_task_ids") != Some(&json!([])) + || receipt.get("reason") != Some(&Value::Null) + || text(receipt, "scope_id")? == "root" + { + return Err(invalid()); + } + text(receipt, "history_event_id")?; + receipt["history_refresh_page_token"] + .as_str() + .filter(|value| !value.trim().is_empty()) + .ok_or_else(invalid) + } + + fn from_history(receipt: &Value, events: &[Value], expected: &Value) -> Result { + Self::acknowledge(receipt, expected)?; + let kind = |index: usize| { + events + .get(index) + .and_then(|event| event.get("event_type").or_else(|| event.get("type"))) + .and_then(Value::as_str) + }; + if kind(0) != Some("WorkflowStarted") + && !(kind(0) == Some("StartAccepted") && kind(1) == Some("WorkflowStarted")) + { + return Err(invalid()); + } + let mut event_ids = BTreeSet::new(); + let mut scopes = BTreeSet::new(); + let mut last_event_sequence = 0; + let mut last_scope_sequence = 0; + let mut found = false; + let mut history = Vec::with_capacity(events.len()); + for event in events { + let event_id = text(event, "id")?; + let event_sequence = event["sequence"].as_u64().ok_or_else(invalid)?; + let payload = &event["payload"]; + let event_kind = event + .get("event_type") + .or_else(|| event.get("type")) + .and_then(Value::as_str) + .filter(|value| !value.trim().is_empty()) + .ok_or_else(invalid)?; + if !event_ids.insert(event_id) + || event_sequence <= last_event_sequence + || event_sequence > MAX_SCOPE_SEQUENCE + || event.get("namespace") != expected.get("namespace") + || !payload.is_object() + { + return Err(invalid()); + } + last_event_sequence = event_sequence; + if event_kind == "CancellationScopeOpened" { + let scope_id = text(payload, "scope_id")?; + let parent = text(payload, "parent_scope_id")?; + let sequence = payload["sequence"].as_u64().ok_or_else(invalid)?; + if payload["schema"] != "durable-workflow.cancellation-scope/v1" + || payload.get("workflow_run_id") != expected.get("workflow_run_id") + || scope_id == "root" + || scopes.contains(scope_id) + || (parent != "root" && !scopes.contains(parent)) + || payload["shield_parent"].as_bool().is_none() + || sequence <= last_scope_sequence + || sequence > MAX_SCOPE_SEQUENCE + { + return Err(invalid()); + } + scopes.insert(scope_id); + last_scope_sequence = sequence; + if Some(event_id) == receipt["history_event_id"].as_str() { + if ["scope_id", "sequence", "parent_scope_id", "shield_parent"] + .iter() + .any(|field| payload.get(*field) != receipt.get(*field)) + { + return Err(invalid()); + } + found = true; + } + } + history.push(serde_json::from_value(event.clone()).map_err(|_| invalid())?); + } + if !found { + return Err(invalid()); + } + CancellationScopeHistory::read(&history, text(expected, "workflow_run_id")?) + .map_err(|_| invalid())?; + Ok(Self { + scope_id: text(receipt, "scope_id")?.into(), + history_event_id: text(receipt, "history_event_id")?.into(), + sequence: receipt["sequence"].as_u64().ok_or_else(invalid)?, + parent_scope_id: text(receipt, "parent_scope_id")?.into(), + shield_parent: receipt["shield_parent"].as_bool().ok_or_else(invalid)?, + duplicate: receipt["duplicate"].as_bool().ok_or_else(invalid)?, + history, + }) + } +} + +impl Client { + /// Open a scope and prove its canonical identity on the original claim. + /// + /// This explicit protocol 1.20 operation keeps one five-second budget for + /// lost acknowledgement recovery and all history pages. No lease, worker + /// capability or cancellation execution authority is added by this proof. + pub async fn open_cancellation_scope_on_claim( + &self, + task: &WorkflowTask, + sequence: u64, + parent_scope_id: &str, + shield_parent: bool, + ) -> Result { + let (owner, run_id) = cooperative_cancellation::cancellation_claim(task)?; + if [ + task.task_id.as_str(), + owner, + run_id, + parent_scope_id, + self.namespace.as_str(), + ] + .iter() + .any(|value| !identity(value)) + || sequence == 0 + || sequence > MAX_SCOPE_SEQUENCE + { + return Err(invalid()); + } + let expected = json!({"task_id":task.task_id, "workflow_run_id":run_id, + "lease_owner":owner, "workflow_task_attempt":task.workflow_task_attempt, + "sequence":sequence, "parent_scope_id":parent_scope_id, + "shield_parent":shield_parent, "namespace":self.namespace}); + let path = format!( + "/worker/workflow-tasks/{}", + percent_encode_path_segment(&task.task_id) + ); + let body = json!({"lease_owner":owner, "workflow_task_attempt":task.workflow_task_attempt, + "sequence":sequence, "parent_scope_id":parent_scope_id,"shield_parent":shield_parent}); + tokio::time::timeout(OPEN_BUDGET, async { + let open_path = format!("{path}/cancellation-scopes/open"); + let request = || self.request_json::(reqwest::Method::POST, + &open_path, RequestProtocol::Worker("1.20"), Some(&body)); + let receipt = match request().await { + Err(error) if worker_operation_is_retryable(&error) => request().await?, + result => result?, + }; + let mut token = Some(CancellationScopeOpenReceipt::acknowledge(&receipt, &expected)?.to_owned()); + let mut seen = BTreeSet::new(); + let mut events = Vec::new(); + while let Some(current) = token.take() { + if seen.len() >= MAX_HISTORY_PAGES || !seen.insert(current.clone()) { return Err(invalid()); } + let page: Value = self.request_json(reqwest::Method::POST, &format!("{path}/history"), + RequestProtocol::Worker("1.20"), Some(&json!({"lease_owner":owner, + "workflow_task_attempt":task.workflow_task_attempt, "next_history_page_token":current, + "history_page_size":WORKFLOW_HISTORY_PAGE_SIZE}))).await?; + if page["task_id"].as_str() != Some(task.task_id.as_str()) + || page["workflow_task_attempt"].as_u64() != Some(task.workflow_task_attempt) + { return Err(invalid()); } + let batch = page["history_events"].as_array().ok_or_else(invalid)?; + if batch.len() > WORKFLOW_HISTORY_PAGE_SIZE as usize || batch.iter().any(|event| !event.is_object()) { + return Err(invalid()); + } + token = match page.get("next_history_page_token") { + Some(Value::Null) => None, + Some(Value::String(next)) if !next.trim().is_empty() && !batch.is_empty() => Some(next.clone()), + _ => return Err(invalid()), + }; + events.extend(batch.iter().cloned()); + } + CancellationScopeOpenReceipt::from_history(&receipt, &events, &expected) + }).await.map_err(|_| Error::Timeout)? + } +} diff --git a/src/cancellation_scope_history.rs b/src/cancellation_scope_history.rs new file mode 100644 index 0000000..f34870e --- /dev/null +++ b/src/cancellation_scope_history.rs @@ -0,0 +1,1613 @@ +//! Original accepted scope requests and frozen v5 delivery projections. +//! These facts alone grant no callback, stop, or publication authority. + +use super::*; +use chrono::{SecondsFormat, Utc}; +use serde::ser::{SerializeMap, SerializeSeq}; + +const FIELDS: [&str; 4] = [ + "activity_members", + "timer_members", + "wait_members", + "child_members", +]; +const GROUP_KEYS: [&str; 11] = [ + "parallel_group_id", + "parallel_group_kind", + "parallel_group_mode", + "parallel_group_base_sequence", + "parallel_group_size", + "parallel_group_index", + "selection_member_key", + "selection_member_index", + "selection_member_base_sequence", + "selection_member_size", + "selection_member_kind", +]; + +fn invalid(detail: &str) -> Error { + invalid_recorded_history( + "invalid_cancellation_scope_history", + 0, + "original accepted scope and frozen operation projection", + "invalid history", + detail, + ) +} + +fn identity(value: &Value) -> bool { + value + .as_str() + .is_some_and(|text| !text.trim().is_empty() && text.len() <= 255) +} + +fn text<'a>(value: &'a Value, key: &str) -> Result<&'a str> { + value + .get(key) + .filter(|value| identity(value)) + .and_then(Value::as_str) + .ok_or_else(|| invalid("scope proof requires its original identity")) +} + +fn positive(value: &Value) -> bool { + value + .as_u64() + .is_some_and(|value| value > 0 && value <= i64::MAX as u64) +} + +pub(super) fn timestamp(value: &Value) -> Result> { + value + .as_str() + .and_then(|value| DateTime::parse_from_rfc3339(value).ok()) + .map(|value| value.with_timezone(&Utc)) + .ok_or_else(|| invalid("scope authority requires an original timestamp with timezone")) +} + +pub(super) fn boundary_scope_states( + context: &ScopedCancellationContext, + preparation: &Value, +) -> Result)>> { + let mut states = BTreeMap::from([( + context.scope_id().to_owned(), + ( + context.clone(), + timestamp(&preparation["authority_deadline_at"])?, + ), + )]); + for member in preparation["descendant_members"] + .as_array() + .ok_or_else(|| invalid("scope preparation lacks its frozen subtree"))? + { + let descendant = ScopedCancellationContext::from_value(&member["cancellation"])?; + if descendant.root_context() != context.root_context() + || !descendant.lineage().starts_with(context.lineage()) + || states.contains_key(descendant.scope_id()) + { + return Err(invalid( + "scope descendant changes its original root or accepted lineage", + )); + } + states.insert( + descendant.scope_id().to_owned(), + (descendant, timestamp(&member["authority_deadline_at"])?), + ); + } + Ok(states) +} + +fn delivery_scope_states( + delivery: &HistoryEvent, + prefix: &[HistoryEvent], +) -> Result)>> { + let preparations: Vec<_> = prefix + .iter() + .filter(|row| { + row.event_type == "CancellationScopeDeliveryPrepared" + && row.raw["id"] == delivery.payload["preparation_history_event_id"] + }) + .collect(); + if preparations.len() != 1 + || preparations[0].raw["sequence"] + .as_u64() + .zip(delivery.raw["sequence"].as_u64()) + .is_none_or(|(before, after)| before >= after) + { + return Err(invalid( + "cleanup timer lacks its original canonical preparation", + )); + } + boundary_scope_states( + &ScopedCancellationContext::from_value(&delivery.payload["cancellation"])?, + &preparations[0].payload, + ) +} + +fn cleanup_timer(event: &HistoryEvent, prefix: &[HistoryEvent]) -> Result { + let p = &event.payload; + let Some(snapshot) = p.get("cancellation_cleanup") else { + if p["cancellation_scope_id"] + .as_str() + .is_some_and(|scope| scope != "root") + { + for delivery in prefix.iter().filter(|row| { + row.event_type == "CancellationScopeDelivered" + && row.raw["sequence"] + .as_u64() + .zip(event.raw["sequence"].as_u64()) + .is_some_and(|(delivery, timer)| delivery < timer) + }) { + if delivery_scope_states(delivery, prefix)? + .contains_key(text(p, "cancellation_scope_id")?) + { + return Err(invalid( + "cleanup timer omits its original delivery snapshot", + )); + } + } + } + return Ok(false); + }; + let matches: Vec<_> = prefix + .iter() + .filter(|row| { + row.event_type == "CancellationScopeDelivered" + && row.raw.get("id") == snapshot.get("delivery_history_event_id") + }) + .collect(); + if matches.len() != 1 { + return Err(invalid( + "cleanup timer lacks its earlier canonical delivery", + )); + } + let delivery = matches[0]; + let original = &delivery.payload; + let states = delivery_scope_states(delivery, prefix)?; + let (context, ceiling) = states + .get(text(p, "cancellation_scope_id")?) + .ok_or_else(|| invalid("cleanup timer changes its original frozen subtree membership"))?; + let ancestor = ScopedCancellationContext::from_value(&original["cancellation"])?; + let expected = json!({"scope_id":ancestor.scope_id(), "operation_scope_id":context.scope_id(), + "request_id":ancestor.request_id(), "root_request_id":ancestor.root_context().root_request_id(), + "delivery_history_event_id":event_id(delivery)?, + "preparation_history_event_id":original["preparation_history_event_id"], + "cleanup_deadline_at":canonical_time(ancestor.deadline()), + "authority_deadline_at":canonical_time(*ceiling)}); + let boundary = CancellationDelivery::from_payload(&json!({ + "workflow_command_id":original["request_id"], "sequence":original["sequence"], + "call_kind":original["call_kind"], "sequence_span":original["sequence_span"], + "operation_sequence":original["operation_sequence"], "operation_sequence_span":original["operation_sequence_span"]}))?; + if snapshot != &expected + || p["cancellation_scope_id"] != context.scope_id() + || p["sequence"] + .as_u64() + .is_none_or(|sequence| sequence < boundary.sequence + boundary.sequence_span) + || event.raw["sequence"].as_u64().is_none_or(|sequence| { + delivery.raw["sequence"] + .as_u64() + .is_none_or(|prior| prior >= sequence) + }) + || event_time(event)? < event_time(delivery)? + || event_time(event)? >= *ceiling + || timestamp(&p["fire_at"])? < event_time(event)? + || timestamp(&p["fire_at"])? >= *ceiling + || !p["timer_kind"].is_null() + { + return Err(invalid( + "cleanup timer changes its original delivery or authority ceiling", + )); + } + Ok(true) +} + +fn event_time(event: &HistoryEvent) -> Result> { + timestamp( + event + .raw + .get("timestamp") + .or_else(|| event.raw.get("recorded_at")) + .unwrap_or(&Value::Null), + ) +} + +fn event_id(event: &HistoryEvent) -> Result<&str> { + event + .raw + .get("id") + .filter(|value| identity(value)) + .and_then(Value::as_str) + .ok_or_else(|| invalid("scope proof lacks its original event identity")) +} +fn event_sequence(event: &HistoryEvent) -> Result { + event + .raw + .get("sequence") + .filter(|value| positive(value)) + .and_then(Value::as_u64) + .ok_or_else(|| invalid("scope proof lacks canonical event order")) +} +fn canonical_time(time: DateTime) -> String { + time.to_rfc3339_opts(SecondsFormat::Micros, true) +} +fn policy(value: &Value) -> bool { + matches!( + value.as_str(), + Some("try_cancel" | "wait_cancellation_completed" | "abandon") + ) +} + +// Only descriptor group objects use this key order. Do not change serde_json's +// global map behavior or any payload codec to reproduce Native's PHP hashes. +struct Descriptor<'a>(&'a Value); +impl Serialize for Descriptor<'_> { + fn serialize( + &self, + serializer: S, + ) -> std::result::Result { + match self.0 { + Value::Array(values) => { + let mut seq = serializer.serialize_seq(Some(values.len()))?; + for value in values { + seq.serialize_element(&Descriptor(value))?; + } + seq.end() + } + Value::Object(object) => { + let mut map = serializer.serialize_map(Some(object.len()))?; + for key in GROUP_KEYS { + if let Some(value) = object.get(key) { + map.serialize_entry(key, &Descriptor(value))?; + } + } + if object.keys().any(|key| !GROUP_KEYS.contains(&key.as_str())) { + return Err(serde::ser::Error::custom( + "unexpected scope descriptor object", + )); + } + map.end() + } + value => value.serialize(serializer), + } + } +} + +fn descriptor_hash(values: Value) -> Result { + let encoded = serde_json::to_string(&Descriptor(&values))?; + let mut php = String::new(); + for character in encoded.chars() { + if character == '/' { + php.push_str("\\/"); + } else if character as u32 > 0x7f { + for unit in character.encode_utf16(&mut [0; 2]) { + use std::fmt::Write; + write!(&mut php, "\\u{unit:04x}").unwrap(); + } + } else { + php.push(character); + } + } + Ok(format!("{:x}", Sha256::digest(php.as_bytes()))) +} + +#[cfg(test)] +mod descriptor_interop { + use super::*; + + #[test] + fn scope_history_descriptor_hash_matches_php_unicode_slashes_del_and_ordered_groups() { + assert_eq!( + descriptor_hash(json!([ + "scope/é😀", + "event/a", + "\u{7f}", + "x\u{2028}y", + true, + null + ])) + .unwrap(), + "afde5922dded878e86ff3e376d9ebd20804265c8df877ac50bcd6d3ca0b22aa8" + ); + assert_eq!( + descriptor_hash(json!([{"parallel_group_id":"parallel-calls:scope/é😀", + "parallel_group_kind":"mixed", "parallel_group_base_sequence":3, + "parallel_group_size":2, "parallel_group_index":0}])) + .unwrap(), + "c31d38786755cb4d32824eb8b765e305757bb627eaa7433eb6f82926928a970a" + ); + } +} + +pub(super) fn normalize_members(field: &str, value: &Value) -> Result { + let (keys, id_key): (&[&str], &str) = match field { + "activity_members" => ( + &["sequence", "activity_execution_id", "descriptor_hash"], + "activity_execution_id", + ), + "timer_members" => (&["sequence", "timer_id", "descriptor_hash"], "timer_id"), + "wait_members" => ( + &["kind", "sequence", "wait_id", "timer_id", "descriptor_hash"], + "wait_id", + ), + "child_members" => ( + &[ + "sequence", + "child_call_id", + "child_workflow_instance_id", + "child_workflow_run_id", + "cancellation_policy", + "descriptor_hash", + ], + "child_call_id", + ), + _ => return Err(invalid("unknown scope projection")), + }; + let members = value + .as_array() + .ok_or_else(|| invalid("scope projection must be a list"))?; + let mut ids = BTreeSet::new(); + let mut sequences = BTreeSet::new(); + for member in members { + let object = member + .as_object() + .ok_or_else(|| invalid("scope member must be an object"))?; + let hash = member["descriptor_hash"].as_str().unwrap_or_default(); + if object.len() != keys.len() + || keys.iter().any(|key| !object.contains_key(*key)) + || !positive(&member["sequence"]) + || !identity(&member[id_key]) + || !ids.insert(member[id_key].as_str().unwrap().to_owned()) + || hash.len() != 64 + || !hash + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + || (matches!(field, "activity_members" | "child_members") + && !sequences.insert(member["sequence"].as_u64().unwrap())) + { + return Err(invalid( + "scope projection changes its original member address", + )); + } + if field == "wait_members" + && (!matches!(member["kind"].as_str(), Some("signal" | "condition")) + || (!member["timer_id"].is_null() && !identity(&member["timer_id"]))) + { + return Err(invalid("scope wait changes its kind or timeout address")); + } + if field == "child_members" + && (!identity(&member["child_workflow_instance_id"]) + || !identity(&member["child_workflow_run_id"]) + || !policy(&member["cancellation_policy"])) + { + return Err(invalid("scope child changes its target or policy")); + } + } + Ok(value.clone()) +} + +fn address<'a>(payload: &'a Value, descriptor_key: &str) -> Result<&'a str> { + let descriptor = payload.get(descriptor_key); + if descriptor.is_some_and(|value| !value.is_object()) { + return Err(invalid("scope operation lacks its original descriptor")); + } + let nested = descriptor.and_then(|value| value.get("cancellation_scope_id")); + let address = payload.get("cancellation_scope_id").or(nested); + if address.is_some_and(|value| !identity(value)) || (nested.is_some() && nested != address) { + return Err(invalid("scope operation changes its immediate membership")); + } + Ok(address.and_then(Value::as_str).unwrap_or("root")) +} + +fn group_entry(payload: &Value) -> Option { + let id = payload["parallel_group_id"].as_str()?; + let kind = payload["parallel_group_kind"].as_str().or_else(|| { + [ + ("parallel-activities:", "activity"), + ("parallel-calls:", "mixed"), + ("select-calls:", "mixed"), + ("parallel-timers:", "timer"), + ("parallel-children:", "child"), + ] + .into_iter() + .find_map(|(prefix, kind)| id.starts_with(prefix).then_some(kind)) + })?; + let mode = payload["parallel_group_mode"] + .as_str() + .filter(|value| !value.is_empty()) + .or_else(|| id.starts_with("select-calls:").then_some("select")); + let select = mode == Some("select"); + if [ + "parallel_group_base_sequence", + "parallel_group_size", + "parallel_group_index", + ] + .iter() + .any(|key| payload[*key].as_i64().is_none()) + || payload["parallel_group_size"].as_i64()? < 1 + || (select + && !(payload["selection_member_key"] + .as_i64() + .is_some_and(|key| key >= 0) + || payload["selection_member_key"] + .as_str() + .is_some_and(|key| !key.is_empty()))) + { + return None; + } + let mut result = json!({"parallel_group_id":id, "parallel_group_kind":kind, + "parallel_group_base_sequence":payload["parallel_group_base_sequence"], + "parallel_group_size":payload["parallel_group_size"], "parallel_group_index":payload["parallel_group_index"]}); + if select { + result["parallel_group_mode"] = json!("select"); + result["selection_member_key"] = payload["selection_member_key"].clone(); + } + for key in [ + "selection_member_index", + "selection_member_base_sequence", + "selection_member_size", + ] { + if payload[key].as_i64().is_some() { + result[key] = payload[key].clone(); + } + } + if payload["selection_member_kind"] + .as_str() + .is_some_and(|value| !value.is_empty()) + { + result["selection_member_kind"] = payload["selection_member_kind"].clone(); + } + Some(result) +} + +fn group_path(payload: &Value) -> Result { + let raw = match payload.get("parallel_group_path") { + None => &[][..], + Some(Value::Array(path)) => path.as_slice(), + _ => { + return Err(invalid( + "scope operation has a malformed original group path", + )) + } + }; + let path: Vec<_> = raw.iter().filter_map(group_entry).collect(); + Ok(json!(if path.is_empty() { + group_entry(payload).into_iter().collect::>() + } else { + path + })) +} + +pub(super) fn members_from_prefix( + field: &str, + prefix: &[HistoryEvent], + scope: &str, + run: &str, +) -> Result { + let mut members = Vec::new(); + let mut activity_ids = BTreeSet::new(); + let mut activity_sequences = BTreeSet::new(); + let mut timer_sequences = BTreeMap::new(); + for event in prefix { + let p = &event.payload; + let sequence = &p["sequence"]; + match (field, event.event_type.as_str()) { + ("activity_members", "ActivityScheduled") => { + let activity = &p["activity"]; + let id = &p["activity_execution_id"]; + if !activity.is_object() + || !identity(id) + || activity["id"] != *id + || !positive(sequence) + || !activity_ids.insert(id.as_str().unwrap().to_owned()) + || !activity_sequences.insert(sequence.as_u64().unwrap()) + || p.get("cancellation_scope_id").unwrap_or( + activity + .get("cancellation_scope_id") + .unwrap_or(&json!("root")), + ) != activity + .get("cancellation_scope_id") + .unwrap_or(&json!("root")) + { + return Err(invalid( + "scope Activity admission changes identity or membership", + )); + } + if address(p, "activity")? != scope { + continue; + } + for key in ["local_preparation", "local_group_admission"] { + if p[key]["cancellation_cleanup"].get("scope_id").is_some() { + return Err(invalid("nested cleanup needs its original cleanup proof")); + } + } + let cancellation_policy = activity + .get("cancellation_policy") + .cloned() + .unwrap_or(json!("try_cancel")); + if !policy(&cancellation_policy) + || p.get("local_activity") + .is_some_and(|value| !value.is_boolean()) + || p.get("execution_mode") + .is_some_and(|value| !value.is_null() && !value.is_string()) + || activity + .get("schedule_to_close_deadline_at") + .is_some_and(|value| !value.is_null() && !value.is_string()) + { + return Err(invalid( + "scope Activity changes its cancellation descriptor", + )); + } + members.push(json!({"sequence":sequence, "activity_execution_id":id, + "descriptor_hash":descriptor_hash(json!([scope, event_id(event)?, cancellation_policy, + p.get("local_activity").unwrap_or(&Value::Bool(false)), p["execution_mode"], + activity["schedule_to_close_deadline_at"]]))?})); + } + ("timer_members", "TimerScheduled") if address(p, "timer")? == scope => { + let id = &p["timer_id"]; + let kind = &p["timer_kind"]; + if !positive(sequence) + || !identity(id) + || p["delay_seconds"].as_i64().is_none_or(|delay| delay < 0) + || timer_sequences + .get(&sequence.as_u64().unwrap_or_default()) + .is_some_and(|previous| { + previous != kind + || !matches!( + kind.as_str(), + Some("signal_timeout" | "condition_timeout") + ) + }) + || p["fire_at"].as_str() + != Some(canonical_time(timestamp(&p["fire_at"])?).as_str()) + { + return Err(invalid("scope timer changes its original descriptor")); + } + timer_sequences.insert(sequence.as_u64().unwrap(), kind.clone()); + if cleanup_timer(event, prefix)? { + continue; + } + members.push(json!({"sequence":sequence, "timer_id":id, + "descriptor_hash":descriptor_hash(json!([scope, event_id(event)?, sequence, id, + p["delay_seconds"], p["fire_at"], kind, p["condition_wait_id"], + p["condition_wait_occurrence_id"], p["signal_wait_id"], group_path(p)?]))?})); + } + ("wait_members", "SignalWaitOpened" | "ConditionWaitOpened") + if p["cancellation_scope_id"].as_str().unwrap_or("root") == scope => + { + let kind = if event.event_type == "SignalWaitOpened" { + "signal" + } else { + "condition" + }; + let id = &p[format!("{kind}_wait_id")]; + if !positive(sequence) || !identity(id) { + return Err(invalid("scope wait changes its original address")); + } + let timers: Vec<_> = prefix + .iter() + .filter(|row| { + row.event_type == "TimerScheduled" + && row.payload[format!("{kind}_wait_id")] == *id + }) + .collect(); + if timers.len() > 1 { + return Err(invalid("scope wait duplicates its timeout")); + } + let timer = timers.first().copied(); + let empty = Value::Null; + let timeout = timer.map_or(&empty, |row| &row.payload); + if let Some(timer) = timer { + if timeout["timer_kind"] != format!("{kind}_timeout") + || timeout["sequence"] != *sequence + || event_sequence(timer)? <= event_sequence(event)? + || timeout["cancellation_scope_id"].as_str().unwrap_or("root") != scope + || !identity(&timeout["timer_id"]) + { + return Err(invalid("scope wait changes its original timeout")); + } + } + members.push(json!({"kind":kind, "sequence":sequence, "wait_id":id, "timer_id":timeout["timer_id"], + "descriptor_hash":descriptor_hash(json!([scope, event_id(event)?, kind, sequence, id, + timer.map(event_id).transpose()?, timeout["timer_id"], p["timeout_seconds"], p["signal_name"], + p["condition_wait_occurrence_id"], p["condition_key"], p["condition_definition_fingerprint"], + group_path(p)?]))?})); + } + ("child_members", "ChildWorkflowScheduled") + if address(p, "child_workflow")? == scope => + { + let id = &p["child_call_id"]; + let instance = &p["child_workflow_instance_id"]; + let mut target = p["child_workflow_run_id"].clone(); + let cancellation_policy = p + .get("cancellation_policy") + .cloned() + .unwrap_or(json!("abandon")); + if !positive(sequence) + || !identity(id) + || !identity(instance) + || !identity(&target) + || target == run + || !policy(&cancellation_policy) + { + return Err(invalid("scope child changes its original target or policy")); + } + let mut last_started = None; + for started in prefix.iter().filter(|row| { + row.event_type == "ChildRunStarted" && row.payload["sequence"] == *sequence + }) { + let next = &started.payload; + if event_sequence(started)? <= event_sequence(event)? + || next["child_call_id"] != *id + || next["child_workflow_instance_id"] != *instance + || !identity(&next["child_workflow_run_id"]) + || next["child_workflow_run_id"] == run + || next + .get("cancellation_scope_id") + .is_some_and(|value| value != scope) + || next + .get("cancellation_policy") + .is_some_and(|value| value != &cancellation_policy) + { + return Err(invalid("scope child changes its original continuation")); + } + target = next["child_workflow_run_id"].clone(); + last_started = Some(event_id(started)?); + } + members.push(json!({"sequence":sequence, "child_call_id":id, "child_workflow_instance_id":instance, + "child_workflow_run_id":target, "cancellation_policy":cancellation_policy, + "descriptor_hash":descriptor_hash(json!([scope, event_id(event)?, sequence, id, instance, + target, cancellation_policy, p["parent_close_policy"], p["child_workflow_type"], + last_started, group_path(p)?]))?})); + } + _ => {} + } + } + normalize_members(field, &json!(members)) +} + +fn descendants_from_prefix( + prefix: &[HistoryEvent], + scope: &str, + run: &str, + authority: DateTime, +) -> Result { + let openings: BTreeMap<_, _> = prefix + .iter() + .filter(|row| row.event_type == "CancellationScopeOpened") + .map(|row| Ok((text(&row.payload, "scope_id")?, row))) + .collect::>()?; + let requests: BTreeMap<_, _> = prefix + .iter() + .filter(|row| row.event_type == "CancellationScopeRequested") + .map(|row| Ok((text(&row.payload, "scope_id")?, row))) + .collect::>()?; + let mut included = BTreeSet::from([scope]); + let mut members = Vec::new(); + // Use canonical opening order, rather than a map's lexical identity order. + for opening in prefix + .iter() + .filter(|row| row.event_type == "CancellationScopeOpened") + { + let id = text(&opening.payload, "scope_id")?; + let parent_id = text(&opening.payload, "parent_scope_id")?; + if id == scope || opening.payload["shield_parent"] == true || !included.contains(parent_id) + { + continue; + } + included.insert(id); + let request = requests + .get(id) + .ok_or_else(|| invalid("descendant lacks its accepted propagation"))?; + let parent_request = requests + .get(parent_id) + .ok_or_else(|| invalid("descendant lacks its accepted parent"))?; + let context = ScopedCancellationContext::from_value(&request.payload["cancellation"])?; + let parent = + ScopedCancellationContext::from_value(&parent_request.payload["cancellation"])?; + let propagation = if context.root_context() == parent.root_context() { + if request.payload["parent_scope_id"] != parent_id { + return Err(invalid("descendant changes its original parent")); + } + *request + } else { + let mut conflict = None; + for event in prefix.iter().filter(|row| { + row.event_type == "CancellationScopeRequestConflicted" + && row.payload["scope_id"] == id + && row.payload["parent_scope_id"] == parent_id + }) { + let p = &event.payload; + if p["schema"] != "durable-workflow.cancellation-scope-request/v1" + || p["workflow_run_id"] != run + || p["reason"] != "cancellation_root_conflict" + || event_sequence(event)? + <= event_sequence(request)?.max(event_sequence(parent_request)?) + { + return Err(invalid("descendant changes its original conflict boundary")); + } + let incoming = ScopedCancellationContext::from_value(&p["incoming_cancellation"])?; + let accepted = ScopedCancellationContext::from_value(&p["accepted_cancellation"])?; + if incoming.root_context() == parent.root_context() + && incoming.lineage()[..incoming.lineage().len() - 1] == *parent.lineage() + && incoming.deadline() == parent.deadline() + && incoming.scope_id() == id + && incoming.workflow_run_id() == run + && incoming.workflow_instance_id() == context.workflow_instance_id() + && accepted == context + { + conflict = Some(event); + break; + } + } + conflict + .ok_or_else(|| invalid("descendant lacks its original competing-root conflict"))? + }; + let mut deadline = authority; + let mut ancestor = id; + while ancestor != "root" { + if let Some(request) = requests.get(ancestor) { + deadline = deadline.min( + ScopedCancellationContext::from_value(&request.payload["cancellation"])? + .deadline(), + ); + } + ancestor = text( + &openings + .get(ancestor) + .ok_or_else(|| invalid("descendant lost its original ancestry"))? + .payload, + "parent_scope_id", + )?; + } + let mut member = json!({"scope_id":id, "parent_scope_id":parent_id, "scope_history_event_id":event_id(opening)?, + "request_history_event_id":event_id(request)?, "request_id":context.request_id(), + "propagation_history_event_id":event_id(propagation)?, "authority_deadline_at":canonical_time(deadline), + "cancellation":context.to_value()}); + for field in FIELDS { + member[field] = members_from_prefix(field, prefix, id, run)?; + } + members.push(member); + } + Ok(json!(members)) +} + +#[derive(Clone, Debug)] +pub(super) struct ScopeRequest { + pub context: ScopedCancellationContext, + pub timestamp: DateTime, + pub history_index: usize, +} + +#[derive(Clone, Debug)] +pub(super) struct ScopeBoundary { + pub context: ScopedCancellationContext, + pub boundary: CancellationDelivery, + pub authority_deadline: DateTime, + pub event: HistoryEvent, +} + +#[derive(Default, Debug)] +pub(super) struct CommittedCancellationScopeHistory { + pub preparations: BTreeMap, + pub deliveries: BTreeMap, + pub pending_requests: BTreeMap, +} + +fn admission(kind: &str) -> Option<(&str, CancellationCallKind)> { + match kind { + "ActivityScheduled" => Some(("activity_members", CancellationCallKind::Activity)), + "TimerScheduled" => Some(("timer_members", CancellationCallKind::Timer)), + "ChildWorkflowScheduled" => Some(("child_members", CancellationCallKind::Child)), + "ConditionWaitOpened" => Some(("wait_members", CancellationCallKind::Condition)), + "SignalWaitOpened" => Some(("wait_members", CancellationCallKind::Signal)), + _ => None, + } +} + +fn assert_complete_group( + boundary: &CancellationDelivery, + scope: &str, + prefix: &[HistoryEvent], + scopes: &cancellation_scope::CancellationScopeHistory, +) -> Result<()> { + let mut members = BTreeSet::new(); + for event in prefix { + let p = &event.payload; + let Some(sequence) = p["sequence"] + .as_u64() + .filter(|value| *value > 0 && boundary.interrupts(*value)) + else { + continue; + }; + if admission(&event.event_type).is_none() + || (event.event_type == "TimerScheduled" + && matches!( + p["timer_kind"].as_str(), + Some("condition_timeout" | "signal_timeout") + )) + { + continue; + } + if scopes + .memberships + .get(&sequence) + .map(String::as_str) + .unwrap_or("root") + != scope + || !members.insert(sequence) + { + return Err(invalid("scope group changes its original member address")); + } + let fallback = vec![p.clone()]; + let path = match p.get("parallel_group_path") { + None => &fallback, + Some(Value::Array(path)) if !path.is_empty() && path.iter().all(Value::is_object) => { + path + } + _ => return Err(invalid("scope group requires its original admitted path")), + }; + if path.iter().any(|entry| { + entry["parallel_group_mode"].as_str().unwrap_or("all") != "all" + || entry["parallel_group_id"] + .as_str() + .unwrap_or_default() + .starts_with("select-calls:") + }) { + return Err(invalid( + "scoped selection needs its original selection proof", + )); + } + if path[0]["parallel_group_base_sequence"].as_u64() != Some(boundary.sequence) + || path[0]["parallel_group_size"].as_u64() != Some(boundary.sequence_span) + || path[0]["parallel_group_index"].as_u64() != Some(sequence - boundary.sequence) + { + return Err(invalid( + "scope group changes its original range or member index", + )); + } + } + if members.len() as u64 != boundary.sequence_span { + return Err(invalid("scope group omits an admitted member")); + } + Ok(()) +} + +impl CommittedCancellationScopeHistory { + pub fn read(history: &[HistoryEvent], run: &str, workflow: &str) -> Result { + let scopes = cancellation_scope::CancellationScopeHistory::read(history, run)?; + let addresses: BTreeMap<_, _> = scopes + .openings + .iter() + .map(|(sequence, opening)| (opening.scope_id.as_str(), (*sequence, opening))) + .collect(); + let run_state = CancellationHistory::from_events(history, run, None)?; + let run_root = run_state + .request + .as_ref() + .and_then(|request| request.context.as_ref()) + .map(ScopedCancellationContext::from_run_context) + .transpose()?; + let mut requests: BTreeMap = BTreeMap::new(); + let mut request_ids = BTreeSet::new(); + let mut committed = Self::default(); + let mut delivered = BTreeSet::new(); + let mut opened = BTreeSet::new(); + let mut admissions: BTreeMap<&str, BTreeMap> = BTreeMap::new(); + for (index, event) in history.iter().enumerate() { + let kind = event.event_type.as_str(); + let p = &event.payload; + let timer_cleanup = + kind == "TimerScheduled" && cleanup_timer(event, &history[..index])?; + if kind == "CancellationScopeOpened" { + opened.insert(text(p, "scope_id")?); + } + if admission(kind).is_some() && positive(&p["sequence"]) && !timer_cleanup { + let sequence = p["sequence"].as_u64().unwrap(); + admissions.entry(kind).or_default().insert( + sequence, + scopes + .memberships + .get(&sequence) + .map(String::as_str) + .unwrap_or("root"), + ); + } + if !matches!( + kind, + "CancellationScopeRequested" + | "CancellationScopeDeliveryPrepared" + | "CancellationScopeDelivered" + ) { + continue; + } + let context = ScopedCancellationContext::from_value(&p["cancellation"])?; + let id = context.scope_id(); + let (opening_sequence, opening) = addresses + .get(id) + .ok_or_else(|| invalid("scope request lacks its original opening"))?; + if !opened.contains(id) + || run.is_empty() + || workflow.is_empty() + || context.workflow_run_id() != run + || context.workflow_instance_id() != workflow + || p["workflow_run_id"] != run + || p["scope_id"] != id + || p["request_id"] != context.request_id() + { + return Err(invalid( + "scope cancellation changes its original run, request or address", + )); + } + let time = event_time(event)?; + if time < context.requested_at() { + return Err(invalid("scope boundary predates its original request")); + } + if kind == "CancellationScopeRequested" { + if p["schema"] != "durable-workflow.cancellation-scope-request/v1" + || p.get("parent_scope_id").is_none() + || requests.contains_key(id) + || !request_ids.insert(context.request_id().to_owned()) + { + return Err(invalid( + "scope cancellation requires one accepted original request", + )); + } + if p["parent_scope_id"].is_null() { + if context.lineage().len() != 1 + || context.request_id() != context.root_context().request_id() + { + return Err(invalid( + "direct scope request substitutes an inherited lineage", + )); + } + } else { + let parent_id = text(p, "parent_scope_id")?; + let parent = if parent_id == "root" && run_state.request_index < index { + run_root.as_ref() + } else { + requests + .get(parent_id) + .filter(|request| request.history_index < index) + .map(|request| &request.context) + } + .ok_or_else(|| invalid("inherited scope request lacks its earlier parent"))?; + if parent.workflow_run_id() != run + || parent.workflow_instance_id() != workflow + || opening.shield_parent + || parent_id != opening.parent_scope_id + || context.root_context() != parent.root_context() + || context.lineage()[..context.lineage().len() - 1] != *parent.lineage() + || context.deadline() != parent.deadline() + { + return Err(invalid("inherited scope request changes its accepted parent or crosses a shield")); + } + } + requests.insert( + id.to_owned(), + ScopeRequest { + context, + timestamp: time, + history_index: index, + }, + ); + continue; + } + let request = requests + .get(id) + .ok_or_else(|| invalid("scope boundary lacks its accepted request"))?; + if context != request.context || time < request.timestamp { + return Err(invalid( + "scope boundary changes its immutable accepted request", + )); + } + if [ + "sequence_span", + "operation_sequence", + "operation_sequence_span", + ] + .iter() + .any(|key| p.get(*key).is_none()) + { + return Err(invalid("scope boundary omits its original operation range")); + } + let mut payload = p.clone(); + payload["workflow_command_id"] = json!(context.request_id()); + let boundary = CancellationDelivery::from_payload(&payload)?; + let deadline = timestamp(&p["authority_deadline_at"])?; + if boundary.sequence <= *opening_sequence + || deadline < context.requested_at() + || deadline > context.deadline() + || time > deadline + { + return Err(invalid( + "scope boundary changes or exceeds its original authority ceiling", + )); + } + if kind == "CancellationScopeDeliveryPrepared" { + if p["schema"] != "durable-workflow.cancellation-scope-preparation/v5" + || committed.preparations.contains_key(id) + || delivered.contains(id) + { + return Err(invalid( + "scope delivery requires one original v5 preparation", + )); + } + let prefix = &history[..index]; + for field in FIELDS { + if normalize_members(field, &p[field])? + != members_from_prefix(field, prefix, id, run)? + { + return Err(invalid( + "scope preparation changes its original member projection", + )); + } + } + if p["descendant_members"] != descendants_from_prefix(prefix, id, run, deadline)? { + return Err(invalid( + "scope preparation changes its original descendant projection", + )); + } + let mut by_scope = BTreeMap::from([(id, p)]); + for member in p["descendant_members"].as_array().unwrap() { + by_scope.insert(text(member, "scope_id")?, member); + } + let operation_scope = scopes + .memberships + .get(&boundary.sequence) + .map(String::as_str) + .unwrap_or(id); + if !by_scope.contains_key(operation_scope) { + return Err(invalid( + "scope boundary consumes an operation outside its frozen subtree", + )); + } + if boundary.call_kind == CancellationCallKind::Parallel { + assert_complete_group(&boundary, operation_scope, prefix, &scopes)?; + } else if !matches!( + boundary.call_kind, + CancellationCallKind::Activity + | CancellationCallKind::LocalActivity + | CancellationCallKind::Timer + | CancellationCallKind::Condition + | CancellationCallKind::Child + ) { + return Err(invalid("scope boundary uses an unsupported operation kind")); + } + for (member_scope, projection) in by_scope { + for (kind, positions) in &admissions { + let (field, mut call_kind) = admission(kind).unwrap(); + let sequences: BTreeSet<_> = projection[field] + .as_array() + .unwrap() + .iter() + .filter(|entry| { + !matches!(*kind, "ConditionWaitOpened" | "SignalWaitOpened") + || entry["kind"] + == if *kind == "ConditionWaitOpened" { + "condition" + } else { + "signal" + } + }) + .map(|entry| entry["sequence"].as_u64().unwrap()) + .collect(); + if positions.iter().any(|(sequence, owner)| { + *owner == member_scope && !sequences.contains(sequence) + }) { + return Err(invalid("scope preparation omits an admitted operation")); + } + if positions.get(&boundary.sequence).copied() != Some(member_scope) { + continue; + } + if call_kind == CancellationCallKind::Activity + && prefix.iter().any(|row| { + row.event_type == *kind + && row.payload["sequence"].as_u64() == Some(boundary.sequence) + && row.payload["local_activity"] == true + }) + { + call_kind = CancellationCallKind::LocalActivity; + } + let timeout = + *kind == "TimerScheduled" + && projection["wait_members"].as_array().unwrap().iter().any( + |entry| entry["sequence"].as_u64() == Some(boundary.sequence), + ); + if !sequences.contains(&boundary.sequence) + || (!timeout + && boundary.call_kind != CancellationCallKind::Parallel + && boundary.call_kind != call_kind) + { + return Err(invalid("scope boundary replaces an admitted operation")); + } + } + } + committed.preparations.insert( + id.to_owned(), + ScopeBoundary { + context, + boundary, + authority_deadline: deadline, + event: event.clone(), + }, + ); + continue; + } + let preparation = committed + .preparations + .get(id) + .ok_or_else(|| invalid("scope delivery lacks its original preparation"))?; + if p["schema"] != "durable-workflow.cancellation-scope-delivery/v1" + || delivered.contains(id) + || committed.deliveries.contains_key(&boundary.sequence) + || p["preparation_history_event_id"].as_str() != Some(event_id(&preparation.event)?) + || boundary != preparation.boundary + || deadline != preparation.authority_deadline + || time < event_time(&preparation.event)? + { + return Err(invalid( + "scope delivery changes its original prepared boundary", + )); + } + delivered.insert(id.to_owned()); + committed.deliveries.insert( + boundary.sequence, + ScopeBoundary { + context, + boundary, + authority_deadline: deadline, + event: event.clone(), + }, + ); + } + let mut covered = delivered; + for delivery in committed.deliveries.values() { + for member in committed.preparations[delivery.context.scope_id()] + .event + .payload["descendant_members"] + .as_array() + .unwrap() + { + covered.insert(text(member, "scope_id")?.to_owned()); + } + } + committed.pending_requests = requests + .into_iter() + .filter(|(scope, _)| !covered.contains(scope)) + .collect(); + Ok(committed) + } + + pub fn pending_request_for_scope<'a>( + &'a self, + scope: &str, + scopes: &cancellation_scope::CancellationScopeHistory, + ) -> Result> { + let addresses: BTreeMap<_, _> = scopes + .openings + .values() + .map(|opening| (opening.scope_id.as_str(), opening)) + .collect(); + let mut id = scope; + let mut request = self.pending_requests.get(scope); + while let Some(opening) = addresses.get(id).filter(|opening| !opening.shield_parent) { + id = &opening.parent_scope_id; + let Some(ancestor) = self.pending_requests.get(id) else { + continue; + }; + if request.is_some_and(|selected| { + ancestor.context.root_context() != selected.context.root_context() + || !selected + .context + .lineage() + .starts_with(ancestor.context.lineage()) + }) { + return Err(invalid( + "pending scope selection changes its original ancestor lineage", + )); + } + request = Some(ancestor); + } + Ok(request) + } +} + +/// One original monotonic budget shared by scope preparation, delivery and proof. +#[doc(hidden)] +pub struct CancellationScopeDeliveryBudget { + expires_at: tokio::time::Instant, +} + +impl Default for CancellationScopeDeliveryBudget { + fn default() -> Self { + Self::new() + } +} + +impl CancellationScopeDeliveryBudget { + pub fn new() -> Self { + Self { + expires_at: tokio::time::Instant::now() + Duration::from_secs(5), + } + } + + /// Narrow the original budget, without granting a new request or lease. + pub fn restrict(&mut self, authority_deadline: DateTime) -> Result<()> { + let unix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| Error::Timeout)?; + let now = DateTime::::from_timestamp( + i64::try_from(unix.as_secs()).map_err(|_| Error::Timeout)?, + unix.subsec_nanos(), + ) + .ok_or(Error::Timeout)?; + let remaining = (authority_deadline - now) + .to_std() + .unwrap_or(Duration::ZERO); + self.expires_at = self.expires_at.min(tokio::time::Instant::now() + remaining); + self.remaining() + } + + fn remaining(&self) -> Result<()> { + if tokio::time::Instant::now() >= self.expires_at { + Err(Error::Timeout) + } else { + Ok(()) + } + } +} + +/// Proof of the original scope preparation and optional committed delivery. +/// This object does not grant callback execution or result publication authority. +#[doc(hidden)] +#[derive(Clone, Debug)] +pub struct CancellationScopeDeliveryReceipt { + pub(super) preparation: ScopeBoundary, + pub(super) delivery: Option, + history: Vec, +} + +impl CancellationScopeDeliveryReceipt { + pub fn context(&self) -> &ScopedCancellationContext { + &self.preparation.context + } + pub fn boundary(&self) -> &CancellationDelivery { + &self.preparation.boundary + } + pub fn authority_deadline(&self) -> DateTime { + self.preparation.authority_deadline + } + pub fn history(&self) -> &[HistoryEvent] { + &self.history + } + pub fn preparation_history_event_id(&self) -> &str { + event_id(&self.preparation.event).unwrap() + } + pub fn delivery_history_event_id(&self) -> Option<&str> { + self.delivery + .as_ref() + .map(|delivery| event_id(&delivery.event).unwrap()) + } + + fn acknowledge<'a>(receipt: &'a Value, expected: &Value, delivering: bool) -> Result<&'a str> { + if expected + .as_object() + .ok_or_else(|| invalid("missing original scope claim"))? + .iter() + .any(|(key, value)| { + !matches!(key.as_str(), "namespace" | "workflow_instance_id") + && receipt.get(key) != Some(value) + }) + || receipt["prepared"].as_bool() != Some(true) + || receipt["delivered"].as_bool() != Some(delivering) + || receipt["claim_released"].as_bool() != Some(false) + || receipt.get("created_task_ids") != Some(&json!([])) + || receipt.get("reason") != Some(&Value::Null) + || (text(receipt, "history_event_id")? + == text(receipt, "preparation_history_event_id")?) + == delivering + { + return Err(invalid( + "scope acknowledgement changes its original claim or retained preparation", + )); + } + for field in FIELDS { + normalize_members(field, &receipt[field])?; + } + let context = ScopedCancellationContext::from_value(&receipt["cancellation"])?; + let authority = timestamp(&receipt["authority_deadline_at"])?; + if context.workflow_run_id() != text(expected, "workflow_run_id")? + || context.workflow_instance_id() != text(expected, "workflow_instance_id")? + || context.scope_id() != text(expected, "scope_id")? + || context.request_id() != text(expected, "request_id")? + || authority < context.requested_at() + || authority > context.deadline() + { + return Err(invalid( + "scope acknowledgement changes its original cancellation authority", + )); + } + receipt["history_refresh_page_token"] + .as_str() + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| { + invalid("scope acknowledgement lacks an opaque canonical history cursor") + }) + } + + fn from_history( + receipt: &Value, + history: Vec, + expected: &Value, + delivering: bool, + ) -> Result { + Self::acknowledge(receipt, expected, delivering)?; + let mut starts = BTreeSet::new(); + for event in &history { + if event.raw.get("namespace") != expected.get("namespace") { + return Err(invalid( + "scope receipt history changes its original namespace", + )); + } + if matches!( + event.event_type.as_str(), + "StartAccepted" | "WorkflowStarted" + ) && (!starts.insert(event.event_type.as_str()) + || event.payload.get("workflow_run_id") != expected.get("workflow_run_id") + || event.payload.get("workflow_instance_id") + != expected.get("workflow_instance_id")) + { + return Err(invalid( + "scope receipt history changes its original workflow start", + )); + } + } + if !starts.contains("WorkflowStarted") { + return Err(invalid("scope receipt history omits its workflow start")); + } + let committed = CommittedCancellationScopeHistory::read( + &history, + text(expected, "workflow_run_id")?, + text(expected, "workflow_instance_id")?, + )?; + let preparation = committed + .preparations + .get(text(expected, "scope_id")?) + .ok_or_else(|| invalid("scope receipt lacks its original committed preparation"))? + .clone(); + let context = ScopedCancellationContext::from_value(&receipt["cancellation"])?; + let mut payload = receipt.clone(); + payload["workflow_command_id"] = expected["request_id"].clone(); + let boundary = CancellationDelivery::from_payload(&payload)?; + if event_id(&preparation.event)? != text(receipt, "preparation_history_event_id")? + || preparation.context != context + || preparation.boundary != boundary + || preparation.authority_deadline != timestamp(&receipt["authority_deadline_at"])? + { + return Err(invalid( + "scope receipt differs from its original committed preparation", + )); + } + for field in FIELDS { + if normalize_members(field, &preparation.event.payload[field])? + != normalize_members(field, &receipt[field])? + { + return Err(invalid( + "scope receipt changes its original frozen projection", + )); + } + } + let delivery = if delivering { + let delivery = committed + .deliveries + .get(&boundary.sequence) + .ok_or_else(|| invalid("scope receipt lacks its original committed delivery"))? + .clone(); + if event_id(&delivery.event)? != text(receipt, "history_event_id")? + || delivery.context != context + || delivery.boundary != boundary + { + return Err(invalid( + "scope receipt changes its original committed delivery", + )); + } + Some(delivery) + } else { + None + }; + Ok(Self { + preparation, + delivery, + history, + }) + } + + fn assert_original_preparation(&self, original: &Self) -> Result<()> { + let before = &original.preparation; + let after = &self.preparation; + if event_id(&before.event)? != event_id(&after.event)? + || before.context != after.context + || before.boundary != after.boundary + || before.authority_deadline != after.authority_deadline + || FIELDS + .into_iter() + .chain(["descendant_members"]) + .any(|field| before.event.payload[field] != after.event.payload[field]) + { + return Err(invalid( + "scope delivery substitutes its previously proved original preparation", + )); + } + Ok(()) + } +} + +impl Client { + /// Candidate preparation proof. All operations share the supplied budget. + #[doc(hidden)] + pub async fn prepare_cancellation_scope_on_claim( + &self, + task: &WorkflowTask, + context: &ScopedCancellationContext, + boundary: &CancellationDelivery, + budget: &CancellationScopeDeliveryBudget, + ) -> Result { + budget.remaining()?; + let run = task.run_id.as_deref().unwrap_or_default(); + let committed = CommittedCancellationScopeHistory::read( + &task.history_events, + run, + task.workflow_id.as_deref().unwrap_or_default(), + )?; + let scopes = cancellation_scope::CancellationScopeHistory::read(&task.history_events, run)?; + let operation_scope = scopes + .memberships + .get(&boundary.sequence) + .map(String::as_str) + .unwrap_or(context.scope_id()); + if committed + .pending_request_for_scope(operation_scope, &scopes)? + .is_none_or(|request| &request.context != context) + { + return Err(invalid( + "scope preparation lacks its original pending request on this claim", + )); + } + self.cancellation_scope_boundary_on_claim(task, context, boundary, None, budget) + .await + } + + /// Candidate delivery requires the earlier original preparation proof. + #[doc(hidden)] + pub async fn deliver_cancellation_scope_on_claim( + &self, + task: &WorkflowTask, + original: &CancellationScopeDeliveryReceipt, + budget: &CancellationScopeDeliveryBudget, + ) -> Result { + self.cancellation_scope_boundary_on_claim( + task, + original.context(), + original.boundary(), + Some(original), + budget, + ) + .await + } + + async fn cancellation_scope_boundary_on_claim( + &self, + task: &WorkflowTask, + context: &ScopedCancellationContext, + boundary: &CancellationDelivery, + original: Option<&CancellationScopeDeliveryReceipt>, + budget: &CancellationScopeDeliveryBudget, + ) -> Result { + let (owner, run) = cooperative_cancellation::cancellation_claim(task)?; + let workflow = task.workflow_id.as_deref().unwrap_or_default(); + if [ + task.task_id.as_str(), + owner, + run, + workflow, + context.scope_id(), + context.request_id(), + self.namespace.as_str(), + ] + .iter() + .any(|value| !identity(&json!(value))) + || context.scope_id() == "root" + || context.workflow_run_id() != run + || context.workflow_instance_id() != workflow + || boundary.request_id != context.request_id() + || (boundary.call_kind != CancellationCallKind::Parallel && boundary.sequence_span != 1) + || boundary.operation_sequence.is_some() + || boundary.operation_sequence_span != 1 + || !matches!( + boundary.call_kind, + CancellationCallKind::Activity + | CancellationCallKind::LocalActivity + | CancellationCallKind::Timer + | CancellationCallKind::Condition + | CancellationCallKind::Child + | CancellationCallKind::Parallel + ) + { + return Err(invalid( + "scope boundary requires its original claim and supported durable call", + )); + } + budget.remaining()?; + let body = json!({"scope_id":context.scope_id(), "request_id":context.request_id(), + "sequence":boundary.sequence, "call_kind":boundary.call_kind, "sequence_span":boundary.sequence_span, + "operation_sequence":boundary.operation_sequence, "operation_sequence_span":boundary.operation_sequence_span, + "lease_owner":owner, "workflow_task_attempt":task.workflow_task_attempt}); + let mut checked = body.clone(); + checked["workflow_command_id"] = json!(boundary.request_id); + CancellationDelivery::from_payload(&checked)?; + let mut expected = body.clone(); + expected["task_id"] = json!(task.task_id); + expected["workflow_run_id"] = json!(run); + expected["workflow_instance_id"] = json!(workflow); + expected["namespace"] = json!(self.namespace); + let path = format!( + "/worker/workflow-tasks/{}", + percent_encode_path_segment(&task.task_id) + ); + let delivering = original.is_some(); + tokio::time::timeout_at(budget.expires_at, async { + let boundary_path = format!("{path}/cancellation-scopes/{}", if delivering { "deliver" } else { "prepare" }); + let request = || self.request_json::(reqwest::Method::POST, &boundary_path, + RequestProtocol::Worker("1.20"), Some(&body)); + loop { + budget.remaining()?; + let mut receipt = match request().await { + Err(error) if worker_operation_is_retryable(&error) => request().await?, + result => result?, + }; + let pending = delivering && receipt["reason"].as_str() + == Some("cancellation_scope_activity_stop_not_acknowledged"); + if pending { + if receipt.get("history_event_id").is_some() { + return Err(invalid("pending scope stop cannot claim a committed delivery")); + } + // A pending stop proves only the retained preparation. Never expose + // it to workflow cleanup as a delivery or renew the original budget. + receipt["reason"] = Value::Null; + receipt["history_event_id"] = json!(original.unwrap().preparation_history_event_id()); + } + let committed = delivering && !pending; + let mut token = Some(CancellationScopeDeliveryReceipt::acknowledge(&receipt, &expected, committed)?.to_owned()); + let mut seen = BTreeSet::new(); + let mut history = Vec::new(); + while let Some(current) = token.take() { + budget.remaining()?; + if seen.len() >= 128 || !seen.insert(current.clone()) { return Err(invalid("scope history repeated or exceeded its opaque cursors")); } + let page: Value = self.request_json(reqwest::Method::POST, &format!("{path}/history"), + RequestProtocol::Worker("1.20"), Some(&json!({"lease_owner":owner, + "workflow_task_attempt":task.workflow_task_attempt, "next_history_page_token":current, + "history_page_size":WORKFLOW_HISTORY_PAGE_SIZE}))).await?; + if page["task_id"].as_str() != Some(task.task_id.as_str()) + || page["workflow_task_attempt"].as_u64() != Some(task.workflow_task_attempt) { + return Err(invalid("scope history page changes its original claim")); + } + let batch = page["history_events"].as_array().ok_or_else(|| invalid("scope history page lacks canonical events"))?; + if batch.len() > WORKFLOW_HISTORY_PAGE_SIZE as usize || batch.iter().any(|event| !event.is_object()) { + return Err(invalid("scope history page exceeds its bounded complete shape")); + } + token = match page.get("next_history_page_token") { + Some(Value::Null) => None, + Some(Value::String(next)) if !next.trim().is_empty() && !batch.is_empty() => Some(next.clone()), + _ => return Err(invalid("scope history page omits its terminal cursor")), + }; + for event in batch { history.push(serde_json::from_value(event.clone()).map_err(|_| invalid("scope history event is malformed"))?); } + } + let proved = CancellationScopeDeliveryReceipt::from_history(&receipt, history, &expected, committed)?; + if let Some(original) = original { proved.assert_original_preparation(original)?; } + budget.remaining()?; + if !pending { return Ok(proved); } + // Yield so the activity supervisor can stop and acknowledge the + // original callback while this workflow retains its hosting claim. + tokio::time::sleep(Duration::from_millis(100)).await; + } + }).await.map_err(|_| Error::Timeout)? + } +} diff --git a/src/cancellation_scope_replay.rs b/src/cancellation_scope_replay.rs new file mode 100644 index 0000000..83fc345 --- /dev/null +++ b/src/cancellation_scope_replay.rs @@ -0,0 +1,707 @@ +//! Candidate scope delivery. Admission remains private and disabled by default. + +use super::*; +use crate::cancellation_scope_history::{ + boundary_scope_states, CommittedCancellationScopeHistory, ScopeBoundary, +}; +use chrono::Utc; + +#[doc(hidden)] +#[derive(Clone, Debug, Error, PartialEq, Eq)] +#[error("scope cancellation {request_id} was requested", request_id = .context.request_id())] +pub struct CancellationScopeRequested { + pub context: ScopedCancellationContext, + pub delivery: CancellationDelivery, +} + +#[derive(Clone, Debug)] +pub(super) struct ScopeDeliveryIntent { + pub context: ScopedCancellationContext, + pub boundary: CancellationDelivery, + pub command_count: usize, +} + +#[derive(Debug)] +pub(super) struct ScopeReplay { + pub canonical: CommittedCancellationScopeHistory, + pub tree: cancellation_scope::CancellationScopeHistory, + pub active_scope: String, + pub consumed: BTreeSet, + pub contexts: BTreeMap)>, + pub boundary_states: + BTreeMap)>>, + pub cleanup_timers: BTreeMap, + pub intent: Option, +} + +fn invalid(sequence: u64, detail: &str) -> Error { + invalid_recorded_history( + "cancellation_scope_call_mismatch", + sequence, + "original scoped call", + "changed call or authority", + detail, + ) +} + +fn scalar(kind: CancellationCallKind) -> bool { + matches!( + kind, + CancellationCallKind::Activity + | CancellationCallKind::Timer + | CancellationCallKind::Condition + | CancellationCallKind::Child + ) +} + +fn scope_operation_terminal(kind: &str) -> bool { + matches!( + kind, + "ActivityCompleted" + | "ActivityFailed" + | "ActivityCancelled" + | "ActivityTimedOut" + | "TimerFired" + | "TimerCancelled" + | "ConditionWaitSatisfied" + | "ConditionWaitTimedOut" + | "ChildRunCompleted" + | "ChildRunFailed" + | "ChildRunCancelled" + | "ChildRunTerminated" + ) +} + +fn validate_scoped_group_definition( + state: &WorkflowState, + descriptor: &ParallelDescriptor, + recorded: &RecordedCommand, +) -> Result<()> { + let sequence = recorded.sequence(); + let path = match (&descriptor.operation, recorded) { + ( + ParallelOperation::Activity { + activity_type, + options, + .. + }, + RecordedCommand::Activity { + activity_type: original_type, + cancellation_policy, + options: original_options, + parallel_group_path, + .. + }, + ) => { + let options = options.validate().map_err(Error::InvalidActivityOptions)?; + if original_type + .as_ref() + .is_some_and(|original| original != activity_type) + || cancellation_policy + != options + .cancellation_policy + .unwrap_or(CancellationPolicy::TryCancel) + .as_str() + { + return Err(invalid( + sequence, + "scoped activity changed its original type or cancellation policy", + )); + } + if let Some(original) = original_options { + let queue = RecordedSnapshotValue::Known(Some( + options + .task_queue + .clone() + .unwrap_or_else(|| state.task_queue.clone()), + )); + if !original.task_queue.matches_current(&queue) + || !original + .execution_mode + .matches_current(&RecordedSnapshotValue::Known(None)) + || !original + .retry_policy + .matches_current(¤t_activity_retry_snapshot(&options)) + { + return Err(invalid(sequence, "scoped activity changed its original queue, execution mode or retry policy")); + } + } + parallel_group_path + } + ( + ParallelOperation::Timer(delay), + RecordedCommand::Timer { + delay_seconds, + parallel_group_path, + .. + }, + ) => { + if delay + .as_secs() + .checked_add(u64::from(delay.subsec_nanos() > 0)) + != Some(*delay_seconds) + { + return Err(invalid(sequence, "scoped timer changed its original delay")); + } + parallel_group_path + } + ( + ParallelOperation::ChildWorkflow { + workflow_type, + options, + .. + }, + RecordedCommand::ChildWorkflow { + workflow_type: original_type, + policies, + parallel_group_path, + .. + }, + ) => { + if original_type + .as_ref() + .is_some_and(|original| original != workflow_type) + { + return Err(invalid( + sequence, + "scoped child changed its original workflow type", + )); + } + ensure_child_policies_match(sequence, policies, options)?; + parallel_group_path + } + ( + ParallelOperation::Condition { options, .. }, + RecordedCommand::ConditionWait { + condition_key, + predicate_identity, + timeout_seconds, + parallel_group_path, + .. + }, + ) => { + let options = options + .validate() + .map_err(Error::InvalidConditionWaitOptions)?; + if condition_key.as_deref() != Some(options.condition_key.as_str()) + || predicate_identity != &options.predicate_identity + || timeout_seconds != &options.timeout_seconds + { + return Err(invalid( + sequence, + "scoped condition changed its original key, predicate identity or timeout", + )); + } + // Delivery interrupts the original occurrence without evaluating the predicate. + parallel_group_path + } + _ => { + return Err(invalid( + sequence, + "scoped group changed its original member kind", + )) + } + }; + ensure_parallel_path_matches(sequence, path.as_deref(), &descriptor.group_path) +} + +impl ScopeReplay { + pub fn read( + history: &[HistoryEvent], + tree: &cancellation_scope::CancellationScopeHistory, + run: &str, + workflow: &str, + ) -> Result { + let canonical = CommittedCancellationScopeHistory::read(history, run, workflow)?; + let mut boundary_states = BTreeMap::new(); + for preparation in canonical.preparations.values() { + let boundary = &preparation.boundary; + let supported = if boundary.call_kind == CancellationCallKind::Parallel { + history + .iter() + .filter(|event| { + durable_event_sequence(event) + .is_some_and(|sequence| boundary.interrupts(sequence)) + }) + .all(|event| match event.event_type.as_str() { + "SignalWaitOpened" => false, + "ActivityScheduled" => { + event.payload["local_activity"] != true + && event.payload["execution_mode"] != "local" + && event.payload["activity"]["local_activity"] != true + && event.payload["activity"]["execution_mode"] != "local" + } + _ => true, + }) + } else { + scalar(boundary.call_kind) && boundary.sequence_span == 1 + }; + if !supported || preparation.boundary.operation_sequence.is_some() { + return Err(Error::CancellationScopeExecutionUnavailable); + } + let states = boundary_scope_states(&preparation.context, &preparation.event.payload)?; + for event in history + .iter() + .take_while(|row| row.raw["id"] != preparation.event.raw["id"]) + { + if event.event_type == "ActivityScheduled" + && durable_event_sequence(event) + .and_then(|sequence| tree.memberships.get(&sequence)) + .is_some_and(|scope| states.contains_key(scope)) + && (event.payload["local_activity"] == true + || event.payload["execution_mode"] == "local" + || event.payload["activity"]["local_activity"] == true + || event.payload["activity"]["execution_mode"] == "local") + { + return Err(Error::CancellationScopeExecutionUnavailable); + } + } + boundary_states.insert(preparation.context.scope_id().to_owned(), states); + } + let mut delivered_scopes = BTreeSet::new(); + for delivery in canonical.deliveries.values() { + for scope in boundary_states[delivery.context.scope_id()].keys() { + if !delivered_scopes.insert(scope.clone()) { + return Err(Error::CancellationScopeExecutionUnavailable); + } + } + } + Ok(Self { + canonical, + tree: tree.clone(), + active_scope: "root".into(), + consumed: BTreeSet::new(), + contexts: BTreeMap::new(), + boundary_states, + cleanup_timers: history.iter().filter(|row| row.event_type == "TimerScheduled") + .filter_map(|row| row.payload.get("cancellation_cleanup").map(|snapshot| + (row.payload["sequence"].as_u64().unwrap(), json!({"scope_id":snapshot["scope_id"], + "request_id":snapshot["request_id"], "delivery_history_event_id":snapshot["delivery_history_event_id"]})))).collect(), + intent: None, + }) + } + + pub(super) fn consumed_delivery_for_scope(&self, scope: &str) -> Option<&ScopeBoundary> { + self.canonical.deliveries.values().find(|delivery| { + self.consumed.contains(&delivery.boundary.sequence) + && self.boundary_states[delivery.context.scope_id()].contains_key(scope) + }) + } + + pub fn bind_commands(&self, commands: &mut Vec) -> Result<()> { + for (&sequence, delivered) in &self.canonical.deliveries { + let index = commands.partition_point(|command| command.sequence() < sequence); + if delivered.boundary.call_kind == CancellationCallKind::Parallel { + let end = commands.partition_point(|command| { + delivered.boundary.interrupts(command.sequence()) + || command.sequence() < sequence + }); + let original: Vec<_> = commands.drain(index..end).collect(); + if original.len() as u64 != delivered.boundary.sequence_span + || original + .iter() + .enumerate() + .any(|(offset, command)| command.sequence() != sequence + offset as u64) + { + return Err(invalid( + sequence, + "scope delivery omits an original group member", + )); + } + commands.insert( + index, + RecordedCommand::CancellationGroup { + sequence, + span: delivered.boundary.sequence_span, + original, + }, + ); + continue; + } + let original = if commands + .get(index) + .is_some_and(|command| command.sequence() == sequence) + { + Some(Box::new(commands.remove(index))) + } else { + let prior = index + .checked_sub(1) + .map_or(0, |index| commands[index].sequence()); + if prior.checked_add(1) != Some(sequence) { + return Err(invalid( + sequence, + "scope delivery skips an earlier authored call", + )); + } + None + }; + commands.insert( + index, + RecordedCommand::CancellationBoundary { + sequence, + call_kind: delivered.boundary.call_kind, + original, + }, + ); + } + Ok(()) + } +} + +impl WorkflowState { + /// Validate the whole group before exposing cancellation to application code. + pub(super) fn prepare_parallel_scope_cancellation( + &mut self, + descriptors: &[ParallelDescriptor], + ) -> Result { + let Some(replay) = &self.scope_delivery else { + return Ok(false); + }; + let index = self.command_cursor; + let sequence = descriptors[0].group_path[0].parallel_group_base_sequence; + if let Some(delivered) = replay.canonical.deliveries.get(&sequence) { + let delivered = delivered.clone(); + if !replay.boundary_states[delivered.context.scope_id()] + .contains_key(&replay.active_scope) + || self.cancellation_shield_depth > 0 + || delivered.boundary.call_kind != CancellationCallKind::Parallel + { + return Err(invalid( + sequence, + "scoped group changed its original membership, kind or shielding", + )); + } + let Some(RecordedCommand::CancellationGroup { span, original, .. }) = + self.recorded_commands.get(index).cloned() + else { + return Err(invalid( + sequence, + "scoped group crossed its original delivery boundary", + )); + }; + if span != descriptors.len() as u64 { + return Err(invalid(sequence, "scoped group changed its original span")); + } + self.validate_scoped_group_members(descriptors, &original)?; + let conditions = descriptors + .iter() + .filter(|descriptor| { + matches!(descriptor.operation, ParallelOperation::Condition { .. }) + }) + .count() as u64; + self.condition_wait_occurrence_counter = self + .condition_wait_occurrence_counter + .checked_add(conditions) + .ok_or_else(|| { + invalid(sequence, "scoped condition occurrence counter overflowed") + })?; + self.recorded_commands.splice(index..=index, original); + return self + .replay_scope_cancellation_at(index, CancellationCallKind::Parallel) + .map(|()| false); + } + if self.cancellation_shield_depth > 0 { + return Ok(false); + } + if replay.intent.is_some() { + return Ok(true); + } + let Some(request) = replay + .canonical + .pending_request_for_scope(&replay.active_scope, &replay.tree)? + .cloned() + else { + return Ok(false); + }; + if replay.contexts.contains_key(request.context.scope_id()) { + return Ok(false); + } + if descriptors + .iter() + .any(|descriptor| matches!(descriptor.operation, ParallelOperation::Signal(_))) + { + return Err(Error::CancellationScopeExecutionUnavailable); + } + let span = descriptors.len() as u64; + let resolved = |sequence| { + self.history_events + .iter() + .take(request.history_index) + .any(|event| { + durable_event_sequence(event) == Some(sequence) + && scope_operation_terminal(&event.event_type) + }) + }; + if (sequence..sequence + span).all(resolved) { + return Ok(false); + } + let original: Vec<_> = self + .recorded_commands + .iter() + .skip(index) + .take(descriptors.len()) + .cloned() + .collect(); + self.validate_scoped_group_members(descriptors, &original)?; + let boundary = CancellationDelivery { + request_id: request.context.request_id().into(), + sequence, + call_kind: CancellationCallKind::Parallel, + sequence_span: span, + operation_sequence: None, + operation_sequence_span: 1, + }; + if replay + .canonical + .preparations + .get(request.context.scope_id()) + .is_some_and(|prepared| prepared.boundary != boundary) + { + return Err(invalid( + sequence, + "prepared scope changed its original authored group", + )); + } + self.scope_delivery.as_mut().unwrap().intent = Some(ScopeDeliveryIntent { + context: request.context, + boundary, + command_count: self.commands.len(), + }); + self.matched_recorded_pending = true; + self.command_cursor = index + descriptors.len(); + Ok(true) + } + + fn validate_scoped_group_members( + &self, + descriptors: &[ParallelDescriptor], + original: &[RecordedCommand], + ) -> Result<()> { + if descriptors.len() != original.len() { + return Err(invalid( + descriptors[0].group_path[0].parallel_group_base_sequence, + "scoped group omits an original admitted member", + )); + } + let replay = self.scope_delivery.as_ref().unwrap(); + for (descriptor, recorded) in descriptors.iter().zip(original) { + let sequence = + descriptor.group_path[0].parallel_group_base_sequence + descriptor.offset as u64; + if recorded.sequence() != sequence + || replay.tree.memberships.get(&sequence).map(String::as_str) + != Some(replay.active_scope.as_str()) + { + return Err(invalid( + sequence, + "scoped group changed its original member address", + )); + } + validate_scoped_group_definition(self, descriptor, recorded)?; + } + Ok(()) + } + + pub(super) fn scope_cleanup_timer_proof(&self, scope: &str) -> Result> { + if self.cancellation_shield_depth == 0 { + return Ok(None); + } + let Some(replay) = &self.scope_delivery else { + return Ok(None); + }; + let Some(_) = replay.contexts.get(scope) else { + return Ok(None); + }; + let delivered = replay.consumed_delivery_for_scope(scope).ok_or_else(|| { + Error::InvalidCooperativeCancellation( + "scoped cleanup lacks its consumed original delivery".into(), + ) + })?; + Ok(Some( + json!({"scope_id":delivered.context.scope_id(), "request_id":delivered.context.request_id(), + "delivery_history_event_id":delivered.event.raw["id"]}), + )) + } + + pub(super) fn validate_scope_cleanup_timer(&self, scope: &str, sequence: u64) -> Result<()> { + let original = self + .scope_delivery + .as_ref() + .and_then(|replay| replay.cleanup_timers.get(&sequence)); + if original != self.scope_cleanup_timer_proof(scope)?.as_ref() { + return Err(invalid( + sequence, + "cleanup timer changed its original delivery or shielding", + )); + } + Ok(()) + } + + pub(super) fn prepare_scalar_scope_cancellation( + &mut self, + index: usize, + kind: CancellationCallKind, + path: &[ParallelGroupMetadata], + ) -> Result { + let Some(replay) = &self.scope_delivery else { + return Ok(false); + }; + if self.cancellation_shield_depth > 0 { + return Ok(false); + } + if replay.intent.is_some() { + self.matched_recorded_pending = true; + return Ok(true); + } + let Some(request) = replay + .canonical + .pending_request_for_scope(&replay.active_scope, &replay.tree)? + else { + return Ok(false); + }; + if replay.contexts.contains_key(request.context.scope_id()) { + return Ok(false); + } + let sequence = self.recorded_commands.get(index).map_or_else( + || self.next_scope_cancellation_sequence(), + |command| Ok(command.sequence()), + )?; + if self + .history_events + .iter() + .take(request.history_index) + .any(|event| { + durable_event_sequence(event) == Some(sequence) + && matches!( + event.event_type.as_str(), + "ActivityCompleted" + | "ActivityFailed" + | "ActivityCancelled" + | "ActivityTimedOut" + | "TimerFired" + | "TimerCancelled" + | "ConditionWaitSatisfied" + | "ConditionWaitTimedOut" + | "ChildRunCompleted" + | "ChildRunFailed" + | "ChildRunCancelled" + | "ChildRunTerminated" + ) + }) + { + return Ok(false); + } + if !scalar(kind) || !path.is_empty() { + return Err(Error::CancellationScopeExecutionUnavailable); + } + let boundary = CancellationDelivery { + request_id: request.context.request_id().into(), + sequence, + call_kind: kind, + sequence_span: 1, + operation_sequence: None, + operation_sequence_span: 1, + }; + if replay + .canonical + .preparations + .get(request.context.scope_id()) + .is_some_and(|prepared| prepared.boundary != boundary) + { + return Err(invalid( + sequence, + "prepared scope changed its original authored boundary", + )); + } + let intent = ScopeDeliveryIntent { + context: request.context.clone(), + boundary, + command_count: self.commands.len(), + }; + self.scope_delivery.as_mut().unwrap().intent = Some(intent); + self.matched_recorded_pending = true; + if index < self.recorded_commands.len() { + self.command_cursor = index + 1; + } + Ok(true) + } + + fn next_scope_cancellation_sequence(&self) -> Result { + self.recorded_commands + .last() + .map_or(0, RecordedCommand::sequence) + .checked_add(self.commands.len() as u64) + .and_then(|sequence| sequence.checked_add(1)) + .filter(|sequence| *sequence < i64::MAX as u64) + .ok_or_else(|| invalid(0, "scope cancellation authored sequence overflowed")) + } + + pub(super) fn replay_scope_cancellation_at( + &mut self, + index: usize, + kind: CancellationCallKind, + ) -> Result<()> { + let Some(sequence) = self + .recorded_commands + .get(index) + .map(RecordedCommand::sequence) + else { + return Ok(()); + }; + let Some(replay) = &self.scope_delivery else { + return Ok(()); + }; + let Some(delivered) = replay.canonical.deliveries.get(&sequence).cloned() else { + return Ok(()); + }; + if replay.consumed.contains(&sequence) + || delivered.boundary.call_kind != kind + || !replay.boundary_states[delivered.context.scope_id()] + .contains_key(&replay.active_scope) + || self.cancellation_shield_depth > 0 + { + return Err(invalid( + sequence, + "committed scope delivery changed its original call, membership or shielding", + )); + } + self.observe_scope_cancellation_delivery(&delivered.event); + let replay = self.scope_delivery.as_mut().unwrap(); + replay.consumed.insert(sequence); + replay + .contexts + .extend(replay.boundary_states[delivered.context.scope_id()].clone()); + let active_context = replay.contexts[&replay.active_scope].0.clone(); + self.command_cursor = index + + if kind == CancellationCallKind::Parallel { + delivered.boundary.sequence_span as usize + } else { + 1 + }; + Err(Error::CancellationScopeRequested( + CancellationScopeRequested { + context: active_context.with_replay(cancellation_replay_clock::active_binding()), + delivery: delivered.boundary, + }, + )) + } +} + +impl WorkflowContext { + /// Original metadata only after consuming this context's committed scope delivery. + #[doc(hidden)] + pub fn scoped_cancellation_context(&self) -> Result> { + let state = self + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)?; + Ok(state + .scope_delivery + .as_ref() + .and_then(|replay| replay.contexts.get(&self.cancellation_scope_id)) + .map(|(context, _)| { + context + .clone() + .with_replay(Some(Arc::downgrade(&self.state))) + })) + } +} diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs new file mode 100644 index 0000000..3170523 --- /dev/null +++ b/src/cooperative_cancellation.rs @@ -0,0 +1,2362 @@ +use super::*; +use std::collections::BTreeSet; + +const CONTROL_BUDGET: Duration = Duration::from_secs(5); +const MAX_REFRESH_PAGES: usize = 128; + +pub(super) fn assert_cancellation_scope_replay_supported( + events: &[HistoryEvent], + allow_authoring: bool, + allow_delivery: bool, +) -> Result<()> { + if allow_delivery && !allow_authoring { + return Err(Error::CancellationScopeExecutionUnavailable); + } + for event in events { + let scope_marker = matches!( + event.event_type.as_str(), + "CancellationScopeRequested" + | "CancellationScopeDeliveryPrepared" + | "CancellationScopeDelivered" + | "CancellationScopeRequestConflicted" + ); + let scoped_membership = std::iter::once(&event.payload) + .chain( + ["activity", "timer", "child_workflow"] + .iter() + .filter_map(|name| event.payload.get(name)), + ) + .any(|payload| { + payload + .get("cancellation_scope_id") + .is_some_and(|scope| scope.as_str() != Some("root")) + }); + if (scope_marker && !allow_delivery) + || (!allow_authoring + && (event.event_type == "CancellationScopeOpened" || scoped_membership)) + { + return Err(Error::CancellationScopeExecutionUnavailable); + } + } + Ok(()) +} + +/// A delivery acknowledgment bound to one workflow task and durable run. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CancellationDeliveryReceipt { + pub task_id: String, + pub run_id: String, + pub delivery: CancellationDelivery, +} + +/// Delivery either commits or parks the parent until canonical child cleanup. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CancellationDeliveryReply { + Delivered(CancellationDeliveryReceipt), + /// The current claim is completed. Return to polling without publishing + /// completion or failure. A child terminal event wakes a successor claim. + ClaimReleased { + task_id: String, + run_id: String, + }, +} + +/// Successful renewal of the exact workflow-task claim with optional observation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WorkflowTaskHeartbeat { + pub task_id: String, + pub workflow_task_attempt: u64, + pub lease_owner: String, + pub lease_expires_at: String, + pub run_status: String, + pub cancellation_request: Option, +} + +/// One actual protocol 1.20 claim and its original pending observation. +/// +/// The claim is read-only. Observing a request does not deliver cancellation to +/// workflow code. Canonical history and a committed delivery are still required. +#[derive(Clone, Debug)] +pub struct CooperativeWorkflowTask { + task: WorkflowTask, + cancellation_request: Option, +} + +impl CooperativeWorkflowTask { + pub fn task(&self) -> &WorkflowTask { + &self.task + } + + pub fn cancellation_request(&self) -> Option<&CancellationRequest> { + self.cancellation_request.as_ref() + } + + /// Renew this exact claim and retain the first observation's identity/token. + /// + /// A refused or malformed renewal leaves the claim and observation intact. + pub async fn heartbeat(&mut self, client: &Client) -> Result { + let receipt = client + .heartbeat_workflow_task(&self.task, self.cancellation_request.as_ref()) + .await?; + self.cancellation_request = receipt.cancellation_request.clone(); + Ok(receipt) + } +} + +/// An explicit cooperative poll, including ordinary idle and stop outcomes. +#[derive(Clone, Debug)] +pub struct CooperativeWorkflowTaskPoll { + pub task: Option, + pub outcome: WorkerPollOutcome, + pub protocol_version: Option, + pub server_capabilities: Option, +} + +/// Optional reason and runtime-owned cleanup limit for a cooperative request. +#[derive(Clone, Debug, Default, Serialize)] +pub struct CooperativeCancellationOptions { + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cleanup_timeout_seconds: Option, +} + +/// The Server's original request identity and immutable cleanup deadline. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CancellationRequest { + pub request_id: String, + pub requested_at: String, + pub cleanup_deadline_at: String, + pub history_refresh_page_token: Option, + pub context: Option, +} + +/// Acknowledgment of a request targeting the current durable run. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WorkflowCancellationRequest { + pub workflow_id: String, + pub run_id: String, + pub duplicate: bool, + pub cancellation_request: CancellationRequest, +} + +/// Cancellation delivered at its committed authored call, with original identity. +#[derive(Clone, Debug, Error, PartialEq, Eq)] +#[error("workflow cancellation {request_id} was requested", request_id = .request.request_id)] +pub struct CooperativeCancellationRequested { + pub request: CancellationRequest, + pub delivery: CancellationDelivery, +} + +/// A workflow-local cleanup scope. Dropping it restores cancellation checks. +/// +/// This does not renew or extend the Server's original cleanup deadline. +#[derive(Debug)] +pub struct CancellationShield { + state: Arc>, +} + +impl Drop for CancellationShield { + fn drop(&mut self) { + if let Ok(mut state) = self.state.lock() { + state.cancellation_shield_depth = state.cancellation_shield_depth.saturating_sub(1); + } + } +} + +fn invalid(detail: impl Into) -> Error { + Error::InvalidCooperativeCancellation(detail.into()) +} + +fn text<'a>(value: &'a Value, field: &str) -> Result<&'a str> { + value + .get(field) + .and_then(Value::as_str) + .filter(|text| !text.trim().is_empty()) + .ok_or_else(|| invalid(format!("{field} must be a non-empty string"))) +} + +impl CancellationRequest { + pub(super) fn preserve_observation(&self, current: &Self) -> Result { + self.validate_observation()?; + current.validate_observation()?; + if self.request_id != current.request_id + || DateTime::parse_from_rfc3339(&self.requested_at).unwrap() + != DateTime::parse_from_rfc3339(¤t.requested_at).unwrap() + || DateTime::parse_from_rfc3339(&self.cleanup_deadline_at).unwrap() + != DateTime::parse_from_rfc3339(¤t.cleanup_deadline_at).unwrap() + { + return Err(invalid( + "observation changed the original request or cleanup deadline", + )); + } + Ok(self.clone()) + } + + pub(super) fn validate_observation(&self) -> Result<()> { + Self::from_observation(&json!({ + "request_id":self.request_id, "requested_at":self.requested_at, + "cleanup_deadline_at":self.cleanup_deadline_at, + "history_refresh_page_token":self.history_refresh_page_token, + })) + .map(|_| ()) + } + + pub(crate) fn from_observation(value: &Value) -> Result { + let requested_at = text(value, "requested_at")?; + let cleanup_deadline_at = text(value, "cleanup_deadline_at")?; + let requested = DateTime::parse_from_rfc3339(requested_at) + .map_err(|_| invalid("requested_at must be a timestamp with a timezone"))?; + let deadline = DateTime::parse_from_rfc3339(cleanup_deadline_at) + .map_err(|_| invalid("cleanup_deadline_at must be a timestamp with a timezone"))?; + if deadline <= requested { + return Err(invalid("cleanup deadline must follow the original request")); + } + Ok(Self { + request_id: text(value, "request_id")?.to_owned(), + requested_at: requested_at.to_owned(), + cleanup_deadline_at: cleanup_deadline_at.to_owned(), + history_refresh_page_token: Some(text(value, "history_refresh_page_token")?.to_owned()), + context: None, + }) + } +} + +const REQUEST_EVENT: &str = "CooperativeCancellationRequested"; +const DELIVERY_EVENT: &str = "CooperativeCancellationDelivered"; +const MAX_SEQUENCE: u64 = i64::MAX as u64; + +/// The authored durable call where canonical cancellation is delivered. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum CancellationCallKind { + Activity, + LocalActivity, + Timer, + Condition, + Signal, + Child, + Parallel, + SelectionHandle, +} + +/// One committed delivery marker, bound to the original cancellation request. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CancellationDelivery { + pub request_id: String, + pub sequence: u64, + pub call_kind: CancellationCallKind, + pub sequence_span: u64, + pub operation_sequence: Option, + pub operation_sequence_span: u64, +} + +fn positive(value: &Value, field: &str, maximum: u64) -> Result { + value + .as_u64() + .filter(|number| (1..=maximum).contains(number)) + .ok_or_else(|| { + invalid(format!( + "{field} must be a positive integer within {maximum}" + )) + }) +} + +impl CancellationDelivery { + pub(crate) fn from_payload(value: &Value) -> Result { + let sequence = positive(&value["sequence"], "sequence", MAX_SEQUENCE)?; + let call_kind: CancellationCallKind = serde_json::from_value(value["call_kind"].clone()) + .map_err(|_| invalid("delivery must name a supported durable call kind"))?; + let span = positive( + value.get("sequence_span").unwrap_or(&json!(1)), + "sequence_span", + 1000, + )?; + let operation_span = positive( + value.get("operation_sequence_span").unwrap_or(&json!(1)), + "operation_sequence_span", + 1000, + )?; + let operation = value + .get("operation_sequence") + .filter(|value| !value.is_null()); + if (call_kind != CancellationCallKind::Parallel && span != 1) + || sequence > MAX_SEQUENCE - span + { + return Err(invalid( + "delivery call span is invalid or overflows the portable sequence range", + )); + } + let operation_sequence = if call_kind == CancellationCallKind::SelectionHandle { + let base = positive( + operation.unwrap_or(&Value::Null), + "operation_sequence", + MAX_SEQUENCE, + )?; + if base >= sequence || operation_span > sequence - base { + return Err(invalid( + "selection handle must name a complete earlier operation range", + )); + } + Some(base) + } else { + if operation.is_some() || operation_span != 1 { + return Err(invalid( + "only selection handles may name an operation range", + )); + } + None + }; + Ok(Self { + request_id: text(value, "workflow_command_id")?.to_owned(), + sequence, + call_kind, + sequence_span: span, + operation_sequence, + operation_sequence_span: operation_span, + }) + } + + fn range(&self) -> (u64, u64) { + self.operation_sequence + .map_or((self.sequence, self.sequence_span), |base| { + (base, self.operation_sequence_span) + }) + } + + /// Whether a sequence belongs to the canonical interrupted operation range. + pub fn interrupts(&self, sequence: u64) -> bool { + let (base, span) = self.range(); + sequence >= base && sequence - base < span + } +} + +/// Validated canonical cancellation state for one workflow run. +/// +/// An observation identifies a request. Only its committed delivery marker +/// authorizes interruption at an authored call. Earlier results retain priority. +#[derive(Clone, Debug)] +pub struct CancellationHistory { + pub request: Option, + pub delivery: Option, + pub request_index: usize, + pub delivery_index: Option, + resolved_before_request: BTreeSet, + latest_condition_sequences_before_request: BTreeMap, + failed_before_request: BTreeSet, + selected_before_request: BTreeSet<(u64, u64)>, +} + +impl CancellationHistory { + pub(super) fn bind_commands( + &self, + mut commands: Vec, + ) -> Result> { + let Some(delivery) = &self.delivery else { + return Ok(commands); + }; + if delivery.call_kind == CancellationCallKind::Parallel { + let index = commands.partition_point(|command| command.sequence() < delivery.sequence); + let end = commands.partition_point(|command| { + command.sequence() < delivery.sequence + delivery.sequence_span + }); + if commands.get(index).map(RecordedCommand::sequence) != Some(delivery.sequence) { + let previous = index + .checked_sub(1) + .map_or(0, |index| commands[index].sequence()); + if previous.checked_add(1) != Some(delivery.sequence) { + return Err(invalid_recorded_history( + "cooperative_cancellation_call_mismatch", + delivery.sequence, + "next authored parallel group", + "missing earlier call", + "parallel cancellation marker skips an unrecorded authored command", + )); + } + } + let original = commands.drain(index..end).collect(); + commands.insert( + index, + RecordedCommand::CancellationGroup { + sequence: delivery.sequence, + span: delivery.sequence_span, + original, + }, + ); + // Physical condition reopens retain the group's authored identity + // but can lie beyond its original sequence span. Bind unresolved + // reopens as well, so replay consumes the latest wait at delivery. + for command in &mut commands[index + 1..] { + let RecordedCommand::ConditionWait { + sequence, + parallel_group_path: Some(path), + .. + } = command + else { + continue; + }; + if self.resolved_before_request.contains(sequence) + || !path.first().is_some_and(|group| { + group.parallel_group_base_sequence == delivery.sequence + && group.parallel_group_size as u64 == delivery.sequence_span + }) + { + continue; + } + *command = RecordedCommand::CancellationBoundary { + sequence: *sequence, + call_kind: CancellationCallKind::Condition, + original: Some(Box::new(command.clone())), + }; + } + return Ok(commands); + } + if !matches!( + delivery.call_kind, + CancellationCallKind::Activity + | CancellationCallKind::Timer + | CancellationCallKind::Condition + | CancellationCallKind::Signal + | CancellationCallKind::Child + | CancellationCallKind::SelectionHandle + ) { + return Ok(commands); + } + let index = commands.partition_point(|command| command.sequence() < delivery.sequence); + let original = if commands + .get(index) + .is_some_and(|command| command.sequence() == delivery.sequence) + { + Some(Box::new(commands.remove(index))) + } else { + let previous = index + .checked_sub(1) + .map_or(0, |index| commands[index].sequence()); + if previous.checked_add(1) != Some(delivery.sequence) { + return Err(invalid_recorded_history( + "cooperative_cancellation_call_mismatch", + delivery.sequence, + "next authored durable call", + "missing earlier call", + "cancellation marker skips an unrecorded authored command", + )); + } + None + }; + commands.insert( + index, + RecordedCommand::CancellationBoundary { + sequence: delivery.sequence, + call_kind: delivery.call_kind, + original, + }, + ); + Ok(commands) + } + + /// Read canonical markers without replacing original request identity. + pub fn from_events( + events: &[HistoryEvent], + run_id: &str, + observation: Option<&CancellationRequest>, + ) -> Result { + Self::read(events, run_id, observation).map_err(|error| { + invalid_recorded_history( + "cooperative_cancellation_history_invalid", + 0, + "one canonical request and matching authored delivery", + "invalid cancellation history", + &error.to_string(), + ) + }) + } + + fn read( + events: &[HistoryEvent], + run_id: &str, + observation: Option<&CancellationRequest>, + ) -> Result { + if let Some(observed) = observation { + let requested = DateTime::parse_from_rfc3339(&observed.requested_at) + .map_err(|_| invalid("observed request timestamp is invalid"))?; + let deadline = DateTime::parse_from_rfc3339(&observed.cleanup_deadline_at) + .map_err(|_| invalid("observed cleanup deadline is invalid"))?; + if observed.request_id.trim().is_empty() + || deadline <= requested + || observed + .history_refresh_page_token + .as_ref() + .is_some_and(|token| token.trim().is_empty()) + { + return Err(invalid( + "observed request identity, deadline or history token is invalid", + )); + } + } + let mut state = Self { + request: observation.cloned().map(|mut request| { + request.context = None; + request + }), + delivery: None, + request_index: events.len(), + delivery_index: None, + resolved_before_request: BTreeSet::new(), + latest_condition_sequences_before_request: BTreeMap::new(), + failed_before_request: BTreeSet::new(), + selected_before_request: BTreeSet::new(), + }; + let mut saw_request = false; + for (index, event) in events.iter().enumerate() { + if !matches!(event.event_type.as_str(), REQUEST_EVENT | DELIVERY_EVENT) { + continue; + } + let request_id = text(&event.payload, "workflow_command_id")?; + let event_run = text(&event.payload, "workflow_run_id")?; + if event + .raw + .get("workflow_command_id") + .is_some_and(|value| value.as_str() != Some(request_id)) + || (!run_id.is_empty() && event_run != run_id) + { + return Err(invalid( + "canonical event does not match its request or workflow run", + )); + } + if event.event_type == REQUEST_EVENT { + if saw_request || state.delivery.is_some() { + return Err(invalid("history must contain one request before delivery")); + } + let recorded_at = event + .raw + .get("recorded_at") + .or_else(|| event.raw.get("timestamp")) + .and_then(Value::as_str) + .ok_or_else(|| invalid("canonical request lacks its recorded timestamp"))?; + let requested = DateTime::parse_from_rfc3339(recorded_at) + .map_err(|_| invalid("canonical request timestamp is invalid"))?; + let deadline_text = text(&event.payload, "cleanup_deadline_at")?; + let deadline = DateTime::parse_from_rfc3339(deadline_text) + .map_err(|_| invalid("canonical cleanup deadline is invalid"))?; + let context = event + .payload + .get("cancellation") + .map(CancellationContext::from_value) + .transpose()?; + if let Some(context) = &context { + let local = context.lineage().last().unwrap(); + if context.request_id() != request_id + || local.workflow_run_id() != event_run + || event + .payload + .get("workflow_instance_id") + .is_some_and(|instance| { + instance.as_str() != Some(local.workflow_instance_id()) + }) + || context.deadline() != deadline + || context.requested_at() > requested + || event + .payload + .get("reason") + .is_some_and(|reason| match reason { + Value::Null => context.reason().is_some(), + Value::String(reason) => Some(reason.as_str()) != context.reason(), + _ => true, + }) + { + return Err(invalid( + "canonical cancellation context does not match its request event", + )); + } + } else if deadline <= requested { + return Err(invalid( + "canonical cleanup deadline must follow the request", + )); + } + if let Some(observed) = observation { + let observed_deadline = + DateTime::parse_from_rfc3339(&observed.cleanup_deadline_at) + .map_err(|_| invalid("observed cleanup deadline is invalid"))?; + if observed.request_id != request_id || observed_deadline != deadline { + return Err(invalid( + "observation changes the original request or cleanup deadline", + )); + } + } + if context.is_some() || observation.is_none() { + state.request = Some(CancellationRequest { + request_id: request_id.to_owned(), + requested_at: context.as_ref().map_or_else( + || recorded_at.to_owned(), + |context| { + context + .requested_at() + .to_rfc3339_opts(chrono::SecondsFormat::Micros, true) + }, + ), + cleanup_deadline_at: deadline_text.to_owned(), + history_refresh_page_token: observation + .and_then(|request| request.history_refresh_page_token.clone()), + context, + }); + } + state.request_index = index; + saw_request = true; + } else { + if !saw_request || state.delivery.is_some() { + return Err(invalid( + "delivery requires one earlier request and one marker", + )); + } + let delivery = CancellationDelivery::from_payload(&event.payload)?; + if state + .request + .as_ref() + .map(|request| request.request_id.as_str()) + != Some(delivery.request_id.as_str()) + { + return Err(invalid("delivery names a different original request")); + } + if let Some(snapshot) = event.payload.get("cancellation") { + let context = CancellationContext::from_value(snapshot)?; + if state + .request + .as_ref() + .and_then(|request| request.context.as_ref()) + != Some(&context) + { + return Err(invalid( + "delivery changes the canonical cancellation context", + )); + } + } + state.delivery = Some(delivery); + state.delivery_index = Some(index); + } + } + if state.request.is_none() { + return Ok(state); + } + let mut condition_occurrences = BTreeMap::new(); + for event in &events[..state.request_index] { + if matches!( + event.event_type.as_str(), + "SelectionResolved" | "SelectionOperationCancelled" + ) { + let (base_field, span_field) = if event.event_type == "SelectionResolved" { + ("selection_group_base_sequence", "selection_group_size") + } else { + ("member_base_sequence", "member_size") + }; + if let (Ok(base), Ok(span)) = ( + positive(&event.payload[base_field], base_field, MAX_SEQUENCE), + positive(&event.payload[span_field], span_field, 1000), + ) { + if base <= MAX_SEQUENCE - span { + if event.event_type == "SelectionResolved" { + state.selected_before_request.insert((base, span)); + } else { + state.resolved_before_request.extend(base..base + span); + } + } + } + } + let Ok(sequence) = positive(&event.payload["sequence"], "sequence", MAX_SEQUENCE) + else { + continue; + }; + // Groups deliver at their original authored span. Scalar condition + // delivery already names the pending physical sequence directly. + if event.event_type == "ConditionWaitOpened" + && (event.payload["parallel_group_id"].is_string() + || event.payload["parallel_group_path"] + .as_array() + .is_some_and(|path| !path.is_empty())) + { + if let Some(occurrence) = event.payload["condition_wait_occurrence_id"].as_str() { + let authored = *condition_occurrences.entry(occurrence).or_insert(sequence); + state + .latest_condition_sequences_before_request + .insert(authored, sequence); + } + } + if matches!( + event.event_type.as_str(), + "ActivityCompleted" + | "ActivityFailed" + | "ActivityCancelled" + | "ActivityTimedOut" + | "TimerFired" + | "TimerCancelled" + | "ConditionWaitSatisfied" + | "ConditionWaitTimedOut" + | "SignalApplied" + | "ChildRunCompleted" + | "ChildRunFailed" + | "ChildRunCancelled" + | "ChildRunTerminated" + ) { + state.resolved_before_request.insert(sequence); + if matches!( + event.event_type.as_str(), + "ActivityFailed" + | "ActivityCancelled" + | "ActivityTimedOut" + | "ChildRunFailed" + | "ChildRunCancelled" + | "ChildRunTerminated" + ) { + state.failed_before_request.insert(sequence); + } + } + } + if let Some(delivery) = &state.delivery { + let (base, span) = delivery.range(); + if !state.range_eligible(base, span) { + return Err(invalid( + "delivery cannot replace an earlier committed result", + )); + } + } + Ok(state) + } + + fn range_eligible(&self, sequence: u64, span: u64) -> bool { + (1..=1000).contains(&span) + && sequence > 0 + && sequence <= MAX_SEQUENCE - span + && !(sequence..sequence + span).all(|sequence| { + let latest = self + .latest_condition_sequences_before_request + .get(&sequence) + .unwrap_or(&sequence); + self.resolved_before_request.contains(latest) + }) + && !(sequence..sequence + span) + .any(|sequence| self.failed_before_request.contains(&sequence)) + && !self.selected_before_request.contains(&(sequence, span)) + } + + /// Whether an unresolved authored call may deliver the pending request. + pub fn eligible(&self, sequence: u64, span: u64) -> bool { + self.request.is_some() && self.delivery.is_none() && self.range_eligible(sequence, span) + } +} + +impl WorkflowState { + fn next_cancellation_sequence(&self) -> Result { + self.recorded_commands.get(self.command_cursor).map_or_else( + || { + self.recorded_commands + .last() + .map_or(0, RecordedCommand::sequence) + .checked_add(self.commands.len() as u64) + .and_then(|sequence| sequence.checked_add(1)) + .filter(|sequence| *sequence < MAX_SEQUENCE) + .ok_or_else(|| invalid("authored cancellation sequence overflowed")) + }, + |command| Ok(command.sequence()), + ) + } + + fn prepare_cancellation_delivery(&mut self, delivery: CancellationDelivery) -> Result { + if !self.cancellation_delivery_enabled || self.cancellation_shield_depth > 0 { + return Ok(false); + } + self.assert_root_cancellation_membership()?; + if self.cancellation_delivery_intent.is_some() { + self.matched_recorded_pending = true; + return Ok(true); + } + let (base, span) = delivery.range(); + if !self.cancellation_history.eligible(base, span) { + return Ok(false); + } + let payload = json!({ + "workflow_command_id":delivery.request_id, "sequence":delivery.sequence, + "call_kind":delivery.call_kind, "sequence_span":delivery.sequence_span, + "operation_sequence":delivery.operation_sequence, + "operation_sequence_span":delivery.operation_sequence_span, + }); + CancellationDelivery::from_payload(&payload)?; + self.cancellation_delivery_command_count = self.commands.len(); + self.cancellation_delivery_intent = Some(delivery); + self.matched_recorded_pending = true; + Ok(true) + } + + fn assert_root_cancellation_membership(&self) -> Result<()> { + if self + .scope_delivery + .as_ref() + .is_some_and(|replay| replay.active_scope != "root") + { + return Err(Error::CancellationScopeExecutionUnavailable); + } + Ok(()) + } + + pub(super) fn prepare_scalar_cancellation( + &mut self, + index: usize, + kind: CancellationCallKind, + group_path: &[ParallelGroupMetadata], + ) -> Result { + if self.prepare_scalar_scope_cancellation(index, kind, group_path)? { + return Ok(true); + } + if !self.cancellation_delivery_enabled || self.cancellation_shield_depth > 0 { + return Ok(false); + } + // Group leaves validate their recorded calls before the enclosing + // parallel/select future suspends. They never choose scalar delivery. + if !group_path.is_empty() && self.cancellation_delivery_intent.is_none() { + return Ok(false); + } + let Some(request) = self.cancellation_history.request.as_ref() else { + return Ok(false); + }; + let sequence = self.recorded_commands.get(index).map_or_else( + || self.next_cancellation_sequence(), + |command| Ok(command.sequence()), + )?; + let pending = self.prepare_cancellation_delivery(CancellationDelivery { + request_id: request.request_id.clone(), + sequence, + call_kind: kind, + sequence_span: 1, + operation_sequence: None, + operation_sequence_span: 1, + })?; + if pending && index < self.recorded_commands.len() { + self.command_cursor = index + 1; + } + Ok(pending) + } + + pub(super) fn prepare_group_cancellation( + &mut self, + descriptors: &[ParallelDescriptor], + ) -> Result<()> { + let Some(request) = self.cancellation_history.request.as_ref() else { + return Ok(()); + }; + let Some(group) = descriptors.first().and_then(|leaf| leaf.group_path.first()) else { + return Ok(()); + }; + self.prepare_cancellation_delivery(CancellationDelivery { + request_id: request.request_id.clone(), + sequence: group.parallel_group_base_sequence, + call_kind: CancellationCallKind::Parallel, + sequence_span: descriptors.len() as u64, + operation_sequence: None, + operation_sequence_span: 1, + })?; + Ok(()) + } + + pub(super) fn prepare_selection_handle_cancellation( + &mut self, + handle: &DurableOperationHandle, + ) -> Result { + if !self.cancellation_delivery_enabled || self.cancellation_shield_depth > 0 { + return Ok(false); + } + validate_selection_delivery_handle(self, handle)?; + if !self + .cancellation_history + .eligible(handle.base_sequence, handle.size as u64) + { + return Ok(false); + } + let request_id = self + .cancellation_history + .request + .as_ref() + .expect("eligible request") + .request_id + .clone(); + self.prepare_cancellation_delivery(CancellationDelivery { + request_id, + sequence: self.next_cancellation_sequence()?, + call_kind: CancellationCallKind::SelectionHandle, + sequence_span: 1, + operation_sequence: Some(handle.base_sequence), + operation_sequence_span: handle.size as u64, + }) + } + + pub(super) fn expand_cancellation_group( + &mut self, + descriptors: &[ParallelDescriptor], + ) -> Result<()> { + let Some(RecordedCommand::CancellationGroup { + sequence, + span, + original, + }) = self.recorded_commands.get(self.command_cursor).cloned() + else { + return Ok(()); + }; + self.validate_cancellation_call( + sequence, + CancellationCallKind::Parallel, + CancellationCallKind::Parallel, + )?; + if descriptors.len() as u64 != span { + return Err(invalid_recorded_history( + "cooperative_cancellation_call_mismatch", + sequence, + &format!("parallel group with {span} durable leaves"), + &format!("{} durable leaves", descriptors.len()), + "authored parallel group span differs from its committed cancellation", + )); + } + let mut original = original.into_iter().peekable(); + let mut leaves = Vec::with_capacity(descriptors.len()); + for descriptor in descriptors { + let leaf_sequence = sequence + descriptor.offset as u64; + let command = if original.peek().map(RecordedCommand::sequence) == Some(leaf_sequence) { + original.next() + } else { + None + }; + if self + .cancellation_history + .resolved_before_request + .contains(&leaf_sequence) + { + let command = command.ok_or_else(|| { + invalid_recorded_history( + "cooperative_cancellation_call_mismatch", + leaf_sequence, + "recorded earlier result", + "missing durable call", + "parallel cancellation cannot discard an earlier committed result", + ) + })?; + leaves.push(command); + continue; + } + let kind = match &descriptor.operation { + ParallelOperation::Activity { .. } => CancellationCallKind::Activity, + ParallelOperation::ChildWorkflow { .. } => CancellationCallKind::Child, + ParallelOperation::Timer(_) => CancellationCallKind::Timer, + ParallelOperation::Signal(_) => CancellationCallKind::Signal, + ParallelOperation::Condition { .. } => CancellationCallKind::Condition, + ParallelOperation::Group(_) => unreachable!("descriptor is a durable leaf"), + }; + leaves.push(RecordedCommand::CancellationBoundary { + sequence: leaf_sequence, + call_kind: kind, + original: command.map(Box::new), + }); + } + self.recorded_commands + .splice(self.command_cursor..=self.command_cursor, leaves); + Ok(()) + } + + pub(super) fn cancellation_error(&mut self) -> Error { + if self.cancellation_consumed { + self.start_cancellation_clock(); + } + match ( + &self.cancellation_history.request, + &self.cancellation_history.delivery, + ) { + (Some(request), Some(delivery)) => { + let mut request = request.clone(); + request.context = request.context.map(|context| { + context.with_replay(cancellation_replay_clock::active_binding()) + }); + Error::CooperativeCancellationRequested(CooperativeCancellationRequested { + request, + delivery: delivery.clone(), + }) + } + _ => Error::WorkflowCancellationRequested(WorkflowCancellationRequested), + } + } + + fn validate_cancellation_call( + &self, + sequence: u64, + kind: CancellationCallKind, + recorded_kind: CancellationCallKind, + ) -> Result<()> { + if kind != recorded_kind || self.cancellation_shield_depth > 0 { + return Err(invalid_recorded_history( + "cooperative_cancellation_call_mismatch", + sequence, + "matching unshielded authored call", + &format!("{kind:?}"), + "committed cancellation call kind or cleanup scope changed", + )); + } + Ok(()) + } + + pub(super) fn cancellation_replay_command( + &mut self, + index: usize, + kind: CancellationCallKind, + ) -> Result> { + match self.recorded_commands.get(index).cloned() { + Some(RecordedCommand::CancellationBoundary { + sequence, + call_kind, + original, + }) => { + if let Some(original) = original { + return Ok(Some(*original)); + } + self.replay_scope_cancellation_at(index, kind)?; + self.assert_root_cancellation_membership()?; + self.validate_cancellation_call(sequence, kind, call_kind)?; + self.cancellation_consumed = true; + self.cancel_requested = true; + self.command_cursor = index + 1; + Err(self.cancellation_error()) + } + command => Ok(command), + } + } + + pub(super) fn replay_cancellation_at( + &mut self, + index: usize, + kind: CancellationCallKind, + ) -> Result<()> { + self.replay_scope_cancellation_at(index, kind)?; + if let Some(RecordedCommand::CancellationBoundary { + sequence, + call_kind, + .. + }) = self.recorded_commands.get(index) + { + self.assert_root_cancellation_membership()?; + self.validate_cancellation_call(*sequence, kind, *call_kind)?; + self.cancellation_consumed = true; + self.cancel_requested = true; + self.command_cursor = index + 1; + return Err(self.cancellation_error()); + } + Ok(()) + } + + pub(super) fn replay_selection_handle_cancellation( + &mut self, + handle: &DurableOperationHandle, + ) -> Result<()> { + let Some(RecordedCommand::CancellationBoundary { + sequence, + call_kind, + original, + }) = self.recorded_commands.get(self.command_cursor) + else { + return Ok(()); + }; + self.validate_cancellation_call( + *sequence, + CancellationCallKind::SelectionHandle, + *call_kind, + )?; + let delivery = self + .cancellation_history + .delivery + .as_ref() + .expect("bound canonical delivery"); + if original.is_some() + || delivery.operation_sequence != Some(handle.base_sequence) + || delivery.operation_sequence_span != handle.size as u64 + { + return Err(invalid_recorded_history( + "cooperative_cancellation_call_mismatch", + *sequence, + "selection handle for the committed operation range", + &format!("{}:{}", handle.base_sequence, handle.size), + "cancellation delivery targets a different authored selection member", + )); + } + validate_selection_delivery_handle(self, handle)?; + self.replay_cancellation_at(self.command_cursor, CancellationCallKind::SelectionHandle) + } +} + +impl WorkflowContext { + /// Original immutable metadata, visible only at committed delivery. + pub fn cancellation_context(&self) -> Result> { + let state = self + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)?; + Ok( + if state.cancellation_consumed && self.cancellation_scope_id == "root" { + state + .cancellation_history + .request + .as_ref() + .and_then(|request| request.context.clone()) + .map(|context| context.with_replay(Some(Arc::downgrade(&self.state)))) + } else { + None + }, + ) + } + + /// Shield explicit cleanup from repeated cancellation checks. + /// + /// Keep the returned guard alive across cleanup awaits. Scopes can nest. + /// The Server retains and enforces the original immutable cleanup deadline. + pub fn cancellation_shield(&self) -> Result { + let mut state = self + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)?; + state.cancellation_shield_depth = state + .cancellation_shield_depth + .checked_add(1) + .ok_or_else(|| invalid("cancellation shield nesting overflowed"))?; + Ok(CancellationShield { + state: Arc::clone(&self.state), + }) + } +} + +pub(super) fn supports_protocol(version: &str) -> bool { + let Some((major, minor)) = version.split_once('.') else { + return false; + }; + major == "1" + && !minor.is_empty() + && minor.bytes().all(|byte| byte.is_ascii_digit()) + && minor.parse::().is_ok_and(|minor| minor >= 20) +} + +fn require_discovery(info: &Value) -> Result<()> { + let protocol = &info["worker_protocol"]; + if protocol["server_capabilities"]["cooperative_cancellation"].as_bool() != Some(true) + || !protocol["version"].as_str().is_some_and(supports_protocol) + { + return Err(Error::CooperativeCancellationUnavailable( + "runtime discovery must explicitly advertise support and compatible protocol 1.20" + .to_string(), + )); + } + Ok(()) +} + +fn acknowledgment( + value: &Value, + workflow_id: &str, + selected_run: Option<&str>, +) -> Result { + if value["accepted"].as_bool() != Some(true) + || text(value, "workflow_id")? != workflow_id + || selected_run.is_some_and(|run| value["run_id"].as_str() != Some(run)) + { + return Err(invalid( + "acknowledgment does not match the accepted workflow/run request", + )); + } + Ok(WorkflowCancellationRequest { + workflow_id: workflow_id.to_owned(), + run_id: text(value, "run_id")?.to_owned(), + duplicate: value["duplicate"] + .as_bool() + .ok_or_else(|| invalid("duplicate must be a boolean"))?, + cancellation_request: CancellationRequest::from_observation( + &value["cancellation_request"], + )?, + }) +} + +impl Client { + /// Report a stopped and dropped remote callback under its original claim. + /// + /// Call only after callback execution has ended. This explicit protocol 1.20 + /// operation preserves the original cancellation request and grants no lease + /// or result publication authority. The request has one five-second budget. + pub async fn acknowledge_activity_cancellation( + &self, + task_id: &str, + activity_attempt_id: &str, + lease_owner: &str, + request_id: &str, + ) -> Result { + if [task_id, activity_attempt_id, lease_owner, request_id] + .iter() + .any(|value| value.trim().is_empty() || value.len() > 255) + { + return Err(invalid( + "activity stop receipt requires bounded original claim and request identities", + )); + } + tokio::time::timeout(CONTROL_BUDGET, async { + let value: Value = activity_task_response( + self.request_json( + reqwest::Method::POST, + &format!("/worker/activity-tasks/{}/acknowledge-cancellation", percent_encode_path_segment(task_id)), + RequestProtocol::Worker("1.20"), + Some(&json!({"activity_attempt_id":activity_attempt_id,"lease_owner":lease_owner,"request_id":request_id})), + ).await, + "acknowledge-cancellation", task_id, activity_attempt_id, + )?; + if value["task_id"].as_str() != Some(task_id) + || value["activity_attempt_id"].as_str() != Some(activity_attempt_id) + || value["lease_owner"].as_str() != Some(lease_owner) + || value["request_id"].as_str() != Some(request_id) + || value["acknowledged"].as_bool() != Some(true) + || value["duplicate"].as_bool().is_none() + || value.get("reason") != Some(&Value::Null) + || value["heartbeat_recorded"].as_bool() != Some(false) + || text(&value, "history_event_id").is_err() + { + return Err(invalid("activity callback-stop acknowledgment did not prove the original claim and receipt")); + } + Ok(value) + }).await.map_err(|_| Error::Timeout)? + } + + /// Observe an exact activity attempt without renewing its lease or progress. + /// + /// This explicit worker-protocol 1.20 operation has one five-second budget. + /// A pending workflow request alone does not cancel an activity. Its reply + /// reports the runtime's current continuation authority after delivery. + pub async fn activity_task_status( + &self, + task_id: &str, + activity_attempt_id: &str, + lease_owner: &str, + ) -> Result { + if [task_id, activity_attempt_id, lease_owner] + .iter() + .any(|value| value.trim().is_empty()) + { + return Err(invalid("activity observation requires the actual claim")); + } + tokio::time::timeout(CONTROL_BUDGET, async { + let value: Value = activity_task_response( + self.request_json( + reqwest::Method::POST, + &format!( + "/worker/activity-tasks/{}/status", + percent_encode_path_segment(task_id) + ), + RequestProtocol::Worker("1.20"), + Some(&json!({ + "activity_attempt_id":activity_attempt_id,"lease_owner":lease_owner + })), + ) + .await, + "status", + task_id, + activity_attempt_id, + )?; + if value["task_id"].as_str() != Some(task_id) + || value["activity_attempt_id"].as_str() != Some(activity_attempt_id) + || value["lease_owner"].as_str() != Some(lease_owner) + || value["can_continue"].as_bool().is_none() + || value["cancel_requested"].as_bool().is_none() + || value["heartbeat_recorded"].as_bool() != Some(false) + { + return Err(invalid( + "activity observation did not acknowledge the exact claim", + )); + } + Ok(value) + }) + .await + .map_err(|_| Error::Timeout)? + } + + /// Acquire a claim with protocol 1.20 and retain its pending observation. + /// + /// The Server must have admitted a capable worker registration. This method + /// registers no capability and does not change ordinary [`Worker`] polling. + /// It never substitutes a worker ID for a missing lease owner or invents a + /// task attempt. Poll retries reuse one request ID. Poll and history loading + /// share one long-poll budget plus five seconds, with at most 128 pages. + pub async fn poll_cooperative_workflow_task( + &self, + worker_id: &str, + task_queue: &str, + timeout: Duration, + ) -> Result { + self.poll_cooperative_workflow_task_with_request_id( + worker_id, + task_queue, + timeout, + &unique_request_id("rust-workflow-poll"), + 1, + ) + .await + } + + async fn poll_cooperative_workflow_task_with_request_id( + &self, + worker_id: &str, + task_queue: &str, + timeout: Duration, + poll_request_id: &str, + transport_retries: usize, + ) -> Result { + if worker_id.trim().is_empty() + || task_queue.trim().is_empty() + || poll_request_id.trim().is_empty() + { + return Err(invalid( + "cooperative polling requires a worker and task queue", + )); + } + let budget = timeout + .checked_add(CONTROL_BUDGET) + .ok_or_else(|| invalid("cooperative poll timeout exceeds its supported budget"))?; + let body = json!({ + "worker_id":worker_id, "task_queue":task_queue, + "poll_request_id":poll_request_id, + "timeout_seconds":long_poll_timeout_seconds(timeout), + "history_page_size":WORKFLOW_HISTORY_PAGE_SIZE, + }); + tokio::time::timeout(budget, async { + let value: Value = self + .poll_request_json( + "/worker/workflow-tasks/poll", + RequestProtocol::Worker("1.20"), + &body, + budget, + transport_retries, + ) + .await?; + let mut response: PollWorkflowTaskResponse = serde_json::from_value(value.clone()) + .map_err(|_| invalid("cooperative poll returned a malformed envelope or task"))?; + let outcome = response.outcome(); + let task = if let Some(task) = response.task.take() { + let (owner, _) = cancellation_claim(&task)?; + if owner != worker_id + || value["task"]["workflow_task_attempt"].as_u64() + != Some(task.workflow_task_attempt) + { + return Err(invalid( + "cooperative poll did not return the caller's actual owner and attempt", + )); + } + let cancellation_request = match value["task"].get("cancellation_request") { + None | Some(Value::Null) => None, + Some(observation) => Some(CancellationRequest::from_observation(observation)?), + }; + let mut claim = CooperativeWorkflowTask { + task, + cancellation_request, + }; + self.load_cooperative_claim_history(&mut claim).await?; + Some(claim) + } else { + None + }; + Ok(CooperativeWorkflowTaskPoll { + task, + outcome, + protocol_version: response.protocol_version, + server_capabilities: response.server_capabilities, + }) + }) + .await + .map_err(|_| Error::Timeout)? + } + + async fn load_cooperative_claim_history( + &self, + claim: &mut CooperativeWorkflowTask, + ) -> Result<()> { + let (owner, _) = cancellation_claim(&claim.task)?; + let owner = owner.to_owned(); + let mut token = claim.task.next_history_page_token.clone(); + let mut seen = BTreeSet::new(); + while let Some(current) = token.take() { + if current.trim().is_empty() + || seen.len() >= MAX_REFRESH_PAGES + || !seen.insert(current.clone()) + { + return Err(invalid( + "claim history exceeded its page bound or returned an invalid/repeated token", + )); + } + let body = json!({ + "lease_owner":owner,"workflow_task_attempt":claim.task.workflow_task_attempt, + "next_history_page_token":current,"history_page_size":WORKFLOW_HISTORY_PAGE_SIZE, + }); + let value: Value = self + .request_json( + reqwest::Method::POST, + &format!( + "/worker/workflow-tasks/{}/history", + percent_encode_path_segment(&claim.task.task_id) + ), + RequestProtocol::Worker("1.20"), + Some(&body), + ) + .await?; + if value["task_id"].as_str() != Some(claim.task.task_id.as_str()) + || value["workflow_task_attempt"].as_u64() != Some(claim.task.workflow_task_attempt) + { + return Err(invalid( + "claim history changed the selected task or attempt", + )); + } + let events = value["history_events"] + .as_array() + .ok_or_else(|| invalid("claim history page must contain an event array"))?; + if events.len() > WORKFLOW_HISTORY_PAGE_SIZE as usize { + return Err(invalid( + "claim history page exceeds its requested event limit", + )); + } + token = match value.get("next_history_page_token") { + Some(Value::Null) => None, + Some(Value::String(next)) if !next.trim().is_empty() && !events.is_empty() => { + Some(next.clone()) + } + _ => return Err(invalid("claim history page token or progress is invalid")), + }; + let page: WorkflowTaskHistoryPage = serde_json::from_value(value) + .map_err(|_| invalid("claim history page contains malformed fields or events"))?; + if page + .history_events + .iter() + .any(|event| event.event_type.trim().is_empty()) + { + return Err(invalid("claim history event type must be non-empty")); + } + claim.task.append_history_page(page); + } + Ok(()) + } + + /// Renew the exact selected workflow-task lease using worker protocol 1.20. + /// + /// The reply must acknowledge the actual task, owner and attempt. Renewal + /// neither commits cancellation delivery nor extends its cleanup deadline. + pub async fn heartbeat_workflow_task( + &self, + task: &WorkflowTask, + original: Option<&CancellationRequest>, + ) -> Result { + let (owner, _) = cancellation_claim(task)?; + if let Some(original) = original { + original.validate_observation()?; + } + tokio::time::timeout(CONTROL_BUDGET, async { + let response: Value = self.request_json( + reqwest::Method::POST, + &format!("/worker/workflow-tasks/{}/heartbeat", percent_encode_path_segment(&task.task_id)), + RequestProtocol::Worker("1.20"), + Some(&json!({"lease_owner":owner,"workflow_task_attempt":task.workflow_task_attempt})), + ).await?; + if response["task_id"].as_str() != Some(task.task_id.as_str()) + || response["workflow_task_attempt"].as_u64() != Some(task.workflow_task_attempt) + || response["lease_owner"].as_str() != Some(owner) + || response["renewed"].as_bool() != Some(true) + || response.get("reason") != Some(&Value::Null) + || response["task_status"].as_str() != Some("leased") + { + return Err(invalid("workflow heartbeat did not acknowledge the exact leased claim")); + } + let expires = text(&response, "lease_expires_at")?; + DateTime::parse_from_rfc3339(expires).map_err(|_| invalid("workflow heartbeat lease expiry must include a timezone"))?; + let run_status = text(&response, "run_status")?; + if !matches!(run_status, "pending" | "running" | "waiting") { + return Err(invalid("workflow heartbeat did not acknowledge an active run")); + } + let cancellation_request = match response.get("cancellation_request") { + None | Some(Value::Null) if original.is_none() => None, + None | Some(Value::Null) => return Err(invalid("workflow heartbeat omitted its original pending request")), + Some(observation) => { + let current = CancellationRequest::from_observation(observation)?; + Some(match original { + Some(original) => original.preserve_observation(¤t)?, + None => current, + }) + }, + }; + Ok(WorkflowTaskHeartbeat { + task_id:task.task_id.clone(), workflow_task_attempt:task.workflow_task_attempt, + lease_owner:owner.to_owned(), lease_expires_at:expires.to_owned(), + run_status:run_status.to_owned(), cancellation_request, + }) + }).await.map_err(|_| Error::Timeout)? + } + + /// Commit cooperative delivery on the exact selected workflow-task claim. + /// + /// This explicit worker-protocol 1.20 operation renews no lease and advertises + /// no worker capability. The Server requires a previously admitted capable + /// claim. Reload canonical history before exposing cancellation to workflow + /// code, including when an acknowledgment is lost. A cancellation wait explicitly + /// releases the claim and must return the worker to polling. + pub async fn deliver_workflow_cancellation( + &self, + task: &WorkflowTask, + delivery: &CancellationDelivery, + ) -> Result { + let (owner, run_id) = cancellation_claim(task)?; + let body = json!({ + "lease_owner":owner, "workflow_task_attempt":task.workflow_task_attempt, + "request_id":delivery.request_id, "sequence":delivery.sequence, + "call_kind":delivery.call_kind, "sequence_span":delivery.sequence_span, + "operation_sequence":delivery.operation_sequence, + "operation_sequence_span":delivery.operation_sequence_span + }); + let mut payload = body.clone(); + payload["workflow_command_id"] = json!(delivery.request_id); + if CancellationDelivery::from_payload(&payload)? != *delivery { + return Err(invalid( + "delivery must name a valid authored operation range", + )); + } + tokio::time::timeout(CONTROL_BUDGET, async { + let path = format!( + "/worker/workflow-tasks/{}/deliver-cancellation", + percent_encode_path_segment(&task.task_id) + ); + let response: Value = self + .request_json( + reqwest::Method::POST, + &path, + RequestProtocol::Worker("1.20"), + Some(&body), + ) + .await?; + if response["delivered"].as_bool() == Some(false) + && match response["reason"].as_str() { + Some("cancellation_waiting_for_child") => matches!( + delivery.call_kind, + CancellationCallKind::Child + | CancellationCallKind::Parallel + | CancellationCallKind::SelectionHandle + ), + Some("cancellation_waiting_for_activity") => matches!( + delivery.call_kind, + CancellationCallKind::Activity + | CancellationCallKind::LocalActivity + | CancellationCallKind::Parallel + | CancellationCallKind::SelectionHandle + ), + _ => false, + } + && response["claim_released"].as_bool() == Some(true) + && response["task_id"].as_str() == Some(task.task_id.as_str()) + && response["workflow_run_id"].as_str() == Some(run_id) + && [ + "request_id", + "sequence", + "call_kind", + "sequence_span", + "operation_sequence", + "operation_sequence_span", + ] + .iter() + .all(|field| response.get(*field).is_none_or(Value::is_null)) + { + return Ok(CancellationDeliveryReply::ClaimReleased { + task_id: task.task_id.clone(), + run_id: run_id.to_owned(), + }); + } + if response["delivered"].as_bool() != Some(true) + || response["task_id"].as_str() != Some(task.task_id.as_str()) + || response["workflow_run_id"].as_str() != Some(run_id) + || response.get("reason") != Some(&Value::Null) + || [ + "sequence_span", + "operation_sequence", + "operation_sequence_span", + ] + .iter() + .any(|field| response.get(*field).is_none()) + { + return Err(invalid( + "delivery acknowledgment does not match the selected task/run", + )); + } + let mut recorded = response.clone(); + recorded["workflow_command_id"] = response["request_id"].clone(); + if CancellationDelivery::from_payload(&recorded)? != *delivery { + return Err(invalid( + "delivery acknowledgment changed its authored operation range", + )); + } + Ok(CancellationDeliveryReply::Delivered( + CancellationDeliveryReceipt { + task_id: task.task_id.clone(), + run_id: run_id.to_owned(), + delivery: delivery.clone(), + }, + )) + }) + .await + .map_err(|_| Error::Timeout)? + } + + /// Reload canonical history with the Server-issued token and exact claim. + /// + /// The refresh has one five-second budget, at most 128 pages, and the existing + /// SDK page-size limit. Invalid, repeated or non-progressing tokens and pages + /// fail closed. This returns fresh history without modifying the caller's + /// previous task snapshot or its original request identity. + pub async fn refresh_workflow_cancellation_history( + &self, + task: &WorkflowTask, + observation: &CancellationRequest, + ) -> Result> { + let (owner, run_id) = cancellation_claim(task)?; + observation.validate_observation()?; + let first_token = observation + .history_refresh_page_token + .clone() + .expect("validated history token"); + tokio::time::timeout(CONTROL_BUDGET, async { + let path = format!("/worker/workflow-tasks/{}/history", percent_encode_path_segment(&task.task_id)); + let mut token = Some(first_token); + let mut seen = BTreeSet::new(); + let mut history = Vec::new(); + while let Some(current) = token.take() { + if seen.len() >= MAX_REFRESH_PAGES || !seen.insert(current.clone()) { + return Err(invalid("canonical history refresh exceeded its page bound or repeated a token")); + } + let body = json!({ + "lease_owner":owner, "workflow_task_attempt":task.workflow_task_attempt, + "next_history_page_token":current, "history_page_size":WORKFLOW_HISTORY_PAGE_SIZE, + }); + let page: Value = self.request_json( + reqwest::Method::POST, &path, RequestProtocol::Worker("1.20"), Some(&body), + ).await?; + if page["task_id"].as_str() != Some(task.task_id.as_str()) + || page["workflow_task_attempt"].as_u64() != Some(task.workflow_task_attempt) + { + return Err(invalid("canonical history page changed the selected task/attempt")); + } + let events = page["history_events"].as_array().ok_or_else(|| invalid("canonical history page must contain an event array"))?; + if events.len() > WORKFLOW_HISTORY_PAGE_SIZE as usize { + return Err(invalid("canonical history page exceeds its requested event limit")); + } + token = match page.get("next_history_page_token") { + Some(Value::Null) => None, + Some(Value::String(next)) if !next.is_empty() && !events.is_empty() => Some(next.clone()), + _ => return Err(invalid("canonical history page token or progress is invalid")), + }; + for event in events { + let event: HistoryEvent = serde_json::from_value(event.clone()).map_err(|_| invalid("canonical history page contains a malformed event"))?; + if event.event_type.trim().is_empty() { + return Err(invalid("canonical history event type must be non-empty")); + } + history.push(event); + } + } + let canonical = CancellationHistory::from_events(&history, run_id, Some(observation))?; + if canonical.request_index >= history.len() { + return Err(invalid("canonical refresh omitted the original request event")); + } + Ok(history) + }).await.map_err(|_| Error::Timeout)? + } + + /// Request bounded workflow-authored cleanup on a capable runtime. + /// + /// This is separate from terminal cancellation. Discovery must advertise + /// cooperative support and compatible protocol 1.20. The Server also + /// refuses an active workflow claim that cannot deliver cooperation. + /// Repeated requests retain its original request identity and deadline. + pub async fn request_workflow_cancellation( + &self, + workflow_id: &str, + options: CooperativeCancellationOptions, + ) -> Result { + self.request_workflow_cancellation_target(workflow_id, None, options) + .await + } + + /// Request cleanup only if the selected run remains current. + pub async fn request_workflow_run_cancellation( + &self, + workflow_id: &str, + run_id: &str, + options: CooperativeCancellationOptions, + ) -> Result { + self.request_workflow_cancellation_target(workflow_id, Some(run_id), options) + .await + } + + async fn request_workflow_cancellation_target( + &self, + workflow_id: &str, + run_id: Option<&str>, + options: CooperativeCancellationOptions, + ) -> Result { + if workflow_id.trim().is_empty() || run_id.is_some_and(|id| id.trim().is_empty()) { + return Err(invalid( + "workflow and selected run identities must be non-empty", + )); + } + if options + .cleanup_timeout_seconds + .is_some_and(|seconds| !(1..=3600).contains(&seconds)) + || options + .reason + .as_ref() + .is_some_and(|reason| reason.chars().count() > 1000) + { + return Err(invalid( + "cleanup timeout must be 1..3600 seconds and reason at most 1000 characters", + )); + } + + // Bound discovery, mutation and any storage retry by one total budget. + tokio::time::timeout(CONTROL_BUDGET, async { + let info: Value = self + .request_json( + reqwest::Method::GET, + "/cluster/info", + RequestProtocol::ControlPlane, + Option::<&Value>::None, + ) + .await?; + require_discovery(&info)?; + let mut path = format!("/workflows/{}", percent_encode_path_segment(workflow_id)); + if let Some(run_id) = run_id { + path.push_str(&format!("/runs/{}", percent_encode_path_segment(run_id))); + } + path.push_str("/request-cancellation"); + let response: Value = self + .request_json( + reqwest::Method::POST, + &path, + RequestProtocol::ControlPlane, + Some(&options), + ) + .await?; + acknowledgment(&response, workflow_id, run_id) + }) + .await + .map_err(|_| Error::Timeout)? + } +} + +impl Worker { + pub(super) async fn poll_cooperative_activity_once(&self) -> Result { + let poll_request_id = unique_request_id("rust-activity-poll"); + let response = self + .retry_worker_operation(|| { + self.client.poll_activity_task_response_with_request_id( + &self.worker_id, + &self.task_queue, + self.poll_timeout, + &poll_request_id, + 0, + ) + }) + .await; + let Some(response) = self.settle_worker_poll_response(response).await? else { + return Ok(ManagedPollOutcome::Idle); + }; + if response.outcome().should_stop() { + return Ok(ManagedPollOutcome::Stop); + } + let Some(task) = response.task else { + return Ok(ManagedPollOutcome::Idle); + }; + let guard = ActivityClaimGuard::new(&self.client, &task, &self.worker_id)?; + let _abandon_on_drop = AbandonActivityOnDrop(guard.clone()); + if guard.observe().await.is_err() { + return Ok(ManagedPollOutcome::Handled); + } + let mut callback_started = false; + let result = { + let invocation = + self.execute_cooperative_activity_task(&task, &guard, &mut callback_started); + tokio::pin!(invocation); + loop { + tokio::select! { + biased; + _ = guard.wait_for_shutdown() => { + guard.abandon(); + break None; + } + result = &mut invocation => break Some(result), + _ = tokio::time::sleep(Duration::from_secs(1)) => { + if guard.observe().await.is_err() { + break None; + } + } + } + } + }; + // The invocation and its callback future are dropped before any receipt. + let Some(result) = result else { + if callback_started { + guard.acknowledge_stopped().await?; + } + return Ok(ManagedPollOutcome::Handled); + }; + // Both genuine application failures and successful results need current + // authority before result upload or completion/failure publication. + if matches!(result, Err(Error::ActivityExecutionAbandoned(_))) + || guard.observe().await.is_err() + { + if callback_started { + guard.acknowledge_stopped().await?; + } + return Ok(ManagedPollOutcome::Handled); + } + let settlement = match result { + Ok(value) => { + self.client + .complete_activity_task( + &guard.task_id, + &guard.attempt_id, + &guard.owner, + value, + &task.payload_codec, + ) + .await + } + Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error), + Err(error) => { + self.client + .fail_activity_task( + &guard.task_id, + &guard.attempt_id, + &guard.owner, + error.to_string(), + false, + ) + .await + } + }; + if let Err(error) = settlement { + if !activity_task_rejection_is_final(&error) { + return Err(error); + } + } + Ok(ManagedPollOutcome::Handled) + } + + async fn execute_cooperative_activity_task( + &self, + task: &ActivityTask, + guard: &ActivityClaimGuard, + callback_started: &mut bool, + ) -> Result { + validate_activity_task_payloads(task)?; + let handler = self + .activities + .get(&task.activity_type) + .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?; + let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?; + let context = ActivityContext { + client: self.client.clone(), + task_id: guard.task_id.clone(), + activity_attempt_id: guard.attempt_id.clone(), + lease_owner: guard.owner.clone(), + activity_type: task.activity_type.clone(), + attempt_number: task.attempt_number, + task_queue: self.task_queue.clone(), + worker_id: self.worker_id.clone(), + claim_guard: Some(guard.clone()), + }; + guard.boundary()?; + *callback_started = true; + handler(context, args).await + } + + pub(super) async fn poll_cooperative_workflow_once(&self) -> Result { + let poll_request_id = unique_request_id("rust-workflow-poll"); + let response = self + .retry_worker_operation(|| { + self.client.poll_cooperative_workflow_task_with_request_id( + &self.worker_id, + &self.task_queue, + self.poll_timeout, + &poll_request_id, + 0, + ) + }) + .await; + let Some(response) = self.settle_worker_poll_response(response).await? else { + return Ok(ManagedPollOutcome::Idle); + }; + if response.outcome.should_stop() { + return Ok(ManagedPollOutcome::Stop); + } + let memo_updates_supported = + runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref()); + let Some(claim) = response.task else { + return Ok(ManagedPollOutcome::Idle); + }; + // An uncertain observation, delivery or canonical refresh cannot become + // an application failure or be published as a workflow-task decision. + let Some(decision) = self.replay_cooperative_workflow_claim(&claim).await? else { + return Ok(ManagedPollOutcome::Handled); + }; + let (owner, _) = cancellation_claim(&claim.task)?; + self.settle_workflow_task_decision( + &claim.task.task_id, + owner, + claim.task.workflow_task_attempt, + claim.task.run_id.as_deref(), + Ok(decision), + memo_updates_supported, + ) + .await + } + + async fn replay_cooperative_workflow_claim( + &self, + claim: &CooperativeWorkflowTask, + ) -> Result> { + let (owner, run_id) = cancellation_claim(&claim.task)?; + if owner != self.worker_id { + return Err(invalid( + "cooperative replay requires this worker's actual claim", + )); + } + let mut task = claim.task.clone(); + let Some(observation) = claim.cancellation_request.as_ref() else { + let canonical = CancellationHistory::from_events(&task.history_events, run_id, None)?; + if canonical.request.is_some() { + return Err(invalid( + "cooperative replay omitted its original pending observation", + )); + } + for _ in 0..1000 { + let mut decision = self.execute_workflow_task_decision(task.clone())?; + if let Some(intent) = decision.cancellation_scope_delivery.as_ref() { + if !decision.commands.is_empty() { + decision.cancellation_scope_delivery = None; + return Ok(Some(decision)); + } + let mut budget = CancellationScopeDeliveryBudget::new(); + budget.restrict(intent.context.deadline())?; + let prepared = self + .client + .prepare_cancellation_scope_on_claim( + &task, + &intent.context, + &intent.boundary, + &budget, + ) + .await?; + budget.restrict(prepared.authority_deadline())?; + let delivered = self + .client + .deliver_cancellation_scope_on_claim(&task, &prepared, &budget) + .await?; + task.history_events = delivered.history().to_vec(); + task.total_history_events = None; + task.history_size_bytes = None; + continue; + } + let Some(opening) = decision.cancellation_scope_opening.as_ref() else { + return Ok(Some(decision)); + }; + if !decision.commands.is_empty() { + decision.cancellation_scope_opening = None; + return Ok(Some(decision)); + } + let receipt = self + .client + .open_cancellation_scope_on_claim( + &task, + opening.sequence, + &opening.parent_scope_id, + opening.shield_parent, + ) + .await?; + task.history_events = receipt.history().to_vec(); + task.total_history_events = None; + task.history_size_bytes = None; + } + return Err(invalid( + "workflow exceeded the canonical scope opening replay limit", + )); + }; + task.history_events = self + .client + .refresh_workflow_cancellation_history(&claim.task, observation) + .await?; + for _ in 0..1000 { + // Count the actual refreshed snapshot. Its byte budget was not + // remeasured, so do not expose the old snapshot's size as current. + task.total_history_events = None; + task.history_size_bytes = None; + let mut decision = self.execute_workflow_task_decision_with_cancellation( + task.clone(), + Some(observation), + )?; + if let Some(intent) = decision.cancellation_scope_delivery.as_ref() { + if !decision.commands.is_empty() { + decision.cancellation_scope_delivery = None; + return Ok(Some(decision)); + } + let mut budget = CancellationScopeDeliveryBudget::new(); + budget.restrict(intent.context.deadline())?; + let prepared = self + .client + .prepare_cancellation_scope_on_claim( + &task, + &intent.context, + &intent.boundary, + &budget, + ) + .await?; + budget.restrict(prepared.authority_deadline())?; + let delivered = self + .client + .deliver_cancellation_scope_on_claim(&task, &prepared, &budget) + .await?; + task.history_events = delivered.history().to_vec(); + continue; + } + let Some(intent) = decision.cancellation_delivery.as_ref() else { + return Ok(Some(decision)); + }; + if !decision.commands.is_empty() { + // Native permits delivery only at/before its next durable call. + // Commit earlier commands first. Completion releases this claim + // and its successor replays their durable results before delivery. + decision.cancellation_delivery = None; + return Ok(Some(decision)); + } + let delivery_error = match self + .client + .deliver_workflow_cancellation(&claim.task, intent) + .await + { + Ok(CancellationDeliveryReply::ClaimReleased { .. }) => return Ok(None), + Ok(CancellationDeliveryReply::Delivered(_)) => None, + Err(error) => Some(error), + }; + task.history_events = self + .client + .refresh_workflow_cancellation_history(&claim.task, observation) + .await?; + let canonical = + CancellationHistory::from_events(&task.history_events, run_id, Some(observation))?; + if canonical.delivery.as_ref() != Some(intent) { + return Err(delivery_error.unwrap_or_else(|| { + invalid( + "canonical history did not prove the exact intended cancellation delivery", + ) + })); + } + } + Err(invalid( + "workflow replay did not converge on its canonical cancellation delivery", + )) + } +} + +#[derive(Clone, Debug)] +pub(super) struct ActivityClaimGuard { + client: Client, + task_id: String, + attempt_id: String, + owner: String, + active: Arc, + cancellation_receipt: Arc>>, + stop: Option>, +} + +struct AbandonActivityOnDrop(ActivityClaimGuard); + +impl Drop for AbandonActivityOnDrop { + fn drop(&mut self) { + self.0.abandon(); + } +} + +impl ActivityClaimGuard { + fn new(client: &Client, task: &ActivityTask, worker_id: &str) -> Result { + let owner = task.lease_owner.as_deref().unwrap_or_default(); + let attempt = task + .activity_attempt_id + .as_deref() + .or(task.attempt_id.as_deref()) + .unwrap_or_default(); + if task.task_id.trim().is_empty() + || attempt.trim().is_empty() + || owner.trim().is_empty() + || owner != worker_id + || matches!((&task.activity_attempt_id, &task.attempt_id), (Some(left), Some(right)) if left != right) + { + return Err(invalid( + "activity execution requires the worker's actual immutable claim", + )); + } + Ok(Self { + client: client.clone(), + task_id: task.task_id.clone(), + attempt_id: attempt.to_owned(), + owner: owner.to_owned(), + active: Arc::new(AtomicBool::new(true)), + cancellation_receipt: Arc::new(Mutex::new(None)), + stop: client + .worker_storage_admission + .as_ref() + .map(|admission| Arc::clone(&admission.stop)), + }) + } + + fn abandon(&self) { + self.active.store(false, Ordering::SeqCst); + } + + /// Caller has already dropped the callback future. Never restores authority. + async fn acknowledge_stopped(&self) -> Result<()> { + self.abandon(); + let receipt = self + .cancellation_receipt + .lock() + .map_err(|_| invalid("activity cancellation receipt lock was poisoned"))? + .clone(); + if let Some(receipt) = receipt { + self.client + .acknowledge_activity_cancellation( + &self.task_id, + &self.attempt_id, + &self.owner, + text(&receipt, "request_id")?, + ) + .await?; + } + Ok(()) + } + + fn retain_cancellation_receipt(&self, value: &Value) -> Result<()> { + if value["can_continue"].as_bool() != Some(false) + || value["cancel_requested"].as_bool() != Some(true) + { + return Ok(()); + } + let Some(receipt) = value + .get("cancellation_acknowledgement") + .filter(|receipt| receipt.is_object()) + else { + return Ok(()); + }; + if !matches!( + receipt["callback_state"].as_str(), + Some("unknown" | "stopped") + ) || [ + "request_id", + "root_request_id", + "cleanup_deadline_at", + "cancellation_history_event_id", + ] + .iter() + .any(|field| text(receipt, field).is_err()) + || DateTime::parse_from_rfc3339(text(receipt, "cleanup_deadline_at")?).is_err() + { + return Ok(()); + } + let mut retained = self + .cancellation_receipt + .lock() + .map_err(|_| invalid("activity cancellation receipt lock was poisoned"))?; + if let Some(original) = retained.as_ref() { + if [ + "request_id", + "root_request_id", + "cleanup_deadline_at", + "cancellation_history_event_id", + ] + .iter() + .any(|field| original[*field] != receipt[*field]) + { + return Err(invalid( + "activity cancellation observation changed its original identity or deadline", + )); + } + } else { + *retained = Some(receipt.clone()); + } + Ok(()) + } + + fn boundary(&self) -> Result<()> { + if !self.active.load(Ordering::SeqCst) + || self + .stop + .as_ref() + .is_some_and(|stop| stop.load(Ordering::SeqCst)) + { + self.abandon(); + return Err(Error::ActivityExecutionAbandoned( + "callback completed, was abandoned, or its worker is stopping".into(), + )); + } + Ok(()) + } + + async fn wait_for_shutdown(&self) { + if let Some(stop) = &self.stop { + wait_for_worker_stop(stop).await; + } else { + std::future::pending::<()>().await; + } + } + + async fn observe(&self) -> Result<()> { + self.boundary()?; + let result = self + .client + .activity_task_status(&self.task_id, &self.attempt_id, &self.owner) + .await + .and_then(|value| { + self.retain_cancellation_receipt(&value)?; + if value["can_continue"].as_bool() != Some(true) + || value["cancel_requested"].as_bool() != Some(false) + || value.get("reason") != Some(&Value::Null) + || value["task_status"].as_str() != Some("leased") + || value["attempt_status"].as_str() != Some("running") + || value["activity_status"].as_str() != Some("running") + { + return Err(invalid("activity observation refused continuation")); + } + let mut bounds = vec![text(&value, "lease_expires_at")?]; + if let Some(deadlines) = value.get("deadlines").filter(|v| !v.is_null()) { + if !deadlines.is_object() { + return Err(invalid("activity execution deadlines must be an object")); + } + for kind in ["heartbeat", "start_to_close", "schedule_to_close"] { + if let Some(deadline) = deadlines.get(kind).filter(|v| !v.is_null()) { + bounds.push( + deadline + .as_str() + .ok_or_else(|| invalid("invalid activity deadline"))?, + ); + } + } + } + if let Some(session) = value.get("worker_session").filter(|v| !v.is_null()) { + if session["status"].as_str() != Some("active") + || session["lease_owner"].as_str() != Some(self.owner.as_str()) + { + return Err(invalid( + "activity no longer owns its required worker session", + )); + } + bounds.push(text(session, "lease_expires_at")?); + bounds.push(text(session, "ttl_expires_at")?); + } + for bound in bounds { + let deadline = DateTime::parse_from_rfc3339(bound) + .map_err(|_| invalid("activity deadline must include a timezone"))?; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| invalid("activity observation clock precedes the epoch"))?; + if !u128::try_from(deadline.timestamp_millis()) + .is_ok_and(|millis| millis > now.as_millis()) + { + return Err(invalid("activity ownership or execution deadline elapsed")); + } + } + self.boundary() + }); + result.map_err(|error| { + self.abandon(); + Error::ActivityExecutionAbandoned(error.to_string()) + }) + } + + pub(super) async fn heartbeat( + &self, + context: &ActivityContext, + details: T, + ) -> Result { + if context.task_id != self.task_id + || context.activity_attempt_id != self.attempt_id + || context.lease_owner != self.owner + || context.worker_id != self.owner + { + self.abandon(); + return Err(Error::ActivityExecutionAbandoned( + "activity context changed its original claim".into(), + )); + } + self.observe().await?; + let result = tokio::time::timeout(CONTROL_BUDGET, async { + let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?; + self.boundary()?; + let value: Value = self.client.request_json( + reqwest::Method::POST, + &format!("/worker/activity-tasks/{}/heartbeat", percent_encode_path_segment(&self.task_id)), + RequestProtocol::Worker("1.20"), + Some(&json!({"activity_attempt_id":self.attempt_id,"lease_owner":self.owner,"details":details})), + ).await?; + if value["task_id"].as_str() != Some(self.task_id.as_str()) + || value["activity_attempt_id"].as_str() != Some(self.attempt_id.as_str()) + || value["lease_owner"].as_str() != Some(self.owner.as_str()) + || value["can_continue"].as_bool() != Some(true) + || value["cancel_requested"].as_bool() != Some(false) + || value["heartbeat_recorded"].as_bool() != Some(true) + { + return Err(invalid("activity heartbeat lost its original claim")); + } + serde_json::from_value(value).map_err(Error::from) + }).await.map_err(|_| Error::Timeout).and_then(|result| result); + let response = result.map_err(|error| { + self.abandon(); + Error::ActivityExecutionAbandoned(error.to_string()) + })?; + self.observe().await?; + Ok(response) + } +} + +pub(super) fn cancellation_claim(task: &WorkflowTask) -> Result<(&str, &str)> { + let owner = task + .lease_owner + .as_deref() + .filter(|owner| !owner.trim().is_empty()); + let run = task.run_id.as_deref().filter(|run| !run.trim().is_empty()); + if task.task_id.trim().is_empty() + || task.workflow_task_attempt == 0 + || task.workflow_task_attempt > MAX_SEQUENCE + { + return Err(invalid( + "cancellation transport requires a valid task and attempt", + )); + } + Ok(( + owner.ok_or_else(|| invalid("cancellation transport requires the actual lease owner"))?, + run.ok_or_else(|| invalid("cancellation transport requires the selected durable run"))?, + )) +} + +impl WorkflowHandle { + /// Request bounded cleanup for whichever run is current. + pub async fn request_cancellation( + &self, + options: CooperativeCancellationOptions, + ) -> Result { + self.client + .request_workflow_cancellation(&self.workflow_id, options) + .await + } + + /// Request bounded cleanup only while this handle's selected run is current. + pub async fn request_selected_run_cancellation( + &self, + options: CooperativeCancellationOptions, + ) -> Result { + let run_id = self + .run_id + .as_deref() + .ok_or_else(|| invalid("selected run_id is required"))?; + self.client + .request_workflow_run_cancellation(&self.workflow_id, run_id, options) + .await + } +} diff --git a/src/lib.rs b/src/lib.rs index 09450f2..387fef7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,8 +1,32 @@ #![doc = include_str!("../README.md")] +mod cancellation_context; +mod cancellation_replay_clock; +mod cancellation_scope; +mod cancellation_scope_history; +mod cancellation_scope_replay; +mod cooperative_cancellation; mod runtime_payloads; mod runtime_uploads; +pub use cancellation_context::{ + CancellationContext, CancellationLineage, ScopedCancellationContext, ScopedCancellationLineage, +}; +pub use cancellation_scope::CancellationScopeOpenReceipt; +#[doc(hidden)] +pub use cancellation_scope_history::{ + CancellationScopeDeliveryBudget, CancellationScopeDeliveryReceipt, +}; +#[doc(hidden)] +pub use cancellation_scope_replay::CancellationScopeRequested; + +pub use cooperative_cancellation::{ + CancellationCallKind, CancellationDelivery, CancellationDeliveryReceipt, + CancellationDeliveryReply, CancellationHistory, CancellationRequest, CancellationShield, + CooperativeCancellationOptions, CooperativeCancellationRequested, CooperativeWorkflowTask, + CooperativeWorkflowTaskPoll, WorkflowCancellationRequest, WorkflowTaskHeartbeat, +}; + use std::{ any::{type_name, Any, TypeId}, collections::{BTreeMap, BTreeSet, HashMap}, @@ -20,7 +44,9 @@ use std::{ use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema}; use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; use chrono::DateTime; -use futures_util::{future::OptionFuture, task::noop_waker_ref}; +use futures_util::{ + future::OptionFuture, stream::FuturesUnordered, task::noop_waker_ref, StreamExt, +}; use serde::{ de::DeserializeOwned, ser::{SerializeMap, SerializeSeq}, @@ -185,6 +211,10 @@ pub enum Error { #[error(transparent)] WorkflowCancellationRequested(WorkflowCancellationRequested), #[error(transparent)] + CooperativeCancellationRequested(CooperativeCancellationRequested), + #[error(transparent)] + CancellationScopeRequested(CancellationScopeRequested), + #[error(transparent)] WorkflowCommandRejected(WorkflowCommandRejection), #[error(transparent)] WorkflowFailed(WorkflowTerminalOutcome), @@ -252,6 +282,14 @@ pub enum Error { "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support" )] WorkflowMemoUpdatesUnavailable, + #[error("cooperative cancellation is unavailable: {0}")] + CooperativeCancellationUnavailable(String), + #[error("cancellation_scope_execution_not_supported: Rust worker cannot replay scoped cancellation history")] + CancellationScopeExecutionUnavailable, + #[error("invalid cooperative cancellation: {0}")] + InvalidCooperativeCancellation(String), + #[error("activity execution no longer owns its claim: {0}")] + ActivityExecutionAbandoned(String), #[error(transparent)] InvalidActivityOptions(ActivityOptionsError), #[error(transparent)] @@ -967,6 +1005,7 @@ pub enum ActivityOptionsErrorKind { BackoffOverflow, EmptyNonRetryableErrorType, TimeoutNotPositive, + MissingTotalTimeout, TimeoutOverflow, TimeoutOrder, } @@ -1254,7 +1293,10 @@ pub struct SagaCompensationFailure { pub enum ParentClosePolicy { #[default] Abandon, + /// Legacy terminal cancellation without cooperative cleanup. RequestCancel, + /// Request cooperative cleanup with the original lineage and deadline. + RequestCancellation, Terminate, } @@ -1263,11 +1305,35 @@ impl ParentClosePolicy { match self { Self::Abandon => "abandon", Self::RequestCancel => "request_cancel", + Self::RequestCancellation => "request_cancellation", Self::Terminate => "terminate", } } } +/// Cancellation at an awaiting operation. Activities default to Try, children to Abandon. +#[non_exhaustive] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum CancellationPolicy { + /// Request cancellation and continue without waiting. Historical Activity default. + TryCancel, + /// Wait for recorded child termination or the original Activity attempt's stop receipt. + WaitCancellationCompleted, + /// Leave work independent. Historical child default. Remote Activities require a total timeout. + #[default] + Abandon, +} + +impl CancellationPolicy { + fn as_str(self) -> &'static str { + match self { + Self::TryCancel => "try_cancel", + Self::WaitCancellationCompleted => "wait_cancellation_completed", + Self::Abandon => "abandon", + } + } +} + /// Durable retry policy for one child workflow invocation. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ChildWorkflowRetryPolicy { @@ -1283,6 +1349,7 @@ pub struct ChildWorkflowRetryPolicy { pub struct ChildWorkflowOptions { pub task_queue: String, pub parent_close_policy: ParentClosePolicy, + pub cancellation_policy: CancellationPolicy, pub retry_policy: Option, pub execution_timeout_seconds: Option, pub run_timeout_seconds: Option, @@ -1293,6 +1360,7 @@ impl ChildWorkflowOptions { Self { task_queue: task_queue.into(), parent_close_policy: ParentClosePolicy::Abandon, + cancellation_policy: CancellationPolicy::Abandon, retry_policy: None, execution_timeout_seconds: None, run_timeout_seconds: None, @@ -1304,6 +1372,11 @@ impl ChildWorkflowOptions { self } + pub fn cancellation_policy(mut self, policy: CancellationPolicy) -> Self { + self.cancellation_policy = policy; + self + } + pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self { self.retry_policy = Some(policy); self @@ -1397,6 +1470,7 @@ pub struct ActivityOptions { pub schedule_to_start_timeout: Option, pub schedule_to_close_timeout: Option, pub heartbeat_timeout: Option, + pub cancellation_policy: Option, } impl ActivityOptions { @@ -1434,7 +1508,21 @@ impl ActivityOptions { self } + pub fn cancellation_policy(mut self, policy: CancellationPolicy) -> Self { + self.cancellation_policy = Some(policy); + self + } + fn validate(&self) -> std::result::Result { + if self.cancellation_policy == Some(CancellationPolicy::Abandon) + && self.schedule_to_close_timeout.is_none() + { + return Err(ActivityOptionsError::new( + ActivityOptionsErrorKind::MissingTotalTimeout, + Some("schedule_to_close_timeout"), + "remote Activity Abandon requires a finite positive total timeout", + )); + } if self .task_queue .as_deref() @@ -1501,6 +1589,7 @@ impl ActivityOptions { self.schedule_to_close_timeout, )?, heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?, + cancellation_policy: self.cancellation_policy, }) } } @@ -1590,6 +1679,7 @@ struct ValidatedActivityOptions { schedule_to_start_timeout: Option, schedule_to_close_timeout: Option, heartbeat_timeout: Option, + cancellation_policy: Option, } fn validate_timeout_order( @@ -2636,6 +2726,7 @@ pub struct Client { namespace: String, max_external_payload_bytes: usize, worker_storage_admission: Option, + cooperative_worker_protocol: bool, runtime_upload_policy: Arc>, } @@ -3554,13 +3645,51 @@ impl Client { body["workflow_definition_fingerprints"] = json!(fingerprints); } - self.request_json( - reqwest::Method::POST, - "/worker/register", - RequestProtocol::Worker(WORKER_PROTOCOL_VERSION), - Some(&body), - ) - .await + let response: Value = self + .request_json( + reqwest::Method::POST, + "/worker/register", + RequestProtocol::Worker(WORKER_PROTOCOL_VERSION), + Some(&body), + ) + .await?; + if self.cooperative_worker_protocol { + let compatible = response["registered"].as_bool() == Some(true) + && response["worker_id"].as_str() == Some(worker_id) + && response["namespace"].as_str() == Some(self.namespace.as_str()) + && response["task_queue"].as_str() == Some(task_queue) + && response["protocol_version"] + .as_str() + .is_some_and(cooperative_cancellation::supports_protocol) + && response["server_capabilities"]["cooperative_cancellation"].as_bool() + == Some(true) + && response["capabilities"].as_array().is_some_and(|accepted| { + capabilities.iter().all(|capability| { + accepted + .iter() + .any(|value| value.as_str() == Some(capability.as_str())) + }) + }); + if !compatible { + let error = Error::CooperativeCancellationUnavailable( + "registration must acknowledge this namespace, queue, worker, capabilities and compatible runtime protocol 1.20".into(), + ); + if response["registered"].as_bool() == Some(true) + && response["worker_id"].as_str() == Some(worker_id) + { + if let Err(deregistration) = + self.deregister_worker_registration(worker_id).await + { + return Err(Error::WorkerShutdown { + primary: Box::new(error), + deregistration: Box::new(deregistration), + }); + } + } + return Err(error); + } + } + Ok(serde_json::from_value(response)?) } /// Gracefully remove one worker's registration through the worker plane. @@ -4136,6 +4265,18 @@ impl Client { body: Option<&B>, timeout: Duration, ) -> Result { + let protocol = match protocol { + RequestProtocol::Worker(version) + if self.cooperative_worker_protocol + && version + .strip_prefix("1.") + .and_then(|minor| minor.parse::().ok()) + .is_some_and(|minor| minor < 20) => + { + RequestProtocol::Worker("1.20") + } + protocol => protocol, + }; let auth_token = self.auth_token(protocol)?; let mut request = self .http @@ -4885,6 +5026,7 @@ impl ClientBuilder { namespace: self.namespace, max_external_payload_bytes: self.max_external_payload_bytes, worker_storage_admission: None, + cooperative_worker_protocol: false, runtime_upload_policy: Arc::new(Mutex::new([None, None])), }) } @@ -6050,6 +6192,9 @@ struct WorkflowTaskDecision { commands: Vec, message_stream_cursors: Vec, message_stream_waits: Vec, + cancellation_delivery: Option, + cancellation_scope_opening: Option, + cancellation_scope_delivery: Option, } impl WorkflowTaskDecision { @@ -6058,6 +6203,9 @@ impl WorkflowTaskDecision { commands, message_stream_cursors: Vec::new(), message_stream_waits: Vec::new(), + cancellation_delivery: None, + cancellation_scope_opening: None, + cancellation_scope_delivery: None, } } } @@ -6152,6 +6300,10 @@ pub struct Worker { heartbeat_interval: Duration, retry_policy: WorkerRetryPolicy, heartbeat_observer: Option, + cooperative_cancellation_enabled: bool, + allow_cancellation_scope_authoring: bool, + allow_cancellation_scope_delivery: bool, + cooperative_registration_confirmed: Arc, } impl Worker { @@ -6170,11 +6322,16 @@ impl Worker { heartbeat_interval: Duration::from_secs(60), retry_policy: WorkerRetryPolicy::default(), heartbeat_observer: None, + cooperative_cancellation_enabled: false, + allow_cancellation_scope_authoring: false, + allow_cancellation_scope_delivery: false, + cooperative_registration_confirmed: Arc::new(AtomicBool::new(false)), } } pub fn worker_id(mut self, worker_id: impl Into) -> Self { self.worker_id = worker_id.into(); + self.cooperative_registration_confirmed = Arc::new(AtomicBool::new(false)); self } @@ -6188,6 +6345,36 @@ impl Worker { self } + /// Opt in to separate cooperative workflow cancellation requests. + /// + /// Registration requires the Server to acknowledge this exact worker, + /// namespace, queue and capabilities with compatible protocol 1.20. Task + /// processing uses canonical delivery and activity ownership fences. + /// Existing terminal cancel and terminate operations remain terminal. + /// The default is disabled, preserving ordinary protocol 1.19 workers. + /// Call [`Worker::register`] before [`Worker::run_once`]. [`Worker::run`] + /// and [`Worker::run_until`] register automatically. + pub fn cooperative_cancellation(mut self, enabled: bool) -> Self { + self.cooperative_cancellation_enabled = enabled; + self.client.cooperative_worker_protocol = enabled; + self.cooperative_registration_confirmed = Arc::new(AtomicBool::new(false)); + self + } + + /// Enable candidate scope authoring without advertising scoped execution. + #[doc(hidden)] + pub fn candidate_cancellation_scope_authoring(mut self, enabled: bool) -> Self { + self.allow_cancellation_scope_authoring = enabled; + self + } + + /// Enable the candidate scoped call consumer with canonical authoring. + #[doc(hidden)] + pub fn candidate_cancellation_scope_delivery(mut self, enabled: bool) -> Self { + self.allow_cancellation_scope_delivery = enabled; + self + } + /// Configure bounded retries for task-poll acquisition and worker heartbeats. pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self { self.retry_policy = policy; @@ -6767,7 +6954,10 @@ impl Worker { ); } - self.client + self.cooperative_registration_confirmed + .store(false, Ordering::SeqCst); + let response = self + .client .register_worker_with_definition_fingerprints( &self.worker_id, &self.task_queue, @@ -6780,6 +6970,8 @@ impl Worker { Some(DURABLE_SELECTION_CAPABILITY.to_string()), Some(MEMO_UPSERTS_CAPABILITY.to_string()), Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()), + self.cooperative_cancellation_enabled + .then(|| "cooperative_cancellation".to_string()), (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()), (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()), worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION) @@ -6801,7 +6993,12 @@ impl Worker { .collect(), ), ) - .await + .await?; + if self.cooperative_cancellation_enabled && response.registered { + self.cooperative_registration_confirmed + .store(true, Ordering::SeqCst); + } + Ok(response) } /// Run until shutdown or a terminal worker error occurs. @@ -6854,6 +7051,8 @@ impl Worker { } let registered_worker_id = registration.worker_id.clone(); let primary = self.run_registered_until(stop, registration).await; + self.cooperative_registration_confirmed + .store(false, Ordering::SeqCst); let deregistration = self .client .deregister_worker_registration(®istered_worker_id) @@ -7019,6 +7218,15 @@ impl Worker { /// Direct callers of [`Client::complete_workflow_task`] continue to receive /// the original [`Error::Http`] status and response body. pub async fn run_once(&self) -> Result { + if self.cooperative_cancellation_enabled + && !self + .cooperative_registration_confirmed + .load(Ordering::SeqCst) + { + return Err(Error::CooperativeCancellationUnavailable( + "register this cooperative worker before polling tasks".into(), + )); + } let worker = self.with_storage_admission(Arc::new(AtomicBool::new(false))); let mut handled = 0; match worker.poll_workflow_once().await? { @@ -7042,6 +7250,9 @@ impl Worker { } async fn poll_workflow_once(&self) -> Result { + if self.cooperative_cancellation_enabled { + return self.poll_cooperative_workflow_once().await; + } let poll_request_id = unique_request_id("rust-workflow-poll"); let response = self .retry_worker_operation(|| { @@ -7074,15 +7285,44 @@ impl Worker { .clone() .unwrap_or_else(|| self.worker_id.clone()); - match self.execute_workflow_task_decision(task) { + self.settle_workflow_task_decision( + &task_id, + &lease_owner, + attempt, + run_id.as_deref(), + self.execute_workflow_task_decision(task), + memo_updates_supported, + ) + .await + } + + async fn settle_workflow_task_decision( + &self, + task_id: &str, + lease_owner: &str, + attempt: u64, + run_id: Option<&str>, + decision: Result, + memo_updates_supported: bool, + ) -> Result { + match decision { + Ok(decision) + if decision.cancellation_delivery.is_some() + || decision.cancellation_scope_delivery.is_some() + || decision.cancellation_scope_opening.is_some() => + { + return Err(Error::CooperativeCancellationUnavailable( + "worker cancellation delivery has not been negotiated".into(), + )); + } Ok(decision) if commands_use_workflow_memo_updates(&decision.commands) && !memo_updates_supported => { self.client .fail_workflow_task( - &task_id, - &lease_owner, + task_id, + lease_owner, attempt, Error::WorkflowMemoUpdatesUnavailable.to_string(), ) @@ -7096,8 +7336,8 @@ impl Worker { // least one executable command. self.client .fail_workflow_task_with_type( - &task_id, - &lease_owner, + task_id, + lease_owner, attempt, WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE, WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE, @@ -7108,8 +7348,8 @@ impl Worker { let completion = self .client .complete_workflow_task_with_message_streams( - &task_id, - &lease_owner, + task_id, + lease_owner, attempt, decision.commands, decision.message_stream_cursors, @@ -7118,18 +7358,16 @@ impl Worker { .await; if let Err(error) = completion { if !workflow_task_completion_is_terminal_timeout( - &error, - &task_id, - attempt, - run_id.as_deref(), + &error, task_id, attempt, run_id, ) { return Err(error); } } } + Err(error @ Error::CancellationScopeExecutionUnavailable) => return Err(error), Err(error) => { self.client - .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string()) + .fail_workflow_task(task_id, lease_owner, attempt, error.to_string()) .await?; } } @@ -7149,6 +7387,9 @@ impl Worker { } async fn poll_activity_once(&self) -> Result { + if self.cooperative_cancellation_enabled { + return self.poll_cooperative_activity_once().await; + } let poll_request_id = unique_request_id("rust-activity-poll"); let response = self .retry_worker_operation(|| { @@ -7219,14 +7460,43 @@ impl Worker { } async fn poll_activities_until_stopped(self, stop: Arc) -> Result<()> { - while !stop.load(Ordering::SeqCst) { - if self.poll_activity_once().await? == ManagedPollOutcome::Stop { - stop.store(true, Ordering::SeqCst); - break; + // Each lane retains its leased response through callback settlement. + // A bounded independent activity must not occupy the cleanup lane too. + // Preserve the existing serial loop for default protocol 1.19 workers. + let lane_count = if self.cooperative_cancellation_enabled { + self.max_concurrent_activity_tasks + } else { + 1 + }; + let mut lanes = FuturesUnordered::new(); + for _ in 0..lane_count { + let worker = self.clone(); + let stop = Arc::clone(&stop); + lanes.push(async move { + while !stop.load(Ordering::SeqCst) { + match worker.poll_activity_once().await { + Ok(ManagedPollOutcome::Stop) => { + stop.store(true, Ordering::SeqCst); + break; + } + Err(error) => { + stop.store(true, Ordering::SeqCst); + return Err(error); + } + _ => {} + } + } + Ok(()) + }); + } + // Join every lane before deregistration, including after one fails. + let mut first_error = None; + while let Some(result) = lanes.next().await { + if let Err(error) = result { + first_error.get_or_insert(error); } } - - Ok(()) + first_error.map_or(Ok(()), Err) } async fn poll_query_once(&self) -> Result { @@ -7515,10 +7785,22 @@ impl Worker { )); let workflow_context = WorkflowContext { state: workflow_state, + cancellation_scope_id: "root".into(), }; let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone()); let mut cx = TaskContext::from_waker(noop_waker_ref()); - match invocation.future.as_mut().poll(&mut cx) { + let outcome = { + let _replay = cancellation_replay_clock::ReplayGuard::enter(&workflow_context) + .map_err(|error| { + QueryTaskExecutionFailure::new( + "query_workflow_state_unavailable", + error.to_string(), + "QueryWorkflowStateUnavailable", + ) + })?; + invocation.future.as_mut().poll(&mut cx) + }; + match outcome { Poll::Ready(Ok(_)) => { workflow_context .ensure_history_consumed() @@ -7592,13 +7874,55 @@ impl Worker { } fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result { - validate_workflow_task_payloads(&task)?; + self.execute_workflow_task_decision_with_cancellation(task, None) + } + fn execute_workflow_task_decision_with_cancellation( + &self, + task: WorkflowTask, + observation: Option<&CancellationRequest>, + ) -> Result { + if (self.allow_cancellation_scope_authoring || self.allow_cancellation_scope_delivery) + && !self.cooperative_cancellation_enabled + { + return Err(Error::CancellationScopeExecutionUnavailable); + } + validate_workflow_task_payloads(&task)?; + cooperative_cancellation::assert_cancellation_scope_replay_supported( + &task.history_events, + self.allow_cancellation_scope_authoring, + self.allow_cancellation_scope_delivery, + )?; + let cancellation_history = if let Some(observation) = observation { + observation.validate_observation()?; + cooperative_cancellation::cancellation_claim(&task)?; + let canonical = CancellationHistory::from_events( + &task.history_events, + task.run_id.as_deref().unwrap_or_default(), + Some(observation), + )?; + if canonical.request_index >= task.history_events.len() { + return Err(Error::InvalidCooperativeCancellation( + "workflow replay requires the observed canonical request".into(), + )); + } + Some(canonical) + } else { + None + }; if let Some(update_id) = task .workflow_update_id .as_deref() .filter(|update_id| !update_id.is_empty()) { + if self.allow_cancellation_scope_delivery { + return Err(Error::CancellationScopeExecutionUnavailable); + } + CancellationHistory::from_events( + &task.history_events, + task.run_id.as_deref().unwrap_or_default(), + None, + )?; return self .execute_update_task(&task, update_id) .map(WorkflowTaskDecision::without_message_streams); @@ -7624,23 +7948,59 @@ impl Worker { .filter(|identity| !identity.is_empty()) .or_else(|| (!task.task_id.is_empty()).then(|| task.task_id.clone())) .unwrap_or_default(); - let mut workflow_state = WorkflowState::new_with_identity( + let mut workflow_state = WorkflowState::new_with_identity_and_scope_delivery( task.history_events, task.workflow_id, task.run_id, self.task_queue.clone(), task.payload_codec.clone(), resume_signal, + self.allow_cancellation_scope_authoring, + self.allow_cancellation_scope_delivery, )?; workflow_state.history_budget = history_budget; workflow_state.workflow_command_identity = workflow_command_identity; - workflow_state.cancel_requested = task.cancel_requested; + if let Some(canonical) = cancellation_history { + workflow_state.cancellation_history = canonical; + workflow_state.cancellation_delivery_enabled = true; + } + if workflow_state.cancellation_history.request.is_none() { + workflow_state.cancel_requested = task.cancel_requested; + } let state = Arc::new(Mutex::new(workflow_state)); - let ctx = WorkflowContext { state }; + let ctx = WorkflowContext { + state, + cancellation_scope_id: "root".into(), + }; let mut future = (workflow.execute)(ctx.clone(), input); let mut cx = TaskContext::from_waker(noop_waker_ref()); - match future.as_mut().poll(&mut cx) { + let outcome = { + let _replay = cancellation_replay_clock::ReplayGuard::enter(&ctx)?; + future.as_mut().poll(&mut cx) + }; + let awaiting_canonical_boundary = { + let state = ctx.state.lock().map_err(|_| Error::WorkflowStatePoisoned)?; + state.cancellation_delivery_intent.is_some() + || state.cancellation_scope_opening.is_some() + || state + .scope_delivery + .as_ref() + .is_some_and(|replay| replay.intent.is_some()) + }; + if awaiting_canonical_boundary { + // A private delivery intent never becomes an application error. + // Even a custom future which ignores a pending durable call cannot + // publish terminal commands before canonical delivery and replay. + if let Poll::Ready(Err(error)) = outcome { + if workflow_task_integrity_error(&error) { + return Err(error); + } + } + return self.message_stream_decision(&ctx, ctx.take_commands()?); + } + + match outcome { Poll::Ready(Ok(result)) => { ctx.ensure_history_consumed()?; let result = encode_typed_envelope(&result, &task.payload_codec)?; @@ -7693,11 +8053,83 @@ impl Worker { ctx: &WorkflowContext, commands: Vec, ) -> Result { + if !self.cooperative_cancellation_enabled + && commands.iter().any(|command| { + command["type"] == "schedule_activity" + && command.get("cancellation_policy").is_some() + }) + { + return Err(Error::CooperativeCancellationUnavailable(format!( + "activity_cancellation_policy_not_supported: Rust worker {} must enable cooperative cancellation with worker protocol 1.20 and a compatible Server/Native backend", + self.worker_id, + ))); + } + if !self.cooperative_cancellation_enabled + && commands.iter().any(|command| { + command["type"] == "start_child_workflow" + && (command["parent_close_policy"] == "request_cancellation" + || matches!( + command["cancellation_policy"].as_str(), + Some("try_cancel" | "wait_cancellation_completed") + )) + }) + { + return Err(Error::CooperativeCancellationUnavailable(format!( + "child_cancellation_policy_not_supported: Rust worker {} must enable cooperative cancellation with worker protocol 1.20 and a compatible Server/Native backend", + self.worker_id, + ))); + } let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?; + let state = ctx.state.lock().map_err(|_| Error::WorkflowStatePoisoned)?; + let scope_delivery = state + .scope_delivery + .as_ref() + .and_then(|replay| replay.intent.clone()); + if scope_delivery + .as_ref() + .is_some_and(|intent| commands.len() != intent.command_count) + { + return Err(invalid_recorded_history( + "cancellation_scope_pending_call_escaped", + scope_delivery.as_ref().unwrap().boundary.sequence, + "commands preceding the pending scoped call", + "commands after suspension", + "workflow cannot publish beyond an uncommitted scope delivery", + )); + } + if let Some(opening) = &state.cancellation_scope_opening { + if commands.len() != opening.command_count { + return Err(invalid_recorded_history( + "cancellation_scope_pending_call_escaped", + opening.sequence, + "commands before original scope opening", + "commands after suspended opening", + "workflow cannot publish beyond an uncommitted scope opening", + )); + } + } + if state.cancellation_delivery_intent.is_some() + && commands.len() != state.cancellation_delivery_command_count + { + return Err(invalid_recorded_history( + "cooperative_cancellation_pending_call_escaped", + state + .cancellation_delivery_intent + .as_ref() + .expect("pending delivery") + .sequence, + "only commands preceding the pending cancellation call", + "commands authored after a suspended call", + "workflow code cannot publish work beyond an uncommitted cancellation boundary", + )); + } Ok(WorkflowTaskDecision { commands, message_stream_cursors, message_stream_waits, + cancellation_delivery: state.cancellation_delivery_intent.clone(), + cancellation_scope_opening: state.cancellation_scope_opening.clone(), + cancellation_scope_delivery: scope_delivery, }) } @@ -7792,6 +8224,7 @@ impl Worker { attempt_number: task.attempt_number, task_queue: self.task_queue.clone(), worker_id: self.worker_id.clone(), + claim_guard: None, }; handler(ctx, args).await @@ -7997,6 +8430,7 @@ impl MessageStream { #[derive(Clone, Debug)] pub struct WorkflowContext { state: Arc>, + cancellation_scope_id: String, } fn valid_memo_key(key: &str) -> bool { @@ -8515,16 +8949,29 @@ impl WorkflowContext { Saga::new(self.clone()) } - /// Whether this task carries a cooperative cancellation request. + /// Whether cancellation has reached this replay's authored boundary. /// /// Server's current `/cancel` route is terminal and does not set this flag. - /// Service-mode cooperative cancellation is not yet available. + /// A canonical service request remains pending until its committed delivery + /// is consumed. Request observation alone does not authorize an exception. pub fn is_cancellation_requested(&self) -> Result { let state = self .state .lock() .map_err(|_| Error::WorkflowStatePoisoned)?; - Ok(state.cancel_requested) + if self.cancellation_scope_id != "root" && state.scope_delivery.is_some() { + return Ok(state + .scope_delivery + .as_ref() + .unwrap() + .contexts + .contains_key(&self.cancellation_scope_id)); + } + Ok(state.cancel_requested + || state + .scope_delivery + .as_ref() + .is_some_and(|replay| replay.contexts.contains_key(&self.cancellation_scope_id))) } /// Raise a typed cooperative cancellation at an author-controlled point. @@ -8532,10 +8979,36 @@ impl WorkflowContext { /// Passing this result to [`Saga::finish`] compensates already registered /// forward steps before the cancellation remains the initiating outcome. pub fn throw_if_cancellation_requested(&self) -> Result<()> { - if self.is_cancellation_requested()? { - return Err(Error::WorkflowCancellationRequested( - WorkflowCancellationRequested, - )); + let mut state = self + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)?; + if state.cancellation_shield_depth == 0 { + if let Some(replay) = &state.scope_delivery { + if let Some((context, _)) = replay.contexts.get(&self.cancellation_scope_id) { + let delivered = replay + .consumed_delivery_for_scope(&self.cancellation_scope_id) + .ok_or_else(|| { + Error::InvalidCooperativeCancellation( + "scoped request lacks its consumed original delivery".into(), + ) + })?; + return Err(Error::CancellationScopeRequested( + CancellationScopeRequested { + context: context + .clone() + .with_replay(Some(Arc::downgrade(&self.state))), + delivery: delivered.boundary.clone(), + }, + )); + } + } + } + if state.cancel_requested && state.cancellation_shield_depth == 0 { + if self.cancellation_scope_id != "root" && state.scope_delivery.is_some() { + return Ok(()); + } + return Err(state.cancellation_error()); } Ok(()) } @@ -9260,6 +9733,22 @@ impl WorkflowContext { .state .lock() .map_err(|_| Error::WorkflowStatePoisoned)?; + if let Some(replay) = &state.scope_delivery { + if let Some((&sequence, _)) = replay + .canonical + .deliveries + .iter() + .find(|(sequence, _)| !replay.consumed.contains(sequence)) + { + return Err(invalid_recorded_history( + "cancellation_scope_delivery_unconsumed", + sequence, + "original scoped delivery", + "workflow completion", + "workflow completed without consuming its committed scope delivery", + )); + } + } if let Some(command) = state.recorded_commands.get(state.command_cursor) { return Err(Error::NonDeterministicReplay(ReplayFailure::new( "recorded_commands_unconsumed", @@ -9269,6 +9758,20 @@ impl WorkflowContext { "workflow completed before consuming all recorded durable commands", ))); } + if let Some(delivery) = state + .cancellation_history + .delivery + .as_ref() + .filter(|_| !state.cancellation_consumed) + { + return Err(invalid_recorded_history( + "cooperative_cancellation_unconsumed", + delivery.sequence, + "committed cancellation call", + "workflow completion", + "workflow completed without reaching its committed cancellation boundary", + )); + } if let Some(sequence) = state .recorded_continue_as_new_sequence .filter(|_| !state.continue_as_new_consumed) @@ -9328,6 +9831,17 @@ struct WorkflowState { history_events: Arc>, history_budget: WorkflowHistoryBudget, cancel_requested: bool, + cancellation_history: CancellationHistory, + cancellation_consumed: bool, + cancellation_clock: Option, + cancellation_shield_depth: u64, + cancellation_delivery_enabled: bool, + cancellation_delivery_intent: Option, + cancellation_delivery_command_count: usize, + allow_cancellation_scope_authoring: bool, + cancellation_scope_opening: Option, + cancellation_scope_memberships: BTreeMap, + scope_delivery: Option, resume_signal: Option, recorded_commands: Vec, selection_markers: Vec, @@ -9373,21 +9887,121 @@ impl WorkflowState { payload_codec: String, resume_signal: Option, ) -> Result { - let recorded_commands = recorded_commands( + Self::new_with_identity_and_scopes( + history, + workflow_id, + run_id, + task_queue, + payload_codec, + resume_signal, + false, + ) + } + + fn new_with_identity_and_scopes( + history: Vec, + workflow_id: Option, + run_id: Option, + task_queue: String, + payload_codec: String, + resume_signal: Option, + allow_cancellation_scope_authoring: bool, + ) -> Result { + Self::new_with_identity_and_scope_delivery( + history, + workflow_id, + run_id, + task_queue, + payload_codec, + resume_signal, + allow_cancellation_scope_authoring, + false, + ) + } + + fn new_with_identity_and_scope_delivery( + history: Vec, + workflow_id: Option, + run_id: Option, + task_queue: String, + payload_codec: String, + resume_signal: Option, + allow_cancellation_scope_authoring: bool, + allow_cancellation_scope_delivery: bool, + ) -> Result { + cooperative_cancellation::assert_cancellation_scope_replay_supported( &history, - &payload_codec, - WorkflowIdentity { - workflow_id: workflow_id.clone(), - run_id: run_id.clone(), - }, + allow_cancellation_scope_authoring, + allow_cancellation_scope_delivery, )?; - let selection_markers = recorded_selection_markers(&history)?; - let cancelled_selection_members = recorded_selection_cancellations(&history)?; - let recorded_continue_as_new = history - .iter() - .filter(|event| event.event_type == "WorkflowContinuedAsNew") - .collect::>(); - if recorded_continue_as_new.len() > 1 { + let scopes = if allow_cancellation_scope_authoring { + cancellation_scope::CancellationScopeHistory::read( + &history, + run_id.as_deref().unwrap_or_default(), + )? + } else { + cancellation_scope::CancellationScopeHistory::default() + }; + let cancellation_history = CancellationHistory::from_events( + &history, + run_id.as_deref().unwrap_or_default(), + None, + )?; + if allow_cancellation_scope_authoring + && cancellation_history.request.is_some() + && !allow_cancellation_scope_delivery + { + return Err(Error::CancellationScopeExecutionUnavailable); + } + let scope_delivery = if allow_cancellation_scope_delivery { + Some(cancellation_scope_replay::ScopeReplay::read( + &history, + &scopes, + run_id.as_deref().unwrap_or_default(), + workflow_id.as_deref().unwrap_or_default(), + )?) + } else { + None + }; + let mut recorded_commands = cancellation_history.bind_commands(recorded_commands( + &history, + &payload_codec, + WorkflowIdentity { + workflow_id: workflow_id.clone(), + run_id: run_id.clone(), + }, + )?)?; + for (sequence, opening) in &scopes.openings { + if recorded_commands + .iter() + .any(|command| command.sequence() == *sequence) + { + return Err(invalid_recorded_history( + "invalid_cancellation_scope_history", + *sequence, + "distinct authored scope opening", + "operation collision", + "scope opening collides with a command", + )); + } + recorded_commands.push(RecordedCommand::CancellationScope { + sequence: *sequence, + scope_id: opening.scope_id.clone(), + parent_scope_id: opening.parent_scope_id.clone(), + shield_parent: opening.shield_parent, + }); + } + recorded_commands.sort_by_key(RecordedCommand::sequence); + if let Some(replay) = &scope_delivery { + replay.bind_commands(&mut recorded_commands)?; + } + let selection_markers = recorded_selection_markers(&history)?; + let cancelled_selection_members = recorded_selection_cancellations(&history)?; + let recorded_continue_as_new = history + .iter() + .filter(|event| event.event_type == "WorkflowContinuedAsNew") + .collect::>(); + if recorded_continue_as_new.len() > 1 { return Err(invalid_recorded_history( "duplicate_continue_as_new_transition", recorded_continue_as_new @@ -9457,6 +10071,22 @@ impl WorkflowState { "WorkflowCancellationRequested" | "WorkflowCancelRequested" ) }); + let cancellation_clock = cancellation_history + .request + .as_ref() + .and_then(|request| request.context.as_ref()) + .map(|_| { + cancellation_replay_clock::ReplayClock::new( + &history, + &recorded_commands, + cancellation_history.delivery_index, + ) + }) + .or_else(|| { + scope_delivery.as_ref().map(|_| { + cancellation_replay_clock::ReplayClock::new(&history, &recorded_commands, None) + }) + }); Ok(Self { workflow_command_identity: String::new(), workflow_stream_command_counter: 0, @@ -9470,6 +10100,17 @@ impl WorkflowState { ..WorkflowHistoryBudget::default() }, cancel_requested, + cancellation_history, + cancellation_consumed: false, + cancellation_clock, + cancellation_shield_depth: 0, + cancellation_delivery_enabled: false, + cancellation_delivery_intent: None, + cancellation_delivery_command_count: 0, + allow_cancellation_scope_authoring, + cancellation_scope_opening: None, + cancellation_scope_memberships: scopes.memberships, + scope_delivery, resume_signal, recorded_commands, selection_markers, @@ -9555,9 +10196,26 @@ fn decode_message_stream_delivery(arguments: Vec) -> Result, + }, + CancellationBoundary { + sequence: u64, + call_kind: CancellationCallKind, + original: Option>, + }, Activity { sequence: u64, activity_type: Option, + cancellation_policy: String, options: Option, outcome: Option, parallel_group_path: Option>, @@ -9571,6 +10229,7 @@ enum RecordedCommand { ChildWorkflow { sequence: u64, workflow_type: Option, + policies: RecordedChildPolicies, outcome: Option, parallel_group_path: Option>, }, @@ -9909,6 +10568,161 @@ struct RecordedActivityOptions { retry_policy: ActivityRetrySnapshot, } +#[derive(Clone, Debug)] +struct RecordedChildPolicies { + parent_close_policy: String, + cancellation_policy: String, +} + +fn recorded_activity_cancellation_policy( + events: &[&HistoryEvent], + sequence: u64, +) -> Result { + let mut policy: Option = None; + for event in events.iter().filter(|event| { + matches!( + event.event_type.as_str(), + "ActivityScheduled" + | "ActivityStarted" + | "ActivityCompleted" + | "ActivityFailed" + | "ActivityTimedOut" + | "ActivityCancelled" + ) + }) { + for source in [Some(&event.payload), event.payload.get("activity")] + .into_iter() + .flatten() + { + let Some(value) = source.get("cancellation_policy") else { + continue; + }; + let Some(incoming @ ("try_cancel" | "wait_cancellation_completed" | "abandon")) = + value.as_str() + else { + return Err(invalid_recorded_history( + "invalid_activity_cancellation_policy_history", + sequence, + "supported Activity cancellation policy", + &value.to_string(), + "Activity history contains an invalid cancellation policy", + )); + }; + if let Some(previous) = policy.as_deref() { + if previous != incoming { + return Err(invalid_recorded_history( + "activity_cancellation_policy_history_conflict", + sequence, + previous, + incoming, + "Activity cancellation policy changed between history events", + )); + } + } + policy = Some(incoming.to_string()); + } + policy.get_or_insert_with(|| "try_cancel".to_string()); + } + Ok(policy.unwrap_or_else(|| "try_cancel".to_string())) +} + +fn recorded_child_policy_value<'a>( + payload: &'a Value, + field: &str, + sequence: u64, +) -> Result> { + let Some(value) = payload.get(field).filter(|value| !value.is_null()) else { + return Ok(None); + }; + let valid = match (field, value.as_str()) { + ( + "parent_close_policy", + Some("abandon" | "request_cancel" | "request_cancellation" | "terminate"), + ) => true, + ("cancellation_policy", Some("abandon" | "try_cancel" | "wait_cancellation_completed")) => { + true + } + _ => false, + }; + if !valid { + return Err(invalid_recorded_history( + "invalid_child_workflow_policy_history", + sequence, + "supported child workflow policy", + &value.to_string(), + "child workflow history contains an invalid policy", + )); + } + Ok(value.as_str()) +} + +fn recorded_child_policies( + events: &[&HistoryEvent], + scheduled: &HistoryEvent, + sequence: u64, +) -> Result { + let parent_close_policy = + recorded_child_policy_value(&scheduled.payload, "parent_close_policy", sequence)? + .unwrap_or("abandon") + .to_string(); + let cancellation_policy = + recorded_child_policy_value(&scheduled.payload, "cancellation_policy", sequence)? + .unwrap_or("abandon") + .to_string(); + for event in events { + for (field, expected) in [ + ("parent_close_policy", parent_close_policy.as_str()), + ("cancellation_policy", cancellation_policy.as_str()), + ] { + if let Some(actual) = recorded_child_policy_value(&event.payload, field, sequence)? { + if actual != expected { + return Err(invalid_recorded_history( + "child_workflow_policy_history_conflict", + sequence, + expected, + actual, + "child workflow policy changed between history events", + )); + } + } + } + } + Ok(RecordedChildPolicies { + parent_close_policy, + cancellation_policy, + }) +} + +fn ensure_child_policies_match( + sequence: u64, + recorded: &RecordedChildPolicies, + current: &ChildWorkflowOptions, +) -> Result<()> { + for (field, expected, actual) in [ + ( + "parent_close_policy", + recorded.parent_close_policy.as_str(), + current.parent_close_policy.as_str(), + ), + ( + "cancellation_policy", + recorded.cancellation_policy.as_str(), + current.cancellation_policy.as_str(), + ), + ] { + if expected != actual { + return Err(Error::NonDeterministicReplay(ReplayFailure::new( + "child_workflow_policy_changed", + Some(sequence), + Some(expected.to_string()), + Some(actual.to_string()), + format!("child workflow {field} changed during replay"), + ))); + } + } + Ok(()) +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize)] enum RecordedSnapshotValue { /// Older history did not persist this field, so it cannot constrain replay. @@ -10068,7 +10882,10 @@ fn activity_options_description(options: &RecordedActivityOptions) -> String { impl RecordedCommand { fn sequence(&self) -> u64 { match self { - Self::Activity { sequence, .. } + Self::CancellationGroup { sequence, .. } + | Self::CancellationScope { sequence, .. } + | Self::CancellationBoundary { sequence, .. } + | Self::Activity { sequence, .. } | Self::Timer { sequence, .. } | Self::ChildWorkflow { sequence, .. } | Self::SignalWait { sequence, .. } @@ -10082,6 +10899,10 @@ impl RecordedCommand { fn shape(&self) -> &'static str { match self { + Self::CancellationScope { .. } => "cancellation scope", + Self::CancellationBoundary { .. } | Self::CancellationGroup { .. } => { + "cooperative cancellation" + } Self::Activity { .. } => "activity", Self::Timer { .. } => "timer", Self::ChildWorkflow { .. } => "child workflow", @@ -10577,6 +11398,7 @@ impl ParallelLeafCall { struct ParallelLeaf { call: ParallelLeafCall, + sequence: u64, member_path: Vec, group_path: Vec, result: Option, @@ -10588,6 +11410,7 @@ pub struct ParallelCall { operations: Option>, shape: Option, leaves: Vec, + pending_scope_delivery: bool, } impl ParallelCall { @@ -10597,6 +11420,7 @@ impl ParallelCall { operations: Some(operations), shape: None, leaves: Vec::new(), + pending_scope_delivery: false, } } @@ -10634,7 +11458,23 @@ impl ParallelCall { } }; - self.leaves = parallel_descriptors(operations, base_sequence)? + let descriptors = parallel_descriptors(operations, base_sequence)?; + { + let mut state = self + .ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)?; + let cursor = state.command_cursor; + self.ctx.validate_scope_membership(&mut state, cursor)?; + if state.prepare_parallel_scope_cancellation(&descriptors)? { + self.pending_scope_delivery = true; + return Ok(()); + } + state.expand_cancellation_group(&descriptors)?; + state.prepare_group_cancellation(&descriptors)?; + } + self.leaves = descriptors .into_iter() .map(|descriptor| { let call = parallel_leaf_call( @@ -10644,6 +11484,7 @@ impl ParallelCall { ); ParallelLeaf { call, + sequence: base_sequence + descriptor.offset as u64, member_path: descriptor.member_path, group_path: descriptor.group_path, result: None, @@ -10662,6 +11503,9 @@ impl ParallelCall { return Poll::Ready(Err(error)); } } + if self.pending_scope_delivery { + return Poll::Pending; + } if self.leaves.is_empty() { return Poll::Ready(Ok(Vec::new())); } @@ -10686,6 +11530,22 @@ impl ParallelCall { { return Poll::Ready(Err(failures.remove(position).1)); } + if self + .ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)? + .cancellation_delivery_intent + .is_some() + { + return Poll::Pending; + } + if let Some(position) = failures + .iter() + .position(|(_, error)| matches!(error, Error::CooperativeCancellationRequested(_))) + { + return Poll::Ready(Err(failures.remove(position).1)); + } failures.sort_by_key(|(index, _)| *index); let (failed_index, cause) = failures.remove(0); let failed = &self.leaves[failed_index]; @@ -10707,6 +11567,16 @@ impl ParallelCall { .first() .map(|entry| entry.parallel_group_id.clone()) .unwrap_or_default(); + let sequences = self + .leaves + .iter() + .map(|leaf| leaf.sequence) + .collect::>(); + self.ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)? + .advance_cancellation_sequences(&sequences, Some(failed.sequence)); return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure { group_id, member_path: failed.member_path.clone(), @@ -10715,7 +11585,15 @@ impl ParallelCall { cause: Box::new(cause), }))); } - if pending { + if pending + || self + .ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)? + .cancellation_delivery_intent + .is_some() + { return Poll::Pending; } @@ -10729,6 +11607,16 @@ impl ParallelCall { self.shape.as_ref().expect("initialized parallel shape"), &mut flat_results, ); + let sequences = self + .leaves + .iter() + .map(|leaf| leaf.sequence) + .collect::>(); + self.ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)? + .advance_cancellation_sequences(&sequences, None); Poll::Ready(Ok(match results { ParallelAvroResult::Group(results) => results, ParallelAvroResult::Activity(_) @@ -11074,7 +11962,12 @@ impl SelectCall { .state .lock() .map_err(|_| Error::WorkflowStatePoisoned)?; - if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) { + if let Some(RecordedCommand::CancellationGroup { sequence, .. }) = + state.recorded_commands.get(state.command_cursor) + { + *sequence + } else if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) + { marker.selection_group_base_sequence } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) { recorded.sequence() @@ -11091,6 +11984,15 @@ impl SelectCall { }; let (descriptors, members) = selection_descriptors(operations, base_sequence)?; let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len()); + { + let mut state = self + .ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)?; + state.expand_cancellation_group(&descriptors)?; + state.prepare_group_cancellation(&descriptors)?; + } self.leaves = descriptors .into_iter() .map(|descriptor| SelectionLeaf { @@ -11130,6 +12032,31 @@ impl Future for SelectCall { } } + if self + .ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)? + .cancellation_delivery_intent + .is_some() + { + return Poll::Pending; + } + + if let Some(leaf) = self.leaves.iter_mut().find(|leaf| { + matches!( + leaf.outcome, + Some(Err(Error::CooperativeCancellationRequested(_))) + ) + }) { + return Poll::Ready( + leaf.outcome + .take() + .expect("matched cancellation") + .map(|_| unreachable!()), + ); + } + let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some()); let selection_member_range = self .members @@ -11335,6 +12262,11 @@ impl Future for SelectCall { None }; let winner = handles[member_position].clone(); + self.ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)? + .advance_cancellation_selection(&marker.selection_group_id); Poll::Ready(Ok(SelectionResult { key: winner.key.clone(), index: winner.index, @@ -11652,9 +12584,12 @@ fn recorded_selection_member_is_terminal( RecordedCommand::SignalWait { value, .. } => value.is_some(), RecordedCommand::ConditionWait { result, .. } => result.is_some(), RecordedCommand::SearchAttributes { .. } + | RecordedCommand::CancellationScope { .. } | RecordedCommand::SideEffect { .. } | RecordedCommand::VersionMarker { .. } - | RecordedCommand::Memo { .. } => false, + | RecordedCommand::Memo { .. } + | RecordedCommand::CancellationBoundary { .. } + | RecordedCommand::CancellationGroup { .. } => false, }; if !terminal { all_completed = false; @@ -11743,14 +12678,26 @@ impl Future for DurableOperationAwaitCall { type Output = Result; fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll { - let state = match self.handle.ctx.state.lock() { + let mut state = match self.handle.ctx.state.lock() { Ok(state) => state, Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)), }; + if let Err(error) = state.replay_selection_handle_cancellation(&self.handle) { + return Poll::Ready(Err(error)); + } + match state.prepare_selection_handle_cancellation(&self.handle) { + Ok(true) => return Poll::Pending, + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } match selection_cancellation_for_handle(&state, &self.handle) { Err(error) => return Poll::Ready(Err(error)), Ok(false) => {} Ok(true) => { + state.advance_cancellation_receipt( + &self.handle.selection_group_id, + self.handle.base_sequence, + ); return Poll::Ready(Err(Error::DurableOperationCancelled( DurableOperationCancelled { selection_group_id: self.handle.selection_group_id.clone(), @@ -11763,13 +12710,77 @@ impl Future for DurableOperationAwaitCall { } } match recorded_selection_member_outcome(&state, &self.handle) { - Ok(Some(result)) => Poll::Ready(Ok(result)), + Ok(Some(result)) => { + state.advance_cancellation_handle(&self.handle, false); + Poll::Ready(Ok(result)) + } Ok(None) => Poll::Pending, - Err(error) => Poll::Ready(Err(error)), + Err(error) => { + if !workflow_task_integrity_error(&error) { + state.advance_cancellation_handle(&self.handle, true); + } + Poll::Ready(Err(error)) + } } } } +fn validate_selection_delivery_handle( + state: &WorkflowState, + handle: &DurableOperationHandle, +) -> Result<()> { + let path = state + .recorded_commands + .iter() + .find(|command| command.sequence() == handle.base_sequence) + .and_then(|command| match command { + RecordedCommand::Activity { + parallel_group_path, + .. + } + | RecordedCommand::ChildWorkflow { + parallel_group_path, + .. + } + | RecordedCommand::Timer { + parallel_group_path, + .. + } + | RecordedCommand::SignalWait { + parallel_group_path, + .. + } + | RecordedCommand::ConditionWait { + parallel_group_path, + .. + } => parallel_group_path.as_ref(), + _ => None, + }) + .and_then(|path| path.first()); + let matches = path.is_some_and(|entry| { + entry.parallel_group_mode.as_deref() == Some("select") + && entry.parallel_group_id == handle.selection_group_id + && entry.selection_member_key.as_ref() == Some(&handle.key) + && entry.selection_member_index == Some(handle.index) + && entry.selection_member_base_sequence == Some(handle.base_sequence) + && entry.selection_member_size == Some(handle.size) + && entry.selection_member_kind.as_deref() == Some(handle.kind.as_str()) + }); + if !matches + || selection_operation_identity(state, &handle.kind, handle.base_sequence, handle.size) + != handle.identity + { + return Err(invalid_recorded_history( + "cooperative_cancellation_call_mismatch", + handle.base_sequence, + "selection handle matching its authored durable identity", + &format!("{handle:?}"), + "selection handle identity or member metadata changed before cancellation replay", + )); + } + Ok(()) +} + /// Future returned by [`DurableOperationHandle::cancel`]. pub struct CancelDurableOperationCall { handle: DurableOperationHandle, @@ -11787,7 +12798,13 @@ impl Future for CancelDurableOperationCall { }; match selection_cancellation_for_handle(&state, &self.handle) { Err(error) => return Poll::Ready(Err(error)), - Ok(true) => return Poll::Ready(Ok(())), + Ok(true) => { + state.advance_cancellation_receipt( + &self.handle.selection_group_id, + self.handle.base_sequence, + ); + return Poll::Ready(Ok(())); + } Ok(false) => {} } if recorded_selection_member_is_terminal(&state, &self.handle) { @@ -11889,6 +12906,10 @@ impl Saga { /// Compensate `initiating_failure` and return the failure that must remain. pub async fn compensate(mut self, initiating_failure: Error) -> Error { + let _shield = match self.ctx.cancellation_shield() { + Ok(shield) => shield, + Err(error) => return error, + }; while let Some(compensation) = self.compensations.pop() { if let Err(compensation_failure) = self .ctx @@ -11964,16 +12985,41 @@ impl ActivityCall { retry_policy: current_activity_retry_snapshot(&options), }; - if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() { + let cursor = state.command_cursor; + if let Err(error) = ctx.validate_scope_membership(&mut state, cursor) { + return Poll::Ready(Err(error)); + } + let recorded = + match state.cancellation_replay_command(cursor, CancellationCallKind::Activity) { + Ok(recorded) => recorded, + Err(error) => return Poll::Ready(Err(error)), + }; + if let Some(recorded) = recorded { let sequence = recorded.sequence(); match recorded { RecordedCommand::Activity { activity_type, + cancellation_policy, options: recorded_options, outcome, parallel_group_path, .. } => { + let current_policy = options + .cancellation_policy + .unwrap_or(CancellationPolicy::TryCancel) + .as_str(); + if cancellation_policy != current_policy { + return Poll::Ready(Err(Error::NonDeterministicReplay( + ReplayFailure::new( + "activity_cancellation_policy_changed", + Some(sequence), + Some(cancellation_policy), + Some(current_policy.to_string()), + "Activity cancellation policy changed during replay", + ), + ))); + } if let Err(error) = ensure_parallel_path_matches( sequence, parallel_group_path.as_deref(), @@ -12038,8 +13084,26 @@ impl ActivityCall { ))); } } + if let Err(error) = + state.replay_cancellation_at(cursor, CancellationCallKind::Activity) + { + return Poll::Ready(Err(error)); + } + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Activity, + &self.parallel_group_path, + ) { + Ok(true) => { + self.scheduled = true; + return Poll::Pending; + } + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } state.command_cursor += 1; if let Some(outcome) = outcome { + state.advance_cancellation_sequence(sequence, &self.parallel_group_path); return Poll::Ready(outcome.map_err(Error::ActivityFailed)); } state.matched_recorded_pending = true; @@ -12094,7 +13158,23 @@ impl ActivityCall { if let Some(retry_policy) = options.retry_policy { command.insert("retry_policy".to_string(), retry_policy); } + if let Some(policy) = options.cancellation_policy { + command.insert("cancellation_policy".to_string(), json!(policy.as_str())); + } apply_parallel_group_path(&mut command, &self.parallel_group_path); + ctx.apply_scope_membership(&mut command); + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Activity, + &self.parallel_group_path, + ) { + Ok(true) => { + self.scheduled = true; + return Poll::Pending; + } + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } state.commands.push(Value::Object(command)); self.scheduled = true; } @@ -12141,7 +13221,16 @@ impl Future for TimerCall { Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)), }; - if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() { + let cursor = state.command_cursor; + if let Err(error) = ctx.validate_scope_membership(&mut state, cursor) { + return Poll::Ready(Err(error)); + } + let recorded = match state.cancellation_replay_command(cursor, CancellationCallKind::Timer) + { + Ok(recorded) => recorded, + Err(error) => return Poll::Ready(Err(error)), + }; + if let Some(recorded) = recorded { match recorded { RecordedCommand::Timer { sequence, @@ -12150,6 +13239,11 @@ impl Future for TimerCall { parallel_group_path, .. } => { + if let Err(error) = + state.validate_scope_cleanup_timer(&ctx.cancellation_scope_id, sequence) + { + return Poll::Ready(Err(error)); + } if let Err(error) = ensure_parallel_path_matches( sequence, parallel_group_path.as_deref(), @@ -12168,8 +13262,27 @@ impl Future for TimerCall { ), ))); } + if let Err(error) = + state.replay_cancellation_at(cursor, CancellationCallKind::Timer) + { + return Poll::Ready(Err(error)); + } + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Timer, + &self.parallel_group_path, + ) { + Ok(true) => { + self.scheduled = true; + self.matched_pending = true; + return Poll::Pending; + } + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } state.command_cursor += 1; if fired { + state.advance_cancellation_sequence(sequence, &self.parallel_group_path); return Poll::Ready(Ok(())); } state.matched_recorded_pending = true; @@ -12186,7 +13299,28 @@ impl Future for TimerCall { ("type".to_string(), json!("start_timer")), ("delay_seconds".to_string(), json!(requested_delay)), ]); + match state.scope_cleanup_timer_proof(&ctx.cancellation_scope_id) { + Ok(Some(proof)) => { + command.insert("cancellation_cleanup".into(), proof); + } + Ok(None) => {} + Err(error) => return Poll::Ready(Err(error)), + } apply_parallel_group_path(&mut command, &self.parallel_group_path); + ctx.apply_scope_membership(&mut command); + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Timer, + &self.parallel_group_path, + ) { + Ok(true) => { + self.scheduled = true; + self.matched_pending = true; + return Poll::Pending; + } + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } state.commands.push(Value::Object(command)); self.scheduled = true; } @@ -12246,7 +13380,17 @@ impl Future for ConditionWaitCall { Ok(state) => state, Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)), }; - let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else { + let initial_cursor = state.command_cursor; + if let Err(error) = ctx.validate_scope_membership(&mut state, initial_cursor) { + return Poll::Ready(Err(error)); + } + let recorded = match state + .cancellation_replay_command(initial_cursor, CancellationCallKind::Condition) + { + Ok(recorded) => recorded, + Err(error) => return Poll::Ready(Err(error)), + }; + let Some(recorded) = recorded else { drop(state); return self.poll_new_condition(options); }; @@ -12256,37 +13400,52 @@ impl Future for ConditionWaitCall { let mut cursor = state.command_cursor; let mut result = None; + let mut pending_delivery = false; loop { + if cursor > initial_cursor + && matches!( + state.recorded_commands.get(cursor), + Some(RecordedCommand::CancellationBoundary { original: None, .. }) + ) + { + break; + } + let recorded = match state + .cancellation_replay_command(cursor, CancellationCallKind::Condition) + { + Ok(recorded) => recorded, + Err(error) => return Poll::Ready(Err(error)), + }; let Some(RecordedCommand::ConditionWait { sequence, - occurrence_id: recorded_occurrence_id, - condition_key, - predicate_identity, + occurrence_id: ref recorded_occurrence_id, + ref condition_key, + ref predicate_identity, timeout_seconds, result: recorded_result, - parallel_group_path, + ref parallel_group_path, .. - }) = state.recorded_commands.get(cursor) + }) = recorded else { break; }; - if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id { + if cursor > initial_cursor && recorded_occurrence_id != &occurrence_id { break; } if let Err(error) = ensure_parallel_path_matches( - *sequence, + sequence, parallel_group_path.as_deref(), &self.parallel_group_path, ) { return Poll::Ready(Err(error)); } if let Err(error) = validate_recorded_condition_wait( - *sequence, + sequence, recorded_occurrence_id, condition_key.as_deref(), predicate_identity, - *timeout_seconds, + timeout_seconds, &occurrence_id, &options, ) { @@ -12295,16 +13454,36 @@ impl Future for ConditionWaitCall { if result == Some(ConditionWaitResult::TimedOut) { return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new( "condition_wait_reopened_after_timeout", - Some(*sequence), + Some(sequence), Some("timed-out condition is terminal".to_string()), Some("another physical wait-open".to_string()), "condition history reopened one logical wait after its durable timeout", )))); } - result = *recorded_result; + if let Err(error) = + state.replay_cancellation_at(cursor, CancellationCallKind::Condition) + { + return Poll::Ready(Err(error)); + } + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Condition, + &self.parallel_group_path, + ) { + Ok(pending) => pending_delivery |= pending, + Err(error) => return Poll::Ready(Err(error)), + } + result = recorded_result; + if result.is_some() { + state.advance_cancellation_sequence(sequence, &self.parallel_group_path); + } cursor += 1; } state.command_cursor = cursor; + if pending_delivery { + self.opened_wait = true; + return Poll::Pending; + } result }; @@ -12329,6 +13508,26 @@ impl ConditionWaitCall { options: ValidatedConditionWaitOptions, recorded_wait: bool, ) -> Poll> { + { + let ctx = self.ctx.clone(); + let mut state = match ctx.state.lock() { + Ok(state) => state, + Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)), + }; + let cursor = state.command_cursor; + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Condition, + &self.parallel_group_path, + ) { + Ok(true) => { + self.opened_wait = true; + return Poll::Pending; + } + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } + } let selection_member = self .parallel_group_path .first() @@ -12365,6 +13564,7 @@ impl ConditionWaitCall { command.insert("timeout_seconds".to_string(), json!(timeout_seconds)); } apply_parallel_group_path(&mut command, &self.parallel_group_path); + ctx.apply_scope_membership(&mut command); state.commands.push(Value::Object(command)); drop(state); self.opened_wait = true; @@ -12446,11 +13646,21 @@ impl ChildWorkflowCall { Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)), }; - if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() { + let cursor = state.command_cursor; + if let Err(error) = ctx.validate_scope_membership(&mut state, cursor) { + return Poll::Ready(Err(error)); + } + let recorded = match state.cancellation_replay_command(cursor, CancellationCallKind::Child) + { + Ok(recorded) => recorded, + Err(error) => return Poll::Ready(Err(error)), + }; + if let Some(recorded) = recorded { let sequence = recorded.sequence(); match recorded { RecordedCommand::ChildWorkflow { workflow_type, + policies, outcome, parallel_group_path, .. @@ -12475,8 +13685,32 @@ impl ChildWorkflowCall { ))); } } + if let Err(error) = + ensure_child_policies_match(sequence, &policies, &self.options) + { + return Poll::Ready(Err(error)); + } + if let Err(error) = + state.replay_cancellation_at(cursor, CancellationCallKind::Child) + { + return Poll::Ready(Err(error)); + } + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Child, + &self.parallel_group_path, + ) { + Ok(true) => { + self.scheduled = true; + self.matched_pending = true; + return Poll::Pending; + } + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } state.command_cursor += 1; if let Some(outcome) = outcome { + state.advance_cancellation_sequence(sequence, &self.parallel_group_path); return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed)); } state.matched_recorded_pending = true; @@ -12534,6 +13768,12 @@ impl ChildWorkflowCall { let object = command .as_object_mut() .expect("child workflow command is always an object"); + if self.options.cancellation_policy != CancellationPolicy::Abandon { + object.insert( + "cancellation_policy".to_string(), + json!(self.options.cancellation_policy.as_str()), + ); + } if let Some(policy) = &self.options.retry_policy { let mut retry_policy = serde_json::Map::new(); if let Some(max_attempts) = policy.max_attempts { @@ -12568,6 +13808,20 @@ impl ChildWorkflowCall { object.insert("run_timeout_seconds".to_string(), json!(seconds)); } apply_parallel_group_path(object, &self.parallel_group_path); + ctx.apply_scope_membership(object); + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Child, + &self.parallel_group_path, + ) { + Ok(true) => { + self.scheduled = true; + self.matched_pending = true; + return Poll::Pending; + } + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } state.commands.push(command); self.scheduled = true; } @@ -12635,7 +13889,16 @@ impl SignalCall { Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)), }; - if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() { + let cursor = state.command_cursor; + if let Err(error) = ctx.validate_scope_membership(&mut state, cursor) { + return Poll::Ready(Err(error)); + } + let recorded = match state.cancellation_replay_command(cursor, CancellationCallKind::Signal) + { + Ok(recorded) => recorded, + Err(error) => return Poll::Ready(Err(error)), + }; + if let Some(recorded) = recorded { match recorded { RecordedCommand::SignalWait { sequence, @@ -12662,8 +13925,27 @@ impl SignalCall { ))); } + if let Err(error) = + state.replay_cancellation_at(cursor, CancellationCallKind::Signal) + { + return Poll::Ready(Err(error)); + } + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Signal, + &self.parallel_group_path, + ) { + Ok(true) => { + self.opened_wait = true; + self.matched_pending = true; + return Poll::Pending; + } + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } state.command_cursor += 1; if let Some(value) = value { + state.advance_cancellation_sequence(sequence, &self.parallel_group_path); return Poll::Ready(Ok(value)); } if state @@ -12675,6 +13957,7 @@ impl SignalCall { .resume_signal .take() .expect("matching resume signal is present"); + state.unavailable_cancellation_boundary(&self.parallel_group_path); return Poll::Ready(Ok(signal.arguments)); } @@ -12692,6 +13975,20 @@ impl SignalCall { } } + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Signal, + &self.parallel_group_path, + ) { + Ok(true) => { + self.opened_wait = true; + self.matched_pending = true; + return Poll::Pending; + } + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } + if state .resume_signal .as_ref() @@ -12701,6 +13998,7 @@ impl SignalCall { .resume_signal .take() .expect("matching resume signal is present"); + state.unavailable_cancellation_boundary(&self.parallel_group_path); return Poll::Ready(Ok(signal.arguments)); } @@ -12710,6 +14008,7 @@ impl SignalCall { ("signal_name".to_string(), json!(self.signal_name)), ]); apply_parallel_group_path(&mut command, &self.parallel_group_path); + ctx.apply_scope_membership(&mut command); state.commands.push(Value::Object(command)); self.opened_wait = true; } @@ -12745,10 +14044,14 @@ pub struct ActivityContext { pub attempt_number: u64, pub task_queue: String, pub worker_id: String, + claim_guard: Option, } impl ActivityContext { pub async fn heartbeat(&self, details: T) -> Result { + if let Some(guard) = &self.claim_guard { + return guard.heartbeat(self, details).await; + } self.client .heartbeat_activity_task( &self.task_id, @@ -13269,6 +14572,7 @@ fn recorded_commands( let is_child_workflow = matches!( event.event_type.as_str(), "ChildWorkflowScheduled" + | "ChildRunStarted" | "ChildRunCompleted" | "ChildRunFailed" | "ChildRunCancelled" @@ -13499,6 +14803,7 @@ fn recorded_commands( return Ok(RecordedCommand::Activity { sequence, activity_type, + cancellation_policy: recorded_activity_cancellation_policy(&activity_events, sequence)?, options, outcome, parallel_group_path, @@ -13521,6 +14826,7 @@ fn recorded_commands( "child workflow replay requires exactly one recorded schedule event", )); } + let policies = recorded_child_policies(&child_events, scheduled[0], sequence)?; let workflow_type = child_events.iter().find_map(|event| { event .payload @@ -13578,6 +14884,7 @@ fn recorded_commands( return Ok(RecordedCommand::ChildWorkflow { sequence, workflow_type, + policies, outcome: outcomes.pop(), parallel_group_path, }); @@ -14766,6 +16073,19 @@ fn workflow_failure_command( "durable_workflow::WorkflowCancellationRequested", json!({"reason": "cancelled"}), ), + Error::CooperativeCancellationRequested(cancellation) => ( + "WorkflowCancellationRequested", + "durable_workflow::CooperativeCancellationRequested", + json!({"reason": "cancelled", "request_id": cancellation.request.request_id, + "cleanup_deadline_at": cancellation.request.cleanup_deadline_at}), + ), + Error::CancellationScopeRequested(cancellation) => ( + "WorkflowCancellationRequested", + "durable_workflow::CancellationScopeRequested", + json!({"reason":"scope_cancelled", "request_id":cancellation.context.request_id(), + "cleanup_deadline_at":cancellation.context.deadline().to_rfc3339(), + "cancellation":cancellation.context.to_value()}), + ), Error::NonDeterministicReplay(_) => ( "NonDeterministicReplay", "durable_workflow::Error", @@ -14780,7 +16100,9 @@ fn workflow_failure_command( Error::SagaCompensationFailed(failure) => { workflow_error_non_retryable(&failure.compensation_failure) } - Error::WorkflowCancellationRequested(_) => true, + Error::WorkflowCancellationRequested(_) + | Error::CooperativeCancellationRequested(_) + | Error::CancellationScopeRequested(_) => true, Error::NonDeterministicReplay(_) => true, _ => false, }; @@ -14821,7 +16143,9 @@ fn workflow_error_type(error: &Error) -> &'static str { }, Error::ParallelFailed(_) => "ParallelFailed", Error::SagaCompensationFailed(_) => "SagaCompensationFailed", - Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested", + Error::WorkflowCancellationRequested(_) + | Error::CooperativeCancellationRequested(_) + | Error::CancellationScopeRequested(_) => "WorkflowCancellationRequested", Error::NonDeterministicReplay(_) => "NonDeterministicReplay", _ => "RustWorkflowError", } @@ -14835,7 +16159,10 @@ fn workflow_error_non_retryable(error: &Error) -> bool { Error::SagaCompensationFailed(failure) => { workflow_error_non_retryable(&failure.compensation_failure) } - Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true, + Error::WorkflowCancellationRequested(_) + | Error::CooperativeCancellationRequested(_) + | Error::CancellationScopeRequested(_) + | Error::NonDeterministicReplay(_) => true, _ => false, } } @@ -14845,6 +16172,7 @@ fn workflow_task_integrity_error(error: &Error) -> bool { error, Error::NonDeterministicReplay(_) | Error::Protocol(_) + | Error::CancellationScopeExecutionUnavailable | Error::MissingWorkflowCommandIdentity | Error::WorkflowStatePoisoned ) @@ -15202,6 +16530,14 @@ fn value_as_u64(value: &Value) -> Option { #[cfg(test)] mod tests { use super::*; + mod activity_cancellation_policies; + mod cancellation_scope_admission; + mod cancellation_scope_authoring; + mod cancellation_scope_history; + mod cancellation_scope_opening; + mod cancellation_scope_replay; + mod child_workflow_policies; + mod cooperative_cancellation; mod runtime_payloads; mod runtime_uploads; use std::{ @@ -15561,6 +16897,7 @@ mod tests { payload_codec: &str, ) -> WorkflowContext { WorkflowContext { + cancellation_scope_id: "root".into(), state: Arc::new(Mutex::new( WorkflowState::new_with_identity( history, @@ -17200,6 +18537,7 @@ mod tests { state.workflow_command_identity = "command-7".to_string(); let context = WorkflowContext { state: Arc::new(Mutex::new(state)), + cancellation_scope_id: "root".into(), }; let item = WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt"); @@ -17238,6 +18576,7 @@ mod tests { replay_state.workflow_command_identity = "command-7".to_string(); let replay_context = WorkflowContext { state: Arc::new(Mutex::new(replay_state)), + cancellation_scope_id: "root".into(), }; replay_context .append_workflow_stream( @@ -18567,6 +19906,7 @@ mod tests { #[test] fn workflow_context_schedules_activity_until_completion_is_in_history() { let ctx = WorkflowContext { + cancellation_scope_id: "root".into(), state: Arc::new(Mutex::new( WorkflowState::new_with_identity( Vec::new(), @@ -21987,6 +23327,7 @@ mod tests { #[test] fn workflow_context_emits_explicit_child_workflow_contract() { let ctx = WorkflowContext { + cancellation_scope_id: "root".into(), state: Arc::new(Mutex::new( WorkflowState::new_with_identity( Vec::new(), @@ -22078,6 +23419,7 @@ mod tests { "child_workflow_instance_id": "wf-child", "child_workflow_run_id": "run-child", "child_workflow_type": "python.child", + "parent_close_policy": "terminate", }), raw: HashMap::new(), }, @@ -22301,6 +23643,7 @@ mod tests { }), ); task.workflow_type = "rust.handled-parent".to_string(); + task.history_events[0].payload["parent_close_policy"] = json!("abandon"); let commands = worker.execute_workflow_task(task).expect("handled failure"); assert_eq!(commands[0]["type"], "complete_workflow"); diff --git a/src/tests/activity_cancellation_policies.rs b/src/tests/activity_cancellation_policies.rs new file mode 100644 index 0000000..e133a16 --- /dev/null +++ b/src/tests/activity_cancellation_policies.rs @@ -0,0 +1,251 @@ +use super::*; + +fn activity_policy_worker( + policy: Option, + mode: &'static str, + enabled: bool, +) -> Worker { + let mut worker = Worker::new( + Client::builder("http://127.0.0.1:1").build().unwrap(), + "queue", + ) + .cooperative_cancellation(enabled); + worker.register_workflow("activity-policy", move |ctx, _| async move { + let mut options = ActivityOptions::new(); + if let Some(policy) = policy { + options = options + .cancellation_policy(policy) + .schedule_to_close_timeout(Duration::from_secs(60)); + } + match mode { + "parallel" => { + ctx.parallel(vec![ParallelOperation::activity_with_options( + "work", + options, + json!([]), + )]) + .await?; + } + "selection" => { + ctx.select_keyed(vec![( + SelectionKey::from("work"), + ParallelOperation::activity_with_options("work", options, json!([])), + )]) + .await?; + } + _ => { + ctx.activity_with_options("work", options, json!([])) + .await?; + } + } + Ok(json!("finished")) + }); + worker +} + +fn activity_policy_event(kind: &str, payload: Value) -> HistoryEvent { + serde_json::from_value(json!({"event_type": kind, "payload": payload})).unwrap() +} + +fn activity_policy_history(command: &Value) -> Vec { + let mut payload = command.clone(); + let object = payload.as_object_mut().unwrap(); + object.remove("type"); + object.remove("arguments"); + object.insert("sequence".into(), json!(1)); + let mut activity = json!({"type": "work"}); + if let Some(policy) = command.get("cancellation_policy") { + activity["cancellation_policy"] = policy.clone(); + } + object.insert("activity".into(), activity); + vec![activity_policy_event("ActivityScheduled", payload)] +} + +fn assert_activity_policy_failure(error: Error, reason: &str) { + let Error::NonDeterministicReplay(failure) = error else { + panic!("expected replay failure, got {error:?}"); + }; + assert_eq!(failure.reason, reason); + assert_eq!(failure.sequence, Some(1)); +} + +#[test] +fn changed_activity_cancellation_policy_is_rejected_during_replay() { + let mut worker = Worker::new( + Client::builder("http://127.0.0.1:1").build().unwrap(), + "queue", + ) + .cooperative_cancellation(true); + worker.register_workflow("activity-policy", |ctx, _| async move { + ctx.activity("work", json!([])).await?; + Ok(json!("finished")) + }); + let event = serde_json::from_value(json!({ + "event_type": "ActivityScheduled", + "payload": {"sequence": 1, "activity_type": "work", "activity": { + "type": "work", "cancellation_policy": "wait_cancellation_completed", + }}, + })) + .unwrap(); + let error = worker + .execute_workflow_task(workflow_task("activity-policy", vec![event], DEFAULT_CODEC)) + .expect_err("changing the original Activity policy must fail replay"); + let Error::NonDeterministicReplay(failure) = error else { + panic!("expected replay failure, got {error:?}"); + }; + assert_eq!(failure.reason, "activity_cancellation_policy_changed"); + assert_eq!(failure.sequence, Some(1)); +} + +#[test] +fn activity_policies_encode_and_replay_original_history_through_completion() { + for policy in [ + None, + Some(CancellationPolicy::TryCancel), + Some(CancellationPolicy::WaitCancellationCompleted), + Some(CancellationPolicy::Abandon), + ] { + let worker = activity_policy_worker(policy, "sequential", true); + let commands = worker + .execute_workflow_task(workflow_task("activity-policy", vec![], DEFAULT_CODEC)) + .unwrap(); + assert_eq!( + commands[0] + .get("cancellation_policy") + .and_then(Value::as_str), + policy.map(CancellationPolicy::as_str) + ); + let mut history = activity_policy_history(&commands[0]); + history.push(activity_policy_event( + "ActivityStarted", + json!({"sequence":1, "activity_type":"work"}), + )); + history.push(activity_policy_event( + "ActivityCompleted", + json!({"sequence":1, "activity_type":"work", + "result": fixture_envelope(json!("recorded"))}), + )); + let completed = worker + .execute_workflow_task(workflow_task( + "activity-policy", + history.clone(), + DEFAULT_CODEC, + )) + .unwrap(); + assert_eq!( + decode_wire_value(&completed[0]["result"], DEFAULT_CODEC).unwrap(), + json!("finished") + ); + let changed = activity_policy_worker( + Some(CancellationPolicy::WaitCancellationCompleted), + "sequential", + true, + ); + if policy != Some(CancellationPolicy::WaitCancellationCompleted) { + assert_activity_policy_failure( + changed + .execute_workflow_task(workflow_task("activity-policy", history, DEFAULT_CODEC)) + .unwrap_err(), + "activity_cancellation_policy_changed", + ); + } + } +} + +#[test] +fn activity_policies_match_ordinary_groups_and_cancellation_delivery() { + for mode in ["sequential", "parallel", "selection"] { + let original = activity_policy_worker( + Some(CancellationPolicy::WaitCancellationCompleted), + mode, + true, + ); + let commands = original + .execute_workflow_task(workflow_task("activity-policy", vec![], DEFAULT_CODEC)) + .unwrap(); + for delivered in [false, true] { + let mut history = activity_policy_history(&commands[0]); + if delivered { + history.push(serde_json::from_value(json!({ + "event_type":"CooperativeCancellationRequested", "recorded_at":"2026-10-01T00:00:00Z", + "payload":{"workflow_run_id":"run", "workflow_command_id":"request-1", "cleanup_deadline_at":"2026-10-01T00:00:30Z"}, + })).unwrap()); + history.push(activity_policy_event("CooperativeCancellationDelivered", json!({ + "workflow_run_id":"run", "workflow_command_id":"request-1", "sequence":1, + "call_kind":if mode == "sequential" {"activity"} else {"parallel"}, "sequence_span":1, + }))); + } + let mut task = workflow_task("activity-policy", history, DEFAULT_CODEC); + task.run_id = Some("run".into()); + original.execute_workflow_task(task.clone()).unwrap(); + let changed = activity_policy_worker(Some(CancellationPolicy::TryCancel), mode, true); + assert_activity_policy_failure( + changed.execute_workflow_task(task).unwrap_err(), + "activity_cancellation_policy_changed", + ); + } + } +} + +#[test] +fn malformed_and_conflicting_activity_policy_history_is_rejected() { + let worker = activity_policy_worker(None, "sequential", true); + for policy in [Value::Null, json!(false), json!([]), json!("unknown")] { + let history = vec![activity_policy_event( + "ActivityScheduled", + json!({"sequence":1, "activity_type":"work", + "activity":{"cancellation_policy":policy}}), + )]; + assert_activity_policy_failure( + worker + .execute_workflow_task(workflow_task("activity-policy", history, DEFAULT_CODEC)) + .unwrap_err(), + "invalid_activity_cancellation_policy_history", + ); + } + let commands = worker + .execute_workflow_task(workflow_task("activity-policy", vec![], DEFAULT_CODEC)) + .unwrap(); + let mut history = activity_policy_history(&commands[0]); + history.push(activity_policy_event( + "ActivityStarted", + json!({"sequence":1, "activity_type":"work", + "activity":{"cancellation_policy":"abandon"}}), + )); + assert_activity_policy_failure( + worker + .execute_workflow_task(workflow_task("activity-policy", history, DEFAULT_CODEC)) + .unwrap_err(), + "activity_cancellation_policy_history_conflict", + ); +} + +#[test] +fn explicit_activity_policies_require_worker_opt_in_and_abandon_requires_total_lifetime() { + for policy in [ + CancellationPolicy::TryCancel, + CancellationPolicy::WaitCancellationCompleted, + CancellationPolicy::Abandon, + ] { + let worker = activity_policy_worker(Some(policy), "sequential", false); + let error = worker + .execute_workflow_task(workflow_task("activity-policy", vec![], DEFAULT_CODEC)) + .unwrap_err(); + let Error::CooperativeCancellationUnavailable(message) = error else { + panic!("unexpected error {error:?}"); + }; + assert!(message.contains("activity_cancellation_policy_not_supported")); + assert!(message.contains("1.20")); + } + let absent = ActivityOptions::new() + .cancellation_policy(CancellationPolicy::Abandon) + .validate() + .unwrap_err(); + assert_eq!(absent.kind, ActivityOptionsErrorKind::MissingTotalTimeout); + let zero = ActivityOptions::new() + .cancellation_policy(CancellationPolicy::Abandon) + .schedule_to_close_timeout(Duration::ZERO) + .validate() + .unwrap_err(); + assert_eq!(zero.kind, ActivityOptionsErrorKind::TimeoutNotPositive); +} diff --git a/src/tests/cancellation_scope_admission.rs b/src/tests/cancellation_scope_admission.rs new file mode 100644 index 0000000..526b352 --- /dev/null +++ b/src/tests/cancellation_scope_admission.rs @@ -0,0 +1,206 @@ +use super::*; + +fn scope_event(kind: &str, payload: Value) -> HistoryEvent { + serde_json::from_value(json!({"event_type": kind, "payload": payload})).unwrap() +} + +fn scope_histories() -> Vec { + let mut events = [ + "CancellationScopeOpened", + "CancellationScopeRequested", + "CancellationScopeDeliveryPrepared", + "CancellationScopeDelivered", + "CancellationScopeRequestConflicted", + ] + .into_iter() + .map(|kind| scope_event(kind, json!({"scope_id": "scope-one", "sequence": 1}))) + .collect::>(); + for location in [ + None, + Some("activity"), + Some("timer"), + Some("child_workflow"), + ] { + let membership = json!({"cancellation_scope_id": "scope-one"}); + let payload = location.map_or_else(|| membership.clone(), |name| json!({name: membership})); + events.push(scope_event("TimerScheduled", payload)); + } + for malformed in [ + Value::Null, + json!(true), + json!(1), + json!(""), + json!([]), + json!({}), + ] { + events.push(scope_event( + "TimerScheduled", + json!({"cancellation_scope_id": malformed}), + )); + } + events +} + +fn scope_worker() -> Worker { + Worker::new(Client::new("http://127.0.0.1:1").unwrap(), "scope-queue").worker_id("scope-worker") +} + +#[test] +fn cancellation_scope_admission_precedes_the_application_factory() { + for event in scope_histories() { + let calls = Arc::new(AtomicUsize::new(0)); + let factory_calls = Arc::clone(&calls); + let mut worker = scope_worker(); + worker.register_workflow("scope-probe", move |_ctx, _input| { + factory_calls.fetch_add(1, Ordering::SeqCst); + async move { Ok(json!("unexpected")) } + }); + let error = worker + .execute_workflow_task(workflow_task("scope-probe", vec![event], DEFAULT_CODEC)) + .expect_err("unqualified scope history must not run application code"); + assert!(error + .to_string() + .contains("cancellation_scope_execution_not_supported")); + assert!(error.to_string().contains("Rust")); + assert_eq!(calls.load(Ordering::SeqCst), 0); + } +} + +#[tokio::test] +async fn cancellation_scope_admission_precedes_replayed_query_factories_and_handlers() { + let calls = Arc::new(AtomicUsize::new(0)); + let factory_calls = Arc::clone(&calls); + let body_calls = Arc::clone(&calls); + let query_calls = Arc::clone(&calls); + let mut worker = scope_worker(); + worker.register_replayed_workflow( + "scope-probe", + move || { + factory_calls.fetch_add(1, Ordering::SeqCst); + 0_u64 + }, + move |_ctx, _input, _state| { + body_calls.fetch_add(1, Ordering::SeqCst); + async move { Ok(Value::Null) } + }, + ); + worker.register_replayed_query::("scope-probe", "state", move |_, _, _| { + query_calls.fetch_add(1, Ordering::SeqCst); + async move { Ok(Value::Null) } + }); + let task = serde_json::from_value(json!({ + "query_task_id": "scope-query", "query_task_attempt": 1, + "workflow_type": "scope-probe", "query_name": "state", "payload_codec": "avro", + "workflow_arguments": fixture_envelope(json!([])), + "query_arguments": fixture_envelope(json!([])), + "history_events": [{"event_type": "CancellationScopeOpened", "payload": {"scope_id": "scope-one"}}], + "run_status": "running" + })).unwrap(); + let error = worker + .execute_query_task(task) + .await + .expect_err("scope query must refuse replay"); + assert!(error + .message + .contains("cancellation_scope_execution_not_supported")); + assert_eq!(calls.load(Ordering::SeqCst), 0); +} + +#[test] +fn cancellation_scope_admission_preserves_omitted_and_explicit_root_membership() { + for explicit_root in [false, true] { + let mut worker = scope_worker(); + worker.register_workflow("root-probe", |ctx, _| async move { + ctx.sleep(Duration::from_secs(1)).await?; + Ok(json!("done")) + }); + let mut payload = json!({"sequence": 1, "timer_id": "timer-one", "delay_seconds": 1}); + if explicit_root { + payload["cancellation_scope_id"] = json!("root"); + } + let history = vec![ + scope_event("TimerScheduled", payload.clone()), + scope_event("TimerFired", payload), + ]; + let commands = worker + .execute_workflow_task(workflow_task("root-probe", history, DEFAULT_CODEC)) + .unwrap(); + assert_eq!(commands.len(), 1); + assert_eq!(commands[0]["type"], "complete_workflow"); + assert_eq!( + decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).unwrap(), + json!("done") + ); + } +} + +#[test] +fn cancellation_scope_admission_preserves_scope_named_application_data() { + let value = json!({"cancellation_scope_id": "application-value", "activity": {"cancellation_scope_id": null}}); + let history = vec![scope_event( + "SideEffectRecorded", + json!({"sequence": 1, "result": fixture_envelope(value.clone())}), + )]; + let mut worker = scope_worker(); + worker.register_workflow("scope-data", |ctx, _| async move { + let recorded: Value = + ctx.side_effect(|| panic!("recorded side effect must not execute"))?; + Ok(recorded) + }); + let commands = worker + .execute_workflow_task(workflow_task("scope-data", history, DEFAULT_CODEC)) + .unwrap(); + assert_eq!( + decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).unwrap(), + value + ); +} + +#[tokio::test] +async fn cancellation_scope_admission_publishes_no_completion_or_task_failure() { + let server = MockWorkerServer::start(); + let worker = Worker::new(Client::new(server.base_url()).unwrap(), "scope-queue"); + let error = worker + .settle_workflow_task_decision( + "scope-task", + "scope-worker", + 2, + Some("scope-run"), + Err(Error::CancellationScopeExecutionUnavailable), + true, + ) + .await + .expect_err("unsupported scope must retain an explicit capability refusal"); + assert!(matches!( + error, + Error::CancellationScopeExecutionUnavailable + )); + assert!(server.requests.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn cancellation_scope_admission_keeps_explicit_snapshot_inspection_available() { + let mut worker = scope_worker(); + worker.register_workflow("scope-probe", |_, _| async move { + panic!("snapshot inspection must not replay the workflow") + }); + worker.register_query("scope-probe", "inspect", |ctx, _| async move { + Ok(json!(ctx.history_events[0].event_type)) + }); + let task = serde_json::from_value(json!({ + "query_task_id": "scope-inspect", "query_task_attempt": 1, + "workflow_type": "scope-probe", "query_name": "inspect", "payload_codec": "avro", + "workflow_arguments": fixture_envelope(json!([])), "query_arguments": fixture_envelope(json!([])), + "history_events": [{"event_type": "CancellationScopeOpened", "payload": {"scope_id": "scope-one"}}], + "run_status": "running" + })).unwrap(); + assert_eq!( + worker + .execute_query_task(task) + .await + .unwrap() + .into_json() + .unwrap(), + json!("CancellationScopeOpened") + ); +} diff --git a/src/tests/cancellation_scope_authoring.rs b/src/tests/cancellation_scope_authoring.rs new file mode 100644 index 0000000..1a29886 --- /dev/null +++ b/src/tests/cancellation_scope_authoring.rs @@ -0,0 +1,377 @@ +use super::*; + +fn event(kind: &str, payload: Value, index: u64) -> HistoryEvent { + serde_json::from_value(json!({"id": format!("event-{index}"), "sequence": index, + "namespace":"tenant", "event_type":kind, "payload":payload})) + .unwrap() +} + +fn started() -> HistoryEvent { + event("WorkflowStarted", json!({}), 1) +} + +fn opening(sequence: u64, id: &str, parent: &str, shield: bool, index: u64) -> HistoryEvent { + event( + "CancellationScopeOpened", + json!({"schema":"durable-workflow.cancellation-scope/v1", + "workflow_run_id":"run-one", "sequence":sequence, "scope_id":id, "parent_scope_id":parent, + "shield_parent":shield}), + index, + ) +} + +fn context(history: Vec) -> WorkflowContext { + WorkflowContext { + cancellation_scope_id: "root".into(), + state: Arc::new(Mutex::new( + WorkflowState::new_with_identity_and_scopes( + history, + Some("workflow-one".into()), + Some("run-one".into()), + "queue".into(), + DEFAULT_CODEC.into(), + None, + true, + ) + .unwrap(), + )), + } +} + +fn poll(future: &mut F) -> Poll { + let mut cx = TaskContext::from_waker(noop_waker_ref()); + Pin::new(future).poll(&mut cx) +} + +#[test] +fn cancellation_scope_authoring_body_waits_for_canonical_opening() { + let ctx = context(vec![started()]); + let calls = Arc::new(AtomicUsize::new(0)); + let body_calls = Arc::clone(&calls); + let mut scope = Box::pin(ctx.cancellation_scope(false, move |_| async move { + body_calls.fetch_add(1, Ordering::SeqCst); + Ok("done") + })); + assert!(poll(&mut scope).is_pending()); + assert_eq!(calls.load(Ordering::SeqCst), 0); + let state = ctx.state.lock().unwrap(); + let intent = state.cancellation_scope_opening.as_ref().unwrap(); + assert_eq!(intent.sequence, 1); + assert_eq!(intent.parent_scope_id, "root"); + assert!(!intent.shield_parent); + assert_eq!(intent.command_count, 0); + assert!(state.commands.is_empty()); +} + +#[test] +fn cancellation_scope_authoring_default_context_refuses_before_body() { + let ctx = workflow_context(Vec::new()); + let mut scope = Box::pin(ctx.cancellation_scope(false, |_| async move { + panic!("unqualified scope cannot enter body"); + #[allow(unreachable_code)] + Ok(()) + })); + assert!(matches!( + poll(&mut scope), + Poll::Ready(Err(Error::CancellationScopeExecutionUnavailable)) + )); +} + +#[test] +fn cancellation_scope_authoring_nested_context_and_deferred_timers_retain_membership() { + let ctx = context(vec![ + started(), + opening(1, "outer", "root", false, 2), + opening(2, "inner", "outer", true, 3), + ]); + let mut scope = Box::pin(ctx.cancellation_scope(false, |outer| async move { + let deferred = outer + .cancellation_scope(true, |inner| async move { + Ok(inner.sleep(Duration::from_secs(1))) + }) + .await?; + Ok((deferred, outer.sleep(Duration::from_secs(2)))) + })); + let Poll::Ready(Ok((mut inner, mut outer))) = poll(&mut scope) else { + panic!("canonical body must resume"); + }; + let mut root = ctx.sleep(Duration::from_secs(3)); + assert!(poll(&mut inner).is_pending()); + assert!(poll(&mut outer).is_pending()); + assert!(poll(&mut root).is_pending()); + let wire = ctx.take_commands().unwrap(); + assert_eq!(wire.len(), 3); + assert_eq!(wire[0]["cancellation_scope_id"], "inner"); + assert_eq!(wire[1]["cancellation_scope_id"], "outer"); + assert!(wire[2].get("cancellation_scope_id").is_none()); + assert_eq!(ctx.cancellation_scope_id, "root"); +} + +#[test] +fn cancellation_scope_authoring_prefix_does_not_enter_body_and_keeps_sequence() { + let ctx = context(vec![started()]); + assert_eq!(ctx.side_effect(|| "prefix".to_string()).unwrap(), "prefix"); + let mut scope = Box::pin(ctx.cancellation_scope(false, |_| async move { + panic!("opening must commit after the prefix"); + #[allow(unreachable_code)] + Ok(()) + })); + assert!(poll(&mut scope).is_pending()); + let state = ctx.state.lock().unwrap(); + assert_eq!( + state.cancellation_scope_opening.as_ref().unwrap().sequence, + 2 + ); + assert_eq!( + state + .cancellation_scope_opening + .as_ref() + .unwrap() + .command_count, + 1 + ); +} + +#[test] +fn cancellation_scope_authoring_changed_parent_shield_or_position_cannot_enter_body() { + for (parent, shield, sequence) in [("root", true, 1), ("root", false, 2)] { + let ctx = context(vec![ + started(), + opening(sequence, "outer", parent, shield, 2), + ]); + let mut scope = Box::pin(ctx.cancellation_scope(false, |_| async move { + panic!("changed opening cannot enter body"); + #[allow(unreachable_code)] + Ok(()) + })); + assert!( + matches!(poll(&mut scope), Poll::Ready(Err(Error::NonDeterministicReplay(f))) + if f.reason == "cancellation_scope_opening_changed") + ); + } + let ctx = context(vec![ + started(), + opening(1, "outer", "root", false, 2), + opening(2, "inner", "root", false, 3), + ]); + let mut scope = Box::pin(ctx.cancellation_scope(false, |outer| async move { + outer + .cancellation_scope(false, |_| async move { + panic!("changed nested parent cannot enter body"); + #[allow(unreachable_code)] + Ok(()) + }) + .await + })); + assert!( + matches!(poll(&mut scope), Poll::Ready(Err(Error::NonDeterministicReplay(f))) + if f.reason == "cancellation_scope_opening_changed") + ); +} + +#[test] +fn cancellation_scope_authoring_invalid_tree_and_operation_membership_refuse_before_factory() { + let base = opening(1, "outer", "root", false, 2); + let mutations = [ + ("scope_id", json!("root")), + ("scope_id", json!("")), + ("scope_id", json!(true)), + ("scope_id", json!("é".repeat(128))), + ("parent_scope_id", json!("missing")), + ("parent_scope_id", json!("outer")), + ("workflow_run_id", json!("foreign")), + ("schema", json!("foreign")), + ("sequence", json!(true)), + ("sequence", json!(0)), + ("sequence", json!(u64::MAX)), + ("shield_parent", json!(1)), + ]; + for (field, value) in mutations { + let mut row = base.clone(); + row.payload[field] = value; + let calls = Arc::new(AtomicUsize::new(0)); + let factory_calls = Arc::clone(&calls); + let mut worker = Worker::new(Client::new("http://127.0.0.1:1").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true); + worker.register_workflow("scope-probe", move |_, _| { + factory_calls.fetch_add(1, Ordering::SeqCst); + async move { Ok(json!("unexpected")) } + }); + let mut task = workflow_task("scope-probe", vec![started(), row], DEFAULT_CODEC); + task.run_id = Some("run-one".into()); + assert!(matches!( + worker.execute_workflow_task(task), + Err(Error::NonDeterministicReplay(_)) + )); + assert_eq!(calls.load(Ordering::SeqCst), 0); + } + for payload in [ + json!({"sequence":2,"cancellation_scope_id":"missing"}), + json!({"sequence":1,"cancellation_scope_id":"outer"}), + json!({"sequence":true,"cancellation_scope_id":"outer"}), + json!({"sequence":2,"cancellation_scope_id":null}), + json!({"sequence":2,"cancellation_scope_id":"outer","timer":{"cancellation_scope_id":"root"}}), + ] { + assert!(cancellation_scope::CancellationScopeHistory::read( + &[started(), base.clone(), event("TimerScheduled", payload, 3)], + "run-one" + ) + .is_err()); + } +} + +#[test] +fn cancellation_scope_authoring_timer_replay_checks_membership_and_consumes_opening() { + let history = vec![ + started(), + opening(1, "outer", "root", false, 2), + event( + "TimerScheduled", + json!({"sequence":2,"timer_id":"timer-one","delay_seconds":1,"cancellation_scope_id":"outer"}), + 3, + ), + event( + "TimerFired", + json!({"sequence":2,"timer_id":"timer-one","delay_seconds":1}), + 4, + ), + ]; + for root_command in [false, true] { + let ctx = context(history.clone()); + let mut scope = Box::pin(ctx.cancellation_scope(false, |scoped| async move { + Ok(scoped.sleep(Duration::from_secs(1))) + })); + let Poll::Ready(Ok(mut timer)) = poll(&mut scope) else { + panic!("scope must replay"); + }; + if root_command { + timer = ctx.sleep(Duration::from_secs(1)); + } + let actual = poll(&mut timer); + if root_command { + assert!( + matches!(actual,Poll::Ready(Err(Error::NonDeterministicReplay(f))) + if f.reason == "cancellation_scope_membership_changed") + ); + } else { + assert!(matches!(actual, Poll::Ready(Ok(())))); + ctx.ensure_history_consumed().unwrap(); + } + } +} + +#[test] +fn cancellation_scope_authoring_unimplemented_delivery_and_closed_history_refuse() { + for kind in [ + "CancellationScopeRequested", + "CancellationScopeDeliveryPrepared", + "CancellationScopeDelivered", + "CancellationScopeRequestConflicted", + ] { + assert!(matches!( + WorkflowState::new_with_identity_and_scopes( + vec![ + started(), + opening(1, "outer", "root", false, 2), + event(kind, json!({}), 3) + ], + None, + Some("run-one".into()), + "queue".into(), + DEFAULT_CODEC.into(), + None, + true + ), + Err(Error::CancellationScopeExecutionUnavailable) + )); + } + for kind in [ + "WorkflowCompleted", + "WorkflowFailed", + "WorkflowCancelled", + "WorkflowTerminated", + ] { + let ctx = context(vec![started(), event(kind, json!({}), 2)]); + let mut scope = Box::pin(ctx.cancellation_scope(false, |_| async move { + panic!("closed history cannot enter new scope"); + #[allow(unreachable_code)] + Ok(()) + })); + assert!( + matches!(poll(&mut scope),Poll::Ready(Err(Error::NonDeterministicReplay(f))) + if f.reason == "cancellation_scope_opening_changed") + ); + } +} + +#[test] +fn cancellation_scope_authoring_all_supported_operation_futures_keep_creation_context() { + let ctx = context(vec![started(), opening(1, "outer", "root", false, 2)]); + let mut scope = Box::pin(ctx.cancellation_scope(false, |scoped| async move { Ok(scoped) })); + let Poll::Ready(Ok(scoped)) = poll(&mut scope) else { + panic!("scope must replay"); + }; + let mut activity = scoped.activity("remote", json!([])); + let mut child = + scoped.start_child_workflow("child", ChildWorkflowOptions::new("queue"), json!([])); + let mut signal = scoped.wait_signal("resume"); + let mut condition = scoped + .wait_condition(ConditionWaitOptions::new("ready", "sha256:ready"), || { + Ok(false) + }); + assert!(poll(&mut activity).is_pending()); + assert!(poll(&mut child).is_pending()); + assert!(poll(&mut signal).is_pending()); + assert!(poll(&mut condition).is_pending()); + let wire = ctx.take_commands().unwrap(); + assert_eq!(wire.len(), 4); + for command in wire { + assert_eq!(command["cancellation_scope_id"], "outer"); + } + assert_eq!(ctx.cancellation_scope_id, "root"); +} + +#[test] +fn cancellation_scope_authoring_worker_cannot_publish_after_ignoring_pending_opening() { + let mut worker = Worker::new(Client::new("http://127.0.0.1:1").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true); + worker.register_workflow("scope-probe", |ctx, _| async move { + let mut opening = Box::pin(ctx.cancellation_scope(false, |_| async move { + panic!("uncommitted scope cannot enter body"); + #[allow(unreachable_code)] + Ok(()) + })); + std::future::poll_fn(|cx| { + assert!(opening.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + ctx.side_effect(|| "escaped".to_string())?; + Ok(json!("must not publish")) + }); + let mut task = workflow_task("scope-probe", vec![started()], DEFAULT_CODEC); + task.run_id = Some("run-one".into()); + assert!( + matches!(worker.execute_workflow_task(task),Err(Error::NonDeterministicReplay(f)) + if f.reason == "cancellation_scope_pending_call_escaped") + ); +} + +#[test] +fn cancellation_scope_authoring_cannot_enable_protocol_or_delivery_implicitly() { + let calls = Arc::new(AtomicUsize::new(0)); + let factory_calls = Arc::clone(&calls); + let mut worker = Worker::new(Client::new("http://127.0.0.1:1").unwrap(), "queue") + .candidate_cancellation_scope_authoring(true); + worker.register_workflow("scope-probe", move |_, _| { + factory_calls.fetch_add(1, Ordering::SeqCst); + async move { Ok(json!("unexpected")) } + }); + assert!(matches!( + worker.execute_workflow_task(workflow_task("scope-probe", vec![started()], DEFAULT_CODEC)), + Err(Error::CancellationScopeExecutionUnavailable) + )); + assert_eq!(calls.load(Ordering::SeqCst), 0); +} diff --git a/src/tests/cancellation_scope_history.rs b/src/tests/cancellation_scope_history.rs new file mode 100644 index 0000000..6157c29 --- /dev/null +++ b/src/tests/cancellation_scope_history.rs @@ -0,0 +1,995 @@ +use super::*; +use crate::cancellation_scope_history::CommittedCancellationScopeHistory; + +fn scalar_fixture() -> Value { + fixture("empty", "unshielded") +} + +fn boundary(value: &Value) -> (ScopedCancellationContext, CancellationDelivery) { + let committed = read(value).unwrap(); + let prepared = committed.preparations.values().next().unwrap(); + (prepared.context.clone(), prepared.boundary.clone()) +} + +fn claim(value: &Value) -> WorkflowTask { + let mut events = value["history"].as_array().unwrap().clone(); + let position = events + .iter() + .position(|row| row["event_type"] == "CancellationScopeDeliveryPrepared") + .unwrap(); + events.truncate(position); + serde_json::from_value(json!({"task_id":"task/one", "run_id":value["task"]["run_id"], + "workflow_id":value["task"]["workflow_id"], "workflow_type":"scope", "lease_owner":"original", + "workflow_task_attempt":4, "payload_codec":DEFAULT_CODEC, "history_events":events})).unwrap() +} + +fn scope_response(path: &str, body: &str, number: usize) -> Option<(&'static str, String)> { + let case = path.split('/').nth(1).unwrap_or_default(); + if case == "budget" { + thread::sleep(Duration::from_millis(900)); + } + if matches!(case, "lost-ack" | "worker-lost-ack") && path.ends_with("/prepare") && number == 1 { + return Some(("invalid-status", String::new())); + } + let mut source = match case { + "group-flat" => fixture("groups", "flat"), + "group-nested" => fixture("groups", "nested"), + _ => scalar_fixture(), + }; + if case.starts_with("worker") { + // Synthetic transport time is safely ahead of the actual request budget. + // The fixture has no admitted members whose descriptor contains dates. + source = + serde_json::from_str(&source.to_string().replace("2026-10-04", "2029-10-04")).unwrap(); + if path.ends_with("/poll") { + let task = claim(&source); + return Some(("200 OK", json!({"protocol_version":"1.20", "task":{ + "task_id":task.task_id, "run_id":task.run_id, "workflow_id":task.workflow_id, + "workflow_type":task.workflow_type, "lease_owner":task.lease_owner, + "workflow_task_attempt":task.workflow_task_attempt, "payload_codec":DEFAULT_CODEC, + "history_events":source["history"].as_array().unwrap().iter() + .take_while(|event| event["event_type"] != "CancellationScopeDeliveryPrepared") + .cloned().collect::>() + }}).to_string())); + } + if path.ends_with("/complete") { + return Some(("200 OK", "{}".into())); + } + } + let delivering = + path.ends_with("/deliver") || (path.ends_with("/history") && body.contains("deliver-")); + if case == "substituted" && delivering { + event_mut(&mut source, "CancellationScopeDeliveryPrepared")["id"] = + json!("borrowed-preparation"); + event_mut(&mut source, "CancellationScopeDelivered")["payload"] + ["preparation_history_event_id"] = json!("borrowed-preparation"); + } + let mut prepared = event_mut(&mut source, "CancellationScopeDeliveryPrepared").clone(); + let delivered = event_mut(&mut source, "CancellationScopeDelivered").clone(); + let mut result; + if path.ends_with("/prepare") || path.ends_with("/deliver") { + result = prepared["payload"].clone(); + result["task_id"] = json!("task/one"); + result["lease_owner"] = json!("original"); + result["workflow_task_attempt"] = json!(4); + result["prepared"] = json!(true); + result["delivered"] = json!(delivering); + result["claim_released"] = json!(false); + result["created_task_ids"] = json!([]); + result["reason"] = Value::Null; + result["preparation_history_event_id"] = prepared["id"].clone(); + result["history_event_id"] = if delivering { + delivered["id"].clone() + } else { + prepared["id"].clone() + }; + result["history_refresh_page_token"] = json!(if delivering { + "deliver-start" + } else { + "prepare-start" + }); + if case.starts_with("pending-") + && path.ends_with("/deliver") + && (number == 1 || case == "pending-budget") + { + result["delivered"] = json!(false); + result["reason"] = json!("cancellation_scope_activity_stop_not_acknowledged"); + result.as_object_mut().unwrap().remove("history_event_id"); + result["history_refresh_page_token"] = json!("prepare-start"); + match case { + "pending-owner" => result["lease_owner"] = json!("replacement"), + "pending-preparation" => result["preparation_history_event_id"] = json!("borrowed"), + "pending-deadline" => { + result["authority_deadline_at"] = json!("2026-10-04T00:00:31.123456Z") + } + "pending-delivery" => result["history_event_id"] = delivered["id"].clone(), + _ => {} + } + } + match case { + "ack-owner" => result["lease_owner"] = json!("replacement"), + "ack-attempt" => result["workflow_task_attempt"] = json!(true), + "ack-run" => result["workflow_run_id"] = json!("foreign"), + "ack-boundary" => result["sequence"] = json!(99), + "ack-range" => result["sequence_span"] = json!(2), + "ack-prepared" => result["prepared"] = json!(1), + "ack-delivered" => result["delivered"] = json!(!delivering), + "ack-released" => result["claim_released"] = json!(true), + "ack-new-task" => result["created_task_ids"] = json!(["new"]), + "ack-deadline" => { + result["authority_deadline_at"] = json!("2026-10-04T00:00:31.123456Z") + } + "ack-token" => result["history_refresh_page_token"] = Value::Null, + "ack-event" => result["history_event_id"] = json!(""), + "ack-reason" => { + result.as_object_mut().unwrap().remove("reason"); + } + _ => {} + } + } else if path.ends_with("/history") { + let events = source["history"].as_array_mut().unwrap(); + if !delivering { + events.pop(); + } + match case { + "history-namespace" => events[0]["namespace"] = json!("foreign"), + "history-start" => events[0]["payload"]["workflow_instance_id"] = json!("foreign"), + "history-context" => { + events[4]["payload"]["cancellation"]["root_context"]["reason"] = json!("changed") + } + "history-range" => { + prepared["payload"] + .as_object_mut() + .unwrap() + .remove("operation_sequence_span"); + *events.last_mut().unwrap() = prepared; + } + "history-preparation" => { + events.pop(); + } + _ => {} + } + let request: Value = serde_json::from_str(body).unwrap(); + let first = request["next_history_page_token"] + .as_str() + .unwrap() + .ends_with("start"); + let prefix = if delivering { "deliver" } else { "prepare" }; + result = json!({"task_id":"task/one", "workflow_task_attempt":4, + "history_events": if first { events.iter().take(2).cloned().collect::>() } + else { events.iter().skip(2).cloned().collect::>() }, + "next_history_page_token": if first { json!(format!("{prefix}-next")) } else { Value::Null }}); + match case { + "page-task" => result["task_id"] = json!("foreign"), + "page-attempt" => result["workflow_task_attempt"] = json!(5), + "page-cycle" => { + result["next_history_page_token"] = request["next_history_page_token"].clone() + } + "page-terminal" => { + result + .as_object_mut() + .unwrap() + .remove("next_history_page_token"); + } + _ => {} + } + } else { + return None; + } + Some(("200 OK", result.to_string())) +} + +fn scope_server() -> MockWorkerServer { + MockWorkerServer::start_with_behavior(MockWorkerBehavior { + request_override: Some(scope_response), + ..MockWorkerBehavior::default() + }) +} + +fn scope_client(server: &MockWorkerServer, case: &str) -> Client { + Client::builder(format!("{}/{case}", server.base_url())) + .namespace("sdk-scope-fixture") + .control_token(Some("control-only".into())) + .worker_token(Some("worker-only".into())) + .build() + .unwrap() +} + +#[tokio::test] +async fn scope_history_waits_for_callback_stop_without_changing_claim_or_preparation() { + let server = scope_server(); + let client = scope_client(&server, "pending-stop"); + let value = scalar_fixture(); + let task = claim(&value); + let (context, boundary) = boundary(&value); + let budget = CancellationScopeDeliveryBudget::new(); + let prepared = client + .prepare_cancellation_scope_on_claim(&task, &context, &boundary, &budget) + .await + .unwrap(); + let delivered = client + .deliver_cancellation_scope_on_claim(&task, &prepared, &budget) + .await + .unwrap(); + assert_eq!( + delivered.preparation_history_event_id(), + prepared.preparation_history_event_id() + ); + assert_eq!(delivered.context(), prepared.context()); + assert_eq!(delivered.boundary(), prepared.boundary()); + assert!(delivered.delivery_history_event_id().is_some()); + let requests = server.requests.lock().unwrap(); + let mutations: Vec<_> = requests + .iter() + .filter(|r| r.path.ends_with("/deliver")) + .collect(); + assert_eq!(mutations.len(), 2); + assert_eq!(mutations[0].body, mutations[1].body); + assert_eq!( + requests + .iter() + .filter(|r| r.path.ends_with("/history")) + .count(), + 6 + ); +} + +#[tokio::test] +async fn scope_history_pending_stop_cannot_change_authority_or_invent_delivery() { + for case in [ + "pending-owner", + "pending-preparation", + "pending-deadline", + "pending-delivery", + ] { + let server = scope_server(); + let client = scope_client(&server, case); + let value = scalar_fixture(); + let task = claim(&value); + let (context, boundary) = boundary(&value); + let budget = CancellationScopeDeliveryBudget::new(); + let prepared = client + .prepare_cancellation_scope_on_claim(&task, &context, &boundary, &budget) + .await + .unwrap(); + assert!( + client + .deliver_cancellation_scope_on_claim(&task, &prepared, &budget) + .await + .is_err(), + "{case}" + ); + assert_eq!( + server + .requests + .lock() + .unwrap() + .iter() + .filter(|r| r.path.ends_with("/deliver")) + .count(), + 1, + "{case}" + ); + } +} + +#[tokio::test] +async fn scope_history_pending_stop_keeps_original_budget() { + let server = scope_server(); + let client = scope_client(&server, "pending-budget"); + let value = scalar_fixture(); + let task = claim(&value); + let (context, boundary) = boundary(&value); + let mut budget = CancellationScopeDeliveryBudget::new(); + let prepared = client + .prepare_cancellation_scope_on_claim(&task, &context, &boundary, &budget) + .await + .unwrap(); + budget + .restrict( + DateTime::::from_timestamp_millis( + i64::try_from( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap() + + 250, + ) + .unwrap(), + ) + .unwrap(); + let started = Instant::now(); + assert!(matches!( + client + .deliver_cancellation_scope_on_claim(&task, &prepared, &budget) + .await, + Err(Error::Timeout) + )); + assert!(started.elapsed() < Duration::from_secs(1)); + let requests = server.requests.lock().unwrap(); + let mutations: Vec<_> = requests + .iter() + .filter(|r| r.path.ends_with("/deliver")) + .collect(); + assert!(mutations.len() >= 2); + assert!(mutations.iter().all(|r| r.body == mutations[0].body)); +} + +#[tokio::test] +async fn cancellation_scope_replay_worker_coordinates_original_claim_and_lost_ack() { + for case in ["worker", "worker-lost-ack"] { + let server = scope_server(); + let cleanup = Arc::new(AtomicUsize::new(0)); + let observed = Arc::clone(&cleanup); + let mut worker = Worker::new(scope_client(&server, case), "queue") + .worker_id("original") + .poll_timeout(Duration::ZERO) + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + worker.register_workflow("scope", move |ctx, _| { + let observed = Arc::clone(&observed); + async move { + ctx.cancellation_scope(false, move |outer| async move { + outer + .cancellation_scope(false, move |inner| async move { + match inner.sleep(Duration::from_secs(3600)).await { + Err(Error::CancellationScopeRequested(cancellation)) => { + observed.fetch_add(1, Ordering::SeqCst); + assert_eq!( + cancellation.context.remaining()?, + Duration::from_micros(22_623_456) + ); + let _shield = inner.cancellation_shield()?; + inner.sleep(Duration::from_secs(1)).await?; + Ok(Value::Null) + } + Err(error) => Err(error), + Ok(()) => panic!("cancelled timer cannot complete ordinarily"), + } + }) + .await + }) + .await + } + }); + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + assert_eq!(cleanup.load(Ordering::SeqCst), 1); + let requests = server.requests.lock().unwrap(); + let preparations: Vec<_> = requests + .iter() + .filter(|request| request.path.ends_with("/prepare")) + .collect(); + assert_eq!( + preparations.len(), + if case == "worker-lost-ack" { 2 } else { 1 } + ); + if preparations.len() == 2 { + assert_eq!(preparations[0].body, preparations[1].body); + } + assert_eq!( + requests + .iter() + .filter(|request| request.path.ends_with("/deliver")) + .count(), + 1 + ); + assert_eq!( + requests + .iter() + .filter(|request| request.path.ends_with("/history")) + .count(), + 4 + ); + let completion = requests.last().unwrap(); + assert_eq!( + completion.path, + format!("/{case}/api/worker/workflow-tasks/task/one/complete") + ); + let body: Value = serde_json::from_str(&completion.body).unwrap(); + assert_eq!(body["workflow_task_attempt"], 4); + assert_eq!(body["lease_owner"], "original"); + assert_eq!(body["commands"][0]["type"], "start_timer"); + assert_eq!(body["commands"][0]["delay_seconds"], 1); + } +} + +#[tokio::test] +async fn scope_history_prepare_and_delivery_prove_original_claim_and_all_pages_after_lost_ack() { + for case in ["valid", "lost-ack", "group-flat", "group-nested"] { + let server = scope_server(); + let client = scope_client(&server, case); + let value = match case { + "group-flat" => fixture("groups", "flat"), + "group-nested" => fixture("groups", "nested"), + _ => scalar_fixture(), + }; + let task = claim(&value); + let (context, boundary) = boundary(&value); + let budget = CancellationScopeDeliveryBudget::new(); + let prepared = client + .prepare_cancellation_scope_on_claim(&task, &context, &boundary, &budget) + .await + .unwrap(); + assert!(prepared.delivery_history_event_id().is_none()); + let delivered = client + .deliver_cancellation_scope_on_claim(&task, &prepared, &budget) + .await + .unwrap(); + assert_eq!( + prepared.preparation_history_event_id(), + delivered.preparation_history_event_id() + ); + assert_ne!( + delivered.delivery_history_event_id().unwrap(), + delivered.preparation_history_event_id() + ); + assert_eq!(delivered.context(), &context); + assert_eq!(delivered.boundary(), &boundary); + assert_eq!( + delivered.history().len(), + value["history"].as_array().unwrap().len() + ); + let requests = server.requests.lock().unwrap(); + let preparations: Vec<_> = requests + .iter() + .filter(|row| row.path.ends_with("/prepare")) + .collect(); + assert_eq!(preparations.len(), if case == "lost-ack" { 2 } else { 1 }); + if case == "lost-ack" { + assert_eq!(preparations[0].body, preparations[1].body); + } + for request in requests.iter() { + assert_eq!(request.worker_protocol.as_deref(), Some("1.20")); + assert_eq!(request.authorization.as_deref(), Some("Bearer worker-only")); + assert_eq!(request.namespace.as_deref(), Some("sdk-scope-fixture")); + assert!(request.path.contains("/task%2Fone/")); + let body: Value = serde_json::from_str(&request.body).unwrap(); + assert_eq!(body["lease_owner"], "original"); + assert_eq!(body["workflow_task_attempt"], 4); + } + } +} + +#[tokio::test] +async fn scope_history_changed_receipts_refuse_before_history_reads() { + for case in [ + "ack-owner", + "ack-attempt", + "ack-run", + "ack-boundary", + "ack-range", + "ack-prepared", + "ack-delivered", + "ack-released", + "ack-new-task", + "ack-deadline", + "ack-token", + "ack-event", + "ack-reason", + ] { + let server = scope_server(); + let value = scalar_fixture(); + let (context, boundary) = boundary(&value); + assert!( + scope_client(&server, case) + .prepare_cancellation_scope_on_claim( + &claim(&value), + &context, + &boundary, + &CancellationScopeDeliveryBudget::new() + ) + .await + .is_err(), + "{case}" + ); + assert_eq!(server.requests.lock().unwrap().len(), 1, "{case}"); + } +} + +#[tokio::test] +async fn scope_history_incomplete_or_changed_canonical_pages_refuse_delivery_authority() { + for case in [ + "history-namespace", + "history-start", + "history-context", + "history-range", + "history-preparation", + "page-task", + "page-attempt", + "page-cycle", + "page-terminal", + ] { + let server = scope_server(); + let value = scalar_fixture(); + let (context, boundary) = boundary(&value); + assert!( + scope_client(&server, case) + .prepare_cancellation_scope_on_claim( + &claim(&value), + &context, + &boundary, + &CancellationScopeDeliveryBudget::new() + ) + .await + .is_err(), + "{case}" + ); + assert!(server + .requests + .lock() + .unwrap() + .iter() + .all(|request| !request.path.ends_with("/deliver"))); + } +} + +#[tokio::test] +async fn scope_history_delivery_cannot_substitute_a_different_valid_preparation() { + let server = scope_server(); + let client = scope_client(&server, "substituted"); + let value = scalar_fixture(); + let task = claim(&value); + let (context, boundary) = boundary(&value); + let budget = CancellationScopeDeliveryBudget::new(); + let prepared = client + .prepare_cancellation_scope_on_claim(&task, &context, &boundary, &budget) + .await + .unwrap(); + assert!(client + .deliver_cancellation_scope_on_claim(&task, &prepared, &budget) + .await + .is_err()); +} + +#[tokio::test] +async fn scope_history_expired_budget_or_borrowed_claim_perform_no_io() { + let server = scope_server(); + let client = scope_client(&server, "valid"); + let value = scalar_fixture(); + let (context, boundary) = boundary(&value); + let mut budget = CancellationScopeDeliveryBudget::new(); + assert!(budget + .restrict(DateTime::::from_timestamp(0, 0).unwrap()) + .is_err()); + assert!(matches!( + client + .prepare_cancellation_scope_on_claim(&claim(&value), &context, &boundary, &budget) + .await, + Err(Error::Timeout) + )); + let mut foreign = claim(&value); + foreign.run_id = Some("borrowed-run".into()); + assert!(client + .prepare_cancellation_scope_on_claim( + &foreign, + &context, + &boundary, + &CancellationScopeDeliveryBudget::new() + ) + .await + .is_err()); + assert!(server.requests.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn scope_history_preparation_delivery_and_history_share_one_original_budget() { + let server = scope_server(); + let client = scope_client(&server, "budget"); + let value = scalar_fixture(); + let task = claim(&value); + let (context, boundary) = boundary(&value); + let budget = CancellationScopeDeliveryBudget::new(); + let started = Instant::now(); + let prepared = client + .prepare_cancellation_scope_on_claim(&task, &context, &boundary, &budget) + .await + .unwrap(); + assert!(matches!( + client + .deliver_cancellation_scope_on_claim(&task, &prepared, &budget) + .await, + Err(Error::Timeout) + )); + assert!(started.elapsed() < Duration::from_millis(5600)); +} + +fn fixture(name: &str, variant: &str) -> Value { + let source = match name { + "operations" => { + include_str!("../../tests/fixtures/committed-scope-operation-projections.json") + } + "empty" => include_str!("../../tests/fixtures/committed-scope-delivery.json"), + "single" => include_str!("../../tests/fixtures/populated-scope-single-calls.json"), + "groups" => include_str!("../../tests/fixtures/populated-scope-groups.json"), + "root" => include_str!("../../tests/fixtures/run-inherited-scope-delivery.json"), + _ => panic!("unknown fixture"), + }; + let value: Value = serde_json::from_str(source).unwrap(); + if variant.is_empty() { + value + } else { + value[variant].clone() + } +} + +fn history(value: &Value) -> Vec { + serde_json::from_value(value["history"].clone()).unwrap() +} + +fn read(value: &Value) -> Result { + CommittedCancellationScopeHistory::read( + &history(value), + value["task"]["run_id"].as_str().unwrap(), + value["task"]["workflow_id"].as_str().unwrap(), + ) +} + +fn event_mut<'a>(value: &'a mut Value, kind: &str) -> &'a mut Value { + value["history"] + .as_array_mut() + .unwrap() + .iter_mut() + .find(|event| event["event_type"] == kind) + .unwrap() +} + +fn renumber(value: &mut Value) { + for (index, event) in value["history"] + .as_array_mut() + .unwrap() + .iter_mut() + .enumerate() + { + event["sequence"] = json!(index + 1); + } +} + +#[test] +fn scope_history_native_v5_operations_groups_descendants_and_competing_roots() { + for variant in ["operations", "groups", "descendants", "competing"] { + let value = fixture("operations", variant); + let committed = read(&value).unwrap_or_else(|error| panic!("{variant}: {error:?}")); + let preparation = &committed.preparations[value["scope_id"].as_str().unwrap()]; + let delivery = &committed.deliveries[&preparation.boundary.sequence]; + assert_eq!(preparation.context, delivery.context); + assert_eq!(preparation.boundary, delivery.boundary); + assert_eq!(preparation.authority_deadline, delivery.authority_deadline); + assert!(committed.pending_requests.is_empty()); + let projection = &preparation.event.payload; + assert_eq!( + projection["timer_members"] + .as_array() + .unwrap() + .iter() + .map(|row| row["timer_id"].as_str().unwrap()) + .collect::>(), + ["timer-plain", "signal-timer-id", "condition-timer-id"] + ); + assert_eq!( + projection["child_members"][1]["child_workflow_run_id"], + "child-continued-run" + ); + assert_eq!( + projection["child_members"] + .as_array() + .unwrap() + .iter() + .map(|row| row["cancellation_policy"].as_str().unwrap()) + .collect::>(), + ["try_cancel", "wait_cancellation_completed", "abandon"] + ); + if matches!(variant, "descendants" | "competing") { + assert_eq!( + projection["descendant_members"][0]["scope_id"], + "desc-child" + ); + assert_eq!( + projection["descendant_members"][1]["scope_id"], + "desc-grandchild" + ); + assert_eq!( + projection["descendant_members"][0]["propagation_history_event_id"], + if variant == "competing" { + "original-child-conflict" + } else { + "inherited-child-accepted" + } + ); + assert_eq!( + projection["descendant_members"][0]["authority_deadline_at"], + if variant == "competing" { + "2026-10-04T00:00:20.123456Z" + } else { + "2026-10-04T00:00:30.123456Z" + } + ); + } else { + assert_eq!(projection["descendant_members"], json!([])); + } + } +} + +#[test] +fn scope_history_empty_delivery_and_populated_scalar_policies() { + for variant in ["shielded", "unshielded"] { + let committed = read(&fixture("empty", variant)).unwrap(); + assert_eq!(committed.deliveries.len(), 1); + let delivery = committed.deliveries.values().next().unwrap(); + assert_eq!(delivery.boundary.call_kind, CancellationCallKind::Timer); + assert_eq!( + delivery.context.request_id(), + delivery.context.root_context().request_id() + ); + assert_eq!(delivery.authority_deadline, delivery.context.deadline()); + } + for variant in [ + "activity-try_cancel", + "activity-wait_cancellation_completed", + "activity-abandon", + "child-try_cancel", + "child-wait_cancellation_completed", + "child-abandon", + "timer", + "condition", + "condition-untimed", + ] { + let committed = read(&fixture("single", variant)) + .unwrap_or_else(|error| panic!("{variant}: {error:?}")); + let preparation = committed.preparations.values().next().unwrap(); + let delivery = &committed.deliveries[&preparation.boundary.sequence]; + assert_eq!(delivery.boundary.sequence, 4); + assert_eq!(delivery.context, preparation.context); + assert!(committed.pending_requests.is_empty()); + assert!(!preparation.event.payload["activity_members"] + .as_array() + .unwrap() + .is_empty()); + } +} + +#[test] +fn scope_history_changed_frozen_members_and_descriptors_are_refused() { + for variant in ["operations", "groups", "descendants", "competing"] { + for field in [ + "activity_members", + "timer_members", + "wait_members", + "child_members", + ] { + for failure in ["hash", "omitted", "boolean", "extra"] { + let mut value = fixture("operations", variant); + let members = event_mut(&mut value, "CancellationScopeDeliveryPrepared")["payload"] + [field] + .as_array_mut() + .unwrap(); + if members.is_empty() { + members.push(json!({})); + assert!(read(&value).is_err(), "{variant}/{field}/invented"); + continue; + } + match failure { + "hash" => members[0]["descriptor_hash"] = json!("0".repeat(64)), + "omitted" => { + members.remove(0); + } + "boolean" => members[0]["sequence"] = json!(true), + "extra" => members[0]["unexpected"] = json!("borrowed"), + _ => unreachable!(), + } + assert!(read(&value).is_err(), "{variant}/{field}/{failure}"); + } + } + } +} + +#[test] +fn scope_history_changed_request_boundary_deadline_ancestry_and_continuation_are_refused() { + for failure in [ + "request", + "run", + "instance", + "deadline", + "boundary", + "preparation", + "before_preparation", + "missing_request", + "duplicate_request", + "duplicate_delivery", + "future_member", + "continuation", + "descendant_deadline", + "descendant_propagation", + "cross_shield", + "inherited_deadline", + ] { + let mut value = fixture("operations", "descendants"); + match failure { + "request" => { + event_mut(&mut value, "CancellationScopeDelivered")["payload"]["request_id"] = + json!("new-request") + } + "run" => { + event_mut(&mut value, "CancellationScopeDeliveryPrepared")["payload"] + ["workflow_run_id"] = json!("new-run") + } + "instance" => { + event_mut(&mut value, "CancellationScopeDelivered")["payload"]["cancellation"] + ["lineage"][0]["workflow_instance_id"] = json!("new-instance") + } + "deadline" => { + event_mut(&mut value, "CancellationScopeDelivered")["payload"] + ["authority_deadline_at"] = json!("2026-10-04T00:00:31.123456Z") + } + "boundary" => { + event_mut(&mut value, "CancellationScopeDelivered")["payload"]["sequence"] = + json!(999) + } + "preparation" => { + event_mut(&mut value, "CancellationScopeDelivered")["payload"] + ["preparation_history_event_id"] = json!("new-preparation") + } + "descendant_deadline" => { + event_mut(&mut value, "CancellationScopeDeliveryPrepared")["payload"] + ["descendant_members"][0]["authority_deadline_at"] = + json!("2026-10-04T00:00:31.123456Z") + } + "descendant_propagation" => { + event_mut(&mut value, "CancellationScopeDeliveryPrepared")["payload"] + ["descendant_members"][0]["propagation_history_event_id"] = + json!("new-propagation") + } + "continuation" | "cross_shield" | "inherited_deadline" => { + let id = match failure { + "continuation" => "child-continued-before-preparation", + "cross_shield" => "desc-child-opened", + _ => "inherited-child-accepted", + }; + let event = value["history"] + .as_array_mut() + .unwrap() + .iter_mut() + .find(|row| row["id"] == id) + .unwrap(); + match failure { + "continuation" => { + event["payload"]["child_workflow_run_id"] = json!("new-child-run") + } + "cross_shield" => event["payload"]["shield_parent"] = json!(true), + _ => { + event["payload"]["cancellation"]["lineage"][1]["cleanup_deadline_at"] = + json!("2026-10-04T00:00:29.123456Z") + } + } + } + _ => { + let events = value["history"].as_array_mut().unwrap(); + let target = match failure { + "missing_request" | "duplicate_request" => "CancellationScopeRequested", + "future_member" => "TimerScheduled", + _ => "CancellationScopeDelivered", + }; + let index = events + .iter() + .position(|row| row["event_type"] == target) + .unwrap(); + if failure.starts_with("duplicate") { + let mut duplicate = events[index].clone(); + duplicate["id"] = + json!(format!("{}-duplicate", duplicate["id"].as_str().unwrap())); + events.insert(index + 1, duplicate); + } else { + let event = events.remove(index); + if failure == "before_preparation" { + let index = events + .iter() + .position(|row| { + row["event_type"] == "CancellationScopeDeliveryPrepared" + }) + .unwrap(); + events.insert(index, event); + } else if failure == "future_member" { + events.push(event); + } + } + } + } + renumber(&mut value); + assert!(read(&value).is_err(), "{failure}"); + } +} + +#[test] +fn scope_history_pending_ancestor_and_original_preparation_survive_replacement() { + let mut value = fixture("operations", "descendants"); + value["history"].as_array_mut().unwrap().pop(); + let committed = read(&value).unwrap(); + let scopes = cancellation_scope::CancellationScopeHistory::read( + &history(&value), + value["task"]["run_id"].as_str().unwrap(), + ) + .unwrap(); + let request = committed + .pending_request_for_scope("desc-grandchild", &scopes) + .unwrap() + .unwrap(); + assert_eq!( + request.context.scope_id(), + value["scope_id"].as_str().unwrap() + ); + let preparation = &committed.preparations[request.context.scope_id()]; + value["task"]["lease_owner"] = json!("replacement"); + value["task"]["workflow_task_attempt"] = json!(17); + let replayed = read(&value).unwrap(); + assert_eq!( + replayed.preparations[request.context.scope_id()].context, + preparation.context + ); + assert_eq!( + replayed.preparations[request.context.scope_id()].boundary, + preparation.boundary + ); + assert_eq!( + replayed.preparations[request.context.scope_id()].authority_deadline, + preparation.authority_deadline + ); +} + +#[test] +fn scope_history_run_inheritance_preserves_original_parent_and_shielding() { + for failure in [ + "valid", "missing", "later", "shield", "metadata", "deadline", + ] { + let mut value = fixture("root", ""); + value["history"].as_array_mut().unwrap().truncate(5); + match failure { + "missing" => { + value["history"].as_array_mut().unwrap().remove(3); + } + "later" => value["history"].as_array_mut().unwrap().swap(3, 4), + "shield" => value["history"][2]["payload"]["shield_parent"] = json!(true), + "metadata" => { + value["history"][4]["payload"]["cancellation"]["root_context"]["reason"] = + json!("changed") + } + "deadline" => { + value["history"][4]["payload"]["cancellation"]["lineage"][1] + ["cleanup_deadline_at"] = json!("2026-10-05T00:00:25.000000Z") + } + _ => {} + } + renumber(&mut value); + let result = read(&value); + if failure != "valid" { + assert!(result.is_err(), "{failure}"); + continue; + } + let committed = result.unwrap(); + let id = value["history"][2]["payload"]["scope_id"].as_str().unwrap(); + let request = &committed.pending_requests[id]; + assert_eq!( + request.context.request_id(), + value["history"][4]["payload"]["request_id"] + .as_str() + .unwrap() + ); + assert_eq!( + request.context.parent_request_id(), + value["history"][3]["payload"]["workflow_command_id"].as_str() + ); + assert_eq!( + request + .context + .lineage() + .iter() + .map(ScopedCancellationLineage::scope_id) + .collect::>(), + ["root", id] + ); + } +} diff --git a/src/tests/cancellation_scope_opening.rs b/src/tests/cancellation_scope_opening.rs new file mode 100644 index 0000000..f219896 --- /dev/null +++ b/src/tests/cancellation_scope_opening.rs @@ -0,0 +1,303 @@ +use super::*; + +fn receipt() -> Value { + json!({"task_id":"task/one", "workflow_run_id":"run-one", "lease_owner":"original", + "workflow_task_attempt":4, "sequence":2, "scope_id":"scope-two", "parent_scope_id":"scope-one", + "shield_parent":true, "opened":true, "duplicate":false, "claim_released":false, + "created_task_ids":[], "reason":null, "history_event_id":"event-scope-two", + "history_refresh_page_token":"opaque-start"}) +} + +fn history() -> Vec { + vec![ + json!({"id":"start", "sequence":1, "namespace":"tenant", "event_type":"WorkflowStarted", "payload":{}}), + json!({"id":"event-scope-one", "sequence":2, "namespace":"tenant", "event_type":"CancellationScopeOpened", + "payload":{"schema":"durable-workflow.cancellation-scope/v1", "workflow_run_id":"run-one", + "sequence":1, "scope_id":"scope-one", "parent_scope_id":"root", "shield_parent":false}}), + json!({"id":"event-scope-two", "sequence":3, "namespace":"tenant", "event_type":"CancellationScopeOpened", + "payload":{"schema":"durable-workflow.cancellation-scope/v1", "workflow_run_id":"run-one", + "sequence":2, "scope_id":"scope-two", "parent_scope_id":"scope-one", "shield_parent":true}}), + ] +} + +fn response(path: &str, body: &str, number: usize) -> Option<(&'static str, String)> { + let case = path.split('/').nth(1).unwrap_or_default(); + if case == "budget" { + thread::sleep(Duration::from_secs(3)); + } + let mut value; + if path.ends_with("/cancellation-scopes/open") { + if case == "lost-ack" && number == 1 { + return Some(("invalid-status", String::new())); + } + if case == "refused" { + return Some(( + "409 Conflict", + r#"{"reason":"lease_owner_mismatch"}"#.into(), + )); + } + value = receipt(); + if case == "lost-ack" || case == "duplicate" { + value["duplicate"] = json!(true); + } + match case { + "ack-owner" => value["lease_owner"] = json!("replacement"), + "ack-attempt" => value["workflow_task_attempt"] = json!(5), + "ack-run" => value["workflow_run_id"] = json!("foreign"), + "ack-sequence" => value["sequence"] = json!(3), + "ack-shield" => value["shield_parent"] = json!(1), + "ack-opened" => value["opened"] = json!(1), + "ack-duplicate" => value["duplicate"] = json!(0), + "ack-released" => value["claim_released"] = json!(true), + "ack-new-task" => value["created_task_ids"] = json!(["new"]), + "ack-root" => value["scope_id"] = json!("root"), + "ack-token" => value["history_refresh_page_token"] = Value::Null, + "ack-event" => value["history_event_id"] = json!(""), + "ack-reason" => { + value.as_object_mut().unwrap().remove("reason"); + } + _ => {} + } + } else if path.ends_with("/history") { + let mut events = history(); + if case == "accepted-prefix" { + events.insert( + 0, + json!({"id":"accepted", "sequence":0, "namespace":"tenant", + "event_type":"StartAccepted", "payload":{}}), + ); + for event in &mut events { + event["sequence"] = json!(event["sequence"].as_u64().unwrap() + 1); + } + } + match case { + "tree-namespace" => events[2]["namespace"] = json!("foreign"), + "tree-id" => events[2]["id"] = json!("start"), + "tree-event-sequence" => events[2]["sequence"] = json!(2), + "tree-run" => events[2]["payload"]["workflow_run_id"] = json!("foreign"), + "tree-scope" => events[2]["payload"]["scope_id"] = json!("scope-one"), + "tree-parent" => events[2]["payload"]["parent_scope_id"] = json!("unknown"), + "tree-self" => events[2]["payload"]["parent_scope_id"] = json!("scope-two"), + "tree-changed-parent" => events[2]["payload"]["parent_scope_id"] = json!("root"), + "tree-shield" => events[2]["payload"]["shield_parent"] = json!(false), + "tree-bool" => events[2]["payload"]["shield_parent"] = json!(1), + "tree-sequence" => events[2]["payload"]["sequence"] = json!(1), + "tree-schema" => events[2]["payload"]["schema"] = json!("other"), + "missing-opening" => { + events.pop(); + } + "missing-start" => { + events.remove(0); + } + "empty-history" => events.clear(), + _ => {} + } + let requested: Value = serde_json::from_str(body).unwrap(); + let (batch, token) = if requested["next_history_page_token"] == "opaque-start" { + ( + events.iter().take(1).cloned().collect::>(), + json!("opaque-next"), + ) + } else { + ( + events.iter().skip(1).cloned().collect::>(), + Value::Null, + ) + }; + value = json!({"task_id":"task/one", "workflow_task_attempt":4, + "history_events":batch,"next_history_page_token":token}); + match case { + "page-task" => value["task_id"] = json!("foreign"), + "page-attempt" => value["workflow_task_attempt"] = json!(5), + "page-cycle" => value["next_history_page_token"] = json!("opaque-start"), + "page-empty" => value["history_events"] = json!([]), + "page-events" => value["history_events"] = json!([null]), + "page-token" => { + value + .as_object_mut() + .unwrap() + .remove("next_history_page_token"); + } + _ => {} + } + } else { + return None; + } + Some(("200 OK", value.to_string())) +} + +fn server() -> MockWorkerServer { + MockWorkerServer::start_with_behavior(MockWorkerBehavior { + request_override: Some(response), + ..MockWorkerBehavior::default() + }) +} + +fn client(server: &MockWorkerServer, case: &str) -> Client { + Client::builder(format!("{}/{case}", server.base_url())) + .namespace("tenant") + .control_token(Some("control-only".into())) + .worker_token(Some("worker-only".into())) + .build() + .unwrap() +} + +fn task() -> WorkflowTask { + serde_json::from_value( + json!({"task_id":"task/one", "run_id":"run-one", "workflow_type":"scope", + "lease_owner":"original", "workflow_task_attempt":4, "payload_codec":DEFAULT_CODEC}), + ) + .unwrap() +} + +#[tokio::test] +async fn scope_opening_proves_complete_paged_tree_and_lost_ack_under_original_worker_credentials() { + for case in ["valid", "duplicate", "lost-ack", "accepted-prefix"] { + let server = server(); + let proof = client(&server, case) + .open_cancellation_scope_on_claim(&task(), 2, "scope-one", true) + .await + .unwrap(); + assert_eq!(proof.scope_id(), "scope-two"); + assert_eq!(proof.history_event_id(), "event-scope-two"); + assert_eq!(proof.sequence(), 2); + assert_eq!(proof.parent_scope_id(), "scope-one"); + assert!(proof.shield_parent()); + assert_eq!(proof.duplicate(), matches!(case, "duplicate" | "lost-ack")); + assert_eq!( + proof.history().len(), + if case == "accepted-prefix" { 4 } else { 3 } + ); + let requests = server.requests.lock().unwrap(); + let opens = requests + .iter() + .filter(|row| row.path.ends_with("/open")) + .collect::>(); + assert_eq!(opens.len(), if case == "lost-ack" { 2 } else { 1 }); + if case == "lost-ack" { + assert_eq!(opens[0].body, opens[1].body); + } + for request in requests.iter() { + assert_eq!(request.worker_protocol.as_deref(), Some("1.20")); + assert_eq!(request.authorization.as_deref(), Some("Bearer worker-only")); + assert_eq!(request.namespace.as_deref(), Some("tenant")); + assert!(request.path.contains("/task%2Fone/")); + let body: Value = serde_json::from_str(&request.body).unwrap(); + assert_eq!(body["lease_owner"], "original"); + assert_eq!(body["workflow_task_attempt"], 4); + } + } +} + +#[tokio::test] +async fn scope_opening_refuses_changed_receipts_before_history_reads() { + for case in [ + "ack-owner", + "ack-attempt", + "ack-run", + "ack-sequence", + "ack-shield", + "ack-opened", + "ack-duplicate", + "ack-released", + "ack-new-task", + "ack-root", + "ack-token", + "ack-event", + "ack-reason", + ] { + let server = server(); + assert!( + matches!( + client(&server, case) + .open_cancellation_scope_on_claim(&task(), 2, "scope-one", true) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + ), + "{case}" + ); + assert_eq!(server.requests.lock().unwrap().len(), 1, "{case}"); + } +} + +#[tokio::test] +async fn scope_opening_refuses_altered_canonical_tree_and_invalid_claim_pages() { + for case in [ + "tree-namespace", + "tree-id", + "tree-event-sequence", + "tree-run", + "tree-scope", + "tree-parent", + "tree-self", + "tree-changed-parent", + "tree-shield", + "tree-bool", + "tree-sequence", + "tree-schema", + "missing-opening", + "missing-start", + "empty-history", + "page-task", + "page-attempt", + "page-cycle", + "page-empty", + "page-events", + "page-token", + ] { + let server = server(); + assert!( + matches!( + client(&server, case) + .open_cancellation_scope_on_claim(&task(), 2, "scope-one", true) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + ), + "{case}" + ); + } +} + +#[tokio::test] +async fn scope_opening_refuses_invalid_original_authority_before_io() { + let server = server(); + let client = client(&server, "valid"); + for field in ["task_id", "run_id", "lease_owner", "workflow_task_attempt"] { + let mut invalid = task(); + match field { + "task_id" => invalid.task_id.clear(), + "run_id" => invalid.run_id = None, + "lease_owner" => invalid.lease_owner = Some("x".repeat(256)), + _ => invalid.workflow_task_attempt = 0, + } + assert!(client + .open_cancellation_scope_on_claim(&invalid, 2, "scope-one", true) + .await + .is_err()); + } + for (sequence, parent) in [(0, "scope-one"), (u64::MAX, "scope-one"), (2, "")] { + assert!(client + .open_cancellation_scope_on_claim(&task(), sequence, parent, true) + .await + .is_err()); + } + assert!(server.requests.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn scope_opening_preserves_claim_refusal_and_bounds_opening_with_all_history_pages() { + let server = server(); + assert!( + matches!(client(&server,"refused").open_cancellation_scope_on_claim(&task(),2,"scope-one",true).await, + Err(Error::Http { status, .. }) if status.as_u16() == 409) + ); + assert_eq!(server.requests.lock().unwrap().len(), 1); + let started = Instant::now(); + assert!(matches!( + client(&server, "budget") + .open_cancellation_scope_on_claim(&task(), 2, "scope-one", true) + .await, + Err(Error::Timeout) + )); + assert!(started.elapsed() < Duration::from_secs(6)); +} diff --git a/src/tests/cancellation_scope_replay.rs b/src/tests/cancellation_scope_replay.rs new file mode 100644 index 0000000..9105fd0 --- /dev/null +++ b/src/tests/cancellation_scope_replay.rs @@ -0,0 +1,663 @@ +use super::*; + +mod descendants; +mod run_scopes; + +fn group_fixture(name: &str) -> Value { + let source: Value = serde_json::from_str(include_str!( + "../../tests/fixtures/populated-scope-groups.json" + )) + .unwrap(); + source[name].clone() +} + +fn group_worker(cleanup: Arc, nested: bool, changed: &'static str) -> Worker { + let mut worker = Worker::new(Client::new("http://unused.invalid").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + worker.register_workflow("scope-replay", move |ctx, _| { + let cleanup = Arc::clone(&cleanup); + async move { + let parent = ctx.clone(); + let result = ctx + .cancellation_scope(false, move |outer| async move { + outer + .cancellation_scope(false, move |inner| async move { + if changed != "prefix" { + inner.activity("prior-step", json!([])).await?; + } + let activity = ParallelOperation::activity_with_options( + if changed == "activity" { + "changed-activity" + } else { + "original-activity" + }, + ActivityOptions::new().cancellation_policy(if changed == "activity_policy" { + CancellationPolicy::WaitCancellationCompleted + } else { CancellationPolicy::TryCancel }), json!([]), + ); + let timer = ParallelOperation::timer(Duration::from_secs( + if changed == "timer" { 3599 } else { 3600 }, + )); + let child = ParallelOperation::child_workflow( + if changed == "child" { "changed-child" } else { "original-child" }, + ChildWorkflowOptions::new("queue") + .parent_close_policy(if changed == "parent_policy" { + ParentClosePolicy::Abandon + } else { ParentClosePolicy::RequestCancel }) + .cancellation_policy(if changed == "child_policy" { + CancellationPolicy::Abandon + } else { + CancellationPolicy::WaitCancellationCompleted + }), + json!([]), + ); + let condition = ParallelOperation::condition(ConditionWaitOptions::new( + if changed == "condition" { "changed-key" } else { "ready" }, + if changed == "predicate" { "sha256:changed" } else { "sha256:3b2b1ad635030d842ab378e8bf021f4613ee6d221dafb4743565467800def433" }) + .timeout(Duration::from_secs(if changed == "condition_timeout" { 31 } else { 30 })), || panic!("committed cancellation must not evaluate the pending predicate")); + let mut group = if nested { + vec![ + activity, + ParallelOperation::group(vec![timer, child]), + condition, + ] + } else { + vec![activity, timer, child, condition] + }; + if changed == "size" { group.pop(); } + if changed == "kind" { group[0] = ParallelOperation::timer(Duration::from_secs(3600)); } + let _shield = if changed == "shield" { + Some(inner.cancellation_shield()?) + } else { + None + }; + match inner.parallel(group).await { + Err(Error::CancellationScopeRequested(cancellation)) => { + cleanup.fetch_add(1, Ordering::SeqCst); + assert_eq!( + cancellation.delivery.call_kind, + CancellationCallKind::Parallel + ); + assert_eq!(cancellation.delivery.sequence_span, 4); + assert_eq!( + cancellation.context.remaining()?, + Duration::from_secs(21) + ); + let _shield = inner.cancellation_shield()?; + inner.sleep(Duration::from_secs(2)).await?; + assert_eq!(cancellation.context.remaining()?, Duration::from_secs(19)); + Ok(json!({"cleaned":true})) + } + Err(error) => Err(error), + Ok(_) => Ok(json!("ordinary")), + } + }) + .await + }) + .await?; + assert!(!parent.is_cancellation_requested()?); + parent.sleep(Duration::from_secs(1)).await?; + Ok(result) + } + }); + worker +} + +#[test] +fn cancellation_scope_replay_flat_and_nested_groups_wait_for_committed_delivery() { + for (name, nested) in [("flat", false), ("nested", true)] { + let source = group_fixture(name); + let cleanup = Arc::new(AtomicUsize::new(0)); + let worker = group_worker(Arc::clone(&cleanup), nested, ""); + for before in [ + "CancellationScopeDeliveryPrepared", + "CancellationScopeDelivered", + ] { + let decision = worker + .execute_workflow_task_decision(task(&prefix(&source, before))) + .unwrap(); + assert!(decision.commands.is_empty()); + let intent = decision.cancellation_scope_delivery.unwrap(); + assert_eq!(intent.boundary.call_kind, CancellationCallKind::Parallel); + assert_eq!(intent.boundary.sequence, 4); + assert_eq!(intent.boundary.sequence_span, 4); + assert_eq!(cleanup.load(Ordering::SeqCst), 0); + } + let decision = worker + .execute_workflow_task_decision(task(&source)) + .unwrap(); + assert!(decision.cancellation_scope_delivery.is_none()); + assert_eq!(cleanup.load(Ordering::SeqCst), 1); + assert_eq!(decision.commands.len(), 1); + assert_eq!(decision.commands[0]["type"], "start_timer"); + assert_eq!(decision.commands[0]["delay_seconds"], 2); + assert_eq!( + decision.commands[0]["cancellation_cleanup"]["request_id"], + cleanup_snapshot(&source)["request_id"] + ); + } +} + +#[test] +fn cancellation_scope_replay_group_rejects_changed_members_before_cleanup() { + for changed in [ + "activity", + "activity_policy", + "timer", + "child", + "child_policy", + "parent_policy", + "condition", + "condition_timeout", + "predicate", + "shield", + "shape", + "prefix", + "size", + "kind", + ] { + let cleanup = Arc::new(AtomicUsize::new(0)); + let worker = group_worker(Arc::clone(&cleanup), changed == "shape", changed); + assert!( + worker + .execute_workflow_task_decision(task(&group_fixture("flat"))) + .is_err(), + "{changed}" + ); + assert_eq!(cleanup.load(Ordering::SeqCst), 0, "{changed}"); + } +} + +#[test] +fn cancellation_scope_replay_unqualified_groups_stop_before_factory() { + for name in [ + "flat-local", + "flat-other-scope", + "flat-selection", + "flat-signal", + "flat-incomplete", + ] { + let invoked = Arc::new(AtomicUsize::new(0)); + let mut worker = Worker::new(Client::new("http://unused.invalid").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + let calls = Arc::clone(&invoked); + worker.register_workflow("scope-replay", move |_, _| { + calls.fetch_add(1, Ordering::SeqCst); + async { Ok(json!(null)) } + }); + assert!( + worker + .execute_workflow_task_decision(task(&group_fixture(name))) + .is_err(), + "{name}" + ); + assert_eq!(invoked.load(Ordering::SeqCst), 0, "{name}"); + } +} + +#[test] +fn cancellation_scope_replay_group_replacement_preserves_cleanup_sequence_and_authority() { + for (name, nested) in [("flat", false), ("nested", true)] { + let mut source = group_fixture(name); + let snapshot = cleanup_snapshot(&source); + let scope = snapshot["scope_id"].clone(); + append( + &mut source, + "TimerScheduled", + json!({"sequence":8, "timer_id":"group-cleanup", + "delay_seconds":2, "cancellation_scope_id":scope, "cancellation_cleanup":snapshot, + "fire_at":"2026-10-04T00:00:11.123456Z"}), + "2026-10-04T00:00:09.123456Z", + ); + append( + &mut source, + "TimerFired", + json!({"sequence":8, "timer_id":"group-cleanup", + "delay_seconds":2, "cancellation_scope_id":scope}), + "2026-10-04T00:00:11.123456Z", + ); + let worker = + group_worker(Arc::new(AtomicUsize::new(0)), nested, "").worker_id("replacement"); + let mut claim = task(&source); + claim.lease_owner = Some("replacement".into()); + claim.workflow_task_attempt = 19; + let decision = worker.execute_workflow_task_decision(claim).unwrap(); + assert_eq!(decision.commands.len(), 1); + assert_eq!(decision.commands[0]["delay_seconds"], 1); + assert!(decision.commands[0].get("cancellation_scope_id").is_none()); + assert!(decision.commands[0].get("cancellation_cleanup").is_none()); + append( + &mut source, + "TimerScheduled", + json!({"sequence":9, "timer_id":"parent-timer", + "delay_seconds":1}), + "2026-10-04T00:00:11.123456Z", + ); + append( + &mut source, + "TimerFired", + json!({"sequence":9, "timer_id":"parent-timer", + "delay_seconds":1}), + "2026-10-04T00:00:12.123456Z", + ); + let decision = worker + .execute_workflow_task_decision(task(&source)) + .unwrap(); + assert_eq!(decision.commands.len(), 1); + assert_eq!(decision.commands[0]["type"], "complete_workflow"); + } +} + +#[test] +fn cancellation_scope_replay_completed_member_does_not_hide_pending_siblings() { + for (name, nested) in [("flat", false), ("nested", true)] { + for prepared in [false, true] { + let source = group_fixture(name); + let mut value = prefix( + &source, + if prepared { + "CancellationScopeDelivered" + } else { + "CancellationScopeDeliveryPrepared" + }, + ); + let events = value["history"].as_array_mut().unwrap(); + let request = events + .iter() + .position(|row| row["event_type"] == "CancellationScopeRequested") + .unwrap(); + let namespace = events[0]["namespace"].clone(); + events.insert( + request, + json!({"id":"completed-first-member", "sequence":1, + "namespace":namespace, "timestamp":"2026-10-04T00:00:02.123456Z", + "event_type":"ActivityCompleted", "payload":{"sequence":4, + "activity_execution_id":"original-id", "activity_type":"original-activity", + "result":{"codec":"avro", "blob":"wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU="}}}), + ); + for (index, row) in events.iter_mut().enumerate() { + row["sequence"] = json!(index + 1); + } + let cleanup = Arc::new(AtomicUsize::new(0)); + let decision = group_worker(Arc::clone(&cleanup), nested, "") + .execute_workflow_task_decision(task(&value)) + .unwrap(); + assert!(decision.commands.is_empty()); + assert_eq!( + decision + .cancellation_scope_delivery + .unwrap() + .boundary + .sequence_span, + 4 + ); + assert_eq!(cleanup.load(Ordering::SeqCst), 0); + } + } +} + +#[test] +fn cancellation_scope_replay_completed_group_preserves_results_and_prepares_next_call() { + let mut source = group_fixture("flat"); + let request = source["history"] + .as_array() + .unwrap() + .iter() + .find(|row| row["event_type"] == "CancellationScopeRequested") + .unwrap()["payload"] + .clone(); + let scope = request["scope_id"].clone(); + source["history"].as_array_mut().unwrap().retain(|row| { + !row["event_type"] + .as_str() + .unwrap() + .starts_with("CancellationScopeRequest") + && !row["event_type"] + .as_str() + .unwrap() + .starts_with("CancellationScopeDeliver") + && row["payload"]["sequence"] + .as_u64() + .is_none_or(|sequence| sequence < 4) + }); + let descriptors = parallel_descriptors( + vec![ + ParallelOperation::timer(Duration::from_secs(1)), + ParallelOperation::timer(Duration::from_secs(1)), + ], + 4, + ) + .unwrap(); + for descriptor in descriptors { + let sequence = 4 + descriptor.offset as u64; + let delay = 1; + let mut payload = serde_json::Map::from_iter([ + ("sequence".into(), json!(sequence)), + ("timer_id".into(), json!(format!("timer-{sequence}"))), + ("delay_seconds".into(), json!(delay)), + ("cancellation_scope_id".into(), scope.clone()), + ]); + apply_parallel_group_path(&mut payload, &descriptor.group_path); + append( + &mut source, + "TimerScheduled", + Value::Object(payload.clone()), + "2026-10-04T00:00:01.123456Z", + ); + append( + &mut source, + "TimerFired", + Value::Object(payload), + "2026-10-04T00:00:02.123456Z", + ); + } + append( + &mut source, + "CancellationScopeRequested", + request, + "2026-10-04T00:00:03.123456Z", + ); + let mut worker = Worker::new(Client::new("http://unused.invalid").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + worker.register_workflow("scope-replay", |ctx, _| async move { + ctx.cancellation_scope(false, |outer| async move { + outer + .cancellation_scope(false, |inner| async move { + inner.activity("prior-step", json!([])).await?; + let results = inner + .parallel(vec![ + ParallelOperation::timer(Duration::from_secs(1)), + ParallelOperation::timer(Duration::from_secs(1)), + ]) + .await?; + assert_eq!(results.len(), 2); + inner.sleep(Duration::from_secs(1)).await?; + Ok(json!(null)) + }) + .await + }) + .await + }); + let decision = worker + .execute_workflow_task_decision(task(&source)) + .unwrap(); + assert!(decision.commands.is_empty()); + let intent = decision.cancellation_scope_delivery.unwrap(); + assert_eq!(intent.boundary.sequence, 6); + assert_eq!(intent.boundary.call_kind, CancellationCallKind::Timer); +} + +fn fixture() -> Value { + let source: Value = serde_json::from_str(include_str!( + "../../tests/fixtures/populated-scope-single-calls.json" + )) + .unwrap(); + source["timer"].clone() +} + +fn task(value: &Value) -> WorkflowTask { + serde_json::from_value(json!({"task_id":"task-one", "workflow_id":value["task"]["workflow_id"], + "run_id":value["task"]["run_id"], "workflow_type":"scope-replay", "payload_codec":DEFAULT_CODEC, + "lease_owner":"original", "workflow_task_attempt":1, "history_events":value["history"]})).unwrap() +} + +fn worker(cleanup: Arc, changed_delay: bool) -> Worker { + let mut worker = Worker::new(Client::new("http://unused.invalid").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + worker.register_workflow("scope-replay", move |ctx, _| { + let cleanup = Arc::clone(&cleanup); + async move { + let parent = ctx.clone(); + let result = ctx.cancellation_scope(false, move |outer| async move { + outer.cancellation_scope(false, move |inner| async move { + assert_eq!(inner.activity("prior-step", json!([])).await?, json!("prior-value")); + match inner.sleep(Duration::from_secs(if changed_delay { 3599 } else { 3600 })).await { + Ok(()) => Ok(json!("ordinary-result")), + Err(Error::CancellationScopeRequested(cancellation)) => { + cleanup.fetch_add(1, Ordering::SeqCst); + assert!(inner.is_cancellation_requested()?); + assert_eq!(inner.scoped_cancellation_context()?.unwrap(), cancellation.context); + let before = cancellation.context.remaining()?; + assert_eq!(before, Duration::from_secs(21)); + let _shield = inner.cancellation_shield()?; + inner.sleep(Duration::from_secs(2)).await?; + let after = cancellation.context.remaining()?; + assert_eq!(after, Duration::from_secs(19)); + Ok(json!({"cleaned":true, "request_id":cancellation.context.request_id()})) + } + Err(error) => Err(error), + } + }).await + }).await?; + assert!(!parent.is_cancellation_requested()?); + assert!(parent.scoped_cancellation_context()?.is_none()); + parent.sleep(Duration::from_secs(1)).await?; + Ok(result) + } + }); + worker +} + +fn prefix(value: &Value, before: &str) -> Value { + let mut value = value.clone(); + let events = value["history"].as_array_mut().unwrap(); + let index = events + .iter() + .position(|event| event["event_type"] == before) + .unwrap(); + events.truncate(index); + value +} + +fn append(value: &mut Value, kind: &str, payload: Value, timestamp: &str) { + let events = value["history"].as_array_mut().unwrap(); + let sequence = events.last().unwrap()["sequence"].as_u64().unwrap() + 1; + let namespace = events[0]["namespace"].clone(); + events.push( + json!({"id":format!("cleanup-event-{sequence}"), "sequence":sequence, + "namespace":namespace, "event_type":kind, "payload":payload, "timestamp":timestamp}), + ); +} + +fn cleanup_snapshot(value: &Value) -> Value { + let delivery = value["history"] + .as_array() + .unwrap() + .iter() + .find(|row| row["event_type"] == "CancellationScopeDelivered") + .unwrap(); + let context = + ScopedCancellationContext::from_value(&delivery["payload"]["cancellation"]).unwrap(); + json!({"scope_id":context.scope_id(), "operation_scope_id":context.scope_id(), + "request_id":context.request_id(), "root_request_id":context.root_context().root_request_id(), + "delivery_history_event_id":delivery["id"], + "preparation_history_event_id":delivery["payload"]["preparation_history_event_id"], + "cleanup_deadline_at":context.deadline().to_rfc3339_opts(chrono::SecondsFormat::Micros, true), + "authority_deadline_at":delivery["payload"]["authority_deadline_at"]}) +} + +#[test] +fn cancellation_scope_replay_waits_for_original_prepare_and_committed_delivery() { + let original = fixture(); + let cleanup = Arc::new(AtomicUsize::new(0)); + let worker = worker(Arc::clone(&cleanup), false); + for before in [ + "CancellationScopeDeliveryPrepared", + "CancellationScopeDelivered", + ] { + let decision = worker + .execute_workflow_task_decision(task(&prefix(&original, before))) + .unwrap(); + let intent = decision.cancellation_scope_delivery.unwrap(); + assert_eq!(intent.boundary.sequence, 4); + assert_eq!(intent.boundary.call_kind, CancellationCallKind::Timer); + assert!(decision.commands.is_empty()); + assert_eq!(cleanup.load(Ordering::SeqCst), 0); + } + let decision = worker + .execute_workflow_task_decision(task(&original)) + .unwrap(); + assert!(decision.cancellation_scope_delivery.is_none()); + assert_eq!(decision.commands.len(), 1); + assert_eq!(decision.commands[0]["type"], "start_timer"); + assert_eq!(decision.commands[0]["delay_seconds"], 2); + assert!(decision.commands[0]["cancellation_scope_id"].is_string()); + let snapshot = cleanup_snapshot(&original); + assert_eq!( + decision.commands[0]["cancellation_cleanup"], + json!({ + "scope_id":snapshot["scope_id"], "request_id":snapshot["request_id"], + "delivery_history_event_id":snapshot["delivery_history_event_id"]}) + ); + assert_eq!(cleanup.load(Ordering::SeqCst), 1); +} + +#[test] +fn cancellation_scope_replay_replacement_keeps_context_clock_and_parent_unaffected() { + let mut source = fixture(); + let snapshot = cleanup_snapshot(&source); + let scope = source["history"] + .as_array() + .unwrap() + .iter() + .find(|event| event["event_type"] == "CancellationScopeDelivered") + .unwrap()["payload"]["scope_id"] + .clone(); + append( + &mut source, + "TimerScheduled", + json!({"sequence":5, "timer_id":"cleanup-timer", + "delay_seconds":2, "cancellation_scope_id":scope, + "fire_at":"2026-10-04T00:00:11.123456Z", "cancellation_cleanup":snapshot}), + "2026-10-04T00:00:09.123456Z", + ); + append( + &mut source, + "TimerFired", + json!({"sequence":5, "timer_id":"cleanup-timer", "delay_seconds":2, + "cancellation_scope_id":scope}), + "2026-10-04T00:00:11.123456Z", + ); + let replacement = worker(Arc::new(AtomicUsize::new(0)), false).worker_id("replacement"); + let mut claim = task(&source); + claim.lease_owner = Some("replacement".into()); + claim.workflow_task_attempt = 2; + let decision = replacement.execute_workflow_task_decision(claim).unwrap(); + assert_eq!(decision.commands.len(), 1); + assert_eq!(decision.commands[0]["delay_seconds"], 1); + assert!(decision.commands[0].get("cancellation_scope_id").is_none()); + append( + &mut source, + "TimerScheduled", + json!({"sequence":6, "timer_id":"parent-timer", "delay_seconds":1}), + "2026-10-04T00:00:11.123456Z", + ); + append( + &mut source, + "TimerFired", + json!({"sequence":6, "timer_id":"parent-timer", "delay_seconds":1}), + "2026-10-04T00:00:12.123456Z", + ); + let decision = replacement + .execute_workflow_task_decision(task(&source)) + .unwrap(); + assert_eq!(decision.commands.len(), 1); + assert_eq!(decision.commands[0]["type"], "complete_workflow"); +} + +#[test] +fn cancellation_scope_replay_cleanup_timer_refuses_changed_authority_before_factory() { + for field in [ + "request_id", + "delivery_history_event_id", + "authority_deadline_at", + "cleanup_deadline_at", + "operation_scope_id", + ] { + let mut value = fixture(); + let mut snapshot = cleanup_snapshot(&value); + let scope = snapshot["scope_id"].clone(); + snapshot[field] = json!("changed"); + append( + &mut value, + "TimerScheduled", + json!({"sequence":5, "timer_id":"cleanup-timer", + "delay_seconds":2, "cancellation_scope_id":scope, + "fire_at":"2026-10-04T00:00:11.123456Z", "cancellation_cleanup":snapshot}), + "2026-10-04T00:00:09.123456Z", + ); + let factories = Arc::new(AtomicUsize::new(0)); + let called = Arc::clone(&factories); + let mut worker = worker(Arc::new(AtomicUsize::new(0)), false); + worker.register_workflow("scope-replay", move |_, _| { + called.fetch_add(1, Ordering::SeqCst); + async { Ok(Value::Null) } + }); + assert!(worker.execute_workflow_task_decision(task(&value)).is_err()); + assert_eq!(factories.load(Ordering::SeqCst), 0); + } +} + +#[test] +fn cancellation_scope_replay_descriptor_change_cannot_enter_cleanup() { + let cleanup = Arc::new(AtomicUsize::new(0)); + let result = + worker(Arc::clone(&cleanup), true).execute_workflow_task_decision(task(&fixture())); + assert!( + matches!(result, Err(Error::NonDeterministicReplay(failure)) if failure.reason == "timer_delay_mismatch") + ); + assert_eq!(cleanup.load(Ordering::SeqCst), 0); +} + +#[test] +fn cancellation_scope_replay_default_and_incomplete_opt_in_refuse_before_factory() { + for (authoring, delivery) in [(false, false), (true, false), (false, true)] { + let calls = Arc::new(AtomicUsize::new(0)); + let observed = Arc::clone(&calls); + let mut worker = Worker::new(Client::new("http://unused.invalid").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(authoring) + .candidate_cancellation_scope_delivery(delivery); + worker.register_workflow("scope-replay", move |_, _| { + observed.fetch_add(1, Ordering::SeqCst); + async { Ok(json!("must-not-enter")) } + }); + assert!(matches!( + worker.execute_workflow_task_decision(task(&fixture())), + Err(Error::CancellationScopeExecutionUnavailable) + )); + assert_eq!(calls.load(Ordering::SeqCst), 0); + } +} + +#[test] +fn cancellation_scope_replay_bad_projection_refuses_before_factory() { + let mut source = fixture(); + for event in source["history"].as_array_mut().unwrap() { + if event["event_type"] == "CancellationScopeDeliveryPrepared" { + event["payload"]["timer_members"][0]["descriptor_hash"] = json!("0".repeat(64)); + } + } + let calls = Arc::new(AtomicUsize::new(0)); + let observed = Arc::clone(&calls); + let mut worker = Worker::new(Client::new("http://unused.invalid").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + worker.register_workflow("scope-replay", move |_, _| { + observed.fetch_add(1, Ordering::SeqCst); + async { Ok(json!("must-not-enter")) } + }); + assert!(worker + .execute_workflow_task_decision(task(&source)) + .is_err()); + assert_eq!(calls.load(Ordering::SeqCst), 0); +} diff --git a/src/tests/cancellation_scope_replay/descendants.rs b/src/tests/cancellation_scope_replay/descendants.rs new file mode 100644 index 0000000..a18cbf7 --- /dev/null +++ b/src/tests/cancellation_scope_replay/descendants.rs @@ -0,0 +1,545 @@ +use super::*; + +type Observed = Arc>>; + +fn descendant_fixture(layout: &str) -> Value { + let fixtures: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/committed-scope-descendants.json" + )) + .unwrap(); + fixtures[layout].clone() +} + +async fn capture( + ctx: &WorkflowContext, + result: Result<()>, + name: &str, + source: &Value, + seen: &Observed, + cleanup: &str, +) -> Result { + let context = match result { + Err(Error::CancellationScopeRequested(request)) => { + assert_eq!( + request.delivery.request_id, + source["contexts"]["parent"]["lineage"] + .as_array() + .unwrap() + .last() + .unwrap()["request_id"] + ); + request.context + } + Err(error) => return Err(error), + Ok(()) => panic!("{name} must retain its original accepted request"), + }; + assert!(ctx.is_cancellation_requested()?); + assert_eq!(ctx.scoped_cancellation_context()?.unwrap(), context); + assert_eq!(context.to_value(), source["contexts"][name]); + let rank = |scope| match scope { + "grandchild" => 0, + "child" => 1, + _ => 2, + }; + let fired = source["history"].as_array().unwrap().iter().any(|row| { + row["event_type"] == "TimerFired" && row["payload"]["timer_id"] == "descendant-cleanup" + }); + assert_eq!( + context.remaining()?, + Duration::from_secs( + if fired && !cleanup.is_empty() && rank(name) > rank(cleanup) { + 15 + } else { + 17 + } + ) + ); + seen.lock().unwrap().insert(name.into(), context); + if cleanup == name { + let _shield = ctx.cancellation_shield()?; + ctx.sleep(Duration::from_secs(1)).await?; + } + Ok(Value::Null) +} + +async fn grandchild_body( + ctx: WorkflowContext, + source: Value, + seen: Observed, + cleanup: &'static str, + changed: &'static str, +) -> Result { + if changed != "prefix" { + assert_eq!( + ctx.activity("prior-step", json!([])).await?, + json!("prior-value") + ); + } + let interrupted = if source["layout"] == "timer" { + ctx.sleep(Duration::from_secs(if changed == "timer" { + 42 + } else { + 3600 + })) + .await + } else { + let predicate = source["history"] + .as_array() + .unwrap() + .iter() + .find(|row| row["event_type"] == "ConditionWaitOpened") + .unwrap()["payload"]["condition_definition_fingerprint"] + .as_str() + .unwrap(); + let activity = ParallelOperation::activity_with_options( + if changed == "activity" { + "changed" + } else { + "original-activity" + }, + ActivityOptions::new().cancellation_policy(if changed == "activity-policy" { + CancellationPolicy::Abandon + } else { + CancellationPolicy::TryCancel + }), + json!([]), + ); + let timer = ParallelOperation::timer(Duration::from_secs(if changed == "timer" { + 42 + } else { + 3600 + })); + let child = ParallelOperation::child_workflow( + if changed == "child" { + "changed" + } else { + "original-child" + }, + ChildWorkflowOptions::new("queue") + .parent_close_policy(ParentClosePolicy::RequestCancel) + .cancellation_policy(if changed == "child-policy" { + CancellationPolicy::Abandon + } else { + CancellationPolicy::WaitCancellationCompleted + }), + json!([]), + ); + let condition = ParallelOperation::condition( + ConditionWaitOptions::new("ready", predicate).timeout(Duration::from_secs(30)), + || panic!("committed cancellation must not evaluate the interrupted predicate"), + ); + let mut group = if changed == "layout" { + vec![activity, timer, child, condition] + } else { + vec![ + activity, + ParallelOperation::group(vec![timer, child]), + condition, + ] + }; + if changed == "size" { + group.pop(); + } + ctx.parallel(group).await.map(|_| ()) + }; + capture(&ctx, interrupted, "grandchild", &source, &seen, cleanup).await +} + +async fn child_body( + ctx: WorkflowContext, + source: Value, + seen: Observed, + cleanup: &'static str, + changed: &'static str, +) -> Result { + let value = source.clone(); + let observed = Arc::clone(&seen); + ctx.cancellation_scope(changed == "shield", move |inner| { + grandchild_body(inner, value, observed, cleanup, changed) + }) + .await?; + capture( + &ctx, + ctx.throw_if_cancellation_requested(), + "child", + &source, + &seen, + cleanup, + ) + .await +} + +async fn parent_body( + ctx: WorkflowContext, + source: Value, + seen: Observed, + cleanup: &'static str, + changed: &'static str, +) -> Result { + ctx.cancellation_scope(true, move |shield| async move { + assert!(!shield.is_cancellation_requested()?); + assert!(shield.scoped_cancellation_context()?.is_none()); + shield + .cancellation_scope(false, move |inner| async move { + assert!(!inner.is_cancellation_requested()?); + assert!(inner.scoped_cancellation_context()?.is_none()); + Ok(Value::Null) + }) + .await + }) + .await?; + let value = source.clone(); + let observed = Arc::clone(&seen); + ctx.cancellation_scope(false, move |inner| { + child_body(inner, value, observed, cleanup, changed) + }) + .await?; + capture( + &ctx, + ctx.throw_if_cancellation_requested(), + "parent", + &source, + &seen, + cleanup, + ) + .await +} + +fn descendant_worker( + source: Value, + seen: Observed, + cleanup: &'static str, + changed: &'static str, +) -> Worker { + let mut worker = Worker::new(Client::new("http://unused.invalid").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + worker.register_workflow("scope-replay", move |ctx, _| { + let source = source.clone(); + let observed = Arc::clone(&seen); + async move { + let value = source.clone(); + let inside = Arc::clone(&observed); + ctx.cancellation_scope(false, move |outer| async move { + outer + .cancellation_scope(false, move |parent| { + parent_body(parent, value, inside, cleanup, changed) + }) + .await?; + assert!(!outer.is_cancellation_requested()?); + assert!(outer.scoped_cancellation_context()?.is_none()); + Ok(Value::Null) + }) + .await?; + assert!(!ctx.is_cancellation_requested()?); + assert!(ctx.scoped_cancellation_context()?.is_none()); + ctx.sleep(Duration::from_secs(1)).await?; + Ok(if cleanup.is_empty() { + json!("survivor") + } else { + json!({"remaining":observed.lock().unwrap()[cleanup].remaining()?.as_secs()}) + }) + } + }); + worker +} + +fn next_sequence(source: &Value) -> u64 { + if source["layout"] == "timer" { + 9 + } else { + 12 + } +} + +fn snapshot_for(source: &Value, target: &str) -> Value { + let context = ScopedCancellationContext::from_value(&source["contexts"][target]).unwrap(); + let ancestor = ScopedCancellationContext::from_value(&source["contexts"]["parent"]).unwrap(); + json!({"scope_id":ancestor.scope_id(), "operation_scope_id":context.scope_id(), + "request_id":ancestor.request_id(), "root_request_id":ancestor.root_context().root_request_id(), + "delivery_history_event_id":"ancestor-delivered", "preparation_history_event_id":"ancestor-prepared", + "cleanup_deadline_at":"2026-10-04T00:00:30.123456Z", "authority_deadline_at":"2026-10-04T00:00:26.123456Z"}) +} + +#[test] +fn descendant_original_contexts_restore_and_leave_outer_and_root_unaffected() { + for layout in ["timer", "group"] { + let source = descendant_fixture(layout); + let seen = Arc::new(Mutex::new(BTreeMap::new())); + let worker = descendant_worker(source.clone(), Arc::clone(&seen), "", ""); + let decision = worker + .execute_workflow_task_decision(task(&source)) + .unwrap(); + assert_eq!(seen.lock().unwrap().len(), 3); + assert_eq!( + decision.commands, + vec![json!({"type":"start_timer", "delay_seconds":1})] + ); + } +} + +#[test] +fn descendant_pending_ancestor_preserves_original_identity_and_range_without_cleanup() { + for layout in ["timer", "group"] { + for (before, inherited) in [ + ("CancellationScopeDeliveryPrepared", false), + ("CancellationScopeDeliveryPrepared", true), + ("CancellationScopeDelivered", true), + ] { + let mut source = prefix(&descendant_fixture(layout), before); + if !inherited { + let parent = source["scopes"]["parent"].clone(); + source["history"].as_array_mut().unwrap().retain(|row| { + row["event_type"] != "CancellationScopeRequested" + || row["payload"]["scope_id"] == parent + }); + } + let seen = Arc::new(Mutex::new(BTreeMap::new())); + let decision = descendant_worker(source.clone(), Arc::clone(&seen), "", "") + .execute_workflow_task_decision(task(&source)) + .unwrap(); + assert!(decision.commands.is_empty()); + assert!(seen.lock().unwrap().is_empty()); + let intent = decision.cancellation_scope_delivery.unwrap(); + assert_eq!(intent.context.to_value(), source["contexts"]["parent"]); + assert_eq!(intent.boundary.sequence, 8); + assert_eq!( + intent.boundary.sequence_span, + if layout == "timer" { 1 } else { 4 } + ); + let mut replacement = task(&source); + replacement.lease_owner = Some("replacement".into()); + replacement.workflow_task_attempt = 17; + let repeated = descendant_worker(source, Arc::clone(&seen), "", "") + .execute_workflow_task_decision(replacement) + .unwrap() + .cancellation_scope_delivery + .unwrap(); + assert_eq!(repeated.context, intent.context); + assert_eq!(repeated.boundary, intent.boundary); + assert!(seen.lock().unwrap().is_empty()); + } + } +} + +#[test] +fn descendant_pending_ancestor_cannot_bypass_competing_intermediate_request() { + let fixtures: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/committed-scope-operation-projections.json" + )) + .unwrap(); + let mut source = prefix(&fixtures["competing"], "CancellationScopeDeliveryPrepared"); + source["history"].as_array_mut().unwrap().retain(|row| { + row["event_type"] != "CancellationScopeRequested" + || row["payload"]["scope_id"] != "desc-grandchild" + }); + let claim = task(&source); + let scopes = cancellation_scope::CancellationScopeHistory::read( + &claim.history_events, + claim.run_id.as_deref().unwrap(), + ) + .unwrap(); + let committed = crate::cancellation_scope_history::CommittedCancellationScopeHistory::read( + &claim.history_events, + claim.run_id.as_deref().unwrap(), + claim.workflow_id.as_deref().unwrap(), + ) + .unwrap(); + let error = committed + .pending_request_for_scope("desc-grandchild", &scopes) + .unwrap_err(); + assert!(error.to_string().contains("original ancestor lineage")); +} + +#[test] +fn descendant_cleanup_retains_ancestor_proof_narrower_authority_and_replacement_clock() { + for layout in ["timer", "group"] { + for target in ["grandchild", "child", "parent"] { + let mut source = descendant_fixture(layout); + let seen = Arc::new(Mutex::new(BTreeMap::new())); + let snapshot = snapshot_for(&source, target); + let sequence = next_sequence(&source); + let wire = json!({"type":"start_timer", "delay_seconds":1, + "cancellation_scope_id":snapshot["operation_scope_id"], "cancellation_cleanup":{ + "scope_id":snapshot["scope_id"], "request_id":snapshot["request_id"], + "delivery_history_event_id":snapshot["delivery_history_event_id"]}}); + assert_eq!( + descendant_worker(source.clone(), Arc::clone(&seen), target, "") + .execute_workflow_task_decision(task(&source)) + .unwrap() + .commands, + vec![wire.clone()] + ); + append( + &mut source, + "TimerScheduled", + json!({"sequence":sequence, + "timer_id":"descendant-cleanup", "delay_seconds":1, + "cancellation_scope_id":snapshot["operation_scope_id"], "cancellation_cleanup":snapshot, + "fire_at":"2026-10-04T00:00:11.123456Z"}), + "2026-10-04T00:00:10.123456Z", + ); + let mut claim = task(&source); + claim.lease_owner = Some("replacement".into()); + claim.workflow_task_attempt = 17; + let blocked = descendant_worker(source.clone(), Arc::clone(&seen), target, "") + .execute_workflow_task_decision(claim) + .unwrap(); + assert!( + blocked.commands.is_empty(), + "an admitted cleanup timer must remain pending" + ); + assert!(blocked.cancellation_scope_delivery.is_none()); + append( + &mut source, + "TimerFired", + json!({"sequence":sequence, + "timer_id":"descendant-cleanup", "delay_seconds":1, + "cancellation_scope_id":snapshot["operation_scope_id"]}), + "2026-10-04T00:00:11.123456Z", + ); + assert_eq!( + descendant_worker(source.clone(), Arc::clone(&seen), target, "") + .execute_workflow_task_decision(task(&source)) + .unwrap() + .commands, + vec![json!({"type":"start_timer", "delay_seconds":1})] + ); + append( + &mut source, + "TimerScheduled", + json!({"sequence":sequence+1, "timer_id":"root-timer", + "delay_seconds":1, "fire_at":"2026-10-04T00:00:12.123456Z"}), + "2026-10-04T00:00:11.123456Z", + ); + append( + &mut source, + "TimerFired", + json!({"sequence":sequence+1, "timer_id":"root-timer", + "delay_seconds":1}), + "2026-10-04T00:00:20.123456Z", + ); + let result = descendant_worker(source.clone(), Arc::clone(&seen), target, "") + .execute_workflow_task_decision(task(&source)) + .unwrap(); + assert_eq!(result.commands[0]["type"], "complete_workflow"); + assert_eq!( + decode_payload::( + &serde_json::from_value(result.commands[0]["result"].clone()).unwrap() + ) + .unwrap(), + json!({"remaining":6}) + ); + } + } +} + +#[test] +fn descendant_changed_subtree_cannot_enter_cleanup() { + for (layout, change) in [ + ("timer", "shield"), + ("timer", "timer"), + ("timer", "prefix"), + ("group", "activity"), + ("group", "activity-policy"), + ("group", "child"), + ("group", "child-policy"), + ("group", "layout"), + ("group", "size"), + ] { + let source = descendant_fixture(layout); + let seen = Arc::new(Mutex::new(BTreeMap::new())); + assert!( + descendant_worker(source.clone(), Arc::clone(&seen), "", change) + .execute_workflow_task_decision(task(&source)) + .is_err(), + "{layout}/{change}" + ); + assert!(seen.lock().unwrap().is_empty(), "{layout}/{change}"); + } +} + +#[test] +fn descendant_changed_cleanup_snapshot_refuses_before_factory() { + for field in [ + "scope_id", + "operation_scope_id", + "request_id", + "root_request_id", + "delivery_history_event_id", + "preparation_history_event_id", + "cleanup_deadline_at", + "authority_deadline_at", + "omitted", + "null", + "retrograde", + ] { + let mut source = descendant_fixture("timer"); + let mut snapshot = snapshot_for(&source, "grandchild"); + if field == "null" { + snapshot = Value::Null; + } else if field != "omitted" && field != "retrograde" { + snapshot[field] = json!("changed"); + } + let mut payload = json!({"sequence":9, "timer_id":"descendant-cleanup", "delay_seconds":1, + "cancellation_scope_id":"grandchild-scope", "cancellation_cleanup":snapshot, + "fire_at":if field=="retrograde" { "2026-10-04T00:00:09.123456Z" } else { "2026-10-04T00:00:11.123456Z" }}); + if field == "omitted" { + payload + .as_object_mut() + .unwrap() + .remove("cancellation_cleanup"); + } + append( + &mut source, + "TimerScheduled", + payload, + "2026-10-04T00:00:10.123456Z", + ); + let invoked = Arc::new(AtomicUsize::new(0)); + let calls = Arc::clone(&invoked); + let mut worker = descendant_worker( + source.clone(), + Arc::new(Mutex::new(BTreeMap::new())), + "", + "", + ); + worker.register_workflow("scope-replay", move |_, _| { + calls.fetch_add(1, Ordering::SeqCst); + async { Ok(Value::Null) } + }); + assert!( + worker + .execute_workflow_task_decision(task(&source)) + .is_err(), + "{field}" + ); + assert_eq!(invoked.load(Ordering::SeqCst), 0, "{field}"); + } +} + +#[test] +fn descendant_competing_root_refuses_before_factory() { + let fixtures: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/committed-scope-operation-projections.json" + )) + .unwrap(); + let source = fixtures["competing"].clone(); + let invoked = Arc::new(AtomicUsize::new(0)); + let calls = Arc::clone(&invoked); + let mut worker = descendant_worker( + source.clone(), + Arc::new(Mutex::new(BTreeMap::new())), + "", + "", + ); + worker.register_workflow("scope-replay", move |_, _| { + calls.fetch_add(1, Ordering::SeqCst); + async { Ok(Value::Null) } + }); + assert!(worker + .execute_workflow_task_decision(task(&source)) + .is_err()); + assert_eq!(invoked.load(Ordering::SeqCst), 0); +} diff --git a/src/tests/cancellation_scope_replay/run_scopes.rs b/src/tests/cancellation_scope_replay/run_scopes.rs new file mode 100644 index 0000000..fde630b --- /dev/null +++ b/src/tests/cancellation_scope_replay/run_scopes.rs @@ -0,0 +1,311 @@ +use super::*; +use std::collections::BTreeMap; + +fn source() -> Value { + serde_json::from_str(include_str!( + "../../../tests/fixtures/run-inherited-scope-timers.json" + )) + .unwrap() +} + +fn at_phase(source: &Value, phase: &str) -> Value { + let mut value = source.clone(); + let end = value["history_ranges"][phase].as_u64().unwrap() as usize; + value["history"].as_array_mut().unwrap().truncate(end); + value +} + +fn probe(seen: Arc>>, changed: &'static str) -> Worker { + let mut worker = Worker::new(Client::new("http://unused.invalid").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + worker.register_workflow("scope-replay", move |ctx, _| { + let seen = Arc::clone(&seen); + async move { + let root = ctx.clone(); + let scoped_seen = Arc::clone(&seen); + let captured = Arc::new(Mutex::new(None::)); + let captured_scope = Arc::clone(&captured); + ctx.cancellation_scope(false, move |outer| async move { + outer + .cancellation_scope(false, move |inner| async move { + *captured_scope.lock().unwrap() = Some(inner.clone()); + let cancelled = match inner.sleep(Duration::from_secs(300)).await { + Err(Error::CancellationScopeRequested(cancelled)) => cancelled, + other => panic!("Expected the original scoped delivery, got {other:?}"), + }; + assert!(inner.cancellation_context()?.is_none()); + assert_eq!( + inner.scoped_cancellation_context()?.unwrap().to_value(), + cancelled.context.to_value() + ); + scoped_seen + .lock() + .unwrap() + .insert("scoped".into(), cancelled.context.to_value()); + scoped_seen.lock().unwrap().insert( + "scoped_remaining".into(), + json!(cancelled.context.remaining()?.as_secs_f64()), + ); + let _shield = inner.cancellation_shield()?; + if changed == "unshielded" { + drop(_shield); + } + inner.sleep(Duration::from_secs(2)).await?; + scoped_seen.lock().unwrap().insert( + "scoped_after".into(), + json!(cancelled.context.remaining()?.as_secs_f64()), + ); + Ok(Value::Null) + }) + .await + }) + .await?; + if matches!( + changed, + "captured_scalar" | "captured_parallel" | "captured_selection" + ) { + let captured = captured.lock().unwrap().as_ref().unwrap().clone(); + match changed { + "captured_scalar" => captured.sleep(Duration::from_secs(300)).await?, + "captured_parallel" => { + captured + .parallel(vec![ParallelOperation::timer(Duration::from_secs(300))]) + .await?; + } + _ => { + captured + .select_keyed(vec![( + "timer", + ParallelOperation::timer(Duration::from_secs(300)), + )]) + .await?; + } + } + } + if changed == "new_scope_before_root" { + return root + .cancellation_scope(false, |scope| async move { + scope.sleep(Duration::from_secs(300)).await?; + Ok(Value::Null) + }) + .await; + } + let Err(Error::CooperativeCancellationRequested(cancelled)) = + root.sleep(Duration::from_secs(300)).await + else { + return Err(Error::CancellationScopeExecutionUnavailable); + }; + let context = root.cancellation_context()?.unwrap(); + seen.lock() + .unwrap() + .insert("root".into(), context.to_value()); + seen.lock().unwrap().insert( + "root_remaining".into(), + json!(context.remaining()?.as_secs_f64()), + ); + { + let _shield = root.cancellation_shield()?; + if changed == "new_scope" { + root.cancellation_scope(false, |scope| async move { + scope.sleep(Duration::from_secs(1)).await?; + Ok(Value::Null) + }) + .await?; + } + root.sleep(Duration::from_secs(2)).await?; + } + seen.lock().unwrap().insert( + "root_after".into(), + json!(context.remaining()?.as_secs_f64()), + ); + Err(Error::CooperativeCancellationRequested(cancelled)) + } + }); + worker +} + +#[test] +fn run_scope_replay_preserves_ancestor_delivery_and_descendant_cleanup_before_root() { + let source = source(); + for phase in [ + "requested", + "prepared", + "scope_delivered", + "scope_cleanup_scheduled", + "scope_cleaned", + "root_delivered", + "root_cleanup_scheduled", + "root_cleaned", + ] { + let seen = Arc::new(Mutex::new(BTreeMap::new())); + let claim = task(&at_phase(&source, phase)); + let mut pending = CancellationHistory::from_events( + &claim.history_events, + claim.run_id.as_deref().unwrap(), + None, + ) + .unwrap() + .request + .unwrap(); + pending.history_refresh_page_token = Some("MA==".into()); + let decision = probe(Arc::clone(&seen), "") + .execute_workflow_task_decision_with_cancellation(claim, Some(&pending)) + .unwrap_or_else(|error| panic!("{phase}: {error:?}")); + let observed = seen.lock().unwrap(); + match phase { + "requested" | "prepared" => { + assert!(observed.is_empty()); + let intent = decision.cancellation_scope_delivery.unwrap(); + assert_eq!(intent.context.scope_id(), source["scopes"]["parent"]); + assert_eq!(intent.boundary.sequence, 3); + assert!(decision.cancellation_delivery.is_none()); + assert!(decision.commands.is_empty()); + } + "scope_delivered" => { + assert_eq!(observed["scoped_remaining"].as_f64(), Some(16.0)); + assert_eq!( + observed["scoped"]["lineage"] + .as_array() + .unwrap() + .last() + .unwrap()["scope_id"], + source["scopes"]["child"] + ); + assert_eq!( + decision.commands[0]["cancellation_scope_id"], + source["scopes"]["child"] + ); + assert_eq!( + decision.commands[0]["cancellation_cleanup"]["scope_id"], + source["scopes"]["parent"] + ); + assert!(decision.cancellation_delivery.is_none()); + } + "scope_cleanup_scheduled" => { + assert!(decision.commands.is_empty()); + assert!(decision.cancellation_delivery.is_none()); + assert!(!observed.contains_key("root")); + } + "scope_cleaned" => { + assert_eq!(observed["scoped_after"].as_f64(), Some(14.0)); + assert!(decision.commands.is_empty()); + assert_eq!(decision.cancellation_delivery.unwrap().sequence, 5); + assert!(!observed.contains_key("root")); + } + "root_delivered" => { + assert_eq!(observed["scoped_after"].as_f64(), Some(14.0)); + assert_eq!(observed["root"], observed["scoped"]["root_context"]); + assert_eq!(observed["root_remaining"].as_f64(), Some(11.0)); + assert_eq!( + decision.commands, + vec![json!({"type":"start_timer", "delay_seconds":2})] + ); + } + "root_cleanup_scheduled" => { + assert!(decision.commands.is_empty()); + assert!(!observed.contains_key("root_after")); + } + "root_cleaned" => { + assert_eq!(observed["root_after"].as_f64(), Some(9.0)); + assert_eq!(decision.commands.len(), 1); + assert_eq!(decision.commands[0]["type"], "fail_workflow"); + assert_eq!( + decision.commands[0]["exception_type"], + "WorkflowCancellationRequested" + ); + } + _ => unreachable!(), + } + } +} + +#[test] +fn run_scope_replay_refuses_changed_shielding_and_new_scope_during_root_cleanup() { + let source = source(); + for changed in [ + "unshielded", + "captured_scalar", + "captured_parallel", + "captured_selection", + "new_scope_before_root", + "new_scope", + ] { + let seen = Arc::new(Mutex::new(BTreeMap::new())); + assert!( + probe(Arc::clone(&seen), changed) + .execute_workflow_task_decision(task(&source)) + .is_err(), + "{changed}" + ); + let observed = seen.lock().unwrap(); + assert!( + observed.contains_key("scoped"), + "{changed}: valid scoped delivery must be reached" + ); + if changed == "new_scope" { + assert!(observed.contains_key("root")); + } else { + assert!(!observed.contains_key("root")); + } + } +} + +#[test] +fn run_scope_replay_refuses_changed_native_cleanup_authority_before_factory() { + let original = source(); + for field in [ + "scope_id", + "operation_scope_id", + "request_id", + "root_request_id", + "delivery_history_event_id", + "preparation_history_event_id", + "cleanup_deadline_at", + "authority_deadline_at", + "omitted", + "null", + "extra", + ] { + let mut changed = original.clone(); + let index = changed["history_ranges"]["scope_cleanup_scheduled"] + .as_u64() + .unwrap() as usize + - 1; + let payload = changed["history"][index]["payload"] + .as_object_mut() + .unwrap(); + match field { + "omitted" => { + payload.remove("cancellation_cleanup"); + } + "null" => { + payload.insert("cancellation_cleanup".into(), Value::Null); + } + "extra" => { + payload.get_mut("cancellation_cleanup").unwrap()["extra"] = json!("borrowed"); + } + _ => { + payload.get_mut("cancellation_cleanup").unwrap()[field] = json!("changed"); + } + } + let invoked = Arc::new(AtomicUsize::new(0)); + let calls = Arc::clone(&invoked); + let mut worker = Worker::new(Client::new("http://unused.invalid").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + worker.register_workflow("scope-replay", move |_, _| { + calls.fetch_add(1, Ordering::SeqCst); + async { Ok(Value::Null) } + }); + assert!( + worker + .execute_workflow_task_decision(task(&changed)) + .is_err(), + "{field}" + ); + assert_eq!(invoked.load(Ordering::SeqCst), 0, "{field}"); + } +} diff --git a/src/tests/child_workflow_policies.rs b/src/tests/child_workflow_policies.rs new file mode 100644 index 0000000..e37df14 --- /dev/null +++ b/src/tests/child_workflow_policies.rs @@ -0,0 +1,296 @@ +use super::*; + +fn policy_worker( + parent: ParentClosePolicy, + operation: CancellationPolicy, + mode: &'static str, + enabled: bool, +) -> Worker { + let mut worker = Worker::new( + Client::builder("http://127.0.0.1:1").build().unwrap(), + "queue", + ) + .cooperative_cancellation(enabled); + worker.register_workflow("child-policy", move |ctx, _input| async move { + let options = ChildWorkflowOptions::new("queue") + .parent_close_policy(parent) + .cancellation_policy(operation); + match mode { + "parallel" => { + ctx.parallel(vec![ParallelOperation::child_workflow( + "child", + options, + json!(["argument"]), + )]) + .await?; + } + "selection" => { + ctx.select_keyed(vec![( + SelectionKey::from("child"), + ParallelOperation::child_workflow("child", options, json!(["argument"])), + )]) + .await?; + } + _ => { + ctx.start_child_workflow("child", options, json!(["argument"])) + .await?; + } + } + Ok(json!("finished")) + }); + worker +} + +fn policy_task(events: Vec) -> WorkflowTask { + let mut task = workflow_task("child-policy", events, DEFAULT_CODEC); + task.run_id = Some("run".into()); + task +} + +fn policy_event(kind: &str, payload: Value) -> HistoryEvent { + serde_json::from_value(json!({"event_type": kind, "payload": payload})).unwrap() +} + +fn scheduled_policy_history(command: &Value) -> Vec { + let mut payload = command.clone(); + let object = payload.as_object_mut().unwrap(); + object.remove("type"); + object.remove("arguments"); + object.remove("queue"); + object.remove("workflow_type"); + object.insert("sequence".into(), json!(1)); + object.insert("child_workflow_type".into(), json!("child")); + object.insert("child_workflow_run_id".into(), json!("child-run")); + vec![policy_event("ChildWorkflowScheduled", payload)] +} + +fn policy_cancellation_history(mut events: Vec, mode: &str) -> Vec { + events.push(serde_json::from_value(json!({ + "event_type": "CooperativeCancellationRequested", "recorded_at": "2026-10-01T00:00:00Z", + "payload": {"workflow_run_id": "run", "workflow_command_id": "request-1", "cleanup_deadline_at": "2026-10-01T00:00:30Z"}, + })).unwrap()); + events.push(policy_event("CooperativeCancellationDelivered", json!({ + "workflow_run_id": "run", "workflow_command_id": "request-1", "sequence": 1, + "call_kind": if mode == "sequential" { "child" } else { "parallel" }, "sequence_span": 1, + }))); + events +} + +fn assert_policy_failure(error: Error, reason: &str) { + let Error::NonDeterministicReplay(failure) = error else { + panic!("expected replay failure, got {error:?}"); + }; + assert_eq!(failure.reason, reason); + assert_eq!(failure.sequence, Some(1)); +} + +#[test] +fn child_policies_encode_all_choices_and_replay_the_same_snapshot() { + for parent in [ + ParentClosePolicy::Abandon, + ParentClosePolicy::RequestCancel, + ParentClosePolicy::RequestCancellation, + ParentClosePolicy::Terminate, + ] { + for operation in [ + CancellationPolicy::Abandon, + CancellationPolicy::TryCancel, + CancellationPolicy::WaitCancellationCompleted, + ] { + let worker = policy_worker(parent, operation, "sequential", true); + let commands = worker.execute_workflow_task(policy_task(vec![])).unwrap(); + assert_eq!(commands[0]["parent_close_policy"], parent.as_str()); + if operation == CancellationPolicy::Abandon { + assert!(commands[0].get("cancellation_policy").is_none()); + } else { + assert_eq!(commands[0]["cancellation_policy"], operation.as_str()); + } + assert_eq!( + decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).unwrap(), + json!(["argument"]) + ); + let mut events = scheduled_policy_history(&commands[0]); + events.push(policy_event( + "ChildRunStarted", + json!({"sequence": 1, "child_workflow_type": "child"}), + )); + events.push(policy_event( + "ChildRunCompleted", + json!({"sequence": 1, "result": fixture_envelope(json!("child-result"))}), + )); + let completed = worker.execute_workflow_task(policy_task(events)).unwrap(); + assert_eq!( + decode_wire_value(&completed[0]["result"], DEFAULT_CODEC).unwrap(), + json!("finished") + ); + } + } +} + +#[test] +fn child_policies_are_checked_in_ordinary_parallel_selection_and_cancellation_replay() { + for mode in ["sequential", "parallel", "selection"] { + let original = policy_worker( + ParentClosePolicy::RequestCancellation, + CancellationPolicy::WaitCancellationCompleted, + mode, + true, + ); + let commands = original.execute_workflow_task(policy_task(vec![])).unwrap(); + let scheduled = scheduled_policy_history(&commands[0]); + for delivered in [false, true] { + let events = if delivered { + policy_cancellation_history(scheduled.clone(), mode) + } else { + scheduled.clone() + }; + let unchanged = original + .execute_workflow_task(policy_task(events.clone())) + .unwrap(); + if delivered { + assert_eq!(unchanged[0]["type"], "fail_workflow"); + } else { + assert!(unchanged.is_empty()); + } + for (parent, operation) in [ + ( + ParentClosePolicy::Abandon, + CancellationPolicy::WaitCancellationCompleted, + ), + ( + ParentClosePolicy::RequestCancellation, + CancellationPolicy::TryCancel, + ), + ] { + let changed = policy_worker(parent, operation, mode, true); + assert_policy_failure( + changed + .execute_workflow_task(policy_task(events.clone())) + .unwrap_err(), + "child_workflow_policy_changed", + ); + } + } + } +} + +#[test] +fn child_policies_preserve_historical_defaults_and_refuse_later_changes() { + let worker = policy_worker( + ParentClosePolicy::Abandon, + CancellationPolicy::Abandon, + "sequential", + false, + ); + let commands = worker.execute_workflow_task(policy_task(vec![])).unwrap(); + assert!(commands[0].get("cancellation_policy").is_none()); + let mut events = scheduled_policy_history(&commands[0]); + events[0] + .payload + .as_object_mut() + .unwrap() + .remove("parent_close_policy"); + assert!(worker + .execute_workflow_task(policy_task(events.clone())) + .unwrap() + .is_empty()); + let changed = policy_worker( + ParentClosePolicy::RequestCancellation, + CancellationPolicy::WaitCancellationCompleted, + "sequential", + true, + ); + assert_policy_failure( + changed + .execute_workflow_task(policy_task(events)) + .unwrap_err(), + "child_workflow_policy_changed", + ); +} + +#[test] +fn child_policies_reject_invalid_and_conflicting_history() { + let worker = policy_worker( + ParentClosePolicy::RequestCancellation, + CancellationPolicy::WaitCancellationCompleted, + "sequential", + true, + ); + let commands = worker.execute_workflow_task(policy_task(vec![])).unwrap(); + let scheduled = scheduled_policy_history(&commands[0]); + for event_type in [ + "ChildRunStarted", + "ChildRunCompleted", + "ChildRunFailed", + "ChildRunCancelled", + "ChildRunTerminated", + ] { + for (field, conflicting) in [ + ("parent_close_policy", "terminate"), + ("cancellation_policy", "try_cancel"), + ] { + let mut events = scheduled.clone(); + let mut payload = json!({"sequence": 1}); + payload[field] = json!(conflicting); + events.push(policy_event(event_type, payload)); + assert_policy_failure( + worker + .execute_workflow_task(policy_task(events)) + .unwrap_err(), + "child_workflow_policy_history_conflict", + ); + } + } + for field in ["parent_close_policy", "cancellation_policy"] { + for invalid in [ + json!("unknown"), + json!(true), + json!(1), + json!([]), + json!({"type": "abandon"}), + ] { + let mut events = scheduled.clone(); + events[0].payload[field] = invalid; + assert_policy_failure( + worker + .execute_workflow_task(policy_task(events)) + .unwrap_err(), + "invalid_child_workflow_policy_history", + ); + } + } +} + +#[test] +fn child_policies_require_worker_opt_in_but_legacy_choices_remain_available() { + for (parent, operation) in [ + ( + ParentClosePolicy::RequestCancellation, + CancellationPolicy::Abandon, + ), + (ParentClosePolicy::Abandon, CancellationPolicy::TryCancel), + ( + ParentClosePolicy::Abandon, + CancellationPolicy::WaitCancellationCompleted, + ), + ] { + let worker = policy_worker(parent, operation, "sequential", false); + let Error::CooperativeCancellationUnavailable(message) = worker + .execute_workflow_task(policy_task(vec![])) + .unwrap_err() + else { + panic!("expected capability diagnostic"); + }; + assert!(message.contains("child_cancellation_policy_not_supported")); + assert!(message.contains(&worker.worker_id)); + assert!(message.contains("1.20")); + } + for parent in [ + ParentClosePolicy::Abandon, + ParentClosePolicy::RequestCancel, + ParentClosePolicy::Terminate, + ] { + let worker = policy_worker(parent, CancellationPolicy::Abandon, "sequential", false); + assert!(worker.execute_workflow_task(policy_task(vec![])).is_ok()); + } +} diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs new file mode 100644 index 0000000..bb08e99 --- /dev/null +++ b/src/tests/cooperative_cancellation.rs @@ -0,0 +1,5567 @@ +use super::*; + +fn context_snapshot() -> Value { + serde_json::from_str(include_str!( + "fixtures/cooperative-cancellation-context.json" + )) + .unwrap() +} + +fn scoped_run_snapshot(name: &str) -> Value { + let fixtures: Value = serde_json::from_str(include_str!( + "fixtures/scoped-run-cancellation-context.json" + )) + .unwrap(); + fixtures[name].clone() +} + +fn scoped_run_history() -> Vec { + vec![serde_json::from_value(json!({ + "event_type":"CooperativeCancellationRequested", "timestamp":"2026-10-04T00:00:05Z", + "payload": {"workflow_run_id":"child-run", "workflow_instance_id":"child-instance", + "workflow_command_id":"child-request", "reason":"maintenance", + "cleanup_deadline_at":"2026-10-04T00:00:15.123456Z", "cancellation":scoped_run_snapshot("child")} + })).unwrap(), remaining_event("CooperativeCancellationDelivered", json!({ + "workflow_run_id":"child-run", "workflow_command_id":"child-request", "sequence":1, + "call_kind":"timer", "cancellation":scoped_run_snapshot("child") + }), "2026-10-04T00:00:08Z")] +} + +fn scoped_run_task(history: Vec) -> WorkflowTask { + let mut task = cancellation_task(history); + task.run_id = Some("child-run".into()); + task +} + +#[test] +fn scoped_run_native_child_and_grandchild_preserve_every_origin_and_original_budget() { + for name in ["child", "grandchild"] { + let snapshot = scoped_run_snapshot(name); + let context = CancellationContext::from_value(&snapshot).unwrap(); + assert_eq!(context.to_value(), snapshot); + assert_eq!( + CancellationContext::from_value(&context.to_value()).unwrap(), + context + ); + assert_eq!(context.root_request_id(), "root-request"); + assert_eq!(context.reason(), Some("maintenance")); + assert_eq!(context.source(), "api"); + assert_eq!(context.requester()["id"], "operator-1"); + assert_eq!( + context + .requested_at() + .to_rfc3339_opts(chrono::SecondsFormat::Micros, true), + "2026-10-04T00:00:00.123456Z" + ); + let origin = context.scope_origin().unwrap(); + assert_eq!( + origin + .root_deadline() + .to_rfc3339_opts(chrono::SecondsFormat::Micros, true), + "2026-10-04T00:00:30.123456Z" + ); + assert_eq!(origin.requested_at(), context.requested_at()); + let (scopes, parent, deadline) = if name == "child" { + ( + vec!["outer", "inner"], + "inner-request", + "2026-10-04T00:00:15.123456Z", + ) + } else { + ( + vec!["outer", "inner", "root", "child-scope"], + "child-scope-request", + "2026-10-04T00:00:12.123456Z", + ) + }; + assert_eq!(context.parent_request_id(), Some(parent)); + assert_eq!( + origin + .lineage() + .iter() + .map(|hop| hop.scope_id()) + .collect::>(), + scopes + ); + assert_eq!( + context + .deadline() + .to_rfc3339_opts(chrono::SecondsFormat::Micros, true), + deadline + ); + assert!(matches!( + context.remaining(), + Err(Error::InvalidCooperativeCancellation(_)) + )); + } +} + +#[test] +fn scoped_run_avro_timezone_and_detached_metadata_keep_the_original_tree() { + let original = scoped_run_snapshot("grandchild"); + let mut snapshot = original.clone(); + snapshot["requested_at"] = json!("2026-10-03T20:00:00.123456-04:00"); + snapshot["cleanup_deadline_at"] = json!("2026-10-03T20:00:12.123456-04:00"); + snapshot["scope_authority_deadline_at"] = snapshot["cleanup_deadline_at"].clone(); + let decoded = decode_wire_value(&fixture_envelope(snapshot), DEFAULT_CODEC).unwrap(); + let context = CancellationContext::from_value(&decoded).unwrap(); + assert_eq!(context.to_value(), original); + let origin = context.scope_origin().unwrap(); + let mut detached = origin.to_value(); + detached["lineage"][3]["scope_id"] = json!("changed"); + assert_eq!(origin.scope_id(), "child-scope"); + assert_eq!(origin.root_scope_id(), "outer"); + assert_eq!(origin.request_id(), "child-scope-request"); + assert_eq!(origin.parent_request_id(), Some("child-request")); + assert_eq!(origin.workflow_instance_id(), "child-instance"); + assert_eq!(origin.workflow_run_id(), "child-run"); + assert_eq!(origin.to_value(), original["scope_origin"]); +} + +#[test] +fn scoped_run_cold_cleanup_replay_consumes_the_same_narrowed_original_clock() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + assert!(ctx.cancellation_context()?.is_none()); + let Err(Error::CooperativeCancellationRequested(cancelled)) = ctx.sleep(Duration::from_secs(10)).await else { + panic!("expected original cancellation"); + }; + let context = cancelled.request.context.unwrap(); + assert_eq!(context.to_value(), scoped_run_snapshot("child")); + assert_eq!(ctx.cancellation_context()?, Some(context.clone())); + let before = context.remaining()?.as_secs_f64(); assert_eq!(before, 7.123456); + let _shield = ctx.cancellation_shield()?; + ctx.activity("cleanup", json!([])).await?; + Ok(json!({"remaining":[before,context.remaining()?.as_secs_f64()], "cancellation":context.to_value()})) + }); + let mut history = scoped_run_history(); + let initial = worker + .execute_workflow_task(scoped_run_task(history.clone())) + .unwrap(); + assert_eq!(initial.len(), 1); + assert_eq!(initial[0]["activity_type"], "cleanup"); + let mut completion = completed_activity(2, "cleanup", json!("cleaned")); + completion + .last_mut() + .unwrap() + .raw + .insert("timestamp".into(), json!("2026-10-04T00:00:12Z")); + history.extend(completion); + history.push(remaining_event( + "WorkflowTaskScheduled", + json!({}), + "2026-10-04T00:00:29Z", + )); + for _replacement in 0..2 { + let result = worker + .execute_workflow_task(scoped_run_task(history.clone())) + .unwrap(); + assert_eq!(result[0]["type"], "complete_workflow"); + assert_eq!( + decode_wire_value(&result[0]["result"], DEFAULT_CODEC).unwrap(), + json!({ + "remaining":[7.123456,3.123456],"cancellation":scoped_run_snapshot("child") + }) + ); + } +} + +#[test] +fn scoped_run_canonical_delivery_cannot_replace_an_original_scope() { + let mut history = scoped_run_history(); + history[1].payload["cancellation"]["scope_origin"]["lineage"][1]["scope_id"] = + json!("different"); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + ctx.sleep(Duration::from_secs(10)).await?; + Ok(Value::Null) + }); + assert!(worker + .execute_workflow_task(scoped_run_task(history)) + .is_err()); +} + +#[test] +fn scoped_run_invalid_origin_identity_and_budget_are_refused() { + let original = scoped_run_snapshot("grandchild"); + for (path, value) in [ + ("/schema", json!("durable-workflow.cancellation-context/v1")), + ("/root_request_id", json!("other")), + ("/root_workflow_instance_id", json!("other")), + ("/root_workflow_run_id", json!("other")), + ("/parent_request_id", json!("root-request")), + ("/reason", json!("other")), + ("/source", json!("other")), + ("/requested_at", json!("2026-10-04T00:00:01.123456Z")), + ("/scope_origin", json!([])), + ("/cleanup_deadline_at", json!("2026-10-04T00:00:15.123456Z")), + ( + "/scope_authority_deadline_at", + json!("2026-10-04T00:00:15.123456Z"), + ), + ("/requester/id", json!("other")), + ("/lineage/1/request_id", json!("child-request")), + ("/lineage/2/workflow_run_id", json!("child-run")), + ( + "/scope_origin/lineage/3/cleanup_deadline_at", + json!("2026-10-04T00:00:16.123456Z"), + ), + ( + "/scope_origin/lineage/3/workflow_instance_id", + json!("other"), + ), + ("/scope_origin/lineage/3/request_id", json!("inner-request")), + ("/scope_origin/lineage/3/scope_id", json!("root")), + ( + "/scope_origin/root_context/schema", + json!("durable-workflow.cancellation-context/v2"), + ), + ] { + let mut snapshot = original.clone(); + *snapshot.pointer_mut(path).unwrap() = value; + assert!( + matches!( + CancellationContext::from_value(&snapshot), + Err(Error::InvalidCooperativeCancellation(_)) + ), + "{path}" + ); + } + for key in [ + "scope_origin", + "scope_authority_deadline_at", + "parent_request_id", + ] { + let mut snapshot = original.clone(); + snapshot.as_object_mut().unwrap().remove(key); + assert!( + CancellationContext::from_value(&snapshot).is_err(), + "missing {key}" + ); + } + for fault in [ + "widened global budget", + "reused request", + "discarded root", + "run reentry", + "unsupported field", + ] { + let mut snapshot = original.clone(); + match fault { + "widened global budget" => { + snapshot["cleanup_deadline_at"] = json!("2026-10-04T00:00:30.123456Z"); + snapshot["scope_authority_deadline_at"] = snapshot["cleanup_deadline_at"].clone(); + } + "reused request" => { + snapshot["request_id"] = json!("inner-request"); + snapshot["lineage"][2]["request_id"] = json!("inner-request"); + } + "discarded root" => { + snapshot["scope_origin"]["lineage"] + .as_array_mut() + .unwrap() + .remove(0); + } + "run reentry" => { + snapshot["scope_origin"]["lineage"][3]["workflow_run_id"] = json!("root-run"); + snapshot["scope_origin"]["lineage"][3]["workflow_instance_id"] = + json!("root-instance"); + } + _ => snapshot["scope_origin"]["lineage"][3]["authority"] = json!("unrecorded"), + } + assert!( + matches!( + CancellationContext::from_value(&snapshot), + Err(Error::InvalidCooperativeCancellation(_)) + ), + "{fault}" + ); + } +} + +fn context_request() -> HistoryEvent { + serde_json::from_value(json!({ + "event_type": "CooperativeCancellationRequested", "recorded_at": "2026-10-01T00:00:05Z", + "payload": { + "workflow_command_id": "request-1", "workflow_run_id": "run-1", "workflow_instance_id": "child-instance", + "reason": "maintenance", "cleanup_deadline_at": "2026-10-01T00:00:30.123456Z", + "cancellation": context_snapshot(), + } + })).unwrap() +} + +fn context_delivery() -> HistoryEvent { + event( + "CooperativeCancellationDelivered", + json!({ + "workflow_command_id": "request-1", "workflow_run_id": "run-1", + "sequence": 1, "call_kind": "timer", "cancellation": context_snapshot(), + }), + ) +} + +fn context_observation() -> CancellationRequest { + CancellationRequest::from_observation(&json!({ + "request_id": "request-1", "requested_at": "2026-10-01T00:00:05Z", + "cleanup_deadline_at": "2026-10-01T00:00:30.123456Z", + "history_refresh_page_token": "opaque-first-page", "cancellation": context_snapshot(), + })) + .unwrap() +} + +fn context_task(events: Vec) -> WorkflowTask { + let mut task = cancellation_task(events); + task.run_id = Some("run-1".into()); + task +} + +fn remaining_delivery() -> HistoryEvent { + let mut delivery = context_delivery(); + delivery + .raw + .insert("timestamp".into(), json!("2026-10-01T00:00:08Z")); + delivery +} + +fn remaining_event(kind: &str, payload: Value, time: &str) -> HistoryEvent { + let mut result = event(kind, payload); + result.raw.insert("timestamp".into(), json!(time)); + result +} + +#[test] +fn cancellation_remaining_preserves_memo_decision_and_consumes_the_timer_on_cold_replay() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + let Err(Error::CooperativeCancellationRequested(cancelled)) = + ctx.sleep(Duration::from_secs(10)).await + else { + panic!("expected delivery"); + }; + let context = cancelled.request.context.unwrap(); + assert_eq!(context.remaining()?, Duration::new(22, 123456000)); + assert_eq!( + ctx.cancellation_context()?.unwrap().remaining()?, + context.remaining()? + ); + assert_eq!( + CancellationContext::from_value(&context.to_value())?, + context + ); + let _shield = ctx.cancellation_shield()?; + ctx.upsert_memo(json!({"phase": "cleanup"}))?; + let delay = if context.remaining()? == Duration::new(22, 123456000) { + 1 + } else { + 2 + }; + ctx.sleep(Duration::from_secs(delay)).await?; + Ok(json!(context.remaining()?.as_secs_f64())) + }); + let initial = vec![context_request(), remaining_delivery()]; + let commands = worker + .execute_workflow_task(context_task(initial.clone())) + .unwrap(); + assert_eq!(commands[0]["type"], "upsert_memo"); + assert_eq!(commands[1]["delay_seconds"], 1); + let mut history = initial; + history.push(remaining_event( + "MemoUpserted", + json!({"sequence":2,"entries":commands[0]["entries"],"merged":commands[0]["entries"]}), + "2026-10-01T00:00:12Z", + )); + history.push(event( + "TimerScheduled", + json!({"sequence":3,"timer_id":"cleanup-timer","delay_seconds":1}), + )); + // A later unrelated row must not move an earlier application decision. + history.push(remaining_event( + "WorkflowTaskScheduled", + json!({}), + "2026-10-01T00:00:29Z", + )); + assert!(worker + .execute_workflow_task(context_task(history.clone())) + .unwrap() + .is_empty()); + history.push(remaining_event( + "TimerFired", + json!({"sequence":3,"timer_id":"cleanup-timer","delay_seconds":1}), + "2026-10-01T00:00:25Z", + )); + for _restart in 0..2 { + let commands = worker + .execute_workflow_task(context_task(history.clone())) + .unwrap(); + assert_eq!( + decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).unwrap(), + json!(5.123456) + ); + } +} + +#[test] +fn cancellation_remaining_parallel_excludes_later_failure_siblings_and_keeps_skew_monotonic() { + for failed in [true, false] { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + let Err(Error::CooperativeCancellationRequested(cancelled)) = + ctx.sleep(Duration::from_secs(10)).await + else { + panic!("expected delivery"); + }; + let context = cancelled.request.context.unwrap(); + let _shield = ctx.cancellation_shield()?; + let outcome = ctx + .parallel(vec![ + ParallelOperation::activity("a", json!([])), + ParallelOperation::activity("b", json!([])), + ]) + .await; + Ok(json!({"failed":outcome.is_err(),"remaining":context.remaining()?.as_secs_f64()})) + }); + let mut history = vec![context_request(), remaining_delivery()]; + let commands = worker + .execute_workflow_task(context_task(history.clone())) + .unwrap(); + let mut payloads = Vec::new(); + for (offset, command) in commands.iter().enumerate() { + let mut payload = command.clone(); + payload.as_object_mut().unwrap().remove("type"); + payload["sequence"] = json!(2 + offset); + history.push(event("ActivityScheduled", payload.clone())); + payloads.push(payload); + } + if failed { + payloads[0]["message"] = json!("failed"); + payloads[0]["exception_type"] = json!("ExampleFailure"); + history.push(remaining_event( + "ActivityFailed", + payloads[0].clone(), + "2026-10-01T00:00:12Z", + )); + payloads[1]["result"] = fixture_envelope(Value::Null); + history.push(remaining_event( + "ActivityCompleted", + payloads[1].clone(), + "2026-10-01T00:00:25Z", + )); + } else { + for (offset, time) in ["2026-10-01T00:00:25Z", "2026-10-01T00:00:20Z"] + .into_iter() + .enumerate() + { + payloads[offset]["result"] = fixture_envelope(Value::Null); + history.push(remaining_event( + "ActivityCompleted", + payloads[offset].clone(), + time, + )); + } + } + for _restart in 0..2 { + let commands = worker + .execute_workflow_task(context_task(history.clone())) + .unwrap(); + assert_eq!( + decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).unwrap(), + json!({"failed":failed,"remaining":if failed {18.123456} else {5.123456}}) + ); + } + } +} + +#[test] +fn cancellation_remaining_refuses_detached_ended_and_missing_or_invalid_delivery_time() { + assert!(CancellationContext::from_value(&context_snapshot()) + .unwrap() + .remaining() + .is_err()); + for time in [ + Value::Null, + json!("tomorrow"), + json!("2026-02-30T00:00:08Z"), + json!("2026-10-01T00:00:08"), + ] { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + let Err(Error::CooperativeCancellationRequested(cancelled)) = + ctx.sleep(Duration::from_secs(10)).await + else { + panic!("expected delivery"); + }; + assert!(cancelled.request.context.unwrap().remaining().is_err()); + Ok(Value::Null) + }); + let mut delivery = context_delivery(); + delivery.raw.insert("timestamp".into(), time); + assert_eq!( + worker + .execute_workflow_task(context_task(vec![context_request(), delivery])) + .unwrap()[0]["type"], + "complete_workflow" + ); + } + let captured = Arc::new(Mutex::new(None)); + let capture = captured.clone(); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _| { + let capture = capture.clone(); + async move { + let Err(Error::CooperativeCancellationRequested(cancelled)) = + ctx.sleep(Duration::from_secs(10)).await + else { + panic!("expected delivery"); + }; + let context = cancelled.request.context.unwrap(); + assert_eq!(context.remaining()?, Duration::new(22, 123456000)); + *capture.lock().unwrap() = Some(context); + Ok(Value::Null) + } + }); + worker + .execute_workflow_task(context_task(vec![context_request(), remaining_delivery()])) + .unwrap(); + assert!(captured + .lock() + .unwrap() + .as_ref() + .unwrap() + .remaining() + .is_err()); +} + +#[test] +fn cancellation_remaining_refuses_missing_consumed_result_and_clamps_expiry_with_timezone_offsets() +{ + for time in [None, Some("2026-10-01T08:00:32+08:00")] { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + let Err(Error::CooperativeCancellationRequested(cancelled)) = + ctx.sleep(Duration::from_secs(10)).await + else { + panic!("expected delivery"); + }; + let context = cancelled.request.context.unwrap(); + let _shield = ctx.cancellation_shield()?; + ctx.sleep(Duration::from_secs(1)).await?; + Ok(json!(context + .remaining() + .map(|remaining| remaining.as_secs_f64()) + .ok())) + }); + let mut fired = event( + "TimerFired", + json!({"sequence":2,"timer_id":"cleanup-timer","delay_seconds":1}), + ); + if let Some(time) = time { + fired.raw.insert("timestamp".into(), json!(time)); + } + let history = vec![ + context_request(), + remaining_delivery(), + event( + "TimerScheduled", + json!({"sequence":2,"timer_id":"cleanup-timer","delay_seconds":1}), + ), + fired, + ]; + let commands = worker.execute_workflow_task(context_task(history)).unwrap(); + assert_eq!( + decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).unwrap(), + if time.is_some() { + json!(0.0) + } else { + Value::Null + } + ); + } +} + +#[test] +fn cancellation_remaining_selection_uses_winner_then_handle_and_first_cancel_receipt() { + for cancel_slow in [false, true] { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + let Err(Error::CooperativeCancellationRequested(cancelled)) = ctx.sleep(Duration::from_secs(10)).await else { panic!("expected delivery"); }; + let context = cancelled.request.context.unwrap(); + let _shield = ctx.cancellation_shield()?; + let selection = ctx.select_keyed(vec![("slow", ParallelOperation::activity("slow", json!([]))), ("fast", ParallelOperation::activity("fast", json!([])))]).await?; + let selected = context.remaining()?.as_secs_f64(); + selection.winner.await_result().await?; + let winner = context.remaining()?.as_secs_f64(); + let result = selection.handle(&SelectionKey::from("slow")).unwrap().await_result().await; + Ok(json!({"selected":selected,"winner":winner,"slow":context.remaining()?.as_secs_f64(),"cancelled":matches!(result, Err(Error::DurableOperationCancelled(_)))})) + }); + let mut history = vec![context_request(), remaining_delivery()]; + let commands = worker + .execute_workflow_task(context_task(history.clone())) + .unwrap(); + assert_eq!(commands.len(), 2); + let mut payloads = Vec::new(); + for (offset, command) in commands.into_iter().enumerate() { + let mut payload = command; + payload.as_object_mut().unwrap().remove("type"); + payload["sequence"] = json!(2 + offset); + payload["activity_execution_id"] = json!(if offset == 0 { + "activity-slow" + } else { + "activity-fast" + }); + history.push(event("ActivityScheduled", payload.clone())); + payloads.push(payload); + } + payloads[1]["result"] = fixture_envelope(json!("fast")); + let mut completed = remaining_event( + "ActivityCompleted", + payloads[1].clone(), + "2026-10-01T00:00:10Z", + ); + completed.raw.insert("id".into(), json!("event-fast")); + history.push(completed); + history.push(remaining_event("SelectionResolved", json!({ + "selection_group_id":"select-calls:2:2","selection_group_base_sequence":2,"selection_group_size":2, + "member_key":"fast","member_index":1,"member_base_sequence":3,"member_size":1, + "operation_kind":"activity","operation_identity":"activity-fast","outcome":"completed", + "resolution_event_id":"event-fast","resolution_event_type":"ActivityCompleted" + }), "2026-10-01T00:00:12Z")); + if cancel_slow { + let receipt = json!({"selection_group_id":"select-calls:2:2","member_key":"slow","member_index":0,"member_base_sequence":2,"member_size":1,"operation_kind":"activity","operation_identity":"activity-slow"}); + history.push(remaining_event( + "SelectionOperationCancelled", + receipt.clone(), + "2026-10-01T00:00:15Z", + )); + history.push(remaining_event( + "SelectionOperationCancelled", + receipt, + "2026-10-01T00:00:28Z", + )); + } else { + payloads[0]["result"] = fixture_envelope(json!("slow")); + history.push(remaining_event( + "ActivityCompleted", + payloads[0].clone(), + "2026-10-01T00:00:20Z", + )); + } + for _restart in 0..2 { + let commands = worker + .execute_workflow_task(context_task(history.clone())) + .unwrap(); + assert_eq!( + decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).unwrap(), + json!({"selected":18.123456,"winner":18.123456,"slow":if cancel_slow {15.123456} else {10.123456},"cancelled":cancel_slow}) + ); + } + } +} + +#[test] +fn cancellation_remaining_parallel_consumes_the_final_physical_condition_wait() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + let Err(Error::CooperativeCancellationRequested(cancelled)) = + ctx.sleep(Duration::from_secs(10)).await + else { + panic!("expected delivery"); + }; + let context = cancelled.request.context.unwrap(); + let _shield = ctx.cancellation_shield()?; + ctx.parallel(vec![ + ParallelOperation::timer(Duration::from_secs(1)), + ParallelOperation::condition( + ConditionWaitOptions::new("ready", "sha256:ready"), + || Ok(false), + ), + ]) + .await?; + Ok(json!(context.remaining()?.as_secs_f64())) + }); + let mut history = vec![context_request(), remaining_delivery()]; + let commands = worker + .execute_workflow_task(context_task(history.clone())) + .unwrap(); + let mut timer = commands[0].clone(); + timer.as_object_mut().unwrap().remove("type"); + timer["sequence"] = json!(2); + timer["timer_id"] = json!("timer-2"); + history.push(event("TimerScheduled", timer.clone())); + let mut condition = commands[1].clone(); + condition.as_object_mut().unwrap().remove("type"); + condition["sequence"] = json!(3); + condition["condition_wait_id"] = json!("condition-3"); + history.push(event("ConditionWaitOpened", condition.clone())); + history.push(remaining_event( + "ConditionWaitSatisfied", + condition.clone(), + "2026-10-01T00:00:12Z", + )); + condition["sequence"] = json!(4); + condition["condition_wait_id"] = json!("condition-4"); + history.push(event("ConditionWaitOpened", condition.clone())); + history.push(remaining_event( + "ConditionWaitSatisfied", + condition, + "2026-10-01T00:00:25Z", + )); + history.push(remaining_event("TimerFired", timer, "2026-10-01T00:00:10Z")); + for _restart in 0..2 { + let commands = worker + .execute_workflow_task(context_task(history.clone())) + .unwrap(); + assert_eq!( + decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).unwrap(), + json!(5.123456) + ); + } +} + +#[test] +fn cancellation_remaining_cannot_escape_into_an_unrelated_nested_worker_poll() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + let Err(Error::CooperativeCancellationRequested(cancelled)) = + ctx.sleep(Duration::from_secs(10)).await + else { + panic!("expected delivery"); + }; + let context = cancelled.request.context.unwrap(); + let nested_context = context.clone(); + let mut nested = cancellation_worker(); + nested.register_workflow("cancel", move |_, _| { + let nested_context = nested_context.clone(); + async move { + assert!( + nested_context.remaining().is_err(), + "another workflow cannot borrow the outer replay binding" + ); + Ok(Value::Null) + } + }); + nested.execute_workflow_task(context_task(Vec::new()))?; + assert_eq!( + context.remaining()?, + Duration::new(22, 123456000), + "outer binding is restored after nested poll" + ); + Ok(Value::Null) + }); + worker + .execute_workflow_task(context_task(vec![context_request(), remaining_delivery()])) + .unwrap(); +} + +#[test] +fn cancellation_remaining_refuses_a_legacy_signal_without_committed_result_time() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + let _ = ctx.sleep(Duration::from_secs(10)).await; + let context = ctx.cancellation_context()?.unwrap(); + assert_eq!(context.remaining()?, Duration::new(22, 123456000)); + let _shield = ctx.cancellation_shield()?; + assert_eq!(ctx.wait_signal("resume").await?, vec![json!(7)]); + assert!( + context.remaining().is_err(), + "missing signal time cannot reuse the delivery budget" + ); + Ok(Value::Null) + }); + let mut task = context_task(vec![context_request(), remaining_delivery()]); + task.signal_name = Some("resume".into()); + task.signal_arguments = Some(fixture_envelope(json!([7]))); + worker.execute_workflow_task(task).unwrap(); +} + +#[test] +fn cooperative_context_retains_original_snapshot_and_detached_copies() { + let mut original = context_snapshot(); + let context = CancellationContext::from_value(&original).unwrap(); + original["requester"]["id"] = json!("changed"); + original["lineage"][0]["request_id"] = json!("changed"); + let mut detached = context.to_value(); + detached["reason"] = json!("changed"); + assert_eq!(context.to_value(), context_snapshot()); + assert_eq!(context.request_id(), "request-1"); + assert_eq!(context.root_request_id(), "root-1"); + assert_eq!(context.parent_request_id(), Some("root-1")); + assert_eq!(context.root_workflow_instance_id(), "parent-instance"); + assert_eq!(context.root_workflow_run_id(), "parent-run"); + assert_eq!(context.reason(), Some("maintenance")); + assert_eq!(context.source(), "control_plane"); + assert_eq!(context.requester()["id"], "operator-1"); + assert_eq!( + context.lineage()[1].workflow_instance_id(), + "child-instance" + ); + assert_eq!( + context.deadline() - context.requested_at(), + chrono::Duration::seconds(30) + ); +} + +#[test] +fn cooperative_context_normalizes_equivalent_timezone_and_object_order() { + let mut value = context_snapshot(); + value["requested_at"] = json!("2026-09-30T20:00:00.123456-04:00"); + value["cleanup_deadline_at"] = json!("2026-09-30T20:00:30.123456-04:00"); + assert_eq!( + CancellationContext::from_value(&value).unwrap(), + CancellationContext::from_value(&context_snapshot()).unwrap() + ); +} + +#[test] +fn cooperative_context_rejects_invalid_snapshots() { + for (field, value) in [ + ("schema", json!("unknown")), + ("request_id", json!("")), + ("source", json!(" ")), + ("parent_request_id", json!("wrong")), + ("root_request_id", json!("wrong")), + ("root_workflow_run_id", json!("wrong")), + ("reason", json!([])), + ("requester", json!([])), + ("lineage", json!([])), + ("requested_at", json!("2026-02-30T00:00:00Z")), + ("cleanup_deadline_at", json!("2026-10-01T00:00:00.123456Z")), + ] { + let mut snapshot = context_snapshot(); + snapshot[field] = value; + assert!( + CancellationContext::from_value(&snapshot).is_err(), + "{field}" + ); + } +} + +#[test] +fn cooperative_context_rejects_lineage_cycles_order_and_unrelated_requester_metadata() { + for field in ["request_id", "workflow_run_id"] { + let mut value = context_snapshot(); + value["lineage"][1][field] = value["lineage"][0][field].clone(); + assert!(CancellationContext::from_value(&value).is_err()); + } + let mut value = context_snapshot(); + value["lineage"].as_array_mut().unwrap().reverse(); + assert!(CancellationContext::from_value(&value).is_err()); + let mut value = context_snapshot(); + value["requester"]["authorization"] = json!("unsupported"); + assert!(CancellationContext::from_value(&value).is_err()); +} + +#[test] +fn cooperative_context_child_keeps_root_time_after_expired_local_admission() { + let mut request = context_request(); + request + .raw + .insert("recorded_at".into(), json!("2026-10-01T00:00:31Z")); + let state = CancellationHistory::from_events(&[request], "run-1", None).unwrap(); + let request = state.request.unwrap(); + assert_eq!(request.requested_at, "2026-10-01T00:00:00.123456Z"); + assert_eq!(request.cleanup_deadline_at, "2026-10-01T00:00:30.123456Z"); + assert_eq!(request.context.unwrap().to_value(), context_snapshot()); + assert!(state.delivery.is_none()); +} + +#[test] +fn cooperative_context_observation_keeps_refresh_route_and_history_supplies_metadata() { + let mut observation = context_observation(); + observation.context = Some(CancellationContext::from_value(&context_snapshot()).unwrap()); + let pending = CancellationHistory::from_events(&[], "run-1", Some(&observation)).unwrap(); + assert!(pending.request.unwrap().context.is_none()); + let state = CancellationHistory::from_events( + &[context_request(), context_delivery()], + "run-1", + Some(&observation), + ) + .unwrap(); + let request = state.request.unwrap(); + assert_eq!( + request.history_refresh_page_token.as_deref(), + Some("opaque-first-page") + ); + assert_eq!(request.requested_at, "2026-10-01T00:00:00.123456Z"); + assert_eq!(request.context.unwrap().to_value(), context_snapshot()); +} + +#[test] +fn cooperative_context_must_match_canonical_local_request_and_run() { + for (field, value) in [ + ("workflow_command_id", json!("other")), + ("workflow_instance_id", json!("other")), + ("cleanup_deadline_at", json!("2026-10-01T00:00:35Z")), + ("reason", json!("changed")), + ("cancellation", Value::Null), + ] { + let mut request = context_request(); + request.payload[field] = value; + assert!( + CancellationHistory::from_events(&[request], "run-1", None).is_err(), + "{field}" + ); + } + let mut request = context_request(); + request.payload["cancellation"]["lineage"][1]["workflow_run_id"] = json!("other"); + assert!(CancellationHistory::from_events(&[request], "run-1", None).is_err()); + let mut request = context_request(); + request + .raw + .insert("recorded_at".into(), json!("2026-09-30T23:59:59Z")); + assert!(CancellationHistory::from_events(&[request], "run-1", None).is_err()); +} + +#[test] +fn cooperative_context_delivery_cannot_change_the_accepted_snapshot() { + let mut delivery = context_delivery(); + delivery.payload["cancellation"]["reason"] = json!("changed"); + assert!( + CancellationHistory::from_events(&[context_request(), delivery], "run-1", None).is_err() + ); +} + +#[test] +fn cooperative_context_cold_worker_replay_restores_same_delivery_and_cleanup() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + assert!(ctx.cancellation_context()?.is_none()); + let Err(Error::CooperativeCancellationRequested(cancelled)) = + ctx.sleep(Duration::from_secs(10)).await + else { + panic!("expected canonical cancellation"); + }; + let context = cancelled.request.context.unwrap(); + assert_eq!(ctx.cancellation_context()?, Some(context.clone())); + let _shield = ctx.cancellation_shield()?; + ctx.throw_if_cancellation_requested()?; + ctx.activity("cleanup", json!([])).await?; + Ok(context.to_value()) + }); + let first = worker + .execute_workflow_task(context_task(vec![context_request(), context_delivery()])) + .unwrap(); + assert_eq!(first.len(), 1); + assert_eq!(first[0]["type"], "schedule_activity"); + assert_eq!(first[0]["activity_type"], "cleanup"); + let mut history = vec![context_request(), context_delivery()]; + history.extend(completed_activity(2, "cleanup", json!("cleaned"))); + for _restart in 0..2 { + let result = worker + .execute_workflow_task(context_task(history.clone())) + .unwrap(); + assert_eq!(result[0]["type"], "complete_workflow"); + assert_eq!( + decode_wire_value(&result[0]["result"], DEFAULT_CODEC).unwrap(), + context_snapshot() + ); + } +} + +#[test] +fn cooperative_context_explicit_check_retains_delivered_metadata() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + let _ = ctx.sleep(Duration::from_secs(10)).await; + let Err(Error::CooperativeCancellationRequested(cancelled)) = + ctx.throw_if_cancellation_requested() + else { + panic!("expected the delivered cancellation"); + }; + assert_eq!( + cancelled.request.context.unwrap().to_value(), + context_snapshot() + ); + Ok(Value::Null) + }); + worker + .execute_workflow_task(context_task(vec![context_request(), context_delivery()])) + .unwrap(); +} + +fn request_observation() -> Value { + json!({ + "request_id": "original-request", + "requested_at": "2026-10-01T08:00:00Z", + "cleanup_deadline_at": "2026-10-01T08:10:00Z", + "history_refresh_page_token": "opaque-server-token" + }) +} + +fn event(kind: &str, payload: Value) -> HistoryEvent { + serde_json::from_value(json!({"event_type":kind, "payload":payload})).unwrap() +} + +fn canonical_request() -> HistoryEvent { + serde_json::from_value(json!({ + "event_type":"CooperativeCancellationRequested", "workflow_command_id":"original-request", + "recorded_at":"2026-10-01T08:00:00.000120Z", "payload":{ + "workflow_command_id":"original-request", "workflow_run_id":"run", + "cleanup_deadline_at":"2026-10-01T08:10:00Z" + } + })) + .unwrap() +} + +fn canonical_delivery(sequence: u64, kind: &str) -> HistoryEvent { + event( + "CooperativeCancellationDelivered", + json!({ + "workflow_command_id":"original-request", "workflow_run_id":"run", + "sequence":sequence, "call_kind":kind + }), + ) +} + +fn history(events: &[HistoryEvent]) -> Result { + CancellationHistory::from_events(events, "run", None) +} + +fn cancellation_worker() -> Worker { + Worker::new( + Client::builder("http://127.0.0.1:1").build().unwrap(), + "queue", + ) +} + +fn cancellation_task(events: Vec) -> WorkflowTask { + let mut task = workflow_task("cancel", events, DEFAULT_CODEC); + task.run_id = Some("run".into()); + task +} + +fn scheduled_timer(sequence: u64) -> HistoryEvent { + event( + "TimerScheduled", + json!({"sequence":sequence, "timer_id":format!("timer-{sequence}"), "delay_seconds":5}), + ) +} + +fn completed_activity(sequence: u64, name: &str, value: Value) -> Vec { + vec![ + event( + "ActivityScheduled", + json!({"sequence":sequence,"activity_type":name,"task_queue":"queue"}), + ), + event( + "ActivityCompleted", + json!({"sequence":sequence,"activity_type":name,"result":fixture_envelope(value)}), + ), + ] +} + +fn cancelled_selection_history() -> Vec { + let mut delivery = canonical_delivery(3, "selection_handle"); + delivery.payload["operation_sequence"] = json!(1); + vec![ + selection_activity_event("ActivityScheduled", 0, "slow", None), + selection_activity_event("ActivityScheduled", 1, "fast", None), + selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner-value"))), + selection_winner_marker(), + canonical_request(), + delivery, + ] +} + +fn cancelled_parallel_history() -> Vec { + let paths = nested_parallel_paths(); + let mut delivery = canonical_delivery(1, "parallel"); + delivery.payload["sequence_span"] = json!(3); + vec![ + parallel_history_event( + "ActivityScheduled", + 1, + "activity_type", + "first", + paths[0].clone(), + None, + ), + parallel_history_event( + "ChildWorkflowScheduled", + 2, + "workflow_type", + "second", + paths[1].clone(), + None, + ), + parallel_history_event( + "ActivityScheduled", + 3, + "activity_type", + "third", + paths[2].clone(), + None, + ), + canonical_request(), + delivery, + ] +} + +fn intent_task(events: Vec) -> WorkflowTask { + let mut task = transport_task(); + task.history_events = events; + task +} + +fn original_observation() -> CancellationRequest { + CancellationRequest::from_observation(&request_observation()).unwrap() +} + +async fn authored_scalar(ctx: WorkflowContext, kind: &str) -> Result<()> { + match kind { + "activity" => { + ctx.activity("forward", json!([])).await?; + } + "timer" => { + ctx.sleep(Duration::from_secs(5)).await?; + } + "child" => { + ctx.start_child_workflow("child", ChildWorkflowOptions::new("queue"), json!([])) + .await?; + } + "signal" => { + ctx.wait_signal("resume").await?; + } + "condition" => { + ctx.wait_condition(ConditionWaitOptions::new("ready", "sha256:ready"), || { + Ok(false) + }) + .await?; + } + _ => unreachable!(), + } + Ok(()) +} + +fn pending_scalar_event(kind: &str) -> HistoryEvent { + match kind { + "activity" => event( + "ActivityScheduled", + json!({"sequence":1,"activity_type":"forward","task_queue":"queue"}), + ), + "timer" => scheduled_timer(1), + "child" => event( + "ChildWorkflowScheduled", + json!({"sequence":1,"workflow_type":"child"}), + ), + "signal" => event( + "SignalWaitOpened", + json!({"sequence":1,"signal_name":"resume"}), + ), + "condition" => event( + "ConditionWaitOpened", + json!({"sequence":1,"condition_wait_id":"condition-1", + "condition_wait_occurrence_id":"rust:condition-wait:0","condition_key":"ready","condition_definition_fingerprint":"sha256:ready"}), + ), + _ => unreachable!(), + } +} + +#[test] +fn cooperative_intent_suspends_actual_worker_scalar_calls_without_exposing_an_error() { + for kind in ["activity", "timer", "child", "signal", "condition"] { + for scheduled in [false, true] { + let resumed = Arc::new(AtomicBool::new(false)); + let observed = resumed.clone(); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _input| { + let observed = observed.clone(); + async move { + assert!(!ctx.is_cancellation_requested()?); + // Even catching every error cannot expose cancellation + // before the worker commits its delivery and replays. + let _ = authored_scalar(ctx, kind).await; + observed.store(true, Ordering::SeqCst); + Ok(Value::Null) + } + }); + let mut events = Vec::new(); + if scheduled { + events.push(pending_scalar_event(kind)); + } + events.push(canonical_request()); + let decision = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(events), + Some(&original_observation()), + ) + .unwrap(); + assert!(!resumed.load(Ordering::SeqCst), "{kind}:{scheduled}"); + assert!(decision.commands.is_empty(), "{kind}:{scheduled}"); + let delivery = decision.cancellation_delivery.unwrap(); + assert_eq!(delivery.sequence, 1); + assert_eq!(serde_json::to_value(delivery.call_kind).unwrap(), kind); + assert_eq!(delivery.request_id, "original-request"); + } + } +} + +#[test] +fn cooperative_intent_suspends_a_resolution_committed_after_the_request() { + let resumed = Arc::new(AtomicBool::new(false)); + let observed = resumed.clone(); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _input| { + let observed = observed.clone(); + async move { + let _ = ctx.sleep(Duration::from_secs(5)).await; + observed.store(true, Ordering::SeqCst); + Ok(Value::Null) + } + }); + let decision = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(vec![ + scheduled_timer(1), + canonical_request(), + event( + "TimerFired", + json!({"sequence":1,"timer_id":"timer-1","delay_seconds":5}), + ), + ]), + Some(&original_observation()), + ) + .unwrap(); + assert!(!resumed.load(Ordering::SeqCst)); + assert!(decision.commands.is_empty()); + assert_eq!(decision.cancellation_delivery, Some(transport_delivery())); +} + +#[test] +fn cooperative_intent_preserves_earlier_commands_and_replayed_side_effects() { + let side_effects = Arc::new(Mutex::new(0)); + let observed = side_effects.clone(); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _input| { + let observed = observed.clone(); + async move { + let value: Value = ctx.side_effect(|| { + *observed.lock().unwrap() += 1; + json!(7) + })?; + assert_eq!(value, 7); + ctx.sleep(Duration::from_secs(5)).await?; + Ok(Value::Null) + } + }); + let first = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(vec![canonical_request()]), + Some(&original_observation()), + ) + .unwrap(); + assert_eq!(first.commands.len(), 1); + assert_eq!(first.commands[0]["type"], "record_side_effect"); + assert_eq!(first.cancellation_delivery.unwrap().sequence, 2); + let second = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(vec![ + event( + "SideEffectRecorded", + json!({"sequence":1,"result":fixture_envelope(json!(7))}), + ), + canonical_request(), + ]), + Some(&original_observation()), + ) + .unwrap(); + assert!(second.commands.is_empty()); + assert_eq!(second.cancellation_delivery.unwrap().sequence, 2); + assert_eq!(*side_effects.lock().unwrap(), 1); +} + +#[test] +fn cooperative_intent_replays_committed_delivery_with_the_original_observation() { + let delivered = Arc::new(Mutex::new(None)); + let observed = delivered.clone(); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _input| { + let observed = observed.clone(); + async move { + match ctx.sleep(Duration::from_secs(5)).await { + Err(Error::CooperativeCancellationRequested(cancellation)) => { + *observed.lock().unwrap() = Some(cancellation.request); + } + other => panic!("{other:?}"), + } + Ok(Value::Null) + } + }); + let observation = original_observation(); + let decision = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(vec![ + scheduled_timer(1), + canonical_request(), + canonical_delivery(1, "timer"), + ]), + Some(&observation), + ) + .unwrap(); + assert!(decision.cancellation_delivery.is_none()); + assert_eq!(decision.commands.len(), 1); + assert_eq!(decision.commands[0]["type"], "complete_workflow"); + assert_eq!(*delivered.lock().unwrap(), Some(observation)); +} + +#[test] +fn cooperative_intent_cannot_publish_a_terminal_result_after_an_ignored_pending_call() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5))); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + assert!(matches!(timer.as_mut().poll(&mut cx), Poll::Pending)); + Ok(json!("ignored pending timer")) + }); + let decision = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(vec![canonical_request()]), + Some(&original_observation()), + ) + .unwrap(); + assert!(decision.commands.is_empty()); + assert_eq!(decision.cancellation_delivery, Some(transport_delivery())); +} + +#[test] +fn cooperative_intent_cannot_publish_commands_after_an_ignored_pending_call() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5))); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + assert!(matches!(timer.as_mut().poll(&mut cx), Poll::Pending)); + let _: Value = ctx.side_effect(|| json!("authored after pending timer"))?; + Ok(Value::Null) + }); + assert!( + matches!(worker.execute_workflow_task_decision_with_cancellation( + intent_task(vec![canonical_request()]), Some(&original_observation()), + ), Err(Error::NonDeterministicReplay(ReplayFailure { reason, .. })) + if reason == "cooperative_cancellation_pending_call_escaped") + ); +} + +#[test] +fn cooperative_intent_preserves_a_prior_selection_winner_then_suspends_the_loser_await() { + for late in [false, true] { + let resumed = Arc::new(AtomicBool::new(false)); + let observed = resumed.clone(); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _input| { + let observed = observed.clone(); + async move { + let selected = keyed_activity_selection(&ctx).await?; + assert_eq!(selected.key, SelectionKey::Name("fast".into())); + assert_eq!( + selected.value, + Some(ParallelResult::Activity(json!("winner-value"))) + ); + let _ = selected + .handle(&SelectionKey::Name("slow".into())) + .unwrap() + .await_result() + .await; + observed.store(true, Ordering::SeqCst); + Ok(Value::Null) + } + }); + let mut events = cancelled_selection_history(); + events.pop(); + if late { + events.push(selection_activity_event( + "ActivityCompleted", + 0, + "slow", + Some(json!("late")), + )); + } + let decision = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(events), + Some(&original_observation()), + ) + .unwrap(); + assert!(!resumed.load(Ordering::SeqCst)); + assert!(decision.commands.is_empty()); + let delivery = decision.cancellation_delivery.unwrap(); + assert_eq!(delivery.call_kind, CancellationCallKind::SelectionHandle); + assert_eq!(delivery.sequence, 3); + assert_eq!(delivery.operation_sequence, Some(1)); + assert_eq!(delivery.operation_sequence_span, 1); + } +} + +#[test] +fn cooperative_intent_validates_all_handle_fields_before_checking_prior_resolution() { + for changed in 0..7 { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _input| async move { + let selected = keyed_activity_selection(&ctx).await?; + let mut handle = selected + .handle(&SelectionKey::Name("slow".into())) + .unwrap() + .clone(); + match changed { + 0 => handle.key = SelectionKey::Name("changed".into()), + 1 => handle.index = 9, + 2 => handle.kind = "timer".into(), + 3 => handle.identity = "changed".into(), + 4 => handle.selection_group_id = "changed".into(), + 5 => handle.base_sequence = 2, + 6 => handle.size = 2, + _ => unreachable!(), + } + handle.await_result().await?; + Ok(Value::Null) + }); + let mut events = cancelled_selection_history(); + events.pop(); + assert!( + matches!( + worker.execute_workflow_task_decision_with_cancellation( + intent_task(events), + Some(&original_observation()), + ), + Err(Error::NonDeterministicReplay(_)) + ), + "{changed}" + ); + } +} + +#[test] +fn cooperative_intent_suspends_nested_parallel_and_an_uncommitted_selection_winner() { + for (selection, mode) in [ + (false, "new"), + (false, "scheduled"), + (false, "partial"), + (true, "new"), + (true, "scheduled"), + ] { + let resumed = Arc::new(AtomicBool::new(false)); + let observed = resumed.clone(); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _input| { + let observed = observed.clone(); + async move { + if selection { + let _ = keyed_activity_selection(&ctx).await; + } else { + let _ = ctx.parallel(nested_parallel_operations()).await; + } + observed.store(true, Ordering::SeqCst); + Ok(Value::Null) + } + }); + let events = if mode == "new" { + vec![canonical_request()] + } else if selection { + vec![ + selection_activity_event("ActivityScheduled", 0, "slow", None), + selection_activity_event("ActivityScheduled", 1, "fast", None), + canonical_request(), + selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("late"))), + selection_winner_marker(), + ] + } else { + let mut events = cancelled_parallel_history(); + events.pop(); + if mode == "partial" { + events.insert( + 3, + parallel_history_event( + "ActivityCompleted", + 1, + "activity_type", + "first", + nested_parallel_paths()[0].clone(), + Some(json!(7)), + ), + ); + } + events.push(event("ActivityFailed", json!({"sequence":3,"activity_type":"third", + "exception_type":"LateFailure","exception":{"class":"LateFailure","message":"too late"}}))); + events + }; + let decision = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(events), + Some(&original_observation()), + ) + .unwrap(); + assert!(!resumed.load(Ordering::SeqCst)); + assert!(decision.commands.is_empty()); + let delivery = decision.cancellation_delivery.unwrap(); + assert_eq!(delivery.call_kind, CancellationCallKind::Parallel); + assert_eq!(delivery.sequence, 1); + assert_eq!(delivery.sequence_span, if selection { 2 } else { 3 }); + } +} + +#[test] +fn cooperative_intent_rejects_changed_later_leaf_details_before_exporting_delivery() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + ctx.parallel(vec![ + ParallelOperation::activity("first", json!([])), + ParallelOperation::group(vec![ + ParallelOperation::child_workflow( + "second", + ChildWorkflowOptions::new("child-workers"), + json!([]), + ), + ParallelOperation::activity("changed-third", json!([])), + ]), + ]) + .await?; + Ok(Value::Null) + }); + let mut events = cancelled_parallel_history(); + events.pop(); + assert!(matches!( + worker.execute_workflow_task_decision_with_cancellation( + intent_task(events), + Some(&original_observation()), + ), + Err(Error::NonDeterministicReplay(_)) + )); +} + +#[test] +fn cooperative_intent_uses_the_pending_physical_condition_reopen() { + let payload = |sequence| { + json!({"sequence":sequence,"condition_wait_id":format!("condition:{sequence}"), + "condition_wait_occurrence_id":"rust:condition-wait:0","condition_key":"ready", + "condition_definition_fingerprint":"sha256:ready"}) + }; + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + ctx.wait_condition(ConditionWaitOptions::new("ready", "sha256:ready"), || { + panic!("delivery proposal must suspend before reevaluating the predicate") + }) + .await?; + Ok(Value::Null) + }); + let decision = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(vec![ + event("ConditionWaitOpened", payload(1)), + event("ConditionWaitSatisfied", payload(1)), + event("ConditionWaitOpened", payload(2)), + canonical_request(), + ]), + Some(&original_observation()), + ) + .unwrap(); + assert!(decision.commands.is_empty()); + let delivery = decision.cancellation_delivery.unwrap(); + assert_eq!(delivery.call_kind, CancellationCallKind::Condition); + assert_eq!(delivery.sequence, 2); +} + +#[test] +fn cooperative_intent_keeps_explicit_shielded_cleanup_pending_without_delivery() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + let _shield = ctx.cancellation_shield()?; + ctx.sleep(Duration::from_secs(5)).await?; + Ok(Value::Null) + }); + let decision = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(vec![scheduled_timer(1), canonical_request()]), + Some(&original_observation()), + ) + .unwrap(); + assert!(decision.commands.is_empty()); + assert!(decision.cancellation_delivery.is_none()); +} + +#[test] +fn cooperative_intent_rejects_unproven_observation_or_claim_before_workflow_code() { + let invoked = Arc::new(AtomicBool::new(false)); + let observed = invoked.clone(); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |_ctx, _input| { + observed.store(true, Ordering::SeqCst); + async { Ok(Value::Null) } + }); + for changed in 0..6 { + let mut task = intent_task(vec![canonical_request()]); + let mut observation = original_observation(); + match changed { + 0 => task.history_events.clear(), + 1 => task.lease_owner = None, + 2 => task.run_id = Some("other-run".into()), + 3 => observation.request_id = "other-request".into(), + 4 => observation.cleanup_deadline_at = "2026-10-01T08:11:00Z".into(), + 5 => observation.history_refresh_page_token = None, + _ => unreachable!(), + } + assert!(worker + .execute_workflow_task_decision_with_cancellation(task, Some(&observation)) + .is_err()); + assert!(!invoked.load(Ordering::SeqCst)); + } +} + +#[test] +fn cooperative_parallel_replays_one_original_cancellation_for_nested_mixed_leaves() { + for _cold_restart in 0..2 { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + ctx.parallel(nested_parallel_operations()).await?; + Ok(Value::Null) + }); + let commands = worker + .execute_workflow_task(cancellation_task(cancelled_parallel_history())) + .unwrap(); + assert_eq!(commands.len(), 1); + assert_eq!( + commands[0]["exception_type"], + "WorkflowCancellationRequested" + ); + assert_eq!( + commands[0]["exception"]["properties"]["request_id"], + "original-request" + ); + assert_eq!(commands[0]["non_retryable"], true); + } +} + +#[test] +fn cooperative_parallel_preserves_prior_result_and_overrides_a_late_leaf_failure() { + let mut events = cancelled_parallel_history(); + events.insert( + 3, + parallel_history_event( + "ActivityCompleted", + 1, + "activity_type", + "first", + nested_parallel_paths()[0].clone(), + Some(json!(7)), + ), + ); + events.push(event("ActivityFailed", json!({"sequence":3, "activity_type":"third", "exception_type":"LateFailure", "exception":{"class":"LateFailure", "message":"too late"}}))); + let ctx = workflow_context(events); + let mut call = Box::pin(ctx.parallel(nested_parallel_operations())); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + let Poll::Ready(Err(Error::CooperativeCancellationRequested(cancellation))) = + call.as_mut().poll(&mut cx) + else { + panic!("group cancellation must retain priority") + }; + assert_eq!(cancellation.delivery.sequence_span, 3); + assert_eq!(ctx.state.lock().unwrap().command_cursor, 3); + ctx.ensure_history_consumed().unwrap(); + assert!(ctx.take_commands().unwrap().is_empty()); +} + +#[test] +fn cooperative_parallel_does_not_hide_changed_later_leaf_details() { + let mut events = cancelled_parallel_history(); + events[2].payload["activity_type"] = json!("changed-third"); + let ctx = workflow_context(events); + let mut call = Box::pin(ctx.parallel(nested_parallel_operations())); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + assert!( + matches!(call.as_mut().poll(&mut cx), Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure { reason, .. }))) if reason == "recorded_command_detail_mismatch") + ); + assert!(ctx.take_commands().unwrap().is_empty()); +} + +#[test] +fn cooperative_parallel_rejects_changed_span_nesting_scalar_and_cleanup_scope() { + for change in 0..4 { + let ctx = workflow_context(cancelled_parallel_history()); + let _shield = if change == 3 { + Some(ctx.cancellation_shield().unwrap()) + } else { + None + }; + let operations = match change { + 0 => vec![ParallelOperation::activity("first", json!([]))], + 1 => vec![ + ParallelOperation::activity("first", json!([])), + ParallelOperation::child_workflow( + "second", + ChildWorkflowOptions::new("child-workers"), + json!([]), + ), + ParallelOperation::activity("third", json!([])), + ], + _ => nested_parallel_operations(), + }; + let mut cx = TaskContext::from_waker(noop_waker_ref()); + let outcome = if change == 2 { + Box::pin(ctx.activity("first", json!([]))) + .as_mut() + .poll(&mut cx) + .map_ok(|_| ()) + } else { + Box::pin(ctx.parallel(operations)) + .as_mut() + .poll(&mut cx) + .map_ok(|_| ()) + }; + assert!(matches!( + outcome, + Poll::Ready(Err(Error::NonDeterministicReplay(_))) + )); + assert!(ctx.take_commands().unwrap().is_empty()); + } +} + +#[test] +fn cooperative_parallel_cold_delivery_covers_all_scalar_leaf_kinds_without_scheduling() { + let mut delivery = canonical_delivery(1, "parallel"); + delivery.payload["sequence_span"] = json!(5); + let ctx = workflow_context(vec![canonical_request(), delivery]); + let mut call = Box::pin(ctx.parallel(vec![ + ParallelOperation::activity("work", json!([])), + ParallelOperation::child_workflow( + "child", + ChildWorkflowOptions::new("child-workers"), + json!([]), + ), + ParallelOperation::timer(Duration::from_secs(5)), + ParallelOperation::signal("ready"), + ParallelOperation::condition(ConditionWaitOptions::new("ready", "sha256:ready"), || { + Ok(false) + }), + ])); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + assert!(matches!( + call.as_mut().poll(&mut cx), + Poll::Ready(Err(Error::CooperativeCancellationRequested(_))) + )); + assert_eq!(ctx.state.lock().unwrap().command_cursor, 5); + ctx.ensure_history_consumed().unwrap(); + assert!(ctx.take_commands().unwrap().is_empty()); +} + +#[test] +fn cooperative_parallel_validates_recorded_timer_signal_and_condition_leaves() { + for changed in 0..3 { + let names = [ + ("ActivityScheduled", "activity_type", "work"), + ("ChildWorkflowScheduled", "workflow_type", "child"), + ("TimerScheduled", "timer_id", "timer"), + ("SignalWaitOpened", "signal_name", "ready"), + ("ConditionWaitOpened", "condition_wait_id", "condition"), + ]; + let mut events = names + .into_iter() + .enumerate() + .map(|(index, (event_type, field, name))| { + parallel_history_event( + event_type, + index as u64 + 1, + field, + name, + vec![parallel_group_entry(1, 5, index, "mixed")], + None, + ) + }) + .collect::>(); + events[2].payload["delay_seconds"] = json!(if changed == 1 { 500 } else { 5 }); + events[4].payload["condition_wait_occurrence_id"] = json!("rust:condition-wait:0"); + events[4].payload["condition_key"] = json!("ready"); + events[4].payload["condition_definition_fingerprint"] = json!(if changed == 2 { + "changed" + } else { + "sha256:ready" + }); + let mut delivery = canonical_delivery(1, "parallel"); + delivery.payload["sequence_span"] = json!(5); + events.extend([canonical_request(), delivery]); + let ctx = workflow_context(events); + let mut call = Box::pin(ctx.parallel(vec![ + ParallelOperation::activity("work", json!([])), + ParallelOperation::child_workflow( + "child", + ChildWorkflowOptions::new("child-workers"), + json!([]), + ), + ParallelOperation::timer(Duration::from_secs(5)), + ParallelOperation::signal("ready"), + ParallelOperation::condition( + ConditionWaitOptions::new("ready", "sha256:ready"), + || panic!("delivered condition must not reevaluate its predicate"), + ), + ])); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + match call.as_mut().poll(&mut cx) { + Poll::Ready(Err(Error::CooperativeCancellationRequested(_))) if changed == 0 => { + ctx.ensure_history_consumed().unwrap(); + } + Poll::Ready(Err(Error::NonDeterministicReplay(failure))) if changed == 1 => { + assert_eq!(failure.reason, "timer_delay_mismatch"); + } + Poll::Ready(Err(Error::NonDeterministicReplay(failure))) if changed == 2 => { + assert_eq!(failure.reason, "condition_wait_predicate_mismatch"); + } + other => panic!("{changed}: {other:?}"), + } + assert!(ctx.take_commands().unwrap().is_empty()); + } +} + +#[test] +fn cooperative_selection_group_delivers_before_an_uncommitted_winner() { + let mut delivery = canonical_delivery(1, "parallel"); + delivery.payload["sequence_span"] = json!(2); + let events = vec![ + selection_activity_event("ActivityScheduled", 0, "slow", None), + selection_activity_event("ActivityScheduled", 1, "fast", None), + canonical_request(), + delivery, + selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("too-late"))), + selection_winner_marker(), + ]; + let ctx = workflow_context(events); + let mut select = Box::pin(keyed_activity_selection(&ctx)); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + assert!(matches!( + select.as_mut().poll(&mut cx), + Poll::Ready(Err(Error::CooperativeCancellationRequested(_))) + )); + ctx.ensure_history_consumed().unwrap(); + assert!(ctx.take_commands().unwrap().is_empty()); +} + +#[test] +fn cooperative_parallel_saga_cleanup_uses_the_slot_after_the_entire_group() { + let mut events = cancelled_parallel_history(); + events.extend(completed_activity(4, "undo", Value::Null)); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + let mut saga = ctx.saga(); + saga.add_compensation("undo", json!([]))?; + saga.finish(ctx.parallel(nested_parallel_operations()).await) + .await?; + Ok(Value::Null) + }); + let commands = worker + .execute_workflow_task(cancellation_task(events)) + .unwrap(); + assert_eq!(commands.len(), 1); + assert_eq!( + commands[0]["exception_type"], + "WorkflowCancellationRequested" + ); + assert_eq!( + commands[0]["exception"]["properties"]["request_id"], + "original-request" + ); +} + +#[test] +fn cooperative_selection_handle_replays_original_request_after_committed_winner() { + for _cold_restart in 0..2 { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + let selected = keyed_activity_selection(&ctx).await?; + assert_eq!(selected.key, SelectionKey::Name("fast".into())); + assert_eq!( + selected.value, + Some(ParallelResult::Activity(json!("winner-value"))) + ); + assert!(!ctx.is_cancellation_requested()?); + selected + .handle(&SelectionKey::Name("slow".into())) + .unwrap() + .await_result() + .await?; + Ok(Value::Null) + }); + let commands = worker + .execute_workflow_task(cancellation_task(cancelled_selection_history())) + .unwrap(); + assert_eq!(commands.len(), 1); + assert_eq!( + commands[0]["exception_type"], + "WorkflowCancellationRequested" + ); + assert_eq!( + commands[0]["exception"]["properties"]["request_id"], + "original-request" + ); + assert_eq!( + commands[0]["exception"]["properties"]["cleanup_deadline_at"], + "2026-10-01T08:10:00Z" + ); + } +} + +#[test] +fn cooperative_selection_handle_delivery_owns_a_late_loser_completion() { + for before_delivery in [true, false] { + let mut events = cancelled_selection_history(); + let late = + selection_activity_event("ActivityCompleted", 0, "slow", Some(json!("too-late"))); + if before_delivery { + events.insert(events.len() - 1, late); + } else { + events.push(late); + } + let ctx = workflow_context(events); + let mut select = Box::pin(keyed_activity_selection(&ctx)); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + let Poll::Ready(Ok(selected)) = select.as_mut().poll(&mut cx) else { + panic!("committed winner must retain priority") + }; + assert_eq!( + selected.value, + Some(ParallelResult::Activity(json!("winner-value"))) + ); + let mut loser = Box::pin( + selected + .handle(&SelectionKey::Name("slow".into())) + .unwrap() + .await_result(), + ); + let Poll::Ready(Err(Error::CooperativeCancellationRequested(cancellation))) = + loser.as_mut().poll(&mut cx) + else { + panic!("late completion must not replace the committed cancellation") + }; + assert_eq!(cancellation.delivery.sequence, 3); + assert_eq!(cancellation.delivery.operation_sequence, Some(1)); + ctx.ensure_history_consumed().unwrap(); + assert!(ctx.take_commands().unwrap().is_empty()); + } +} + +#[test] +fn cooperative_selection_handle_rejects_changed_public_identity_and_member_range() { + for changed in 0..7 { + let ctx = workflow_context(cancelled_selection_history()); + let mut select = Box::pin(keyed_activity_selection(&ctx)); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + let Poll::Ready(Ok(selected)) = select.as_mut().poll(&mut cx) else { + panic!("winner") + }; + let mut handle = selected + .handle(&SelectionKey::Name("slow".into())) + .unwrap() + .clone(); + match changed { + 0 => handle.key = SelectionKey::Name("changed".into()), + 1 => handle.index = 9, + 2 => handle.kind = "timer".into(), + 3 => handle.identity = "changed".into(), + 4 => handle.selection_group_id = "changed".into(), + 5 => handle.base_sequence = 2, + 6 => handle.size = 2, + _ => unreachable!(), + } + let mut wait = Box::pin(handle.await_result()); + assert!( + matches!(wait.as_mut().poll(&mut cx), Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure { reason, .. }))) if reason == "cooperative_cancellation_call_mismatch") + ); + assert!(!ctx.is_cancellation_requested().unwrap()); + assert!(ctx.take_commands().unwrap().is_empty()); + } +} + +#[test] +fn cooperative_selection_handle_request_only_keeps_the_loser_pending() { + let mut events = cancelled_selection_history(); + events.pop(); + let ctx = workflow_context(events); + let mut select = Box::pin(keyed_activity_selection(&ctx)); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + let Poll::Ready(Ok(selected)) = select.as_mut().poll(&mut cx) else { + panic!("winner") + }; + let mut wait = Box::pin( + selected + .handle(&SelectionKey::Name("slow".into())) + .unwrap() + .await_result(), + ); + assert!(matches!(wait.as_mut().poll(&mut cx), Poll::Pending)); + assert!(!ctx.is_cancellation_requested().unwrap()); + assert!(ctx.take_commands().unwrap().is_empty()); +} + +#[test] +fn cooperative_selection_handle_cannot_be_skipped_or_replaced_by_a_scalar_call() { + for skip in [true, false] { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _input| async move { + let selected = keyed_activity_selection(&ctx).await?; + if !skip { + ctx.sleep(Duration::from_secs(5)).await?; + } + selected.into_result()?; + Ok(Value::Null) + }); + assert!(matches!( + worker.execute_workflow_task(cancellation_task(cancelled_selection_history())), + Err(Error::NonDeterministicReplay(_)) + )); + } +} + +#[test] +fn cooperative_selection_handle_cannot_hide_delivery_inside_a_cleanup_shield() { + let ctx = workflow_context(cancelled_selection_history()); + let mut select = Box::pin(keyed_activity_selection(&ctx)); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + let Poll::Ready(Ok(selected)) = select.as_mut().poll(&mut cx) else { + panic!("winner") + }; + let _shield = ctx.cancellation_shield().unwrap(); + let mut wait = Box::pin( + selected + .handle(&SelectionKey::Name("slow".into())) + .unwrap() + .await_result(), + ); + assert!(matches!( + wait.as_mut().poll(&mut cx), + Poll::Ready(Err(Error::NonDeterministicReplay(_))) + )); +} + +#[test] +fn cooperative_selection_handle_runs_saga_cleanup_as_the_next_durable_command() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + let selected = keyed_activity_selection(&ctx).await?; + let mut saga = ctx.saga(); + saga.add_compensation("undo", json!([]))?; + let result = selected + .handle(&SelectionKey::Name("slow".into())) + .unwrap() + .await_result() + .await; + saga.finish(result).await?; + Ok(Value::Null) + }); + let commands = worker + .execute_workflow_task(cancellation_task(cancelled_selection_history())) + .unwrap(); + assert_eq!(commands.len(), 1); + assert_eq!(commands[0]["type"], "schedule_activity"); + assert_eq!(commands[0]["activity_type"], "undo"); +} + +#[test] +fn cooperative_replay_preserves_completed_forward_result_and_saga_cleanup_identity() { + for _cold_restart in 0..2 { + let mut events = completed_activity(1, "forward", json!(7)); + events.extend([ + scheduled_timer(2), + canonical_request(), + canonical_delivery(2, "timer"), + ]); + events.extend(completed_activity(3, "undo", Value::Null)); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + let result = ctx.activity("forward", json!([])).await?; + assert_eq!(result, json!(7)); + assert!(!ctx.is_cancellation_requested()?); + let mut saga = ctx.saga(); + saga.add_compensation("undo", json!([]))?; + saga.finish(ctx.sleep(Duration::from_secs(5)).await).await?; + Ok(Value::Null) + }); + let commands = worker + .execute_workflow_task(cancellation_task(events)) + .unwrap(); + assert_eq!(commands.len(), 1); + assert_eq!( + commands[0]["exception_type"], + "WorkflowCancellationRequested" + ); + assert_eq!( + commands[0]["exception"]["properties"]["request_id"], + "original-request" + ); + assert_eq!( + commands[0]["exception"]["properties"]["cleanup_deadline_at"], + "2026-10-01T08:10:00Z" + ); + } +} + +#[test] +fn cooperative_replay_saga_schedules_cleanup_once_after_committed_delivery() { + let mut events = completed_activity(1, "forward", json!(7)); + events.extend([ + scheduled_timer(2), + canonical_request(), + canonical_delivery(2, "timer"), + ]); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + ctx.activity("forward", json!([])).await?; + let mut saga = ctx.saga(); + saga.add_compensation("undo", json!([]))?; + saga.finish(ctx.sleep(Duration::from_secs(5)).await).await?; + Ok(Value::Null) + }); + let commands = worker + .execute_workflow_task(cancellation_task(events)) + .unwrap(); + assert_eq!(commands.len(), 1); + assert_eq!(commands[0]["type"], "schedule_activity"); + assert_eq!(commands[0]["activity_type"], "undo"); +} + +#[test] +fn cooperative_replay_consumes_reopened_condition_once_at_its_physical_delivery() { + let payload = |sequence| { + json!({"sequence":sequence,"condition_wait_id":format!("condition:{sequence}"), + "condition_wait_occurrence_id":"rust:condition-wait:0", "condition_key":"ready", + "condition_definition_fingerprint":"sha256:ready"}) + }; + let events = vec![ + event("ConditionWaitOpened", payload(1)), + event("ConditionWaitSatisfied", payload(1)), + event("ConditionWaitOpened", payload(2)), + canonical_request(), + canonical_delivery(2, "condition"), + ]; + for _cold_restart in 0..2 { + let ctx = workflow_context(events.clone()); + let mut wait = Box::pin( + ctx.wait_condition(ConditionWaitOptions::new("ready", "sha256:ready"), || { + Ok(false) + }), + ); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + let Poll::Ready(Err(Error::CooperativeCancellationRequested(cancellation))) = + wait.as_mut().poll(&mut cx) + else { + panic!("reopened condition must receive canonical cancellation") + }; + assert_eq!(cancellation.delivery.sequence, 2); + assert_eq!( + ctx.state.lock().unwrap().condition_wait_occurrence_counter, + 1 + ); + ctx.ensure_history_consumed().unwrap(); + assert!(ctx.take_commands().unwrap().is_empty()); + } +} + +fn cancelled_reopened_group_history() -> Vec { + let fixture: Value = serde_json::from_str(include_str!( + "../../tests/fixtures/replay-regressions/cooperative-grouped-condition-reopen.json" + )) + .unwrap(); + let mut events: Vec = serde_json::from_value(fixture["history"].clone()).unwrap(); + for event in &mut events { + if matches!( + event.event_type.as_str(), + "CooperativeCancellationRequested" | "CooperativeCancellationDelivered" + ) { + event.payload["workflow_run_id"] = json!("run"); + } + } + events +} + +#[test] +fn cooperative_group_delivery_consumes_pending_physical_condition_without_reopening() { + for reopens in 1..=2 { + let mut events = cancelled_reopened_group_history(); + if reopens == 2 { + let mut satisfied = events[3].clone(); + satisfied.event_type = "ConditionWaitSatisfied".into(); + let mut reopened = events[3].clone(); + reopened.payload["sequence"] = json!(4); + reopened.payload["condition_wait_id"] = json!("condition-4"); + events.splice(4..4, [satisfied, reopened]); + } + let ctx = workflow_context(events); + let mut call = Box::pin(ctx.parallel(vec![ + ParallelOperation::timer(Duration::from_secs(300)), + ParallelOperation::condition( + ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"), + || panic!("canonical cancellation must not reevaluate or reopen the condition"), + ), + ])); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + assert!(matches!( + call.as_mut().poll(&mut cx), + Poll::Ready(Err(Error::CooperativeCancellationRequested(_))) + )); + ctx.ensure_history_consumed().unwrap(); + assert!(ctx.take_commands().unwrap().is_empty()); + } +} + +#[test] +fn cooperative_group_eligibility_uses_the_latest_physical_condition_result() { + for latest_satisfied in [false, true] { + let mut events = cancelled_reopened_group_history(); + events.pop(); + let mut fired = events[0].clone(); + fired.event_type = "TimerFired".into(); + events.insert(4, fired); + if latest_satisfied { + let mut satisfied = events[3].clone(); + satisfied.event_type = "ConditionWaitSatisfied".into(); + events.insert(5, satisfied); + } + let cancellation = history(&events).unwrap(); + assert_eq!(cancellation.eligible(1, 2), !latest_satisfied); + } +} + +#[test] +fn cooperative_replay_keeps_adjacent_condition_occurrences_and_cleanup_scopes_separate() { + let payload = |sequence, occurrence| { + json!({"sequence":sequence,"condition_wait_id":format!("condition:{sequence}"), + "condition_wait_occurrence_id":format!("rust:condition-wait:{occurrence}"), "condition_key":"ready", + "condition_definition_fingerprint":"sha256:ready"}) + }; + let ctx = workflow_context(vec![ + event("ConditionWaitOpened", payload(1, 0)), + event("ConditionWaitSatisfied", payload(1, 0)), + event("ConditionWaitOpened", payload(2, 1)), + canonical_request(), + canonical_delivery(2, "condition"), + ]); + let shield = ctx.cancellation_shield().unwrap(); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + let mut first = Box::pin( + ctx.wait_condition(ConditionWaitOptions::new("ready", "sha256:ready"), || { + Ok(false) + }), + ); + assert!(matches!( + first.as_mut().poll(&mut cx), + Poll::Ready(Ok(ConditionWaitResult::Satisfied)) + )); + drop(shield); + let mut second = Box::pin( + ctx.wait_condition(ConditionWaitOptions::new("ready", "sha256:ready"), || { + Ok(false) + }), + ); + assert!(matches!( + second.as_mut().poll(&mut cx), + Poll::Ready(Err(Error::CooperativeCancellationRequested(_))) + )); + assert_eq!( + ctx.state.lock().unwrap().condition_wait_occurrence_counter, + 2 + ); + ctx.ensure_history_consumed().unwrap(); +} + +#[test] +fn cooperative_replay_marker_keeps_priority_over_resolution_committed_after_request() { + let ctx = workflow_context(vec![ + scheduled_timer(1), + canonical_request(), + canonical_delivery(1, "timer"), + event( + "TimerFired", + json!({"sequence":1,"timer_id":"timer-1","delay_seconds":5}), + ), + ]); + let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5))); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + assert!(matches!( + timer.as_mut().poll(&mut cx), + Poll::Ready(Err(Error::CooperativeCancellationRequested(_))) + )); + ctx.ensure_history_consumed().unwrap(); +} + +#[test] +fn cooperative_replay_actual_worker_delivers_scalar_calls_with_original_identity() { + for kind in ["activity", "timer", "child", "signal", "condition"] { + for _cold_restart in 0..2 { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _input| async move { + match kind { + "activity" => { + ctx.activity("forward", json!([])).await?; + } + "timer" => { + ctx.sleep(Duration::from_secs(5)).await?; + } + "child" => { + ctx.start_child_workflow( + "child", + ChildWorkflowOptions::new("queue"), + json!([]), + ) + .await?; + } + "signal" => { + ctx.wait_signal("resume").await?; + } + "condition" => { + ctx.wait_condition( + ConditionWaitOptions::new("ready", "sha256:ready"), + || Ok(false), + ) + .await?; + } + _ => unreachable!(), + } + Ok(Value::Null) + }); + let commands = worker + .execute_workflow_task(cancellation_task(vec![ + canonical_request(), + canonical_delivery(1, kind), + ])) + .unwrap(); + assert_eq!(commands.len(), 1, "{kind}"); + assert_eq!(commands[0]["type"], "fail_workflow", "{kind}"); + assert_eq!( + commands[0]["exception_type"], "WorkflowCancellationRequested", + "{kind}" + ); + assert_eq!( + commands[0]["exception"]["properties"]["request_id"], "original-request", + "{kind}" + ); + assert_eq!( + commands[0]["exception"]["properties"]["cleanup_deadline_at"], + "2026-10-01T08:10:00Z", + "{kind}" + ); + assert_eq!(commands[0]["non_retryable"], true, "{kind}"); + } + } +} + +#[test] +fn cooperative_replay_observed_request_and_task_flag_do_not_inject_without_delivery() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + assert!(!ctx.is_cancellation_requested()?); + ctx.throw_if_cancellation_requested()?; + ctx.sleep(Duration::from_secs(5)).await?; + Ok(Value::Null) + }); + let mut task = cancellation_task(vec![scheduled_timer(1), canonical_request()]); + task.cancel_requested = true; + assert!(worker.execute_workflow_task(task).unwrap().is_empty()); +} + +#[test] +fn cooperative_replay_recorded_timer_validates_authored_details_before_delivery() { + for delay in [5, 500] { + let ctx = workflow_context(vec![ + scheduled_timer(1), + canonical_request(), + canonical_delivery(1, "timer"), + ]); + let mut timer = Box::pin(ctx.sleep(Duration::from_secs(delay))); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + match timer.as_mut().poll(&mut cx) { + Poll::Ready(Err(Error::CooperativeCancellationRequested(cancellation))) + if delay == 5 => + { + assert_eq!(cancellation.request.request_id, "original-request"); + assert_eq!(cancellation.delivery.sequence, 1); + ctx.ensure_history_consumed().unwrap(); + } + Poll::Ready(Err(Error::NonDeterministicReplay(failure))) if delay == 500 => { + assert_eq!(failure.reason, "timer_delay_mismatch"); + } + other => panic!("{delay}: {other:?}"), + } + assert!(ctx.take_commands().unwrap().is_empty()); + } +} + +#[test] +fn cooperative_replay_changed_call_kind_and_unconsumed_boundary_are_rejected() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + ctx.activity("changed", json!([])).await?; + Ok(Value::Null) + }); + assert!( + matches!(worker.execute_workflow_task(cancellation_task(vec![canonical_request(), canonical_delivery(1, "timer")])), + Err(Error::NonDeterministicReplay(ReplayFailure { reason, .. })) if reason == "cooperative_cancellation_call_mismatch") + ); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |_ctx, _input| async { Ok(Value::Null) }); + assert!( + matches!(worker.execute_workflow_task(cancellation_task(vec![canonical_request(), canonical_delivery(1, "timer")])), + Err(Error::NonDeterministicReplay(ReplayFailure { reason, .. })) if reason == "recorded_commands_unconsumed") + ); +} + +#[test] +fn cooperative_replay_rejects_a_marker_that_skips_unrecorded_authored_calls() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + ctx.sleep(Duration::from_secs(5)).await?; + Ok(Value::Null) + }); + assert!( + matches!(worker.execute_workflow_task(cancellation_task(vec![canonical_request(), canonical_delivery(999, "timer")])), + Err(Error::NonDeterministicReplay(ReplayFailure { reason, .. })) if reason == "cooperative_cancellation_call_mismatch") + ); +} + +#[test] +fn cooperative_replay_shields_nested_cleanup_and_preserves_original_request_after_drop() { + let ctx = workflow_context(vec![canonical_request(), canonical_delivery(1, "timer")]); + let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5))); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + assert!(matches!( + timer.as_mut().poll(&mut cx), + Poll::Ready(Err(Error::CooperativeCancellationRequested(_))) + )); + assert!(ctx.is_cancellation_requested().unwrap()); + let outer = ctx.cancellation_shield().unwrap(); + let inner = ctx.cancellation_shield().unwrap(); + ctx.throw_if_cancellation_requested().unwrap(); + drop(inner); + ctx.throw_if_cancellation_requested().unwrap(); + drop(outer); + let Error::CooperativeCancellationRequested(cancellation) = + ctx.throw_if_cancellation_requested().unwrap_err() + else { + panic!("original cancellation missing") + }; + assert_eq!(cancellation.request.request_id, "original-request"); + assert_eq!( + cancellation.request.cleanup_deadline_at, + "2026-10-01T08:10:00Z" + ); +} + +#[test] +fn cooperative_replay_cannot_shield_away_a_committed_authored_delivery() { + let ctx = workflow_context(vec![canonical_request(), canonical_delivery(1, "timer")]); + let _shield = ctx.cancellation_shield().unwrap(); + let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5))); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + assert!(matches!( + timer.as_mut().poll(&mut cx), + Poll::Ready(Err(Error::NonDeterministicReplay(_))) + )); +} + +#[test] +fn cooperative_history_preserves_observation_without_authorizing_delivery() { + let observation = CancellationRequest::from_observation(&request_observation()).unwrap(); + let state = CancellationHistory::from_events(&[canonical_request()], "run", Some(&observation)) + .unwrap(); + assert_eq!(state.request.unwrap(), observation); + assert_eq!(state.delivery, None); + let pending = history(&[canonical_request()]).unwrap(); + assert!(pending.eligible(1, 1)); + assert_eq!(pending.request.unwrap().history_refresh_page_token, None); +} + +#[test] +fn cooperative_history_preserves_earlier_results_and_allows_later_unresolved_calls() { + let state = history(&[ + event("ActivityCompleted", json!({"sequence":1})), + canonical_request(), + event("TimerFired", json!({"sequence":2})), + ]) + .unwrap(); + assert!(!state.eligible(1, 1)); + assert!(state.eligible(2, 1)); + assert!(state.eligible(1, 2)); +} + +#[test] +fn cooperative_history_preserves_prior_parallel_failure_and_committed_selection() { + for prior in [ + event("ActivityFailed", json!({"sequence":1})), + event( + "SelectionResolved", + json!({ + "selection_group_base_sequence":1,"selection_group_size":2 + }), + ), + ] { + let state = history(&[prior.clone(), canonical_request()]).unwrap(); + assert!(!state.eligible(1, 2)); + let mut marker = canonical_delivery(1, "parallel"); + marker.payload["sequence_span"] = json!(2); + assert!(matches!( + history(&[prior, canonical_request(), marker]), + Err(Error::NonDeterministicReplay(_)) + )); + } +} + +#[test] +fn cooperative_history_preserves_cancelled_selection_member_ranges() { + let state = history(&[ + event( + "SelectionOperationCancelled", + json!({"member_base_sequence":3,"member_size":2}), + ), + canonical_request(), + ]) + .unwrap(); + assert!(!state.eligible(3, 2)); + assert!(state.eligible(4, 2)); +} + +#[test] +fn cooperative_history_cold_delivery_retains_parallel_and_selection_operation_ranges() { + let mut parallel = canonical_delivery(2, "parallel"); + parallel.payload["sequence_span"] = json!(3); + let state = history(&[canonical_request(), parallel]).unwrap(); + assert_eq!(state.request_index, 0); + assert_eq!(state.delivery_index, Some(1)); + assert!(!state.eligible(5, 1)); + let delivery = state.delivery.unwrap(); + assert_eq!( + (1..=5) + .map(|sequence| delivery.interrupts(sequence)) + .collect::>(), + vec![false, true, true, true, false] + ); + let mut selected = canonical_delivery(6, "selection_handle"); + selected.payload["operation_sequence"] = json!(2); + selected.payload["operation_sequence_span"] = json!(2); + let delivered = history(&[canonical_request(), selected]) + .unwrap() + .delivery + .unwrap(); + assert!(delivered.interrupts(2)); + assert!(delivered.interrupts(3)); + assert!(!delivered.interrupts(4)); + assert!(!delivered.interrupts(6)); +} + +#[test] +fn cooperative_history_rejects_reordered_duplicate_and_mismatched_markers() { + for events in [ + vec![canonical_delivery(2, "timer")], + vec![canonical_request(), canonical_request()], + vec![ + canonical_request(), + canonical_delivery(2, "timer"), + canonical_delivery(2, "timer"), + ], + vec![canonical_delivery(2, "timer"), canonical_request()], + ] { + assert!(matches!( + history(&events), + Err(Error::NonDeterministicReplay(_)) + )); + } + for (field, value) in [ + ("workflow_command_id", json!("different")), + ("workflow_run_id", json!("different")), + ("workflow_command_id", json!(" ")), + ("workflow_run_id", Value::Null), + ] { + let mut marker = canonical_delivery(2, "timer"); + marker.payload[field] = value; + assert!(matches!( + history(&[canonical_request(), marker]), + Err(Error::NonDeterministicReplay(_)) + )); + } + let mut request = canonical_request(); + request + .raw + .insert("workflow_command_id".into(), json!("different")); + assert!(matches!( + history(&[request]), + Err(Error::NonDeterministicReplay(_)) + )); +} + +#[test] +fn cooperative_history_rejects_changed_observation_identity_deadline_and_malformed_time() { + for (field, value) in [ + ("request_id", json!("different")), + ("cleanup_deadline_at", json!("2026-10-01T08:11:00Z")), + ("requested_at", json!("no timestamp")), + ("history_refresh_page_token", json!(" ")), + ] { + let mut observation = request_observation(); + observation[field] = value; + match CancellationRequest::from_observation(&observation) { + Ok(observed) => assert!(matches!( + CancellationHistory::from_events(&[canonical_request()], "run", Some(&observed)), + Err(Error::NonDeterministicReplay(_)) + )), + Err(error) => assert!(matches!(error, Error::InvalidCooperativeCancellation(_))), + } + } + let mut request = canonical_request(); + request.raw.insert("recorded_at".into(), json!("bad")); + assert!(matches!( + history(&[request]), + Err(Error::NonDeterministicReplay(_)) + )); +} + +#[test] +fn cooperative_history_accepts_equivalent_deadline_timezone_without_changing_original_text() { + let mut observation = request_observation(); + observation["cleanup_deadline_at"] = json!("2026-10-01T10:10:00+02:00"); + let observed = CancellationRequest::from_observation(&observation).unwrap(); + let state = + CancellationHistory::from_events(&[canonical_request()], "run", Some(&observed)).unwrap(); + assert_eq!(state.request.unwrap(), observed); +} + +#[test] +fn cooperative_history_rejects_invalid_call_ranges_and_portable_integer_overflow() { + for (field, value) in [ + ("sequence", json!(0)), + ("sequence", json!(true)), + ("sequence", json!("2")), + ("sequence", json!(i64::MAX)), + ("sequence", json!(u64::MAX)), + ("call_kind", json!("unknown")), + ("sequence_span", json!(2)), + ("sequence_span", Value::Null), + ("operation_sequence", json!(1)), + ("operation_sequence_span", json!(2)), + ] { + let mut marker = canonical_delivery(2, "timer"); + marker.payload[field] = value; + assert!(matches!( + history(&[canonical_request(), marker]), + Err(Error::NonDeterministicReplay(_)) + )); + } + for (base, span) in [(0, 1), (6, 1), (5, 2), (2, 1001)] { + let mut selected = canonical_delivery(6, "selection_handle"); + selected.payload["operation_sequence"] = json!(base); + selected.payload["operation_sequence_span"] = json!(span); + assert!(matches!( + history(&[canonical_request(), selected]), + Err(Error::NonDeterministicReplay(_)) + )); + } + let mut parallel = canonical_delivery(2, "parallel"); + parallel.payload["sequence_span"] = json!(1001); + assert!(matches!( + history(&[canonical_request(), parallel]), + Err(Error::NonDeterministicReplay(_)) + )); + assert!(!history(&[canonical_request()]) + .unwrap() + .eligible(u64::MAX, 1)); +} + +#[test] +fn cooperative_history_rejects_delivery_over_an_earlier_resolved_call() { + for kind in [ + "ActivityCompleted", + "ActivityFailed", + "ActivityCancelled", + "ActivityTimedOut", + "TimerFired", + "TimerCancelled", + "ConditionWaitSatisfied", + "ConditionWaitTimedOut", + "SignalApplied", + "ChildRunCompleted", + "ChildRunFailed", + "ChildRunCancelled", + "ChildRunTerminated", + ] { + assert!( + matches!( + history(&[ + event(kind, json!({"sequence":2})), + canonical_request(), + canonical_delivery(2, "timer") + ]), + Err(Error::NonDeterministicReplay(_)) + ), + "{kind}" + ); + } +} + +fn responses(path: &str, _body: &str, number: usize) -> Option<(&'static str, String)> { + let case = path.split('/').nth(1).unwrap_or_default(); + if path.ends_with("/api/cluster/info") { + if case == "discovery-http" { + return Some(("403 Forbidden", r#"{"reason":"forbidden"}"#.into())); + } + if case == "budget" { + thread::sleep(Duration::from_millis(2500)); + } + let mut discovery = json!({"worker_protocol": { + "version": "1.20", "server_capabilities": {"cooperative_cancellation": true} + }}); + match case { + "unsupported" => { + discovery["worker_protocol"]["server_capabilities"]["cooperative_cancellation"] = + json!(false) + } + "missing" => discovery = json!({}), + "string-flag" => { + discovery["worker_protocol"]["server_capabilities"]["cooperative_cancellation"] = + json!("true") + } + "old" => discovery["worker_protocol"]["version"] = json!("1.19"), + "future-major" => discovery["worker_protocol"]["version"] = json!("2.20"), + "malformed-version" => discovery["worker_protocol"]["version"] = json!("1.+20"), + "new-minor" => discovery["worker_protocol"]["version"] = json!("1.21"), + _ => {} + } + return Some(("200 OK", discovery.to_string())); + } + if !path.ends_with("/request-cancellation") { + return None; + } + if case == "refused" { + return Some(( + "409 Conflict", + r#"{"reason":"active_claim_cancellation_not_supported"}"#.into(), + )); + } + if case == "budget" { + thread::sleep(Duration::from_secs(3)); + } + let mut response = json!({ + "accepted": true, "duplicate": number > 1, + "workflow_id": "workflow", "run_id": "run", + "cancellation_request": request_observation() + }); + match case { + "escaped" => { + response["workflow_id"] = json!("workflow/with?path"); + response["run_id"] = json!("run/selected"); + } + "wrong-workflow" => response["workflow_id"] = json!("other"), + "wrong-run" => response["run_id"] = json!("other"), + "empty-run" => response["run_id"] = json!(" "), + "not-accepted" => response["accepted"] = json!(false), + "accepted-string" => response["accepted"] = json!("true"), + "duplicate-integer" => response["duplicate"] = json!(0), + "missing-request" => response["cancellation_request"] = Value::Null, + "blank-request" => response["cancellation_request"]["request_id"] = json!(" "), + "blank-token" => { + response["cancellation_request"]["history_refresh_page_token"] = json!(" ") + } + "no-timezone" => { + response["cancellation_request"]["requested_at"] = json!("2026-10-01T08:00:00") + } + "invalid-date" => { + response["cancellation_request"]["requested_at"] = json!("2026-02-30T08:00:00Z") + } + "equal-deadline" => { + response["cancellation_request"]["cleanup_deadline_at"] = json!("2026-10-01T08:00:00Z") + } + "earlier-deadline" => { + response["cancellation_request"]["cleanup_deadline_at"] = json!("2026-10-01T07:59:00Z") + } + _ => {} + } + Some(("202 Accepted", response.to_string())) +} + +fn server() -> MockWorkerServer { + MockWorkerServer::start_with_behavior(MockWorkerBehavior { + request_override: Some(responses), + ..MockWorkerBehavior::default() + }) +} + +fn client(server: &MockWorkerServer, case: &str) -> Client { + Client::builder(format!("{}/{case}", server.base_url())) + .control_token(Some("control-only".to_string())) + .worker_token(Some("worker-only".to_string())) + .namespace("caller-namespace") + .build() + .unwrap() +} + +fn transport_task() -> WorkflowTask { + let mut task = cancellation_task(vec![scheduled_timer(1)]); + task.task_id = "task/selected".into(); + task.workflow_task_attempt = 7; + task.lease_owner = Some("actual-owner".into()); + task +} + +fn transport_delivery() -> CancellationDelivery { + CancellationDelivery::from_payload(&canonical_delivery(1, "timer").payload).unwrap() +} + +fn pending_child_delivery(task_id: &str) -> Value { + json!({"delivered":false, "task_id":task_id, "workflow_run_id":"run", + "reason":"cancellation_waiting_for_child", "claim_released":true, + "request_id":null, "sequence":null, "call_kind":null, "sequence_span":null, + "operation_sequence":null, "operation_sequence_span":null}) +} + +fn transport_responses(path: &str, body: &str, number: usize) -> Option<(&'static str, String)> { + let case = path.split('/').nth(1).unwrap_or_default(); + let body: Value = serde_json::from_str(body).unwrap(); + if case == "refused" || (case == "lost-ack" && path.ends_with("/deliver-cancellation")) { + return Some(( + "409 Conflict", + json!({"reason":"lease_owner_mismatch"}).to_string(), + )); + } + if case == "budget" { + thread::sleep(Duration::from_secs(3)); + } + if case == "heartbeat-budget" { + thread::sleep(Duration::from_millis(5200)); + } + if case == "claim-budget" { + thread::sleep(Duration::from_secs(3)); + } + if path.ends_with("/workflow-tasks/poll") { + if case == "claim-retry" && number == 1 { + // The first HTTP reply is unreadable after acquisition. Retrying + // must reuse the same durable poll identity rather than claim twice. + return Some(("invalid-status", String::new())); + } + if matches!(case, "claim-idle" | "claim-stop") { + return Some(( + "200 OK", + json!({"task":null, + "poll_status":if case == "claim-stop" { "draining" } else { "idle" }, + "reason":if case == "claim-stop" { "worker_draining" } else { "no_tasks" }, + "protocol_version":"1.20" + }) + .to_string(), + )); + } + let mut task = json!({ + "task_id":"task/selected", "workflow_type":"cancel", "run_id":"run", + "lease_owner":"actual-owner", "workflow_task_attempt":7, + "payload_codec":DEFAULT_CODEC, "history_events":[], + "cancel_requested":true, "cancellation_request":request_observation() + }); + match case { + "claim-no-observation" => { + task.as_object_mut().unwrap().remove("cancellation_request"); + } + "claim-missing-attempt" => { + task.as_object_mut() + .unwrap() + .remove("workflow_task_attempt"); + } + "claim-zero-attempt" => task["workflow_task_attempt"] = json!(0), + "claim-owner" => task["lease_owner"] = json!("replacement"), + "claim-missing-owner" => task["lease_owner"] = Value::Null, + "claim-missing-run" => task["run_id"] = Value::Null, + "claim-missing-id" => task["task_id"] = json!(" "), + "claim-bad-observation" => task["cancellation_request"] = json!(false), + "claim-bad-token" => { + task["cancellation_request"]["history_refresh_page_token"] = json!(" ") + } + _ => {} + } + if matches!( + case, + "claim-history" + | "claim-budget" + | "cycle" + | "page-bound" + | "empty-progress" + | "oversized-page" + | "wrong-task" + | "wrong-attempt" + | "malformed-event" + | "empty-token" + | "missing-token" + ) { + task["next_history_page_token"] = json!("opaque-server-token"); + } + return Some(( + "200 OK", + json!({"task":task,"protocol_version":"1.20", + "server_capabilities":{"workflow_memo_updates":{"supported":true}} + }) + .to_string(), + )); + } + if path.ends_with("/deliver-cancellation") + && (case.starts_with("child-wait") || case.starts_with("activity-wait")) + { + let mut pending = pending_child_delivery("task/selected"); + if case.starts_with("activity-wait") { + pending["reason"] = json!("cancellation_waiting_for_activity"); + } + let suffix = case + .strip_prefix("child-wait") + .or_else(|| case.strip_prefix("activity-wait")) + .unwrap(); + match suffix { + "-wrong-task" => pending["task_id"] = json!("other"), + "-wrong-run" => pending["workflow_run_id"] = json!("other"), + "-not-released" => pending["claim_released"] = json!(false), + "-string-released" => pending["claim_released"] = json!("true"), + "-missing-release" => { + pending.as_object_mut().unwrap().remove("claim_released"); + } + "-wrong-reason" => pending["reason"] = json!("other"), + "-request" => pending["request_id"] = body["request_id"].clone(), + "-sequence" => pending["sequence"] = json!(1), + "-kind" => pending["call_kind"] = json!("child"), + "-span" => pending["sequence_span"] = json!(1), + "-operation" => pending["operation_sequence"] = json!(1), + "-operation-span" => pending["operation_sequence_span"] = json!(1), + "-string-delivered" => pending["delivered"] = json!("false"), + _ => {} + } + return Some(("200 OK", pending.to_string())); + } + let mut response = if path.ends_with("/deliver-cancellation") { + json!({"delivered":true, "task_id":"task/selected", "workflow_run_id":"run", + "request_id":body["request_id"], "sequence":body["sequence"], "call_kind":body["call_kind"], + "sequence_span":body["sequence_span"], "operation_sequence":body["operation_sequence"], + "operation_sequence_span":body["operation_sequence_span"], "reason":null}) + } else if path.ends_with("/heartbeat") { + json!({"task_id":"task/selected", "workflow_task_attempt":7, + "lease_owner":"actual-owner", "renewed":true, + "lease_expires_at":"2026-10-01T08:00:30Z", "run_status":"running", + "task_status":"leased", "reason":null, "cancellation_request":request_observation()}) + } else if path.ends_with("/history") { + let first = body["next_history_page_token"] == "opaque-server-token"; + let events = if first { + vec![scheduled_timer(1), canonical_request()] + } else { + vec![canonical_delivery(1, "timer")] + }; + let events = events + .into_iter() + .map(|event| { + let mut value = event.raw; + value.insert("event_type".into(), json!(event.event_type)); + value.insert("payload".into(), event.payload); + Value::Object(value.into_iter().collect()) + }) + .collect::>(); + let mut page = json!({"task_id":"task/selected", "workflow_task_attempt":7, + "history_events":events, "total_history_events":3, + "next_history_page_token":if first { json!("page-next") } else { Value::Null }}); + match case { + "cycle" if !first => page["next_history_page_token"] = json!("opaque-server-token"), + "page-bound" => { + page["history_events"] = + json!([{"event_type":"OpaqueUnrelatedEvent", "payload":{}}]); + page["next_history_page_token"] = json!(format!("next-{number}")); + } + "empty-progress" => page["history_events"] = json!([]), + "oversized-page" => { + page["history_events"] = json!(vec![ + json!({"event_type":"OpaqueUnrelatedEvent"}); + WORKFLOW_HISTORY_PAGE_SIZE as usize + 1 + ]) + } + "missing-request" => { + page["history_events"] = if first { + json!([{"event_type":"TimerScheduled", "payload":{"sequence":1,"delay_seconds":5}}]) + } else { + json!([]) + }; + } + "changed-canonical" if !first => { + page["history_events"][0]["payload"]["workflow_command_id"] = json!("changed") + } + "malformed-event" => page["history_events"] = json!([{"event_type":false}]), + "wrong-attempt" => page["workflow_task_attempt"] = json!(8), + "empty-token" => page["next_history_page_token"] = json!(""), + "missing-token" => { + page.as_object_mut() + .unwrap() + .remove("next_history_page_token"); + } + _ => {} + } + page + } else { + return None; + }; + match case { + "wrong-task" => response["task_id"] = json!("other-task"), + "wrong-run" => response["workflow_run_id"] = json!("other-run"), + "heartbeat-attempt" => response["workflow_task_attempt"] = json!(8), + "heartbeat-owner" => response["lease_owner"] = json!("replacement-owner"), + "heartbeat-not-renewed" => response["renewed"] = json!(false), + "heartbeat-string-renewed" => response["renewed"] = json!("true"), + "heartbeat-closed" => response["run_status"] = json!("cancelled"), + "heartbeat-unknown-run" => response["run_status"] = json!("unknown"), + "heartbeat-finished-task" => response["task_status"] = json!("completed"), + "heartbeat-expiry" => response["lease_expires_at"] = json!("bad"), + "heartbeat-expiry-no-zone" => response["lease_expires_at"] = json!("2026-10-01T08:00:30"), + "heartbeat-malformed-request" => response["cancellation_request"] = json!(false), + "heartbeat-missing-token" => { + response["cancellation_request"] + .as_object_mut() + .unwrap() + .remove("history_refresh_page_token"); + } + "heartbeat-changed-request" => { + response["cancellation_request"]["request_id"] = json!("changed") + } + "heartbeat-changed-request-time" => { + response["cancellation_request"]["requested_at"] = json!("2026-10-01T08:00:01Z") + } + "heartbeat-changed-deadline" => { + response["cancellation_request"]["cleanup_deadline_at"] = json!("2026-10-01T08:11:00Z") + } + "heartbeat-equivalent-time" => { + response["cancellation_request"]["requested_at"] = json!("2026-10-01T10:00:00+02:00"); + response["cancellation_request"]["cleanup_deadline_at"] = + json!("2026-10-01T10:10:00+02:00"); + response["cancellation_request"]["history_refresh_page_token"] = + json!("fresh-server-token"); + } + "heartbeat-no-request" => { + response + .as_object_mut() + .unwrap() + .remove("cancellation_request"); + } + "wrong-request" => response["request_id"] = json!("other-request"), + "wrong-sequence" => response["sequence"] = json!(2), + "wrong-kind" => response["call_kind"] = json!("activity"), + "wrong-span" => response["sequence_span"] = json!(2), + "false-delivered" => response["delivered"] = json!(false), + "string-delivered" => response["delivered"] = json!("true"), + "wrong-reason" => response["reason"] = json!("not_committed"), + "missing-span" => { + response.as_object_mut().unwrap().remove("sequence_span"); + } + "missing-operation" => { + response + .as_object_mut() + .unwrap() + .remove("operation_sequence"); + } + "missing-operation-span" => { + response + .as_object_mut() + .unwrap() + .remove("operation_sequence_span"); + } + _ => {} + } + Some(("200 OK", response.to_string())) +} + +fn transport_server() -> MockWorkerServer { + MockWorkerServer::start_with_behavior(MockWorkerBehavior { + request_override: Some(transport_responses), + ..Default::default() + }) +} + +fn remote_activity_responses( + path: &str, + body: &str, + number: usize, +) -> Option<(&'static str, String)> { + let case = path.split('/').nth(1).unwrap_or_default(); + if path.ends_with("/cluster/info") { + return responses(path, body, number); + } + if path.ends_with("/worker/register") { + let request: Value = serde_json::from_str(body).unwrap(); + let mut reply = json!({"registered":true,"worker_id":request["worker_id"], + "namespace":"caller-namespace","task_queue":request["task_queue"], + "capabilities":request["capabilities"],"heartbeat_interval_seconds":3600, + "protocol_version":"1.20","server_capabilities":{"cooperative_cancellation":true}}); + match case { + "register-old-protocol" => reply["protocol_version"] = json!("1.19"), + "register-wrong-major" => reply["protocol_version"] = json!("2.0"), + "register-malformed-protocol" => reply["protocol_version"] = json!("1.20garbage"), + "register-missing-protocol" => { + reply.as_object_mut().unwrap().remove("protocol_version"); + } + "register-unsupported" | "register-cleanup-refused" => { + reply["server_capabilities"]["cooperative_cancellation"] = json!(false) + } + "register-string-support" => { + reply["server_capabilities"]["cooperative_cancellation"] = json!("true") + } + "register-missing-support" => reply["server_capabilities"] = json!({}), + "register-wrong-worker" => reply["worker_id"] = json!("other-worker"), + "register-wrong-namespace" => reply["namespace"] = json!("other-namespace"), + "register-wrong-queue" => reply["task_queue"] = json!("other-queue"), + "register-missing-capability" => { + reply["capabilities"] = json!(["cooperative_cancellation"]) + } + "register-no-cooperative" => reply["capabilities"] + .as_array_mut() + .unwrap() + .retain(|value| value != "cooperative_cancellation"), + "register-declined" => reply["registered"] = json!(false), + "register-newer" => reply["protocol_version"] = json!("1.21"), + "register-replaced" if number >= 2 => reply["worker_id"] = json!("replacement"), + _ => {} + } + return Some(("200 OK", reply.to_string())); + } + if path.ends_with("/worker/heartbeat") { + return Some(("200 OK", json!({}).to_string())); + } + if path.ends_with("/worker/registrations/actual-owner") { + if case == "register-cleanup-refused" { + return Some(( + "503 Service Unavailable", + json!({"reason":"backend_unavailable"}).to_string(), + )); + } + return Some(("200 OK", json!({"worker_id":"actual-owner","outcome":"deregistered","recovered_workflow_task_count":1}).to_string())); + } + if path.ends_with("/activity-tasks/poll") { + if number > if case == "remote-concurrency" { 3 } else { 1 } { + return Some(( + "200 OK", + json!({"task":null,"poll_status":"timeout"}).to_string(), + )); + } + let mut task = json!({"task_id":"activity","activity_attempt_id":"attempt-original", + "activity_type":"work","payload_codec":"avro","lease_owner":"actual-owner","attempt_number":3}); + if case == "remote-concurrency" { + task["task_id"] = json!(format!("activity-{number}")); + task["activity_attempt_id"] = json!(format!("attempt-{number}")); + } + match case { + "remote-missing-owner" => { + task.as_object_mut().unwrap().remove("lease_owner"); + } + "remote-missing-attempt" => { + task.as_object_mut().unwrap().remove("activity_attempt_id"); + } + "remote-invented-owner" => task["lease_owner"] = json!("other-worker"), + "remote-conflicting-attempt" => task["attempt_id"] = json!("other-attempt"), + _ => {} + } + return Some(( + "200 OK", + json!({"task":task,"poll_status":"leased"}).to_string(), + )); + } + if path.ends_with("/status") { + let request: Value = serde_json::from_str(body).unwrap(); + if case == "remote-refused" || (case == "remote-replaced" && number >= 2) { + return Some(( + "409 Conflict", + json!({"reason":"lease_owner_mismatch"}).to_string(), + )); + } + if case == "remote-offline" { + return Some(( + "503 Service Unavailable", + json!({"reason":"backend_unavailable"}).to_string(), + )); + } + if case == "remote-status-slow" { + std::thread::sleep(Duration::from_millis(5250)); + } + let mut reply = json!({"task_id":if path.ends_with("/activity%2Fselected/status") { "activity/selected" } else { "activity" }, + "activity_attempt_id":request["activity_attempt_id"],"lease_owner":request["lease_owner"], + "can_continue":true,"cancel_requested":false,"reason":null,"heartbeat_recorded":false, + "lease_expires_at":"2099-01-01T00:00:00Z","deadlines":null,"worker_session":null, + "task_status":"leased","attempt_status":"running","activity_status":"running"}); + if case == "remote-concurrency" { + reply["task_id"] = json!(path.rsplit('/').nth(1).unwrap()); + } + match case { + "remote-wrong-task" => reply["task_id"] = json!("other-task"), + "remote-wrong-attempt" => reply["activity_attempt_id"] = json!("other-attempt"), + "remote-wrong-owner" => reply["lease_owner"] = json!("other-worker"), + "remote-malformed" => reply["can_continue"] = json!("true"), + "remote-recorded-progress" => reply["heartbeat_recorded"] = json!(true), + "remote-closed-task" => reply["task_status"] = json!("completed"), + "remote-closed-attempt" => reply["attempt_status"] = json!("completed"), + "remote-closed-activity" => reply["activity_status"] = json!("completed"), + "remote-expired" => reply["lease_expires_at"] = json!("2000-01-01T00:00:00Z"), + "remote-no-timezone" => reply["lease_expires_at"] = json!("2099-01-01T00:00:00"), + "remote-deadline" => reply["deadlines"] = json!({"heartbeat":"2000-01-01T00:00:00Z"}), + "remote-bad-deadlines" => reply["deadlines"] = json!(false), + "remote-session-expired" => { + reply["worker_session"] = json!({"status":"active","lease_owner":"actual-owner","lease_expires_at":"2099-01-01T00:00:00Z","ttl_expires_at":"2000-01-01T00:00:00Z"}) + } + "remote-session-replaced" => { + reply["worker_session"] = json!({"status":"active","lease_owner":"replacement","lease_expires_at":"2099-01-01T00:00:00Z","ttl_expires_at":"2099-01-01T00:00:00Z"}) + } + "remote-cancelled" | "remote-late-result" + if case == "remote-cancelled" || number >= 2 => + { + reply["can_continue"] = json!(false); + reply["cancel_requested"] = json!(true); + reply["reason"] = json!("activity_cancelled"); + } + "remote-heartbeat-post-loss" if number >= 3 => reply["can_continue"] = json!(false), + _ => {} + } + if case.starts_with("remote-stop-joined-") && number >= 2 { + reply["can_continue"] = json!(false); + reply["cancel_requested"] = json!(true); + reply["reason"] = json!("activity_cancelled"); + reply["cancellation_acknowledgement"] = json!({"request_id":"local-request","root_request_id":"root-request", + "cleanup_deadline_at":"2099-01-01T00:00:30Z","cancellation_history_event_id":"cancel-history","callback_state":"unknown"}); + } + return Some(("200 OK", reply.to_string())); + } + if path.ends_with("/acknowledge-cancellation") { + let request: Value = serde_json::from_str(body).unwrap(); + if case.starts_with("remote-stop-joined-") + && std::fs::read_to_string(std::env::temp_dir().join(case)) + .ok() + .as_deref() + != Some("dropped") + { + return Some(( + "409 Conflict", + json!({"reason":"callback_not_dropped"}).to_string(), + )); + } + if case == "remote-ack-refused" { + return Some(( + "409 Conflict", + json!({"reason":"cancellation_request_mismatch"}).to_string(), + )); + } + let mut reply = json!({"task_id":"activity/selected","activity_attempt_id":request["activity_attempt_id"], + "lease_owner":request["lease_owner"],"request_id":request["request_id"],"acknowledged":true, + "duplicate":number > 1,"history_event_id":"stop-history","reason":null,"heartbeat_recorded":false}); + if case.starts_with("remote-stop-joined-") { + reply["task_id"] = json!("activity"); + } + match case { + "remote-ack-task" => reply["task_id"] = json!("wrong"), + "remote-ack-attempt" => reply["activity_attempt_id"] = json!("wrong"), + "remote-ack-owner" => reply["lease_owner"] = json!("wrong"), + "remote-ack-request" => reply["request_id"] = json!("wrong"), + "remote-ack-unproved" => reply["history_event_id"] = Value::Null, + "remote-ack-declined" => reply["acknowledged"] = json!(false), + "remote-ack-progress" => reply["heartbeat_recorded"] = json!(true), + "remote-ack-duplicate" => reply["duplicate"] = json!("true"), + "remote-ack-slow" => std::thread::sleep(Duration::from_millis(5250)), + _ => {} + } + return Some(("200 OK", reply.to_string())); + } + if path.ends_with("/activity/heartbeat") { + let mut reply = json!({"task_id":"activity","activity_attempt_id":"attempt-original", + "lease_owner":"actual-owner","can_continue":true,"cancel_requested":false,"heartbeat_recorded":true}); + match case { + "remote-heartbeat-wrong-attempt" => reply["activity_attempt_id"] = json!("replacement"), + "remote-heartbeat-cancelled" => reply["cancel_requested"] = json!(true), + _ => {} + } + return Some(("200 OK", reply.to_string())); + } + if path.ends_with("/complete") || path.ends_with("/fail") { + return Some(("200 OK", json!({"recorded":true}).to_string())); + } + None +} + +fn remote_activity_server() -> MockWorkerServer { + MockWorkerServer::start_with_behavior(MockWorkerBehavior { + request_override: Some(remote_activity_responses), + ..Default::default() + }) +} + +#[tokio::test] +async fn cooperative_registration_uses_explicit_protocol_and_preserves_other_clients() { + for case in ["register-valid", "register-newer"] { + let server = remote_activity_server(); + let original = client(&server, case); + let worker = Worker::new(original.clone(), "queue") + .worker_id("actual-owner") + .cooperative_cancellation(true); + let registration = worker.register().await.unwrap(); + assert!(registration.registered); + assert!(worker + .cooperative_registration_confirmed + .load(Ordering::SeqCst)); + original + .poll_activity_task("actual-owner", "queue", Duration::ZERO) + .await + .unwrap(); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].worker_protocol.as_deref(), Some("1.20")); + assert_eq!(requests[1].worker_protocol.as_deref(), Some("1.19")); + assert_eq!( + requests[0].authorization.as_deref(), + Some("Bearer worker-only") + ); + assert_eq!(requests[0].namespace.as_deref(), Some("caller-namespace")); + let body: Value = serde_json::from_str(&requests[0].body).unwrap(); + assert!(body["capabilities"] + .as_array() + .unwrap() + .contains(&json!("cooperative_cancellation"))); + assert_eq!( + body["capability_manifest"], + portable_worker_affinity_capability_manifest() + ); + } +} + +#[tokio::test] +async fn cooperative_registration_refuses_incompatible_receipts_before_polling() { + for case in [ + "register-old-protocol", + "register-wrong-major", + "register-malformed-protocol", + "register-missing-protocol", + "register-unsupported", + "register-string-support", + "register-missing-support", + "register-wrong-worker", + "register-wrong-namespace", + "register-wrong-queue", + "register-missing-capability", + "register-no-cooperative", + "register-declined", + ] { + let server = remote_activity_server(); + let worker = coordinator_worker(&server, case); + assert!( + matches!( + worker.register().await, + Err(Error::CooperativeCancellationUnavailable(_)) + ), + "{case}" + ); + assert!( + matches!( + worker.run_once().await, + Err(Error::CooperativeCancellationUnavailable(_)) + ), + "{case}" + ); + let requests = server.requests.lock().unwrap(); + assert_eq!( + requests.len(), + if matches!(case, "register-wrong-worker" | "register-declined") { + 1 + } else { + 2 + }, + "{case}" + ); + assert!(requests[0].path.ends_with("/register")); + assert!( + !requests + .iter() + .any(|request| request.path.ends_with("/poll")), + "{case}" + ); + if requests.len() > 1 { + assert_eq!(requests[1].method, "DELETE"); + assert!(requests[1].path.ends_with("/registrations/actual-owner")); + assert_eq!(requests[1].worker_protocol.as_deref(), Some("1.20")); + } + } +} + +#[tokio::test] +async fn cooperative_registration_preserves_cleanup_failure_and_requires_registration() { + let server = remote_activity_server(); + let worker = coordinator_worker(&server, "register-cleanup-refused"); + assert!(matches!( + worker.run_once().await, + Err(Error::CooperativeCancellationUnavailable(_)) + )); + assert!(server.requests.lock().unwrap().is_empty()); + let error = worker.register().await.unwrap_err(); + assert!( + matches!(error, Error::WorkerShutdown {primary, deregistration} + if matches!(*primary, Error::CooperativeCancellationUnavailable(_)) + && matches!(*deregistration, Error::Http { status, .. } if status.as_u16() == 503)) + ); + assert!(!worker + .cooperative_registration_confirmed + .load(Ordering::SeqCst)); +} + +#[tokio::test] +async fn cooperative_registration_refusal_invalidates_previous_confirmation_and_changed_identity() { + let server = remote_activity_server(); + let worker = coordinator_worker(&server, "register-replaced"); + worker.register().await.unwrap(); + assert!(worker + .cooperative_registration_confirmed + .load(Ordering::SeqCst)); + assert!(matches!( + worker.register().await, + Err(Error::CooperativeCancellationUnavailable(_)) + )); + assert!(matches!( + worker.run_once().await, + Err(Error::CooperativeCancellationUnavailable(_)) + )); + assert_eq!(server.requests.lock().unwrap().len(), 2); + + let worker = coordinator_worker(&server, "register-valid"); + worker.register().await.unwrap(); + let changed = worker.worker_id("replacement"); + assert!(matches!( + changed.run_once().await, + Err(Error::CooperativeCancellationUnavailable(_)) + )); + assert_eq!(server.requests.lock().unwrap().len(), 3); +} + +#[tokio::test] +async fn cooperative_registration_default_and_disabled_workers_keep_ordinary_protocol() { + for explicit in [false, true] { + let server = remote_activity_server(); + let worker = + Worker::new(client(&server, "register-valid"), "queue").worker_id("actual-owner"); + let worker = if explicit { + worker + .cooperative_cancellation(true) + .cooperative_cancellation(false) + } else { + worker + }; + worker.register().await.unwrap(); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests[0].worker_protocol.as_deref(), Some("1.19")); + let body: Value = serde_json::from_str(&requests[0].body).unwrap(); + assert!(!body["capabilities"] + .as_array() + .unwrap() + .contains(&json!("cooperative_cancellation"))); + } +} + +#[tokio::test] +async fn cooperative_registration_requires_only_worker_credentials() { + let server = remote_activity_server(); + let client = Client::builder(format!("{}/register-valid", server.base_url())) + .worker_token(Some("worker-only".into())) + .namespace("caller-namespace") + .build() + .unwrap(); + let worker = Worker::new(client, "queue") + .worker_id("actual-owner") + .cooperative_cancellation(true); + worker.register().await.unwrap(); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert_eq!( + requests[0].authorization.as_deref(), + Some("Bearer worker-only") + ); + assert!(requests[0].control_protocol.is_none()); +} + +struct PendingActivityCallback(Arc); + +impl Future for PendingActivityCallback { + type Output = Result; + + fn poll(self: Pin<&mut Self>, _: &mut TaskContext<'_>) -> Poll { + Poll::Pending + } +} + +impl Drop for PendingActivityCallback { + fn drop(&mut self) { + self.0.store(true, Ordering::SeqCst); + } +} + +#[tokio::test] +async fn cooperative_activity_observation_uses_the_exact_worker_claim_without_renewal() { + let server = remote_activity_server(); + let reply = client(&server, "remote-valid") + .activity_task_status("activity/selected", "attempt-original", "actual-owner") + .await + .unwrap(); + assert_eq!(reply["heartbeat_recorded"], false); + assert_eq!(reply["can_continue"], true); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert_eq!( + requests[0].path, + "/remote-valid/api/worker/activity-tasks/activity%2Fselected/status" + ); + assert_eq!(requests[0].worker_protocol.as_deref(), Some("1.20")); + assert_eq!( + requests[0].authorization.as_deref(), + Some("Bearer worker-only") + ); + assert_eq!(requests[0].namespace.as_deref(), Some("caller-namespace")); + let body: Value = serde_json::from_str(&requests[0].body).unwrap(); + assert_eq!( + body, + json!({"activity_attempt_id":"attempt-original","lease_owner":"actual-owner"}) + ); +} + +#[tokio::test] +async fn cooperative_activity_observation_rejects_changed_receipts_and_preserves_refusals() { + for case in [ + "remote-wrong-task", + "remote-wrong-attempt", + "remote-wrong-owner", + "remote-malformed", + "remote-recorded-progress", + ] { + let server = remote_activity_server(); + assert!( + matches!( + client(&server, case) + .activity_task_status("activity", "attempt-original", "actual-owner") + .await, + Err(Error::InvalidCooperativeCancellation(_)) + ), + "{case}" + ); + } + let server = remote_activity_server(); + let result = client(&server, "remote-refused") + .activity_task_status("activity", "attempt-original", "actual-owner") + .await; + assert!( + matches!(result, Err(Error::ActivityTaskRejected(ref error)) if error.operation == "status" + && error.status == 409 && error.reason == "lease_owner_mismatch") + ); + for (task, attempt, owner) in [ + ("", "attempt", "owner"), + ("task", "", "owner"), + ("task", "attempt", ""), + ] { + assert!(client(&server, "remote-valid") + .activity_task_status(task, attempt, owner) + .await + .is_err()); + } + let control_only = Client::builder(format!("{}/remote-valid", server.base_url())) + .control_token(Some("control-only".to_string())) + .build() + .unwrap(); + assert!(matches!( + control_only + .activity_task_status("activity", "attempt", "owner") + .await, + Err(Error::MissingRoleCredentials { role: "worker", .. }) + )); + assert_eq!(server.requests.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn cooperative_activity_stop_receipt_preserves_worker_authentication_and_duplicate_identity() +{ + let server = remote_activity_server(); + let original = client(&server, "remote-ack-valid"); + let first = original + .acknowledge_activity_cancellation( + "activity/selected", + "attempt-original", + "actual-owner", + "local-request", + ) + .await + .unwrap(); + let duplicate = original + .acknowledge_activity_cancellation( + "activity/selected", + "attempt-original", + "actual-owner", + "local-request", + ) + .await + .unwrap(); + assert_eq!(first["history_event_id"], duplicate["history_event_id"]); + assert_eq!(first["duplicate"], false); + assert_eq!(duplicate["duplicate"], true); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 2); + for request in requests.iter() { + assert!(request + .path + .ends_with("/activity%2Fselected/acknowledge-cancellation")); + assert_eq!(request.worker_protocol.as_deref(), Some("1.20")); + assert_eq!(request.authorization.as_deref(), Some("Bearer worker-only")); + assert_eq!(request.namespace.as_deref(), Some("caller-namespace")); + let body: Value = serde_json::from_str(&request.body).unwrap(); + assert_eq!( + body, + json!({"activity_attempt_id":"attempt-original","lease_owner":"actual-owner","request_id":"local-request"}) + ); + } + assert_eq!(WORKER_PROTOCOL_VERSION, "1.19"); +} + +#[tokio::test] +async fn cooperative_activity_stop_receipt_refuses_changed_fences_unproved_receipts_and_missing_identity( +) { + for case in [ + "remote-ack-task", + "remote-ack-attempt", + "remote-ack-owner", + "remote-ack-request", + "remote-ack-unproved", + "remote-ack-declined", + "remote-ack-progress", + "remote-ack-duplicate", + ] { + let server = remote_activity_server(); + assert!( + matches!( + client(&server, case) + .acknowledge_activity_cancellation( + "activity/selected", + "attempt-original", + "actual-owner", + "local-request" + ) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + ), + "{case}" + ); + } + let server = remote_activity_server(); + assert!( + matches!(client(&server, "remote-ack-refused").acknowledge_activity_cancellation( + "activity/selected", "attempt-original", "actual-owner", "local-request").await, + Err(Error::ActivityTaskRejected(ref error)) if error.operation == "acknowledge-cancellation" && error.reason == "cancellation_request_mismatch") + ); + for (task, attempt, owner, request) in [ + ("", "attempt", "owner", "request"), + ("task", "", "owner", "request"), + ("task", "attempt", "", "request"), + ("task", "attempt", "owner", ""), + ] { + assert!(client(&server, "remote-ack-valid") + .acknowledge_activity_cancellation(task, attempt, owner, request) + .await + .is_err()); + } + assert_eq!(server.requests.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn cooperative_activity_stop_receipt_has_one_five_second_budget() { + let server = remote_activity_server(); + let started = Instant::now(); + assert!(matches!( + client(&server, "remote-ack-slow") + .acknowledge_activity_cancellation( + "activity/selected", + "attempt-original", + "actual-owner", + "local-request" + ) + .await, + Err(Error::Timeout) + )); + assert!(started.elapsed() < Duration::from_millis(5200)); + assert_eq!(server.requests.lock().unwrap().len(), 1); +} + +struct StoppedCallbackMarker(std::path::PathBuf); + +impl Drop for StoppedCallbackMarker { + fn drop(&mut self) { + std::fs::write(&self.0, "dropped").unwrap(); + } +} + +#[tokio::test] +async fn cooperative_activity_callback_is_dropped_before_stop_receipt_and_cloned_context_stays_fenced( +) { + let case = unique_request_id("remote-stop-joined"); + let marker = std::env::temp_dir().join(&case); + let server = remote_activity_server(); + let mut worker = coordinator_worker(&server, &case); + let saved = Arc::new(Mutex::new(None::)); + let capture = Arc::clone(&saved); + let callback_marker = marker.clone(); + worker.register_activity("work", move |ctx, _| { + *capture.lock().unwrap() = Some(ctx); + let stopped = StoppedCallbackMarker(callback_marker.clone()); + async move { + let _stopped = stopped; + std::future::pending::<()>().await; + Ok(Value::Null) + } + }); + let result = worker.poll_activity_once().await; + let dropped = std::fs::read_to_string(&marker); + let _ = std::fs::remove_file(&marker); + assert_eq!(result.unwrap(), ManagedPollOutcome::Handled); + assert_eq!(dropped.unwrap(), "dropped"); + let before = server.requests.lock().unwrap().len(); + let context = saved.lock().unwrap().as_ref().unwrap().clone(); + assert!(matches!( + context.heartbeat(json!({"late":true})).await, + Err(Error::ActivityExecutionAbandoned(_)) + )); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), before); + assert_eq!( + requests.last().unwrap().path.rsplit('/').next(), + Some("acknowledge-cancellation") + ); + assert!(!requests + .iter() + .any(|request| request.path.ends_with("/heartbeat") + || request.path.ends_with("/complete") + || request.path.ends_with("/fail"))); +} + +#[tokio::test] +async fn cooperative_activity_observation_has_one_five_second_budget() { + let server = remote_activity_server(); + let start = Instant::now(); + assert!(matches!( + client(&server, "remote-status-slow") + .activity_task_status("activity", "attempt-original", "actual-owner") + .await, + Err(Error::Timeout) + )); + assert!(start.elapsed() < Duration::from_millis(5200)); + assert_eq!(server.requests.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn cooperative_activity_refuses_execution_without_current_claim_deadlines_and_session() { + for case in [ + "remote-refused", + "remote-offline", + "remote-cancelled", + "remote-wrong-task", + "remote-wrong-attempt", + "remote-wrong-owner", + "remote-malformed", + "remote-recorded-progress", + "remote-expired", + "remote-closed-task", + "remote-closed-attempt", + "remote-closed-activity", + "remote-no-timezone", + "remote-deadline", + "remote-bad-deadlines", + "remote-session-expired", + "remote-session-replaced", + ] { + let server = remote_activity_server(); + let mut worker = coordinator_worker(&server, case); + let called = Arc::new(AtomicBool::new(false)); + let invoked = Arc::clone(&called); + worker.register_activity("work", move |_, _| { + invoked.store(true, Ordering::SeqCst); + async { Ok(Value::Null) } + }); + assert_eq!( + worker.poll_activity_once().await.unwrap(), + ManagedPollOutcome::Handled, + "{case}" + ); + assert!(!called.load(Ordering::SeqCst), "{case}"); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 2, "{case}"); + assert!(requests.last().unwrap().path.ends_with("/status"), "{case}"); + } + for case in [ + "remote-missing-owner", + "remote-missing-attempt", + "remote-invented-owner", + "remote-conflicting-attempt", + ] { + let server = remote_activity_server(); + let worker = coordinator_worker(&server, case); + assert!( + matches!( + worker.poll_activity_once().await, + Err(Error::InvalidCooperativeCancellation(_)) + ), + "{case}" + ); + assert_eq!(server.requests.lock().unwrap().len(), 1, "{case}"); + } +} + +#[tokio::test] +async fn cooperative_activity_fences_success_failure_heartbeat_and_late_context() { + for fail in [false, true] { + let server = remote_activity_server(); + let mut worker = coordinator_worker(&server, "remote-valid"); + let saved = Arc::new(Mutex::new(None::)); + let capture = Arc::clone(&saved); + worker.register_activity("work", move |ctx, _| { + *capture.lock().unwrap() = Some(ctx.clone()); + async move { + assert!( + ctx.heartbeat(json!({"completed":1})) + .await? + .heartbeat_recorded + ); + if fail { + return Err(Error::WorkerLoop("application failure".into())); + } + Ok(json!("done")) + } + }); + assert_eq!( + worker.poll_activity_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + let context = saved.lock().unwrap().as_ref().unwrap().clone(); + let before = server.requests.lock().unwrap().len(); + assert!(matches!( + context.heartbeat(json!({"late":true})).await, + Err(Error::ActivityExecutionAbandoned(_)) + )); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), before); + let paths = requests + .iter() + .map(|r| r.path.rsplit('/').next().unwrap()) + .collect::>(); + let mut expected = vec!["poll", "status", "status", "heartbeat", "status", "status"]; + if !fail { + expected.push("info"); + } + expected.push(if fail { "fail" } else { "complete" }); + assert_eq!(paths, expected); + let body: Value = serde_json::from_str(&requests.last().unwrap().body).unwrap(); + assert_eq!(body["activity_attempt_id"], "attempt-original"); + assert_eq!(body["lease_owner"], "actual-owner"); + if fail { + assert_eq!( + body["failure"]["message"], + "worker loop error: application failure" + ); + } + } +} + +#[tokio::test] +async fn cooperative_activity_discards_results_and_failure_after_attempt_replacement() { + for (case, fail) in [ + ("remote-replaced", false), + ("remote-replaced", true), + ("remote-late-result", false), + ] { + let server = remote_activity_server(); + let mut worker = coordinator_worker(&server, case); + worker.register_activity("work", move |_, _| async move { + if fail { + Err(Error::WorkerLoop("application failure".into())) + } else { + Ok(json!("late")) + } + }); + assert_eq!( + worker.poll_activity_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 3); + assert!(requests.last().unwrap().path.ends_with("/status")); + assert!(!requests.iter().any(|r| r.path.ends_with("/complete") + || r.path.ends_with("/fail") + || r.path.ends_with("/cluster/info"))); + } +} + +#[tokio::test] +async fn cooperative_activity_heartbeat_abandons_changed_context_or_lost_authority() { + for case in [ + "remote-context-changed", + "remote-heartbeat-wrong-attempt", + "remote-heartbeat-cancelled", + "remote-heartbeat-post-loss", + ] { + let server = remote_activity_server(); + let mut worker = coordinator_worker(&server, case); + let changed = case == "remote-context-changed"; + worker.register_activity("work", move |mut ctx, _| async move { + if changed { + ctx.activity_attempt_id = "replacement".into(); + } + assert!(matches!( + ctx.heartbeat(json!({"progress":1})).await, + Err(Error::ActivityExecutionAbandoned(_)) + )); + // Catching abandonment cannot restore publication authority. + Ok(json!("late")) + }); + assert_eq!( + worker.poll_activity_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + let requests = server.requests.lock().unwrap(); + assert!( + !requests + .iter() + .any(|r| r.path.ends_with("/complete") || r.path.ends_with("/fail")), + "{case}" + ); + assert_eq!( + requests + .iter() + .filter(|r| r.path.ends_with("/activity/heartbeat")) + .count(), + if changed { 0 } else { 1 } + ); + } +} + +#[tokio::test] +async fn cooperative_activity_observer_drops_a_pending_callback_and_fences_its_context() { + let server = remote_activity_server(); + let mut worker = coordinator_worker(&server, "remote-replaced"); + let saved = Arc::new(Mutex::new(None::)); + let capture = Arc::clone(&saved); + let dropped = Arc::new(AtomicBool::new(false)); + let finished = Arc::clone(&dropped); + worker.register_activity("work", move |ctx, _| { + *capture.lock().unwrap() = Some(ctx); + PendingActivityCallback(Arc::clone(&finished)) + }); + assert_eq!( + tokio::time::timeout(Duration::from_secs(3), worker.poll_activity_once()) + .await + .unwrap() + .unwrap(), + ManagedPollOutcome::Handled + ); + let context = saved.lock().unwrap().as_ref().unwrap().clone(); + assert!(dropped.load(Ordering::SeqCst)); + let before = server.requests.lock().unwrap().len(); + assert!(matches!( + context.heartbeat(Value::Null).await, + Err(Error::ActivityExecutionAbandoned(_)) + )); + assert_eq!(server.requests.lock().unwrap().len(), before); + assert_eq!(before, 3); +} + +#[tokio::test] +async fn cooperative_activity_worker_shutdown_abandons_a_pending_callback_before_deregistration() { + let server = remote_activity_server(); + let mut worker = coordinator_worker(&server, "remote-valid"); + let saved = Arc::new(Mutex::new(None::)); + let capture = Arc::clone(&saved); + let started = Arc::new(tokio::sync::Notify::new()); + let invoked = Arc::clone(&started); + let dropped = Arc::new(AtomicBool::new(false)); + let finished = Arc::clone(&dropped); + worker.register_activity("work", move |ctx, _| { + *capture.lock().unwrap() = Some(ctx); + invoked.notify_one(); + PendingActivityCallback(Arc::clone(&finished)) + }); + tokio::time::timeout(Duration::from_secs(3), worker.run_until(started.notified())) + .await + .unwrap() + .unwrap(); + let context = saved.lock().unwrap().as_ref().unwrap().clone(); + assert!(dropped.load(Ordering::SeqCst)); + let before = server.requests.lock().unwrap().len(); + assert!(matches!( + context.heartbeat(Value::Null).await, + Err(Error::ActivityExecutionAbandoned(_)) + )); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), before); + let deregistration = requests.last().unwrap(); + assert!(deregistration + .path + .ends_with("/worker/registrations/actual-owner")); + assert_eq!(deregistration.method, "DELETE"); + assert!(!requests + .iter() + .any(|r| r.path.ends_with("/complete") || r.path.ends_with("/fail"))); +} + +#[tokio::test] +async fn cooperative_activity_dropped_poll_future_abandons_cloned_context() { + let server = remote_activity_server(); + let mut worker = coordinator_worker(&server, "remote-valid"); + let saved = Arc::new(Mutex::new(None::)); + let capture = Arc::clone(&saved); + let started = Arc::new(tokio::sync::Notify::new()); + let invoked = Arc::clone(&started); + let dropped = Arc::new(AtomicBool::new(false)); + let finished = Arc::clone(&dropped); + worker.register_activity("work", move |ctx, _| { + *capture.lock().unwrap() = Some(ctx); + invoked.notify_one(); + PendingActivityCallback(Arc::clone(&finished)) + }); + let poll = tokio::spawn(async move { worker.poll_activity_once().await }); + tokio::time::timeout(Duration::from_secs(3), started.notified()) + .await + .unwrap(); + poll.abort(); + assert!(poll.await.unwrap_err().is_cancelled()); + assert!(dropped.load(Ordering::SeqCst)); + let context = saved.lock().unwrap().as_ref().unwrap().clone(); + let before = server.requests.lock().unwrap().len(); + assert!(matches!( + context.heartbeat(Value::Null).await, + Err(Error::ActivityExecutionAbandoned(_)) + )); + assert_eq!(server.requests.lock().unwrap().len(), before); +} + +#[tokio::test] +async fn cooperative_activity_concurrency_reuses_slots_and_stops_before_deregistering() { + struct ActiveCallback(Arc); + impl Drop for ActiveCallback { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } + } + let server = remote_activity_server(); + let mut worker = + coordinator_worker(&server, "remote-concurrency").max_concurrent_activity_tasks(2); + let active = Arc::new(AtomicUsize::new(0)); + let release = Arc::new(tokio::sync::Semaphore::new(0)); + let (entered_tx, mut entered_rx) = tokio::sync::mpsc::unbounded_channel(); + let callbacks = Arc::clone(&active); + let permits = Arc::clone(&release); + worker.register_activity("work", move |ctx, _| { + let callbacks = Arc::clone(&callbacks); + let permits = Arc::clone(&permits); + let entered_tx = entered_tx.clone(); + async move { + callbacks.fetch_add(1, Ordering::SeqCst); + let _active = ActiveCallback(callbacks); + entered_tx.send(ctx.task_id).unwrap(); + permits.acquire().await.unwrap().forget(); + Ok(Value::Null) + } + }); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let running = tokio::spawn(async move { + worker + .run_until(async { + let _ = shutdown_rx.await; + }) + .await + }); + let mut entered = Vec::new(); + for _ in 0..2 { + entered.push( + tokio::time::timeout(Duration::from_secs(3), entered_rx.recv()) + .await + .expect("both configured slots must start callbacks") + .unwrap(), + ); + } + assert_eq!(active.load(Ordering::SeqCst), 2); + assert!( + tokio::time::timeout(Duration::from_millis(100), entered_rx.recv()) + .await + .is_err(), + "a third callback cannot exceed the configured capacity" + ); + release.add_permits(1); + entered.push( + tokio::time::timeout(Duration::from_secs(3), entered_rx.recv()) + .await + .expect("a settled callback must release its slot") + .unwrap(), + ); + entered.sort(); + assert_eq!(entered, ["activity-1", "activity-2", "activity-3"]); + assert_eq!(active.load(Ordering::SeqCst), 2); + shutdown_tx.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(3), running) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + active.load(Ordering::SeqCst), + 0, + "shutdown must drop all managed callbacks" + ); + let requests = server.requests.lock().unwrap(); + assert_eq!( + requests + .iter() + .filter(|request| request.path.ends_with("/registrations/actual-owner")) + .count(), + 1 + ); + assert!( + requests + .last() + .unwrap() + .path + .ends_with("/registrations/actual-owner"), + "all lanes must stop before deregistration" + ); +} + +fn coordinator_responses(path: &str, body: &str, number: usize) -> Option<(&'static str, String)> { + let case = path.split('/').nth(1).unwrap_or_default(); + if path.ends_with("/cluster/info") { + return responses(path, body, number); + } + if path.ends_with("/poll") { + if case == "coordinator-retry" && number == 1 { + return Some(("invalid-status", String::new())); + } + let (status, response) = transport_responses(path, body, number)?; + let mut response: Value = serde_json::from_str(&response).unwrap(); + if case == "coordinator-prefix" && number >= 2 { + response["task"]["task_id"] = json!("successor"); + response["task"]["workflow_task_attempt"] = json!(8); + } + if case == "coordinator-saga" && number >= 2 { + response["task"]["task_id"] = json!("cleanup"); + response["task"]["workflow_task_attempt"] = json!(9); + } + if case == "coordinator-child-wait" && number == 2 { + response["task"]["task_id"] = json!("other-task"); + response["task"]["workflow_type"] = json!("other"); + response["task"]["run_id"] = json!("other-run"); + response["task"]["cancel_requested"] = json!(false); + response["task"] + .as_object_mut() + .unwrap() + .remove("cancellation_request"); + } + if case == "coordinator-child-wait" && number >= 3 { + response["task"]["task_id"] = json!("parent-resume-task"); + response["task"]["workflow_task_attempt"] = json!(8); + } + return Some((status, response.to_string())); + } + if path.ends_with("/history") { + if case == "coordinator-owner-lost" && number >= 2 { + return Some(( + "409 Conflict", + json!({"reason":"lease_owner_mismatch"}).to_string(), + )); + } + let prefix = case == "coordinator-prefix"; + let successor = prefix && path.ends_with("/successor/history"); + let child_wait = case == "coordinator-child-wait"; + let child_resume = child_wait && path.ends_with("/parent-resume-task/history"); + let mut events = if child_wait { + vec![pending_scalar_event("child"), canonical_request()] + } else if prefix { + vec![canonical_request()] + } else { + vec![scheduled_timer(1), canonical_request()] + }; + if successor { + events.push(event( + "SideEffectRecorded", + json!({"sequence":1,"result":fixture_envelope(json!(7))}), + )); + } + let cleanup = case == "coordinator-saga" && path.ends_with("/cleanup/history"); + let delivered = if child_wait { + child_resume && number >= 2 + } else if prefix { + successor && number >= 2 + } else { + number >= 2 || case == "coordinator-cold" || cleanup + }; + if delivered && !matches!(case, "coordinator-unproved" | "coordinator-shield") { + events.push(canonical_delivery( + if prefix { 2 } else { 1 }, + if child_wait { + "child" + } else if case == "coordinator-mismatch" { + "activity" + } else { + "timer" + }, + )); + } + if cleanup { + events.extend(completed_activity(2, "undo", Value::Null)); + } + if case == "coordinator-missing-request" { + events.retain(|event| event.event_type != "CooperativeCancellationRequested"); + } + let events = events + .into_iter() + .map(|event| { + let mut value = event.raw; + value.insert("event_type".into(), json!(event.event_type)); + value.insert("payload".into(), event.payload); + Value::Object(value.into_iter().collect()) + }) + .collect::>(); + return Some(( + "200 OK", + json!({ + "task_id":if successor { "successor" } else if child_resume { "parent-resume-task" } else if cleanup { "cleanup" } else { "task/selected" }, + "workflow_task_attempt":if successor || child_resume { 8 } else if cleanup { 9 } else { 7 }, + "total_history_events":events.len(),"history_events":events, + "next_history_page_token":null, + }) + .to_string(), + )); + } + if path.ends_with("/deliver-cancellation") { + if case == "coordinator-child-wait" + && !path.ends_with("/parent-resume-task/deliver-cancellation") + { + return Some(( + "200 OK", + pending_child_delivery("task/selected").to_string(), + )); + } + if case == "coordinator-lost-ack" { + return Some(( + "409 Conflict", + json!({"reason":"lease_owner_mismatch"}).to_string(), + )); + } + let (status, response) = transport_responses(path, body, number)?; + let mut response: Value = serde_json::from_str(&response).unwrap(); + if case == "coordinator-prefix" { + response["task_id"] = json!("successor"); + } + if case == "coordinator-child-wait" { + response["task_id"] = json!("parent-resume-task"); + } + return Some((status, response.to_string())); + } + if path.ends_with("/complete") || path.ends_with("/fail") { + return Some(("200 OK", json!({"recorded":true}).to_string())); + } + None +} + +fn coordinator_server() -> MockWorkerServer { + MockWorkerServer::start_with_behavior(MockWorkerBehavior { + request_override: Some(coordinator_responses), + ..Default::default() + }) +} + +fn coordinator_worker(server: &MockWorkerServer, case: &str) -> Worker { + let mut worker = Worker::new(client(server, case), "queue") + .worker_id("actual-owner") + .cooperative_cancellation(true); + worker.poll_timeout = Duration::ZERO; + worker.retry_policy = WorkerRetryPolicy { + max_retries: 1, + initial_backoff: Duration::from_millis(1), + max_backoff: Duration::from_millis(1), + }; + worker +} + +fn register_coordinator_timer( + worker: &mut Worker, + observed: Arc>>, +) { + worker.register_workflow("cancel", move |ctx, _| { + let observed = observed.clone(); + async move { + match ctx.sleep(Duration::from_secs(5)).await { + Err(Error::CooperativeCancellationRequested(request)) => { + observed.lock().unwrap().push(request) + } + result => result?, + } + Ok(Value::Null) + } + }); +} + +#[tokio::test] +async fn cooperative_coordinator_commits_delivery_proves_history_and_replays_before_completion() { + for case in [ + "coordinator-valid", + "coordinator-lost-ack", + "coordinator-cold", + ] { + let server = coordinator_server(); + let mut worker = coordinator_worker(&server, case); + let observed = Arc::new(Mutex::new(Vec::new())); + register_coordinator_timer(&mut worker, observed.clone()); + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + let observed = observed.lock().unwrap(); + assert_eq!(observed.len(), 1); + assert_eq!(observed[0].request, original_observation()); + assert_eq!(observed[0].delivery, transport_delivery()); + let requests = server.requests.lock().unwrap(); + let delivery_count = requests + .iter() + .filter(|request| request.path.ends_with("/deliver-cancellation")) + .count(); + assert_eq!( + delivery_count, + if case == "coordinator-cold" { 0 } else { 1 } + ); + let completion = requests.last().unwrap(); + assert!(completion.path.ends_with("/complete")); + assert!(requests[requests.len() - 2].path.ends_with("/history")); + let body: Value = serde_json::from_str(&completion.body).unwrap(); + assert_eq!(body["lease_owner"], "actual-owner"); + assert_eq!(body["workflow_task_attempt"], 7); + assert_eq!(body["commands"][0]["type"], "complete_workflow"); + assert!(!requests + .iter() + .any(|request| request.path.ends_with("/fail"))); + } +} + +#[tokio::test] +async fn cooperative_coordinator_returns_to_other_work_then_replays_the_parent_on_a_new_claim() { + let server = coordinator_server(); + let mut worker = coordinator_worker(&server, "coordinator-child-wait"); + let observed = Arc::new(Mutex::new(Vec::new())); + let cleanup = Arc::clone(&observed); + worker.register_workflow("cancel", move |ctx, _| { + let cleanup = Arc::clone(&cleanup); + async move { + match ctx + .start_child_workflow("child", ChildWorkflowOptions::new("queue"), json!([])) + .await + { + Err(Error::CooperativeCancellationRequested(request)) => { + cleanup.lock().unwrap().push(request) + } + result => { + result?; + } + } + Ok(Value::Null) + } + }); + worker.register_workflow("other", |_, _| async { Ok(Value::Null) }); + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + assert!(observed.lock().unwrap().is_empty()); + assert!(!server + .requests + .lock() + .unwrap() + .iter() + .any(|request| request.path.ends_with("/complete") || request.path.ends_with("/fail"))); + + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + assert!(observed.lock().unwrap().is_empty()); + assert!(server + .requests + .lock() + .unwrap() + .iter() + .any(|request| request.path.ends_with("/other-task/complete"))); + + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + let observed = observed.lock().unwrap(); + assert_eq!(observed.len(), 1); + assert_eq!(observed[0].request, original_observation()); + assert_eq!(observed[0].delivery.call_kind, CancellationCallKind::Child); + assert_eq!(observed[0].delivery.sequence, 1); + let requests = server.requests.lock().unwrap(); + let deliveries = requests + .iter() + .filter(|request| request.path.ends_with("/deliver-cancellation")) + .map(|request| serde_json::from_str::(&request.body).unwrap()) + .collect::>(); + assert_eq!(deliveries.len(), 2); + assert_eq!(deliveries[0]["workflow_task_attempt"], 7); + assert_eq!(deliveries[1]["workflow_task_attempt"], 8); + assert!(deliveries + .iter() + .all(|body| body["request_id"] == "original-request")); + assert!(!requests + .iter() + .any(|request| request.path.ends_with("/fail"))); +} + +#[tokio::test] +async fn cooperative_coordinator_refuses_unproved_changed_or_lost_claim_history_without_publication( +) { + for case in [ + "coordinator-unproved", + "coordinator-mismatch", + "coordinator-owner-lost", + "coordinator-missing-request", + ] { + let server = coordinator_server(); + let mut worker = coordinator_worker(&server, case); + let observed = Arc::new(Mutex::new(Vec::new())); + register_coordinator_timer(&mut worker, observed.clone()); + let result = worker.poll_workflow_once().await; + if case == "coordinator-owner-lost" { + assert!(matches!(result, Err(Error::Http { status, .. }) if status.as_u16() == 409)); + } else { + assert!( + matches!(result, Err(Error::InvalidCooperativeCancellation(_))), + "{case}: {result:?}" + ); + } + assert!(observed.lock().unwrap().is_empty()); + let requests = server.requests.lock().unwrap(); + assert!(!requests + .iter() + .any(|request| request.path.ends_with("/complete") || request.path.ends_with("/fail"))); + } +} + +#[tokio::test] +async fn cooperative_coordinator_commits_earlier_prefix_and_delivers_only_on_its_successor_claim() { + let server = coordinator_server(); + let mut worker = coordinator_worker(&server, "coordinator-prefix"); + let side_effects = Arc::new(Mutex::new(0)); + let executions = side_effects.clone(); + let observed = Arc::new(Mutex::new(Vec::new())); + let errors = observed.clone(); + worker.register_workflow("cancel", move |ctx, _| { + let executions = executions.clone(); + let errors = errors.clone(); + async move { + let value: Value = ctx.side_effect(|| { + *executions.lock().unwrap() += 1; + json!(7) + })?; + assert_eq!(value, 7); + match ctx.sleep(Duration::from_secs(5)).await { + Err(Error::CooperativeCancellationRequested(request)) => { + errors.lock().unwrap().push(request) + } + result => result?, + } + Ok(Value::Null) + } + }); + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + assert!(observed.lock().unwrap().is_empty()); + { + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 3); + let completion: Value = serde_json::from_str(&requests[2].body).unwrap(); + assert_eq!(completion["commands"].as_array().unwrap().len(), 1); + assert_eq!(completion["commands"][0]["type"], "record_side_effect"); + assert_eq!(completion["workflow_task_attempt"], 7); + assert!(!requests + .iter() + .any(|request| request.path.ends_with("/deliver-cancellation"))); + } + let result = worker.poll_workflow_once().await; + assert!( + matches!(result, Ok(ManagedPollOutcome::Handled)), + "{result:?}, paths: {:?}", + server + .requests + .lock() + .unwrap() + .iter() + .map(|request| &request.path) + .collect::>() + ); + assert_eq!(*side_effects.lock().unwrap(), 1); + let observed = observed.lock().unwrap(); + assert_eq!(observed.len(), 1); + assert_eq!(observed[0].delivery.sequence, 2); + assert_eq!(observed[0].request, original_observation()); + let requests = server.requests.lock().unwrap(); + let delivery = requests + .iter() + .find(|request| request.path.ends_with("/deliver-cancellation")) + .unwrap(); + assert!(delivery.path.ends_with("/successor/deliver-cancellation")); + let body: Value = serde_json::from_str(&delivery.body).unwrap(); + assert_eq!(body["workflow_task_attempt"], 8); + assert_eq!(body["sequence"], 2); + let body: Value = serde_json::from_str(&requests.last().unwrap().body).unwrap(); + assert_eq!(body["commands"].as_array().unwrap().len(), 1); + assert_eq!(body["commands"][0]["type"], "complete_workflow"); +} + +#[tokio::test] +async fn cooperative_coordinator_leaves_shielded_cleanup_pending_without_delivery() { + let server = coordinator_server(); + let mut worker = coordinator_worker(&server, "coordinator-shield"); + worker.register_workflow("cancel", |ctx, _| async move { + let _shield = ctx.cancellation_shield()?; + ctx.sleep(Duration::from_secs(5)).await?; + Ok(Value::Null) + }); + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 3); + assert!(requests.last().unwrap().path.ends_with("/fail")); + let body: Value = serde_json::from_str(&requests.last().unwrap().body).unwrap(); + assert_eq!( + body["failure"]["type"], + WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE + ); + assert!(!requests + .iter() + .any(|request| request.path.ends_with("/deliver-cancellation"))); +} + +#[tokio::test] +async fn cooperative_coordinator_unhandled_canonical_request_publishes_typed_workflow_cancellation() +{ + let server = coordinator_server(); + let mut worker = coordinator_worker(&server, "coordinator-valid"); + worker.register_workflow("cancel", |ctx, _| async move { + ctx.sleep(Duration::from_secs(5)).await?; + Ok(Value::Null) + }); + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + let requests = server.requests.lock().unwrap(); + let completion = requests.last().unwrap(); + assert!(completion.path.ends_with("/complete")); + let body: Value = serde_json::from_str(&completion.body).unwrap(); + assert_eq!(body["commands"].as_array().unwrap().len(), 1); + let command = &body["commands"][0]; + assert_eq!(command["type"], "fail_workflow"); + assert_eq!(command["exception_type"], "WorkflowCancellationRequested"); + assert_eq!( + command["exception_class"], + "durable_workflow::CooperativeCancellationRequested" + ); + assert_eq!(command["non_retryable"], true); + assert_eq!(command["exception"]["properties"]["reason"], "cancelled"); + assert_eq!( + command["exception"]["properties"]["request_id"], + "original-request" + ); + assert_eq!( + command["exception"]["properties"]["cleanup_deadline_at"], + "2026-10-01T08:10:00Z" + ); + assert!(!requests + .iter() + .any(|request| request.path.ends_with("/fail"))); +} + +#[tokio::test] +async fn cooperative_coordinator_replays_saga_cleanup_on_a_cold_successor_before_terminal_cancellation( +) { + let server = coordinator_server(); + let mut worker = coordinator_worker(&server, "coordinator-saga"); + worker.register_workflow("cancel", |ctx, _| async move { + let mut saga = ctx.saga(); + saga.add_compensation("undo", json!([]))?; + let result = ctx.sleep(Duration::from_secs(5)).await; + saga.finish(result).await?; + Ok(Value::Null) + }); + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + { + let requests = server.requests.lock().unwrap(); + let body: Value = serde_json::from_str(&requests.last().unwrap().body).unwrap(); + assert_eq!(body["commands"].as_array().unwrap().len(), 1); + assert_eq!(body["commands"][0]["type"], "schedule_activity"); + assert_eq!(body["commands"][0]["activity_type"], "undo"); + assert_eq!(body["workflow_task_attempt"], 7); + } + // A fresh Worker represents process loss after cleanup was scheduled. The + // Server history proves its activity completion before terminal publication. + let mut replacement = coordinator_worker(&server, "coordinator-saga"); + replacement.register_workflow("cancel", |ctx, _| async move { + let mut saga = ctx.saga(); + saga.add_compensation("undo", json!([]))?; + let result = ctx.sleep(Duration::from_secs(5)).await; + saga.finish(result).await?; + Ok(Value::Null) + }); + assert_eq!( + replacement.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + let requests = server.requests.lock().unwrap(); + assert_eq!( + requests + .iter() + .filter(|request| request.path.ends_with("/deliver-cancellation")) + .count(), + 1 + ); + let completion = requests.last().unwrap(); + assert!(completion.path.ends_with("/cleanup/complete")); + let body: Value = serde_json::from_str(&completion.body).unwrap(); + assert_eq!(body["workflow_task_attempt"], 9); + assert_eq!(body["commands"].as_array().unwrap().len(), 1); + let command = &body["commands"][0]; + assert_eq!(command["type"], "fail_workflow"); + assert_eq!(command["exception_type"], "WorkflowCancellationRequested"); + assert_eq!( + command["exception_class"], + "durable_workflow::CooperativeCancellationRequested" + ); + assert_eq!(command["non_retryable"], true); + assert_eq!(command["exception"]["properties"]["reason"], "cancelled"); + assert_eq!( + command["exception"]["properties"]["request_id"], + "original-request" + ); + assert_eq!( + command["exception"]["properties"]["cleanup_deadline_at"], + "2026-10-01T08:10:00Z" + ); +} + +#[tokio::test] +async fn cooperative_coordinator_poll_retry_retains_the_same_claim_acquisition_identity() { + let server = coordinator_server(); + let mut worker = coordinator_worker(&server, "coordinator-retry"); + register_coordinator_timer(&mut worker, Arc::new(Mutex::new(Vec::new()))); + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + let requests = server.requests.lock().unwrap(); + let polls = requests + .iter() + .filter(|request| request.path.ends_with("/poll")) + .collect::>(); + assert_eq!(polls.len(), 2); + assert_eq!(polls[0].body, polls[1].body); + assert_eq!(polls[0].worker_protocol.as_deref(), Some("1.20")); +} + +#[tokio::test] +async fn cooperative_poll_retains_the_actual_claim_observation_and_fenced_history() { + let server = transport_server(); + let response = client(&server, "claim-history") + .poll_cooperative_workflow_task("actual-owner", "queue", Duration::ZERO) + .await + .unwrap(); + assert_eq!(response.outcome, WorkerPollOutcome::Task); + assert_eq!(response.protocol_version.as_deref(), Some("1.20")); + assert_eq!( + response.server_capabilities.unwrap()["workflow_memo_updates"]["supported"], + true + ); + let claim = response.task.unwrap(); + assert_eq!(claim.task().task_id, "task/selected"); + assert_eq!(claim.task().lease_owner.as_deref(), Some("actual-owner")); + assert_eq!(claim.task().workflow_task_attempt, 7); + assert_eq!(claim.cancellation_request(), Some(&original_observation())); + assert_eq!(claim.task().history_events.len(), 3); + assert_eq!(claim.task().next_history_page_token, None); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 3); + for request in requests.iter() { + assert_eq!(request.worker_protocol.as_deref(), Some("1.20")); + assert_eq!(request.authorization.as_deref(), Some("Bearer worker-only")); + assert_eq!(request.namespace.as_deref(), Some("caller-namespace")); + let body: Value = serde_json::from_str(&request.body).unwrap(); + if request.path.ends_with("/poll") { + assert_eq!(body["worker_id"], "actual-owner"); + assert_eq!(body["task_queue"], "queue"); + assert_eq!(body["history_page_size"], WORKFLOW_HISTORY_PAGE_SIZE); + assert!(!body["poll_request_id"].as_str().unwrap().is_empty()); + } else { + assert!(request.path.ends_with("/task%2Fselected/history")); + assert_eq!(body["lease_owner"], "actual-owner"); + assert_eq!(body["workflow_task_attempt"], 7); + } + } +} + +#[tokio::test] +async fn cooperative_poll_rejects_invented_claims_and_malformed_observations() { + for case in [ + "claim-missing-attempt", + "claim-zero-attempt", + "claim-owner", + "claim-missing-owner", + "claim-missing-run", + "claim-missing-id", + "claim-bad-observation", + "claim-bad-token", + ] { + let server = transport_server(); + let result = client(&server, case) + .poll_cooperative_workflow_task("actual-owner", "queue", Duration::ZERO) + .await; + assert!( + matches!(result, Err(Error::InvalidCooperativeCancellation(_))), + "{case}: {result:?}" + ); + assert_eq!(server.requests.lock().unwrap().len(), 1); + } +} + +#[tokio::test] +async fn cooperative_poll_transport_retry_preserves_the_same_acquisition_identity() { + let server = transport_server(); + let response = client(&server, "claim-retry") + .poll_cooperative_workflow_task("actual-owner", "queue", Duration::ZERO) + .await + .unwrap(); + assert_eq!(response.task.unwrap().task().workflow_task_attempt, 7); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].body, requests[1].body); + let body: Value = serde_json::from_str(&requests[0].body).unwrap(); + assert!(!body["poll_request_id"].as_str().unwrap().is_empty()); +} + +#[tokio::test] +async fn cooperative_claim_history_rejects_changed_claims_bad_progress_and_unbounded_pages() { + for case in [ + "cycle", + "page-bound", + "empty-progress", + "oversized-page", + "wrong-task", + "wrong-attempt", + "malformed-event", + "empty-token", + "missing-token", + ] { + let server = transport_server(); + let result = client(&server, case) + .poll_cooperative_workflow_task("actual-owner", "queue", Duration::ZERO) + .await; + assert!( + matches!(result, Err(Error::InvalidCooperativeCancellation(_))), + "{case}: {result:?}" + ); + assert!(server.requests.lock().unwrap().len() <= 129); + } +} + +#[tokio::test] +async fn cooperative_claim_heartbeat_preserves_the_first_observation_and_refuses_replacement() { + let server = transport_server(); + let mut claim = client(&server, "valid") + .poll_cooperative_workflow_task("actual-owner", "queue", Duration::ZERO) + .await + .unwrap() + .task + .unwrap(); + assert!(claim.task().history_events.is_empty()); + claim + .heartbeat(&client(&server, "heartbeat-equivalent-time")) + .await + .unwrap(); + assert_eq!(claim.cancellation_request(), Some(&original_observation())); + assert!(matches!( + claim.heartbeat(&client(&server, "heartbeat-owner")).await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + assert_eq!(claim.cancellation_request(), Some(&original_observation())); + assert_eq!(claim.task().lease_owner.as_deref(), Some("actual-owner")); + assert_eq!(claim.task().workflow_task_attempt, 7); + assert!(claim.task().history_events.is_empty()); +} + +#[tokio::test] +async fn cooperative_poll_preserves_idle_stop_and_no_observation_despite_task_flags() { + let server = transport_server(); + for case in ["claim-idle", "claim-stop"] { + let response = client(&server, case) + .poll_cooperative_workflow_task("actual-owner", "queue", Duration::ZERO) + .await + .unwrap(); + assert!(response.task.is_none()); + assert_eq!(response.outcome.should_stop(), case == "claim-stop"); + } + let mut claim = client(&server, "claim-no-observation") + .poll_cooperative_workflow_task("actual-owner", "queue", Duration::ZERO) + .await + .unwrap() + .task + .unwrap(); + assert!(claim.task().cancel_requested); + assert!(claim.cancellation_request().is_none()); + claim.heartbeat(&client(&server, "valid")).await.unwrap(); + assert_eq!(claim.cancellation_request(), Some(&original_observation())); +} + +#[tokio::test] +async fn cooperative_poll_preflights_inputs_and_shares_the_poll_and_history_budget() { + let server = transport_server(); + let client = client(&server, "claim-budget"); + for (owner, queue, timeout) in [ + (" ", "queue", Duration::ZERO), + ("actual-owner", " ", Duration::ZERO), + ("actual-owner", "queue", Duration::MAX), + ] { + assert!(matches!( + client + .poll_cooperative_workflow_task(owner, queue, timeout) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + } + assert!(server.requests.lock().unwrap().is_empty()); + let control_only = Client::builder(server.base_url()) + .control_token(Some("control-only".into())) + .build() + .unwrap(); + assert!(matches!( + control_only + .poll_cooperative_workflow_task("actual-owner", "queue", Duration::ZERO) + .await, + Err(Error::MissingRoleCredentials { role: "worker", .. }) + )); + assert!(server.requests.lock().unwrap().is_empty()); + let started = Instant::now(); + assert!(matches!( + client + .poll_cooperative_workflow_task("actual-owner", "queue", Duration::ZERO) + .await, + Err(Error::Timeout) + )); + assert!(started.elapsed() < Duration::from_secs(6)); + assert_eq!(server.requests.lock().unwrap().len(), 2); +} + +#[tokio::test] +async fn cooperative_heartbeat_renews_the_exact_worker_claim_and_retains_original_observation() { + let server = transport_server(); + let original = original_observation(); + let receipt = client(&server, "heartbeat-equivalent-time") + .heartbeat_workflow_task(&transport_task(), Some(&original)) + .await + .unwrap(); + assert_eq!(receipt.task_id, "task/selected"); + assert_eq!(receipt.workflow_task_attempt, 7); + assert_eq!(receipt.lease_owner, "actual-owner"); + assert_eq!(receipt.cancellation_request, Some(original)); + let requests = server.requests.lock().unwrap(); + let request = &requests[0]; + assert_eq!( + request.path, + "/heartbeat-equivalent-time/api/worker/workflow-tasks/task%2Fselected/heartbeat" + ); + assert_eq!(request.worker_protocol.as_deref(), Some("1.20")); + assert_eq!(request.authorization.as_deref(), Some("Bearer worker-only")); + assert_eq!(request.namespace.as_deref(), Some("caller-namespace")); + let body: Value = serde_json::from_str(&request.body).unwrap(); + assert_eq!( + body, + json!({"lease_owner":"actual-owner", "workflow_task_attempt":7}) + ); +} + +#[tokio::test] +async fn cooperative_heartbeat_rejects_changed_claims_malformed_renewal_and_changed_identity() { + for case in [ + "wrong-task", + "heartbeat-attempt", + "heartbeat-owner", + "heartbeat-not-renewed", + "heartbeat-string-renewed", + "heartbeat-closed", + "heartbeat-unknown-run", + "heartbeat-finished-task", + "heartbeat-expiry", + "heartbeat-expiry-no-zone", + "heartbeat-malformed-request", + "heartbeat-missing-token", + "heartbeat-changed-request", + "heartbeat-changed-request-time", + "heartbeat-changed-deadline", + "heartbeat-no-request", + "wrong-reason", + ] { + let server = transport_server(); + let result = client(&server, case) + .heartbeat_workflow_task(&transport_task(), Some(&original_observation())) + .await; + assert!( + matches!(result, Err(Error::InvalidCooperativeCancellation(_))), + "{case}: {result:?}" + ); + } +} + +#[tokio::test] +async fn cooperative_heartbeat_accepts_new_observation_and_does_not_invent_a_request() { + for case in ["valid", "heartbeat-no-request"] { + let server = transport_server(); + let receipt = client(&server, case) + .heartbeat_workflow_task(&transport_task(), None) + .await + .unwrap(); + assert_eq!( + receipt.cancellation_request, + if case == "valid" { + Some(original_observation()) + } else { + None + } + ); + assert_eq!(receipt.lease_expires_at, "2026-10-01T08:00:30Z"); + } +} + +#[tokio::test] +async fn cooperative_heartbeat_validates_the_claim_and_original_observation_before_network_io() { + let server = transport_server(); + let client = client(&server, "valid"); + let mut task = transport_task(); + task.lease_owner = None; + assert!(matches!( + client.heartbeat_workflow_task(&task, None).await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + let mut original = original_observation(); + original.history_refresh_page_token = None; + assert!(matches!( + client + .heartbeat_workflow_task(&transport_task(), Some(&original)) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + assert!(server.requests.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn cooperative_heartbeat_preserves_ownership_refusal_and_the_total_budget() { + let server = transport_server(); + assert!( + matches!(client(&server, "refused").heartbeat_workflow_task(&transport_task(), None).await, + Err(Error::Http { status, .. }) if status.as_u16() == 409) + ); + let started = Instant::now(); + assert!(matches!( + client(&server, "heartbeat-budget") + .heartbeat_workflow_task(&transport_task(), None) + .await, + Err(Error::Timeout) + )); + assert!(started.elapsed() < Duration::from_secs(6)); +} + +#[tokio::test] +async fn cooperative_transport_delivers_the_exact_claim_with_worker_credentials() { + let server = transport_server(); + let client = client(&server, "valid"); + let task = transport_task(); + let delivery = transport_delivery(); + let reply = client + .deliver_workflow_cancellation(&task, &delivery) + .await + .unwrap(); + let CancellationDeliveryReply::Delivered(receipt) = reply else { + panic!("delivery must be committed") + }; + assert_eq!(receipt.task_id, task.task_id); + assert_eq!(receipt.run_id, "run"); + assert_eq!(receipt.delivery, delivery); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 1); + let request = &requests[0]; + assert_eq!( + request.path, + "/valid/api/worker/workflow-tasks/task%2Fselected/deliver-cancellation" + ); + assert_eq!(request.worker_protocol.as_deref(), Some("1.20")); + assert_eq!(request.authorization.as_deref(), Some("Bearer worker-only")); + assert_eq!(request.namespace.as_deref(), Some("caller-namespace")); + let body: Value = serde_json::from_str(&request.body).unwrap(); + assert_eq!(body["lease_owner"], "actual-owner"); + assert_eq!(body["workflow_task_attempt"], 7); + assert_eq!(body["request_id"], "original-request"); +} + +#[tokio::test] +async fn cooperative_transport_accepts_only_explicit_cancellation_claim_release() { + for (case, kinds) in [ + ( + "child-wait", + vec![ + CancellationCallKind::Child, + CancellationCallKind::Parallel, + CancellationCallKind::SelectionHandle, + ], + ), + ( + "activity-wait", + vec![ + CancellationCallKind::Activity, + CancellationCallKind::LocalActivity, + CancellationCallKind::Parallel, + CancellationCallKind::SelectionHandle, + ], + ), + ] { + for kind in kinds { + let server = transport_server(); + let mut delivery = transport_delivery(); + delivery.call_kind = kind; + if kind == CancellationCallKind::SelectionHandle { + delivery.sequence = 2; + delivery.operation_sequence = Some(1); + } + assert_eq!( + client(&server, case) + .deliver_workflow_cancellation(&transport_task(), &delivery) + .await + .unwrap(), + CancellationDeliveryReply::ClaimReleased { + task_id: "task/selected".into(), + run_id: "run".into() + } + ); + } + } + for (case, kind) in [ + ("child-wait", CancellationCallKind::Timer), + ("activity-wait", CancellationCallKind::Timer), + ("child-wait", CancellationCallKind::Activity), + ("activity-wait", CancellationCallKind::Child), + ] { + let server = transport_server(); + let mut delivery = transport_delivery(); + delivery.call_kind = kind; + assert!(matches!( + client(&server, case) + .deliver_workflow_cancellation(&transport_task(), &delivery) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + } + for (prefix, kind) in [ + ("child-wait", CancellationCallKind::Child), + ("activity-wait", CancellationCallKind::Activity), + ] { + for suffix in [ + "wrong-task", + "wrong-run", + "not-released", + "string-released", + "missing-release", + "wrong-reason", + "request", + "sequence", + "kind", + "span", + "operation", + "operation-span", + "string-delivered", + ] { + let case = format!("{prefix}-{suffix}"); + let server = transport_server(); + let mut delivery = transport_delivery(); + delivery.call_kind = kind; + let result = client(&server, &case) + .deliver_workflow_cancellation(&transport_task(), &delivery) + .await; + assert!( + matches!(result, Err(Error::InvalidCooperativeCancellation(_))), + "{case}: {result:?}" + ); + } + } +} + +#[tokio::test] +async fn cooperative_transport_rejects_malformed_or_changed_delivery_acknowledgments() { + for case in [ + "wrong-task", + "wrong-run", + "wrong-request", + "wrong-sequence", + "wrong-kind", + "wrong-span", + "false-delivered", + "string-delivered", + "wrong-reason", + "missing-span", + "missing-operation", + "missing-operation-span", + ] { + let server = transport_server(); + let result = client(&server, case) + .deliver_workflow_cancellation(&transport_task(), &transport_delivery()) + .await; + assert!( + matches!(result, Err(Error::InvalidCooperativeCancellation(_))), + "{case}: {result:?}" + ); + } +} + +#[tokio::test] +async fn cooperative_transport_refreshes_with_the_opaque_token_and_preserves_the_snapshot() { + let server = transport_server(); + let client = client(&server, "valid"); + let task = transport_task(); + let observation = CancellationRequest::from_observation(&request_observation()).unwrap(); + let original = observation.clone(); + let fresh = client + .refresh_workflow_cancellation_history(&task, &observation) + .await + .unwrap(); + assert_eq!(fresh.len(), 3); + assert_eq!(task.history_events.len(), 1); + assert_eq!(observation, original); + let canonical = CancellationHistory::from_events(&fresh, "run", Some(&observation)).unwrap(); + assert_eq!(canonical.delivery, Some(transport_delivery())); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 2); + for (index, request) in requests.iter().enumerate() { + assert_eq!(request.worker_protocol.as_deref(), Some("1.20")); + let body: Value = serde_json::from_str(&request.body).unwrap(); + assert_eq!(body["lease_owner"], "actual-owner"); + assert_eq!(body["workflow_task_attempt"], 7); + assert_eq!(body["history_page_size"], WORKFLOW_HISTORY_PAGE_SIZE); + assert_eq!( + body["next_history_page_token"], + if index == 0 { + "opaque-server-token" + } else { + "page-next" + } + ); + } +} + +#[tokio::test] +async fn cooperative_transport_rejects_changed_claims_bad_pages_and_token_cycles() { + for case in [ + "wrong-task", + "wrong-attempt", + "cycle", + "empty-progress", + "oversized-page", + "missing-request", + "changed-canonical", + "malformed-event", + "empty-token", + "missing-token", + ] { + let server = transport_server(); + let observation = CancellationRequest::from_observation(&request_observation()).unwrap(); + let result = client(&server, case) + .refresh_workflow_cancellation_history(&transport_task(), &observation) + .await; + assert!( + matches!( + result, + Err(Error::InvalidCooperativeCancellation(_) | Error::NonDeterministicReplay(_)) + ), + "{case}: {result:?}" + ); + assert!(server.requests.lock().unwrap().len() <= 2); + } +} + +#[tokio::test] +async fn cooperative_transport_bounds_unique_history_pages() { + let server = transport_server(); + let observation = CancellationRequest::from_observation(&request_observation()).unwrap(); + let result = client(&server, "page-bound") + .refresh_workflow_cancellation_history(&transport_task(), &observation) + .await; + assert!(matches!( + result, + Err(Error::InvalidCooperativeCancellation(_)) + )); + assert_eq!(server.requests.lock().unwrap().len(), 128); +} + +#[tokio::test] +async fn cooperative_transport_can_prove_committed_history_after_a_lost_acknowledgment() { + let server = transport_server(); + let client = client(&server, "lost-ack"); + let task = transport_task(); + let intent = transport_delivery(); + assert!(matches!( + client.deliver_workflow_cancellation(&task, &intent).await, + Err(Error::Http { status, .. }) if status.as_u16() == 409 + )); + let observation = CancellationRequest::from_observation(&request_observation()).unwrap(); + let fresh = client + .refresh_workflow_cancellation_history(&task, &observation) + .await + .unwrap(); + assert_eq!( + CancellationHistory::from_events(&fresh, "run", Some(&observation)) + .unwrap() + .delivery, + Some(intent) + ); +} + +#[tokio::test] +async fn cooperative_transport_validates_inputs_and_never_substitutes_control_credentials() { + let server = transport_server(); + let client = client(&server, "valid"); + for changed in 0..6 { + let mut task = transport_task(); + match changed { + 0 => task.task_id.clear(), + 1 => task.workflow_task_attempt = 0, + 2 => task.workflow_task_attempt = u64::MAX, + 3 => task.run_id = None, + 4 => task.lease_owner = None, + 5 => task.lease_owner = Some(" ".into()), + _ => unreachable!(), + } + assert!(matches!( + client + .deliver_workflow_cancellation(&task, &transport_delivery()) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + } + let mut invalid = transport_delivery(); + invalid.sequence_span = 2; + assert!(matches!( + client + .deliver_workflow_cancellation(&transport_task(), &invalid) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + let mut observation = CancellationRequest::from_observation(&request_observation()).unwrap(); + observation.history_refresh_page_token = None; + assert!(matches!( + client + .refresh_workflow_cancellation_history(&transport_task(), &observation) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + let control_only = Client::builder(server.base_url()) + .control_token(Some("control-only".into())) + .build() + .unwrap(); + assert!(matches!( + control_only + .deliver_workflow_cancellation(&transport_task(), &transport_delivery()) + .await, + Err(Error::MissingRoleCredentials { role: "worker", .. }) + )); + assert!(server.requests.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn cooperative_transport_preserves_claim_refusals_and_bounds_the_total_exchange() { + let server = transport_server(); + assert!(matches!( + client(&server, "refused") + .deliver_workflow_cancellation(&transport_task(), &transport_delivery()) + .await, + Err(Error::Http { status, .. }) if status.as_u16() == 409 + )); + let started = Instant::now(); + let observation = CancellationRequest::from_observation(&request_observation()).unwrap(); + let result = client(&server, "budget") + .refresh_workflow_cancellation_history(&transport_task(), &observation) + .await; + assert!(matches!(result, Err(Error::Timeout))); + assert!(started.elapsed() < Duration::from_secs(6)); + assert_eq!( + server + .requests + .lock() + .unwrap() + .iter() + .filter(|request| request.path.ends_with("/history")) + .count(), + 2 + ); +} + +#[tokio::test] +async fn cooperative_request_uses_control_role_namespace_and_preserves_server_identity() { + let server = server(); + let client = client(&server, "valid"); + let options = CooperativeCancellationOptions { + reason: Some("caller stopped".into()), + cleanup_timeout_seconds: Some(60), + }; + let first = client + .request_workflow_cancellation("workflow", options) + .await + .unwrap(); + let repeated = client + .request_workflow_cancellation( + "workflow", + CooperativeCancellationOptions { + reason: Some("changed reason".into()), + cleanup_timeout_seconds: Some(120), + }, + ) + .await + .unwrap(); + assert!(!first.duplicate); + assert!(repeated.duplicate); + assert_eq!(first.cancellation_request, repeated.cancellation_request); + assert_eq!(first.cancellation_request.request_id, "original-request"); + assert_eq!( + first.cancellation_request.cleanup_deadline_at, + "2026-10-01T08:10:00Z" + ); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 4); + for request in requests.iter() { + assert_eq!( + request.authorization.as_deref(), + Some("Bearer control-only") + ); + assert_eq!(request.namespace.as_deref(), Some("caller-namespace")); + assert_eq!(request.control_protocol.as_deref(), Some("2")); + assert_eq!(request.worker_protocol, None); + } + assert_eq!(requests[0].method, "GET"); + assert_eq!(requests[1].method, "POST"); + assert_eq!( + serde_json::from_str::(&requests[1].body).unwrap(), + json!({ + "reason":"caller stopped", "cleanup_timeout_seconds":60 + }) + ); +} + +#[tokio::test] +async fn cooperative_request_refuses_missing_unsupported_or_incompatible_discovery_before_mutation() +{ + for case in [ + "unsupported", + "missing", + "string-flag", + "old", + "future-major", + "malformed-version", + ] { + let server = server(); + let result = client(&server, case) + .request_workflow_cancellation("workflow", Default::default()) + .await; + assert!( + matches!(result, Err(Error::CooperativeCancellationUnavailable(_))), + "{case}: {result:?}" + ); + assert_eq!(server.requests.lock().unwrap().len(), 1, "{case}"); + } +} + +#[tokio::test] +async fn cooperative_request_accepts_additive_minor_discovery_without_changing_worker_default() { + let server = server(); + client(&server, "new-minor") + .request_workflow_cancellation("workflow", Default::default()) + .await + .unwrap(); + assert_eq!(WORKER_PROTOCOL_VERSION, "1.19"); + let ordinary = Client::builder(server.base_url()).build().unwrap(); + let mut worker = Worker::new(ordinary, "queue"); + worker.register_workflow("example", |_ctx, _input| async { Ok(Value::Null) }); + worker.register().await.unwrap(); + let requests = server.requests.lock().unwrap(); + let registration = requests + .iter() + .find(|request| request.path.ends_with("/worker/register")) + .unwrap(); + assert_eq!(registration.worker_protocol.as_deref(), Some("1.19")); + let body: Value = serde_json::from_str(®istration.body).unwrap(); + assert!(!body["capabilities"] + .as_array() + .unwrap() + .contains(&json!("cooperative_cancellation"))); +} + +#[tokio::test] +async fn cooperative_request_validates_selected_run_and_encodes_path_segments() { + let server = server(); + let result = client(&server, "escaped") + .request_workflow_run_cancellation("workflow/with?path", "run/selected", Default::default()) + .await + .unwrap(); + assert_eq!(result.run_id, "run/selected"); + assert_eq!(server.request_count("/escaped/api/workflows/workflow%2Fwith%3Fpath/runs/run%2Fselected/request-cancellation"), 1); +} + +#[tokio::test] +async fn cooperative_request_rejects_malformed_or_mismatched_acknowledgments() { + for case in [ + "wrong-workflow", + "wrong-run", + "empty-run", + "not-accepted", + "accepted-string", + "duplicate-integer", + "missing-request", + "blank-request", + "blank-token", + "no-timezone", + "invalid-date", + "equal-deadline", + "earlier-deadline", + ] { + let server = server(); + let result = client(&server, case) + .request_workflow_run_cancellation("workflow", "run", Default::default()) + .await; + assert!( + matches!(result, Err(Error::InvalidCooperativeCancellation(_))), + "{case}: {result:?}" + ); + } +} + +#[tokio::test] +async fn cooperative_request_rejects_invalid_inputs_without_network_activity() { + let server = server(); + let client = client(&server, "valid"); + for options in [ + CooperativeCancellationOptions { + cleanup_timeout_seconds: Some(0), + ..Default::default() + }, + CooperativeCancellationOptions { + cleanup_timeout_seconds: Some(3601), + ..Default::default() + }, + CooperativeCancellationOptions { + reason: Some("x".repeat(1001)), + ..Default::default() + }, + ] { + assert!(matches!( + client + .request_workflow_cancellation("workflow", options) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + } + assert!(matches!( + client + .request_workflow_cancellation(" ", Default::default()) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + assert!(matches!( + client + .request_workflow_run_cancellation("workflow", " ", Default::default()) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + assert!(server.requests.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn cooperative_request_does_not_substitute_worker_credentials() { + let server = server(); + let client = Client::builder(server.base_url()) + .worker_token(Some("worker-only".into())) + .build() + .unwrap(); + assert!(matches!( + client + .request_workflow_cancellation("workflow", Default::default()) + .await, + Err(Error::MissingRoleCredentials { + role: "control", + .. + }) + )); + assert!(server.requests.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn cooperative_request_preserves_discovery_and_active_claim_refusals() { + for (case, expected) in [ + ("discovery-http", reqwest::StatusCode::FORBIDDEN), + ("refused", reqwest::StatusCode::CONFLICT), + ] { + let server = server(); + let error = client(&server, case) + .request_workflow_cancellation("workflow", Default::default()) + .await + .unwrap_err(); + assert!(matches!(error, Error::Http {status, ..} if status==expected)); + assert_eq!( + server.requests.lock().unwrap().len(), + if case == "discovery-http" { 1 } else { 2 } + ); + } +} + +#[tokio::test] +async fn cooperative_request_handles_keep_current_and_selected_run_routing_separate() { + let server = server(); + let handle = WorkflowHandle { + client: client(&server, "valid"), + workflow_id: "workflow".into(), + run_id: Some("run".into()), + workflow_type: "example".into(), + }; + handle + .request_cancellation(Default::default()) + .await + .unwrap(); + handle + .request_selected_run_cancellation(Default::default()) + .await + .unwrap(); + assert_eq!( + server.request_count("/valid/api/workflows/workflow/request-cancellation"), + 1 + ); + assert_eq!( + server.request_count("/valid/api/workflows/workflow/runs/run/request-cancellation"), + 1 + ); + let missing = WorkflowHandle { + run_id: None, + ..handle + }; + assert!(matches!( + missing + .request_selected_run_cancellation(Default::default()) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + assert_eq!(server.requests.lock().unwrap().len(), 4); +} + +#[tokio::test] +async fn cooperative_request_bounds_discovery_and_mutation_with_one_total_budget() { + let server = server(); + let started = Instant::now(); + let result = client(&server, "budget") + .request_workflow_cancellation("workflow", Default::default()) + .await; + assert!(matches!(result, Err(Error::Timeout)), "{result:?}"); + assert!(started.elapsed() >= Duration::from_secs(4)); + assert!( + started.elapsed() < Duration::from_secs(7), + "total budget exceeded: {:?}", + started.elapsed() + ); + assert_eq!(server.requests.lock().unwrap().len(), 2); +} diff --git a/src/tests/fixtures/cooperative-cancellation-context.json b/src/tests/fixtures/cooperative-cancellation-context.json new file mode 100644 index 0000000..dbe7e89 --- /dev/null +++ b/src/tests/fixtures/cooperative-cancellation-context.json @@ -0,0 +1,17 @@ +{ + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "request-1", + "root_request_id": "root-1", + "root_workflow_instance_id": "parent-instance", + "root_workflow_run_id": "parent-run", + "parent_request_id": "root-1", + "reason": "maintenance", + "requester": {"type": "operator", "id": "operator-1", "label": "Maintainer"}, + "source": "control_plane", + "requested_at": "2026-10-01T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-01T00:00:30.123456Z", + "lineage": [ + {"request_id": "root-1", "workflow_instance_id": "parent-instance", "workflow_run_id": "parent-run"}, + {"request_id": "request-1", "workflow_instance_id": "child-instance", "workflow_run_id": "run-1"} + ] +} diff --git a/src/tests/fixtures/scoped-run-cancellation-context.json b/src/tests/fixtures/scoped-run-cancellation-context.json new file mode 100644 index 0000000..7d9d0a4 --- /dev/null +++ b/src/tests/fixtures/scoped-run-cancellation-context.json @@ -0,0 +1,164 @@ +{ + "native_commit": "a51395da59ef70945f91f00581ecb6bad8765bdd", + "child": { + "schema": "durable-workflow.cancellation-context/v2", + "request_id": "child-request", + "root_request_id": "root-request", + "root_workflow_instance_id": "root-instance", + "root_workflow_run_id": "root-run", + "parent_request_id": "inner-request", + "reason": "maintenance", + "requester": { + "type": "operator", + "id": "operator-1" + }, + "source": "api", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:15.123456Z", + "lineage": [ + { + "request_id": "root-request", + "workflow_instance_id": "root-instance", + "workflow_run_id": "root-run" + }, + { + "request_id": "child-request", + "workflow_instance_id": "child-instance", + "workflow_run_id": "child-run" + } + ], + "scope_origin": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "root-request", + "root_request_id": "root-request", + "root_workflow_instance_id": "root-instance", + "root_workflow_run_id": "root-run", + "parent_request_id": null, + "reason": "maintenance", + "requester": { + "type": "operator", + "id": "operator-1" + }, + "source": "api", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "root-request", + "workflow_instance_id": "root-instance", + "workflow_run_id": "root-run" + } + ] + }, + "lineage": [ + { + "request_id": "root-request", + "workflow_instance_id": "root-instance", + "workflow_run_id": "root-run", + "scope_id": "outer", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "inner-request", + "workflow_instance_id": "root-instance", + "workflow_run_id": "root-run", + "scope_id": "inner", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z" + } + ] + }, + "scope_authority_deadline_at": "2026-10-04T00:00:15.123456Z" + }, + "grandchild": { + "schema": "durable-workflow.cancellation-context/v2", + "request_id": "grandchild-request", + "root_request_id": "root-request", + "root_workflow_instance_id": "root-instance", + "root_workflow_run_id": "root-run", + "parent_request_id": "child-scope-request", + "reason": "maintenance", + "requester": { + "type": "operator", + "id": "operator-1" + }, + "source": "api", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:12.123456Z", + "lineage": [ + { + "request_id": "root-request", + "workflow_instance_id": "root-instance", + "workflow_run_id": "root-run" + }, + { + "request_id": "child-scope-request", + "workflow_instance_id": "child-instance", + "workflow_run_id": "child-run" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "grandchild-instance", + "workflow_run_id": "grandchild-run" + } + ], + "scope_origin": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "root-request", + "root_request_id": "root-request", + "root_workflow_instance_id": "root-instance", + "root_workflow_run_id": "root-run", + "parent_request_id": null, + "reason": "maintenance", + "requester": { + "type": "operator", + "id": "operator-1" + }, + "source": "api", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "root-request", + "workflow_instance_id": "root-instance", + "workflow_run_id": "root-run" + } + ] + }, + "lineage": [ + { + "request_id": "root-request", + "workflow_instance_id": "root-instance", + "workflow_run_id": "root-run", + "scope_id": "outer", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "inner-request", + "workflow_instance_id": "root-instance", + "workflow_run_id": "root-run", + "scope_id": "inner", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "child-instance", + "workflow_run_id": "child-run", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-04T00:00:15.123456Z" + }, + { + "request_id": "child-scope-request", + "workflow_instance_id": "child-instance", + "workflow_run_id": "child-run", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:12.123456Z" + } + ] + }, + "scope_authority_deadline_at": "2026-10-04T00:00:12.123456Z" + } +} diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs new file mode 100644 index 0000000..6d450e9 --- /dev/null +++ b/tests/cooperative_server.rs @@ -0,0 +1,2485 @@ +//! Explicit source qualification against an isolated Server protocol 1.20 stack. +//! Ordinary cargo tests leave these cases ignored. They require a real runtime. + +use durable_workflow::{ + json, ActivityContext, ActivityOptions, CancellationPolicy, Client, ConditionWaitOptions, + CooperativeCancellationOptions, Error, ParallelOperation, ParallelResult, + ScopedCancellationContext, SelectionKey, Value, Worker, WorkflowCommandOptions, + WorkflowContext, WorkflowHandle, WorkflowResultOptions, +}; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, Mutex, + }, + time::Duration, +}; + +const WORKFLOW: &str = "tests.rust-cooperative-timer"; +const UNDO: &str = "tests.rust-cooperative-undo"; +const BLOCKED: &str = "tests.rust-cooperative-blocked"; +const REPLAY: &str = "tests.rust-cooperative-replay"; +const PROCESS: &str = "tests.rust-cooperative-process-reclaim"; +const RUN_SCOPE: &str = "tests.rust-run-scope-cleanup"; +const REMOTE_SCOPE: &str = "tests.rust-scope-remote"; + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +struct ActivityGrant { + task_id: String, + attempt_id: String, + owner: String, + attempt_number: u64, +} + +struct WorkerProcess(std::process::Child); + +impl Drop for WorkerProcess { + fn drop(&mut self) { + if !matches!(self.0.try_wait(), Ok(Some(_))) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } +} + +struct ProcessScratch(std::path::PathBuf); + +impl Drop for ProcessScratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +struct AbortWorkerOnDrop(tokio::task::AbortHandle); + +impl Drop for AbortWorkerOnDrop { + fn drop(&mut self) { + self.0.abort(); + } +} + +fn register_process_workflow(worker: &mut Worker) { + worker.register_workflow(PROCESS, |ctx, _| async move { + let mut saga = ctx.saga(); + saga.add_compensation(UNDO, json!([]))?; + let result = ctx.activity(BLOCKED, json!([])).await; + saga.finish(result).await?; + Ok(Value::Null) + }); +} + +// Invoked only by the isolated process-death case. Ordinary tests return +// without starting a worker. The parent kills this actual Worker with SIGKILL. +#[tokio::test] +async fn cooperative_process_worker_child() { + let Ok(queue) = std::env::var("DURABLE_WORKFLOW_PROCESS_CHILD_QUEUE") else { + return; + }; + let ready = std::env::var("DURABLE_WORKFLOW_PROCESS_CHILD_READY").unwrap(); + let grant = std::env::var("DURABLE_WORKFLOW_PROCESS_CHILD_GRANT").unwrap(); + let scoped = + std::env::var("DURABLE_WORKFLOW_PROCESS_CHILD_PROFILE").as_deref() == Ok("scope-root"); + let mut worker = if scoped { + run_scope_worker(&client(), &queue, &format!("{queue}-killed")) + } else { + let mut worker = Worker::new(client(), &queue) + .worker_id(format!("{queue}-killed")) + .cooperative_cancellation(true) + .poll_timeout(Duration::from_secs(1)); + register_process_workflow(&mut worker); + worker + }; + worker.register_activity(BLOCKED, move |ctx, _| { + let grant = grant.clone(); + async move { + let bytes = serde_json::to_vec(&ActivityGrant { + task_id: ctx.task_id, + attempt_id: ctx.activity_attempt_id, + owner: ctx.lease_owner, + attempt_number: ctx.attempt_number, + }) + .unwrap(); + let pending = format!("{grant}.pending"); + std::fs::write(&pending, bytes).unwrap(); + std::fs::rename(pending, grant).unwrap(); + std::future::pending::>().await + } + }); + worker.register().await.unwrap(); + std::fs::write(ready, b"registered").unwrap(); + loop { + worker.run_once().await.expect("child actual Worker tick"); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +async fn await_child_file(process: &mut WorkerProcess, path: &std::path::Path) { + tokio::time::timeout(Duration::from_secs(15), async { + while !path.is_file() { + assert!( + process.0.try_wait().unwrap().is_none(), + "actual child Worker exited before publishing {}", + path.display() + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("actual child Worker claim within 15 seconds"); +} + +#[cfg(unix)] +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with its real activity lease and repair daemon"] +async fn server_sigkill_activity_worker_reclaims_attempt_and_fences_old_publication() { + use std::os::unix::process::ExitStatusExt; + + let client = client(); + let queue = queue(); + let scratch = ProcessScratch(std::env::temp_dir().join(format!("{queue}-process"))); + std::fs::create_dir(&scratch.0).unwrap(); + let ready = scratch.0.join("ready"); + let grant_path = scratch.0.join("grant.json"); + let mut original_process = WorkerProcess( + std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "cooperative_process_worker_child", "--nocapture"]) + .env("DURABLE_WORKFLOW_PROCESS_CHILD_QUEUE", &queue) + .env("DURABLE_WORKFLOW_PROCESS_CHILD_READY", &ready) + .env("DURABLE_WORKFLOW_PROCESS_CHILD_GRANT", &grant_path) + .spawn() + .unwrap(), + ); + await_child_file(&mut original_process, &ready).await; + let handle = client + .start_workflow(PROCESS, &queue, &queue, json!([])) + .await + .unwrap(); + await_child_file(&mut original_process, &grant_path).await; + let original: ActivityGrant = + serde_json::from_slice(&std::fs::read(&grant_path).unwrap()).unwrap(); + assert_eq!(original.attempt_number, 1); + let leased = client + .activity_task_status(&original.task_id, &original.attempt_id, &original.owner) + .await + .unwrap(); + assert_eq!(leased["can_continue"], true); + eprintln!("SIGKILL original grant: {original:?}, actual lease: {leased}"); + original_process.0.kill().unwrap(); + let status = original_process.0.wait().unwrap(); + assert_eq!( + status.signal(), + Some(9), + "expected actual SIGKILL: {status}" + ); + + let (entered_tx, mut entered_rx) = tokio::sync::mpsc::unbounded_channel(); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let cleanups = Arc::new(AtomicUsize::new(0)); + let mut successor = Worker::new(client.clone(), &queue) + .worker_id(format!("{queue}-successor")) + .cooperative_cancellation(true) + .max_concurrent_workflow_tasks(1) + .max_concurrent_activity_tasks(1) + .poll_timeout(Duration::from_secs(1)); + register_process_workflow(&mut successor); + successor.register_activity(BLOCKED, move |ctx, _| { + let entered_tx = entered_tx.clone(); + async move { + ctx.heartbeat(json!({"qualification":"reclaimed"})).await?; + entered_tx.send(ctx).unwrap(); + std::future::pending::>().await + } + }); + let completed = Arc::clone(&cleanups); + successor.register_activity(UNDO, move |ctx, _| { + let completed = Arc::clone(&completed); + async move { + ctx.heartbeat(json!({"qualification":"cleanup"})).await?; + completed.fetch_add(1, Ordering::SeqCst); + Ok(Value::Null) + } + }); + let run = tokio::spawn(async move { + successor + .run_until(async { + let _ = shutdown_rx.await; + }) + .await + }); + let _abort_on_failure = AbortWorkerOnDrop(run.abort_handle()); + let reclaimed: ActivityContext = + // Native's actual activity-task lease is five minutes. This case waits + // for real wall-clock expiry, without changing timestamps or storage. + tokio::time::timeout(Duration::from_secs(330), entered_rx.recv()) + .await + .expect("repair daemon and actual successor reclaim within 330 seconds") + .expect("actual successor callback entered"); + assert_eq!(reclaimed.task_id, original.task_id); + assert_ne!(reclaimed.activity_attempt_id, original.attempt_id); + assert_ne!(reclaimed.lease_owner, original.owner); + assert_eq!(reclaimed.attempt_number, 2); + eprintln!( + "SIGKILL successor grant: {}/{}/{}/{}", + reclaimed.task_id, + reclaimed.activity_attempt_id, + reclaimed.lease_owner, + reclaimed.attempt_number + ); + + let closed = client + .activity_task_status(&original.task_id, &original.attempt_id, &original.owner) + .await + .unwrap(); + assert_eq!(closed["attempt_status"], "expired"); + assert_eq!(closed["can_continue"], false); + let before = history(&handle).await; + for result in [ + client + .complete_activity_task( + &original.task_id, + &original.attempt_id, + &original.owner, + json!("late"), + "avro", + ) + .await, + client + .fail_activity_task( + &original.task_id, + &original.attempt_id, + &original.owner, + "late failure", + true, + ) + .await, + ] { + assert!( + matches!(result, Err(Error::ActivityTaskRejected(rejection)) if rejection.status == 409) + ); + } + let heartbeat = client + .heartbeat_activity_task( + &original.task_id, + &original.attempt_id, + &original.owner, + json!({"late":true}), + ) + .await + .expect("dead-attempt heartbeat returns its stop status"); + assert_eq!(heartbeat.can_continue, Some(false)); + assert!(!heartbeat.heartbeat_recorded); + assert!(heartbeat.should_stop()); + assert_eq!(heartbeat.reason.as_deref(), Some("attempt_closed")); + assert!(!heartbeat.cancel_requested); + assert_eq!( + heartbeat.last_heartbeat_at.as_deref(), + closed["last_heartbeat_at"].as_str() + ); + assert_eq!( + heartbeat.lease_expires_at.as_deref(), + closed["lease_expires_at"].as_str(), + "dead attempt renewed its lease" + ); + eprintln!("SIGKILL dead-attempt heartbeat: {heartbeat:?}"); + assert_eq!( + closed, + client + .activity_task_status(&original.task_id, &original.attempt_id, &original.owner) + .await + .unwrap(), + "dead heartbeat changed attempt authority or lifetime" + ); + assert_eq!( + before, + history(&handle).await, + "dead attempt changed canonical history" + ); + + let request = handle + .request_cancellation(CooperativeCancellationOptions::default()) + .await + .unwrap(); + let result = handle + .result_selected_run(WorkflowResultOptions { + timeout: Duration::from_secs(20), + poll_interval: Duration::from_millis(50), + }) + .await; + assert!( + matches!(result, Err(Error::WorkflowCancelled(_))), + "reclaimed result: {result:?}" + ); + let snapshot = assert_cancelled(&handle, &request.cancellation_request.request_id, true).await; + assert_eq!(cleanups.load(Ordering::SeqCst), 1); + assert_eq!(count(&snapshot, "ActivityCancelled"), 1); + eprintln!( + "SIGKILL activity recovery: old={original:?}, successor={}/{}/{}, history={snapshot}", + reclaimed.task_id, reclaimed.activity_attempt_id, reclaimed.lease_owner + ); + shutdown_tx.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(10), run) + .await + .unwrap() + .unwrap() + .unwrap(); +} + +struct CallbackDrop(Arc); + +async fn blocked_cleanup_cutoff(terminate: bool) { + let client = client(); + let queue = queue(); + let dropped = Arc::new(AtomicUsize::new(0)); + let cleanup_dropped = Arc::clone(&dropped); + let (entered_tx, mut entered_rx) = tokio::sync::mpsc::unbounded_channel(); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let mut worker = Worker::new(client.clone(), &queue) + .cooperative_cancellation(true) + .max_concurrent_workflow_tasks(1) + .max_concurrent_activity_tasks(1) + .poll_timeout(Duration::from_secs(1)); + register_process_workflow(&mut worker); + worker.register_activity(UNDO, move |ctx, _| { + let entered_tx = entered_tx.clone(); + let cleanup_dropped = Arc::clone(&cleanup_dropped); + async move { + let _drop = CallbackDrop(cleanup_dropped); + entered_tx.send(ctx).unwrap(); + std::future::pending::>().await + } + }); + let handle = client + .start_workflow(PROCESS, &queue, &queue, json!([])) + .await + .unwrap(); + let request = handle + .request_cancellation(CooperativeCancellationOptions { + cleanup_timeout_seconds: Some(if terminate { 60 } else { 5 }), + ..CooperativeCancellationOptions::default() + }) + .await + .unwrap(); + let run = tokio::spawn(async move { + worker + .run_until(async { + let _ = shutdown_rx.await; + }) + .await + }); + let _abort_on_failure = AbortWorkerOnDrop(run.abort_handle()); + let cleanup: ActivityContext = tokio::time::timeout(Duration::from_secs(10), entered_rx.recv()) + .await + .unwrap() + .expect("actual shielded cleanup entered"); + if terminate { + handle + .terminate_selected_run(WorkflowCommandOptions::default()) + .await + .unwrap(); + } + let result = handle + .result_selected_run(WorkflowResultOptions { + // Include the production ten-second repair cadence without extending + // the original cleanup deadline. + timeout: Duration::from_secs(25), + poll_interval: Duration::from_millis(50), + }) + .await; + if terminate { + assert!( + matches!(result, Err(Error::WorkflowTerminated(_))), + "{result:?}" + ); + } else { + assert!( + matches!(result, Err(Error::WorkflowCancelled(_))), + "{result:?}" + ); + } + tokio::time::timeout(Duration::from_secs(10), async { + while dropped.load(Ordering::SeqCst) == 0 { + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("blocked cleanup future drops after terminal authority loss"); + assert_eq!(dropped.load(Ordering::SeqCst), 1); + assert!(matches!( + cleanup.heartbeat(json!({"late":true})).await, + Err(Error::ActivityExecutionAbandoned(_)) + )); + let receipt_expected = !terminate + && std::env::var("DURABLE_WORKFLOW_NATIVE_SOURCE_QUALIFICATION").as_deref() == Ok("1"); + if receipt_expected { + // Callback drop precedes the asynchronous Server receipt. Stabilize + // that diagnostic before checking stale result/failure publication. + let status = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let status = client + .activity_task_status( + &cleanup.task_id, + &cleanup.activity_attempt_id, + &cleanup.lease_owner, + ) + .await + .unwrap(); + if status["cancellation_acknowledgement"]["callback_state"] == "stopped" { + break status; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("dropped cleanup callback must leave its original stop receipt"); + let receipt = &status["cancellation_acknowledgement"]; + assert_eq!( + receipt["request_id"], + request.cancellation_request.request_id + ); + assert_eq!( + receipt["root_request_id"], + request.cancellation_request.request_id + ); + assert_eq!( + receipt["cleanup_deadline_at"], + request.cancellation_request.cleanup_deadline_at + ); + assert_eq!(receipt["received_after_deadline"], true); + assert_eq!(status["heartbeat_recorded"], false); + assert_eq!(status["can_continue"], false); + eprintln!("Blocked cleanup stop receipt: {status}"); + } + let snapshot = history(&handle).await; + assert_eq!( + count(&snapshot, "ActivityCancellationAcknowledged"), + usize::from(receipt_expected) + ); + for kind in [ + "CooperativeCancellationRequested", + "CooperativeCancellationDelivered", + ] { + assert_eq!(count(&snapshot, kind), 1, "{snapshot}"); + let event = snapshot["events"] + .as_array() + .unwrap() + .iter() + .find(|e| e["event_type"] == kind) + .unwrap(); + assert_eq!( + event["payload"]["workflow_command_id"], + request.cancellation_request.request_id + ); + } + assert_eq!( + count(&snapshot, "WorkflowCancelled"), + usize::from(!terminate) + ); + assert_eq!( + count(&snapshot, "WorkflowTerminated"), + usize::from(terminate) + ); + assert_eq!(count(&snapshot, "ActivityScheduled"), 1); + assert_eq!(count(&snapshot, "ActivityCancelled"), 1); + for kind in [ + "ActivityCompleted", + "ActivityFailed", + "ActivityTimedOut", + "WorkflowCompleted", + "WorkflowFailed", + ] { + assert_eq!(count(&snapshot, kind), 0, "{snapshot}"); + } + for result in [ + client + .complete_activity_task( + &cleanup.task_id, + &cleanup.activity_attempt_id, + &cleanup.lease_owner, + json!("late"), + "avro", + ) + .await, + client + .fail_activity_task( + &cleanup.task_id, + &cleanup.activity_attempt_id, + &cleanup.lease_owner, + "late", + true, + ) + .await, + ] { + assert!( + matches!(result, Err(Error::ActivityTaskRejected(rejection)) if rejection.status == 409) + ); + } + assert_eq!( + snapshot, + history(&handle).await, + "late cleanup changed canonical history" + ); + eprintln!("blocked cleanup cutoff terminate={terminate}: {snapshot}"); + shutdown_tx.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(10), run) + .await + .unwrap() + .unwrap() + .unwrap(); +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_cleanup_deadline_stops_blocked_compensation() { + blocked_cleanup_cutoff(false).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_termination_stops_blocked_compensation() { + blocked_cleanup_cutoff(true).await; +} + +impl Drop for CallbackDrop { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} + +fn client() -> Client { + Client::builder(std::env::var("DURABLE_WORKFLOW_SERVER_URL").expect("isolated Server URL")) + .token(Some( + std::env::var("DURABLE_WORKFLOW_AUTH_TOKEN").expect("isolated Server token"), + )) + .namespace("default") + .build() + .unwrap() +} + +fn queue() -> String { + format!("rust-cooperative-{}", durable_workflow::Uuid::new_v4()) +} + +fn worker(client: &Client, queue: &str, saga: bool, prefix: bool) -> Worker { + let mut worker = Worker::new(client.clone(), queue) + .worker_id(format!("{queue}-{}", durable_workflow::Uuid::new_v4())) + .cooperative_cancellation(true) + .poll_timeout(Duration::from_secs(1)); + worker.register_workflow(WORKFLOW, move |ctx, _| async move { + if prefix { + assert_eq!(ctx.side_effect(|| json!(7))?, json!(7)); + } + if saga { + let mut saga = ctx.saga(); + saga.add_compensation(UNDO, json!([]))?; + let result = ctx.sleep(Duration::from_secs(300)).await; + saga.finish(result).await?; + } else { + ctx.sleep(Duration::from_secs(300)).await?; + } + Ok(Value::Null) + }); + worker +} + +async fn history(handle: &WorkflowHandle) -> Value { + let url = std::env::var("DURABLE_WORKFLOW_SERVER_URL").unwrap(); + let token = std::env::var("DURABLE_WORKFLOW_AUTH_TOKEN").unwrap(); + let response = reqwest::Client::new() + .get(format!( + "{url}/api/workflows/{}/runs/{}/history", + handle.workflow_id, + handle.run_id.as_deref().unwrap() + )) + .query(&[("page_size", "1000")]) + .bearer_auth(token) + .header("X-Namespace", "default") + .header("X-Durable-Workflow-Control-Plane-Version", "2") + .send() + .await + .unwrap(); + let status = response.status(); + let body: Value = response.json().await.unwrap(); + assert!(status.is_success(), "history {status}: {body}"); + assert!( + body["next_page_token"].is_null(), + "bounded scenario history was paginated" + ); + body +} + +fn count(history: &Value, kind: &str) -> usize { + history["events"] + .as_array() + .unwrap() + .iter() + .filter(|event| event["event_type"] == kind) + .count() +} + +async fn tick_until(worker: &Worker, handle: &WorkflowHandle, kind: &str) -> Value { + tick_until_count(worker, handle, kind, 1).await +} + +async fn tick_until_count( + worker: &Worker, + handle: &WorkflowHandle, + kind: &str, + target: usize, +) -> Value { + let mut last_snapshot = Value::Null; + let observed = tokio::time::timeout(Duration::from_secs(30), async { + loop { + worker.run_once().await.expect("actual Worker tick"); + let snapshot = history(handle).await; + if count(&snapshot, kind) >= target { + return snapshot; + } + last_snapshot = snapshot; + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await; + observed.unwrap_or_else(|_| { + panic!( + "expected {target} {kind} within 30s for {}/{:?}, last history: {last_snapshot}", + handle.workflow_id, handle.run_id + ) + }) +} + +async fn assert_cancelled(handle: &WorkflowHandle, request_id: &str, cleanup: bool) -> Value { + let result = handle + .result_selected_run(WorkflowResultOptions { + timeout: Duration::from_secs(5), + poll_interval: Duration::from_millis(50), + }) + .await; + assert!( + matches!(result, Err(Error::WorkflowCancelled(_))), + "terminal result: {result:?}" + ); + let snapshot = history(handle).await; + for kind in [ + "CooperativeCancellationRequested", + "CooperativeCancellationDelivered", + "WorkflowCancelled", + ] { + assert_eq!(count(&snapshot, kind), 1, "{kind}: {snapshot}"); + let event = snapshot["events"] + .as_array() + .unwrap() + .iter() + .find(|event| event["event_type"] == kind) + .unwrap(); + assert_eq!( + event["payload"]["workflow_command_id"], request_id, + "{kind}: {event}" + ); + } + for kind in [ + "WorkflowCompleted", + "WorkflowFailed", + "ActivityFailed", + "ActivityTimedOut", + ] { + assert_eq!(count(&snapshot, kind), 0, "{kind}: {snapshot}"); + } + assert_eq!(count(&snapshot, "ActivityCompleted"), usize::from(cleanup)); + snapshot +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_scope_opening_proves_real_native_tree_on_original_claim() { + let client = client(); + let queue = queue(); + let owner = format!("{queue}-scope-owner"); + let worker = worker(&client, &queue, false, false).worker_id(&owner); + worker.register().await.unwrap(); + let handle = client + .start_workflow(WORKFLOW, &queue, &queue, json!([])) + .await + .unwrap(); + let claim = tokio::time::timeout(Duration::from_secs(30), async { + loop { + let reply = client + .poll_cooperative_workflow_task(&owner, &queue, Duration::from_secs(1)) + .await + .unwrap(); + if let Some(claim) = reply.task { + return claim; + } + } + }) + .await + .unwrap(); + let parent = client + .open_cancellation_scope_on_claim(claim.task(), 1, "root", false) + .await + .unwrap(); + let child = client + .open_cancellation_scope_on_claim(claim.task(), 2, parent.scope_id(), true) + .await + .unwrap(); + let duplicate = client + .open_cancellation_scope_on_claim(claim.task(), 2, parent.scope_id(), true) + .await + .unwrap(); + assert_eq!(parent.parent_scope_id(), "root"); + assert!(!parent.shield_parent() && !parent.duplicate()); + assert_eq!(child.parent_scope_id(), parent.scope_id()); + assert!(child.shield_parent() && !child.duplicate() && duplicate.duplicate()); + assert_eq!(duplicate.scope_id(), child.scope_id()); + assert_eq!(duplicate.history_event_id(), child.history_event_id()); + let openings = child + .history() + .iter() + .filter(|event| event.event_type == "CancellationScopeOpened") + .map(|event| event.payload["scope_id"].as_str().unwrap()) + .collect::>(); + assert_eq!(openings, vec![parent.scope_id(), child.scope_id()]); + let mut stale = claim.task().clone(); + stale.workflow_task_attempt += 1; + assert!( + matches!(client.open_cancellation_scope_on_claim(&stale,3,"root",false).await, + Err(Error::Http { status, .. }) if status.as_u16()==409) + ); + let snapshot = history(&handle).await; + assert_eq!(count(&snapshot, "CancellationScopeOpened"), 2); + for kind in ["TimerScheduled", "WorkflowFailed", "WorkflowCompleted"] { + assert_eq!(count(&snapshot, kind), 0); + } + eprintln!("Native scope opening proof: {:?}", child.history()); + handle + .terminate_selected_run(WorkflowCommandOptions::default()) + .await + .unwrap(); +} + +fn scope_authoring_worker(client: &Client, queue: &str, owner: &str) -> Worker { + let mut worker = Worker::new(client.clone(), queue) + .worker_id(owner) + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .poll_timeout(Duration::from_millis(100)); + worker.register_workflow("tests.rust-candidate-scope-replay", |ctx, _| async move { + let _: String = ctx.side_effect(|| "original prefix".to_string())?; + ctx.cancellation_scope(false, |outer| async move { + let deferred = outer + .cancellation_scope(true, |inner| async move { + Ok(inner.sleep(Duration::from_secs(1))) + }) + .await?; + deferred.await?; + outer.sleep(Duration::from_secs(1)).await + }) + .await?; + ctx.sleep(Duration::from_secs(1)).await?; + Ok(json!("replayed original scopes")) + }); + worker +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with the Native scope candidate"] +async fn server_scope_authoring_replays_nested_tree_and_deferred_membership_on_replacement() { + let client = client(); + let queue = queue(); + let original = scope_authoring_worker(&client, &queue, &format!("{queue}-original")); + original.register().await.unwrap(); + let handle = client + .start_workflow( + "tests.rust-candidate-scope-replay", + &queue, + &queue, + json!([]), + ) + .await + .unwrap(); + let first = tick_until(&original, &handle, "TimerScheduled").await; + assert_eq!(count(&first, "SideEffectRecorded"), 1); + assert_eq!(count(&first, "CancellationScopeOpened"), 2); + assert_eq!(count(&first, "TimerScheduled"), 1); + + let replacement = scope_authoring_worker(&client, &queue, &format!("{queue}-replacement")); + replacement.register().await.unwrap(); + let final_history = tick_until(&replacement, &handle, "WorkflowCompleted").await; + assert_eq!(count(&final_history, "SideEffectRecorded"), 1); + assert_eq!(count(&final_history, "CancellationScopeOpened"), 2); + assert_eq!(count(&final_history, "TimerScheduled"), 3); + assert_eq!(count(&final_history, "TimerFired"), 3); + assert_eq!(count(&final_history, "WorkflowFailed"), 0); + let rows = final_history["events"].as_array().unwrap(); + let openings: Vec<_> = rows + .iter() + .filter(|row| row["event_type"] == "CancellationScopeOpened") + .map(|row| &row["payload"]) + .collect(); + assert_eq!(openings[0]["sequence"], 2); + assert_eq!(openings[0]["parent_scope_id"], "root"); + assert_eq!(openings[0]["shield_parent"], false); + assert_eq!(openings[1]["sequence"], 3); + assert_eq!(openings[1]["parent_scope_id"], openings[0]["scope_id"]); + assert_eq!(openings[1]["shield_parent"], true); + let timers: Vec<_> = rows + .iter() + .filter(|row| row["event_type"] == "TimerScheduled") + .map(|row| &row["payload"]) + .collect(); + assert_eq!( + timers + .iter() + .map(|row| row["sequence"].as_u64().unwrap()) + .collect::>(), + vec![4, 5, 6] + ); + assert_eq!(timers[0]["cancellation_scope_id"], openings[1]["scope_id"]); + assert_eq!(timers[1]["cancellation_scope_id"], openings[0]["scope_id"]); + assert!(timers[2] + .get("cancellation_scope_id") + .is_none_or(|id| id == "root")); + assert_eq!( + handle + .result_selected_run(WorkflowResultOptions { + timeout: Duration::from_secs(10), + ..WorkflowResultOptions::default() + }) + .await + .unwrap(), + json!("replayed original scopes") + ); + println!("Native scope authoring and replacement replay: {final_history}"); +} + +fn request_native_scope_fixture(handle: &WorkflowHandle, scope: &str) -> Value { + use std::io::Write; + use std::process::{Command, Stdio}; + let mut process = Command::new("timeout") + .args([ + "--kill-after=1", + "10", + "docker", + "compose", + "exec", + "-T", + "--user", + "1000:1000", + "--env", + "DURABLE_WORKFLOW_NATIVE_SCOPE_FIXTURE=1", + "server", + "timeout", + "--kill-after=1", + "5", + "php", + "/app/sdk-source-fixtures/native-scope-request.php", + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let input = serde_json::to_vec(&json!({"run_id":handle.run_id.as_deref().unwrap(), + "workflow_id":handle.workflow_id, "scope_id":scope})) + .unwrap(); + process.stdin.take().unwrap().write_all(&input).unwrap(); + let result = process.wait_with_output().unwrap(); + assert!( + result.status.success(), + "Native scope fixture: {}", + String::from_utf8_lossy(&result.stderr) + ); + serde_json::from_slice(&result.stdout).unwrap() +} + +async fn scope_cleanup_body( + scope: WorkflowContext, + grouped: bool, +) -> durable_workflow::Result<(ScopedCancellationContext, Value)> { + let pending = if grouped { + scope + .parallel(vec![ + ParallelOperation::timer(Duration::from_secs(300)), + ParallelOperation::group(vec![ParallelOperation::timer(Duration::from_secs(600))]), + ]) + .await + .map(|_| ()) + } else { + scope.sleep(Duration::from_secs(300)).await + }; + let cancellation = match pending { + Err(Error::CancellationScopeRequested(request)) => request.context, + Err(error) => return Err(error), + Ok(()) => { + return Err(Error::InvalidCooperativeCancellation( + "original timer was not interrupted".into(), + )) + } + }; + assert!(scope.is_cancellation_requested()?); + let _shield = scope.cancellation_shield()?; + let metadata = scope.side_effect(|| cancellation.to_value())?; + scope.sleep(Duration::from_secs(1)).await?; + Ok((cancellation, metadata)) +} + +fn scope_cleanup_worker( + client: &Client, + queue: &str, + owner: &str, + grouped: bool, + descendants: bool, +) -> Worker { + let mut worker = Worker::new(client.clone(), queue) + .worker_id(owner) + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true) + .poll_timeout(Duration::from_millis(100)); + worker.register_workflow( + "tests.rust-candidate-scope-cleanup", + move |ctx, _| async move { + let _: String = ctx.side_effect(|| "original prefix".to_owned())?; + let (cancellation, metadata, descendant_metadata) = ctx + .cancellation_scope(false, move |scope| async move { + if !descendants { + let (cancellation, metadata) = scope_cleanup_body(scope, grouped).await?; + return Ok((cancellation, metadata, None)); + } + let (_, child_metadata) = scope + .cancellation_scope(false, move |child| scope_cleanup_body(child, grouped)) + .await?; + let cancellation = match scope.throw_if_cancellation_requested() { + Err(Error::CancellationScopeRequested(request)) => request.context, + Err(error) => return Err(error), + Ok(()) => { + return Err(Error::InvalidCooperativeCancellation( + "parent lost its original accepted request".into(), + )) + } + }; + let metadata = cancellation.to_value(); + assert_eq!(metadata["root_context"], child_metadata["root_context"]); + assert_eq!(scope.scoped_cancellation_context()?.unwrap(), cancellation); + let _shield = scope.cancellation_shield()?; + scope.sleep(Duration::from_secs(1)).await?; + Ok((cancellation, metadata, Some(child_metadata))) + }) + .await?; + assert!(!ctx.is_cancellation_requested()?); + assert!(ctx.scoped_cancellation_context()?.is_none()); + ctx.sleep(Duration::from_secs(1)).await?; + Ok( + json!({"context":metadata, "remaining":cancellation.remaining()?.as_secs_f64(), + "descendant_context":descendant_metadata}), + ) + }, + ); + worker +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with the Native scope candidate"] +async fn server_scope_cleanup_preserves_original_delivery_and_budget_after_replacement() { + qualify_scope_cleanup_replacement(false, false).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with the Native scope candidate"] +async fn server_scope_group_cleanup_preserves_original_delivery_and_budget_after_replacement() { + qualify_scope_cleanup_replacement(true, false).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with the Native scope candidate"] +async fn server_scope_descendant_cleanup_preserves_original_delivery_after_replacement() { + qualify_scope_cleanup_replacement(false, true).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with the Native scope candidate"] +async fn server_scope_descendant_group_cleanup_preserves_original_delivery_after_replacement() { + qualify_scope_cleanup_replacement(true, true).await; +} + +async fn qualify_scope_cleanup_replacement(grouped: bool, descendants: bool) { + let client = client(); + let queue = format!( + "rust-cooperative-scope-boundary-{}", + durable_workflow::Uuid::new_v4().simple() + ); + let original = scope_cleanup_worker( + &client, + &queue, + &format!("{queue}-original"), + grouped, + descendants, + ); + original.register().await.unwrap(); + let handle = client + .start_workflow( + "tests.rust-candidate-scope-cleanup", + &queue, + &queue, + json!([]), + ) + .await + .unwrap(); + let first = tick_until(&original, &handle, "TimerScheduled").await; + assert_eq!(count(&first, "SideEffectRecorded"), 1); + let scope = first["events"] + .as_array() + .unwrap() + .iter() + .find(|row| row["event_type"] == "CancellationScopeOpened") + .unwrap()["payload"]["scope_id"] + .as_str() + .unwrap(); + let accepted = request_native_scope_fixture(&handle, scope); + assert_eq!(accepted, request_native_scope_fixture(&handle, scope)); + let cleaning = tick_until_count(&original, &handle, "SideEffectRecorded", 2).await; + assert_eq!( + count(&cleaning, "TimerScheduled"), + if grouped { 3 } else { 2 } + ); + assert_eq!(count(&cleaning, "CancellationScopeDelivered"), 1); + let replacement = scope_cleanup_worker( + &client, + &queue, + &format!("{queue}-replacement"), + grouped, + descendants, + ); + replacement.register().await.unwrap(); + let final_history = tick_until(&replacement, &handle, "WorkflowCompleted").await; + for kind in [ + "CancellationScopeDeliveryPrepared", + "CancellationScopeDelivered", + "WorkflowCompleted", + ] { + assert_eq!(count(&final_history, kind), 1, "{kind}"); + } + for kind in ["CancellationScopeOpened", "CancellationScopeRequested"] { + assert_eq!( + count(&final_history, kind), + if descendants { 2 } else { 1 }, + "{kind}" + ); + } + assert_eq!(count(&final_history, "SideEffectRecorded"), 2); + assert_eq!( + count(&final_history, "TimerCancelled"), + if grouped { 2 } else { 1 } + ); + assert_eq!( + count(&final_history, "TimerScheduled"), + (if grouped { 4 } else { 3 }) + usize::from(descendants) + ); + assert_eq!( + count(&final_history, "TimerFired"), + 2 + usize::from(descendants) + ); + for kind in [ + "CooperativeCancellationRequested", + "WorkflowCancelled", + "WorkflowFailed", + ] { + assert_eq!(count(&final_history, kind), 0, "{kind}"); + } + let result = handle + .result_selected_run(WorkflowResultOptions { + timeout: Duration::from_secs(5), + ..WorkflowResultOptions::default() + }) + .await + .unwrap(); + assert_eq!(result["context"], accepted["payload"]["cancellation"]); + if descendants { + let child = final_history["events"] + .as_array() + .unwrap() + .iter() + .find(|row| { + row["event_type"] == "CancellationScopeRequested" + && row["payload"]["scope_id"] != scope + }) + .unwrap()["payload"]["cancellation"] + .clone(); + assert_eq!(result["descendant_context"], child); + assert_eq!(child["root_context"], result["context"]["root_context"]); + let mut lineage = child["lineage"].as_array().unwrap().clone(); + lineage.pop(); + assert_eq!(json!(lineage), result["context"]["lineage"]); + } + assert!(result["remaining"] + .as_f64() + .is_some_and(|remaining| remaining > 0.0 && remaining < 30.0)); + println!("Native scope cleanup and replacement replay: {final_history}"); +} + +fn run_scope_worker(client: &Client, queue: &str, owner: &str) -> Worker { + let mut worker = Worker::new(client.clone(), queue) + .worker_id(owner) + .cooperative_cancellation(true) + .poll_timeout(Duration::from_secs(1)) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + worker.register_workflow(RUN_SCOPE, |root, _| async move { + let scoped = root + .cancellation_scope(false, |parent| async move { + parent + .cancellation_scope(false, |child| async move { + match child.sleep(Duration::from_secs(300)).await { + Err(Error::CancellationScopeRequested(cancelled)) => { + assert!(child.cancellation_context()?.is_none()); + let original = cancelled.context.to_value(); + { + let _shield = child.cancellation_shield()?; + child.sleep(Duration::from_secs(5)).await?; + } + Ok(original) + } + Err(error) => Err(error), + Ok(()) => { + panic!("original scoped timer completed without cancellation") + } + } + }) + .await + }) + .await?; + match root.sleep(Duration::from_secs(300)).await { + Err(Error::CooperativeCancellationRequested(cancelled)) => { + assert_eq!( + scoped["root_context"], + root.cancellation_context()?.unwrap().to_value() + ); + { + let _shield = root.cancellation_shield()?; + root.sleep(Duration::from_secs(1)).await?; + } + Err(Error::CooperativeCancellationRequested(cancelled)) + } + Err(error) => Err(error), + Ok(()) => panic!("root timer completed without canonical root delivery"), + } + }); + worker +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with the Native scope candidate"] +async fn server_scope_try_stops_actual_remote_callback_without_application_heartbeats() { + qualify_scoped_remote_stop(false, CancellationPolicy::TryCancel).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with the Native scope candidate"] +async fn server_scope_wait_stops_actual_remote_callback_before_cleanup() { + qualify_scoped_remote_stop(false, CancellationPolicy::WaitCancellationCompleted).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with the Native scope candidate"] +async fn server_scope_group_wait_stops_actual_remote_callback_before_cleanup() { + qualify_scoped_remote_stop(true, CancellationPolicy::WaitCancellationCompleted).await; +} + +async fn qualify_scoped_remote_stop(grouped: bool, policy: CancellationPolicy) { + let client = client(); + let queue = format!( + "rust-cooperative-scope-boundary-{}", + durable_workflow::Uuid::new_v4().simple() + ); + let dropped = Arc::new(AtomicUsize::new(0)); + let cleanup_dropped = Arc::clone(&dropped); + let (entered_tx, mut entered_rx) = tokio::sync::mpsc::unbounded_channel::(); + let (heartbeat_tx, mut heartbeat_rx) = tokio::sync::mpsc::unbounded_channel(); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let mut worker = Worker::new(client.clone(), &queue) + .worker_id(format!("{queue}-managed")) + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true) + .max_concurrent_workflow_tasks(1) + .max_concurrent_activity_tasks(1) + .poll_timeout(Duration::from_secs(1)) + .on_worker_heartbeat(move |observation| { + if observation.acknowledgement["acknowledged"] == true { + let _ = heartbeat_tx.send(()); + } + }); + worker.register_workflow(REMOTE_SCOPE, move |root, _| { + let dropped = Arc::clone(&cleanup_dropped); + async move { + let result = root + .cancellation_scope(false, move |scope| async move { + let options = ActivityOptions::new().cancellation_policy(policy); + let pending = if grouped { + scope + .parallel(vec![ + ParallelOperation::activity_with_options( + BLOCKED, + options, + json!([]), + ), + ParallelOperation::group(vec![ParallelOperation::timer( + Duration::from_secs(300), + )]), + ]) + .await + .map(|_| ()) + } else { + scope + .activity_with_options(BLOCKED, options, json!([])) + .await + .map(|_| ()) + }; + let cancelled = match pending { + Err(Error::CancellationScopeRequested(cancelled)) => cancelled, + Err(error) => return Err(error), + Ok(()) => panic!("blocked remote callback completed without cancellation"), + }; + if policy == CancellationPolicy::WaitCancellationCompleted { + assert_eq!( + dropped.load(Ordering::SeqCst), + 1, + "Wait cleanup preceded callback drop" + ); + } + let _shield = scope.cancellation_shield()?; + scope.sleep(Duration::from_secs(1)).await?; + Ok(cancelled.context.to_value()) + }) + .await?; + assert!(!root.is_cancellation_requested()?); + root.sleep(Duration::from_secs(1)).await?; + Ok(result) + } + }); + let callback_dropped = Arc::clone(&dropped); + worker.register_activity(BLOCKED, move |ctx, _| { + let entered_tx = entered_tx.clone(); + let dropped = Arc::clone(&callback_dropped); + async move { + let _drop = CallbackDrop(dropped); + entered_tx.send(ctx).unwrap(); + std::future::pending::>().await + } + }); + let run = tokio::spawn(async move { + let result = worker + .run_until(async { + let _ = shutdown_rx.await; + }) + .await; + if let Err(error) = &result { + eprintln!("Scoped remote worker stopped before convergence: {error:?}"); + } + result + }); + let _abort_on_failure = AbortWorkerOnDrop(run.abort_handle()); + tokio::time::timeout(Duration::from_secs(10), heartbeat_rx.recv()) + .await + .unwrap() + .expect("actual accepted worker heartbeat"); + let handle = client + .start_workflow(REMOTE_SCOPE, &queue, &queue, json!([])) + .await + .unwrap(); + let original = tokio::time::timeout(Duration::from_secs(10), entered_rx.recv()) + .await + .unwrap() + .expect("actual scoped activity callback entered"); + let initial = history(&handle).await; + let scope = initial["events"] + .as_array() + .unwrap() + .iter() + .find(|row| row["event_type"] == "CancellationScopeOpened") + .unwrap()["payload"]["scope_id"] + .as_str() + .unwrap(); + let accepted = request_native_scope_fixture(&handle, scope); + assert_eq!(accepted, request_native_scope_fixture(&handle, scope)); + let context = + ScopedCancellationContext::from_value(&accepted["payload"]["cancellation"]).unwrap(); + let result = handle + .result_selected_run(WorkflowResultOptions { + timeout: Duration::from_secs(20), + poll_interval: Duration::from_millis(50), + }) + .await + .unwrap(); + assert_eq!(result, context.to_value()); + assert_eq!(dropped.load(Ordering::SeqCst), 1); + let status = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let status = client + .activity_task_status( + &original.task_id, + &original.activity_attempt_id, + &original.lease_owner, + ) + .await + .unwrap(); + if status["cancellation_acknowledgement"]["callback_state"] == "stopped" { + return status; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("actual callback drop must leave its original stop receipt"); + let receipt = &status["cancellation_acknowledgement"]; + assert_eq!(receipt["callback_state"], "stopped"); + assert_eq!(receipt["request_id"], context.request_id()); + assert_eq!( + receipt["root_request_id"], + context.root_context().request_id() + ); + assert_eq!( + receipt["cleanup_deadline_at"], + accepted["payload"]["cancellation"]["root_context"]["cleanup_deadline_at"] + ); + assert_eq!(receipt["received_after_deadline"], false); + assert_eq!(status["can_continue"], false); + let duplicate = client + .acknowledge_activity_cancellation( + &original.task_id, + &original.activity_attempt_id, + &original.lease_owner, + context.request_id(), + ) + .await + .unwrap(); + assert_eq!(duplicate["duplicate"], true); + assert_eq!(duplicate["history_event_id"], receipt["history_event_id"]); + let snapshot = history(&handle).await; + for kind in [ + "CancellationScopeRequested", + "CancellationScopeDeliveryPrepared", + "CancellationScopeDelivered", + "ActivityCancelled", + "ActivityCancellationAcknowledged", + "WorkflowCompleted", + ] { + assert_eq!(count(&snapshot, kind), 1, "{kind}"); + } + for kind in [ + "CooperativeCancellationRequested", + "CooperativeCancellationDelivered", + "WorkflowCancelled", + "WorkflowFailed", + "ActivityHeartbeatRecorded", + "ActivityCompleted", + ] { + assert_eq!(count(&snapshot, kind), 0, "{kind}"); + } + let events = snapshot["events"].as_array().unwrap(); + assert_eq!(count(&snapshot, "TimerCancelled"), usize::from(grouped)); + assert_eq!(count(&snapshot, "TimerScheduled"), 2 + usize::from(grouped)); + assert_eq!(count(&snapshot, "TimerFired"), 2); + let terminal = events + .iter() + .find(|row| row["event_type"] == "WorkflowCompleted") + .unwrap(); + assert!( + chrono::DateTime::parse_from_rfc3339(terminal["timestamp"].as_str().unwrap()) + .unwrap() + .with_timezone(&chrono::Utc) + < context.deadline() + ); + if policy == CancellationPolicy::WaitCancellationCompleted { + let stopped = events + .iter() + .position(|row| row["event_type"] == "ActivityCancellationAcknowledged") + .unwrap(); + let delivered = events + .iter() + .position(|row| row["event_type"] == "CancellationScopeDelivered") + .unwrap(); + assert!(stopped < delivered); + } + assert!(matches!( + original.heartbeat(json!({"late":true})).await, + Err(Error::ActivityExecutionAbandoned(_)) + )); + assert!(matches!(client.complete_activity_task(&original.task_id, + &original.activity_attempt_id, &original.lease_owner, json!("late"), "avro").await, + Err(Error::ActivityTaskRejected(rejection)) if rejection.status == 409)); + assert!(matches!(client.fail_activity_task(&original.task_id, + &original.activity_attempt_id, &original.lease_owner, "late failure", true).await, + Err(Error::ActivityTaskRejected(rejection)) if rejection.status == 409)); + assert_eq!(snapshot, history(&handle).await); + println!( + "Actual scoped callback stop and unchanged parent: {}", + json!({"status":status,"history":snapshot}) + ); + shutdown_tx.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(10), run) + .await + .unwrap() + .unwrap() + .unwrap(); +} + +#[cfg(unix)] +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with the Native scope candidate"] +async fn server_sigkill_scoped_cleanup_replays_before_root_with_original_30_second_deadline() { + use std::os::unix::process::ExitStatusExt; + + let client = client(); + let queue = queue(); + let scratch = ProcessScratch(std::env::temp_dir().join(format!("{queue}-run-scope"))); + std::fs::create_dir(&scratch.0).unwrap(); + let ready = scratch.0.join("ready"); + let mut original = WorkerProcess( + std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "cooperative_process_worker_child", "--nocapture"]) + .env("DURABLE_WORKFLOW_PROCESS_CHILD_QUEUE", &queue) + .env("DURABLE_WORKFLOW_PROCESS_CHILD_PROFILE", "scope-root") + .env("DURABLE_WORKFLOW_PROCESS_CHILD_READY", &ready) + .env( + "DURABLE_WORKFLOW_PROCESS_CHILD_GRANT", + scratch.0.join("unused-grant"), + ) + .spawn() + .unwrap(), + ); + await_child_file(&mut original, &ready).await; + let handle = client + .start_workflow(RUN_SCOPE, &queue, &queue, json!([])) + .await + .unwrap(); + let observe = |target| { + let handle = handle.clone(); + async move { + tokio::time::timeout(Duration::from_secs(15), async { + loop { + let snapshot = history(&handle).await; + if count(&snapshot, "TimerScheduled") >= target { + return snapshot; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("actual process must admit its original timer") + } + }; + observe(1).await; + let accepted = handle + .request_cancellation(CooperativeCancellationOptions { + reason: Some("bounded scope cleanup".into()), + cleanup_timeout_seconds: Some(30), + }) + .await + .unwrap(); + let cleaning = observe(2).await; + assert_eq!(count(&cleaning, "CancellationScopeDelivered"), 1); + assert_eq!(count(&cleaning, "CooperativeCancellationDelivered"), 0); + let snapshot = cleaning["events"] + .as_array() + .unwrap() + .iter() + .find(|row| { + row["event_type"] == "TimerScheduled" + && row["payload"]["cancellation_cleanup"].is_object() + }) + .unwrap()["payload"]["cancellation_cleanup"] + .clone(); + assert_eq!(snapshot.as_object().unwrap().len(), 8); + assert_ne!(snapshot["scope_id"], snapshot["operation_scope_id"]); + assert_eq!( + snapshot["root_request_id"], + accepted.cancellation_request.request_id + ); + assert_eq!( + snapshot["cleanup_deadline_at"], + accepted.cancellation_request.cleanup_deadline_at + ); + original.0.kill().unwrap(); + assert_eq!(original.0.wait().unwrap().signal(), Some(9)); + let duplicate = handle + .request_cancellation(CooperativeCancellationOptions { + reason: Some("duplicate request".into()), + cleanup_timeout_seconds: Some(90), + }) + .await + .unwrap(); + assert!(duplicate.duplicate); + assert_eq!( + duplicate.cancellation_request.request_id, + accepted.cancellation_request.request_id + ); + assert_eq!( + duplicate.cancellation_request.requested_at, + accepted.cancellation_request.requested_at + ); + assert_eq!( + duplicate.cancellation_request.cleanup_deadline_at, + accepted.cancellation_request.cleanup_deadline_at + ); + let replacement = run_scope_worker(&client, &queue, &format!("{queue}-replacement")); + replacement.register().await.unwrap(); + let deadline = + chrono::DateTime::parse_from_rfc3339(&accepted.cancellation_request.cleanup_deadline_at) + .unwrap(); + let wall = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap(); + let now = + chrono::DateTime::::from_timestamp(wall.as_secs() as i64, wall.subsec_nanos()) + .unwrap(); + let remaining = (deadline.with_timezone(&chrono::Utc) - now) + .to_std() + .unwrap(); + let final_history = tokio::time::timeout( + remaining, + tick_until(&replacement, &handle, "WorkflowCancelled"), + ) + .await + .expect("replacement must converge before the original deadline"); + for kind in [ + "CooperativeCancellationRequested", + "CancellationScopeDeliveryPrepared", + "CancellationScopeDelivered", + "CooperativeCancellationDelivered", + "WorkflowCancelled", + ] { + assert_eq!(count(&final_history, kind), 1, "{kind}"); + } + assert_eq!(count(&final_history, "CancellationScopeOpened"), 2); + assert_eq!(count(&final_history, "CancellationScopeRequested"), 2); + assert_eq!(count(&final_history, "TimerScheduled"), 3); + assert_eq!(count(&final_history, "TimerFired"), 2); + assert_eq!(count(&final_history, "WorkflowCompleted"), 0); + assert_eq!(count(&final_history, "WorkflowFailed"), 0); + let events = final_history["events"].as_array().unwrap(); + let recovered = events + .iter() + .find(|row| { + row["event_type"] == "TimerScheduled" + && row["payload"]["cancellation_cleanup"].is_object() + }) + .unwrap(); + assert_eq!(recovered["payload"]["cancellation_cleanup"], snapshot); + let terminal = events + .iter() + .find(|row| row["event_type"] == "WorkflowCancelled") + .unwrap(); + assert!( + chrono::DateTime::parse_from_rfc3339(terminal["timestamp"].as_str().unwrap()).unwrap() + < deadline + ); + println!("Root/scope SIGKILL and original budget: {final_history}"); +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_request_before_claim_replays_canonical_typed_cancellation() { + let client = client(); + let queue = queue(); + let worker = worker(&client, &queue, false, false); + worker.register().await.unwrap(); + let handle = client + .start_workflow(WORKFLOW, &queue, &queue, json!([])) + .await + .unwrap(); + let request = handle + .request_cancellation(CooperativeCancellationOptions::default()) + .await + .unwrap(); + tick_until(&worker, &handle, "WorkflowCancelled").await; + assert_cancelled(&handle, &request.cancellation_request.request_id, false).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_waiting_timer_observes_the_original_request_and_delivery() { + let client = client(); + let queue = queue(); + let worker = worker(&client, &queue, false, false); + worker.register().await.unwrap(); + let handle = client + .start_workflow(WORKFLOW, &queue, &queue, json!([])) + .await + .unwrap(); + tick_until(&worker, &handle, "TimerScheduled").await; + let request = handle + .request_cancellation(CooperativeCancellationOptions::default()) + .await + .unwrap(); + tick_until(&worker, &handle, "WorkflowCancelled").await; + assert_cancelled(&handle, &request.cancellation_request.request_id, false).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_duplicate_request_preserves_identity_and_cleanup_deadline() { + let client = client(); + let queue = queue(); + let worker = worker(&client, &queue, false, false); + worker.register().await.unwrap(); + let handle = client + .start_workflow(WORKFLOW, &queue, &queue, json!([])) + .await + .unwrap(); + let first = handle + .request_cancellation(CooperativeCancellationOptions { + cleanup_timeout_seconds: Some(60), + reason: Some("first".into()), + }) + .await + .unwrap(); + let duplicate = handle + .request_cancellation(CooperativeCancellationOptions { + cleanup_timeout_seconds: Some(120), + reason: Some("duplicate".into()), + }) + .await + .unwrap(); + assert!(duplicate.duplicate); + assert_eq!(first.cancellation_request, duplicate.cancellation_request); + tick_until(&worker, &handle, "WorkflowCancelled").await; + assert_cancelled(&handle, &first.cancellation_request.request_id, false).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_commits_earlier_side_effect_before_cancellation_delivery() { + let client = client(); + let queue = queue(); + let original = worker(&client, &queue, false, true); + original.register().await.unwrap(); + let handle = client + .start_workflow(WORKFLOW, &queue, &queue, json!([])) + .await + .unwrap(); + let request = handle + .request_cancellation(CooperativeCancellationOptions::default()) + .await + .unwrap(); + let prefix = tick_until(&original, &handle, "SideEffectRecorded").await; + assert_eq!(count(&prefix, "CooperativeCancellationDelivered"), 0); + drop(original); + let successor = worker(&client, &queue, false, true); + successor.register().await.unwrap(); + tick_until(&successor, &handle, "WorkflowCancelled").await; + let snapshot = assert_cancelled(&handle, &request.cancellation_request.request_id, false).await; + assert_eq!(count(&snapshot, "SideEffectRecorded"), 1); + let delivery = snapshot["events"] + .as_array() + .unwrap() + .iter() + .find(|event| event["event_type"] == "CooperativeCancellationDelivered") + .unwrap(); + assert_eq!(delivery["payload"]["sequence"], 2); +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_cold_successor_runs_saga_cleanup_before_terminal_cancellation() { + let client = client(); + let queue = queue(); + let observations = Arc::new(Mutex::new(Vec::new())); + let mut original = worker(&client, &queue, true, false); + register_remaining_workflow(&mut original, observations.clone(), "original"); + original.register().await.unwrap(); + let handle = client + .start_workflow(WORKFLOW, &queue, &queue, json!([])) + .await + .unwrap(); + let request = handle + .request_cancellation(CooperativeCancellationOptions { + cleanup_timeout_seconds: Some(30), + ..CooperativeCancellationOptions::default() + }) + .await + .unwrap(); + let pending = tick_until(&original, &handle, "ActivityScheduled").await; + assert_eq!(count(&pending, "ActivityCompleted"), 0); + assert_eq!(count(&pending, "WorkflowCancelled"), 0); + drop(original); + let mut successor = worker(&client, &queue, true, false); + register_remaining_workflow(&mut successor, observations.clone(), "replacement"); + let completed = Arc::new(AtomicUsize::new(0)); + let called = Arc::clone(&completed); + successor.register_activity(UNDO, move |ctx, _| { + let called = Arc::clone(&called); + async move { + ctx.heartbeat(json!({"cleanup":true})).await?; + called.fetch_add(1, Ordering::SeqCst); + Ok(Value::Null) + } + }); + successor.register().await.unwrap(); + tick_until(&successor, &handle, "WorkflowCancelled").await; + let snapshot = assert_cancelled(&handle, &request.cancellation_request.request_id, true).await; + assert_eq!(completed.load(Ordering::SeqCst), 1); + let events = snapshot["events"].as_array().unwrap(); + let event_time = |kind| { + chrono::DateTime::parse_from_rfc3339( + events + .iter() + .find(|event| event["event_type"] == kind) + .unwrap()["timestamp"] + .as_str() + .unwrap(), + ) + .unwrap() + }; + let deadline = + chrono::DateTime::parse_from_rfc3339(&request.cancellation_request.cleanup_deadline_at) + .unwrap(); + let delivery_budget = (deadline - event_time("CooperativeCancellationDelivered")) + .to_std() + .unwrap() + .as_secs_f64(); + let completed_budget = (deadline - event_time("ActivityCompleted")) + .to_std() + .unwrap() + .as_secs_f64(); + let observed = observations.lock().unwrap(); + let deliveries = observed + .iter() + .filter(|entry| entry["phase"] == "delivery") + .collect::>(); + assert!(deliveries.iter().any(|entry| entry["worker"] == "original")); + assert!(deliveries + .iter() + .any(|entry| entry["worker"] == "replacement")); + for entry in &deliveries { + assert_eq!(entry["context"], deliveries[0]["context"]); + assert!((entry["remaining"].as_f64().unwrap() - delivery_budget).abs() < 0.000001); + } + let completed = observed + .iter() + .find(|entry| entry["phase"] == "completed") + .unwrap(); + assert_eq!(completed["context"], deliveries[0]["context"]); + assert!((completed["remaining"].as_f64().unwrap() - completed_budget).abs() < 0.000001); + assert!(0.0 < completed_budget && completed_budget < delivery_budget); + assert!(event_time("WorkflowCancelled") < deadline); + assert_eq!(count(&snapshot, "MemoUpserted"), 1); + eprintln!( + "cold cleanup remaining-time observations: {}", + json!(*observed) + ); + eprintln!("cold cleanup remaining-time history: {snapshot}"); +} + +fn register_remaining_workflow( + worker: &mut Worker, + observations: Arc>>, + label: &'static str, +) { + worker.register_workflow(WORKFLOW, move |ctx, _| { + let observations = observations.clone(); + async move { + let mut saga = ctx.saga(); + saga.add_compensation(UNDO, json!([]))?; + let result = ctx.sleep(Duration::from_secs(300)).await; + let context = ctx.cancellation_context()?.expect("committed rich delivery"); + let remaining = context.remaining()?; + observations.lock().unwrap().push(json!({"worker":label,"phase":"delivery","context":context.to_value(),"remaining":remaining.as_secs_f64()})); + ctx.upsert_memo(json!({"phase":"cleanup"}))?; + assert_eq!(context.remaining()?, remaining, "synchronous memo preserves the consumed budget"); + let finished = saga.finish(result).await; + observations.lock().unwrap().push(json!({"worker":label,"phase":"completed","context":context.to_value(),"remaining":context.remaining()?.as_secs_f64()})); + finished?; + Ok(Value::Null) + } + }); +} + +async fn managed_remote_cancellation(user_heartbeat: bool, policy: Option) { + let client = client(); + let queue = queue(); + let dropped = Arc::new(AtomicUsize::new(0)); + let completed = Arc::new(AtomicUsize::new(0)); + let (entered_tx, mut entered_rx) = tokio::sync::mpsc::unbounded_channel::(); + let (heartbeat_tx, mut heartbeat_rx) = tokio::sync::mpsc::unbounded_channel(); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let mut worker = Worker::new(client.clone(), &queue) + .worker_id(format!("{queue}-managed")) + .cooperative_cancellation(true) + .max_concurrent_workflow_tasks(1) + .max_concurrent_activity_tasks(1) + .poll_timeout(Duration::from_secs(1)) + .on_worker_heartbeat(move |observation| { + if observation.acknowledgement["acknowledged"] == true { + let _ = heartbeat_tx.send(()); + } + }); + worker.register_workflow(WORKFLOW, move |ctx, _| async move { + let mut saga = ctx.saga(); + saga.add_compensation(UNDO, json!([]))?; + let result = match policy { + Some(policy) => { + ctx.activity_with_options( + BLOCKED, + ActivityOptions::new().cancellation_policy(policy), + json!([]), + ) + .await + } + None => ctx.activity(BLOCKED, json!([])).await, + }; + saga.finish(result).await?; + Ok(Value::Null) + }); + let callback_dropped = Arc::clone(&dropped); + worker.register_activity(BLOCKED, move |ctx, _| { + let entered_tx = entered_tx.clone(); + let callback_dropped = Arc::clone(&callback_dropped); + async move { + let _drop = CallbackDrop(callback_dropped); + if user_heartbeat { + ctx.heartbeat(json!({"qualification":"blocked"})).await?; + } + entered_tx.send(ctx.clone()).unwrap(); + if user_heartbeat { + loop { + tokio::time::sleep(Duration::from_millis(100)).await; + ctx.heartbeat(json!({"qualification":"blocked"})).await?; + } + } else { + std::future::pending::>().await + } + } + }); + let cleanup_completed = Arc::clone(&completed); + worker.register_activity(UNDO, move |ctx, _| { + let cleanup_completed = Arc::clone(&cleanup_completed); + async move { + ctx.heartbeat(json!({"qualification":"cleanup"})).await?; + cleanup_completed.fetch_add(1, Ordering::SeqCst); + Ok(Value::Null) + } + }); + let run = tokio::spawn(async move { + worker + .run_until(async { + let _ = shutdown_rx.await; + }) + .await + }); + tokio::time::timeout(Duration::from_secs(10), heartbeat_rx.recv()) + .await + .unwrap() + .expect("actual accepted worker heartbeat"); + let handle = client + .start_workflow(WORKFLOW, &queue, &queue, json!([])) + .await + .unwrap(); + let original = tokio::time::timeout(Duration::from_secs(10), entered_rx.recv()) + .await + .unwrap() + .expect("actual blocked callback entered"); + let request = handle + .request_cancellation(CooperativeCancellationOptions { + cleanup_timeout_seconds: Some(30), + ..CooperativeCancellationOptions::default() + }) + .await + .unwrap(); + let result = handle + .result_selected_run(WorkflowResultOptions { + timeout: Duration::from_secs(20), + poll_interval: Duration::from_millis(50), + }) + .await; + assert!( + matches!(result, Err(Error::WorkflowCancelled(_))), + "managed result: {result:?}" + ); + let mut snapshot = + assert_cancelled(&handle, &request.cancellation_request.request_id, true).await; + assert_eq!( + dropped.load(Ordering::SeqCst), + 1, + "blocked callback future must drop" + ); + assert_eq!( + completed.load(Ordering::SeqCst), + 1, + "cleanup runs once after freeing its only activity slot" + ); + assert_eq!(count(&snapshot, "ActivityCancelled"), 1); + let original_progress = snapshot["events"] + .as_array() + .unwrap() + .iter() + .filter(|event| { + event["event_type"] == "ActivityHeartbeatRecorded" + && event["payload"]["activity_type"] == BLOCKED + }) + .count(); + assert_eq!( + original_progress > 0, + user_heartbeat, + "readonly checks must not record progress" + ); + assert!( + matches!( + original.heartbeat(json!({"late":true})).await, + Err(Error::ActivityExecutionAbandoned(_)) + ), + "cloned context remains abandoned" + ); + if std::env::var("DURABLE_WORKFLOW_NATIVE_SOURCE_QUALIFICATION").as_deref() == Ok("1") { + let status = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let status = client + .activity_task_status( + &original.task_id, + &original.activity_attempt_id, + &original.lease_owner, + ) + .await + .unwrap(); + if status["cancellation_acknowledgement"]["callback_state"] == "stopped" { + break status; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("dropped callback must leave a durable stop receipt"); + let receipt = &status["cancellation_acknowledgement"]; + assert_eq!( + receipt["request_id"], + request.cancellation_request.request_id + ); + assert_eq!( + receipt["root_request_id"], + request.cancellation_request.request_id + ); + assert_eq!( + receipt["cleanup_deadline_at"], + request.cancellation_request.cleanup_deadline_at + ); + assert_eq!(receipt["received_after_deadline"], false); + assert_eq!(status["heartbeat_recorded"], false); + assert_eq!(status["can_continue"], false); + let duplicate = client + .acknowledge_activity_cancellation( + &original.task_id, + &original.activity_attempt_id, + &original.lease_owner, + &request.cancellation_request.request_id, + ) + .await + .unwrap(); + assert_eq!(duplicate["duplicate"], true); + assert_eq!(duplicate["history_event_id"], receipt["history_event_id"]); + snapshot = history(&handle).await; + assert_eq!(count(&snapshot, "ActivityCancellationAcknowledged"), 1); + if let Some(policy) = policy { + let events = snapshot["events"].as_array().unwrap(); + let scheduled = events + .iter() + .find(|event| event["event_type"] == "ActivityScheduled") + .unwrap(); + let expected = match policy { + CancellationPolicy::TryCancel => "try_cancel", + CancellationPolicy::WaitCancellationCompleted => "wait_cancellation_completed", + CancellationPolicy::Abandon => unreachable!(), + _ => unreachable!(), + }; + assert_eq!( + scheduled["payload"]["activity"]["cancellation_policy"], + expected + ); + if policy == CancellationPolicy::WaitCancellationCompleted { + let stop = events + .iter() + .position(|event| event["event_type"] == "ActivityCancellationAcknowledged") + .unwrap(); + let delivery = events + .iter() + .position(|event| event["event_type"] == "CooperativeCancellationDelivered") + .unwrap(); + assert!( + stop < delivery, + "Wait must record actual callback drop before workflow delivery" + ); + } + } + let recorded = snapshot["events"] + .as_array() + .unwrap() + .iter() + .find(|event| event["event_type"] == "ActivityCancellationAcknowledged") + .unwrap(); + assert_eq!(recorded["payload"]["callback_state"], "stopped"); + assert_eq!(recorded["payload"]["evidence_source"], "activity_worker"); + assert_eq!( + recorded["payload"]["activity_attempt_id"], + original.activity_attempt_id + ); + assert_eq!(recorded["payload"]["request_id"], receipt["request_id"]); + assert_eq!( + recorded["payload"]["root_request_id"], + receipt["root_request_id"] + ); + assert_eq!( + recorded["payload"]["cleanup_deadline_at"], + receipt["cleanup_deadline_at"] + ); + assert_eq!( + recorded["payload"]["cancellation_history_event_id"], + receipt["cancellation_history_event_id"] + ); + eprintln!("remote stop receipt: user_heartbeat={user_heartbeat}, status={status}, duplicate={duplicate}, history={recorded}"); + } + let completion = client + .complete_activity_task( + &original.task_id, + &original.activity_attempt_id, + &original.lease_owner, + json!("late"), + "avro", + ) + .await; + assert!( + matches!(completion, Err(Error::ActivityTaskRejected(rejection)) if rejection.status == 409), + "Server must reject the cancelled attempt's late completion" + ); + let failure = client + .fail_activity_task( + &original.task_id, + &original.activity_attempt_id, + &original.lease_owner, + "late failure", + true, + ) + .await; + assert!( + matches!(failure, Err(Error::ActivityTaskRejected(rejection)) if rejection.status == 409), + "Server must reject the cancelled attempt's late failure" + ); + assert_eq!( + snapshot, + history(&handle).await, + "late publication changed canonical history" + ); + shutdown_tx.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(10), run) + .await + .unwrap() + .unwrap() + .unwrap(); +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_managed_worker_fences_blocked_callback_without_user_heartbeats() { + managed_remote_cancellation(false, None).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_managed_worker_fences_blocked_callback_with_user_heartbeats() { + managed_remote_cancellation(true, None).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_explicit_try_cancellation_stops_callback_without_user_heartbeats() { + managed_remote_cancellation(false, Some(CancellationPolicy::TryCancel)).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_explicit_wait_records_callback_stop_before_delivery() { + managed_remote_cancellation(false, Some(CancellationPolicy::WaitCancellationCompleted)).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_bounded_abandon_completes_independent_activity_after_parent_cancellation() { + let client = client(); + let queue = queue(); + let dropped = Arc::new(AtomicUsize::new(0)); + let (entered_tx, mut entered_rx) = tokio::sync::mpsc::unbounded_channel::(); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let mut worker = Worker::new(client.clone(), &queue) + .worker_id(format!("{queue}-abandon")) + .cooperative_cancellation(true) + .max_concurrent_workflow_tasks(1) + .max_concurrent_activity_tasks(2) + .poll_timeout(Duration::from_secs(1)); + worker.register_workflow(WORKFLOW, |ctx, _| async move { + let mut saga = ctx.saga(); + saga.add_compensation(UNDO, json!([]))?; + let result = ctx + .activity_with_options( + BLOCKED, + ActivityOptions::new() + .cancellation_policy(CancellationPolicy::Abandon) + .schedule_to_close_timeout(Duration::from_secs(60)), + json!([]), + ) + .await; + saga.finish(result).await?; + Ok(Value::Null) + }); + let callback_dropped = Arc::clone(&dropped); + worker.register_activity(BLOCKED, move |ctx, _| { + let entered_tx = entered_tx.clone(); + let callback_dropped = Arc::clone(&callback_dropped); + async move { + let _drop = CallbackDrop(callback_dropped); + entered_tx.send(ctx).unwrap(); + tokio::time::sleep(Duration::from_secs(25)).await; + Ok(json!("independent-completion")) + } + }); + worker.register_activity(UNDO, |_ctx, _| async { Ok(Value::Null) }); + let run = tokio::spawn(async move { + worker + .run_until(async { + let _ = shutdown_rx.await; + }) + .await + }); + let handle = client + .start_workflow(WORKFLOW, &queue, &queue, json!([])) + .await + .unwrap(); + let original = tokio::time::timeout(Duration::from_secs(10), entered_rx.recv()) + .await + .unwrap() + .unwrap(); + let request = handle + .request_cancellation(CooperativeCancellationOptions { + cleanup_timeout_seconds: Some(30), + ..CooperativeCancellationOptions::default() + }) + .await + .unwrap(); + let duplicate = handle + .request_cancellation(CooperativeCancellationOptions { + cleanup_timeout_seconds: Some(300), + ..CooperativeCancellationOptions::default() + }) + .await + .unwrap(); + assert!(duplicate.duplicate); + assert_eq!(request.cancellation_request, duplicate.cancellation_request); + let snapshot = assert_cancelled(&handle, &request.cancellation_request.request_id, true).await; + assert_eq!( + dropped.load(Ordering::SeqCst), + 0, + "Abandon must preserve the callback after parent closure" + ); + assert_eq!(count(&snapshot, "ActivityCancelled"), 0); + assert_eq!(count(&snapshot, "ActivityCancellationAcknowledged"), 0); + let scheduled = snapshot["events"] + .as_array() + .unwrap() + .iter() + .find(|event| event["event_type"] == "ActivityScheduled") + .unwrap(); + assert_eq!( + scheduled["payload"]["activity"]["cancellation_policy"], + "abandon" + ); + let total_deadline = scheduled["payload"]["activity"]["schedule_to_close_deadline_at"].clone(); + assert!(total_deadline.is_string()); + let status = client + .activity_task_status( + &original.task_id, + &original.activity_attempt_id, + &original.lease_owner, + ) + .await + .unwrap(); + assert_eq!(status["can_continue"], true); + let snapshot = tokio::time::timeout(Duration::from_secs(35), async { + loop { + let snapshot = history(&handle).await; + if snapshot["events"].as_array().unwrap().iter().any(|event| { + event["event_type"] == "ActivityCompleted" + && event["payload"]["activity_type"] == BLOCKED + }) { + break snapshot; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await + .expect("independent callback must complete within its original lifetime"); + let completed: Vec<_> = snapshot["events"] + .as_array() + .unwrap() + .iter() + .filter(|event| { + event["event_type"] == "ActivityCompleted" + && event["payload"]["activity_type"] == BLOCKED + }) + .collect(); + assert_eq!(completed.len(), 1); + let envelope: durable_workflow::PayloadEnvelope = + serde_json::from_value(completed[0]["payload"]["result"].clone()).unwrap(); + assert_eq!( + durable_workflow::decode_payload::(&envelope).unwrap(), + "independent-completion" + ); + assert_eq!( + completed[0]["payload"]["activity"]["schedule_to_close_deadline_at"], + total_deadline + ); + assert_eq!( + completed[0]["payload"]["activity_attempt_id"], + original.activity_attempt_id + ); + assert_eq!(count(&snapshot, "WorkflowCancelled"), 1); + assert_eq!(count(&snapshot, "WorkflowCompleted"), 0); + assert_eq!(count(&snapshot, "ActivityCancellationAcknowledged"), 0); + for outcome in [ + client + .complete_activity_task( + &original.task_id, + &original.activity_attempt_id, + &original.lease_owner, + json!("stale"), + "avro", + ) + .await, + client + .fail_activity_task( + &original.task_id, + &original.activity_attempt_id, + &original.lease_owner, + "stale", + true, + ) + .await, + ] { + assert!( + matches!(outcome, Err(Error::ActivityTaskRejected(rejection)) if rejection.status == 409) + ); + } + assert_eq!(snapshot, history(&handle).await); + eprintln!( + "Bounded remote Abandon: root={}, total_deadline={}, completed={}", + request.cancellation_request.request_id, total_deadline, completed[0] + ); + shutdown_tx.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(10), run) + .await + .unwrap() + .unwrap() + .unwrap(); +} + +fn replay_worker(client: &Client, queue: &str, mode: &'static str) -> Worker { + let mut worker = Worker::new(client.clone(), queue) + .worker_id(format!("{queue}-{}", durable_workflow::Uuid::new_v4())) + .cooperative_cancellation(true) + .poll_timeout(Duration::from_secs(1)); + worker.register_workflow(REPLAY, move |ctx, _| async move { + let predicate_ctx = ctx.clone(); + let condition = || ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"); + match mode { + "condition" => { + ctx.wait_condition(condition(), move || { + Ok(predicate_ctx.signals("vote")?.len() >= 2) + }) + .await?; + } + "parallel" => { + ctx.parallel(vec![ + ParallelOperation::timer(Duration::from_secs(300)), + ParallelOperation::group(vec![ + ParallelOperation::signal("never"), + ParallelOperation::condition(condition(), move || { + Ok(predicate_ctx.signals("vote")?.len() >= 2) + }), + ]), + ]) + .await?; + } + "selection" => { + ctx.select_keyed(vec![ + ("timer", ParallelOperation::timer(Duration::from_secs(300))), + ( + "votes", + ParallelOperation::condition(condition(), move || { + Ok(predicate_ctx.signals("vote")?.len() >= 2) + }), + ), + ]) + .await?; + } + "winner" => { + let selected = ctx + .select_keyed(vec![ + ("slow", ParallelOperation::signal("never")), + ("fast", ParallelOperation::timer(Duration::from_secs(1))), + ]) + .await?; + assert_eq!(selected.key, SelectionKey::Name("fast".into())); + let slow = selected + .handle(&SelectionKey::Name("slow".into())) + .unwrap() + .clone(); + assert_eq!(selected.into_result()?, ParallelResult::Timer); + slow.await_result().await?; + } + _ => panic!("unknown fixture mode"), + } + Ok(Value::Null) + }); + worker +} + +async fn reopened_condition_cancellation(mode: &'static str, span: u64) { + let client = client(); + let queue = queue(); + let original = replay_worker(&client, &queue, mode); + original.register().await.unwrap(); + let handle = client + .start_workflow(REPLAY, &queue, &queue, json!([])) + .await + .unwrap(); + let first = tick_until(&original, &handle, "ConditionWaitOpened").await; + assert_eq!(count(&first, "ConditionWaitOpened"), 1); + handle + .signal_selected_run("vote", json!(["first"])) + .await + .unwrap(); + let reopened = tick_until_count(&original, &handle, "ConditionWaitOpened", 2).await; + assert_eq!(count(&reopened, "ConditionWaitOpened"), 2); + assert_eq!(count(&reopened, "ConditionWaitSatisfied"), 1); + assert_eq!(count(&reopened, "SelectionResolved"), 0); + let opens = reopened["events"] + .as_array() + .unwrap() + .iter() + .filter(|event| event["event_type"] == "ConditionWaitOpened") + .collect::>(); + assert_eq!( + opens[0]["payload"]["condition_wait_occurrence_id"], + opens[1]["payload"]["condition_wait_occurrence_id"] + ); + assert_ne!( + opens[0]["payload"]["sequence"], + opens[1]["payload"]["sequence"] + ); + let pending_sequence = opens[1]["payload"]["sequence"].as_u64().unwrap(); + let request = handle + .request_cancellation(CooperativeCancellationOptions::default()) + .await + .unwrap(); + drop(original); + let successor = replay_worker(&client, &queue, mode); + successor.register().await.unwrap(); + tick_until(&successor, &handle, "WorkflowCancelled").await; + let snapshot = assert_cancelled(&handle, &request.cancellation_request.request_id, false).await; + let events = snapshot["events"].as_array().unwrap(); + let delivery_index = events + .iter() + .position(|event| event["event_type"] == "CooperativeCancellationDelivered") + .unwrap(); + // Delivery consumes the latest physical wait under the original authored + // identity. It must not create another opening at either side of delivery. + assert_eq!(count(&snapshot, "ConditionWaitOpened"), 2); + for event in events + .iter() + .filter(|event| event["event_type"] == "ConditionWaitOpened") + { + assert_eq!( + event["payload"]["condition_wait_occurrence_id"], + opens[0]["payload"]["condition_wait_occurrence_id"], + "physical prefix reopen changed authored identity: {snapshot}" + ); + } + assert!( + events[delivery_index + 1..] + .iter() + .all(|event| event["event_type"] != "ConditionWaitOpened"), + "condition reopened after cancellation delivery: {snapshot}" + ); + eprintln!("cooperative reopened condition {mode}: {snapshot}"); + assert_eq!(count(&snapshot, "SelectionResolved"), 0); + let delivery = snapshot["events"] + .as_array() + .unwrap() + .iter() + .find(|event| event["event_type"] == "CooperativeCancellationDelivered") + .unwrap(); + assert_eq!( + delivery["payload"]["sequence_span"].as_u64().unwrap_or(1), + span + ); + if mode == "condition" { + assert_eq!(delivery["payload"]["call_kind"], "condition"); + assert_eq!(delivery["payload"]["sequence"], pending_sequence); + } else { + assert_eq!(delivery["payload"]["call_kind"], "parallel"); + assert_eq!(delivery["payload"]["sequence"], 1); + } +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_cold_condition_delivery_uses_the_pending_physical_reopen() { + reopened_condition_cancellation("condition", 1).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_cold_nested_parallel_cancellation_replays_reopened_condition() { + reopened_condition_cancellation("parallel", 3).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_cold_selection_cancellation_replays_reopened_condition() { + reopened_condition_cancellation("selection", 2).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_cold_selection_preserves_committed_winner_before_loser_cancellation() { + let client = client(); + let queue = queue(); + let original = replay_worker(&client, &queue, "winner"); + original.register().await.unwrap(); + let handle = client + .start_workflow(REPLAY, &queue, &queue, json!([])) + .await + .unwrap(); + let winner = tick_until(&original, &handle, "SelectionResolved").await; + assert_eq!(count(&winner, "SelectionResolved"), 1); + assert_eq!(count(&winner, "WorkflowCompleted"), 0); + let request = handle + .request_cancellation(CooperativeCancellationOptions::default()) + .await + .unwrap(); + drop(original); + let successor = replay_worker(&client, &queue, "winner"); + successor.register().await.unwrap(); + tick_until(&successor, &handle, "WorkflowCancelled").await; + let snapshot = assert_cancelled(&handle, &request.cancellation_request.request_id, false).await; + assert_eq!(count(&snapshot, "SelectionResolved"), 1); + let delivery = snapshot["events"] + .as_array() + .unwrap() + .iter() + .find(|event| event["event_type"] == "CooperativeCancellationDelivered") + .unwrap(); + assert_eq!(delivery["payload"]["call_kind"], "selection_handle"); + assert_eq!(delivery["payload"]["operation_sequence"], 1); + assert_eq!( + delivery["payload"]["operation_sequence_span"] + .as_u64() + .unwrap_or(1), + 1 + ); +} diff --git a/tests/fixtures/committed-scope-delivery.json b/tests/fixtures/committed-scope-delivery.json new file mode 100644 index 0000000..b58f412 --- /dev/null +++ b/tests/fixtures/committed-scope-delivery.json @@ -0,0 +1,695 @@ +{ + "native_commit": "6d484654c7a966770cd6bbce90d9007a1c75e0ba", + "unshielded": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "outer_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 5, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 6, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 3, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:05.250000Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 7, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 3, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:07.500000Z" + } + ] + }, + "shielded": { + "task": { + "workflow_id": "scope-fixture-shielded", + "run_id": "01m43xf6c15h003132br5fcabx" + }, + "outer_scope_id": "01M43XF6CN7YF8HKCNJ5RQX0CN", + "scope_id": "01M43XF6CSSPQNT6VMA6WKG7Z6", + "history": [ + { + "id": "01m43xf6c4f0n5d26xy0j05w1s", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf6c2avkvv3aybv47qtvp", + "workflow_instance_id": "scope-fixture-shielded", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf6c2avkvv3aybv47qtvp", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf6c15h003132br5fcabx", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6c5gcfdww4t1z91nzhx", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-shielded", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "workflow_command_id": "01m43xf6c2avkvv3aybv47qtvp", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf6c2avkvv3aybv47qtvp", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf6c15h003132br5fcabx", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6cpajmgwqsrxxcvsv7s", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "sequence": 1, + "scope_id": "01M43XF6CN7YF8HKCNJ5RQX0CN", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf6c6ctyp5qgqscpch981", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6ctdkeyg4bxp8shzkc4", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "sequence": 2, + "scope_id": "01M43XF6CSSPQNT6VMA6WKG7Z6", + "parent_scope_id": "01M43XF6CN7YF8HKCNJ5RQX0CN", + "shield_parent": true, + "task": { + "id": "01m43xf6c6ctyp5qgqscpch981", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6czvbndcpec0kab3nwb", + "namespace": "sdk-scope-fixture", + "sequence": 5, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "scope_id": "01M43XF6CSSPQNT6VMA6WKG7Z6", + "parent_scope_id": null, + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "root_request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "root_workflow_instance_id": "scope-fixture-shielded", + "root_workflow_run_id": "01m43xf6c15h003132br5fcabx", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "workflow_instance_id": "scope-fixture-shielded", + "workflow_run_id": "01m43xf6c15h003132br5fcabx" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "workflow_instance_id": "scope-fixture-shielded", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "scope_id": "01M43XF6CSSPQNT6VMA6WKG7Z6", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6dg25ypwscmh8dfft10", + "namespace": "sdk-scope-fixture", + "sequence": 6, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "scope_id": "01M43XF6CSSPQNT6VMA6WKG7Z6", + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "root_request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "root_workflow_instance_id": "scope-fixture-shielded", + "root_workflow_run_id": "01m43xf6c15h003132br5fcabx", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "workflow_instance_id": "scope-fixture-shielded", + "workflow_run_id": "01m43xf6c15h003132br5fcabx" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "workflow_instance_id": "scope-fixture-shielded", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "scope_id": "01M43XF6CSSPQNT6VMA6WKG7Z6", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 3, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m43xf6c6ctyp5qgqscpch981", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:05.250000Z" + }, + { + "id": "01m43xf6eb0t411jspfy8yv3n1", + "namespace": "sdk-scope-fixture", + "sequence": 7, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "scope_id": "01M43XF6CSSPQNT6VMA6WKG7Z6", + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "root_request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "root_workflow_instance_id": "scope-fixture-shielded", + "root_workflow_run_id": "01m43xf6c15h003132br5fcabx", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "workflow_instance_id": "scope-fixture-shielded", + "workflow_run_id": "01m43xf6c15h003132br5fcabx" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "workflow_instance_id": "scope-fixture-shielded", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "scope_id": "01M43XF6CSSPQNT6VMA6WKG7Z6", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 3, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6dg25ypwscmh8dfft10", + "task": { + "id": "01m43xf6c6ctyp5qgqscpch981", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:07.500000Z" + } + ] + } +} diff --git a/tests/fixtures/committed-scope-descendants.json b/tests/fixtures/committed-scope-descendants.json new file mode 100644 index 0000000..c96b3b5 --- /dev/null +++ b/tests/fixtures/committed-scope-descendants.json @@ -0,0 +1,3512 @@ +{ + "_provenance": { + "native_commit": "0471f1ebbb98c61fe53663086d62dc2db74db9e3", + "history_store": "isolated SQLite synthetic accepted-history projection inputs", + "effect_authority": false, + "producers": [ + "ScopedCancellationContext::forDescendant", + "CancellationScopeDescendants::members", + "CancellationScopeDelivery::activityMembers", + "ScopedTimerCancellation::members", + "ScopedWaitCancellation::members", + "ScopedChildCancellation::members" + ], + "input_fixture_sha256": "85320e25c7c19521bca4a1cff138911df1f19f76471ea033e961695e8ed761be" + }, + "timer": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scopes": { + "outer": "outer-scope", + "parent": "parent-scope", + "shield": "shield-scope", + "shield-child": "shield-child-scope", + "child": "child-scope", + "grandchild": "grandchild-scope" + }, + "layout": "timer", + "contexts": { + "parent": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "child": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "grandchild": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "opened-outer", + "sequence": 3, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "outer-scope", + "parent_scope_id": "root", + "shield_parent": false + } + }, + { + "id": "opened-parent", + "sequence": 4, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "parent-scope", + "parent_scope_id": "outer-scope", + "shield_parent": false + } + }, + { + "id": "opened-shield", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 3, + "scope_id": "shield-scope", + "parent_scope_id": "parent-scope", + "shield_parent": true + } + }, + { + "id": "opened-shield-child", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 4, + "scope_id": "shield-child-scope", + "parent_scope_id": "shield-scope", + "shield_parent": false + } + }, + { + "id": "opened-child", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 5, + "scope_id": "child-scope", + "parent_scope_id": "parent-scope", + "shield_parent": false + } + }, + { + "id": "opened-grandchild", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 6, + "scope_id": "grandchild-scope", + "parent_scope_id": "child-scope", + "shield_parent": false + } + }, + { + "id": "prior-scheduled", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "grandchild-scope", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "grandchild-scope", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "grandchild-scope", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-timer", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 8, + "cancellation_scope_id": "grandchild-scope", + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "requested-parent", + "sequence": 12, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "parent_scope_id": null, + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "requested-child", + "sequence": 13, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "request_id": "child-request", + "parent_scope_id": "parent-scope", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "requested-grandchild", + "sequence": 14, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "request_id": "grandchild-request", + "parent_scope_id": "child-scope", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "ancestor-prepared", + "namespace": "sdk-scope-fixture", + "sequence": 15, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 8, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [ + { + "scope_id": "child-scope", + "parent_scope_id": "parent-scope", + "scope_history_event_id": "opened-child", + "request_history_event_id": "requested-child", + "request_id": "child-request", + "propagation_history_event_id": "requested-child", + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [] + }, + { + "scope_id": "grandchild-scope", + "parent_scope_id": "child-scope", + "scope_history_event_id": "opened-grandchild", + "request_history_event_id": "requested-grandchild", + "request_id": "grandchild-request", + "propagation_history_event_id": "requested-grandchild", + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [ + { + "sequence": 7, + "activity_execution_id": "prior-id", + "descriptor_hash": "f60e9f5c60924a14222285670b881c3e456b61bc18e79addfd3d3308ef5ccbae" + } + ], + "timer_members": [ + { + "sequence": 8, + "timer_id": "original-timer", + "descriptor_hash": "4ceb714bc8fed33200f9b15a44469e5d70db95e41357293fa6a210eeb31d53c8" + } + ], + "wait_members": [], + "child_members": [] + } + ], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "ancestor-delivered", + "namespace": "sdk-scope-fixture", + "sequence": 16, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 8, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "preparation_history_event_id": "ancestor-prepared", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "group": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scopes": { + "outer": "outer-scope", + "parent": "parent-scope", + "shield": "shield-scope", + "shield-child": "shield-child-scope", + "child": "child-scope", + "grandchild": "grandchild-scope" + }, + "layout": "group", + "contexts": { + "parent": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "child": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "grandchild": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "opened-outer", + "sequence": 3, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "outer-scope", + "parent_scope_id": "root", + "shield_parent": false + } + }, + { + "id": "opened-parent", + "sequence": 4, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "parent-scope", + "parent_scope_id": "outer-scope", + "shield_parent": false + } + }, + { + "id": "opened-shield", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 3, + "scope_id": "shield-scope", + "parent_scope_id": "parent-scope", + "shield_parent": true + } + }, + { + "id": "opened-shield-child", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 4, + "scope_id": "shield-child-scope", + "parent_scope_id": "shield-scope", + "shield_parent": false + } + }, + { + "id": "opened-child", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 5, + "scope_id": "child-scope", + "parent_scope_id": "parent-scope", + "shield_parent": false + } + }, + { + "id": "opened-grandchild", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 6, + "scope_id": "grandchild-scope", + "parent_scope_id": "child-scope", + "shield_parent": false + } + }, + { + "id": "prior-scheduled", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "grandchild-scope", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "grandchild-scope", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "grandchild-scope", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "group-activity", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 8, + "cancellation_scope_id": "grandchild-scope", + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "grandchild-scope", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "group-timer", + "sequence": 12, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 9, + "cancellation_scope_id": "grandchild-scope", + "parallel_group_id": "parallel-calls:9:2", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 9, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "parallel-calls:9:2", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 9, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ], + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "group-child", + "sequence": 13, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 10, + "cancellation_scope_id": "grandchild-scope", + "parallel_group_id": "parallel-calls:9:2", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 9, + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 2 + }, + { + "parallel_group_id": "parallel-calls:9:2", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 9, + "parallel_group_size": 2, + "parallel_group_index": 1 + } + ], + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "group-condition", + "sequence": 14, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "grandchild-scope", + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "condition_wait_id": "original-wait", + "condition_key": "ready", + "timeout_seconds": 30, + "condition_wait_occurrence_id": "original-occurrence", + "condition_definition_fingerprint": "sha256:3b2b1ad635030d842ab378e8bf021f4613ee6d221dafb4743565467800def433" + } + }, + { + "id": "condition-timeout", + "sequence": 15, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "grandchild-scope", + "timer_id": "original-timeout", + "timer_kind": "condition_timeout", + "condition_wait_id": "original-wait", + "condition_wait_occurrence_id": "original-occurrence", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:32.123456Z" + } + }, + { + "id": "requested-parent", + "sequence": 16, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "parent_scope_id": null, + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "requested-child", + "sequence": 17, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "request_id": "child-request", + "parent_scope_id": "parent-scope", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "requested-grandchild", + "sequence": 18, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "request_id": "grandchild-request", + "parent_scope_id": "child-scope", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "ancestor-prepared", + "namespace": "sdk-scope-fixture", + "sequence": 19, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 8, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [ + { + "scope_id": "child-scope", + "parent_scope_id": "parent-scope", + "scope_history_event_id": "opened-child", + "request_history_event_id": "requested-child", + "request_id": "child-request", + "propagation_history_event_id": "requested-child", + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [] + }, + { + "scope_id": "grandchild-scope", + "parent_scope_id": "child-scope", + "scope_history_event_id": "opened-grandchild", + "request_history_event_id": "requested-grandchild", + "request_id": "grandchild-request", + "propagation_history_event_id": "requested-grandchild", + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [ + { + "sequence": 7, + "activity_execution_id": "prior-id", + "descriptor_hash": "f60e9f5c60924a14222285670b881c3e456b61bc18e79addfd3d3308ef5ccbae" + }, + { + "sequence": 8, + "activity_execution_id": "original-id", + "descriptor_hash": "1f575cec410a08e47f0bd32b2a95b40f317eacb774f17ec542f4a85dd244503c" + } + ], + "timer_members": [ + { + "sequence": 9, + "timer_id": "original-timer", + "descriptor_hash": "bb99cb1a1ff6ac1a6923dd29afbc845055f997356283f437b56dbd0f4d42383a" + }, + { + "sequence": 11, + "timer_id": "original-timeout", + "descriptor_hash": "72ff6f4ce79da9fa15a173cd173c2ffc79f791f1b3af7a44f2cdad0cdbf9bc0c" + } + ], + "wait_members": [ + { + "kind": "condition", + "sequence": 11, + "wait_id": "original-wait", + "timer_id": "original-timeout", + "descriptor_hash": "c51fd1d0e85fb66892f7b623278b7c89da4bccb287ff4397494ac596adf8da84" + } + ], + "child_members": [ + { + "sequence": 10, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "a482bfd51e2c03fe2a285880a24e614586e621aa3ad21fb67db151503514834e" + } + ] + } + ], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "ancestor-delivered", + "namespace": "sdk-scope-fixture", + "sequence": 20, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 8, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "preparation_history_event_id": "ancestor-prepared", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "partial-group": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scopes": { + "outer": "outer-scope", + "parent": "parent-scope", + "shield": "shield-scope", + "shield-child": "shield-child-scope", + "child": "child-scope", + "grandchild": "grandchild-scope" + }, + "layout": "partial-group", + "contexts": { + "parent": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "child": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "grandchild": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "opened-outer", + "sequence": 3, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "outer-scope", + "parent_scope_id": "root", + "shield_parent": false + } + }, + { + "id": "opened-parent", + "sequence": 4, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "parent-scope", + "parent_scope_id": "outer-scope", + "shield_parent": false + } + }, + { + "id": "opened-shield", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 3, + "scope_id": "shield-scope", + "parent_scope_id": "parent-scope", + "shield_parent": true + } + }, + { + "id": "opened-shield-child", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 4, + "scope_id": "shield-child-scope", + "parent_scope_id": "shield-scope", + "shield_parent": false + } + }, + { + "id": "opened-child", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 5, + "scope_id": "child-scope", + "parent_scope_id": "parent-scope", + "shield_parent": false + } + }, + { + "id": "opened-grandchild", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 6, + "scope_id": "grandchild-scope", + "parent_scope_id": "child-scope", + "shield_parent": false + } + }, + { + "id": "prior-scheduled", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "grandchild-scope", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "grandchild-scope", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "grandchild-scope", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "group-activity", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 8, + "cancellation_scope_id": "grandchild-scope", + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "grandchild-scope", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "group-timer", + "sequence": 12, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 9, + "cancellation_scope_id": "grandchild-scope", + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "group-child", + "sequence": 13, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 10, + "cancellation_scope_id": "grandchild-scope", + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "requested-parent", + "sequence": 14, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "parent_scope_id": null, + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "requested-child", + "sequence": 15, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "request_id": "child-request", + "parent_scope_id": "parent-scope", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "requested-grandchild", + "sequence": 16, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "request_id": "grandchild-request", + "parent_scope_id": "child-scope", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "ancestor-prepared", + "namespace": "sdk-scope-fixture", + "sequence": 17, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 8, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [ + { + "scope_id": "child-scope", + "parent_scope_id": "parent-scope", + "scope_history_event_id": "opened-child", + "request_history_event_id": "requested-child", + "request_id": "child-request", + "propagation_history_event_id": "requested-child", + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [] + }, + { + "scope_id": "grandchild-scope", + "parent_scope_id": "child-scope", + "scope_history_event_id": "opened-grandchild", + "request_history_event_id": "requested-grandchild", + "request_id": "grandchild-request", + "propagation_history_event_id": "requested-grandchild", + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [ + { + "sequence": 7, + "activity_execution_id": "prior-id", + "descriptor_hash": "f60e9f5c60924a14222285670b881c3e456b61bc18e79addfd3d3308ef5ccbae" + }, + { + "sequence": 8, + "activity_execution_id": "original-id", + "descriptor_hash": "1f575cec410a08e47f0bd32b2a95b40f317eacb774f17ec542f4a85dd244503c" + } + ], + "timer_members": [ + { + "sequence": 9, + "timer_id": "original-timer", + "descriptor_hash": "6881ed38e69f1e10fd153d33d725657e4cf25cfe5df525fc1780324a29136aff" + } + ], + "wait_members": [], + "child_members": [ + { + "sequence": 10, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "30bd3db70d54bc9c67e4f961bccbca06ee1f3749d2842022610e0e2d4f11bd66" + } + ] + } + ], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "ancestor-delivered", + "namespace": "sdk-scope-fixture", + "sequence": 18, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 8, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "preparation_history_event_id": "ancestor-prepared", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "shielded-operation": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scopes": { + "outer": "outer-scope", + "parent": "parent-scope", + "shield": "shield-scope", + "shield-child": "shield-child-scope", + "child": "child-scope", + "grandchild": "grandchild-scope" + }, + "layout": "shielded-operation", + "contexts": { + "parent": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "child": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "grandchild": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "opened-outer", + "sequence": 3, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "outer-scope", + "parent_scope_id": "root", + "shield_parent": false + } + }, + { + "id": "opened-parent", + "sequence": 4, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "parent-scope", + "parent_scope_id": "outer-scope", + "shield_parent": false + } + }, + { + "id": "opened-shield", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 3, + "scope_id": "shield-scope", + "parent_scope_id": "parent-scope", + "shield_parent": true + } + }, + { + "id": "opened-shield-child", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 4, + "scope_id": "shield-child-scope", + "parent_scope_id": "shield-scope", + "shield_parent": false + } + }, + { + "id": "opened-child", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 5, + "scope_id": "child-scope", + "parent_scope_id": "parent-scope", + "shield_parent": false + } + }, + { + "id": "opened-grandchild", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 6, + "scope_id": "grandchild-scope", + "parent_scope_id": "child-scope", + "shield_parent": false + } + }, + { + "id": "prior-scheduled", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "grandchild-scope", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "grandchild-scope", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "grandchild-scope", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-timer", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 8, + "cancellation_scope_id": "shield-scope", + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "requested-parent", + "sequence": 12, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "parent_scope_id": null, + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "requested-child", + "sequence": 13, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "request_id": "child-request", + "parent_scope_id": "parent-scope", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "requested-grandchild", + "sequence": 14, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "request_id": "grandchild-request", + "parent_scope_id": "child-scope", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "ancestor-prepared", + "namespace": "sdk-scope-fixture", + "sequence": 15, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 8, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [ + { + "scope_id": "child-scope", + "parent_scope_id": "parent-scope", + "scope_history_event_id": "opened-child", + "request_history_event_id": "requested-child", + "request_id": "child-request", + "propagation_history_event_id": "requested-child", + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [] + }, + { + "scope_id": "grandchild-scope", + "parent_scope_id": "child-scope", + "scope_history_event_id": "opened-grandchild", + "request_history_event_id": "requested-grandchild", + "request_id": "grandchild-request", + "propagation_history_event_id": "requested-grandchild", + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [ + { + "sequence": 7, + "activity_execution_id": "prior-id", + "descriptor_hash": "f60e9f5c60924a14222285670b881c3e456b61bc18e79addfd3d3308ef5ccbae" + } + ], + "timer_members": [], + "wait_members": [], + "child_members": [] + } + ], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "ancestor-delivered", + "namespace": "sdk-scope-fixture", + "sequence": 16, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 8, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "preparation_history_event_id": "ancestor-prepared", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + } +} diff --git a/tests/fixtures/committed-scope-operation-projections.json b/tests/fixtures/committed-scope-operation-projections.json new file mode 100644 index 0000000..86bda4b --- /dev/null +++ b/tests/fixtures/committed-scope-operation-projections.json @@ -0,0 +1,3285 @@ +{ + "_provenance": { + "native_commit": "0471f1ebbb98c61fe53663086d62dc2db74db9e3", + "input_fixture_sha256": "85320e25c7c19521bca4a1cff138911df1f19f76471ea033e961695e8ed761be", + "producers": [ + "CancellationScopeDelivery::activityMembers", + "ScopedTimerCancellation::members", + "ScopedWaitCancellation::members", + "ScopedChildCancellation::members", + "CancellationScopeDescendants::members" + ], + "history_store": "isolated SQLite read-only projection inputs", + "effect_authority": false + }, + "operations": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "plain-timer", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 10, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "timer-plain", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:00.123456Z" + } + }, + { + "id": "signal-wait", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "SignalWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "signal_wait_id": "signal-wait-id", + "timeout_seconds": 30, + "signal_name": "inbox", + "condition_key": null, + "condition_definition_fingerprint": null, + "condition_wait_occurrence_id": null + } + }, + { + "id": "signal-timer", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "signal-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z", + "timer_kind": "signal_timeout", + "signal_wait_id": "signal-wait-id", + "condition_wait_occurrence_id": null + } + }, + { + "id": "condition-wait", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 12, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "condition-wait-id", + "timeout_seconds": 30, + "signal_name": null, + "condition_key": "ready", + "condition_definition_fingerprint": "predicate-v1", + "condition_wait_occurrence_id": "occurrence-1" + } + }, + { + "id": "condition-timer", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 12, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "condition-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z", + "timer_kind": "condition_timeout", + "condition_wait_id": "condition-wait-id", + "condition_wait_occurrence_id": "occurrence-1" + } + }, + { + "id": "wait-no-timeout", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 13, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "wait-no-timeout-id", + "condition_key": "done" + } + }, + { + "id": "child-0", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 14, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-0", + "child_workflow_instance_id": "child-instance-0", + "child_workflow_run_id": "child-run-0", + "cancellation_policy": "try_cancel", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child" + } + }, + { + "id": "child-1", + "sequence": 12, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 15, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-run-1", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child" + } + }, + { + "id": "child-2", + "sequence": 13, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 16, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-2", + "child_workflow_instance_id": "child-instance-2", + "child_workflow_run_id": "child-run-2", + "cancellation_policy": "abandon", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child" + } + }, + { + "id": "child-continued-before-preparation", + "sequence": 14, + "namespace": "sdk-scope-fixture", + "event_type": "ChildRunStarted", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 15, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-continued-run", + "cancellation_policy": "wait_cancellation_completed" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 15, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 16, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 20, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [], + "timer_members": [ + { + "sequence": 10, + "timer_id": "timer-plain", + "descriptor_hash": "d78a4b0d2c2f326857a31358e3ddfa20bd567e2364315ca30cef3cb11eae6c0f" + }, + { + "sequence": 11, + "timer_id": "signal-timer-id", + "descriptor_hash": "cb7fd119a14082cae54f6f418beb5769a3506cc5769df7ac10318017b1360b70" + }, + { + "sequence": 12, + "timer_id": "condition-timer-id", + "descriptor_hash": "167249f4220f18c56597a3a14165aa464443b4e71e7e28ffd708f271b6f0460e" + } + ], + "wait_members": [ + { + "kind": "signal", + "sequence": 11, + "wait_id": "signal-wait-id", + "timer_id": "signal-timer-id", + "descriptor_hash": "68a60d453bbc004c3c17ca4e33f95bbad92e24508e680152c1140be5f247a63f" + }, + { + "kind": "condition", + "sequence": 12, + "wait_id": "condition-wait-id", + "timer_id": "condition-timer-id", + "descriptor_hash": "e8f0ce4c1242168f2a97286c5ceea4bb9458822aedc8301734226b139510e960" + }, + { + "kind": "condition", + "sequence": 13, + "wait_id": "wait-no-timeout-id", + "timer_id": null, + "descriptor_hash": "b462fc9baefdd0f5b792ed8f41177d3a97a3d4bce08a58f04782a31215d4f6f6" + } + ], + "child_members": [ + { + "sequence": 14, + "child_call_id": "child-call-0", + "child_workflow_instance_id": "child-instance-0", + "child_workflow_run_id": "child-run-0", + "cancellation_policy": "try_cancel", + "descriptor_hash": "19c48c0ad10c781d4c08900a51fa1d98e4606bc7b65556b12d705d3f49367d07" + }, + { + "sequence": 15, + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-continued-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "0d65b5082c06e33b7db952b1d61085a344a4b97b60d982998a4fba6b6e6748ef" + }, + { + "sequence": 16, + "child_call_id": "child-call-2", + "child_workflow_instance_id": "child-instance-2", + "child_workflow_run_id": "child-run-2", + "cancellation_policy": "abandon", + "descriptor_hash": "a6a5e39e958de9cf34a656dc01f417209dae2a5787ab4ee0e169c9e3f08421bd" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 17, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 20, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "groups": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "plain-timer", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 10, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "timer-plain", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:00.123456Z", + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:10:10", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 10, + "parallel_group_size": 10, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ] + } + }, + { + "id": "signal-wait", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "SignalWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "signal_wait_id": "signal-wait-id", + "timeout_seconds": 30, + "signal_name": "inbox", + "condition_key": null, + "condition_definition_fingerprint": null, + "condition_wait_occurrence_id": null, + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:10:10", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 10, + "parallel_group_size": 10, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ] + } + }, + { + "id": "signal-timer", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "signal-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z", + "timer_kind": "signal_timeout", + "signal_wait_id": "signal-wait-id", + "condition_wait_occurrence_id": null, + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:10:10", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 10, + "parallel_group_size": 10, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ] + } + }, + { + "id": "condition-wait", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 12, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "condition-wait-id", + "timeout_seconds": 30, + "signal_name": null, + "condition_key": "ready", + "condition_definition_fingerprint": "predicate-v1", + "condition_wait_occurrence_id": "occurrence-1", + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:10:10", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 10, + "parallel_group_size": 10, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ] + } + }, + { + "id": "condition-timer", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 12, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "condition-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z", + "timer_kind": "condition_timeout", + "condition_wait_id": "condition-wait-id", + "condition_wait_occurrence_id": "occurrence-1", + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:10:10", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 10, + "parallel_group_size": 10, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ] + } + }, + { + "id": "wait-no-timeout", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 13, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "wait-no-timeout-id", + "condition_key": "done", + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:10:10", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 10, + "parallel_group_size": 10, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ] + } + }, + { + "id": "child-0", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 14, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-0", + "child_workflow_instance_id": "child-instance-0", + "child_workflow_run_id": "child-run-0", + "cancellation_policy": "try_cancel", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child", + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:10:10", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 10, + "parallel_group_size": 10, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ] + } + }, + { + "id": "child-1", + "sequence": 12, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 15, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-run-1", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child", + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:10:10", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 10, + "parallel_group_size": 10, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ] + } + }, + { + "id": "child-2", + "sequence": 13, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 16, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-2", + "child_workflow_instance_id": "child-instance-2", + "child_workflow_run_id": "child-run-2", + "cancellation_policy": "abandon", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child", + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:10:10", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 10, + "parallel_group_size": 10, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ] + } + }, + { + "id": "child-continued-before-preparation", + "sequence": 14, + "namespace": "sdk-scope-fixture", + "event_type": "ChildRunStarted", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 15, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-continued-run", + "cancellation_policy": "wait_cancellation_completed" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 15, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 16, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 20, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [], + "timer_members": [ + { + "sequence": 10, + "timer_id": "timer-plain", + "descriptor_hash": "d3012a07431c6e6fccdadde3185bdea12d2c5bae3c83c631704bab9e84c2badc" + }, + { + "sequence": 11, + "timer_id": "signal-timer-id", + "descriptor_hash": "3cc49b05da15311eac9d5791fb2099a5ade355ba4ea9ae90f86b597bfdd1ecb8" + }, + { + "sequence": 12, + "timer_id": "condition-timer-id", + "descriptor_hash": "4793559947b722bcc9f8d3a5d983f0c496b1d3da62127b14d677d241be3862f9" + } + ], + "wait_members": [ + { + "kind": "signal", + "sequence": 11, + "wait_id": "signal-wait-id", + "timer_id": "signal-timer-id", + "descriptor_hash": "0c73d277e69cb9a54a442fe8c16b461aca8218d80db5780ed57f41a4a85da3b3" + }, + { + "kind": "condition", + "sequence": 12, + "wait_id": "condition-wait-id", + "timer_id": "condition-timer-id", + "descriptor_hash": "dbf2614f9457f7e94ec5e6858923b8381beaaefe3d322dbc4b1acb33d360eb86" + }, + { + "kind": "condition", + "sequence": 13, + "wait_id": "wait-no-timeout-id", + "timer_id": null, + "descriptor_hash": "c7c84bd9d2dc024450a9c520deba77ac10239e019a587d42201ad7dd03fe2236" + } + ], + "child_members": [ + { + "sequence": 14, + "child_call_id": "child-call-0", + "child_workflow_instance_id": "child-instance-0", + "child_workflow_run_id": "child-run-0", + "cancellation_policy": "try_cancel", + "descriptor_hash": "d4c2dd2ddf569b6b6b156b4d82f30a59262f30fff94a5d579c737266cfa228a2" + }, + { + "sequence": 15, + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-continued-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "e3eec098ec70e85f527e9bb6cf14faaed9092540e32638b022e23f4c9ef50763" + }, + { + "sequence": 16, + "child_call_id": "child-call-2", + "child_workflow_instance_id": "child-instance-2", + "child_workflow_run_id": "child-run-2", + "cancellation_policy": "abandon", + "descriptor_hash": "ec6572e50f0c0eb8738ec8d523f93991476d5e501428d03c6e9af0824e53c022" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 17, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 20, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "descendants": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "desc-child-opened", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "parent_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "shield_parent": false, + "sequence": 3 + } + }, + { + "id": "desc-grandchild-opened", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-grandchild", + "parent_scope_id": "desc-child", + "shield_parent": false, + "sequence": 4 + } + }, + { + "id": "desc-shield-opened", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-shield", + "parent_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "shield_parent": true, + "sequence": 5 + } + }, + { + "id": "desc-shield-child-opened", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-shield-child", + "parent_scope_id": "desc-shield", + "shield_parent": false, + "sequence": 6 + } + }, + { + "id": "plain-timer", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 10, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "timer-plain", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:00.123456Z" + } + }, + { + "id": "signal-wait", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "SignalWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "signal_wait_id": "signal-wait-id", + "timeout_seconds": 30, + "signal_name": "inbox", + "condition_key": null, + "condition_definition_fingerprint": null, + "condition_wait_occurrence_id": null + } + }, + { + "id": "signal-timer", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "signal-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z", + "timer_kind": "signal_timeout", + "signal_wait_id": "signal-wait-id", + "condition_wait_occurrence_id": null + } + }, + { + "id": "condition-wait", + "sequence": 12, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 12, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "condition-wait-id", + "timeout_seconds": 30, + "signal_name": null, + "condition_key": "ready", + "condition_definition_fingerprint": "predicate-v1", + "condition_wait_occurrence_id": "occurrence-1" + } + }, + { + "id": "condition-timer", + "sequence": 13, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 12, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "condition-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z", + "timer_kind": "condition_timeout", + "condition_wait_id": "condition-wait-id", + "condition_wait_occurrence_id": "occurrence-1" + } + }, + { + "id": "wait-no-timeout", + "sequence": 14, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 13, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "wait-no-timeout-id", + "condition_key": "done" + } + }, + { + "id": "child-0", + "sequence": 15, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 14, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-0", + "child_workflow_instance_id": "child-instance-0", + "child_workflow_run_id": "child-run-0", + "cancellation_policy": "try_cancel", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child" + } + }, + { + "id": "child-1", + "sequence": 16, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 15, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-run-1", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child" + } + }, + { + "id": "child-2", + "sequence": 17, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 16, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-2", + "child_workflow_instance_id": "child-instance-2", + "child_workflow_run_id": "child-run-2", + "cancellation_policy": "abandon", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child" + } + }, + { + "id": "child-continued-before-preparation", + "sequence": 18, + "namespace": "sdk-scope-fixture", + "event_type": "ChildRunStarted", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 15, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-continued-run", + "cancellation_policy": "wait_cancellation_completed" + } + }, + { + "id": "descendant-timer", + "sequence": 19, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 17, + "cancellation_scope_id": "desc-child", + "timer_id": "descendant-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 20, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "inherited-child-accepted", + "sequence": 21, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "parent_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "inherited-child-request", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "inherited-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "inherited-grandchild-accepted", + "sequence": 22, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-grandchild", + "parent_scope_id": "desc-child", + "request_id": "inherited-grandchild-request", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "inherited-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "inherited-grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-grandchild", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 23, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 20, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [], + "timer_members": [ + { + "sequence": 10, + "timer_id": "timer-plain", + "descriptor_hash": "d78a4b0d2c2f326857a31358e3ddfa20bd567e2364315ca30cef3cb11eae6c0f" + }, + { + "sequence": 11, + "timer_id": "signal-timer-id", + "descriptor_hash": "cb7fd119a14082cae54f6f418beb5769a3506cc5769df7ac10318017b1360b70" + }, + { + "sequence": 12, + "timer_id": "condition-timer-id", + "descriptor_hash": "167249f4220f18c56597a3a14165aa464443b4e71e7e28ffd708f271b6f0460e" + } + ], + "wait_members": [ + { + "kind": "signal", + "sequence": 11, + "wait_id": "signal-wait-id", + "timer_id": "signal-timer-id", + "descriptor_hash": "68a60d453bbc004c3c17ca4e33f95bbad92e24508e680152c1140be5f247a63f" + }, + { + "kind": "condition", + "sequence": 12, + "wait_id": "condition-wait-id", + "timer_id": "condition-timer-id", + "descriptor_hash": "e8f0ce4c1242168f2a97286c5ceea4bb9458822aedc8301734226b139510e960" + }, + { + "kind": "condition", + "sequence": 13, + "wait_id": "wait-no-timeout-id", + "timer_id": null, + "descriptor_hash": "b462fc9baefdd0f5b792ed8f41177d3a97a3d4bce08a58f04782a31215d4f6f6" + } + ], + "child_members": [ + { + "sequence": 14, + "child_call_id": "child-call-0", + "child_workflow_instance_id": "child-instance-0", + "child_workflow_run_id": "child-run-0", + "cancellation_policy": "try_cancel", + "descriptor_hash": "19c48c0ad10c781d4c08900a51fa1d98e4606bc7b65556b12d705d3f49367d07" + }, + { + "sequence": 15, + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-continued-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "0d65b5082c06e33b7db952b1d61085a344a4b97b60d982998a4fba6b6e6748ef" + }, + { + "sequence": 16, + "child_call_id": "child-call-2", + "child_workflow_instance_id": "child-instance-2", + "child_workflow_run_id": "child-run-2", + "cancellation_policy": "abandon", + "descriptor_hash": "a6a5e39e958de9cf34a656dc01f417209dae2a5787ab4ee0e169c9e3f08421bd" + } + ], + "descendant_members": [ + { + "scope_id": "desc-child", + "parent_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "scope_history_event_id": "desc-child-opened", + "request_history_event_id": "inherited-child-accepted", + "request_id": "inherited-child-request", + "propagation_history_event_id": "inherited-child-accepted", + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "inherited-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [], + "timer_members": [ + { + "sequence": 17, + "timer_id": "descendant-timer-id", + "descriptor_hash": "b37b71a960ac7b3edf51dbf31e3a0d22c863c10d05abf2ac362f74d6ac131bf6" + } + ], + "wait_members": [], + "child_members": [] + }, + { + "scope_id": "desc-grandchild", + "parent_scope_id": "desc-child", + "scope_history_event_id": "desc-grandchild-opened", + "request_history_event_id": "inherited-grandchild-accepted", + "request_id": "inherited-grandchild-request", + "propagation_history_event_id": "inherited-grandchild-accepted", + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "inherited-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "inherited-grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-grandchild", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [] + } + ], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 24, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 20, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "competing": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "desc-child-opened", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "parent_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "shield_parent": false, + "sequence": 3 + } + }, + { + "id": "desc-grandchild-opened", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-grandchild", + "parent_scope_id": "desc-child", + "shield_parent": false, + "sequence": 4 + } + }, + { + "id": "desc-shield-opened", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-shield", + "parent_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "shield_parent": true, + "sequence": 5 + } + }, + { + "id": "desc-shield-child-opened", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-shield-child", + "parent_scope_id": "desc-shield", + "shield_parent": false, + "sequence": 6 + } + }, + { + "id": "plain-timer", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 10, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "timer-plain", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:00.123456Z" + } + }, + { + "id": "signal-wait", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "SignalWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "signal_wait_id": "signal-wait-id", + "timeout_seconds": 30, + "signal_name": "inbox", + "condition_key": null, + "condition_definition_fingerprint": null, + "condition_wait_occurrence_id": null + } + }, + { + "id": "signal-timer", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "signal-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z", + "timer_kind": "signal_timeout", + "signal_wait_id": "signal-wait-id", + "condition_wait_occurrence_id": null + } + }, + { + "id": "condition-wait", + "sequence": 12, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 12, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "condition-wait-id", + "timeout_seconds": 30, + "signal_name": null, + "condition_key": "ready", + "condition_definition_fingerprint": "predicate-v1", + "condition_wait_occurrence_id": "occurrence-1" + } + }, + { + "id": "condition-timer", + "sequence": 13, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 12, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "condition-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z", + "timer_kind": "condition_timeout", + "condition_wait_id": "condition-wait-id", + "condition_wait_occurrence_id": "occurrence-1" + } + }, + { + "id": "wait-no-timeout", + "sequence": 14, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 13, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "wait-no-timeout-id", + "condition_key": "done" + } + }, + { + "id": "child-0", + "sequence": 15, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 14, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-0", + "child_workflow_instance_id": "child-instance-0", + "child_workflow_run_id": "child-run-0", + "cancellation_policy": "try_cancel", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child" + } + }, + { + "id": "child-1", + "sequence": 16, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 15, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-run-1", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child" + } + }, + { + "id": "child-2", + "sequence": 17, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 16, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-2", + "child_workflow_instance_id": "child-instance-2", + "child_workflow_run_id": "child-run-2", + "cancellation_policy": "abandon", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child" + } + }, + { + "id": "child-continued-before-preparation", + "sequence": 18, + "namespace": "sdk-scope-fixture", + "event_type": "ChildRunStarted", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 15, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-continued-run", + "cancellation_policy": "wait_cancellation_completed" + } + }, + { + "id": "descendant-timer", + "sequence": 19, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 17, + "cancellation_scope_id": "desc-child", + "timer_id": "descendant-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z" + } + }, + { + "id": "independent-child-accepted", + "sequence": 20, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "parent_scope_id": null, + "request_id": "independent-child-request", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "independent-child-request", + "root_request_id": "independent-child-request", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z", + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z" + } + ] + } + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 21, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "original-child-conflict", + "sequence": 22, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequestConflicted", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "scope_id": "desc-child", + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "reason": "cancellation_root_conflict", + "parent_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "incoming_cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "inherited-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "accepted_cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "independent-child-request", + "root_request_id": "independent-child-request", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z", + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z" + } + ] + } + } + }, + { + "id": "inherited-grandchild-accepted", + "sequence": 23, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-grandchild", + "parent_scope_id": "desc-child", + "request_id": "inherited-grandchild-request", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "independent-child-request", + "root_request_id": "independent-child-request", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z", + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z" + }, + { + "request_id": "inherited-grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-grandchild", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z" + } + ] + } + } + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 24, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 20, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [], + "timer_members": [ + { + "sequence": 10, + "timer_id": "timer-plain", + "descriptor_hash": "d78a4b0d2c2f326857a31358e3ddfa20bd567e2364315ca30cef3cb11eae6c0f" + }, + { + "sequence": 11, + "timer_id": "signal-timer-id", + "descriptor_hash": "cb7fd119a14082cae54f6f418beb5769a3506cc5769df7ac10318017b1360b70" + }, + { + "sequence": 12, + "timer_id": "condition-timer-id", + "descriptor_hash": "167249f4220f18c56597a3a14165aa464443b4e71e7e28ffd708f271b6f0460e" + } + ], + "wait_members": [ + { + "kind": "signal", + "sequence": 11, + "wait_id": "signal-wait-id", + "timer_id": "signal-timer-id", + "descriptor_hash": "68a60d453bbc004c3c17ca4e33f95bbad92e24508e680152c1140be5f247a63f" + }, + { + "kind": "condition", + "sequence": 12, + "wait_id": "condition-wait-id", + "timer_id": "condition-timer-id", + "descriptor_hash": "e8f0ce4c1242168f2a97286c5ceea4bb9458822aedc8301734226b139510e960" + }, + { + "kind": "condition", + "sequence": 13, + "wait_id": "wait-no-timeout-id", + "timer_id": null, + "descriptor_hash": "b462fc9baefdd0f5b792ed8f41177d3a97a3d4bce08a58f04782a31215d4f6f6" + } + ], + "child_members": [ + { + "sequence": 14, + "child_call_id": "child-call-0", + "child_workflow_instance_id": "child-instance-0", + "child_workflow_run_id": "child-run-0", + "cancellation_policy": "try_cancel", + "descriptor_hash": "19c48c0ad10c781d4c08900a51fa1d98e4606bc7b65556b12d705d3f49367d07" + }, + { + "sequence": 15, + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-continued-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "0d65b5082c06e33b7db952b1d61085a344a4b97b60d982998a4fba6b6e6748ef" + }, + { + "sequence": 16, + "child_call_id": "child-call-2", + "child_workflow_instance_id": "child-instance-2", + "child_workflow_run_id": "child-run-2", + "cancellation_policy": "abandon", + "descriptor_hash": "a6a5e39e958de9cf34a656dc01f417209dae2a5787ab4ee0e169c9e3f08421bd" + } + ], + "descendant_members": [ + { + "scope_id": "desc-child", + "parent_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "scope_history_event_id": "desc-child-opened", + "request_history_event_id": "independent-child-accepted", + "request_id": "independent-child-request", + "propagation_history_event_id": "original-child-conflict", + "authority_deadline_at": "2026-10-04T00:00:20.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "independent-child-request", + "root_request_id": "independent-child-request", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z", + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z" + } + ] + }, + "activity_members": [], + "timer_members": [ + { + "sequence": 17, + "timer_id": "descendant-timer-id", + "descriptor_hash": "b37b71a960ac7b3edf51dbf31e3a0d22c863c10d05abf2ac362f74d6ac131bf6" + } + ], + "wait_members": [], + "child_members": [] + }, + { + "scope_id": "desc-grandchild", + "parent_scope_id": "desc-child", + "scope_history_event_id": "desc-grandchild-opened", + "request_history_event_id": "inherited-grandchild-accepted", + "request_id": "inherited-grandchild-request", + "propagation_history_event_id": "inherited-grandchild-accepted", + "authority_deadline_at": "2026-10-04T00:00:20.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "independent-child-request", + "root_request_id": "independent-child-request", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z", + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z" + }, + { + "request_id": "inherited-grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-grandchild", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z" + } + ] + }, + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [] + } + ], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 25, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 20, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + } +} diff --git a/tests/fixtures/populated-scope-groups.json b/tests/fixtures/populated-scope-groups.json new file mode 100644 index 0000000..d93139f --- /dev/null +++ b/tests/fixtures/populated-scope-groups.json @@ -0,0 +1,3919 @@ +{ + "_provenance": { + "native_commit": "0471f1ebbb98c61fe53663086d62dc2db74db9e3", + "input_fixture_sha256": "85320e25c7c19521bca4a1cff138911df1f19f76471ea033e961695e8ed761be", + "history_store": "isolated SQLite projection inputs", + "effect_authority": false, + "producers": [ + "CancellationScopeDelivery::activityMembers", + "ScopedTimerCancellation::members", + "ScopedWaitCancellation::members", + "ScopedChildCancellation::members" + ] + }, + "flat": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "layout": "flat", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "group-activity", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "group-timer", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 5, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "group-child", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 6, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "group-condition", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "condition_wait_id": "original-wait", + "condition_key": "ready", + "timeout_seconds": 30, + "condition_wait_occurrence_id": "original-occurrence", + "condition_definition_fingerprint": "sha256:3b2b1ad635030d842ab378e8bf021f4613ee6d221dafb4743565467800def433" + } + }, + { + "id": "condition-timeout", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "original-timeout", + "timer_kind": "condition_timeout", + "condition_wait_id": "original-wait", + "condition_wait_occurrence_id": "original-occurrence", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:32.123456Z" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 12, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 13, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "c3a3d20bd8882dfc59673847c3f8ed67c66cdaee74e989810002e2b622764699" + } + ], + "timer_members": [ + { + "sequence": 5, + "timer_id": "original-timer", + "descriptor_hash": "bae0466390116ff60cbd3597bcdaf8e0097fae4c7f8b789db6f31ee9553b62c1" + }, + { + "sequence": 7, + "timer_id": "original-timeout", + "descriptor_hash": "f5518182dcbfc2737f54b9a1754e941e8efa416815a2aad9c8e4cec7f266d423" + } + ], + "wait_members": [ + { + "kind": "condition", + "sequence": 7, + "wait_id": "original-wait", + "timer_id": "original-timeout", + "descriptor_hash": "0bd2f375d595a9433320f9b48a03831232d40aac2258f1193d2026248f822d3f" + } + ], + "child_members": [ + { + "sequence": 6, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "ecced7a15fa4e651c44eb0b9f2c0962d94f73a3caae0798d375ec65d3618ee1f" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 14, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "nested": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "layout": "nested", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "group-activity", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "group-timer", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 5, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:5:2", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 5, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "parallel-calls:5:2", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 5, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ], + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "group-child", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 6, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:5:2", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 5, + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2 + }, + { + "parallel_group_id": "parallel-calls:5:2", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 5, + "parallel_group_size": 2, + "parallel_group_index": 1 + } + ], + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "group-condition", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "condition_wait_id": "original-wait", + "condition_key": "ready", + "timeout_seconds": 30, + "condition_wait_occurrence_id": "original-occurrence", + "condition_definition_fingerprint": "sha256:3b2b1ad635030d842ab378e8bf021f4613ee6d221dafb4743565467800def433" + } + }, + { + "id": "condition-timeout", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "original-timeout", + "timer_kind": "condition_timeout", + "condition_wait_id": "original-wait", + "condition_wait_occurrence_id": "original-occurrence", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:32.123456Z" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 12, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 13, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "c3a3d20bd8882dfc59673847c3f8ed67c66cdaee74e989810002e2b622764699" + } + ], + "timer_members": [ + { + "sequence": 5, + "timer_id": "original-timer", + "descriptor_hash": "63a2bd9bc43403237583a4025f6dd94fd3af405b54ec9d8dc52d6f786924bcf8" + }, + { + "sequence": 7, + "timer_id": "original-timeout", + "descriptor_hash": "f5518182dcbfc2737f54b9a1754e941e8efa416815a2aad9c8e4cec7f266d423" + } + ], + "wait_members": [ + { + "kind": "condition", + "sequence": 7, + "wait_id": "original-wait", + "timer_id": "original-timeout", + "descriptor_hash": "0bd2f375d595a9433320f9b48a03831232d40aac2258f1193d2026248f822d3f" + } + ], + "child_members": [ + { + "sequence": 6, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "83bc0b6e32d165958c0009a10112a72c3ed757a8c4c398138fe324c5613445ac" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 14, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "flat-local": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "layout": "flat-local", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "group-activity", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + }, + "local_activity": true, + "execution_mode": "local" + } + }, + { + "id": "group-timer", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 5, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "group-child", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 6, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "group-condition", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "condition_wait_id": "original-wait", + "condition_key": "ready", + "timeout_seconds": 30, + "condition_wait_occurrence_id": "original-occurrence", + "condition_definition_fingerprint": "sha256:3b2b1ad635030d842ab378e8bf021f4613ee6d221dafb4743565467800def433" + } + }, + { + "id": "condition-timeout", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "original-timeout", + "timer_kind": "condition_timeout", + "condition_wait_id": "original-wait", + "condition_wait_occurrence_id": "original-occurrence", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:32.123456Z" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 12, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 13, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "938c445dfa05b9573b85e49ab0839f77d706dc694ac7801a13c621eff301e727" + } + ], + "timer_members": [ + { + "sequence": 5, + "timer_id": "original-timer", + "descriptor_hash": "bae0466390116ff60cbd3597bcdaf8e0097fae4c7f8b789db6f31ee9553b62c1" + }, + { + "sequence": 7, + "timer_id": "original-timeout", + "descriptor_hash": "f5518182dcbfc2737f54b9a1754e941e8efa416815a2aad9c8e4cec7f266d423" + } + ], + "wait_members": [ + { + "kind": "condition", + "sequence": 7, + "wait_id": "original-wait", + "timer_id": "original-timeout", + "descriptor_hash": "0bd2f375d595a9433320f9b48a03831232d40aac2258f1193d2026248f822d3f" + } + ], + "child_members": [ + { + "sequence": 6, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "ecced7a15fa4e651c44eb0b9f2c0962d94f73a3caae0798d375ec65d3618ee1f" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 14, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "flat-selection": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "layout": "flat-selection", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "group-activity", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "select-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_mode": "select", + "selection_member_key": 0, + "selection_member_index": 0, + "selection_member_base_sequence": 4, + "selection_member_size": 1, + "selection_member_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_mode": "select", + "selection_member_key": 0, + "selection_member_index": 0, + "selection_member_base_sequence": 4, + "selection_member_size": 1, + "selection_member_kind": "activity" + } + ], + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "group-timer", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 5, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "select-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_mode": "select", + "selection_member_key": 1, + "selection_member_index": 1, + "selection_member_base_sequence": 5, + "selection_member_size": 1, + "selection_member_kind": "timer", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_mode": "select", + "selection_member_key": 1, + "selection_member_index": 1, + "selection_member_base_sequence": 5, + "selection_member_size": 1, + "selection_member_kind": "timer" + } + ], + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "group-child", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 6, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "select-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_mode": "select", + "selection_member_key": 2, + "selection_member_index": 2, + "selection_member_base_sequence": 6, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_mode": "select", + "selection_member_key": 2, + "selection_member_index": 2, + "selection_member_base_sequence": 6, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ], + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "group-condition", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "select-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_mode": "select", + "selection_member_key": 3, + "selection_member_index": 3, + "selection_member_base_sequence": 7, + "selection_member_size": 1, + "selection_member_kind": "condition", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_mode": "select", + "selection_member_key": 3, + "selection_member_index": 3, + "selection_member_base_sequence": 7, + "selection_member_size": 1, + "selection_member_kind": "condition" + } + ], + "condition_wait_id": "original-wait", + "condition_key": "ready", + "timeout_seconds": 30, + "condition_wait_occurrence_id": "original-occurrence", + "condition_definition_fingerprint": "sha256:3b2b1ad635030d842ab378e8bf021f4613ee6d221dafb4743565467800def433" + } + }, + { + "id": "condition-timeout", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "original-timeout", + "timer_kind": "condition_timeout", + "condition_wait_id": "original-wait", + "condition_wait_occurrence_id": "original-occurrence", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:32.123456Z" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 12, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 13, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "c3a3d20bd8882dfc59673847c3f8ed67c66cdaee74e989810002e2b622764699" + } + ], + "timer_members": [ + { + "sequence": 5, + "timer_id": "original-timer", + "descriptor_hash": "72554083760b2fa13f5a8050ca408c127e31d3d31406e5a15c36aa58ccad6fc2" + }, + { + "sequence": 7, + "timer_id": "original-timeout", + "descriptor_hash": "f5518182dcbfc2737f54b9a1754e941e8efa416815a2aad9c8e4cec7f266d423" + } + ], + "wait_members": [ + { + "kind": "condition", + "sequence": 7, + "wait_id": "original-wait", + "timer_id": "original-timeout", + "descriptor_hash": "2226681f0dfa1f9cbd32dde39187b68a7bc2c8271849babe6e09dd799f79de0d" + } + ], + "child_members": [ + { + "sequence": 6, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "cf830fe095b233d4d675edb3ab40d03ef634b68b4ed8c17b58cea04b00e3bb44" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 14, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "flat-incomplete": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "layout": "flat-incomplete", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "group-activity", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "group-timer", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 5, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "group-child", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 6, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 11, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "c3a3d20bd8882dfc59673847c3f8ed67c66cdaee74e989810002e2b622764699" + } + ], + "timer_members": [ + { + "sequence": 5, + "timer_id": "original-timer", + "descriptor_hash": "bae0466390116ff60cbd3597bcdaf8e0097fae4c7f8b789db6f31ee9553b62c1" + } + ], + "wait_members": [], + "child_members": [ + { + "sequence": 6, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "ecced7a15fa4e651c44eb0b9f2c0962d94f73a3caae0798d375ec65d3618ee1f" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 12, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "flat-signal": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "layout": "flat-signal", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "group-activity", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "group-timer", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 5, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "group-child", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 6, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "group-condition", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "SignalWaitOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "timeout_seconds": 30, + "signal_wait_id": "original-wait", + "signal_name": "ready" + } + }, + { + "id": "condition-timeout", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "original-timeout", + "timer_kind": "signal_timeout", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:32.123456Z", + "signal_wait_id": "original-wait" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 12, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 13, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "c3a3d20bd8882dfc59673847c3f8ed67c66cdaee74e989810002e2b622764699" + } + ], + "timer_members": [ + { + "sequence": 5, + "timer_id": "original-timer", + "descriptor_hash": "bae0466390116ff60cbd3597bcdaf8e0097fae4c7f8b789db6f31ee9553b62c1" + }, + { + "sequence": 7, + "timer_id": "original-timeout", + "descriptor_hash": "c02f931bce4a2a8528bfd5ec9eb522c5402f88b96c77a0b211d43948e1484244" + } + ], + "wait_members": [ + { + "kind": "signal", + "sequence": 7, + "wait_id": "original-wait", + "timer_id": "original-timeout", + "descriptor_hash": "4386073479943cef92b70c14b427ad2bb9cbad16b2481af8e6406b98db9c780d" + } + ], + "child_members": [ + { + "sequence": 6, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "ecced7a15fa4e651c44eb0b9f2c0962d94f73a3caae0798d375ec65d3618ee1f" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 14, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "flat-other-scope": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "layout": "flat-other-scope", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "group-activity", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "group-timer", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 5, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "group-child", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 6, + "cancellation_scope_id": "root", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "group-condition", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "condition_wait_id": "original-wait", + "condition_key": "ready", + "timeout_seconds": 30, + "condition_wait_occurrence_id": "original-occurrence", + "condition_definition_fingerprint": "sha256:3b2b1ad635030d842ab378e8bf021f4613ee6d221dafb4743565467800def433" + } + }, + { + "id": "condition-timeout", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "original-timeout", + "timer_kind": "condition_timeout", + "condition_wait_id": "original-wait", + "condition_wait_occurrence_id": "original-occurrence", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:32.123456Z" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 12, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 13, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "c3a3d20bd8882dfc59673847c3f8ed67c66cdaee74e989810002e2b622764699" + } + ], + "timer_members": [ + { + "sequence": 5, + "timer_id": "original-timer", + "descriptor_hash": "bae0466390116ff60cbd3597bcdaf8e0097fae4c7f8b789db6f31ee9553b62c1" + }, + { + "sequence": 7, + "timer_id": "original-timeout", + "descriptor_hash": "f5518182dcbfc2737f54b9a1754e941e8efa416815a2aad9c8e4cec7f266d423" + } + ], + "wait_members": [ + { + "kind": "condition", + "sequence": 7, + "wait_id": "original-wait", + "timer_id": "original-timeout", + "descriptor_hash": "0bd2f375d595a9433320f9b48a03831232d40aac2258f1193d2026248f822d3f" + } + ], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 14, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + } +} diff --git a/tests/fixtures/populated-scope-single-calls.json b/tests/fixtures/populated-scope-single-calls.json new file mode 100644 index 0000000..d083851 --- /dev/null +++ b/tests/fixtures/populated-scope-single-calls.json @@ -0,0 +1,3756 @@ +{ + "_provenance": { + "native_commit": "0471f1ebbb98c61fe53663086d62dc2db74db9e3", + "input_fixture_sha256": "85320e25c7c19521bca4a1cff138911df1f19f76471ea033e961695e8ed761be", + "history_store": "isolated SQLite projection inputs", + "effect_authority": false, + "producers": [ + "CancellationScopeDelivery::activityMembers", + "ScopedTimerCancellation::members", + "ScopedWaitCancellation::members", + "ScopedChildCancellation::members" + ] + }, + "activity-try_cancel": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "kind": "activity", + "policy": "try_cancel", + "timeout": 30, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-admission", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 8, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 9, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "activity", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "a26f8a0ff7beb14634f3846cf6f6998166bedd9a468920cf2ccf911762ca9f03" + } + ], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "activity", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "activity-wait_cancellation_completed": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "kind": "activity", + "policy": "wait_cancellation_completed", + "timeout": 30, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-admission", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "wait_cancellation_completed", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 8, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 9, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "activity", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "70125d7921b68151c2e28d6a383487f75b6adab62decd19418ccf636ac540e03" + } + ], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "activity", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "activity-abandon": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "kind": "activity", + "policy": "abandon", + "timeout": 30, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-admission", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "abandon", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 8, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 9, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "activity", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "271cff6e0f86da5383c36fed6e1e3b291a29dbc4cd1ce4105910ddad5c4cb318" + } + ], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "activity", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "child-try_cancel": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "kind": "child", + "policy": "try_cancel", + "timeout": 30, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-admission", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "try_cancel", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 8, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 9, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "child", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + } + ], + "timer_members": [], + "wait_members": [], + "child_members": [ + { + "sequence": 4, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "try_cancel", + "descriptor_hash": "a958cabe4cf43b14934fed381fcd5ea2fe2ad1b83788dfc3de16c442231a1b23" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "child", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "child-wait_cancellation_completed": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "kind": "child", + "policy": "wait_cancellation_completed", + "timeout": 30, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-admission", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 8, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 9, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "child", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + } + ], + "timer_members": [], + "wait_members": [], + "child_members": [ + { + "sequence": 4, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "ef3b5c2dea5496740fceb297f0715a49d8ed675e9a529df754a19d1aacc60a54" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "child", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "child-abandon": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "kind": "child", + "policy": "abandon", + "timeout": 30, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-admission", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "abandon", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 8, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 9, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "child", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + } + ], + "timer_members": [], + "wait_members": [], + "child_members": [ + { + "sequence": 4, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "abandon", + "descriptor_hash": "2798ee25796bf6132d9b8020055ce55b4a46dbf2ea7ad6a2a9261b875247f3b6" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "child", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "timer": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "kind": "timer", + "policy": null, + "timeout": 30, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-admission", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 8, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 9, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + } + ], + "timer_members": [ + { + "sequence": 4, + "timer_id": "original-timer", + "descriptor_hash": "0120b0c0164724ed0fd3b589206d8e67a6177689a25a77f66add6dd7b9aea8f7" + } + ], + "wait_members": [], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "condition": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "kind": "condition", + "policy": null, + "timeout": 30, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-admission", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "original-wait", + "condition_key": "ready", + "timeout_seconds": 30, + "condition_wait_occurrence_id": "original-occurrence", + "condition_definition_fingerprint": "sha256:a1afc3faf0d393d76f4c4f2dfa825e189d8b8294ed8ac63b8a1d260b3076fdc5" + } + }, + { + "id": "condition-timeout", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "original-timeout", + "timer_kind": "condition_timeout", + "condition_wait_id": "original-wait", + "condition_wait_occurrence_id": "original-occurrence", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:32.123456Z" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 9, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "condition", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + } + ], + "timer_members": [ + { + "sequence": 4, + "timer_id": "original-timeout", + "descriptor_hash": "4e574cb5fe9e89df7c963b3d473e1d5ec232af084af565707e3405936b35a6be" + } + ], + "wait_members": [ + { + "kind": "condition", + "sequence": 4, + "wait_id": "original-wait", + "timer_id": "original-timeout", + "descriptor_hash": "3b1c0ac20ee7b4b205a4c8d10959c5ef76721bef02e4fc7cf2ae8e1f35590ebe" + } + ], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 11, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "condition", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "condition-untimed": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "kind": "condition", + "policy": "untimed", + "timeout": null, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-admission", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "original-wait", + "condition_key": "ready", + "timeout_seconds": null, + "condition_wait_occurrence_id": "original-occurrence", + "condition_definition_fingerprint": "sha256:a1afc3faf0d393d76f4c4f2dfa825e189d8b8294ed8ac63b8a1d260b3076fdc5" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 8, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 9, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "condition", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + } + ], + "timer_members": [], + "wait_members": [ + { + "kind": "condition", + "sequence": 4, + "wait_id": "original-wait", + "timer_id": null, + "descriptor_hash": "cf5a40353f288f5d9a4eb75f7a922fc32224e665386b2f61db40e75c77d4cb4c" + } + ], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "condition", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + } +} diff --git a/tests/fixtures/replay-regressions/activity-cancellation-policy-changed.json b/tests/fixtures/replay-regressions/activity-cancellation-policy-changed.json new file mode 100644 index 0000000..3fd68c2 --- /dev/null +++ b/tests/fixtures/replay-regressions/activity-cancellation-policy-changed.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-activity-cancellation-policy-changed", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": {"type": "corpus.activity-policy-author", "input": [], "payload_codec": "avro"}, + "history": [ + {"event_type": "ActivityScheduled", "payload": {"sequence": 1, "activity_type": "work", "activity": {"type": "work", "cancellation_policy": "wait_cancellation_completed"}}} + ], + "command_sequence": [{"type": "complete_workflow", "result": "unreachable"}], + "expected": {"type": "complete_workflow", "result": "unreachable"}, + "expected_failure": "activity_cancellation_policy_changed" +} diff --git a/tests/fixtures/replay-regressions/cancellation-remaining-final-condition.json b/tests/fixtures/replay-regressions/cancellation-remaining-final-condition.json new file mode 100644 index 0000000..7032bca --- /dev/null +++ b/tests/fixtures/replay-regressions/cancellation-remaining-final-condition.json @@ -0,0 +1,56 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-cancellation-remaining-final-condition", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": {"type": "corpus.cancellation-remaining-condition", "input": [], "payload_codec": "avro"}, + "history": [ + { + "event_type": "CooperativeCancellationRequested", "recorded_at": "2026-10-01T00:00:00.123456Z", + "payload": { + "workflow_command_id": "original-request", "workflow_run_id": "regression-corpus-run", + "cleanup_deadline_at": "2026-10-01T00:00:30.123456Z", + "cancellation": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "original-request", "root_request_id": "original-request", + "root_workflow_instance_id": "regression-corpus-rust-cancellation-remaining-final-condition", + "root_workflow_run_id": "regression-corpus-run", "parent_request_id": null, + "reason": null, "requester": {"type": "operator", "id": "fixture-operator"}, "source": "control_plane", + "requested_at": "2026-10-01T00:00:00.123456Z", "cleanup_deadline_at": "2026-10-01T00:00:30.123456Z", + "lineage": [{"request_id": "original-request", "workflow_instance_id": "regression-corpus-rust-cancellation-remaining-final-condition", "workflow_run_id": "regression-corpus-run"}] + } + } + }, + { + "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-01T00:00:08Z", + "payload": {"workflow_command_id": "original-request", "workflow_run_id": "regression-corpus-run", "sequence": 1, "call_kind": "timer"} + }, + { + "event_type": "TimerScheduled", + "payload": {"sequence": 2, "timer_id": "timer-2", "delay_seconds": 1, "parallel_group_id": "parallel-calls:2:2", "parallel_group_kind": "mixed", "parallel_group_base_sequence": 2, "parallel_group_size": 2, "parallel_group_index": 0} + }, + { + "event_type": "ConditionWaitOpened", + "payload": {"sequence": 3, "condition_wait_id": "condition-3", "condition_wait_occurrence_id": "rust:condition-wait:0", "condition_key": "ready", "condition_definition_fingerprint": "sha256:ready", "parallel_group_id": "parallel-calls:2:2", "parallel_group_kind": "mixed", "parallel_group_base_sequence": 2, "parallel_group_size": 2, "parallel_group_index": 1} + }, + { + "event_type": "ConditionWaitSatisfied", "timestamp": "2026-10-01T00:00:12Z", + "payload": {"sequence": 3, "condition_wait_id": "condition-3", "condition_wait_occurrence_id": "rust:condition-wait:0", "condition_key": "ready", "condition_definition_fingerprint": "sha256:ready", "parallel_group_id": "parallel-calls:2:2", "parallel_group_kind": "mixed", "parallel_group_base_sequence": 2, "parallel_group_size": 2, "parallel_group_index": 1} + }, + { + "event_type": "ConditionWaitOpened", + "payload": {"sequence": 4, "condition_wait_id": "condition-4", "condition_wait_occurrence_id": "rust:condition-wait:0", "condition_key": "ready", "condition_definition_fingerprint": "sha256:ready", "parallel_group_id": "parallel-calls:2:2", "parallel_group_kind": "mixed", "parallel_group_base_sequence": 2, "parallel_group_size": 2, "parallel_group_index": 1} + }, + { + "event_type": "ConditionWaitSatisfied", "timestamp": "2026-10-01T00:00:25Z", + "payload": {"sequence": 4, "condition_wait_id": "condition-4", "condition_wait_occurrence_id": "rust:condition-wait:0", "condition_key": "ready", "condition_definition_fingerprint": "sha256:ready", "parallel_group_id": "parallel-calls:2:2", "parallel_group_kind": "mixed", "parallel_group_base_sequence": 2, "parallel_group_size": 2, "parallel_group_index": 1} + }, + { + "event_type": "TimerFired", "timestamp": "2026-10-01T00:00:10Z", + "payload": {"sequence": 2, "timer_id": "timer-2", "delay_seconds": 1, "parallel_group_id": "parallel-calls:2:2", "parallel_group_kind": "mixed", "parallel_group_base_sequence": 2, "parallel_group_size": 2, "parallel_group_index": 0} + } + ], + "command_sequence": [{"type": "complete_workflow", "result": 5.123456}], + "expected": {"type": "complete_workflow", "result": 5.123456} +} diff --git a/tests/fixtures/replay-regressions/cancellation-scope-unqualified-worker-rejected.json b/tests/fixtures/replay-regressions/cancellation-scope-unqualified-worker-rejected.json new file mode 100644 index 0000000..278fae8 --- /dev/null +++ b/tests/fixtures/replay-regressions/cancellation-scope-unqualified-worker-rejected.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-cancellation-scope-unqualified-worker-rejected", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": { + "type": "corpus.side-effect-version", + "input": [], + "payload_codec": "avro" + }, + "history": [ + { + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "regression-corpus-run", + "scope_id": "scope-one", + "parent_scope_id": "root", + "sequence": 1, + "shield_parent": false + } + } + ], + "expected": {"command_sequence": []}, + "expected_failure": "cancellation_scope_execution_not_supported: Rust worker" +} diff --git a/tests/fixtures/replay-regressions/child-cancellation-policy-changed.json b/tests/fixtures/replay-regressions/child-cancellation-policy-changed.json new file mode 100644 index 0000000..d3089d7 --- /dev/null +++ b/tests/fixtures/replay-regressions/child-cancellation-policy-changed.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-child-cancellation-policy-changed", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": {"type": "corpus.child-policy-author", "input": [], "payload_codec": "avro"}, + "history": [ + {"event_type": "ChildWorkflowScheduled", "payload": {"sequence": 1, "child_workflow_type": "child", "parent_close_policy": "abandon", "cancellation_policy": "wait_cancellation_completed"}} + ], + "command_sequence": [{"type": "complete_workflow", "result": "unreachable"}], + "expected": {"type": "complete_workflow", "result": "unreachable"}, + "expected_failure": "child_workflow_policy_changed" +} diff --git a/tests/fixtures/replay-regressions/child-start-policy-conflict.json b/tests/fixtures/replay-regressions/child-start-policy-conflict.json new file mode 100644 index 0000000..d1523d0 --- /dev/null +++ b/tests/fixtures/replay-regressions/child-start-policy-conflict.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-child-start-policy-conflict", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": {"type": "corpus.child-policy-author", "input": [], "payload_codec": "avro"}, + "history": [ + {"event_type": "ChildWorkflowScheduled", "payload": {"sequence": 1, "child_workflow_type": "child", "parent_close_policy": "abandon", "cancellation_policy": "abandon"}}, + {"event_type": "ChildRunStarted", "payload": {"sequence": 1, "child_workflow_type": "child", "cancellation_policy": "try_cancel"}} + ], + "command_sequence": [{"type": "complete_workflow", "result": "unreachable"}], + "expected": {"type": "complete_workflow", "result": "unreachable"}, + "expected_failure": "child_workflow_policy_history_conflict" +} diff --git a/tests/fixtures/replay-regressions/cooperative-cold-activity-delivery.json b/tests/fixtures/replay-regressions/cooperative-cold-activity-delivery.json new file mode 100644 index 0000000..ab2c3d3 --- /dev/null +++ b/tests/fixtures/replay-regressions/cooperative-cold-activity-delivery.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-cooperative-cold-activity-delivery", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": {"type":"corpus.redrive-boundary", "input":[], "payload_codec":"avro"}, + "history": [ + { + "event_type":"CooperativeCancellationRequested", + "workflow_command_id":"original-request", + "recorded_at":"2026-10-01T08:00:00Z", + "payload": { + "workflow_command_id":"original-request", "workflow_run_id":"regression-corpus-run", + "cleanup_deadline_at":"2026-10-01T08:10:00Z" + } + }, + { + "event_type":"CooperativeCancellationDelivered", + "workflow_command_id":"original-request", + "payload": { + "workflow_command_id":"original-request", "workflow_run_id":"regression-corpus-run", + "sequence":1, "call_kind":"activity" + } + } + ], + "command_sequence": [{ + "type":"fail_workflow", "exception_type":"WorkflowCancellationRequested", "non_retryable":true, + "exception": {"properties": {"request_id":"original-request", "cleanup_deadline_at":"2026-10-01T08:10:00Z"}} + }], + "expected": {"type":"fail_workflow", "exception_type":"WorkflowCancellationRequested", "non_retryable":true, + "exception": {"properties": {"request_id":"original-request", "cleanup_deadline_at":"2026-10-01T08:10:00Z"}}} +} diff --git a/tests/fixtures/replay-regressions/cooperative-cold-parallel-group.json b/tests/fixtures/replay-regressions/cooperative-cold-parallel-group.json new file mode 100644 index 0000000..ac673ab --- /dev/null +++ b/tests/fixtures/replay-regressions/cooperative-cold-parallel-group.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-cooperative-cold-parallel-group", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": {"type":"corpus.nested-parallel", "input":[], "payload_codec":"avro"}, + "history": [ + { + "event_type":"CooperativeCancellationRequested", "workflow_command_id":"original-request", + "recorded_at":"2026-10-01T08:00:00Z", "payload": { + "workflow_command_id":"original-request", "workflow_run_id":"regression-corpus-run", + "cleanup_deadline_at":"2026-10-01T08:10:00Z" + } + }, + { + "event_type":"CooperativeCancellationDelivered", "workflow_command_id":"original-request", "payload": { + "workflow_command_id":"original-request", "workflow_run_id":"regression-corpus-run", + "sequence":1, "call_kind":"parallel", "sequence_span":3 + } + } + ], + "command_sequence": [{ + "type":"fail_workflow", "exception_type":"WorkflowCancellationRequested", "non_retryable":true, + "exception": {"properties": {"request_id":"original-request", "cleanup_deadline_at":"2026-10-01T08:10:00Z"}} + }], + "expected": { + "type":"fail_workflow", "exception_type":"WorkflowCancellationRequested", "non_retryable":true, + "exception": {"properties": {"request_id":"original-request", "cleanup_deadline_at":"2026-10-01T08:10:00Z"}} + } +} diff --git a/tests/fixtures/replay-regressions/cooperative-cold-selection-handle.json b/tests/fixtures/replay-regressions/cooperative-cold-selection-handle.json new file mode 100644 index 0000000..0f970a7 --- /dev/null +++ b/tests/fixtures/replay-regressions/cooperative-cold-selection-handle.json @@ -0,0 +1,69 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-cooperative-cold-selection-handle", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": {"type":"corpus.durable-selection", "input":[], "payload_codec":"avro"}, + "history": [ + { + "id":"event-slow-open", "event_type":"ActivityScheduled", "payload": { + "sequence":1, "activity_execution_id":"activity-slow", "activity_type":"slow-activity", + "parallel_group_id":"select-calls:1:2", "parallel_group_kind":"activity", "parallel_group_mode":"select", + "parallel_group_base_sequence":1, "parallel_group_size":2, "parallel_group_index":0, + "selection_member_key":"slow", "selection_member_index":0, "selection_member_base_sequence":1, + "selection_member_size":1, "selection_member_kind":"activity" + } + }, + { + "id":"event-fast-open", "event_type":"ActivityScheduled", "payload": { + "sequence":2, "activity_execution_id":"activity-fast", "activity_type":"fast-activity", + "parallel_group_id":"select-calls:1:2", "parallel_group_kind":"activity", "parallel_group_mode":"select", + "parallel_group_base_sequence":1, "parallel_group_size":2, "parallel_group_index":1, + "selection_member_key":"fast", "selection_member_index":1, "selection_member_base_sequence":2, + "selection_member_size":1, "selection_member_kind":"activity" + } + }, + { + "id":"event-fast-completed", "event_type":"ActivityCompleted", "payload": { + "sequence":2, "activity_execution_id":"activity-fast", "activity_type":"fast-activity", + "result":"wwHioz3/VYAiNwoYd2lubmVyLXZhbHVl", "payload_codec":"avro" + } + }, + { + "id":"event-selection", "event_type":"SelectionResolved", "payload": { + "selection_group_id":"select-calls:1:2", "selection_group_base_sequence":1, "selection_group_size":2, + "member_key":"fast", "member_index":1, "member_base_sequence":2, "member_size":1, + "operation_kind":"activity", "operation_identity":"activity-fast", "outcome":"completed", + "resolution_event_id":"event-fast-completed", "resolution_event_type":"ActivityCompleted" + } + }, + { + "event_type":"CooperativeCancellationRequested", "workflow_command_id":"original-request", + "recorded_at":"2026-10-01T08:00:00Z", "payload": { + "workflow_command_id":"original-request", "workflow_run_id":"regression-corpus-run", + "cleanup_deadline_at":"2026-10-01T08:10:00Z" + } + }, + { + "event_type":"CooperativeCancellationDelivered", "workflow_command_id":"original-request", "payload": { + "workflow_command_id":"original-request", "workflow_run_id":"regression-corpus-run", + "sequence":3, "call_kind":"selection_handle", "operation_sequence":1 + } + }, + { + "id":"event-slow-late", "event_type":"ActivityCompleted", "payload": { + "sequence":1, "activity_execution_id":"activity-slow", "activity_type":"slow-activity", + "result":"wwHioz3/VYAiNwoWbG9zZXItdmFsdWU=", "payload_codec":"avro" + } + } + ], + "command_sequence": [{ + "type":"fail_workflow", "exception_type":"WorkflowCancellationRequested", "non_retryable":true, + "exception": {"properties": {"request_id":"original-request", "cleanup_deadline_at":"2026-10-01T08:10:00Z"}} + }], + "expected": { + "type":"fail_workflow", "exception_type":"WorkflowCancellationRequested", "non_retryable":true, + "exception": {"properties": {"request_id":"original-request", "cleanup_deadline_at":"2026-10-01T08:10:00Z"}} + } +} diff --git a/tests/fixtures/replay-regressions/cooperative-duplicate-request-rejected.json b/tests/fixtures/replay-regressions/cooperative-duplicate-request-rejected.json new file mode 100644 index 0000000..c9cb5d2 --- /dev/null +++ b/tests/fixtures/replay-regressions/cooperative-duplicate-request-rejected.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-cooperative-duplicate-request-rejected", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": { + "type": "corpus.side-effect-version", + "input": [], + "payload_codec": "avro" + }, + "history": [ + { + "event_type": "CooperativeCancellationRequested", + "workflow_command_id": "original-request", + "recorded_at": "2026-10-01T08:00:00Z", + "payload": { + "workflow_command_id": "original-request", + "workflow_run_id": "regression-corpus-run", + "cleanup_deadline_at": "2026-10-01T08:10:00Z" + } + }, + { + "event_type": "CooperativeCancellationRequested", + "workflow_command_id": "original-request", + "recorded_at": "2026-10-01T08:00:00Z", + "payload": { + "workflow_command_id": "original-request", + "workflow_run_id": "regression-corpus-run", + "cleanup_deadline_at": "2026-10-01T08:10:00Z" + } + } + ], + "command_sequence": [{"type":"complete_workflow", "result":"unreachable"}], + "expected": {"type":"complete_workflow", "result":"unreachable"}, + "expected_failure": "cooperative_cancellation_history_invalid" +} diff --git a/tests/fixtures/replay-regressions/cooperative-grouped-condition-reopen.json b/tests/fixtures/replay-regressions/cooperative-grouped-condition-reopen.json new file mode 100644 index 0000000..80adff9 --- /dev/null +++ b/tests/fixtures/replay-regressions/cooperative-grouped-condition-reopen.json @@ -0,0 +1,69 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-cooperative-grouped-condition-reopen", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": { + "type": "corpus.grouped-condition-reopen", + "input": [], + "payload_codec": "avro" + }, + "history": [ + { + "event_type": "TimerScheduled", + "payload": { + "sequence": 1, "timer_id": "timer-1", "delay_seconds": 300, + "parallel_group_id": "parallel-calls:1:2", "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 1, "parallel_group_size": 2, "parallel_group_index": 0 + } + }, + { + "event_type": "ConditionWaitOpened", + "payload": { + "sequence": 2, "condition_wait_id": "condition-2", + "condition_wait_occurrence_id": "rust:condition-wait:0", "condition_key": "two-votes", + "condition_definition_fingerprint": "sha256:two-votes-v1", + "parallel_group_id": "parallel-calls:1:2", "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 1, "parallel_group_size": 2, "parallel_group_index": 1 + } + }, + { + "event_type": "ConditionWaitSatisfied", + "payload": { + "sequence": 2, "condition_wait_id": "condition-2", + "condition_wait_occurrence_id": "rust:condition-wait:0", "condition_key": "two-votes", + "condition_definition_fingerprint": "sha256:two-votes-v1", + "parallel_group_id": "parallel-calls:1:2", "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 1, "parallel_group_size": 2, "parallel_group_index": 1 + } + }, + { + "event_type": "ConditionWaitOpened", + "payload": { + "sequence": 3, "condition_wait_id": "condition-3", + "condition_wait_occurrence_id": "rust:condition-wait:0", "condition_key": "two-votes", + "condition_definition_fingerprint": "sha256:two-votes-v1", + "parallel_group_id": "parallel-calls:1:2", "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 1, "parallel_group_size": 2, "parallel_group_index": 1 + } + }, + { + "event_type": "CooperativeCancellationRequested", + "recorded_at": "2026-10-01T08:00:00Z", + "payload": { + "workflow_command_id": "original-request", "workflow_run_id": "regression-corpus-run", + "cleanup_deadline_at": "2026-10-01T08:10:00Z" + } + }, + { + "event_type": "CooperativeCancellationDelivered", + "payload": { + "workflow_command_id": "original-request", "workflow_run_id": "regression-corpus-run", + "sequence": 1, "call_kind": "parallel", "sequence_span": 2 + } + } + ], + "command_sequence": [{"type": "fail_workflow", "exception_type": "WorkflowCancellationRequested"}], + "expected": {"type": "fail_workflow", "exception_type": "WorkflowCancellationRequested"} +} diff --git a/tests/fixtures/run-inherited-scope-delivery.json b/tests/fixtures/run-inherited-scope-delivery.json new file mode 100644 index 0000000..14ef74c --- /dev/null +++ b/tests/fixtures/run-inherited-scope-delivery.json @@ -0,0 +1,937 @@ +{ + "native_commit": "afce2bcff8eab9a945545521a7b57695d52b6736", + "task": { + "workflow_id": "sdk-run-scope-fixture", + "run_id": "01m46rrz72hsttdx8y9m2df4nz" + }, + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "history_ranges": { + "requested": 5, + "prepared": 6, + "scope_delivered": 7, + "scope_cleaned": 10, + "root_delivered": 11, + "root_cleaned": 14 + }, + "history": [ + { + "id": "01m46rrz7bef6cws9ebad38zjy", + "namespace": "sdk-root-scopes", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m46rrz77fw3g5bmn3h6cep9m", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m46rrz77fw3g5bmn3h6cep9m", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-05T00:00:00.000000Z", + "applied_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:00.000000Z" + }, + { + "id": "01m46rrz7ftv911jxc3qgkk92a", + "namespace": "sdk-root-scopes", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "workflow_command_id": "01m46rrz77fw3g5bmn3h6cep9m", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m46rrz77fw3g5bmn3h6cep9m", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-05T00:00:00.000000Z", + "applied_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:00.000000Z" + }, + { + "id": "01m46rrzdj99vk7az33x50ndz7", + "namespace": "sdk-root-scopes", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "sequence": 1, + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:01:00.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:00.000000Z" + }, + { + "id": "01m46rrzep84mhpck4txrhwmt3", + "namespace": "sdk-root-scopes", + "sequence": 4, + "event_type": "CooperativeCancellationRequested", + "payload": { + "workflow_command_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "command_type": "request_cancellation", + "reason": "finish the entire run", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z", + "cancellation": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_workflow_instance_id": "sdk-run-scope-fixture", + "root_workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "parent_request_id": null, + "reason": "finish the entire run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-05T00:00:00.000000Z", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z", + "lineage": [ + { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz" + } + ] + }, + "command": { + "id": "01M46RRZECZJ27TVWWFQRF3YXD", + "sequence": 2, + "type": "request_cancellation", + "target_scope": "instance", + "resolved_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNw4GDHJlYXNvbgoqZmluaXNoIHRoZSBlbnRpcmUgcnVuJmNsZWFudXBfZGVhZGxpbmVfYXQKNjIwMjYtMTAtMDVUMDA6MDA6MzAuMDAwMDAwWhhjYW5jZWxsYXRpb24OGAxzY2hlbWEKUGR1cmFibGUtd29ya2Zsb3cuY2FuY2VsbGF0aW9uLWNvbnRleHQvdjEUcmVxdWVzdF9pZAo0MDFNNDZSUlpFQ1pKMjdUVldXRlFSRjNZWEQecm9vdF9yZXF1ZXN0X2lkCjQwMU00NlJSWkVDWkoyN1RWV1dGUVJGM1lYRDJyb290X3dvcmtmbG93X2luc3RhbmNlX2lkCipzZGstcnVuLXNjb3BlLWZpeHR1cmUocm9vdF93b3JrZmxvd19ydW5faWQKNDAxbTQ2cnJ6NzJoc3R0ZHg4eTltMmRmNG56InBhcmVudF9yZXF1ZXN0X2lkAAxyZWFzb24KKmZpbmlzaCB0aGUgZW50aXJlIHJ1bhJyZXF1ZXN0ZXIOBAh0eXBlCgZwaHAKbGFiZWwKDlBIUCBBUEkADHNvdXJjZQoGcGhwGHJlcXVlc3RlZF9hdAo2MjAyNi0xMC0wNVQwMDowMDowMC4wMDAwMDBaJmNsZWFudXBfZGVhZGxpbmVfYXQKNjIwMjYtMTAtMDVUMDA6MDA6MzAuMDAwMDAwWg5saW5lYWdlDAIOBhRyZXF1ZXN0X2lkCjQwMU00NlJSWkVDWkoyN1RWV1dGUVJGM1lYRCh3b3JrZmxvd19pbnN0YW5jZV9pZAoqc2RrLXJ1bi1zY29wZS1maXh0dXJlHndvcmtmbG93X3J1bl9pZAo0MDFtNDZycno3MmhzdHRkeDh5OW0yZGY0bnoAAAAA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "cancellation_requested", + "accepted_at": "2026-10-05T00:00:00.000000Z", + "applied_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:00.000000Z" + }, + { + "id": "01m46rrzf8cdgxx95rzzdrx2hm", + "namespace": "sdk-root-scopes", + "sequence": 5, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "parent_scope_id": "root", + "request_id": "01M46RRZF2NQZNW5BH8TP04RZ7", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_workflow_instance_id": "sdk-run-scope-fixture", + "root_workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "parent_request_id": null, + "reason": "finish the entire run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-05T00:00:00.000000Z", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z", + "lineage": [ + { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz" + } + ] + }, + "lineage": [ + { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z" + }, + { + "request_id": "01M46RRZF2NQZNW5BH8TP04RZ7", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z" + } + ] + } + }, + "timestamp": "2026-10-05T00:00:00.000000Z" + }, + { + "id": "01m46rrzk55z6pg33e55n885er", + "namespace": "sdk-root-scopes", + "sequence": 6, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "request_id": "01M46RRZF2NQZNW5BH8TP04RZ7", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_workflow_instance_id": "sdk-run-scope-fixture", + "root_workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "parent_request_id": null, + "reason": "finish the entire run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-05T00:00:00.000000Z", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z", + "lineage": [ + { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz" + } + ] + }, + "lineage": [ + { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z" + }, + { + "request_id": "01M46RRZF2NQZNW5BH8TP04RZ7", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z" + } + ] + }, + "sequence": 2, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-05T00:00:30.000000Z", + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:00:10.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:05.000000Z" + }, + { + "id": "01m46rrzqan7q27ajs2dtte082", + "namespace": "sdk-root-scopes", + "sequence": 7, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "request_id": "01M46RRZF2NQZNW5BH8TP04RZ7", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_workflow_instance_id": "sdk-run-scope-fixture", + "root_workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "parent_request_id": null, + "reason": "finish the entire run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-05T00:00:00.000000Z", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z", + "lineage": [ + { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz" + } + ] + }, + "lineage": [ + { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z" + }, + { + "request_id": "01M46RRZF2NQZNW5BH8TP04RZ7", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z" + } + ] + }, + "sequence": 2, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-05T00:00:30.000000Z", + "preparation_history_event_id": "01m46rrzk55z6pg33e55n885er", + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:00:15.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:05.000000Z" + }, + { + "id": "01m46rrzw27psk234pnmdv6fvj", + "namespace": "sdk-root-scopes", + "sequence": 8, + "event_type": "ActivityScheduled", + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "01m46rrzw12cmnyxcqq3g3ysjp", + "activity_class": "tests.scoped-cleanup", + "activity_type": "tests.scoped-cleanup", + "sequence": 3, + "workflow_task_id": "01m46rrz7gd3k22qas007tjrt7", + "activity": { + "id": "01m46rrzw12cmnyxcqq3g3ysjp", + "idempotency_key": "01m46rrzw12cmnyxcqq3g3ysjp", + "sequence": 3, + "type": "tests.scoped-cleanup", + "class": "tests.scoped-cleanup", + "execution_mode": "local", + "local_activity": true, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "start_to_close_timeout": null, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null, + "non_retryable_error_types": [] + }, + "cancellation_scope_id": "01M46RRZDG58VBRDV14REK37H3", + "queue": "default", + "created_at": "2026-10-05T00:00:05.000000Z", + "arguments": "wwHioz3/VYAiNwwA" + }, + "local_preparation": { + "version": 1, + "descriptor_fingerprint": "644cb65803c9776806936fa612567447325318b32ce65d7c6096fb6d793d1b2c", + "worker_attempt_id": "scope-cleanup-attempt", + "workflow_task_attempt": 1, + "cancellation_cleanup": { + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "operation_scope_id": "01M46RRZDG58VBRDV14REK37H3", + "request_id": "01M46RRZF2NQZNW5BH8TP04RZ7", + "root_request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "delivery_history_event_id": "01m46rrzqan7q27ajs2dtte082", + "preparation_history_event_id": "01m46rrzk55z6pg33e55n885er", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z", + "authority_deadline_at": "2026-10-05T00:00:30.000000Z" + } + }, + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:00:15.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:05.000000Z" + }, + { + "id": "01m46rrzwr8pnjmh25t0267099", + "namespace": "sdk-root-scopes", + "sequence": 9, + "event_type": "ActivityStarted", + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "01m46rrzw12cmnyxcqq3g3ysjp", + "activity_attempt_id": "01M46RRZW5PH9WA5VWN52TV9DK", + "activity_class": "tests.scoped-cleanup", + "activity_type": "tests.scoped-cleanup", + "sequence": 3, + "attempt_number": 1, + "workflow_task_id": "01m46rrz7gd3k22qas007tjrt7", + "lease_expires_at": "2026-10-05T00:00:15.000000Z", + "activity": { + "id": "01m46rrzw12cmnyxcqq3g3ysjp", + "idempotency_key": "01m46rrzw12cmnyxcqq3g3ysjp", + "sequence": 3, + "type": "tests.scoped-cleanup", + "class": "tests.scoped-cleanup", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "01M46RRZW5PH9WA5VWN52TV9DK", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "start_to_close_timeout": null, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null, + "non_retryable_error_types": [] + }, + "cancellation_scope_id": "01M46RRZDG58VBRDV14REK37H3", + "queue": "default", + "last_heartbeat_at": "2026-10-05T00:00:05.000000Z", + "created_at": "2026-10-05T00:00:05.000000Z", + "started_at": "2026-10-05T00:00:05.000000Z", + "arguments": "wwHioz3/VYAiNwwA" + }, + "activity_attempt": { + "id": "01M46RRZW5PH9WA5VWN52TV9DK", + "worker_attempt_id": "scope-cleanup-attempt", + "activity_execution_id": "01m46rrzw12cmnyxcqq3g3ysjp", + "task_id": "01m46rrz7gd3k22qas007tjrt7", + "attempt_number": 1, + "status": "running", + "lease_owner": "original", + "started_at": "2026-10-05T00:00:05.000000Z", + "last_heartbeat_at": "2026-10-05T00:00:05.000000Z", + "lease_expires_at": "2026-10-05T00:00:15.000000Z" + }, + "worker_attempt_id": "scope-cleanup-attempt", + "local_preparation": { + "version": 1, + "descriptor_fingerprint": "644cb65803c9776806936fa612567447325318b32ce65d7c6096fb6d793d1b2c", + "worker_attempt_id": "scope-cleanup-attempt", + "workflow_task_attempt": 1, + "cancellation_cleanup": { + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "operation_scope_id": "01M46RRZDG58VBRDV14REK37H3", + "request_id": "01M46RRZF2NQZNW5BH8TP04RZ7", + "root_request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "delivery_history_event_id": "01m46rrzqan7q27ajs2dtte082", + "preparation_history_event_id": "01m46rrzk55z6pg33e55n885er", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z", + "authority_deadline_at": "2026-10-05T00:00:30.000000Z" + }, + "schedule_to_close_deadline_at": "2026-10-05T00:00:30.000000Z" + }, + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:00:15.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:05.000000Z" + }, + { + "id": "01m46rs0217fay9tdxg92rf9hb", + "namespace": "sdk-root-scopes", + "sequence": 10, + "event_type": "ActivityCompleted", + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "01m46rrzw12cmnyxcqq3g3ysjp", + "activity_attempt_id": "01M46RRZW5PH9WA5VWN52TV9DK", + "activity_class": "tests.scoped-cleanup", + "activity_type": "tests.scoped-cleanup", + "sequence": 3, + "attempt_number": 1, + "result": "wwHioz3/VYAiNwoac2NvcGUtY2xlYW5lZA==", + "payload_codec": "avro", + "workflow_task_id": "01m46rrz7gd3k22qas007tjrt7", + "activity": { + "id": "01m46rrzw12cmnyxcqq3g3ysjp", + "idempotency_key": "01m46rrzw12cmnyxcqq3g3ysjp", + "sequence": 3, + "type": "tests.scoped-cleanup", + "class": "tests.scoped-cleanup", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "01M46RRZW5PH9WA5VWN52TV9DK", + "status": "completed", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "start_to_close_timeout": null, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null, + "non_retryable_error_types": [] + }, + "cancellation_scope_id": "01M46RRZDG58VBRDV14REK37H3", + "queue": "default", + "last_heartbeat_at": "2026-10-05T00:00:05.000000Z", + "created_at": "2026-10-05T00:00:05.000000Z", + "started_at": "2026-10-05T00:00:05.000000Z", + "closed_at": "2026-10-05T00:00:06.000000Z", + "arguments": "wwHioz3/VYAiNwwA", + "result": "wwHioz3/VYAiNwoac2NvcGUtY2xlYW5lZA==" + }, + "activity_attempt": { + "id": "01M46RRZW5PH9WA5VWN52TV9DK", + "worker_attempt_id": "scope-cleanup-attempt", + "activity_execution_id": "01m46rrzw12cmnyxcqq3g3ysjp", + "task_id": "01m46rrz7gd3k22qas007tjrt7", + "attempt_number": 1, + "status": "completed", + "lease_owner": "original", + "started_at": "2026-10-05T00:00:05.000000Z", + "last_heartbeat_at": "2026-10-05T00:00:05.000000Z", + "closed_at": "2026-10-05T00:00:06.000000Z" + }, + "worker_attempt_id": "scope-cleanup-attempt", + "local_outcome": { + "version": 1, + "report_fingerprint": "c32cb4bcc458921215e509d0b9d301da67c823423a013ea8013f40fe53e03184", + "workflow_task_attempt": 1, + "submitted_outcome": "completed" + }, + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:00:15.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:06.000000Z" + }, + { + "id": "01m46rs04d8jqrcamnx2mk6sk0", + "namespace": "sdk-root-scopes", + "sequence": 11, + "event_type": "CooperativeCancellationDelivered", + "payload": { + "workflow_command_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "sequence": 4, + "call_kind": "timer", + "cancellation": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_workflow_instance_id": "sdk-run-scope-fixture", + "root_workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "parent_request_id": null, + "reason": "finish the entire run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-05T00:00:00.000000Z", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z", + "lineage": [ + { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz" + } + ] + }, + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:00:15.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:07.000000Z" + }, + { + "id": "01m46rs06q8qxmmzy16h3y1dx5", + "namespace": "sdk-root-scopes", + "sequence": 12, + "event_type": "ActivityScheduled", + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "01m46rs06ny40t1ne7js0b8nac", + "activity_class": "tests.root-cleanup", + "activity_type": "tests.root-cleanup", + "sequence": 5, + "workflow_task_id": "01m46rrz7gd3k22qas007tjrt7", + "activity": { + "id": "01m46rs06ny40t1ne7js0b8nac", + "idempotency_key": "01m46rs06ny40t1ne7js0b8nac", + "sequence": 5, + "type": "tests.root-cleanup", + "class": "tests.root-cleanup", + "execution_mode": "local", + "local_activity": true, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "start_to_close_timeout": null, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null, + "non_retryable_error_types": [] + }, + "queue": "default", + "created_at": "2026-10-05T00:00:07.000000Z", + "arguments": "wwHioz3/VYAiNwwA" + }, + "local_preparation": { + "version": 1, + "descriptor_fingerprint": "49c9f59482e82cc4a310fd1878bd1677d3a6af46a14e94395ef953bb5db10c9a", + "worker_attempt_id": "root-cleanup-attempt", + "workflow_task_attempt": 1, + "cancellation_cleanup": { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "delivery_history_event_id": "01m46rs04d8jqrcamnx2mk6sk0", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z" + } + }, + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:00:17.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:07.000000Z" + }, + { + "id": "01m46rs06v4hr2nbq37cc217j9", + "namespace": "sdk-root-scopes", + "sequence": 13, + "event_type": "ActivityStarted", + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "01m46rs06ny40t1ne7js0b8nac", + "activity_attempt_id": "01M46RS06R879KJC0T0V7RBSCJ", + "activity_class": "tests.root-cleanup", + "activity_type": "tests.root-cleanup", + "sequence": 5, + "attempt_number": 1, + "workflow_task_id": "01m46rrz7gd3k22qas007tjrt7", + "lease_expires_at": "2026-10-05T00:00:17.000000Z", + "activity": { + "id": "01m46rs06ny40t1ne7js0b8nac", + "idempotency_key": "01m46rs06ny40t1ne7js0b8nac", + "sequence": 5, + "type": "tests.root-cleanup", + "class": "tests.root-cleanup", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "01M46RS06R879KJC0T0V7RBSCJ", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "start_to_close_timeout": null, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null, + "non_retryable_error_types": [] + }, + "queue": "default", + "last_heartbeat_at": "2026-10-05T00:00:07.000000Z", + "created_at": "2026-10-05T00:00:07.000000Z", + "started_at": "2026-10-05T00:00:07.000000Z", + "arguments": "wwHioz3/VYAiNwwA" + }, + "activity_attempt": { + "id": "01M46RS06R879KJC0T0V7RBSCJ", + "worker_attempt_id": "root-cleanup-attempt", + "activity_execution_id": "01m46rs06ny40t1ne7js0b8nac", + "task_id": "01m46rrz7gd3k22qas007tjrt7", + "attempt_number": 1, + "status": "running", + "lease_owner": "original", + "started_at": "2026-10-05T00:00:07.000000Z", + "last_heartbeat_at": "2026-10-05T00:00:07.000000Z", + "lease_expires_at": "2026-10-05T00:00:17.000000Z" + }, + "worker_attempt_id": "root-cleanup-attempt", + "local_preparation": { + "version": 1, + "descriptor_fingerprint": "49c9f59482e82cc4a310fd1878bd1677d3a6af46a14e94395ef953bb5db10c9a", + "worker_attempt_id": "root-cleanup-attempt", + "workflow_task_attempt": 1, + "cancellation_cleanup": { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "delivery_history_event_id": "01m46rs04d8jqrcamnx2mk6sk0", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z" + }, + "schedule_to_close_deadline_at": "2026-10-05T00:00:30.000000Z" + }, + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:00:17.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:07.000000Z" + }, + { + "id": "01m46rs090gqkd6hzzs8nx9t8m", + "namespace": "sdk-root-scopes", + "sequence": 14, + "event_type": "ActivityCompleted", + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "01m46rs06ny40t1ne7js0b8nac", + "activity_attempt_id": "01M46RS06R879KJC0T0V7RBSCJ", + "activity_class": "tests.root-cleanup", + "activity_type": "tests.root-cleanup", + "sequence": 5, + "attempt_number": 1, + "result": "wwHioz3/VYAiNwoYcm9vdC1jbGVhbmVk", + "payload_codec": "avro", + "workflow_task_id": "01m46rrz7gd3k22qas007tjrt7", + "activity": { + "id": "01m46rs06ny40t1ne7js0b8nac", + "idempotency_key": "01m46rs06ny40t1ne7js0b8nac", + "sequence": 5, + "type": "tests.root-cleanup", + "class": "tests.root-cleanup", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "01M46RS06R879KJC0T0V7RBSCJ", + "status": "completed", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "start_to_close_timeout": null, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null, + "non_retryable_error_types": [] + }, + "queue": "default", + "last_heartbeat_at": "2026-10-05T00:00:07.000000Z", + "created_at": "2026-10-05T00:00:07.000000Z", + "started_at": "2026-10-05T00:00:07.000000Z", + "closed_at": "2026-10-05T00:00:08.000000Z", + "arguments": "wwHioz3/VYAiNwwA", + "result": "wwHioz3/VYAiNwoYcm9vdC1jbGVhbmVk" + }, + "activity_attempt": { + "id": "01M46RS06R879KJC0T0V7RBSCJ", + "worker_attempt_id": "root-cleanup-attempt", + "activity_execution_id": "01m46rs06ny40t1ne7js0b8nac", + "task_id": "01m46rrz7gd3k22qas007tjrt7", + "attempt_number": 1, + "status": "completed", + "lease_owner": "original", + "started_at": "2026-10-05T00:00:07.000000Z", + "last_heartbeat_at": "2026-10-05T00:00:07.000000Z", + "closed_at": "2026-10-05T00:00:08.000000Z" + }, + "worker_attempt_id": "root-cleanup-attempt", + "local_outcome": { + "version": 1, + "report_fingerprint": "8cfab1f8c3d2b9f046c4652ac1fe520c8714ed495593cb07be048f9f5f1424ce", + "workflow_task_attempt": 1, + "submitted_outcome": "completed" + }, + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:00:17.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:08.000000Z" + } + ], + "native_terminal_status": "cancelled" +} diff --git a/tests/fixtures/run-inherited-scope-timers.json b/tests/fixtures/run-inherited-scope-timers.json new file mode 100644 index 0000000..1b48585 --- /dev/null +++ b/tests/fixtures/run-inherited-scope-timers.json @@ -0,0 +1,892 @@ +{ + "native_commit": "bdb6e005154f27fb48eef231beaff507eed3b4c6", + "task": { + "workflow_id": "run-scope-timers", + "run_id": "01m47mzvj3fzggw9shywzxs59p" + }, + "scopes": { + "parent": "01M47MZVM91EK5APNK3F6MQFPR", + "child": "01M47MZVMDP91A23HCY5YXK9V2" + }, + "history_ranges": { + "requested": 8, + "prepared": 9, + "scope_delivered": 11, + "scope_cleanup_scheduled": 12, + "scope_cleaned": 13, + "root_delivered": 14, + "root_cleanup_scheduled": 15, + "root_cleaned": 16 + }, + "history": [ + { + "id": "01m47mzvjaqqsx2wt3pbzvrbqp", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "sdk-root-scopes", + "payload": { + "workflow_command_id": "01m47mzvj722a7s0djfqp6xceb", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m47mzvj722a7s0djfqp6xceb", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m47mzvj3fzggw9shywzxs59p", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": null, + "workflow_command_id": "01m47mzvj722a7s0djfqp6xceb", + "recorded_at": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m47mzvjd0b5ee9bdh2nj3ys2", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "sdk-root-scopes", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "workflow_command_id": "01m47mzvj722a7s0djfqp6xceb", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m47mzvj722a7s0djfqp6xceb", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m47mzvj3fzggw9shywzxs59p", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": null, + "workflow_command_id": "01m47mzvj722a7s0djfqp6xceb", + "recorded_at": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m47mzvmaz45g5fg429em4pdf", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "namespace": "sdk-root-scopes", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "sequence": 1, + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m47mzvje3sws2b0q7c4c6rdb", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "leased_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:05:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": "01m47mzvje3sws2b0q7c4c6rdb", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m47mzvmeearb1rhh0sje8192", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "namespace": "sdk-root-scopes", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "sequence": 2, + "scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "parent_scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "shield_parent": false, + "task": { + "id": "01m47mzvje3sws2b0q7c4c6rdb", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "leased_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:05:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": "01m47mzvje3sws2b0q7c4c6rdb", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m47mzvmnsm9zxfas2zn0sg85", + "sequence": 5, + "event_type": "TimerScheduled", + "namespace": "sdk-root-scopes", + "payload": { + "timer_id": "01m47mzvmmr7hq3m5kzqq9hdwb", + "sequence": 3, + "delay_seconds": 300, + "fire_at": "2026-10-04T00:05:00.123456Z", + "cancellation_scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "task": { + "id": "01m47mzvje3sws2b0q7c4c6rdb", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "leased_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:05:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": "01m47mzvje3sws2b0q7c4c6rdb", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m47mzvp0cj76bdwk49593d34", + "sequence": 6, + "event_type": "CooperativeCancellationRequested", + "namespace": "sdk-root-scopes", + "payload": { + "workflow_command_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "command_type": "request_cancellation", + "reason": "clean up the run", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "cancellation": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_workflow_instance_id": "run-scope-timers", + "root_workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "parent_request_id": null, + "reason": "clean up the run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p" + } + ] + }, + "command": { + "id": "01M47MZVNV78SG2539WCVZGP7Z", + "sequence": 2, + "type": "request_cancellation", + "target_scope": "run", + "requested_run_id": "01m47mzvj3fzggw9shywzxs59p", + "resolved_run_id": "01m47mzvj3fzggw9shywzxs59p", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNw4GDHJlYXNvbgogY2xlYW4gdXAgdGhlIHJ1biZjbGVhbnVwX2RlYWRsaW5lX2F0CjYyMDI2LTEwLTA0VDAwOjAwOjMwLjEyMzQ1NloYY2FuY2VsbGF0aW9uDhgMc2NoZW1hClBkdXJhYmxlLXdvcmtmbG93LmNhbmNlbGxhdGlvbi1jb250ZXh0L3YxFHJlcXVlc3RfaWQKNDAxTTQ3TVpWTlY3OFNHMjUzOVdDVlpHUDdaHnJvb3RfcmVxdWVzdF9pZAo0MDFNNDdNWlZOVjc4U0cyNTM5V0NWWkdQN1oycm9vdF93b3JrZmxvd19pbnN0YW5jZV9pZAogcnVuLXNjb3BlLXRpbWVycyhyb290X3dvcmtmbG93X3J1bl9pZAo0MDFtNDdtenZqM2Z6Z2d3OXNoeXd6eHM1OXAicGFyZW50X3JlcXVlc3RfaWQADHJlYXNvbgogY2xlYW4gdXAgdGhlIHJ1bhJyZXF1ZXN0ZXIOBAh0eXBlCgZwaHAKbGFiZWwKDlBIUCBBUEkADHNvdXJjZQoGcGhwGHJlcXVlc3RlZF9hdAo2MjAyNi0xMC0wNFQwMDowMDowMC4xMjM0NTZaJmNsZWFudXBfZGVhZGxpbmVfYXQKNjIwMjYtMTAtMDRUMDA6MDA6MzAuMTIzNDU2Wg5saW5lYWdlDAIOBhRyZXF1ZXN0X2lkCjQwMU00N01aVk5WNzhTRzI1MzlXQ1ZaR1A3Wih3b3JrZmxvd19pbnN0YW5jZV9pZAogcnVuLXNjb3BlLXRpbWVycx53b3JrZmxvd19ydW5faWQKNDAxbTQ3bXp2ajNmemdndzlzaHl3enhzNTlwAAAAAA==", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "cancellation_requested", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": null, + "workflow_command_id": "01M47MZVNV78SG2539WCVZGP7Z", + "recorded_at": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m47mzvp92r71hw38e435wxr7", + "sequence": 7, + "event_type": "CancellationScopeRequested", + "namespace": "sdk-root-scopes", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "parent_scope_id": "root", + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_workflow_instance_id": "run-scope-timers", + "root_workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "parent_request_id": null, + "reason": "clean up the run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p" + } + ] + }, + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "workflow_task_id": null, + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m47mzvq9dr7mc2qz7s4a94z5", + "sequence": 8, + "event_type": "CancellationScopeRequested", + "namespace": "sdk-root-scopes", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "parent_scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "request_id": "01M47MZVQ283NWZQW43SM4NCZA", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_workflow_instance_id": "run-scope-timers", + "root_workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "parent_request_id": null, + "reason": "clean up the run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p" + } + ] + }, + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "01M47MZVQ283NWZQW43SM4NCZA", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "workflow_task_id": null, + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m47mzvv4fev6k1h72hawz9ec", + "sequence": 9, + "event_type": "CancellationScopeDeliveryPrepared", + "namespace": "sdk-root-scopes", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_workflow_instance_id": "run-scope-timers", + "root_workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "parent_request_id": null, + "reason": "clean up the run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p" + } + ] + }, + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 3, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:25.123456Z", + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [ + { + "scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "parent_scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "scope_history_event_id": "01m47mzvmeearb1rhh0sje8192", + "request_history_event_id": "01m47mzvq9dr7mc2qz7s4a94z5", + "request_id": "01M47MZVQ283NWZQW43SM4NCZA", + "propagation_history_event_id": "01m47mzvq9dr7mc2qz7s4a94z5", + "authority_deadline_at": "2026-10-04T00:00:25.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_workflow_instance_id": "run-scope-timers", + "root_workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "parent_request_id": null, + "reason": "clean up the run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p" + } + ] + }, + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "01M47MZVQ283NWZQW43SM4NCZA", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [], + "timer_members": [ + { + "sequence": 3, + "timer_id": "01m47mzvmmr7hq3m5kzqq9hdwb", + "descriptor_hash": "17f35c9f66c474e8dc05aee8884967bca0d73e7d19b304248fc600c7e400fcd3" + } + ], + "wait_members": [], + "child_members": [] + } + ], + "task": { + "id": "01m47mzvpabc0pd223wam7wb3t", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "leased_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:00:10.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": "01m47mzvpabc0pd223wam7wb3t", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m47mzw11e8n4es101s5gn6dw", + "sequence": 10, + "event_type": "TimerCancelled", + "namespace": "sdk-root-scopes", + "payload": { + "timer_id": "01m47mzvmmr7hq3m5kzqq9hdwb", + "sequence": 3, + "delay_seconds": 300, + "fire_at": "2026-10-04T00:05:00.123456Z", + "cancelled_at": "2026-10-04T00:00:09.123456Z", + "cancellation_scope": { + "schema": "durable-workflow.scoped-timer-cancellation/v1", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "request_id": "01M47MZVQ283NWZQW43SM4NCZA", + "request_history_event_id": "01m47mzvq9dr7mc2qz7s4a94z5", + "preparation_history_event_id": "01m47mzvv4fev6k1h72hawz9ec", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_workflow_instance_id": "run-scope-timers", + "root_workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "parent_request_id": null, + "reason": "clean up the run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p" + } + ] + }, + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "01M47MZVQ283NWZQW43SM4NCZA", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "authority_deadline_at": "2026-10-04T00:00:25.123456Z" + }, + "task": { + "id": "01m47mzvpabc0pd223wam7wb3t", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "leased_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:00:19.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": "01m47mzvpabc0pd223wam7wb3t", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m47mzw4xxwj9rznrtcserxsh", + "sequence": 11, + "event_type": "CancellationScopeDelivered", + "namespace": "sdk-root-scopes", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_workflow_instance_id": "run-scope-timers", + "root_workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "parent_request_id": null, + "reason": "clean up the run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p" + } + ] + }, + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 3, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:25.123456Z", + "preparation_history_event_id": "01m47mzvv4fev6k1h72hawz9ec", + "task": { + "id": "01m47mzvpabc0pd223wam7wb3t", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "leased_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:00:19.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": "01m47mzvpabc0pd223wam7wb3t", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m47mzwkbp380vd6tx5dxgjf1", + "sequence": 12, + "event_type": "TimerScheduled", + "namespace": "sdk-root-scopes", + "payload": { + "timer_id": "01m47mzwf7fzstazjgsb7g7wwn", + "sequence": 4, + "delay_seconds": 2, + "fire_at": "2026-10-04T00:00:11.123456Z", + "cancellation_cleanup": { + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "operation_scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "root_request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "delivery_history_event_id": "01m47mzw4xxwj9rznrtcserxsh", + "preparation_history_event_id": "01m47mzvv4fev6k1h72hawz9ec", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "authority_deadline_at": "2026-10-04T00:00:25.123456Z" + }, + "cancellation_scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "task": { + "id": "01m47mzvpabc0pd223wam7wb3t", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "leased_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:00:19.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": "01m47mzvpabc0pd223wam7wb3t", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m47mzwnbzg488ypk71skz0nn", + "sequence": 13, + "event_type": "TimerFired", + "namespace": "sdk-root-scopes", + "payload": { + "timer_id": "01m47mzwf7fzstazjgsb7g7wwn", + "sequence": 4, + "delay_seconds": 2, + "fire_at": "2026-10-04T00:00:11.123456Z", + "fired_at": "2026-10-04T00:00:11.123456Z", + "task": { + "id": "01m47mzwkcmek99kv4vh0fs1pe", + "type": "timer", + "status": "leased", + "available_at": "2026-10-04T00:00:11.123456Z", + "leased_at": "2026-10-04T00:00:11.123456Z", + "lease_owner": "01m47mzwkcmek99kv4vh0fs1pe", + "lease_expires_at": "2026-10-04T00:05:11.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default" + } + }, + "workflow_task_id": "01m47mzwkcmek99kv4vh0fs1pe", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:11.123456Z" + }, + { + "id": "01m47mzwqhqnc2kk3k59sc0a6g", + "sequence": 14, + "event_type": "CooperativeCancellationDelivered", + "namespace": "sdk-root-scopes", + "payload": { + "workflow_command_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "sequence": 5, + "call_kind": "timer", + "cancellation": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_workflow_instance_id": "run-scope-timers", + "root_workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "parent_request_id": null, + "reason": "clean up the run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p" + } + ] + }, + "task": { + "id": "01m47mzwndpke4wqztb0y5x2xg", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:11.123456Z", + "leased_at": "2026-10-04T00:00:19.123456Z", + "lease_owner": "replacement", + "lease_expires_at": "2026-10-04T00:00:29.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:11.123456Z", + "last_dispatched_at": "2026-10-04T00:00:11.123456Z" + } + }, + "workflow_task_id": "01m47mzwndpke4wqztb0y5x2xg", + "workflow_command_id": "01M47MZVNV78SG2539WCVZGP7Z", + "recorded_at": "2026-10-04T00:00:19.123456Z" + }, + { + "id": "01m47mzwsb92kb88fhpsqrvnjy", + "sequence": 15, + "event_type": "TimerScheduled", + "namespace": "sdk-root-scopes", + "payload": { + "timer_id": "01m47mzws9w9qz1vrxrqj0jsvn", + "sequence": 6, + "delay_seconds": 2, + "fire_at": "2026-10-04T00:00:21.123456Z", + "task": { + "id": "01m47mzwndpke4wqztb0y5x2xg", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:11.123456Z", + "leased_at": "2026-10-04T00:00:19.123456Z", + "lease_owner": "replacement", + "lease_expires_at": "2026-10-04T00:00:29.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:11.123456Z", + "last_dispatched_at": "2026-10-04T00:00:11.123456Z" + } + }, + "workflow_task_id": "01m47mzwndpke4wqztb0y5x2xg", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:19.123456Z" + }, + { + "id": "01m47mzwvgn43gp86tbtjfs51p", + "sequence": 16, + "event_type": "TimerFired", + "namespace": "sdk-root-scopes", + "payload": { + "timer_id": "01m47mzws9w9qz1vrxrqj0jsvn", + "sequence": 6, + "delay_seconds": 2, + "fire_at": "2026-10-04T00:00:21.123456Z", + "fired_at": "2026-10-04T00:00:21.123456Z", + "task": { + "id": "01m47mzwsb92kb88fhpsqrvnk0", + "type": "timer", + "status": "leased", + "available_at": "2026-10-04T00:00:21.123456Z", + "leased_at": "2026-10-04T00:00:21.123456Z", + "lease_owner": "01m47mzwsb92kb88fhpsqrvnk0", + "lease_expires_at": "2026-10-04T00:05:21.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default" + } + }, + "workflow_task_id": "01m47mzwsb92kb88fhpsqrvnk0", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:21.123456Z" + } + ] +} diff --git a/tests/fixtures/source/native-scope-request.php b/tests/fixtures/source/native-scope-request.php new file mode 100644 index 0000000..457a751 --- /dev/null +++ b/tests/fixtures/source/native-scope-request.php @@ -0,0 +1,31 @@ +make(Kernel::class)->bootstrap(); +if (!$app->environment('testing')) { + throw new RuntimeException('Native scope admission requires the disposable testing application.'); +} +$input = json_decode(stream_get_contents(STDIN), true, flags: JSON_THROW_ON_ERROR); +if (!is_array($input) || count($input) !== 3 || !is_string($input['run_id'] ?? null) + || !is_string($input['workflow_id'] ?? null) || !is_string($input['scope_id'] ?? null) + || preg_match('/\Arust-cooperative-scope-boundary-[a-f0-9]{32}\z/', $input['workflow_id']) !== 1 + || $input['scope_id'] === '' || $input['scope_id'] === 'root') { + throw new RuntimeException('Native scope fixture requires its original synthetic workflow and scope.'); +} +$run = WorkflowRun::query()->where('namespace', 'default')->findOrFail($input['run_id']); +if ($run->workflow_instance_id !== $input['workflow_id']) { + throw new RuntimeException('Native scope fixture cannot cross its original workflow.'); +} +$accepted = CancellationScopeRequests::request($run, $input['scope_id'], '1.20', 30, 'connected scope receipt fixture'); +echo json_encode(['history_event_id' => $accepted->id, 'payload' => $accepted->payload], JSON_THROW_ON_ERROR), "\n"; diff --git a/tests/replay_regression_corpus.rs b/tests/replay_regression_corpus.rs index 8a0de8d..14d0568 100644 --- a/tests/replay_regression_corpus.rs +++ b/tests/replay_regression_corpus.rs @@ -408,9 +408,12 @@ async fn execute_fixture_delivery(fixture: &Value, delivery_id: &str) -> Result< | "corpus.condition-search" | "corpus.condition-search-adjacent" | "corpus.grouped-condition-reopen" + | "corpus.cancellation-remaining-condition" | "corpus.durable-selection" | "corpus.durable-selection-portable-affinity" | "corpus.redrive-boundary" + | "corpus.child-policy-author" + | "corpus.activity-policy-author" ) { return Err(format!( "replay fixture {fixture_id} has no registered Rust workflow {workflow_type:?}" @@ -542,6 +545,24 @@ async fn execute_fixture_delivery(fixture: &Value, delivery_id: &str) -> Result< ])) }); } + "corpus.cancellation-remaining-condition" => { + worker.register_workflow(workflow_type, |ctx, _| async move { + let _ = ctx.sleep(Duration::from_secs(10)).await; + let context = ctx + .cancellation_context()? + .expect("committed rich cancellation"); + let _shield = ctx.cancellation_shield()?; + ctx.parallel(vec![ + ParallelOperation::timer(Duration::from_secs(1)), + ParallelOperation::condition( + ConditionWaitOptions::new("ready", "sha256:ready"), + || Ok(false), + ), + ]) + .await?; + Ok(json!(context.remaining()?.as_secs_f64())) + }); + } "corpus.grouped-condition-reopen" => { worker.register_workflow(workflow_type, |ctx, _| async move { ctx.parallel(vec![ @@ -610,6 +631,23 @@ async fn execute_fixture_delivery(fixture: &Value, delivery_id: &str) -> Result< }, ); } + "corpus.activity-policy-author" => { + worker.register_workflow(workflow_type, |ctx, _input| async move { + ctx.activity("work", json!([])).await?; + Ok(json!("unreachable")) + }); + } + "corpus.child-policy-author" => { + worker.register_workflow(workflow_type, |ctx, _input| async move { + ctx.start_child_workflow( + "child", + ChildWorkflowOptions::new("regression-corpus"), + json!([]), + ) + .await?; + Ok(json!("unreachable")) + }); + } "corpus.redrive-boundary" => { worker.register_workflow(workflow_type, |ctx, _input| async move { ctx.activity("corpus.redrive.activity", json!([])).await?;