Implement service cooperative cancellation in Rust

.github/workflows/ci.yml

@@ -6,6 +6,19 @@ on:
   push:
     branches: [main]
   workflow_dispatch:
+    inputs:
+      cooperative_qualification:
+        description: Run cooperative protocol 1.20 cases against an isolated Server candidate
+        type: boolean
+        default: false
+      server_commit:
+        description: Exact 40-character public Server candidate SHA
+        type: string
+        default: ""
+      native_commit:
+        description: Optional exact public Native candidate SHA for readonly source qualification
+        type: string
+        default: ""
 
 permissions:
   contents: read
@@ -181,10 +194,107 @@ jobs:
           python scripts/ci/test-release-package.py
           python scripts/ci/test-example-base-urls.py
 
+  cooperative-server:
+    name: Cooperative Server source qualification
+    if: ${{ github.event_name == 'workflow_dispatch' && inputs.cooperative_qualification }}
+    needs: [action-policy, test]
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    env:
+      COMPOSE_PROJECT_NAME: rust-cooperative-${{ github.run_id }}-${{ github.run_attempt }}
+      SERVER_CANDIDATE_SHA: ${{ inputs.server_commit }}
+      NATIVE_CANDIDATE_SHA: ${{ inputs.native_commit }}
+      DURABLE_WORKFLOW_SERVER_SOURCE: ${{ github.workspace }}/.cooperative-server-source
+      DURABLE_WORKFLOW_NATIVE_SOURCE: ${{ github.workspace }}/.cooperative-native-source
+      DURABLE_WORKFLOW_AUTH_TOKEN: test-token
+      COMPOSE_FILE: docker-compose.cooperative.yml
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803  # v6
+        with:
+          persist-credentials: false
+      - name: Require an exact Server candidate
+        run: '[[ "$SERVER_CANDIDATE_SHA" =~ ^[0-9a-f]{40}$ ]]'
+      - name: Check out the public Server candidate
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803  # v6
+        with:
+          repository: durable-workflow/server
+          ref: ${{ inputs.server_commit }}
+          path: .cooperative-server-source
+          persist-credentials: false
+      - name: Verify checked-out candidate identity
+        run: test "$(git -C .cooperative-server-source rev-parse HEAD)" = "$SERVER_CANDIDATE_SHA"
+      - name: Require an exact optional Native candidate
+        if: ${{ inputs.native_commit != '' }}
+        run: '[[ "$NATIVE_CANDIDATE_SHA" =~ ^[0-9a-f]{40}$ ]]'
+      - name: Check out the public Native candidate
+        if: ${{ inputs.native_commit != '' }}
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803  # v6
+        with:
+          repository: durable-workflow/workflow
+          ref: ${{ inputs.native_commit }}
+          path: .cooperative-native-source
+          persist-credentials: false
+      - name: Verify Native identity and select its readonly source overlay
+        if: ${{ inputs.native_commit != '' }}
+        run: |
+          test "$(git -C .cooperative-native-source rev-parse HEAD)" = "$NATIVE_CANDIDATE_SHA"
+          printf 'COMPOSE_FILE=docker-compose.cooperative.yml:docker-compose.native-cancellation.yml\n' >> "$GITHUB_ENV"
+          printf 'DURABLE_WORKFLOW_NATIVE_SOURCE_QUALIFICATION=1\n' >> "$GITHUB_ENV"
+      - name: Install Rust 1.86
+        uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4  # stable
+        with:
+          toolchain: 1.86.0
+      - name: Start isolated Server stack
+        run: docker compose up -d --build --wait --wait-timeout 300
+      - name: Retain runtime identity and select endpoint
+        run: |
+          docker compose exec -T server cat /app/.package-provenance > cooperative-package-provenance.txt
+          docker compose images --format json > cooperative-images.json
+          jq --null-input --arg sdk "$GITHUB_SHA" --arg server "$SERVER_CANDIDATE_SHA" --arg native "$NATIVE_CANDIDATE_SHA" \
+            '{qualification:"source",sdk_commit:$sdk,server_commit:$server,native_source_overlay:($native | if length > 0 then . else null end)}' \
+            > cooperative-source-provenance.json
+          endpoint="$(docker compose port server 8080)"
+          echo "DURABLE_WORKFLOW_SERVER_URL=http://$endpoint" >> "$GITHUB_ENV"
+      - name: Run actual Worker cooperative scenarios
+        shell: bash
+        run: |
+          arguments=(--ignored --nocapture --test-threads=1)
+          if [ -z "$NATIVE_CANDIDATE_SHA" ]; then
+            arguments+=(--skip server_scope_opening_proves_real_native_tree_on_original_claim)
+            arguments+=(--skip server_scope_authoring_replays_nested_tree_and_deferred_membership_on_replacement)
+            arguments+=(--skip server_scope_cleanup_preserves_original_delivery_and_budget_after_replacement)
+            arguments+=(--skip server_scope_group_cleanup_preserves_original_delivery_and_budget_after_replacement)
+            arguments+=(--skip server_sigkill_scoped_cleanup_replays_before_root_with_original_30_second_deadline)
+            arguments+=(--skip server_scope_try_stops_actual_remote_callback_without_application_heartbeats)
+            arguments+=(--skip server_scope_wait_stops_actual_remote_callback_before_cleanup)
+            arguments+=(--skip server_scope_group_wait_stops_actual_remote_callback_before_cleanup)
+          fi
+          cargo test --test cooperative_server -- "${arguments[@]}" 2>&1 | tee cooperative-results.log
+      - name: Retain bounded scenario evidence
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7
+        with:
+          name: cooperative-server-${{ github.sha }}
+          path: |
+            cooperative-results.log
+            cooperative-package-provenance.txt
+            cooperative-images.json
+            cooperative-source-provenance.json
+          retention-days: 90
+          if-no-files-found: warn
+      - name: Emit failure diagnostics
+        if: ${{ failure() }}
+        run: docker compose logs --no-color --tail 100 bootstrap server worker repair
+      - name: Remove task runtime resources
+        if: ${{ always() }}
+        run: |
+          docker compose down -v
+          docker image rm "dw-rust-cooperative-server:$SERVER_CANDIDATE_SHA"
+
   target-branch-qualification:
     name: Target branch qualification
     if: ${{ always() }}
-    needs: [action-policy, regression-corpus, test, protocol, docs, package]
+    needs: [action-policy, regression-corpus, test, protocol, docs, package, cooperative-server]
     runs-on: ubuntu-latest
     timeout-minutes: 2
     steps:
@@ -200,9 +310,14 @@ jobs:
           PACKAGE_RESULT: ${{ needs.package.result }}
           PROTOCOL_RESULT: ${{ needs.protocol.result }}
           TEST_RESULT: ${{ needs.test.result }}
+          COOPERATIVE_REQUESTED: ${{ inputs.cooperative_qualification }}
+          COOPERATIVE_RESULT: ${{ needs.cooperative-server.result }}
         run: |
           test "$CORPUS_RESULT" = success
           test "$DOCS_RESULT" = success
           test "$PACKAGE_RESULT" = success
           test "$PROTOCOL_RESULT" = success
           test "$TEST_RESULT" = success
+          if [ "$COOPERATIVE_REQUESTED" = true ]; then
+            test "$COOPERATIVE_RESULT" = success
+          fi

CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+## 3.0.0-rc.1
+
+- Add cooperative whole-run cancellation with immutable request metadata,
+  lineage, deterministic cleanup time helpers and the original bounded deadline.
+- Add explicit Activity and Child TryCancel, WaitCancellationCompleted and
+  Abandon policies, and cooperative RequestCancellation on parent closure.
+- Supervise opted-in async Activity callbacks independently of application
+  heartbeats, acknowledge their stop and reject stale publication. Resume
+  shielded durable cleanup from the original delivery boundary after worker loss.
+- Keep cooperation opt-in and ordinary worker protocol 1.19. Independently
+  cancellable scopes remain a disabled source preview. Portable local activities,
+  worker sessions and sticky execution remain unsupported by this Rust profile.
+- This major release adds public cancellation fields and enum variants. Follow
+  the [v3 migration guide](https://github.com/durable-workflow/sdk-rust/blob/main/docs/migrating-to-v3.md)
+  when updating struct literals and exhaustive matches. The wire protocol remains
+  compatible with older same-major workers on a supporting Server.
+
 ## 2.1.5
 
 - Allow the ordinary Worker to append and close workflow streams when Server

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "durable-workflow"
-version = "2.1.5"
+version = "3.0.0-rc.1"
 edition = "2021"
 description = "Rust client and worker SDK for Durable Workflow Cloud and self-hosted Server"
 license = "MIT"
@@ -14,10 +14,10 @@ categories = ["api-bindings", "asynchronous"]
 include = ["/src/**", "/schema/**", "/examples/**", "/Cargo.toml", "/README.md", "/CHANGELOG.md", "/LICENSE"]
 
 [package.metadata.durable-workflow]
-product-train = "2.1.5"
+product-train = "3.0.0-rc.1"
 compatibility-authority = "protocol-manifests"
 supported-server-versions = "2.0.0"
-qualified-server-version = "2.4.0"
+qualified-server-version = "2.5.0-rc.1"
 worker-protocol-version = "1.19"
 server-worker-protocol-versions = ">=1.19,<2.0"
 portable-worker-affinity-minimum-worker-protocol-version = "1.18"
@@ -110,7 +110,7 @@ apache-avro = { version = "0.21", default-features = false }
 base64 = "0.22"
 # Keep apache-avro's dependency resolution compatible with the Rust 1.86 MSRV.
 bon = { version = "=3.8.1", default-features = false }
-chrono = { version = "0.4", default-features = false }
+chrono = { version = "0.4", default-features = false, features = ["alloc"] }
 futures-util = "0.3"
 # Keep reqwest's IDNA dependency resolution compatible with the Rust 1.86 MSRV.
 icu_collections = { version = "=2.2.0", default-features = false }

README.md

@@ -109,6 +109,14 @@ payload fetches never follow redirects.
 
 ## Compatibility
 
+The cancellation release candidate adds cooperative requests and bounded,
+replayable cleanup. Enable `Worker::cooperative_cancellation(true)` against a
+Server that advertises protocol 1.20 and the required capabilities. This profile
+supervises async Activity futures independently of application heartbeats.
+Independently cancellable scopes remain disabled. See the
+[cancellation guide](https://github.com/durable-workflow/sdk-rust/blob/main/docs/cooperative-cancellation-design.md)
+and [v3 migration guide](https://github.com/durable-workflow/sdk-rust/blob/main/docs/migrating-to-v3.md).
+
 The crate publishes its supported Server and worker-protocol ranges in
 `[package.metadata.durable-workflow]` in [`Cargo.toml`](https://github.com/durable-workflow/sdk-rust/blob/main/Cargo.toml). Runtime
 capability manifests, not matching package version strings, determine protocol

docker-compose.cooperative.yml

@@ -0,0 +1,79 @@
+services:
+  bootstrap:
+    image: dw-rust-cooperative-server:${SERVER_CANDIDATE_SHA:-candidate}
+    build:
+      context: ${DURABLE_WORKFLOW_SERVER_SOURCE:-../server}
+    command: ["server-bootstrap"]
+    environment: &runtime-env
+      APP_ENV: testing
+      APP_KEY: "base64:dGVzdGluZy1rZXktMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMg=="
+      APP_DEBUG: "false"
+      DB_CONNECTION: mysql
+      DB_HOST: mysql
+      DB_DATABASE: durable_workflow
+      DB_USERNAME: workflow
+      DB_PASSWORD: workflow
+      REDIS_HOST: redis
+      QUEUE_CONNECTION: redis
+      CACHE_STORE: redis
+      WORKFLOW_SERVER_AUTH_DRIVER: token
+      WORKFLOW_SERVER_AUTH_TOKEN: test-token
+      DW_WORKFLOW_TASK_TIMEOUT: 10
+      DW_WORKER_PROTOCOL_VERSION: "1.20"
+    depends_on:
+      mysql:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+  server:
+    image: dw-rust-cooperative-server:${SERVER_CANDIDATE_SHA:-candidate}
+    build:
+      context: ${DURABLE_WORKFLOW_SERVER_SOURCE:-../server}
+    environment: *runtime-env
+    ports:
+      - "127.0.0.1::8080"
+    depends_on:
+      bootstrap:
+        condition: service_completed_successfully
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8080/api/health"]
+      interval: 5s
+      timeout: 3s
+      retries: 12
+  worker:
+    image: dw-rust-cooperative-server:${SERVER_CANDIDATE_SHA:-candidate}
+    build:
+      context: ${DURABLE_WORKFLOW_SERVER_SOURCE:-../server}
+    command: ["php", "artisan", "queue:work", "--sleep=1", "--tries=3", "--max-time=1800"]
+    environment: *runtime-env
+    depends_on:
+      server:
+        condition: service_healthy
+  repair:
+    image: dw-rust-cooperative-server:${SERVER_CANDIDATE_SHA:-candidate}
+    # Match the production scheduler's unscoped, unthrottled repair cadence.
+    command: sh -c 'while true; do php artisan workflow:v2:repair-pass --json; sleep 10; done'
+    environment: *runtime-env
+    depends_on:
+      server:
+        condition: service_healthy
+  mysql:
+    image: mysql:8.0
+    environment:
+      MYSQL_DATABASE: durable_workflow
+      MYSQL_USER: workflow
+      MYSQL_PASSWORD: workflow
+      MYSQL_ROOT_PASSWORD: root
+    tmpfs: ["/var/lib/mysql"]
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost"]
+      interval: 3s
+      timeout: 2s
+      retries: 20
+  redis:
+    image: redis:7-alpine
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 3s
+      timeout: 2s
+      retries: 10

docker-compose.native-cancellation.yml

@@ -0,0 +1,15 @@
+# Source qualification only. Published Composer authority is retained in the image.
+services:
+  bootstrap:
+    volumes:
+      - ${DURABLE_WORKFLOW_NATIVE_SOURCE:?exact Native checkout}:/app/vendor/durable-workflow/workflow:ro
+  server:
+    volumes:
+      - ${DURABLE_WORKFLOW_NATIVE_SOURCE:?exact Native checkout}:/app/vendor/durable-workflow/workflow:ro
+      - ./tests/fixtures/source/native-scope-request.php:/app/sdk-source-fixtures/native-scope-request.php:ro
+  worker:
+    volumes:
+      - ${DURABLE_WORKFLOW_NATIVE_SOURCE:?exact Native checkout}:/app/vendor/durable-workflow/workflow:ro
+  repair:
+    volumes:
+      - ${DURABLE_WORKFLOW_NATIVE_SOURCE:?exact Native checkout}:/app/vendor/durable-workflow/workflow:ro