diff --git a/.github/feature-test-timings.json b/.github/feature-test-timings.json index 72980acaa..3f9e29a57 100644 --- a/.github/feature-test-timings.json +++ b/.github/feature-test-timings.json @@ -39,6 +39,11 @@ "tests/Feature/V2/V2AvroAdapterCodecTest.php": 5.000000, "tests/Feature/V2/V2AvroParitySuiteTest.php": 2.612838, "tests/Feature/V2/V2AwaitWorkflowTest.php": 23.893981, + "tests/Feature/V2/V2CancellationCascadeViewTest.php": 30.0, + "tests/Feature/V2/V2CancellationCleanupOutcomeTest.php": 30.0, + "tests/Feature/V2/V2CancellationRequestConcurrencyTest.php": 10.0, + "tests/Feature/V2/V2CancellationScopeDescendantsTest.php": 134.011351, + "tests/Feature/V2/V2CancellationScopeHistoryTest.php": 10.0, "tests/Feature/V2/V2ChildWorkflowExternalOutputReplayTest.php": 0.114765, "tests/Feature/V2/V2ChildWorkflowNamespaceProjectionTest.php": 0.162660, "tests/Feature/V2/V2CompatibilityWorkflowTest.php": 7.354094, @@ -138,6 +143,11 @@ "tests/Feature/V2/V2AvroAdapterCodecTest.php": 6.000000, "tests/Feature/V2/V2AvroParitySuiteTest.php": 2.820501, "tests/Feature/V2/V2AwaitWorkflowTest.php": 16.564047, + "tests/Feature/V2/V2CancellationCascadeViewTest.php": 30.0, + "tests/Feature/V2/V2CancellationCleanupOutcomeTest.php": 30.0, + "tests/Feature/V2/V2CancellationRequestConcurrencyTest.php": 10.0, + "tests/Feature/V2/V2CancellationScopeDescendantsTest.php": 182.205493, + "tests/Feature/V2/V2CancellationScopeHistoryTest.php": 10.0, "tests/Feature/V2/V2ChildWorkflowExternalOutputReplayTest.php": 0.191675, "tests/Feature/V2/V2ChildWorkflowNamespaceProjectionTest.php": 0.216388, "tests/Feature/V2/V2CompatibilityWorkflowTest.php": 11.312729, diff --git a/.github/pr-test-selection.json b/.github/pr-test-selection.json index dae6af760..60215538d 100644 --- a/.github/pr-test-selection.json +++ b/.github/pr-test-selection.json @@ -4,10 +4,13 @@ "unit_contracts": [ "tests/Unit/Serializers/ExternalStorageEnvelopeTest.php", "tests/Unit/Traits/ResolvesMethodDependenciesTest.php", + "tests/Unit/V2/ActivityCancellationCompletionTest.php", + "tests/Unit/V2/CancellationContextTest.php", "tests/Unit/V2/CompiledWorkflowDefinitionTest.php", "tests/Unit/V2/ExternalPayloadStorageTest.php", "tests/Unit/V2/PlatformProtocolSpecsTest.php", "tests/Unit/V2/ReadmeBranchDestinationsTest.php", + "tests/Unit/V2/ScopedCancellationContextTest.php", "tests/Unit/V2/StartOptionsTest.php", "tests/Unit/V2/VersionMarkerWireFormatTest.php", "tests/Unit/V2/WorkflowFiberRunnerTest.php", diff --git a/.github/workflows/php.yml b/.github/workflows/php.yml index ed945ef5f..035adaaf6 100644 --- a/.github/workflows/php.yml +++ b/.github/workflows/php.yml @@ -330,6 +330,18 @@ jobs: vendor/bin/phpunit --testdox \ --log-junit build/test-results/pr-unit-contracts.xml + - name: Run workflow command normalizer unit cases + run: >- + vendor/bin/phpunit tests/Unit/V2/WorkflowCommandNormalizerTest.php + --fail-on-warning --log-junit build/test-results/pr-workflow-command-normalizer.xml + env: + REDIS_HOST: 127.0.0.1 + + - name: Run cancellation timeline unit cases + run: >- + vendor/bin/phpunit tests/Unit/V2/CancellationHistoryTimelineTest.php + --fail-on-warning --log-junit build/test-results/pr-cancellation-timeline.xml + pr-feature-mysql: needs: preflight if: ${{ github.event_name == 'pull_request' }} @@ -478,6 +490,138 @@ jobs: QUEUE_CONNECTION: redis XDEBUG_MODE: off + - name: Run scoped Activity fencing and original stop receipt cases + run: >- + vendor/bin/phpunit tests/Feature/V2/V2ScopedActivityCancellationTest.php + --testsuite feature + --fail-on-warning --log-junit build/test-results/pr-smoke-scoped-activities.xml + env: + DB_CONNECTION: mysql + QUEUE_CONNECTION: redis + XDEBUG_MODE: off + + - name: Run scoped timer fencing and late fire cases + run: >- + vendor/bin/phpunit tests/Feature/V2/V2ScopedTimerCancellationTest.php + --testsuite feature + --fail-on-warning --log-junit build/test-results/pr-smoke-scoped-timers.xml + env: + DB_CONNECTION: mysql + QUEUE_CONNECTION: redis + XDEBUG_MODE: off + + - name: Run canonical operation scope delivery boundary cases + run: >- + vendor/bin/phpunit tests/Feature/V2/V2CancellationScopeDeliveryTest.php + --testsuite feature + --fail-on-warning --log-junit build/test-results/pr-smoke-scope-delivery.xml + env: + DB_CONNECTION: mysql + QUEUE_CONNECTION: redis + XDEBUG_MODE: off + + - name: Run canonical operation scope request authority cases + run: >- + vendor/bin/phpunit tests/Feature/V2/V2CancellationScopeRequestsTest.php + --testsuite feature + --fail-on-warning --log-junit build/test-results/pr-smoke-scope-requests.xml + env: + DB_CONNECTION: mysql + QUEUE_CONNECTION: redis + XDEBUG_MODE: off + + - name: Run canonical operation scope admission cases + run: >- + vendor/bin/phpunit tests/Feature/V2/V2CancellationScopeHistoryTest.php + --testsuite feature + --fail-on-warning --log-junit build/test-results/pr-smoke-scope-admission.xml + env: + DB_CONNECTION: mysql + QUEUE_CONNECTION: redis + XDEBUG_MODE: off + + - name: Run activity cancellation policy and portable admission cases + run: >- + vendor/bin/phpunit tests/Feature/V2/V2PortableCancellationDeliveryTest.php + --testsuite feature + --filter '/testInternal.*(ActivityWait|ActivityAbandon)|testPortableRemoteActivityPolicy/' + --fail-on-warning --log-junit build/test-results/pr-smoke-activity-cancellation-wait.xml + env: + DB_CONNECTION: mysql + QUEUE_CONNECTION: redis + XDEBUG_MODE: off + + - name: Run local callback original-claim receipt cases + run: >- + vendor/bin/phpunit tests/Feature/V2/V2PortableCancellationDeliveryTest.php + --testsuite feature + --filter '/(testLocalStopReceipt|testLocalOriginalOwner|testLateLocalReceipt)/' + --fail-on-warning --log-junit build/test-results/pr-smoke-local-claim.xml + env: + DB_CONNECTION: mysql + QUEUE_CONNECTION: redis + XDEBUG_MODE: off + + - name: Run portable local preparation cases + run: >- + vendor/bin/phpunit tests/Feature/V2/V2PortableLocalActivityPreparationTest.php + --testsuite feature + --fail-on-warning --log-junit build/test-results/pr-smoke-local-preparation.xml + env: + DB_CONNECTION: mysql + QUEUE_CONNECTION: redis + XDEBUG_MODE: off + + - name: Run portable local retained-claim checkpoint cases + run: >- + vendor/bin/phpunit tests/Feature/V2/V2PortableLocalActivityCheckpointTest.php + --testsuite feature + --fail-on-warning --log-junit build/test-results/pr-smoke-local-checkpoint.xml + env: + DB_CONNECTION: mysql + QUEUE_CONNECTION: redis + XDEBUG_MODE: off + + - name: Run portable local prepared outcome cases + run: >- + vendor/bin/phpunit tests/Feature/V2/V2PortableLocalActivityOutcomeTest.php + --testsuite feature + --fail-on-warning --log-junit build/test-results/pr-smoke-local-outcomes.xml + env: + DB_CONNECTION: mysql + QUEUE_CONNECTION: redis + XDEBUG_MODE: off + + - name: Run portable local cold recovery cases + run: >- + vendor/bin/phpunit tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php + --testsuite feature + --fail-on-warning --log-junit build/test-results/pr-smoke-local-recovery.xml + env: + DB_CONNECTION: mysql + QUEUE_CONNECTION: redis + XDEBUG_MODE: off + + - name: Run portable local supervisor control cases + run: >- + vendor/bin/phpunit tests/Feature/V2/V2PortableLocalActivityControlTest.php + --testsuite feature + --fail-on-warning --log-junit build/test-results/pr-smoke-local-control.xml + env: + DB_CONNECTION: mysql + QUEUE_CONNECTION: redis + XDEBUG_MODE: off + + - name: Run portable local cleanup and application heartbeat cases + run: >- + vendor/bin/phpunit tests/Feature/V2/V2PortableLocalActivityCleanupTest.php + --testsuite feature + --fail-on-warning --log-junit build/test-results/pr-smoke-local-cleanup.xml + env: + DB_CONNECTION: mysql + QUEUE_CONNECTION: redis + XDEBUG_MODE: off + - name: Upload pull-request MySQL smoke timing report if: ${{ always() && github.server_url == 'https://github.com' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -498,7 +642,7 @@ jobs: needs: preflight if: ${{ github.event_name != 'pull_request' }} runs-on: ubuntu-latest - timeout-minutes: 18 + timeout-minutes: 21 strategy: fail-fast: false @@ -597,7 +741,7 @@ jobs: - name: Run feature test suite (MySQL shard ${{ matrix.shard }}) id: mysql_suite - timeout-minutes: 12 + timeout-minutes: 15 run: | mkdir -p build/test-results php scripts/ci/split-feature-tests.php \ @@ -612,7 +756,7 @@ jobs: cp /tmp/mysql-shard-files "build/test-results/mysql-shard-${{ matrix.shard }}-files.txt" started_at=$(date +%s) set +e - timeout --foreground 12m xargs -a /tmp/mysql-shard-files \ + timeout --foreground 15m xargs -a /tmp/mysql-shard-files \ vendor/bin/phpunit --testdox --testsuite feature \ --coverage-clover "build/test-results/mysql-shard-${{ matrix.shard }}-coverage.xml" \ --log-junit "build/test-results/mysql-shard-${{ matrix.shard }}.xml" diff --git a/composer.json b/composer.json index 99b96e5f2..fdb8aad8f 100644 --- a/composer.json +++ b/composer.json @@ -79,7 +79,7 @@ "dev-main": "2.0.x-dev" }, "durable-workflow": { - "product-train": "2.3.5", + "product-train": "2.4.0-rc.1", "laravel-embedded-upgrade-contract": "resources/laravel-embedded-upgrade-contract.json", "laravel-dependency-security-policy": "resources/laravel-dependency-security-policy.json" }, diff --git a/docs/api-stability.md b/docs/api-stability.md index bc417bd13..f45bf2aa7 100644 --- a/docs/api-stability.md +++ b/docs/api-stability.md @@ -220,6 +220,22 @@ request admission, observation, worker capability checks, compatible replay and a versioned wire specification before advertising cooperative cancellation to service workers. +## Prepared local activity bridge + +The candidate protocol 1.20 `PreparedLocalActivityTaskBridge` is a separate +optional role for local preparation, prefix checkpointing, outcome, recovery, +supervisor control, application heartbeat and joined-stop persistence. Check +the actual bound bridge before advertising it. Ordinary and cooperative-only +bridges retain their existing interfaces, and protocol 1.19 keeps its published +local execution path. The role is not yet a published capability contract. + +Prepared cleanup requires canonical request and delivery identities and a +later authored sequence. Its immutable root budget comes from recorded +cancellation history. Supervisor renewal changes both leases without an +application heartbeat. Application heartbeat records progress and updates +only its heartbeat timeout. Neither can move the original total or cleanup +deadline, revive expired authority or establish physical callback stop. + ## Workflow service operation caller API PHP workflow code can initiate a durable Nexus service operation from inside @@ -485,21 +501,23 @@ class, source file path, or stack trace to replay or handle a failure. | `ActivityRetryScheduled` | `activity_execution_id`, `activity_attempt_id`, `activity_class`, `activity_type`, `sequence`, `retry_task_id`, `retry_of_task_id`, `retry_available_at`, `retry_backoff_seconds`, `retry_after_attempt_id`, `retry_after_attempt`, `retry_reason`, `max_attempts`, `retry_policy`, `timeout_kind`, `execution_mode`, `local_activity`, `workflow_task_id`, `exception_type`, `exception_class`, `message`, `code`, `exception`, `activity`, `parallel_group_id`, `parallel_group_kind`, `parallel_group_base_sequence`, `parallel_group_size`, `parallel_group_index`, `parallel_group_path` | `ActivitySnapshot`, `ActivityAttemptSnapshots`, `HistoryTimeline`, `RunTaskView`, `ParallelChildGroup` | | `ActivityCompleted` | `activity_execution_id`, `activity_attempt_id`, `activity_class`, `activity_type`, `sequence`, `attempt_number`, `result`, `payload_codec`, `execution_mode`, `local_activity`, `workflow_task_id`, `activity`, `activity_attempt`, `parallel_group_id`, `parallel_group_kind`, `parallel_group_base_sequence`, `parallel_group_size`, `parallel_group_index`, `parallel_group_path` | `WorkflowExecutor`, `QueryStateReplayer`, `ParallelChildGroup`, `ActivityRecovery` | | `ActivityFailed` | `activity_execution_id`, `activity_attempt_id`, `activity_class`, `activity_type`, `sequence`, `attempt_number`, `failure_id`, `failure_category`, `non_retryable`, `exception_type`, `exception_class`, `message`, `code`, `exception`, `execution_mode`, `local_activity`, `workflow_task_id`, `activity`, `activity_attempt`, `parallel_group_id`, `parallel_group_kind`, `parallel_group_base_sequence`, `parallel_group_size`, `parallel_group_index`, `parallel_group_path`, `structural_limit_kind`, `structural_limit_value`, `structural_limit_configured` | `ActivitySnapshot`, `FailureSnapshots`, `HistoryTimeline`, `ParallelFailureSelector`, `ParallelChildGroup` | -| `ActivityCancelled` | `workflow_command_id`, `activity_execution_id`, `activity_attempt_id`, `activity_class`, `activity_type`, `sequence`, `attempt_number`, `cancelled_at`, `execution_mode`, `local_activity`, `workflow_task_id`, `activity`, `activity_attempt` | `ActivitySnapshot`, `ActivityAttemptSnapshots`, `HistoryTimeline`, cancellation repair projections | +| `ActivityCancelled` | `workflow_command_id`, `activity_execution_id`, `activity_attempt_id`, `worker_attempt_id`, `activity_class`, `activity_type`, `sequence`, `attempt_number`, `cancelled_at`, `execution_mode`, `local_activity`, `workflow_task_id`, `activity`, `activity_attempt`, `parallel_group_id`, `parallel_group_kind`, `parallel_group_base_sequence`, `parallel_group_size`, `parallel_group_index`, `parallel_group_path` | `ActivitySnapshot`, `ActivityAttemptSnapshots`, `HistoryTimeline`, cancellation repair projections | | `ActivityTimedOut` | `activity_execution_id`, `activity_attempt_id`, `activity_class`, `activity_type`, `sequence`, `attempt_number`, `failure_id`, `failure_category`, `timeout_kind`, `message`, `exception_class`, `schedule_deadline_at`, `close_deadline_at`, `schedule_to_close_deadline_at`, `heartbeat_deadline_at`, `execution_mode`, `local_activity`, `workflow_task_id`, `activity`, `activity_attempt`, `parallel_group_id`, `parallel_group_kind`, `parallel_group_base_sequence`, `parallel_group_size`, `parallel_group_index`, `parallel_group_path` | `ActivitySnapshot`, `FailureSnapshots`, `HistoryTimeline`, `ParallelChildGroup`, timeout repair projections | | `TimerScheduled` | `timer_id`, `sequence`, `delay_seconds`, `fire_at`, `timer_kind`, `condition_wait_id`, `condition_key`, `condition_definition_fingerprint`, `signal_wait_id`, `signal_name` | `WorkflowStepHistory`, `QueryStateReplayer`, `RunTimerView`, `ConditionWaits`, `SignalWaits` | -| `TimerFired` | `timer_id`, `sequence`, `delay_seconds`, `fire_at`, `fired_at`, `timer_kind`, `condition_wait_id`, `condition_key`, `condition_definition_fingerprint`, `signal_wait_id`, `signal_name` | `WorkflowStepHistory`, `QueryStateReplayer`, `RunTimerView`, `ConditionWaits`, `SignalWaits` | +| `TimerFired` | `timer_id`, `sequence`, `delay_seconds`, `fire_at`, `fired_at`, `timer_kind`, `condition_wait_id`, `condition_key`, `condition_definition_fingerprint`, `signal_wait_id`, `signal_name`, `cancellation_scope_id` (candidate 1.20) | `WorkflowStepHistory`, `QueryStateReplayer`, `RunTimerView`, `ConditionWaits`, `SignalWaits` | | `TimerCancelled` | `timer_id`, `sequence`, `delay_seconds`, `fire_at`, `timer_kind`, `condition_wait_id`, `condition_key`, `condition_definition_fingerprint`, `signal_wait_id`, `signal_name`, `cancelled_at` | `WorkflowStepHistory`, `RunTimerView`, `ConditionWaits`, `SignalWaits`, `HistoryTimeline` | | `SignalReceived` | `workflow_command_id`, `signal_id`, `workflow_instance_id`, `workflow_run_id`, `signal_name`, `signal_wait_id`, `arguments`, `payload_codec` | `SignalWaits`, `RunSignalView`, public and worker history payload consumers | | `SignalApplied` | `workflow_command_id`, `signal_id`, `signal_name`, `signal_wait_id`, `sequence`, `value` | `WorkflowStepHistory`, `SignalWaits`, `RunSignalView`, `QueryStateReplayer` | -| `SignalWaitOpened` | `signal_name`, `signal_wait_id`, `sequence`, `timeout_seconds` | `WorkflowStepHistory`, `SignalWaits`, `RunSignalView`, `HistoryTimeline` | +| `SignalWaitOpened` | `signal_name`, `signal_wait_id`, `sequence`, `timeout_seconds`, `cancellation_scope_id` (candidate 1.20) | `WorkflowStepHistory`, `SignalWaits`, `RunSignalView`, `HistoryTimeline` | +| `SignalWaitCancelled` (candidate 1.20) | `signal_name`, `signal_wait_id`, `sequence`, `timeout_seconds`, `timer_id`, `cancelled_at`, `cancellation_scope` | `ScopedWaitCancellation`, `SignalWaits`, `HistoryTimeline`, scope delivery and worker history consumers | | `UpdateAccepted` | `workflow_command_id`, `update_id`, `workflow_instance_id`, `workflow_run_id`, `update_name`, `arguments`, `ordering_state`, `queued_behind_command_id`, `queued_behind_command_sequence`, `queued_behind_command_type` | `RunUpdateView`, worker history payload consumers | | `UpdateRejected` | `workflow_command_id`, `update_id`, `workflow_instance_id`, `workflow_run_id`, `update_name`, `arguments`, `validation_errors` | `RunUpdateView`, `HistoryTimeline`, command-contract projections | | `UpdateApplied` | `workflow_command_id`, `update_id`, `workflow_instance_id`, `workflow_run_id`, `update_name`, `arguments`, `sequence` | `QueryStateReplayer`, `RunUpdateView`, worker history payload consumers | | `UpdateCompleted` | `workflow_command_id`, `update_id`, `workflow_instance_id`, `workflow_run_id`, `update_name`, `sequence`, `result`, `failure_id`, `failure_category`, `non_retryable`, `exception_type`, `exception_class`, `message`, `code`, `exception`, `terminal_reason` | `RunUpdateView`, worker history payload consumers | -| `ConditionWaitOpened` | `condition_wait_id`, `condition_key`, `condition_definition_fingerprint`, `sequence`, `timeout_seconds` | `WorkflowStepHistory`, `ConditionWaits`, worker history payload consumers | +| `ConditionWaitOpened` | `condition_wait_id`, `condition_key`, `condition_definition_fingerprint`, `sequence`, `timeout_seconds`, `cancellation_scope_id` (candidate 1.20) | `WorkflowStepHistory`, `ConditionWaits`, worker history payload consumers | | `ConditionWaitSatisfied` | `condition_wait_id`, `condition_key`, `condition_definition_fingerprint`, `sequence`, `timer_id`, `timeout_seconds`, `workflow_signal_id`, `signal_name`, `signal_wait_id` | `WorkflowStepHistory`, `ConditionWaits`, `QueryStateReplayer`, `HistoryTimeline` | | `ConditionWaitTimedOut` | `condition_wait_id`, `condition_key`, `condition_definition_fingerprint`, `sequence`, `timer_id`, `timeout_seconds` | `WorkflowStepHistory`, `ConditionWaits`, `QueryStateReplayer`, `HistoryTimeline` | +| `ConditionWaitCancelled` (candidate 1.20) | `condition_wait_id`, `condition_wait_occurrence_id`, `condition_key`, `condition_definition_fingerprint`, `sequence`, `timer_id`, `timeout_seconds`, `cancelled_at`, `cancellation_scope` | `ScopedWaitCancellation`, `ConditionWaits`, `HistoryTimeline`, scope delivery and worker history consumers | | `SideEffectRecorded` | `sequence`, `result` | `WorkflowStepHistory`, `WorkflowExecutor`, `QueryStateReplayer` | | `VersionMarkerRecorded` | `sequence`, `change_id`, `version`, `min_supported`, `max_supported` | `WorkflowStepHistory`, `WorkflowExecutor`, `QueryStateReplayer` | | `ChildWorkflowScheduled` | `sequence`, `workflow_link_id`, `child_call_id`, `child_workflow_instance_id`, `child_workflow_run_id`, `child_workflow_class`, `child_workflow_type`, `child_run_number`, `parent_close_policy`, `retry_policy`, `timeout_policy`, `parallel_group_id`, `parallel_group_kind`, `parallel_group_base_sequence`, `parallel_group_size`, `parallel_group_index`, `parallel_group_path` | `WorkflowStepHistory`, `WorkflowExecutor`, `QueryStateReplayer`, `ChildRunHistory`, `ParallelChildGroup`, `RunLineageView` | @@ -516,7 +534,7 @@ class, source file path, or stack trace to replay or handle a failure. | `MemoUpserted` | `sequence`, `entries`, `merged` | `WorkflowStepHistory`, `HistoryTimeline`, run detail projections, history export | | `RepairRequested` | `workflow_command_id`, `workflow_instance_id`, `workflow_run_id`, `command_type`, `outcome`, `liveness_state`, `wait_kind`, `task_id`, `task_type` | `HistoryTimeline`, `RunCommandContract`, repair diagnostics, operator detail projections | | `CancelRequested` | `workflow_command_id`, `workflow_instance_id`, `workflow_run_id`, `command_type`, `reason` | `HistoryTimeline`, `RunCommandContract`, cancellation projections | -| `WorkflowCancelled` | `workflow_command_id`, `workflow_instance_id`, `workflow_run_id`, `failure_id`, `failure_category`, `closed_reason`, `exception_class`, `message`, `reason` | `HistoryTimeline`, `FailureSnapshots`, `ChildRunHistory`, cancellation projections | +| `WorkflowCancelled` | `workflow_command_id`, `workflow_instance_id`, `workflow_run_id`, `failure_id`, `failure_category`, `closed_reason`, `exception_class`, `message`, `reason`, `cancellation_cleanup` | `HistoryTimeline`, `FailureSnapshots`, `ChildRunHistory`, cancellation projections | | `TerminateRequested` | `workflow_command_id`, `workflow_instance_id`, `workflow_run_id`, `command_type`, `reason` | `HistoryTimeline`, `RunCommandContract`, termination projections | | `WorkflowTerminated` | `workflow_command_id`, `workflow_instance_id`, `workflow_run_id`, `failure_id`, `failure_category`, `closed_reason`, `exception_class`, `message`, `reason` | `HistoryTimeline`, `FailureSnapshots`, `ChildRunHistory`, termination projections | | `ArchiveRequested` | `workflow_command_id`, `workflow_instance_id`, `workflow_run_id`, `command_type`, `outcome`, `reason` | `HistoryTimeline`, `RunCommandContract`, archive/export projections | diff --git a/docs/architecture/cancellation-scope.md b/docs/architecture/cancellation-scope.md index a59751185..c28ff05c6 100644 --- a/docs/architecture/cancellation-scope.md +++ b/docs/architecture/cancellation-scope.md @@ -71,18 +71,17 @@ It does not cover: ## Terminology -- **Cancel** — the cooperative close path. The engine throws - `Workflow\V2\Exceptions\WorkflowCancelledException` on the parent - fiber, records a `CancelRequested` event when the command is - received, and a terminal `WorkflowCancelled` event when the run - closes. `WorkflowCancelledException` extends `Error` rather than - `Exception` so a generic `catch (\Exception)` block cannot - accidentally swallow a cancellation. -- **Terminate** — the forceful close path. The engine throws - `Workflow\V2\Exceptions\WorkflowTerminatedException`, records - `TerminateRequested`, and a terminal `WorkflowTerminated` event. - Terminate does not give authoring code or activities a chance to - cooperate; it closes the run immediately. +- **Cancel** — terminal closure with a Cancelled outcome. The engine + records `CancelRequested` and `WorkflowCancelled`, closes the run + immediately and revokes open work. The recorded failure uses + `Workflow\V2\Exceptions\WorkflowCancelledException`. This does + not execute workflow-code `finally` cleanup. Use the separate + `requestCancellation()` path for bounded cooperative cleanup. +- **Terminate** — immediate terminal closure with a Terminated + outcome. The engine records `TerminateRequested` and + `WorkflowTerminated`, with a + `Workflow\V2\Exceptions\WorkflowTerminatedException` failure. + It does not wait for workflow or activity cleanup. - **Run-level scope** — the property that a cancel or terminate command closes the entire workflow run. Propagation to child runs follows the per-call `ParentClosePolicy`; there is no @@ -101,7 +100,7 @@ It does not cover: `Workflow\V2\Enums\CommandType` names the two close-driven command types that enter this contract: -- `CommandType::Cancel` (`'cancel'`) — cooperative close request. +- `CommandType::Cancel` (`'cancel'`) — terminal close with a Cancelled outcome. - `CommandType::Terminate` (`'terminate'`) — forceful close request. Both command types flow through the same external-command ingress @@ -344,7 +343,7 @@ The typed history events used by this contract: ### On the parent run -- `CancelRequested` — command ingress for a cooperative close. +- `CancelRequested` — command ingress for terminal cancellation. Payload: `workflow_command_id`, `workflow_instance_id`, `workflow_run_id`, `command_type = 'cancel'`, optional `reason`. diff --git a/docs/architecture/cooperative-cancellation-model.md b/docs/architecture/cooperative-cancellation-model.md new file mode 100644 index 000000000..5fd7fba46 --- /dev/null +++ b/docs/architecture/cooperative-cancellation-model.md @@ -0,0 +1,613 @@ +# Cooperative cancellation model + +Design work for [the shared cancellation issue](https://github.com/durable-workflow/.github/issues/136). +This document describes the next model. It does not advertise these additions +as released capabilities. + +## Customer outcome + +A cancellation request should let an application stop work and recover its +resources within an explicit budget. The user should be able to inspect the +request, its propagation, cleanup progress and final outcome. Worker crashes, +replay and duplicate requests must preserve that meaning. + +The cancellation model must demonstrate useful advantages over competing +products before the shared issue closes. Passing portable compatibility checks +alone does not establish that outcome. + +## Preserve existing contracts + +### Initial portable release boundary + +The initial portable release covers cooperative whole-run requests, the rich +immutable context, one bounded cancellation tree across children and Activities, +explicit operation policies, shielded cleanup, replay after worker loss, stale +attempt fencing and the shared API/CLI/Waterline cascade view. The published +mixed-language demonstration and comparative qualification remain required. + +Remote Activities support TryCancel, WaitCancellationCompleted and Abandon. +Abandon requires an original finite independent lifetime. Portable local +Activities support TryCancel and WaitCancellationCompleted. Local Abandon is +rejected before admission because it cannot outlive its hosting claim safely. +This limitation is part of the advertised capability contract. + +Independently cancellable operation scopes remain a source preview. The SDK +profiles default off and no general scope execution capability is advertised. +The existing scalar/all-group consumers test the recording and authority model. +Partial local supervision, mixed-membership groups, overlapping deliveries and +the full scope inspection view remain outside the initial supported release. +The separate hierarchical-scope qualification gate still applies before those +capabilities can be enabled or advertised. Those preview extensions do not +extend the stable whole-run contract or acquire a compatibility guarantee. + +The competitive claim must identify demonstrated customer advantages and the +remaining tradeoffs. A bounded cleanup tree and automatic callback supervision +are useful stronger contracts. They do not establish that every operation or +scope authoring API is better than every competing product. + +`WorkflowStub::requestCancellation()` is cooperative. `cancel()` immediately +closes a run and revokes outstanding durable work. `terminate()` also closes +the run immediately with a distinct outcome. None of these can roll back an +external side effect. + +Existing `ParentClosePolicy::RequestCancel` snapshots use `request_cancel` and +currently issue terminal cancellation. Preserve that recorded contract during +replay. The candidate `ParentClosePolicy::RequestCancellation` uses +`request_cancellation` for cooperative cleanup. The enum marks the historical +terminal policy clearly and retains its old meaning. + +Parent-close policy and per-operation cancellation policy answer different +questions. The former controls a child after its parent closes. The latter +controls what an awaiting workflow does when its scope receives cancellation. + +## Request identity and budget + +Each target run has a local request ID used to validate delivery and ownership. +A propagated request additionally carries the root request ID, root run, +immediate parent request and target run. Do not replace local delivery identity +with the root ID. + +Keep reason, requester, source, original requested-at and root cleanup deadline +in canonical history. A descendant receives the remaining budget. Propagation, +retry, continue-as-new and worker replacement must never grant a fresh budget. + +The first accepted request on a run owns its local identity, root, delivery +route and deadline. A later direct request returns that original request. +Propagation from the same root also returns the original local request, even +when another recorded parent supplies a different route. Propagation from a +different root returns `cancellation_root_conflict` with the existing and +incoming identities and deadlines. It does not replace either request or +extend the accepted budget. This conflict does not mean the two independent +cascades became one tree. Awaiting-operation policies must report the conflict +and remain bounded by their original budget. Reproducible simultaneous-request +database tests remain required before publication. + +Workflow code receives an immutable cancellation context. Its `deadline()` and +`remaining()` helpers use deterministic workflow time. Worker control checks +use runtime authority and wall-clock deadlines. A replay must not take a +different authored branch because the host clock advanced. + +The candidate Native implementation records a versioned `cancellation` snapshot +in the accepted command and canonical request history. A root request's local +and root IDs are equal. `CommandResult::cancellationContext()` exposes this +snapshot to the caller, and `WorkflowCancellationRequestedException::cancellation` +exposes it to workflow cleanup. The object contains reason, requester, source, +original requested-at, immutable deadline and lineage. Requester metadata is +restricted to caller type, ID and label. + +`remaining()` is bound to the Fiber that received canonical delivery. Detached +metadata, ended replays and other workflow Fibers cannot borrow its clock. The +binding retains neither context metadata nor the Fiber and leaves portable +serialization and value equality unchanged. + +`remaining()` reads the consumed blocking-boundary clock. It starts at recorded +delivery and advances through awaited cleanup outcomes. Synchronous side effects, +version markers, memo updates and search-attribute updates preserve it because +the service runner returns their first results before persistence. A later +persistence timestamp cannot change the same authored decision during cold +replay. The existing `now()` clock keeps its current behavior. + +Fractional seconds are preserved, expiry clamps to zero, and recorded clock +skew cannot increase the consumed budget. Missing blocking timestamps and calls +outside an active workflow Fiber fail without a host-time fallback. The runtime +supervisor independently enforces the actual deadline. Older histories that +lack the snapshot keep their existing delivery behavior and expose a null context. + +### Cleanup outcome + +Supervised portable cleanup uses renewable ownership windows of at most ten +seconds, or the configured workflow lease when it is shorter. Each window also +ends at the original cancellation deadline. Portable delivery changes the hosting claim +to this cleanup window. Replacement claims, workflow heartbeats and local +callback supervision preserve the same bound. Ordinary workflow lease settings +remain unchanged. + +Embedded Laravel callbacks retain their existing configured lease. They execute +inside the workflow transaction and do not have the portable SDK supervisor +that renews short ownership windows. The cancellation deadline still bounds +their cleanup outcome. + +Prepared local callback supervision renews the task and Activity attempt in +one transaction without requiring an application heartbeat. A dead worker +therefore leaves a reclaimable ownership window within a longer cleanup +budget. Recovery retains the original request, delivery boundary and deadline. +Lease expiry fences publication and still leaves physical callback stop state +unknown until a matching worker reports it. Replacement availability, polling +and queue repair affect recovery time. An exhausted budget ends as +`deadline_expired` rather than granting another cleanup window. + +The candidate cooperative `WorkflowCancelled` terminal event includes an +optional `cancellation_cleanup` object. It retains the local `request_id`, +original `cleanup_deadline_at`, `finished_at` and, when the canonical delivery +matches that request and authored sequence, `delivery_history_event_id` and +`delivery_sequence`. Duplicate requests leave this terminal record intact. +Legacy terminal cancellation and historical events can omit the object. + +| Outcome | Meaning | +| --- | --- | +| `completed` | Workflow cleanup reached a terminal boundary after the matching canonical cancellation delivery and before the original deadline. | +| `deadline_expired` | The original cleanup deadline was reached. This applies whether or not cancellation had been delivered to workflow code. | +| `not_delivered` | The run closed before the deadline without a recorded cancellation delivery. Ordinary shielded cleanup can still have run. | +| `unavailable` | Delivery history and its run projection disagree. The terminal record does not claim cleanup completion. | + +These outcomes describe workflow cleanup. Per-operation policies and callback +stop receipts separately establish whether activities stopped or were +abandoned. A completed workflow cleanup does not establish reversal of external +effects. Termination remains a distinct run outcome. Losing a worker lease +describes that attempt's authority and does not establish that its process +stopped or that the replacement cleanup failed. + +### Cascade inspection + +The Source `CancellationCascadeView::forRun()` candidate resolves the exact +selected run and its original root. It provides one request budget, related +run nodes and child or continuation edges. It scopes each run and instance +to the selected namespace before loading history. An unavailable relation has +no target ID or target metadata in the response. Selecting a historical run +does not follow its instance's current-run pointer. + +Each node reports the canonical terminal event separately from projected run +status. It retains local request identity, the original root budget, authored +delivery range, cleanup outcome, child propagation, activity fences, matching +worker stop reports and cleanup attempt recovery. An independent child request +keeps its own identity and budget, with the parent's canonical propagation +conflict visible. A parent-close origin preserves the parent's ordinary +terminal outcome while explaining its children's shared cancellation budget. + +The lifecycle describes requested, delivered, cleaning up, cancelled, +deadline expired or a distinct completed, failed, timed out or terminated run. +Cleaning up requires a canonical activity or timer command after the recorded +cancellation boundary. Lease expiry alone does not establish this phase. +Callback reports are labelled `reported_stopped`. A fence without a matching +receipt has unknown callback stop state. Recovery retains that unknown state +without inventing a process-kill cause. + +The response uses `durable-workflow.cancellation-cascade/v1`. Its initial limits +are 20 runs, 100 relations, 128 recent relevant history events per run and 512 +recent events overall. Original requests are also resolved separately, with +at most two request rows per lookup, so the original budget can survive an +exceeded recent-history window. Request text is capped at 8192 bytes per field. +Missing, conflicting, inaccessible or truncated evidence sets +`inspection_complete: false` and supplies a named finding with an explanation. +`inspection_complete` describes the evidence inventory, not successful cleanup +or agreement between independently cancelling roots. Server, CLI, Waterline +parity and published qualification remain separate gates for this candidate. + +The candidate Native request primitive +`attemptRequestCancellationFromParent()` reads its parent's accepted context +from storage and requires a recorded direct child link to the selected current +run. It assigns a distinct local request ID, appends lineage, and inherits the +root identity, requester, reason, original requested-at and deadline. It does +not immediately terminate the child. A late propagation retains an already +expired deadline so repair can close it without granting another budget. +Unlinked parents, ancestor cycles and legacy parents without the context +capability receive explicit diagnostics. Transaction retries clear their +by-reference results before retrying. + +Cooperative parent-close enforcement uses that same primitive. It does not +mark the child call cancelled before canonical child completion. If the parent +already accepted cancellation, every child inherits that original root and +deadline, including a deadline that has expired. A child with an independently +accepted root keeps it and produces a recorded `cancellation_root_conflict`. +The parent history and lineage view expose the request and rejection details. + +A parent that closes without accepting cancellation records one policy-owned +`ParentCloseCancellationRequested` origin. This event leaves the parent's +completed, failed, cancelled or terminated outcome intact. Its original request +time is the canonical closing event's recorded time. Its default cleanup budget +is 600 seconds from that event, shared by all affected children. Enforcement +after a delay cannot grant another 600 seconds. Repeated enforcement preserves +the origin and each accepted child request without appending duplicate applied +receipts. A mutable terminal status without canonical closing history cannot +create this budget. The parent run lock serializes origin creation and receipts. + +The candidate child-operation policy now propagates from cancellation delivery. +PHP, Python and Rust source drafts expose both child policies and preserve their +identity through cold replay. Connected PHP qualification uses one workflow +worker for parent and child, including a replacement after a released child +wait. Activity policies, simultaneous-root qualification, continuation behavior +and the complete published cascade remain required before this model is +published. + +## Operation policies + +Activities and children need three explicit choices: + +| Policy | Durable behavior | +| --- | --- | +| Try cancel | Request cancellation and release the await immediately. Cleanup can overlap work that has not yet stopped. | +| Wait for cancellation completion | Request cancellation and hold the await until canonical acknowledgement or a defined terminal outcome, bounded by the original cleanup budget. | +| Abandon | Release the await without requesting cancellation of the operation. Detached work remains inspectable. | + +Define defaults, child propagation, retry behavior and recorded policy identity +before adding SDK options. Changing a policy on replay must not reinterpret an +operation already recorded under another policy. + +The candidate Native child API exposes `CancellationPolicy::TryCancel`, +`WaitCancellationCompleted` and `Abandon` through +`ChildWorkflowOptions::cancellationPolicy`. The default is `Abandon`, preserving +the existing behavior. Each new child schedule records the selected policy. +Replay reads that history, including an `Abandon` fallback for older schedules +that have no policy field. + +`TryCancel` durably requests child cooperation before delivering cancellation +to parent code. `WaitCancellationCompleted` additionally parks the parent until +the child has canonical terminal history. Mixed parallel waits and selected +child handles apply the same rule. A terminal child projection without terminal +history does not acknowledge completion. Waiting cannot extend the parent's +original deadline. An independently cancelling child keeps its own accepted +root and deadline, and the propagation conflict remains visible. + +Canonical `ChildCancellationRequested` and `ChildCancellationResolved` events +record the policy, parent and child identities, original budgets, conflicts and +terminal history reference. The run timeline exposes these details with the +child outcome. This is a durable workflow outcome, not proof that every external +callback stopped. + +The portable command bridge accepts the same child policy and returns +`delivered: false` with `cancellation_waiting_for_child` while acknowledgement is +pending. It parks the parent and completes the current task claim atomically, +returning `claim_released: true`. The SDK leaves that claim without publishing +completion or failure. This frees a single worker to execute the child rather +than occupying it with a blocking wait. Child terminal history wakes the parent +for a new claim, which replays the same authored cancellation boundary. +First-party SDK drafts handle this pending state. Their source checks and the +connected single-worker PHP test do not qualify a published child-policy tuple. + +An expired lease proves loss of authority to commit a durable result. It does +not prove that a remote process stopped or that an external system performed +an undo. Waiting must distinguish callback acknowledgement, durable fencing and +external reconciliation. + +### Remote callback-stop acknowledgement + +The candidate internal `ActivityCancellationAcknowledgement::recordStopped()` +records a remote owner's callback-stop report separately from `ActivityCancelled`. +The latter fences durable publication. It does not establish that a callback +has stopped. An SDK may send the new report only after stopping and joining its +callback. `ActivityCancellationAcknowledged` names that worker report and the +server's receipt time, with `evidence_source: activity_worker`. It does not +describe reversal of external effects. + +The report requires the original Server-issued activity attempt, lease owner +and local cancellation request. Remote claims use that attempt identity and +do not require the separate worker attempt ID used by local activities. +Current rows and the canonical +cancellation snapshot must match every fence. Mutable cancelled rows without +that history, a snapshot without its attempt identity, another owner or a +replacement attempt cannot authorize the report. The run lock serializes +duplicates, which return the original acknowledgement event. No report renews +a lease, restores result authority or grants a new cleanup budget. + +The receipt preserves the original root identity and deadline. A late report +has `received_after_deadline: true`, so an expired budget cannot appear to have +completed on time. The timeline identifies the original activity attempt and +explains the worker's stop report. Its `cancellation_acknowledgement` metadata +retains the local and root request IDs, original deadline, cancellation event, +receipt time and whether the receipt was late. Projection of this diagnostic +event must remain safe during cleanup, repair and stale publication refusal. +Local callback acknowledgements need the workflow task's distinct authority +and are explicitly refused by this remote primitive. The Server remote route +and PHP, Python and Rust stop/join emission have connected source qualification. +Python uses a separate callback process supervised independently of application +progress. The complete published cascade and activity waiting policies remain +required. + +### Portable local callback authority + +The candidate `PortableLocalActivityPreparation` kernel records local Scheduled +and Started history before callback admission. It preserves encoded inputs, +the original workflow task and attempt, a Server-issued activity attempt ID, +the SDK attempt ID and the original deadlines. A same-claim retry after response +loss returns that preparation. Changed descriptors, a reclaimed claim, +unshielded cancellation or expiry refuse invocation. This is an internal protocol 1.20 +primitive. Published protocol 1.19 keeps its existing local execution path. + +Before preparation, `checkpointLocalActivityPrefix()` can atomically commit +earlier nonterminal commands while retaining the workflow claim. Each batch is +bounded to 100 commands. Its latest receipt is stored on that claim, including +the normalized command fingerprint and authored sequence range. A lost response +can be retried before sending a subsequent checkpoint. Changed contents or +ownership cannot relabel a receipt or repeat child creation, side effects or +memo updates. Checkpointing does not renew the lease or admit application code. +An accepted cancellation refuses new prefix work until its delivery is recorded. +After delivery, the retained claim can checkpoint cleanup commands before the +original deadline. A checkpoint receipt alone never admits a callback. +The SDK must replay committed history before submitting later commands. + +The separate optional `PreparedLocalActivityGroupTaskBridge` adds an unfrozen +Source `checkpointLocalActivityGroup()` operation for complete `all` groups. +Its bounded batch uses `prepare_local_activity` descriptors without fabricated +results. The transaction commits every sibling and local Scheduled event, +retains the original workflow claim and stores the receipt before returning. +Local members are pending with no callback attempt. Their total budget starts +at admission, and the canonical `local_group_admission` records descriptor and +batch fingerprints, the checkpoint identity and original claim epoch. + +Preparation separately validates that durable admission before starting a local +attempt. Replacement before preparation creates its own first attempt without +inventing recovery or a stop receipt. A lost group response cannot create another +child or extend a deadline. Partial groups, changed descriptors, wrong claims +and unqualified cleanup proofs refuse without creating siblings. Nested `all` +paths are retained through Started and outcome history. Selection groups remain +refused until their policy and physical-stop behavior are qualified. Existing +custom prepared bridges do not acquire the optional group role from an alias. +Server admission, SDK group consumers and connected mixed-language qualification +remain required before advertising or publishing this Source extension. + +A shielded cleanup descriptor explicitly supplies `cancellation_cleanup` with +`request_id` and `delivery_history_event_id`. Admission validates that request +and canonical delivery on this run and requires a later authored command +sequence. Workers cannot supply or extend a cleanup deadline. The runtime +records the original root identity and deadline in the execution and Started +authority, and bounds local execution deadlines by that budget. Retries and +replacement-worker recovery retain the same delivery and budget. Pre-request +activities remain fenced. A cleanup result at or after the original deadline +is refused, and a supervisor stop instruction still requires a separate joined +callback report. + +The candidate local stop receipt uses the saved Started snapshot and the +canonical cancellation fence. The original workflow owner can report stop after +task takeover without changing the replacement claim. A missing or +unqualified post-cancellation preparation cannot authorize a stop report. Local history and +timeline retain the original workflow claim rather than inventing an ordinary +activity queue task. + +`recordPortableOutcome()` commits results against that prepared attempt. It +preserves encoded Avro bytes and reuses the existing local failure, retry and +timeout recorders. A repeated report returns its original canonical receipt +after claim takeover, expiry or later cancellation. Changed reports cannot +replace an outcome. A retry creates one durable workflow task and releases the +hosting claim. It preserves the original total activity deadline. + +An accepted cancellation can fence the original local attempt without changing +a replacement workflow claim. This refuses result publication. It does not +acknowledge physical callback stop. The original owner must separately report +that its supervisor has stopped and joined the callback. + +Retry preparation validates the canonical retry event, its hosting workflow +task, original descriptor and backoff before allocating a distinct attempt. +It shares the embedded attempt recorder without renewing the hosting claim. +The SDK must use a fresh attempt identity. A lost preparation response returns +the same current attempt. The total deadline remains recorded in Started +history. An expired total budget records a terminal timeout without admitting +another callback. A renewed workflow claim cannot readmit an expired local +attempt. + +`recover()` records an interrupted prepared attempt under a valid replacement +claim. It requires the original Started authority, an expired attempt lease +and loss of the original workflow claim. It records that attempt as Expired and +uses the existing retry or terminal recorder. Timeout and retry exhaustion +remain authoritative. Recovery never admits a callback or grants a new total +deadline. A scheduled retry releases the replacement claim for polling, while +a terminal outcome retains it for replay. Response loss returns the original +recovery receipt after takeover or later cancellation. + +For an admitted local group, recovering another interrupted member may create +a successor workflow task before any callback resumes. An earlier member can +prepare on that final claim only through the recorded sibling retry chain. +Every link must belong to the same original admission batch, retain its original +attempt authority, and follow a completed hosting task. Unrelated claims, +cycles, missing links and changed batches are refused. Each member still keeps +its own recorded backoff and total deadline, and cleanup keeps the original +cancellation delivery and deadline. + +The recovery receipt and timeline retain the original and replacement claims +and explicitly report callback stop as unknown. Lease expiry fences publication +and does not establish physical stop. An accepted cancellation fences the +prepared attempt immediately without waiting for expiry, changing the +replacement claim or fabricating an acknowledgement. + +The optional `PreparedLocalActivityTaskBridge` exposes checkpoint, preparation, +outcome, recovery, supervisor control, application heartbeat and stop-receipt persistence through the existing workflow +bridge binding. Consumers must check the actual bound instance. An existing +custom workflow or cooperative bridge does not acquire this role from an alias. +The published protocol default still refuses these candidate operations. This +role does not qualify physical SDK supervision. + +`controlLocalActivity()` polls the original prepared attempt independently of +application heartbeats. Active status polling alone is read-only. Optional +renewal commits the hosting workflow lease and local attempt lease together +under the original canonical owner and workflow epoch. It never records an +application heartbeat or renews the start-to-close, total, heartbeat, run or +cancellation deadline. Expired authority cannot be revived by polling. + +`heartbeatLocalActivity()` records an actual application heartbeat using the +same prepared authority checks and bounded progress format as other activities. +It updates heartbeat time and timeout and records canonical heartbeat history. +Prepared group heartbeats retain the complete authored membership path, including +nested groups. SDK heartbeat details use the existing `progress.details` format +and survive in canonical history. +It does not renew either lease or move start-to-close, total or root deadlines. +Cleanup heartbeat timeouts and supervisor lease renewals stay bounded by the +original cleanup deadline. Neither path revives expired authority. SDKs must +keep the latest acknowledged heartbeat timeout separate from fixed execution +deadlines when validating control responses. + +For a pre-request activity, accepted cancellation returns its original context even to the original +supervisor after takeover. It fences publication without changing the +replacement cleanup claim. The supervisor must stop and join its callback, +then report the separate acknowledgement. A fence or stop instruction is not +physical-stop evidence. A prepared cleanup call continues only under its +recorded delivery, original budget and live claim. Server admission for cleanup +and application heartbeats and SDK control loops still need connected qualification. + +Server admission and SDK physical local supervisors must connect these +primitives, dispatch created work and qualify response loss, cancellation and +worker replacement together. A portable SDK must not reconstruct already +prepared local rows from a posthoc completion report. + +Activity `WaitCancellationCompleted` must wait for this callback acknowledgement +or a prior canonical completion. Lease expiry alone leaves stop state unknown. +The Source Native kernel now uses `ActivityCancellationCompletion` to distinguish +the latest attempt's canonical callback outcome, an atomically fenced operation +that never started, and the original owner's stop receipt. A timeout, cancellation +row or receipt for another owner/attempt/request/root/deadline does not resolve +this wait. An outcome recorded after its cancellation fence cannot replace the +stop receipt. + +An internal recorded activity policy can park the awaiting workflow and complete +its hosting claim while the callback stops. The completed claim retains the +original requested delivery boundary, root and deadline. All required receipts +must be present before one fresh workflow claim resumes that boundary. Receipts +arriving after the original deadline remain late evidence and do not create a +fresh cleanup claim or budget. Remote SDK policy admission and detached +`Abandon` have separate connected Source qualification. Prepared-local policy +admission and its consumers remain separate gates. Internal kernel tests do +not qualify consumer APIs or physical SDK supervision. + +An `Abandon` activity must remain independently tracked and capable of finishing +after the awaiting scope is cancelled. Implement its retention and terminal-run +behavior deliberately rather than routing it through the normal terminal +activity fence. These operation semantics match the supported policy choices +in [Temporal's activity cancellation contract](https://docs.temporal.io/develop/typescript/workflows/cancellation). +DW's independent cancellation observation and original bounded cascade remain +the advantages to qualify. + +The internal Source remote lifetime now reads `Abandon` from the canonical +`ActivityScheduled` snapshot and captures the original absolute +`schedule_to_close_deadline_at` with it. Detachment requires that finite total +budget. Cooperative parent closure preserves the activity task and attempt. +The original owner can continue and commit a canonical outcome after the +parent closes, including after the parent's cleanup deadline. The activity's +own deadline remains unchanged. Outcomes and timeout closure do not create a +workflow task or reopen the cancelled parent. Failure retries and expired-owner +recovery retain the same total budget and reject stale publication. + +Retention holds the parent detail and external payloads until canonical terminal +history resolves the latest activity attempt. A mutable completed row is +insufficient. `WorkflowRunRetentionCleanup::retentionHoldReason()` lets hosts +check this before reclaiming objects, and `pruneRun()` also enforces the hold. +An older backend that cannot evaluate this policy must preserve its records and +return an explicit unsupported-backend diagnostic until a compatible backend +is restored. Legacy terminal cancellation and termination retain their authority +revocation contracts. + +### Prepared-local policy admission + +The Source preparation descriptor accepts explicit `try_cancel` and +`wait_cancellation_completed`. It persists the selected policy before callback +admission and retains it in canonical Scheduled/Started snapshots and the +descriptor fingerprint. A response-loss retry cannot change that policy. +Omission preserves the historical TryCancel behavior. + +TryCancel fences publication and releases the durable await without claiming +physical callback settlement. WaitCancellationCompleted releases the hosting +claim while the matching original-owner stop receipt is missing. That receipt +resumes one successor workflow claim at the same authored delivery boundary, +retaining the original root identity and deadline. Expired authority or a +wrong-owner receipt cannot substitute for callback-stop evidence. + +`DefaultWorkflowTaskBridge::supportedLocalActivityCancellationPolicies()` +reports the policies supported by the actual installed prepared-local bridge. +A custom implementation of the older optional role does not acquire this +policy admission capability from an alias. Server discovery/admission and +PHP/Python/Rust authoring, replay and physical supervision require separate +qualification before the candidate can be published. + +Local `Abandon` is refused before callback admission, including with a finite +timeout. The callback currently depends on its hosting workflow claim. It +cannot acquire an independent lifetime by changing the policy field. Use a +remote Activity for independently tracked work. No local-to-remote conversion +is implicit. A full local Abandon lifetime and independently cancellable +subtree scopes require separate authority. The +[hierarchical scope decision](hierarchical-cancellation-scopes.md) defines the +chosen durable operation-tree boundary, canonical membership, shield inheritance, +shared local claim handling and the implementation/qualification gate. The current +source tuple has preview consumers for a restricted subset and does not advertise +general scope execution. + +## Lifecycle and diagnostics + +Expose requested, delivered and cleaning-up progress without guessing from a +run's terminal status. Record cleanup completion, deadline expiry, termination +and loss of authority explicitly. Define whether these outcomes describe the +workflow, an operation or one worker attempt. + +API, CLI and Waterline should present the same request lineage, original budget, +pending operations and safe next action. Capability rejection should identify +the unsupported worker or SDK and the required capability. + +## Evidence required + +### Required mixed-language cascade before closure + +One isolated stack must execute this complete scenario using exact published +Native, Server and PHP/Python/Rust artifacts: + +```text +Parent PHP workflow + ├── Python child workflow + │ └── Rust remote activity + └── PHP local activity +``` + +Request cancellation once with a cleanup deadline 30 seconds after the +original request. Require all of the following in the same run: + +1. Parent, child and activities expose one root cancellation identity and the + original deadline. Local delivery and attempt identities remain distinct. +2. The Python child receives a genuine cooperative request and enters its + authored cleanup. Parent propagation must not immediately close the child. +3. Both activity callbacks stop without application heartbeats. Capture actual + callback-stop observations and SDK/runtime acknowledgements separately from + durable fencing. A stale activity attempt cannot publish a result. +4. SIGKILL a workflow worker during cleanup. A fresh replacement worker replays + the same canonical delivery boundary and resumes cleanup. Use supported + published lease and recovery settings, without editing lease rows or + substituting a virtual clock. +5. Repeat the cancellation request before and after recovery. Each duplicate + returns the original identity and deadline, without granting a new budget. +6. Cleanup completes and all workflow runs converge to `Cancelled` before the + original deadline. Deadline expiry cannot substitute for successful cleanup + and recovery in this scenario. +7. One API/UI view explains the cascade: root request, original deadline, + lineage, activity stops and fences, worker loss and replacement, cleanup + progress, and final outcomes. + +Retain raw observations, canonical histories, commands, package versions, +image digests and the inspection response. Independently passing language +tests or source-only qualification cannot substitute for this published +end-to-end result. Shared issue 136 remains open until this gate and its +competitive-strength decision are satisfied. + +### Broader contract qualification + +Qualify the exact published Native, Server and PHP/Python/Rust artifacts. Cover +request-before-claim, in-flight local and remote work without application +heartbeats, every operation policy, nested shielding, parent/child propagation, +duplicate and competing requests, worker loss, cold replay, cleanup expiry and +termination. Verify canonical history and reject late results from lost owners. + +Use current first-party competitor contracts and their supported configurations: + +- [Temporal operation policies](https://docs.temporal.io/develop/typescript/workflows/cancellation) + and [nested scopes](https://docs.temporal.io/develop/typescript/workflows/cancellation-scopes). +- [Restate cancellation and recursive call propagation](https://docs.restate.dev/services/invocation/managing-invocations). +- [DBOS workflow cancellation](https://docs.dbos.dev/python/tutorials/workflow-management) + and [preemptible async steps](https://docs.dbos.dev/python/reference/contexts). + +Exercise competing implementations where a behavior or timing comparison +depends on execution. Distinguish a documented contract from a measured result. +Report genuine advantages and remaining tradeoffs. Do not equate a competitor's +default configuration or absent documentation with its strongest supported model. diff --git a/docs/architecture/hierarchical-cancellation-scopes.md b/docs/architecture/hierarchical-cancellation-scopes.md new file mode 100644 index 000000000..502374584 --- /dev/null +++ b/docs/architecture/hierarchical-cancellation-scopes.md @@ -0,0 +1,527 @@ +# Hierarchical cancellation scopes + +Design decision for [shared cancellation work](https://github.com/durable-workflow/.github/issues/136) +and [the Native implementation](https://github.com/durable-workflow/workflow/pull/603). +General scope execution is not advertised by the current source tuple. Restricted +source consumers remain behind disabled SDK preview profiles. Native has a canonical +registration kernel: `CancellationScopeHistory` records `CancellationScopeOpened` +at a typed durable command position under the configured storage connection +and matching live claim. It preserves the recorded parent and shield mode on +fresh reads and replacement-claim replay, and includes those facts in the +history timeline. The optional internal `CancellationScopeTaskBridge` now +exposes this kernel through `openCancellationScope()`. Candidate Server endpoints +authenticate scope opening, prefix checkpointing, preparation and delivery. +No SDK scope capability is enabled by that foundation. The database tests exercise +the real bridge with authoritative claim fixtures, not an end-to-end SDK scope +execution. +The initial portable release boundary is defined in the +[cooperative cancellation model](cooperative-cancellation-model.md#initial-portable-release-boundary). +Its whole-run contract is qualified separately. Scope execution remains disabled +until the authority, replay and complete consumer gates below pass. Internal +preview extensions do not acquire a stable compatibility promise from sharing +the candidate protocol number. + +## Outcome and chosen boundary + +An application can cooperatively cancel one durable operation subtree while +unrelated and shielded operations continue. The application can inspect each +request, its original budget, callback stop evidence and cleanup outcome after +a worker restart. Handling a scope cancellation can let the enclosing workflow +complete successfully. Cancelling the workflow still cancels the run. + +Use an explicit tree of durable operation scopes. Scope membership belongs to +the scheduled operation and survives the lexical block that created it. +Nested activity/child/timer groups and their returned handles retain that +membership. A lexical authoring helper can assign the current scope, but a +Fiber-local flag alone is not the durable scope. + +This provides independently cancellable operation trees without introducing +implicit child workflow runs or a second workflow scheduler. It does not add +concurrent arbitrary workflow-code callbacks. Temporal's concurrent lexical +scope ergonomics remain a competing strength. Do not claim API equivalence +from a scope wrapper around today's sequential workflow Fiber. + +Preserve existing `DurableOperationHandle::cancel()` and terminal routes. +The new cooperative entry point is `requestCancellation()` on the scope. +It must not delegate to terminal handle cancellation. Operations outside a +scope continue to use the existing run boundary and historical defaults. + +## Canonical membership + +Opening a scope is a durable authoring command. Its canonical history records +an opaque scope ID, the owning exact run, its parent scope, authored creation +boundary and whether it shields parent propagation. The implicit root denotes +the run. An SDK obtains the recorded identity on first execution and cold +replay. It cannot manufacture a new identity from host time or process state. + +Scope creation must reject a foreign run, an unknown parent, duplicate authored +identity or a cycle. Replaying a creation with a changed parent or shield mode +is a nondeterminism failure before work admission. + +The optional opening bridge requires candidate protocol 1.20, the exact issued +workflow task, its live lease owner and attempt, and the next authored command +sequence. It locks the owning run before the task, then invokes the canonical +kernel. A successful receipt contains the recorded scope and history identities, +parent, shield mode and sequence. Opening retains the workflow claim, creates no +operation and grants no new lease or cancellation budget. The SDK must receive +that durable acknowledgement before entering the scope body. + +A response-loss retry or replacement claim returns the same scope and history +event, marked as a duplicate. It cannot change the recorded parent or shield. +Expired, stale, wrong-namespace, non-workflow or foreign-run claims are refused +before mutation. Malformed or unsupported requests have explicit reasons. +Existing bridge adapters need not implement this optional role. Server must +report that absence before accepting scope authoring. Default protocol 1.19, +scope delivery, scoped clocks and physical supervision remain unchanged. + +Each scheduled leaf records its immediate scope ID. Descendant membership is +resolved through canonical parent links. A prepared local descriptor includes +that identity in its admission fingerprint, Scheduled and Started snapshots. +Child, remote activity, timer and selection/group history retain it too. +Response loss, retry, selection and replacement cannot reparent work. + +The internal prepared-local descriptor now accepts optional +`cancellation_scope_id`. Admission checks the exact run's canonical scope and +requires its creation before the operation's authored sequence. The descriptor +fingerprint, execution options and Scheduled/Started activity snapshots preserve +that identity. Original-claim inspection refuses contradictory membership. +Atomic local group admission checks every member before creating any sibling. +Historical omission leaves unscoped descriptors and snapshots unchanged. +Remote activity, timer and child scheduling now carry the same optional address. +Candidate protocol 1.20 is required when transport commands name a scope. Both +ordinary completion and retained-claim checkpointing verify every named scope +before applying any sibling command. Remote activity snapshots, timer scheduling +history and task locators, and child scheduling/started history retain the +address. Child operator metadata is a projection, not its authority. Historical +unscoped scheduling shapes remain unchanged. + +Signal and condition wait commands also retain their optional address on the +canonical opening event. Timed waits retain it on timeout scheduling and task +locators, including immediate timeout history. Untimed waits retain membership +without creating a timer. Foreign or unknown scope addresses refuse the entire +command batch before effects. Waits still close a turn and cannot enter a +retained-claim prefix. Once delivery is prepared, the original wait can replay +but new work in that scope cannot be admitted. + +The candidate backend records canonical `SignalWaitCancelled` and +`ConditionWaitCancelled` decisions for timed and untimed original waits. +Preparation freezes each wait's identity, descriptor and associated timer. +The wait decision and timeout fence commit together under the timer-task, run, +hosting-claim and timer-row lock order. Untimed waits use run and claim locks. +Every timer entry point delegates a wait-owned timer to this atomic actor. +Already received/applied signals, satisfied conditions and natural timeouts keep +their original outcome. Cancellation retains buffered signal bytes, closes only +the original wait occurrence, and denies stale publication into that occurrence. +Replacement and lost-acknowledgement retries use the first receipt and deadline. +Source qualification remains required before exposing SDK scope execution. + +Preparation v5 also freezes each child call's sequence, call identity, instance, +typed-history run target and recorded cancellation policy. It selects the latest +child start committed before preparation. Later child starts, changed current +run pointers and missing operator projections cannot retarget a cold retry. +Natural completion leaves the original prepared membership intact. The snapshot +is exposed through candidate worker responses and the history timeline, without +copying application payload bytes. The internal child actor now commits a genuine +cooperative child request and its parent receipt atomically. TryCancel waits for +that request receipt. WaitCancellationCompleted also requires a separately +recorded canonical child terminal outcome. Abandon records the policy without +changing the child. Cold delivery checks the original preparation, child command, +typed request history and terminal receipt before accepting a delivery marker. +The hosting claim, unrelated siblings and shielded addresses remain intact. +Portable delivery invokes that actor for each original direct or inherited child +member and returns its durable request and resolution receipts, including partial progress +when a policy barrier remains pending. A duplicate or replacement claim reconciles +those same receipts under the original preparation and deadline. + +The same preparation freezes the original unshielded descendant tree. Inherited +requests and that inventory commit atomically under the original run and hosting +claim. Each descendant retains its accepted request, canonical parent, original +root lineage and deadline, operation membership and any competing-root conflict. +An existing independent request is preserved, and its descendants inherit that +accepted identity. The authority ceiling remains bounded by the original parent +preparation and every accepted ancestor budget. Shields exclude their complete +branch from propagation without extending its authority. + +Cold replay validates that original history prefix, so later openings, requests, +mutable limits or worker replacement cannot resample the inventory or deadline. +New scopes, including new shields, cannot open inside the prepared subtree. +Existing openings replay, and unrelated root work and sibling branches continue. +A lost claim or expired budget rolls back the preparation and its nested requests. +The internal descendant actors consume an immutable reference to the original +parent preparation. Each reference retains its actual history ID and sequence, +the descendant's accepted request and membership, and the captured authority +ceiling. It creates no new history event or authored command position. Activity, +timer, wait and child receipts retain their original ownership checks. Timed +waits and their timer fence still commit atomically. WaitCancellationCompleted +requires the original activity owner's stop receipt before the parent delivery +marker can commit. Portable dispatch reconciles direct and inherited receipts +under that one preparation, preserving siblings and shielded branches. + +Reading an original reference or delivery marker after expiry grants no new +effect authority. Live actors still require the current hosting claim and both +the current and captured budget. SDK scope execution, the response contract and +published mixed-language qualification remain separate unfinished gates. + +The optional internal `CancellationScopeAdmission` bridge role exposes these +canonical membership checks to Server. Server preserves the optional field, +refuses a backend without that role, checks before payload resolution and +rechecks under the run lock. Stream directives commit only after successful +Native admission in the same transaction. A refusal does not publish stream +items, while timeout or cancellation decisions made by Native remain durable. + +These are backend admission contracts. Scope-aware authoring/replay, request +delivery and physical sibling supervision remain separate implementation and +qualification gates. No scope capability is enabled. + +Direct prepared-local admission checks the same cancellation guard as batched +commands before creating an execution or attempt. An implicit-root command, +including a metadata write, also cannot consume a position reserved for an +unfinished scoped delivery. This covers ordinary completion, both retained-claim +prefixes and atomic local groups. Previously admitted original operations retain +their membership and can replay, including an unaffected root local member in a +mixed group. Once delivery consumes its position, the unaffected parent can +admit its next operation. A delivery marker alone still grants no new scoped +durable cleanup authority. + +Operation policy and shielding are independent. TryCancel, WaitCancellationCompleted +and Abandon retain their meanings. A shield blocks inherited cooperative +delivery. It does not silently change an activity to Abandon. Prepared local +Abandon remains refused until it has a qualified independent lifetime. + +## Request identity and finite authority + +The target address is `(exact run, scope)`. Each address has a distinct local +request identity and delivery record. Propagation carries one original root +identity, original request time and immutable cleanup deadline. Opening another +scope, restarting a worker or retrying a request grants no new budget. + +The current `cancellation-context/v1` lineage permits only one entry per run. +It cannot represent several scope hops inside that run. Preserve its parser and +historical snapshots. Define a separately versioned scoped context with explicit +scope addresses before adding consumers. Do not squeeze scope hops into repeated +run entries or replace a local delivery ID with the root ID. The rich authored +context retains requester, source, reason and deterministic remaining-time helpers. + +The internal metadata encoding is +`durable-workflow.scoped-cancellation-context/v1`. `root_context` contains the +original rich v1 root snapshot with its original single root request. The +separate `lineage` contains `request_id`, `workflow_instance_id`, +`workflow_run_id`, `scope_id` and `cleanup_deadline_at` for each accepted address. +The first address must match that root and its deadline. A descendant deadline +can shorten but cannot increase. `rootDeadline()` keeps the original global +budget while `deadline()` returns the selected scope's accepted budget. + +`ScopedCancellationContext` permits different scopes within one run, rejects +repeated request IDs or run/scope addresses and contradictory run/instance +mapping, and explicitly adapts old run lineage to implicit root scopes. It +normalizes scope address field order independently of database JSON key order. +The existing v1 parser and snapshots are unchanged. This metadata parser grants +no authority: backend consumers must verify the recorded scope, propagation +edge and accepted root under lock. Later ancestor authority ceilings and +competing roots are separate records, not mutations of the accepted context. +When a scoped request enters a child run, the internal +`durable-workflow.cancellation-context/v2` encoding retains the full accepted +`scope_origin`. Its ordinary run lineage contains one entry per run, while the +origin preserves every scope hop and the immediate causal parent request ID. +The child keeps the original requester, reason, source and requested-at time, +and uses the accepted scope budget narrowed by the original preparation and +current parent authority ceilings. The immutable child +`scope_authority_deadline_at` binds that limit without extending the global root +budget or rewriting the accepted scope origin. +Further run-to-scope-to-run propagation retains that origin. Canonical child +requests read it from the original prepared member, never caller-supplied +metadata or a mutable current-run pointer. Competing origins cannot replace an +accepted child context. Existing v1 snapshots remain readable. This candidate +encoding is source-qualified for parsing in the PHP, Python and Rust SDK +candidates. Authored scope execution and the published mixed-language cascade +remain required before scope support is advertised. +Native's internal `CancellationScopeRequests` kernel now accepts requests at +recorded non-root addresses under task-before-run locks on the configured +storage connection. The accepted +`CancellationScopeRequested` history event owns its context and identity. +Duplicates return that event, including after run closure. Inheritance reads +the immediate canonical parent's accepted context, or the run's validated +request for an implicit-root edge. A conflicting inherited root records +`CancellationScopeRequestConflicted` with both contexts and leaves the accepted +event unchanged. Request and conflict facts appear in the history timeline. +Repeated propagation of the same conflicting root returns its original conflict +event and incoming address identity rather than growing history on every retry. + +The separate authority inspection takes the earliest accepted ancestor, run +cancellation, execution or run deadline, and marks terminal or expired authority +inactive. Shielding does not remove those ceilings. Requests do not set run +cancellation fields, release claims, wake or stop callbacks, or deliver an +exception. Root scope requests still use the existing whole-run boundary. +Canonical reads reject contradictory addresses or inherited contexts. + +Acceptance also makes shared workflow ownership recoverable before preparation. +It projects the original scope authority ceiling on the run and shortens live +workflow leases to at most ten seconds, honoring a smaller configured lease. +It never extends an already shorter lease, revives an expired claim or changes +its owner, attempt or activity leases. The run-summary lease projection changes +with the shortened claim. A pending task acquires that interval when claimed. +Duplicates and conflicting roots do not renew ownership or replace the accepted +budget. While any projected original scope budget is live, claims and heartbeats +keep the bounded ownership interval. When all those budgets expire, unrelated +work can retain the ordinary configured lease. This projection does not grant +callback authority or deliver cancellation. SDK scope consumers still need +qualified supervision that renews these intervals in time. + +The internal `CancellationScopeDelivery` kernel records one +`CancellationScopeDelivered` boundary per accepted exact-run scope address. +It rechecks the live claim owner/attempt, accepted identity, current authority, +authored call shape and canonical operation membership under the configured run +lock. Response loss and replacement claims return the original event, context +and recorded clock. A changed boundary is refused. An inherited request is +deferred at a parent shield, while a direct request can reach the shielded scope. +Every member of a recorded parallel or selection boundary must belong to the +address. A mixed-scope barrier requires selective member delivery and is refused +by this boundary kernel. + +Delivery consumes the interrupted durable command range even when it was not +scheduled. It does not set whole-run cancellation fields or replace the workflow +claim. Preparation and delivery renew its bounded hosting interval while retaining +owner and attempt. Its `authority_deadline_at` preserves the ceiling observed at delivery. +Later ancestors can shorten live authority without rewriting this receipt, so +the receipt never authorizes a new effect. Cold inspection remains possible +after expiry or run closure. This is a recording kernel, not a consumer that +injects cancellation or physically stops callbacks. Scope-aware SDK replay, +selective supervision and scoped `remaining()` remain unimplemented. No scope +capability is advertised. + +The first accepted request at an address owns its identity and deadline. +Duplicates return that context. A different root is a recorded conflict with +both identities and deadlines, not an implicit merge. The accepted context is +unchanged. Qualification must race these requests on supported databases. + +When a later ancestor preparation includes an independently cancelled descendant, +its strictly earlier preparation continues to own the original delivery boundary. +While that delivery is unfinished, portable delivery returns +`cancellation_scope_descendant_delivery_pending` before dispatching any actor. +Direct activity, timer, wait and child actors also reject an ancestor preparation +that would replace the unfinished boundary. The original preparation can finish +within its remaining authority. + +Once delivery completes, reconcile its original preparation and canonical marker. +The marker can follow the new preparation, but must precede the ancestor marker +that consumes it. It must prove the same accepted context and exact activity, +timer, wait and child inventory. A descendant may use the original marker of its +own earlier ancestor. Read receipts from that marker's history prefix without +rerunning actors, changing projections or rebinding the captured budget. +Completed cleanup remains readable after its original deadline or when a later +parent imposes a tighter ceiling. Future history cannot justify an earlier marker +or authorize another effect. + +An inherited deadline is never later than its originating deadline. A stricter +authored scope limit can shorten authority and is recorded once. The inspection +view distinguishes the original request deadline from any ancestor authority +ceiling. An independently cancelled scope cannot stay alive after the enclosing +run loses authority. Neither shielding nor a conflicting root can extend a +run's execution limit, termination or accepted run cancellation deadline. + +Inherited cancellation is pending at a shielded scope. Existing operations in +that scope can continue within the remaining authority. Delivery occurs when +the shield boundary is left. A direct request targeting that scope is delivered +there even if it shields its parent. New normal work in an already requested +unshielded scope is refused before callback admission. Cleanup work is explicitly +bound to the accepted request and remaining budget. + +### Prepared local cleanup authority + +The candidate prepared-local descriptor accepts an explicit `cancellation_cleanup` +proof containing only `scope_id`, `request_id` and `delivery_history_event_id`. +The requested scope and delivery must belong to this run's canonical history. +The new operation must follow that delivery's authored range and belong to its +original prepared scope or an original unshielded descendant with the same root +context. A shielded or unaffected address cannot borrow the proof. An earlier +independent preparation still owns its own cleanup authority. + +The runtime records the original request/root identity, delivery and preparation +IDs, operation membership, accepted cleanup deadline and captured authority +ceiling in the execution and original start. The worker supplies no deadline. +Both callback deadlines and retry deadlines are bounded by that ceiling. A later +increase to a mutable run limit cannot extend it. Current run/execution limits, +accepted ancestor deadlines, terminal state and later whole-run cancellation +still end authority. A scoped proof grants no immunity from whole-run cancellation. + +When a scope inherited the current whole-run request, its delivered cleanup can +continue before root delivery. The runtime verifies the exact root context and +lineage prefix for this run's request. Sharing an ancestor's root ID is insufficient. +Recovery, atomic group admission, control and publication retain that authority +and the original deadline. Its hosting lease is also bounded by that deadline. +An independent scope request remains subject to later whole-run cancellation. + +A later ancestor preparation excludes these explicitly shielded cleanup calls +from its ordinary cancellation inventory. It verifies their original scheduled +snapshot against the earlier canonical delivery before excluding them. This also +covers atomic group members that have been admitted but have not started. +Corrupt cleanup history fails preparation rather than hiding ordinary work or +rebinding cleanup to the later ancestor's identity. + +Authenticated scoped preparation immediately sets a hosting ownership interval +of at most ten seconds, or the configured workflow lease when it is shorter. While any committed +scope preparation's captured authority budget remains live, workflow heartbeats, +surviving callback controls, embedded local renewals and replacement claims use +that same interval. Preparation and delivery record the latest captured scope +expiry in the run within their transaction, so ordinary claims and renewals need +no history query. +Pending child completion and activity stop receipts do not delay this recovery +interval. A valid retry renews only the authenticated live claim and retains +the first preparation. An expired or replaced owner cannot revive its claim. +Preparation and delivery lock the hosting task before its run, matching worker +claim, heartbeat and watchdog repair. +This recovery projection grants no callback authority or new cleanup budget. +The subtree's absolute deadline does not end the shared +hosting claim. Its own callback lease stays bounded by current and captured +authority. Once every captured scope budget expires, unaffected work can renew +the normal configured workflow lease. An accepted whole-run request bounds shared +ownership by its original cancellation deadline, including before root delivery. +Neither renewal nor replacement +changes a scope's delivery record or cleanup budget. Real scoped worker SIGKILL +recovery remains required before enabling SDK scope execution. +Synchronous embedded callbacks outside scoped recovery use the configured +ownership interval, clipped to the original run deadline. They cannot renew +during execution. Supervised portable callbacks retain short renewable ownership. +Accepted scope requests also bound shared hosting ownership before preparation, +using the recovery projection described above. Replacement can therefore reach +the first preparation while the original cleanup budget remains live. + +Admission validates every atomic local-group member before creating any sibling. +Renewal and publication validate the recorded cleanup fact. Missing or rewritten +execution/start snapshots cannot turn cleanup into normal work. A late reported +success cannot publish its result. Replacement preserves the same original +delivery and budget, and an old attempt cannot publish. Native lease recovery +still reports an unknown physical callback-stop state until the original owner +provides real stop proof. + +Shielded scoped timers carry `cancellation_cleanup` with only the original +scope, request and delivery history identities. Admission derives the immutable +budget and preparation ceiling from canonical history. It refuses timers that +would fire at or beyond that ceiling and preserves the authored delay. The +scheduled history retains the derived snapshot so replacement can verify the +same authority and later ancestor cancellation can preserve the shielded timer. +Malformed proofs, invented deadlines and missing shielding cannot admit new work. + +These are Native prepared-local and timer primitives. Other durable cleanup operations, +complete SDK consumption/supervision, cleanup outcomes in scope inspection and real scoped +SIGKILL recovery remain qualification work. Scope execution stays unadvertised +and the SDK Worker does not enable it from this internal descriptor alone. + +## Delivery and operation resolution + +Only awaits that belong to the requested unshielded subtree receive the scoped +cancellation exception. Record its request, authored boundary, operation range +and context canonically before returning it. An unrelated await does not receive +it. Cold replay must deliver at that same boundary even when later results exist. + +The receiving address can be the requested scope or any canonically recorded +descendant reached without crossing a parent shield. Each scheduled leaf must +retain a valid scope opened before that leaf. Conflicting flat and nested +membership, unknown scopes and later scope openings cannot justify delivery. +Cold reads establish membership from the original marker's history prefix. + +TryCancel releases the scoped await after durable fencing/request propagation. +WaitCancellationCompleted requires the same matching original-owner callback +stop receipt or canonical child terminal outcome used by run cancellation. +Abandon leaves independently supported work running and inspectable. None of +these imply an external side effect was undone. + +Mixed groups retain each leaf's scope and policy. A cancelled member must not +fence its sibling merely because both share a group. The group can expose its +cancelled outcome while surviving handles remain available. A scope join resolves +the requested member policies and its own cleanup, not every operation in the +enclosing run. A caught scoped exception does not set run cancellation fields +or force a successful enclosing workflow to end Cancelled. + +A parallel or selection address is eligible when at least one leaf belongs to +the requested unshielded subtree. Other leaves retain their original scope, +task and result authority. A group consisting entirely of unrelated or shielded +leaves cannot receive that request. Delivery checks stop proof against the +frozen scope inventories and their original contexts, rather than requiring +every leaf in the await range to be stopped. These Native kernel checks do not +yet advertise authored scope execution in the published SDKs. + +## Prepared local callback ownership + +Control of a running scalar callback now follows its accepted original ancestor +request. It releases SQL locks, commits the original local delivery preparation, +then fences that exact callback under the preparation. A run request cannot +replace the scope's cancellation identity. Preparation and fencing keep their +own transactions, and a takeover between them cannot grant the old supervisor +the replacement claim. Physical exit still requires the original supervisor's +joined stop receipt. This source path does not enable general SDK scope support. + +Today's callbacks share a hosting workflow claim. Run-level waiting can release +that whole claim. Reusing that release for a partial scope would revoke unrelated +local siblings and is not a valid implementation. + +For partial cancellation, keep the hosting claim while supported siblings are +active. Independently poll and stop only attempts in the requested scope. +Continue control/lease renewal for surviving callbacks while a scoped wait is +pending. The cancelled attempt's result is fenced even though the hosting claim +remains valid. Scope authority and attempt authority are checked separately. + +Once all hosted callbacks are settled, the workflow can park its claim while +waiting for remote/child outcomes. Persist the original scoped delivery boundary +and wait inventory first. A fresh claim resumes that boundary. An SDK that cannot +maintain sibling supervision must refuse this capability before any callback +starts. Do not silently stop siblings or convert local work to remote work. + +SIGKILL can interrupt every callback hosted by the lost worker. Report those +attempts' stop states as unknown until evidence exists. Recovery may retry +uncommitted sibling work under ordinary at-least-once semantics. It must preserve +already committed results, scope membership, accepted request identities and +deadlines. A replacement cannot acknowledge a stopped original attempt merely +because it has acquired the workflow lease. + +## Inspectable states and admission + +An unfinished preparation reserves its authored delivery position, including +an interrupted await that has no scheduled operation yet. A new scope opening, +operation prefix or another scope's preparation cannot consume that position. +Recorded commands still replay, and already scheduled unrelated or shielded +operations continue under their own authority. The delivered marker consumes +the original position before workflow authoring advances to another new command. +Reusing a preparation revalidates its live command shape before dispatching +actors. Historical delivery replay keeps the original boundary and clock. + +Extend the existing cascade inventory with scope nodes and parent edges. For each +node show accepted/conflicting requests, deferred shield propagation, delivered +boundary, pending stop receipts, cleanup progress and final outcome. Keep callback +fencing separate from reported physical stop. A scope can finish cancelled while +its workflow and unrelated scopes remain running or complete successfully. + +Server discovery describes actual installed bridge support. Worker admission +requires the SDK's real scope replay and supervisor implementation for each +operation kind it claims. Missing support identifies the Worker/SDK and operation +before scheduling or callback invocation. Rust's missing prepared-local executor +is separate foundation work, not a capability flag this design can enable. + +## Required qualification before advertising scopes + +1. One parent scope contains an unshielded timer/child subtree and a shielded + activity. An independent sibling activity runs alongside it. Request only + the parent scope, observe genuine child cooperation and matching callback + stop evidence, then let the shield and sibling complete. Catch the scope + cancellation and finish the enclosing workflow successfully. +2. Repeat with two prepared local siblings in the same hosting claim. Cancel + one with WaitCancellationCompleted. The other retains authority and publishes + its result. Stale completion of the cancelled attempt is refused. +3. SIGKILL during scoped cleanup. A fresh replacement replays the same scope + identity, delivery boundary and consumed budget, resumes cleanup and preserves + committed sibling results. Stop observations without receipts remain unknown. +4. Race direct, ancestor and duplicate requests on supported databases. The + winning accepted identity and deadline remain immutable. Conflicts are visible. + A shield or independently accepted request cannot outlive run authority. +5. Verify cold replay, query/update replay and changed membership/policy refusal, + including mixed nested groups and history preceding scope support. +6. Run the exact published PHP/Python/Rust supported-operation tuple and inspect + the same scope cascade through API, CLI and Waterline. Run the original full + workflow cancellation/SIGKILL cascade again. Unsupported local executors are + diagnosed explicitly rather than counted as qualified. + +The existing published competitor experiments supply comparison cases, not +timing baselines for a capacity claim. Retain the customer tradeoffs: a shared +claim needs an available supervising worker during partial local waits, external +effects still need reconciliation, and concurrent arbitrary workflow-code +branches are outside this operation-scope contract. diff --git a/scripts/ci/validate-regression-corpus.py b/scripts/ci/validate-regression-corpus.py index 4cd587f0c..8f6310240 100644 --- a/scripts/ci/validate-regression-corpus.py +++ b/scripts/ci/validate-regression-corpus.py @@ -29,6 +29,7 @@ GOLDEN_HISTORY_SCHEMA = "durable-workflow.golden-history.v1" MALFORMED_SERVICE_RESPONSE_ENVELOPE = "malformed_service_response_envelope" SEARCH_ATTRIBUTE_TYPE_IDENTITY_MISMATCH = "search_attribute_type_identity_mismatch" +WORKFLOW_HISTORY_SHAPE_MISMATCH = "workflow_history_shape_mismatch" UNSUPPORTED_PAYLOAD_CODEC = "unsupported_payload_codec" PHP_GOLDEN_REPLAY_WORKFLOW = "Tests\\Fixtures\\V2\\TestGoldenReplayWorkflow" PHP_REPLAY_CONSUMERS = { @@ -162,6 +163,7 @@ "tests/Fixtures/V2/TestServiceResponseReplayWorkflow.php", "tests/Fixtures/V2/TestServiceGroupedConditionReopenWorkflow.php", "tests/Fixtures/V2/TestSignalResumedParallelWorkflow.php", + "tests/Fixtures/V2/TestCancellationContextWorkflow.php", "tests/Unit/V2/ReplayRegressionCorpusTest.php", ), } @@ -1257,7 +1259,7 @@ def _replay_expected( def _replay_expected_failure(value: Any, context: str) -> Mapping[str, str]: - """Return the one fail-closed replay outcome supported by this corpus format.""" + """Return an explicitly supported rejection from the official replay binding.""" expected = _object(value, context) required = {"type", "exception"} @@ -1267,6 +1269,7 @@ def _replay_expected_failure(value: Any, context: str) -> Mapping[str, str]: if failure_type not in { MALFORMED_SERVICE_RESPONSE_ENVELOPE, SEARCH_ATTRIBUTE_TYPE_IDENTITY_MISMATCH, + WORKFLOW_HISTORY_SHAPE_MISMATCH, UNSUPPORTED_PAYLOAD_CODEC, }: raise CorpusError(f"{context}.type is unsupported") diff --git a/src/Providers/WorkflowServiceProvider.php b/src/Providers/WorkflowServiceProvider.php index 638b57a13..ff9515b9c 100644 --- a/src/Providers/WorkflowServiceProvider.php +++ b/src/Providers/WorkflowServiceProvider.php @@ -30,6 +30,8 @@ use Workflow\V2\Contracts\LongPollWakeStore; use Workflow\V2\Contracts\MatchingRole; use Workflow\V2\Contracts\OperatorObservabilityRepository; +use Workflow\V2\Contracts\PreparedLocalActivityGroupTaskBridge; +use Workflow\V2\Contracts\PreparedLocalActivityTaskBridge; use Workflow\V2\Contracts\RuntimeSignalControlPlane; use Workflow\V2\Contracts\SchedulerRole; use Workflow\V2\Contracts\ScheduleWorkflowStarter; @@ -97,6 +99,8 @@ static function ($app): HistoryProjectionMaintenanceRole { $this->app->singleton(WorkflowTaskBridge::class, DefaultWorkflowTaskBridge::class); $this->app->alias(WorkflowTaskBridge::class, CooperativeWorkflowTaskBridge::class); + $this->app->alias(WorkflowTaskBridge::class, PreparedLocalActivityTaskBridge::class); + $this->app->alias(WorkflowTaskBridge::class, PreparedLocalActivityGroupTaskBridge::class); $this->app->singleton(ActivityTaskBridge::class, DefaultActivityTaskBridge::class); diff --git a/src/V2/Activity.php b/src/V2/Activity.php index ca42bffc5..cdb7b1194 100644 --- a/src/V2/Activity.php +++ b/src/V2/Activity.php @@ -199,7 +199,7 @@ public function heartbeat(array $progress = []): void } if ($this->ownsLocalWorkflowTask($execution, $attempt, $task)) { - $leaseExpiresAt = LocalActivityRuntime::renewWorkflowTask($task); + $leaseExpiresAt = LocalActivityRuntime::renewWorkflowTask($task, run: $run); if ($attempt->status === ActivityAttemptStatus::Running) { $attempt->forceFill([ diff --git a/src/V2/CancellationContext.php b/src/V2/CancellationContext.php new file mode 100644 index 000000000..d250c8aca --- /dev/null +++ b/src/V2/CancellationContext.php @@ -0,0 +1,359 @@ +>|null + */ + private static ?WeakMap $replayFibers = null; + + /** + * @param array $requester + * @param list $lineage + */ + private function __construct( + public readonly string $requestId, + public readonly string $rootRequestId, + public readonly string $rootWorkflowInstanceId, + public readonly string $rootWorkflowRunId, + public readonly ?string $parentRequestId, + public readonly ?string $reason, + public readonly array $requester, + public readonly string $source, + private readonly CarbonImmutable $originalRequestedAt, + private readonly CarbonImmutable $cleanupDeadline, + public readonly array $lineage, + public readonly ?ScopedCancellationContext $scopeOrigin = null, + ) { + } + + /** + * @param array $snapshot + */ + public static function fromArray(array $snapshot): self + { + $schema = $snapshot['schema'] ?? null; + if (! in_array($schema, ['durable-workflow.cancellation-context/v1', + 'durable-workflow.cancellation-context/v2'], true)) { + throw new InvalidArgumentException('Unsupported cancellation context schema.'); + } + $scopeOrigin = null; + if ($schema === 'durable-workflow.cancellation-context/v2') { + if (! is_array($snapshot['scope_origin'] ?? null)) { + throw new InvalidArgumentException('Scoped run cancellation requires its original scope context.'); + } + $scopeOrigin = ScopedCancellationContext::fromArray($snapshot['scope_origin']); + } elseif (array_key_exists('scope_origin', $snapshot) || array_key_exists( + 'scope_authority_deadline_at', + $snapshot + )) { + throw new InvalidArgumentException('Legacy cancellation context cannot discard a scoped origin.'); + } + $requester = $snapshot['requester'] ?? null; + if (! is_array($requester) || array_is_list($requester) || $requester === []) { + throw new InvalidArgumentException('Cancellation requester must identify its caller.'); + } + foreach ($requester as $key => $value) { + if (! in_array($key, ['type', 'id', 'label'], true) || ! is_string($value) || $value === '') { + throw new InvalidArgumentException('Cancellation requester contains unsupported metadata.'); + } + } + // JSON databases may reorder object keys. Identity comparisons must + // preserve the values without depending on their storage order. + $normalizedRequester = []; + foreach (['type', 'id', 'label'] as $key) { + if (isset($requester[$key])) { + $normalizedRequester[$key] = $requester[$key]; + } + } + $lineage = $snapshot['lineage'] ?? null; + if (! is_array($lineage) || ! array_is_list($lineage) || $lineage === []) { + throw new InvalidArgumentException('Cancellation lineage must contain the root request.'); + } + $normalizedLineage = []; + foreach ($lineage as $entry) { + if (! is_array($entry)) { + throw new InvalidArgumentException('Cancellation lineage entry is invalid.'); + } + $normalizedLineage[] = [ + 'request_id' => self::text($entry, 'request_id'), + 'workflow_instance_id' => self::text($entry, 'workflow_instance_id'), + 'workflow_run_id' => self::text($entry, 'workflow_run_id'), + ]; + } + $requestId = self::text($snapshot, 'request_id'); + $rootRequestId = self::text($snapshot, 'root_request_id'); + $rootInstanceId = self::text($snapshot, 'root_workflow_instance_id'); + $rootRunId = self::text($snapshot, 'root_workflow_run_id'); + $parentRequestId = $snapshot['parent_request_id'] ?? null; + $reason = $snapshot['reason'] ?? null; + if (count(array_unique(array_column($normalizedLineage, 'request_id'))) !== count($normalizedLineage) + || count(array_unique(array_column($normalizedLineage, 'workflow_run_id'))) !== count($normalizedLineage)) { + throw new InvalidArgumentException('Cancellation lineage cannot contain a cycle.'); + } + if (($parentRequestId !== null && (! is_string($parentRequestId) || $parentRequestId === '')) + || ($reason !== null && ! is_string($reason))) { + throw new InvalidArgumentException('Cancellation parent identity or reason is invalid.'); + } + if ($normalizedLineage[0] !== [ + 'request_id' => $rootRequestId, + 'workflow_instance_id' => $rootInstanceId, + 'workflow_run_id' => $rootRunId, + ] || $normalizedLineage[count($normalizedLineage) - 1]['request_id'] !== $requestId + || ($scopeOrigin === null && (count($normalizedLineage) === 1 + ? $parentRequestId !== null + : $parentRequestId !== $normalizedLineage[count($normalizedLineage) - 2]['request_id']))) { + throw new InvalidArgumentException('Cancellation lineage does not match its request identities.'); + } + $requestedAt = self::timestamp(self::text($snapshot, 'requested_at')); + $deadline = self::timestamp(self::text($snapshot, 'cleanup_deadline_at')); + if ($deadline->lessThanOrEqualTo($requestedAt)) { + throw new InvalidArgumentException('Cancellation deadline must follow the original request.'); + } + if ($scopeOrigin !== null) { + $last = $normalizedLineage[count($normalizedLineage) - 1]; + $expected = self::scopedDescendantSnapshot( + $scopeOrigin, + $requestId, + $last['workflow_instance_id'], + $last['workflow_run_id'], + self::timestamp(self::text($snapshot, 'scope_authority_deadline_at')), + ); + $normalized = [ + ...$snapshot, + 'lineage' => $normalizedLineage, + 'requested_at' => $requestedAt->toISOString(), + 'cleanup_deadline_at' => $deadline->toISOString(), + 'scope_authority_deadline_at' => self::timestamp( + self::text($snapshot, 'scope_authority_deadline_at') + )->toISOString(), + ]; + foreach ($expected as $key => $value) { + $actual = $normalized[$key] ?? null; + if ($key === 'requester') { + ksort($actual); + ksort($value); + } + if ($key !== 'scope_origin' && $actual !== $value) { + throw new InvalidArgumentException( + 'Run cancellation does not preserve its original scope context.' + ); + } + } + } + + return new self( + $requestId, + $rootRequestId, + $rootInstanceId, + $rootRunId, + $parentRequestId, + $reason, + $normalizedRequester, + self::text($snapshot, 'source'), + $requestedAt, + $deadline, + $normalizedLineage, + $scopeOrigin, + ); + } + + public function requestedAt(): CarbonImmutable + { + return $this->originalRequestedAt; + } + + public function deadline(): CarbonImmutable + { + return $this->cleanupDeadline; + } + + /** + * Derive a local delivery identity without granting another cleanup budget. + */ + public function forDescendant(string $requestId, string $workflowInstanceId, string $workflowRunId): self + { + if ($this->scopeOrigin !== null) { + return self::fromScopeContext( + ScopedCancellationContext::fromRunContext($this), + $requestId, + $workflowInstanceId, + $workflowRunId, + ); + } + $snapshot = $this->toArray(); + $snapshot['request_id'] = $requestId; + $snapshot['parent_request_id'] = $this->requestId; + $snapshot['lineage'][] = [ + 'request_id' => $requestId, + 'workflow_instance_id' => $workflowInstanceId, + 'workflow_run_id' => $workflowRunId, + ]; + + return self::fromArray($snapshot); + } + + /** + * Preserve every scope address when cooperative cancellation enters a child run. + */ + public static function fromScopeContext( + ScopedCancellationContext $origin, + string $requestId, + string $workflowInstanceId, + string $workflowRunId, + ?CarbonImmutable $authorityDeadline = null, + ): self { + return self::fromArray(self::scopedDescendantSnapshot( + $origin, + $requestId, + $workflowInstanceId, + $workflowRunId, + $authorityDeadline, + )); + } + + /** + * Remaining seconds at the consumed blocking boundary, never the host clock. + */ + public function remaining(): float + { + $fiber = Fiber::getCurrent(); + $binding = self::$replayFibers === null ? null : (self::$replayFibers[$this] ?? null); + if (! $fiber instanceof Fiber || $binding?->get() !== $fiber || ! WorkflowFiberContext::active()) { + throw new LogicException('Cancellation remaining() requires its active workflow replay.'); + } + + $time = WorkflowFiberContext::getCancellationTime(); + $seconds = $this->cleanupDeadline->getTimestamp() - $time->getTimestamp(); + $microseconds = (int) $this->cleanupDeadline->format('u') - (int) $time->format('u'); + + return max(0.0, $seconds + $microseconds / 1_000_000); + } + + /** + * @internal Bind only at canonical executor delivery, without retaining the Fiber. + */ + public function bindToReplayFiber(Fiber $fiber): void + { + self::$replayFibers ??= new WeakMap(); + self::$replayFibers[$this] = WeakReference::create($fiber); + } + + /** + * @return array + */ + public function toArray(): array + { + return [ + 'schema' => $this->scopeOrigin === null ? 'durable-workflow.cancellation-context/v1' + : 'durable-workflow.cancellation-context/v2', + 'request_id' => $this->requestId, + 'root_request_id' => $this->rootRequestId, + 'root_workflow_instance_id' => $this->rootWorkflowInstanceId, + 'root_workflow_run_id' => $this->rootWorkflowRunId, + 'parent_request_id' => $this->parentRequestId, + 'reason' => $this->reason, + 'requester' => $this->requester, + 'source' => $this->source, + 'requested_at' => $this->originalRequestedAt->toISOString(), + 'cleanup_deadline_at' => $this->cleanupDeadline->toISOString(), + 'lineage' => $this->lineage, + ...($this->scopeOrigin === null ? [] : [ + 'scope_origin' => $this->scopeOrigin->toArray(), + 'scope_authority_deadline_at' => $this->cleanupDeadline->toISOString(), + ]), + ]; + } + + /** + * @return array + */ + private static function scopedDescendantSnapshot( + ScopedCancellationContext $origin, + string $requestId, + string $workflowInstanceId, + string $workflowRunId, + ?CarbonImmutable $authorityDeadline = null, + ): array { + if (in_array($workflowRunId, array_column($origin->lineage, 'workflow_run_id'), true) + || in_array($requestId, array_column($origin->lineage, 'request_id'), true) + || trim($requestId) === '' || trim($workflowInstanceId) === '' || trim($workflowRunId) === '') { + throw new InvalidArgumentException('Scoped run cancellation cannot repeat a request or run.'); + } + $root = $origin->rootContext->toArray(); + $deadline = $authorityDeadline ?? $origin->deadline(); + if ($deadline->greaterThan($origin->deadline()) || $deadline->lessThanOrEqualTo($origin->requestedAt())) { + throw new InvalidArgumentException('Child cancellation authority cannot extend its original scope budget.'); + } + $lineage = [$root['lineage'][0]]; + foreach ($origin->lineage as $entry) { + if ($entry['workflow_run_id'] === $origin->rootContext->rootWorkflowRunId) { + continue; + } + $address = array_intersect_key($entry, array_flip([ + 'request_id', 'workflow_instance_id', 'workflow_run_id', + ])); + $index = count($lineage) - 1; + if ($lineage[$index]['workflow_run_id'] === $entry['workflow_run_id']) { + $lineage[$index] = $address; + } else { + $lineage[] = $address; + } + } + return [ + ...$root, + 'schema' => 'durable-workflow.cancellation-context/v2', + 'request_id' => $requestId, + 'parent_request_id' => $origin->requestId, + 'cleanup_deadline_at' => $deadline + ->toISOString(), + 'lineage' => [...$lineage, [ + 'request_id' => $requestId, + 'workflow_instance_id' => $workflowInstanceId, + 'workflow_run_id' => $workflowRunId, + ]], + 'scope_origin' => $origin->toArray(), + 'scope_authority_deadline_at' => $deadline->toISOString(), + ]; + } + + /** + * @param array $value + */ + private static function text(array $value, string $key): string + { + $text = $value[$key] ?? null; + if (! is_string($text) || trim($text) === '') { + throw new InvalidArgumentException(sprintf('Cancellation %s must be a non-empty string.', $key)); + } + + return $text; + } + + private static function timestamp(string $value): CarbonImmutable + { + if (preg_match('/\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})\z/', $value) !== 1) { + throw new InvalidArgumentException('Cancellation timestamp requires an ISO date, time and timezone.'); + } + $time = CarbonImmutable::parse($value); + $errors = CarbonImmutable::getLastErrors(); + if ($errors !== false && ($errors['warning_count'] !== 0 || $errors['error_count'] !== 0)) { + throw new InvalidArgumentException('Cancellation timestamp is invalid.'); + } + + return $time; + } +} diff --git a/src/V2/CommandResult.php b/src/V2/CommandResult.php index aee6181b2..118715338 100644 --- a/src/V2/CommandResult.php +++ b/src/V2/CommandResult.php @@ -143,6 +143,13 @@ public function reason(): ?string return $this->command->commandReason(); } + public function cancellationContext(): ?CancellationContext + { + $snapshot = $this->command->payloadValues(['cancellation'])['cancellation'] ?? null; + + return is_array($snapshot) ? CancellationContext::fromArray($snapshot) : null; + } + public function message(): ?string { return $this->command->commandMessage(); diff --git a/src/V2/Contracts/CancellationScopeAdmission.php b/src/V2/Contracts/CancellationScopeAdmission.php new file mode 100644 index 000000000..2438f76d2 --- /dev/null +++ b/src/V2/Contracts/CancellationScopeAdmission.php @@ -0,0 +1,24 @@ + $commands + * @return string|null The durable refusal reason, or null when valid. + */ + public function validateCancellationScopeMembership(WorkflowRun $run, array $commands, int $sequence): ?string; +} diff --git a/src/V2/Contracts/CancellationScopeTaskBridge.php b/src/V2/Contracts/CancellationScopeTaskBridge.php new file mode 100644 index 000000000..1a029df7e --- /dev/null +++ b/src/V2/Contracts/CancellationScopeTaskBridge.php @@ -0,0 +1,46 @@ + $commands + * @return array + */ + public function checkpointCancellationScopePrefix( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + string $checkpointId, + int $startSequence, + array $commands, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; + + /** + * @return array + */ + public function openCancellationScope( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + string $parentScopeId = 'root', + bool $shieldParent = false, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; +} diff --git a/src/V2/Contracts/PreparedCancellationScopeTaskBridge.php b/src/V2/Contracts/PreparedCancellationScopeTaskBridge.php new file mode 100644 index 000000000..209498ada --- /dev/null +++ b/src/V2/Contracts/PreparedCancellationScopeTaskBridge.php @@ -0,0 +1,51 @@ + + */ + public function prepareCancellationScopeDelivery( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + string $scopeId, + string $requestId, + int $sequence, + string $callKind, + int $sequenceSpan = 1, + ?int $operationSequence = null, + int $operationSequenceSpan = 1, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; + + /** + * Dispatch the original scoped operations, then record their policy barrier. + * Preparation and each actor commit separately, allowing partial retry. + * Pending stop proof retains the preparation and workflow claim. + * @return array + */ + public function deliverCancellationScope( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + string $scopeId, + string $requestId, + int $sequence, + string $callKind, + int $sequenceSpan = 1, + ?int $operationSequence = null, + int $operationSequenceSpan = 1, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; +} diff --git a/src/V2/Contracts/PreparedLocalActivityGroupTaskBridge.php b/src/V2/Contracts/PreparedLocalActivityGroupTaskBridge.php new file mode 100644 index 000000000..e306c5044 --- /dev/null +++ b/src/V2/Contracts/PreparedLocalActivityGroupTaskBridge.php @@ -0,0 +1,28 @@ + $commands + * @return array + */ + public function checkpointLocalActivityGroup( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + string $checkpointId, + int $startSequence, + array $commands, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; +} diff --git a/src/V2/Contracts/PreparedLocalActivityTaskBridge.php b/src/V2/Contracts/PreparedLocalActivityTaskBridge.php new file mode 100644 index 000000000..1411e1d4c --- /dev/null +++ b/src/V2/Contracts/PreparedLocalActivityTaskBridge.php @@ -0,0 +1,102 @@ + $commands + * @return array + */ + public function checkpointLocalActivityPrefix( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + string $checkpointId, + int $startSequence, + array $commands, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; + + /** @param array $descriptor + * @return array + */ + public function prepareLocalActivity( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + string $workerAttemptId, + array $descriptor, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; + + /** @param array $report + * @return array + */ + public function recordLocalActivityOutcome( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + array $report, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; + + /** @param array $descriptor + * @return array + */ + public function recoverLocalActivity( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + array $descriptor, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; + + /** + * Poll cancellation and authority independently of application heartbeats. + * Optional renewal commits the attempt and hosting claim together. + * @return array + */ + public function controlLocalActivity( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + bool $renewLease = false, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; + + /** + * Record a real application heartbeat. This does not renew either lease. + * @param array $progress + * @return array + */ + public function heartbeatLocalActivity( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + array $progress = [], + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; + + /** + * @return array + */ + public function acknowledgeLocalActivityCancellation( + string $attemptId, + string $leaseOwner, + string $requestId, + int $workflowTaskAttempt, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; +} diff --git a/src/V2/Contracts/WorkflowTaskBridge.php b/src/V2/Contracts/WorkflowTaskBridge.php index a47bbb52d..8b2cb0d2e 100644 --- a/src/V2/Contracts/WorkflowTaskBridge.php +++ b/src/V2/Contracts/WorkflowTaskBridge.php @@ -303,7 +303,8 @@ public function heartbeat(string $taskId): array; * payload_codec?: string|null, * connection?: string|null, * queue?: string|null, - * parent_close_policy?: 'abandon'|'request_cancel'|'terminate', + * parent_close_policy?: 'abandon'|'request_cancel'|'request_cancellation'|'terminate', + * cancellation_policy?: 'try_cancel'|'wait_cancellation_completed'|'abandon', * retry_policy?: array, * execution_timeout_seconds?: int, * run_timeout_seconds?: int diff --git a/src/V2/Enums/CancellationPolicy.php b/src/V2/Enums/CancellationPolicy.php new file mode 100644 index 000000000..fb9d92784 --- /dev/null +++ b/src/V2/Enums/CancellationPolicy.php @@ -0,0 +1,13 @@ +historyEvents->contains(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::TimerCancelled + && ($event->payload['timer_id'] ?? null) === $timerId); + $firedInHistory = $run->historyEvents->contains(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::TimerFired + && ($event->payload['timer_id'] ?? null) === $timerId); if ( in_array($run->status, [RunStatus::Cancelled, RunStatus::Terminated], true) || $timer->status === TimerStatus::Cancelled + || $cancelledInHistory ) { $task->forceFill([ - 'status' => $task->status === TaskStatus::Cancelled ? TaskStatus::Cancelled : TaskStatus::Completed, + 'status' => $task->status === TaskStatus::Cancelled || $cancelledInHistory + ? TaskStatus::Cancelled : TaskStatus::Completed, + 'lease_owner' => null, 'lease_expires_at' => null, ])->save(); @@ -144,6 +153,19 @@ public function handle(): void return null; } + if ($firedInHistory) { + $task->forceFill([ + 'status' => TaskStatus::Completed, + 'lease_owner' => null, + 'lease_expires_at' => null, + ])->save(); + $timer->forceFill([ + 'status' => TimerStatus::Fired, + ])->save(); + $this->projectRun($run, self::PROJECTION_RUN_RELATIONS); + return null; + } + $timer->forceFill([ 'status' => TimerStatus::Fired, 'fired_at' => now(), diff --git a/src/V2/Models/WorkflowRun.php b/src/V2/Models/WorkflowRun.php index b96122ed3..c36b5f71e 100644 --- a/src/V2/Models/WorkflowRun.php +++ b/src/V2/Models/WorkflowRun.php @@ -49,6 +49,7 @@ class WorkflowRun extends Model 'cancellation_deadline_at' => 'datetime', 'cancellation_delivered_at' => 'datetime', 'cancellation_delivery_sequence' => 'integer', + 'cancellation_scope_recovery_until' => UtcScheduleTimestamp::class, 'sticky_until' => 'datetime', 'started_at' => 'datetime', 'closed_at' => 'datetime', diff --git a/src/V2/ScopedCancellationContext.php b/src/V2/ScopedCancellationContext.php new file mode 100644 index 000000000..f35ebc2be --- /dev/null +++ b/src/V2/ScopedCancellationContext.php @@ -0,0 +1,226 @@ + $lineage + */ + private function __construct( + public readonly CancellationContext $rootContext, + public readonly array $lineage, + private readonly CarbonImmutable $cleanupDeadline, + ) { + $last = $lineage[count($lineage) - 1]; + $this->requestId = $last['request_id']; + $this->parentRequestId = count($lineage) === 1 ? null : $lineage[count($lineage) - 2]['request_id']; + $this->workflowInstanceId = $last['workflow_instance_id']; + $this->workflowRunId = $last['workflow_run_id']; + $this->scopeId = $last['scope_id']; + $this->rootScopeId = $lineage[0]['scope_id']; + } + + /** + * @param array $snapshot + */ + public static function fromArray(array $snapshot): self + { + self::assertKeys($snapshot, ['schema', 'root_context', 'lineage']); + if ($snapshot['schema'] !== self::SCHEMA || ! is_array($snapshot['root_context'])) { + throw new InvalidArgumentException('Unsupported scoped cancellation context schema or root.'); + } + $root = CancellationContext::fromArray($snapshot['root_context']); + self::assertKeys($snapshot['root_context'], array_keys($root->toArray())); + if ($root->requestId !== $root->rootRequestId || $root->parentRequestId !== null || count( + $root->lineage + ) !== 1) { + throw new InvalidArgumentException('Scoped cancellation root must contain the original root request.'); + } + $lineage = $snapshot['lineage']; + if (! is_array($lineage) || ! array_is_list($lineage) || $lineage === []) { + throw new InvalidArgumentException('Scoped cancellation lineage must contain its root address.'); + } + $requests = []; + $addresses = []; + $instancesByRun = []; + $lastRunId = null; + $normalized = []; + $deadline = $root->deadline(); + foreach ($lineage as $index => $entry) { + if (! is_array($entry)) { + throw new InvalidArgumentException('Scoped cancellation lineage entry is invalid.'); + } + self::assertKeys($entry, [ + 'request_id', 'workflow_instance_id', 'workflow_run_id', 'scope_id', 'cleanup_deadline_at', + ]); + $entry = [ + 'request_id' => self::text($entry, 'request_id'), + 'workflow_instance_id' => self::text($entry, 'workflow_instance_id'), + 'workflow_run_id' => self::text($entry, 'workflow_run_id'), + 'scope_id' => self::text($entry, 'scope_id'), + 'cleanup_deadline_at' => self::text($entry, 'cleanup_deadline_at'), + ]; + if ($index === 0 && ($entry['request_id'] !== $root->requestId + || $entry['workflow_instance_id'] !== $root->rootWorkflowInstanceId + || $entry['workflow_run_id'] !== $root->rootWorkflowRunId)) { + throw new InvalidArgumentException('Scoped cancellation root address does not match its request.'); + } + $address = json_encode([$entry['workflow_run_id'], $entry['scope_id']], JSON_THROW_ON_ERROR); + if (isset($requests[$entry['request_id']]) || isset($addresses[$address])) { + throw new InvalidArgumentException('Scoped cancellation lineage cannot repeat a request or address.'); + } + if (isset($instancesByRun[$entry['workflow_run_id']]) + && $instancesByRun[$entry['workflow_run_id']] !== $entry['workflow_instance_id']) { + throw new InvalidArgumentException( + 'Scoped cancellation lineage assigns conflicting instances to one run.' + ); + } + if ($lastRunId !== null && $lastRunId !== $entry['workflow_run_id'] + && isset($instancesByRun[$entry['workflow_run_id']])) { + throw new InvalidArgumentException('Scoped cancellation lineage cannot reenter an earlier run.'); + } + $deadlineSnapshot = $root->toArray(); + $deadlineSnapshot['cleanup_deadline_at'] = $entry['cleanup_deadline_at']; + $nextDeadline = CancellationContext::fromArray($deadlineSnapshot)->deadline(); + if (($index === 0 && ! $nextDeadline->equalTo($root->deadline())) || $nextDeadline->greaterThan( + $deadline + )) { + throw new InvalidArgumentException( + 'Scoped cancellation lineage cannot change its root or extend a descendant budget.' + ); + } + $entry['cleanup_deadline_at'] = $nextDeadline->toISOString(); + $normalized[] = $entry; + $requests[$entry['request_id']] = true; + $addresses[$address] = true; + $instancesByRun[$entry['workflow_run_id']] = $entry['workflow_instance_id']; + $lastRunId = $entry['workflow_run_id']; + $deadline = $nextDeadline; + } + return new self($root, $normalized, $deadline); + } + + public static function fromRunContext(CancellationContext $context): self + { + if ($context->scopeOrigin !== null) { + $last = $context->lineage[count($context->lineage) - 1]; + return $context->scopeOrigin->forDescendant( + $context->requestId, + $last['workflow_instance_id'], + $last['workflow_run_id'], + CancellationScopeHistory::ROOT_SCOPE_ID, + $context->deadline(), + ); + } + $root = $context->toArray(); + $root['request_id'] = $context->rootRequestId; + $root['parent_request_id'] = null; + $root['lineage'] = [$context->lineage[0]]; + return self::fromArray([ + 'schema' => self::SCHEMA, + 'root_context' => $root, + 'lineage' => array_map(static fn (array $entry): array => [ + ...$entry, + 'scope_id' => CancellationScopeHistory::ROOT_SCOPE_ID, + 'cleanup_deadline_at' => $context->deadline() + ->toISOString(), + ], $context->lineage), + ]); + } + + public function forDescendant( + string $requestId, + string $workflowInstanceId, + string $workflowRunId, + string $scopeId, + ?CarbonImmutable $deadline = null, + ): self { + $snapshot = $this->toArray(); + $snapshot['lineage'][] = [ + 'request_id' => $requestId, + 'workflow_instance_id' => $workflowInstanceId, + 'workflow_run_id' => $workflowRunId, + 'scope_id' => $scopeId, + 'cleanup_deadline_at' => ($deadline ?? $this->cleanupDeadline) + ->toISOString(), + ]; + return self::fromArray($snapshot); + } + + public function requestedAt(): CarbonImmutable + { + return $this->rootContext->requestedAt(); + } + + public function rootDeadline(): CarbonImmutable + { + return $this->rootContext->deadline(); + } + + public function deadline(): CarbonImmutable + { + return $this->cleanupDeadline; + } + + /** + * @return array + */ + public function toArray(): array + { + return [ + 'schema' => self::SCHEMA, + 'root_context' => $this->rootContext->toArray(), + 'lineage' => $this->lineage, + ]; + } + + /** + * @param array $value + */ + private static function text(array $value, string $key): string + { + $text = $value[$key]; + if (! is_string($text) || trim($text) === '') { + throw new InvalidArgumentException('Scoped cancellation address and deadline must be nonempty strings.'); + } + return $text; + } + + /** @param array $value + * @param list $keys + */ + private static function assertKeys(array $value, array $keys): void + { + $actual = array_keys($value); + sort($actual); + sort($keys); + if ($actual !== $keys) { + throw new InvalidArgumentException('Scoped cancellation context has missing or unsupported fields.'); + } + } +} diff --git a/src/V2/Support/ActivityAbandonment.php b/src/V2/Support/ActivityAbandonment.php new file mode 100644 index 000000000..496dcbba3 --- /dev/null +++ b/src/V2/Support/ActivityAbandonment.php @@ -0,0 +1,97 @@ +relationLoaded('historyEvents')) { + $run->loadMissing('historyEvents'); + } + return $run->historyEvents->first(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::ActivityScheduled + && ($event->payload['sequence'] ?? null) === $execution->sequence + && ($event->payload['activity_execution_id'] ?? null) === $execution->id + && ($event->payload['activity']['id'] ?? null) === $execution->id + && ($event->payload['activity']['cancellation_policy'] ?? null) === CancellationPolicy::Abandon->value); + } + + public static function prepare(WorkflowRun $run, ActivityExecution $execution): void + { + if (LocalActivityRuntime::isExecution($execution)) { + throw new LogicException('Local Activity Abandon requires an independent callback lifetime.'); + } + if (! self::allows($run, $execution)) { + throw new LogicException( + 'Activity Abandon requires its canonical policy and original finite total deadline.' + ); + } + } + + public static function hasTerminalHistory(WorkflowRun $run, WorkflowHistoryEvent $scheduled): bool + { + $executionId = $scheduled->payload['activity_execution_id'] ?? null; + $sequence = $scheduled->payload['sequence'] ?? null; + if (! is_string($executionId) || ($scheduled->payload['activity']['id'] ?? null) !== $executionId + || ! is_int($sequence) || $sequence < 1) { + return false; + } + $started = $run->historyEvents->filter(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::ActivityStarted + && ($event->payload['activity_execution_id'] ?? null) === $executionId)->sortByDesc('sequence')->first(); + return $run->historyEvents->contains(static fn (WorkflowHistoryEvent $event): bool => + in_array($event->event_type, [HistoryEventType::ActivityCompleted, HistoryEventType::ActivityFailed, + HistoryEventType::ActivityCancelled, HistoryEventType::ActivityTimedOut], true) + && ($event->payload['activity_execution_id'] ?? null) === $executionId + && ($event->payload['sequence'] ?? null) === $sequence + && ($started !== null || in_array( + $event->event_type, + [HistoryEventType::ActivityCancelled, HistoryEventType::ActivityTimedOut], + true + )) + && $event->sequence > ($started?->sequence ?? $scheduled->sequence) + && array_key_exists('activity_attempt_id', $event->payload) + && ($event->payload['activity_attempt_id'] ?? null) === ($started?->payload['activity_attempt_id'] ?? null)); + } + + public static function allows(WorkflowRun $run, ActivityExecution $execution): bool + { + if ($run->cancellation_request_command_id === null + || $execution->workflow_run_id !== $run->id || LocalActivityRuntime::isExecution($execution)) { + return false; + } + $scheduled = self::scheduled($run, $execution); + $deadline = $scheduled?->payload['activity']['schedule_to_close_deadline_at'] ?? null; + if (! is_string($deadline) || $deadline === '' || $execution->schedule_to_close_deadline_at === null + || ! $execution->schedule_to_close_deadline_at->equalTo(CarbonImmutable::parse($deadline)) + || CooperativeCancellationDelivery::context($run) === null + || $run->historyEvents->contains(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::ActivityCancelled + && ($event->payload['activity_execution_id'] ?? null) === $execution->id)) { + return false; + } + if (! $run->status->isTerminal()) { + return true; + } + // A legacy terminal cancel or termination cannot borrow cooperation's + // identity to retain authority. The canonical cooperative close must match. + return $run->status === RunStatus::Cancelled + && CooperativeCancellationDelivery::recorded($run) !== null + && $run->historyEvents->contains(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::WorkflowCancelled + && $event->workflow_command_id === $run->cancellation_request_command_id); + } +} diff --git a/src/V2/Support/ActivityCancellation.php b/src/V2/Support/ActivityCancellation.php index d568a9284..1b6fc0471 100644 --- a/src/V2/Support/ActivityCancellation.php +++ b/src/V2/Support/ActivityCancellation.php @@ -8,6 +8,7 @@ use Workflow\V2\Enums\ActivityStatus; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\TaskStatus; +use Workflow\V2\Enums\TaskType; use Workflow\V2\Models\ActivityAttempt; use Workflow\V2\Models\ActivityExecution; use Workflow\V2\Models\WorkflowCommand; @@ -17,12 +18,45 @@ final class ActivityCancellation { + /** + * @param array|null $scopeCancellation + */ public static function record( WorkflowRun $run, ActivityExecution $execution, ?WorkflowTask $task = null, WorkflowCommand|string|null $command = null, + ?array $scopeCancellation = null, + ?ScopedCancellationPreparation $scopePreparation = null, ): ?WorkflowHistoryEvent { + if ($scopeCancellation !== null) { + $context = ActivityCancellationContext::forScopeSnapshot( + $run, + $scopeCancellation, + $execution->id, + $execution->sequence + ); + $authority = is_string($scopeCancellation['scope_id'] ?? null) + ? CancellationScopeRequests::authority($run, $scopeCancellation['scope_id']) : null; + $deadlineMatches = $scopePreparation === null + ? ($authority['deadline_at'] ?? null) === ($scopeCancellation['authority_deadline_at'] ?? null) + : $scopePreparation->workflowRunId === $run->id + && $scopePreparation->scopeId === ($scopeCancellation['scope_id'] ?? null) + && $scopePreparation->requestId === $command + && $scopePreparation->requestHistoryEventId === ($scopeCancellation['request_history_event_id'] ?? null) + && $scopePreparation->context->toArray() === ($scopeCancellation['cancellation'] ?? null) + && $scopePreparation->authorityDeadlineAt === ($scopeCancellation['authority_deadline_at'] ?? null) + && now() + ->lt(\Carbon\CarbonImmutable::parse($scopePreparation->authorityDeadlineAt)) + && collect($scopePreparation->activityMembers) + ->contains(static fn (array $member): bool => + $member['activity_execution_id'] === $execution->id && $member['sequence'] === $execution->sequence); + if ($context === null || $context->requestId !== $command || $execution->workflow_run_id !== $run->id + || ($execution->activity_options['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID) !== $context->scopeId + || ! ($authority['active'] ?? false) || ! $deadlineMatches) { + throw new \LogicException('cancellation_scope_activity_context_mismatch'); + } + } $cancelledAt = now(); if ($execution->status !== ActivityStatus::Cancelled || $execution->closed_at === null) { @@ -34,7 +68,12 @@ public static function record( $attempt = self::currentAttempt($execution); - if ($attempt instanceof ActivityAttempt && $attempt->status !== ActivityAttemptStatus::Cancelled) { + if ($attempt instanceof ActivityAttempt && $attempt->status !== ActivityAttemptStatus::Cancelled + && ($scopeCancellation === null || ! in_array( + $attempt->status, + [ActivityAttemptStatus::Completed, ActivityAttemptStatus::Failed], + true + ))) { $attempt->forceFill([ 'status' => ActivityAttemptStatus::Cancelled, 'lease_expires_at' => null, @@ -42,7 +81,10 @@ public static function record( ])->save(); } - if ($task instanceof WorkflowTask && ($task->status !== TaskStatus::Cancelled || $task->lease_expires_at !== null)) { + $retainWorkflowClaim = $scopeCancellation !== null && LocalActivityRuntime::isExecution($execution) + && $task?->task_type === TaskType::Workflow; + if ($task instanceof WorkflowTask && ! $retainWorkflowClaim + && ($task->status !== TaskStatus::Cancelled || $task->lease_expires_at !== null)) { $task->forceFill([ 'status' => TaskStatus::Cancelled, 'lease_expires_at' => null, @@ -69,6 +111,9 @@ public static function record( 'activity' => ActivitySnapshot::fromExecution($execution), 'activity_attempt' => self::attemptSnapshot($attempt), ]; + if ($scopeCancellation !== null) { + $payload['cancellation_scope'] = $scopeCancellation; + } if (LocalActivityRuntime::isExecution($execution)) { $payload = LocalActivityRuntime::eventPayload($payload); @@ -118,6 +163,7 @@ private static function attemptSnapshot(?ActivityAttempt $attempt): ?array return array_filter([ 'id' => $attempt->id, + 'worker_attempt_id' => $attempt->worker_attempt_id, 'activity_execution_id' => $attempt->activity_execution_id, 'task_id' => $attempt->workflow_task_id, 'attempt_number' => $attempt->attempt_number, diff --git a/src/V2/Support/ActivityCancellationAcknowledgement.php b/src/V2/Support/ActivityCancellationAcknowledgement.php new file mode 100644 index 000000000..c086b59a3 --- /dev/null +++ b/src/V2/Support/ActivityCancellationAcknowledgement.php @@ -0,0 +1,261 @@ +lockForUpdate() + ->find($attempt->workflow_run_id); + /** @var WorkflowTask|null $task */ + $task = ConfiguredV2Models::query('task_model', WorkflowTask::class) + ->lockForUpdate() + ->find($attempt->workflow_task_id); + if (! $run instanceof WorkflowRun || ! $task instanceof WorkflowTask) { + return self::refused('activity_claim_not_found'); + } + if ($leaseOwner === '' || $requestId === '' + || $execution->current_attempt_id !== $attemptId + || $execution->workflow_run_id !== $run->id + || $task->workflow_run_id !== $run->id + || $attempt->lease_owner !== $leaseOwner + || (! $local && $task->lease_owner !== $leaseOwner) + || ($local && $task->task_type !== TaskType::Workflow)) { + return self::refused('activity_cancellation_acknowledgement_fence_mismatch'); + } + if ($run->cancellation_request_command_id !== $requestId && ! $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeRequested) + ->where('payload->request_id', $requestId) + ->exists()) { + return self::refused('cancellation_request_mismatch'); + } + $events = $run->historyEvents() + ->where('workflow_command_id', $requestId) + ->where('payload->activity_execution_id', $execution->id) + ->whereIn('event_type', [ + HistoryEventType::ActivityCancelled->value, + HistoryEventType::ActivityCancellationAcknowledged->value, + ]) + ->get(); + /** @var WorkflowHistoryEvent|null $cancelled */ + $cancelled = $events->first(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::ActivityCancelled + && ($event->payload['activity_execution_id'] ?? null) === $execution->id + && ($event->payload['activity_attempt_id'] ?? null) === $attemptId); + if (! $cancelled instanceof WorkflowHistoryEvent) { + return self::refused('activity_cancellation_not_recorded'); + } + $snapshot = $cancelled->payload['activity_attempt'] ?? []; + if (! is_array($snapshot) || array_is_list($snapshot) + || ($snapshot['id'] ?? null) !== $attemptId + || ($snapshot['status'] ?? null) !== ActivityAttemptStatus::Cancelled->value + || ($snapshot['lease_owner'] ?? null) !== $leaseOwner + || ($snapshot['task_id'] ?? null) !== $task->id + || ($snapshot['activity_execution_id'] ?? null) !== $execution->id) { + return self::refused('activity_cancellation_snapshot_mismatch'); + } + $originalClaim = null; + if ($local) { + $originalClaim = self::originalLocalClaim( + $run, + $task, + $cancelled, + $attemptId, + $leaseOwner, + $workflowTaskAttempt + ); + if ($originalClaim === null) { + return self::refused('local_activity_workflow_claim_mismatch'); + } + } + $context = ActivityCancellationContext::forEvent($run, $cancelled); + if ($context === null || $context->requestId !== $requestId) { + return self::refused('cancellation_context_not_recorded'); + } + /** @var WorkflowHistoryEvent|null $existing */ + $existing = $events->first(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::ActivityCancellationAcknowledged + && ($event->payload['cancellation_history_event_id'] ?? null) === $cancelled->id + && ($event->payload['activity_attempt_id'] ?? null) === $attemptId); + if ($existing instanceof WorkflowHistoryEvent) { + return [ + 'acknowledged' => true, + 'duplicate' => true, + 'reason' => null, + 'history_event_id' => $existing->id, + ]; + } + if ($attempt->status !== ActivityAttemptStatus::Cancelled + || $execution->status !== ActivityStatus::Cancelled + || (! $local && $task->status !== TaskStatus::Cancelled)) { + return self::refused('activity_cancellation_not_fenced'); + } + $receivedAt = now(); + $payload = [ + 'sequence' => $execution->sequence, + 'activity_execution_id' => $execution->id, + 'activity_attempt_id' => $attemptId, + 'lease_owner' => $leaseOwner, + 'cancellation_history_event_id' => $cancelled->id, + 'request_id' => $requestId, + 'root_request_id' => ActivityCancellationContext::rootRequestId($context), + 'cleanup_deadline_at' => $context->deadline() + ->toISOString(), + 'callback_state' => 'stopped', + 'evidence_source' => $local ? 'workflow_worker' : 'activity_worker', + 'acknowledged_at' => $receivedAt->toISOString(), + 'received_after_deadline' => $receivedAt->gte($context->deadline()), + ]; + if ($originalClaim !== null) { + $payload = LocalActivityRuntime::eventPayload($payload); + $payload['workflow_task_id'] = $task->id; + // Keep the receipt's claim snapshot bound to the original owner. + $payload['task'] = $originalClaim; + } + $event = WorkflowHistoryEvent::record( + $run, + HistoryEventType::ActivityCancellationAcknowledged, + $payload, + $task, + $requestId + ); + ActivityCancellationWait::resume($run, $event); + + return [ + 'acknowledged' => true, + 'duplicate' => false, + 'reason' => null, + 'history_event_id' => $event->id, + ]; + }, 5); + } + + /** + * @return array|null + */ + private static function originalLocalClaim( + WorkflowRun $run, + WorkflowTask $task, + WorkflowHistoryEvent $cancelled, + string $attemptId, + string $leaseOwner, + ?int $workflowTaskAttempt, + ): ?array { + if (($cancelled->payload['local_activity'] ?? null) !== true + || ($cancelled->payload['execution_mode'] ?? null) !== LocalActivityRuntime::EXECUTION_MODE) { + return null; + } + /** @var WorkflowHistoryEvent|null $started */ + $started = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted) + ->where('payload->activity_attempt_id', $attemptId) + ->where('sequence', '<', $cancelled->sequence) + ->first(); + if (! $started instanceof WorkflowHistoryEvent + || ($started->payload['local_activity'] ?? null) !== true + || ($started->payload['execution_mode'] ?? null) !== LocalActivityRuntime::EXECUTION_MODE + || ($started->payload['activity_execution_id'] ?? null) !== $cancelled->payload['activity_execution_id'] + || ($started->payload['workflow_task_id'] ?? null) !== $task->id) { + return null; + } + $claim = $started->payload['task'] ?? null; + if (! is_array($claim) || array_is_list($claim) + || ($claim['id'] ?? null) !== $task->id + || ($claim['type'] ?? null) !== TaskType::Workflow->value + || ($claim['status'] ?? null) !== TaskStatus::Leased->value + || ($claim['lease_owner'] ?? null) !== $leaseOwner + || ! is_int($workflowTaskAttempt) || $workflowTaskAttempt < 1 + || ($claim['attempt_count'] ?? null) !== $workflowTaskAttempt) { + return null; + } + + return $claim; + } + + /** + * @return array{acknowledged: bool, duplicate: bool, reason: string, history_event_id: null} + */ + private static function refused(string $reason): array + { + return [ + 'acknowledged' => false, + 'duplicate' => false, + 'reason' => $reason, + 'history_event_id' => null, + ]; + } +} diff --git a/src/V2/Support/ActivityCancellationCompletion.php b/src/V2/Support/ActivityCancellationCompletion.php new file mode 100644 index 000000000..06e480eaf --- /dev/null +++ b/src/V2/Support/ActivityCancellationCompletion.php @@ -0,0 +1,118 @@ +relationLoaded('historyEvents')) { + $run->loadMissing('historyEvents'); + } + $scheduled = $run->historyEvents->first(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::ActivityScheduled + && ($event->payload['activity_execution_id'] ?? null) === $executionId); + if (! $scheduled instanceof WorkflowHistoryEvent + || ! is_int($scheduled->payload['sequence'] ?? null) || $scheduled->payload['sequence'] < 1 + || ($request === null && CooperativeCancellationDelivery::context($run) === null)) { + return false; + } + + $cancelled = $run->historyEvents->first(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::ActivityCancelled + && ($event->payload['activity_execution_id'] ?? null) === $executionId); + if ($cancelled !== null && ActivityCancellationContext::forEvent($run, $cancelled) === null) { + return false; + } + $started = $run->historyEvents->filter(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::ActivityStarted + && ($event->payload['activity_execution_id'] ?? null) === $executionId)->sortByDesc('sequence')->first(); + if ($started instanceof WorkflowHistoryEvent && is_string($started->payload['activity_attempt_id'] ?? null) + && $run->historyEvents->contains(static fn (WorkflowHistoryEvent $event): bool => + (in_array( + $event->event_type, + [HistoryEventType::ActivityCompleted, HistoryEventType::ActivityFailed], + true + ) || ($event->event_type === HistoryEventType::ActivityRetryScheduled + && ($event->payload['activity_attempt']['id'] ?? null) === $started->payload['activity_attempt_id'] + && ($event->payload['activity_attempt']['activity_execution_id'] ?? null) === $executionId + && ($event->payload['activity_attempt']['status'] ?? null) === ActivityAttemptStatus::Failed->value + && is_string($event->payload['activity_attempt']['closed_at'] ?? null) + && ($event->payload['timeout_kind'] ?? null) === null)) + && ($event->payload['activity_execution_id'] ?? null) === $executionId + && ($event->payload['activity_attempt_id'] ?? null) === $started->payload['activity_attempt_id'] + && ($event->payload['sequence'] ?? null) === ($scheduled->payload['sequence'] ?? null) + && $event->sequence > $started->sequence + && ($cancelled === null || $event->sequence < $cancelled->sequence))) { + return true; + } + + if (! $cancelled instanceof WorkflowHistoryEvent) { + return false; + } + if ($cancelled->sequence <= $scheduled->sequence + || ($cancelled->payload['sequence'] ?? null) !== ($scheduled->payload['sequence'] ?? null) + || ! array_key_exists('activity_attempt_id', $cancelled->payload) + || ! array_key_exists('activity_attempt', $cancelled->payload)) { + return false; + } + $attemptId = $cancelled->payload['activity_attempt_id'] ?? null; + if ($attemptId === null) { + // No Started event and a canonical no-attempt fence prove that atomic + // cancellation won before callback admission. Missing history cannot. + return ($cancelled->payload['activity_attempt'] ?? null) === null + && ! $run->historyEvents->contains(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::ActivityStarted + && ($event->payload['activity_execution_id'] ?? null) === $executionId); + } + + return self::stopReceipt($run, $cancelled) !== null; + } + + public static function stopReceipt(WorkflowRun $run, WorkflowHistoryEvent $cancelled): ?WorkflowHistoryEvent + { + $context = ActivityCancellationContext::forEvent($run, $cancelled); + $snapshot = $cancelled->payload['activity_attempt'] ?? null; + $attemptId = $cancelled->payload['activity_attempt_id'] ?? null; + $executionId = $cancelled->payload['activity_execution_id'] ?? null; + if ($context === null || ! is_string($cancelled->id) || $cancelled->id === '' + || $cancelled->event_type !== HistoryEventType::ActivityCancelled + || $cancelled->workflow_command_id !== $context->requestId + || ! is_string($attemptId) || $attemptId === '' + || ! is_string($executionId) || $executionId === '' + || ! is_array($snapshot) || ($snapshot['id'] ?? null) !== $attemptId + || ($snapshot['activity_execution_id'] ?? null) !== $executionId + || ($snapshot['status'] ?? null) !== ActivityAttemptStatus::Cancelled->value + || ! is_string($snapshot['lease_owner'] ?? null) || $snapshot['lease_owner'] === '') { + return null; + } + $source = ($cancelled->payload['local_activity'] ?? false) === true ? 'workflow_worker' : 'activity_worker'; + return $run->historyEvents->first(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::ActivityCancellationAcknowledged + && $event->sequence > $cancelled->sequence + && $event->workflow_command_id === $context->requestId + && ($event->payload['sequence'] ?? null) === ($cancelled->payload['sequence'] ?? null) + && ($event->payload['activity_execution_id'] ?? null) === $executionId + && ($event->payload['activity_attempt_id'] ?? null) === $attemptId + && ($event->payload['lease_owner'] ?? null) === $snapshot['lease_owner'] + && ($event->payload['cancellation_history_event_id'] ?? null) === $cancelled->id + && ($event->payload['request_id'] ?? null) === $context->requestId + && ($event->payload['root_request_id'] ?? null) === ActivityCancellationContext::rootRequestId($context) + && ($event->payload['cleanup_deadline_at'] ?? null) === $context->deadline()->toISOString() + && ($event->payload['callback_state'] ?? null) === 'stopped' + && ($event->payload['evidence_source'] ?? null) === $source); + } +} diff --git a/src/V2/Support/ActivityCancellationContext.php b/src/V2/Support/ActivityCancellationContext.php new file mode 100644 index 000000000..d8e6ce954 --- /dev/null +++ b/src/V2/Support/ActivityCancellationContext.php @@ -0,0 +1,115 @@ +workflow_run_id !== $run->id || $event->event_type !== HistoryEventType::ActivityCancelled) { + return null; + } + if (! array_key_exists('cancellation_scope', $event->payload)) { + $context = CooperativeCancellationDelivery::context($run); + return $context !== null && $event->workflow_command_id === $context->requestId ? $context : null; + } + $context = self::forScopeSnapshot( + $run, + $event->payload['cancellation_scope'], + $event->payload['activity_execution_id'] ?? null, + $event->payload['sequence'] ?? null, + $event->sequence, + ); + return $context !== null && $event->workflow_command_id === $context->requestId ? $context : null; + } + + /** + * Validate canonical membership and the accepted immutable snapshot, including on cold reads. + */ + public static function forScopeSnapshot( + WorkflowRun $run, + mixed $snapshot, + mixed $executionId, + mixed $sequence, + ?int $fenceHistorySequence = null, + ): ?ScopedCancellationContext { + if (! is_array($snapshot) || ($snapshot['schema'] ?? null) !== self::SCOPE_SCHEMA + || ($snapshot['workflow_run_id'] ?? null) !== $run->id + || ! is_string($snapshot['scope_id'] ?? null) || ! is_array($snapshot['cancellation'] ?? null) + || ! is_string( + $snapshot['request_history_event_id'] ?? null + ) || $snapshot['request_history_event_id'] === '' + || ! is_string($executionId) || $executionId === '' || ! is_int($sequence) || $sequence < 1) { + return null; + } + try { + $context = CancellationScopeRequests::context($run, $snapshot['scope_id']); + $encoded = ScopedCancellationContext::fromArray($snapshot['cancellation']); + if ($context === null || $encoded->toArray() !== $context->toArray() + || ($snapshot['request_id'] ?? null) !== $context->requestId) { + return null; + } + $request = $run->historyEvents() + ->whereKey($snapshot['request_history_event_id']) + ->where('event_type', HistoryEventType::CancellationScopeRequested)->first(); + $scope = CancellationScopeHistory::forRun($run)[$context->scopeId]; + if ($request === null || ($request->payload['scope_id'] ?? null) !== $context->scopeId + || ($request->payload['request_id'] ?? null) !== $context->requestId + || ($scope['shield_parent'] && ($request->payload['parent_scope_id'] ?? null) !== null) + || ($fenceHistorySequence !== null && $request->sequence >= $fenceHistorySequence) + || ! CancellationScopeHistory::isRecordedBefore($run, $context->scopeId, $sequence)) { + return null; + } + $scheduled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityScheduled) + ->where('payload->activity_execution_id', $executionId) + ->get(); + if ($scheduled->count() !== 1) { + return null; + } + $event = $scheduled->sole(); + $payload = $event->payload; + $activity = $payload['activity'] ?? null; + if (($payload['sequence'] ?? null) !== $sequence || ! is_array($activity) + || ($activity['id'] ?? null) !== $executionId + || ($activity['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID) !== $context->scopeId + || (array_key_exists( + 'cancellation_scope_id', + $payload + ) && $payload['cancellation_scope_id'] !== $context->scopeId) + || ($fenceHistorySequence !== null && $event->sequence >= $fenceHistorySequence)) { + return null; + } + $deadline = CancellationContext::fromArray([ + ...$context->rootContext->toArray(), + 'cleanup_deadline_at' => $snapshot['authority_deadline_at'] ?? null, + ])->deadline(); + if ($deadline->greaterThan($context->deadline())) { + return null; + } + return $context; + } catch (InvalidArgumentException|LogicException) { + return null; + } + } + + public static function rootRequestId(CancellationContext|ScopedCancellationContext $context): string + { + return $context instanceof ScopedCancellationContext ? $context->rootContext->rootRequestId : $context->rootRequestId; + } +} diff --git a/src/V2/Support/ActivityCancellationWait.php b/src/V2/Support/ActivityCancellationWait.php new file mode 100644 index 000000000..36e42fdf6 --- /dev/null +++ b/src/V2/Support/ActivityCancellationWait.php @@ -0,0 +1,193 @@ +relationLoaded('historyEvents')) { + $run->loadMissing('historyEvents'); + } + $scheduled = $run->historyEvents->first(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::ActivityScheduled + && ($event->payload['sequence'] ?? null) === $sequence); + $value = $scheduled?->payload['activity']['cancellation_policy'] ?? CancellationPolicy::TryCancel->value; + return is_string($value) ? CancellationPolicy::from($value) + : throw new LogicException('Recorded activity cancellation policy is invalid.'); + } + + /** + * The caller owns the run lock. True means release the workflow claim until callback exit is proved. + */ + public static function prepare(WorkflowRun $run, WorkflowTask $workflowTask, ActivityExecution $execution): bool + { + $context = CooperativeCancellationDelivery::context($run); + if ($context === null || $execution->workflow_run_id !== $run->id) { + throw new LogicException('Activity cancellation waiting requires the original run context.'); + } + if (in_array($execution->status, [ActivityStatus::Pending, ActivityStatus::Running], true)) { + $activityTask = $run->tasks->first(static fn (WorkflowTask $task): bool => + $task->task_type === TaskType::Activity + && ($task->payload['activity_execution_id'] ?? null) === $execution->id); + $event = ActivityCancellation::record($run, $execution, $activityTask, $context->requestId); + if ($event !== null) { + $run->historyEvents->push($event); + } + } + if (ActivityCancellationCompletion::resolved($run, $execution->id)) { + return false; + } + $payload = $workflowTask->payload; + $ids = $payload['cancellation_activity_wait']['execution_ids'] ?? []; + $payload['cancellation_activity_wait'] = [ + 'request_id' => $context->requestId, + 'root_request_id' => $context->rootRequestId, + 'cleanup_deadline_at' => $context->deadline() + ->toISOString(), + 'execution_ids' => array_values(array_unique([...$ids, $execution->id])), + ]; + $workflowTask->payload = $payload; + return true; + } + + public static function bindBoundary( + WorkflowTask $task, + int $sequence, + string $callKind, + int $sequenceSpan, + ?int $operationSequence, + int $operationSequenceSpan, + ): void { + $payload = $task->payload; + if (isset($payload['cancellation_activity_wait'])) { + $payload['cancellation_activity_wait']['boundary'] = [ + 'sequence' => $sequence, + 'call_kind' => $callKind, + 'sequence_span' => $sequenceSpan, + 'operation_sequence' => $operationSequence, + 'operation_sequence_span' => $operationSequenceSpan, + ]; + $task->payload = $payload; + } + } + + public static function validateBoundary( + WorkflowRun $run, + int $sequence, + string $callKind, + int $sequenceSpan, + ?int $operationSequence, + int $operationSequenceSpan, + ): ?string { + // Pure replay models can supply their task relation. Persisted runs read + // completed hosting claims under the run lock even after replacement. + if (! $run->relationLoaded('tasks')) { + if (! $run->exists) { + return null; + } + $run->loadMissing('tasks'); + } + $expected = [ + 'sequence' => $sequence, + 'call_kind' => $callKind, + 'sequence_span' => $sequenceSpan, + 'operation_sequence' => $operationSequence, + 'operation_sequence_span' => $operationSequenceSpan, + ]; + foreach ($run->tasks as $task) { + $wait = $task->payload['cancellation_activity_wait'] ?? null; + if ($task->status === TaskStatus::Completed && is_array($wait) + && ($wait['request_id'] ?? null) === $run->cancellation_request_command_id) { + $context = CooperativeCancellationDelivery::context($run); + if ($context === null || ($wait['root_request_id'] ?? null) !== $context->rootRequestId + || ($wait['cleanup_deadline_at'] ?? null) !== $context->deadline()->toISOString()) { + return 'cancellation_wait_context_mismatch'; + } + if (($wait['boundary'] ?? null) !== $expected) { + return 'cancellation_wait_boundary_mismatch'; + } + } + } + return null; + } + + /** + * Called within stop-receipt persistence under the same run lock. + */ + public static function resume(WorkflowRun $run, WorkflowHistoryEvent $receipt): ?WorkflowTask + { + if ($run->status !== RunStatus::Waiting || $run->cancellation_deadline_at === null + || now() + ->gte($run->cancellation_deadline_at)) { + return null; + } + $run->setRelation('historyEvents', $run->historyEvents()->get()); + if (CooperativeCancellationDelivery::recorded($run) !== null) { + return null; + } + $context = CooperativeCancellationDelivery::context($run); + if ($context === null || $receipt->workflow_command_id !== $context->requestId) { + return null; + } + $tasks = $run->tasks() + ->where('task_type', TaskType::Workflow->value)->lockForUpdate()->get(); + if ($tasks->contains(static fn (WorkflowTask $task): bool => + in_array($task->status, [TaskStatus::Ready, TaskStatus::Leased], true))) { + return null; + } + $waitingTask = $tasks->sortByDesc('created_at') + ->first(static fn (WorkflowTask $task): bool => + $task->status === TaskStatus::Completed + && ($task->payload['cancellation_activity_wait']['request_id'] ?? null) === $context->requestId); + $wait = $waitingTask?->payload['cancellation_activity_wait'] ?? null; + if (! is_array($wait) || ($wait['root_request_id'] ?? null) !== $context->rootRequestId + || ($wait['cleanup_deadline_at'] ?? null) !== $context->deadline()->toISOString() + || ! is_array($wait['execution_ids'] ?? null) || $wait['execution_ids'] === [] + || ! in_array($receipt->payload['activity_execution_id'] ?? null, $wait['execution_ids'], true)) { + return null; + } + foreach ($wait['execution_ids'] as $executionId) { + if (! is_string($executionId) || ! ActivityCancellationCompletion::resolved($run, $executionId)) { + return null; + } + } + $task = WorkflowTask::query()->create([ + 'workflow_run_id' => $run->id, + 'namespace' => $run->namespace, + 'task_type' => TaskType::Workflow->value, + 'status' => TaskStatus::Ready->value, + 'available_at' => now(), + 'payload' => [ + 'resume_source_kind' => 'activity_cancellation_acknowledged', + 'resume_source_id' => $receipt->id, + 'workflow_command_id' => $context->requestId, + ], + 'connection' => $run->connection, + 'queue' => $run->queue, + 'compatibility' => $run->compatibility, + ]); + app(HistoryProjectionRole::class)->projectRun( + $run->fresh(['instance', 'tasks', 'activityExecutions', 'timers', 'failures', 'historyEvents']) + ); + DB::afterCommit(static fn () => TaskDispatcher::dispatch($task)); + return $task; + } +} diff --git a/src/V2/Support/ActivityOutcomeRecorder.php b/src/V2/Support/ActivityOutcomeRecorder.php index caa5668ee..d5014bc93 100644 --- a/src/V2/Support/ActivityOutcomeRecorder.php +++ b/src/V2/Support/ActivityOutcomeRecorder.php @@ -76,7 +76,8 @@ public static function record( ->lockForUpdate() ->findOrFail($lockedExecution->workflow_run_id); - if (in_array($run->status, [RunStatus::Cancelled, RunStatus::Terminated], true)) { + $abandoned = ActivityAbandonment::allows($run, $lockedExecution); + if (in_array($run->status, [RunStatus::Cancelled, RunStatus::Terminated], true) && ! $abandoned) { $reason = $run->status === RunStatus::Terminated ? 'run_terminated' : 'run_cancelled'; @@ -113,6 +114,10 @@ public static function record( return self::ignored('stale_attempt'); } + if ($abandoned && ActivityTimeoutEnforcer::hasExpiredDeadline($lockedExecution, now())) { + return self::ignored('activity_deadline_elapsed'); + } + $runCodec = is_string($run->payload_codec) && $run->payload_codec !== '' ? $run->payload_codec : null; @@ -230,6 +235,10 @@ public static function record( self::closeAttempt($attemptId, ActivityAttemptStatus::Failed); + // Preserve the callback owner's completed failure as a history + // fact. A timeout-created retry has no such stop evidence. + $closedAttempt = ActivityAttempt::query()->findOrFail($attemptId); + $lockedExecution->forceFill([ 'status' => ActivityStatus::Pending, 'exception' => self::serializeWithCodec($exceptionPayload, null, $runCodec)['blob'], @@ -278,6 +287,14 @@ public static function record( 'max_attempts' => $maxAttempts === PHP_INT_MAX ? null : $maxAttempts, 'retry_policy' => $lockedExecution->retry_policy, 'exception_type' => $exceptionPayload['type'] ?? null, + 'activity_attempt' => [ + 'id' => $closedAttempt->id, + 'activity_execution_id' => $closedAttempt->activity_execution_id, + 'task_id' => $closedAttempt->workflow_task_id, + 'status' => $closedAttempt->status->value, + 'lease_owner' => $closedAttempt->lease_owner, + 'closed_at' => $closedAttempt->closed_at?->toJSON(), + ], 'exception_class' => $exceptionPayload['class'] ?? get_class($throwable), 'message' => $exceptionPayload['message'] ?? $throwable->getMessage(), 'code' => $throwable->getCode(), @@ -387,6 +404,11 @@ public static function record( return self::recorded(null); } + if ($abandoned && $run->status->isTerminal()) { + self::projectRun($run->fresh(['instance', 'tasks', 'activityExecutions', 'failures'])); + return self::recorded(null); + } + // A standalone-activity host run has no workflow code to resume: // the activity execution IS the work, so close the host run with // the activity's outcome instead of scheduling a workflow-task diff --git a/src/V2/Support/ActivityRowLockOrder.php b/src/V2/Support/ActivityRowLockOrder.php index cff2fa5ef..d699bf80a 100644 --- a/src/V2/Support/ActivityRowLockOrder.php +++ b/src/V2/Support/ActivityRowLockOrder.php @@ -24,12 +24,12 @@ final class ActivityRowLockOrder * snapshot_attempt_id: string|null * } */ - public static function lockForExecution(string $executionId): array + public static function lockForExecution(string $executionId, bool $includeClosedAttempt = false): array { /** @var ActivityExecution|null $snapshot */ $snapshot = ActivityExecution::query()->find($executionId); $snapshotAttemptId = $snapshot instanceof ActivityExecution - ? self::runningAttemptId($snapshot) + ? ($includeClosedAttempt ? $snapshot->current_attempt_id : self::runningAttemptId($snapshot)) : null; /** @var ActivityAttempt|null $attempt */ diff --git a/src/V2/Support/ActivitySnapshot.php b/src/V2/Support/ActivitySnapshot.php index c90b5b92b..a93e0f106 100644 --- a/src/V2/Support/ActivitySnapshot.php +++ b/src/V2/Support/ActivitySnapshot.php @@ -6,6 +6,7 @@ use Carbon\CarbonInterface; use Workflow\V2\Enums\ActivityStatus; +use Workflow\V2\Enums\CancellationPolicy; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Models\ActivityExecution; use Workflow\V2\Models\WorkflowHistoryEvent; @@ -34,6 +35,11 @@ public static function fromExecution(ActivityExecution $execution): array 'payload_codec' => self::stringValue($execution->payload_codec), 'attempt_count' => self::executionAttemptCount($execution), 'retry_policy' => self::arrayValue($execution->retry_policy), + 'cancellation_policy' => self::stringValue($execution->activity_options['cancellation_policy'] ?? null), + 'cancellation_scope_id' => self::stringValue($execution->activity_options['cancellation_scope_id'] ?? null), + 'schedule_to_close_deadline_at' => ($execution->activity_options['cancellation_policy'] ?? null) + === CancellationPolicy::Abandon->value + ? self::timestamp($execution->schedule_to_close_deadline_at) : null, 'connection' => $execution->connection, 'queue' => $execution->queue, 'last_heartbeat_at' => self::timestamp($execution->last_heartbeat_at), @@ -86,6 +92,7 @@ public static function fromEvent(WorkflowHistoryEvent $event): ?array 'sequence' => self::intValue($payload['sequence'] ?? null), 'type' => self::stringValue($payload['activity_type'] ?? null), 'class' => self::stringValue($payload['activity_class'] ?? null), + 'attempt_count' => self::intValue($payload['attempt_number'] ?? null), 'execution_mode' => self::stringValue($payload['execution_mode'] ?? null), 'local_activity' => ($payload['execution_mode'] ?? null) === LocalActivityRuntime::EXECUTION_MODE || ($payload['local_activity'] ?? null) === true, @@ -96,6 +103,10 @@ public static function fromEvent(WorkflowHistoryEvent $event): ?array 'parallel_group_index' => self::intValue($payload['parallel_group_index'] ?? null), 'parallel_group_path' => self::parallelGroupPath($payload), 'retry_policy' => self::arrayValue($payload['retry_policy'] ?? null), + 'cancellation_policy' => self::stringValue($payload['cancellation_policy'] ?? null), + ...(isset($payload['cancellation_scope_id']) ? [ + 'cancellation_scope_id' => self::stringValue($payload['cancellation_scope_id']), + ] : []), 'result' => self::payloadValue($payload['result'] ?? null), 'reused_from_run_id' => self::stringValue($payload['reused_from_run_id'] ?? null), 'reused_activity_execution_id' => self::stringValue($payload['reused_activity_execution_id'] ?? null), @@ -155,8 +166,13 @@ private static function sanitizeSnapshot(array $snapshot): array 'type' => self::stringValue($snapshot['type'] ?? null), 'class' => self::stringValue($snapshot['class'] ?? null), 'execution_mode' => self::stringValue($snapshot['execution_mode'] ?? null), - 'local_activity' => ($snapshot['execution_mode'] ?? null) === LocalActivityRuntime::EXECUTION_MODE - || ($snapshot['local_activity'] ?? null) === true, + 'local_activity' => array_key_exists('execution_mode', $snapshot) || array_key_exists( + 'local_activity', + $snapshot + ) + ? (($snapshot['execution_mode'] ?? null) === LocalActivityRuntime::EXECUTION_MODE + || ($snapshot['local_activity'] ?? null) === true) + : null, 'parallel_group_kind' => self::stringValue($snapshot['parallel_group_kind'] ?? null), 'parallel_group_id' => self::stringValue($snapshot['parallel_group_id'] ?? null), 'parallel_group_base_sequence' => self::intValue($snapshot['parallel_group_base_sequence'] ?? null), @@ -168,6 +184,9 @@ private static function sanitizeSnapshot(array $snapshot): array 'payload_codec' => self::stringValue($snapshot['payload_codec'] ?? null), 'attempt_count' => self::intValue($snapshot['attempt_count'] ?? null), 'retry_policy' => self::arrayValue($snapshot['retry_policy'] ?? null), + 'cancellation_policy' => self::stringValue($snapshot['cancellation_policy'] ?? null), + 'cancellation_scope_id' => self::stringValue($snapshot['cancellation_scope_id'] ?? null), + 'schedule_to_close_deadline_at' => self::stringValue($snapshot['schedule_to_close_deadline_at'] ?? null), 'connection' => self::stringValue($snapshot['connection'] ?? null), 'queue' => self::stringValue($snapshot['queue'] ?? null), 'last_heartbeat_at' => self::stringValue($snapshot['last_heartbeat_at'] ?? null), @@ -208,7 +227,12 @@ private static function eventAttemptCount( array $snapshot, array $taskSnapshot, ): int { - $taskAttemptCount = in_array($eventType, [ + // A local callback attempt and its hosting workflow claim have + // independent retry counters. The workflow counter is not an + // activity attempt, including on a repaired workflow claim. + $local = ($snapshot['local_activity'] ?? null) === true + || ($snapshot['execution_mode'] ?? null) === LocalActivityRuntime::EXECUTION_MODE; + $taskAttemptCount = ! $local && in_array($eventType, [ HistoryEventType::ActivityStarted, HistoryEventType::ActivityHeartbeatRecorded, HistoryEventType::ActivityRetryScheduled, diff --git a/src/V2/Support/ActivityTimeoutEnforcer.php b/src/V2/Support/ActivityTimeoutEnforcer.php index ac24834c7..2ff536e49 100644 --- a/src/V2/Support/ActivityTimeoutEnforcer.php +++ b/src/V2/Support/ActivityTimeoutEnforcer.php @@ -118,7 +118,7 @@ public static function enforce(string $executionId): array ->lockForUpdate() ->findOrFail($execution->workflow_run_id); - if ($run->status->isTerminal()) { + if ($run->status->isTerminal() && ! ActivityAbandonment::allows($run, $execution)) { return self::skipped('run_already_terminal'); } @@ -421,6 +421,15 @@ private static function recordTerminalTimeout( $message, ); + if ($run->status->isTerminal() && ActivityAbandonment::allows($run, $execution)) { + self::projectRun($run->fresh(['instance', 'tasks', 'activityExecutions', 'failures'])); + return [ + 'enforced' => true, + 'reason' => null, + 'next_task' => null, + ]; + } + // A standalone-activity host run has no workflow code to resume: // close the host run as Failed instead of scheduling a workflow-task // resume row that no worker will be able to drive. diff --git a/src/V2/Support/CancellationCascadeView.php b/src/V2/Support/CancellationCascadeView.php new file mode 100644 index 000000000..cbaca9b05 --- /dev/null +++ b/src/V2/Support/CancellationCascadeView.php @@ -0,0 +1,624 @@ +> + */ + private array $findings = []; + + private bool $truncated = false; + + private int $historyCount = 0; + + /** + * @var array + */ + private array $visibleRuns = []; + + private function __construct( + private readonly WorkflowRun $selected + ) { + } + + /** + * Resolve the exact selected run, including historical runs. Every related + * run and instance is scoped before its history or metadata is inspected. + * + * @return array|null + */ + public static function forRun(WorkflowRun $selected): ?array + { + return (new self($selected))->inspect(); + } + + /** + * @return array|null + */ + private function inspect(): ?array + { + $selected = $this->visibleRun($this->selected->id); + if ($selected === null) { + return null; + } + [$selectedContext] = $this->request($selected); + if ($selectedContext === null && $selected->cancellation_request_command_id === null + && $this->findings === []) { + return null; + } + + $root = $selected; + $origin = $selectedContext; + if ($selectedContext !== null && $selectedContext->rootWorkflowRunId !== $selected->id) { + $visibleRoot = $this->visibleRun($selectedContext->rootWorkflowRunId); + if ($visibleRoot === null) { + $this->incomplete('root_unavailable', $selected->id); + $origin = null; + } else { + $root = $visibleRoot; + [$origin] = $this->request($root); + if ($origin === null || ! $this->sameOrigin($origin, $selectedContext)) { + $this->incomplete('root_request_mismatch', $selected->id); + $origin = null; + } + } + } + + $pending = [$root]; + $seen = []; + $nodes = []; + $edges = []; + while ($pending !== [] && count($nodes) < self::RUN_LIMIT) { + $run = array_shift($pending); + if (isset($seen[$run->id])) { + continue; + } + $seen[$run->id] = true; + $nodes[] = $this->node($run, $origin); + foreach ($this->references($run, self::EDGE_LIMIT - count($edges)) as $reference) { + $childId = $reference['target_run_id']; + $child = is_string($childId) ? $this->visibleRun($childId) : null; + $edge = [ + 'parent_run_id' => $run->id, + 'child_run_id' => $child?->id, + 'kind' => $reference['kind'], + 'reference_id' => $reference['id'], + 'reference_source' => $reference['source'], + 'reference_state' => $childId === null ? 'pending' : ($child === null ? 'unavailable' : 'resolved'), + ]; + $edges[] = $edge; + if ($childId !== null && $child === null) { + // Do not disclose an inaccessible target's ID, type or request. + $this->incomplete('related_run_unavailable', $run->id); + } elseif ($child !== null && ! isset($seen[$child->id])) { + $pending[] = $child; + } + } + } + if ($pending !== []) { + $this->truncate('run_limit', $root->id); + } + if (! isset($seen[$selected->id])) { + $this->incomplete('selected_run_not_reachable', $selected->id); + if (count($nodes) < self::RUN_LIMIT) { + $nodes[] = $this->node($selected, $origin); + } + } + + return [ + 'schema' => 'durable-workflow.cancellation-cascade/v1', + 'selected_run_id' => $selected->id, + 'root' => $origin === null ? null : $this->metadata($origin), + 'runs' => $nodes, + 'edges' => $edges, + 'inspection_complete' => $this->findings === [], + 'truncated' => $this->truncated, + 'findings' => $this->findings, + 'limits' => [ + 'runs' => self::RUN_LIMIT, + 'edges' => self::EDGE_LIMIT, + 'history_events_per_run' => self::HISTORY_LIMIT, + 'history_events_total' => self::TOTAL_HISTORY_LIMIT, + 'request_text_bytes' => self::REQUEST_TEXT_BYTES, + 'scope_origin_addresses' => self::EDGE_LIMIT, + ], + ]; + } + + private function visibleRun(string $id): ?WorkflowRun + { + if (array_key_exists($id, $this->visibleRuns)) { + return $this->visibleRuns[$id]; + } + return $this->visibleRuns[$id] = $this->selected->newQuery()->whereKey($id) + ->where('namespace', $this->selected->namespace) + ->whereHas( + 'instance', + fn (Builder $query): Builder => $query->where('namespace', $this->selected->namespace) + ) + ->first(); + } + + /** + * @return array{CancellationContext|null, WorkflowHistoryEvent|null} + */ + private function request(WorkflowRun $run): array + { + $query = $run->historyEvents() + ->orderBy('sequence'); + if (is_string($run->cancellation_request_command_id)) { + $query->where('event_type', HistoryEventType::CooperativeCancellationRequested->value) + ->where('workflow_command_id', $run->cancellation_request_command_id); + } else { + $query->whereIn('event_type', [ + HistoryEventType::CooperativeCancellationRequested->value, + HistoryEventType::ParentCloseCancellationRequested->value, + ]); + } + $events = $query->limit(2) + ->get(); + $event = $events->first(); + if ($events->count() > 1) { + $this->incomplete('request_history_conflict', $run->id); + } + if ($event === null) { + if ($run->cancellation_request_command_id !== null) { + $this->incomplete('request_history_unavailable', $run->id); + } + return [null, null]; + } + $snapshot = $event->payload['cancellation'] ?? null; + if (! is_array($snapshot) || ! is_array($snapshot['lineage'] ?? null) + || count($snapshot['lineage']) > self::RUN_LIMIT) { + $this->incomplete('request_context_unavailable', $run->id); + return [null, $event]; + } + try { + $context = CancellationContext::fromArray($snapshot); + if ($event->event_type === HistoryEventType::ParentCloseCancellationRequested) { + $context = ParentCloseCancellation::context($run); + if ($context === null) { + throw new LogicException('Parent-close cancellation origin is unavailable.'); + } + } elseif ($event->workflow_command_id !== $context->requestId + || ($event->payload['workflow_command_id'] ?? null) !== $context->requestId) { + throw new InvalidArgumentException('Cancellation history request mismatch.'); + } + $target = $context->lineage[count($context->lineage) - 1]; + if ($target['workflow_run_id'] !== $run->id + || $target['workflow_instance_id'] !== $run->workflow_instance_id + || ($run->cancellation_request_command_id !== null + && $context->requestId !== $run->cancellation_request_command_id)) { + throw new InvalidArgumentException('Cancellation target mismatch.'); + } + } catch (LogicException) { + $this->incomplete('request_context_invalid', $run->id); + return [null, $event]; + } + if ($event->event_type === HistoryEventType::CooperativeCancellationRequested + && ($run->cancellation_request_command_id !== $context->requestId + || $run->cancellation_deadline_at === null + || ! $run->cancellation_deadline_at->equalTo($context->deadline()))) { + $this->incomplete('request_projection_mismatch', $run->id); + } + return [$context, $event]; + } + + /** + * @return array + */ + private function node(WorkflowRun $run, ?CancellationContext $origin): array + { + [$context, $request] = $this->request($run); + if ($context !== null && $this->visibleRun($context->rootWorkflowRunId) === null) { + $this->incomplete('request_root_unavailable', $run->id); + $context = null; + } + $limit = min(self::HISTORY_LIMIT, self::TOTAL_HISTORY_LIMIT - $this->historyCount); + $events = $run->historyEvents() + ->whereIn('event_type', [ + HistoryEventType::CooperativeCancellationRequested->value, + HistoryEventType::CooperativeCancellationDelivered->value, + HistoryEventType::ActivityCancelled->value, + HistoryEventType::ActivityCancellationAcknowledged->value, + HistoryEventType::ActivityScheduled->value, + HistoryEventType::ActivityRetryScheduled->value, + HistoryEventType::TimerScheduled->value, + HistoryEventType::ChildCancellationRequested->value, + HistoryEventType::ChildCancellationResolved->value, + HistoryEventType::WorkflowCancelled->value, + HistoryEventType::WorkflowTerminated->value, + HistoryEventType::WorkflowFailed->value, + HistoryEventType::WorkflowCompleted->value, + HistoryEventType::WorkflowTimedOut->value, + ])->orderByDesc('sequence') + ->limit($limit + 1) + ->get(); + if ($events->count() > $limit) { + $this->truncate('history_limit', $run->id); + } + $events = new Collection($events->take($limit)->reverse()->values()->all()); + $this->historyCount += $events->count(); + if ($request !== null && ! $events->contains('id', $request->id)) { + $events->prepend($request); + } + $run->setRelation('historyEvents', $events); + $deliveries = $events->filter(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::CooperativeCancellationDelivered); + $delivery = $deliveries->first(); + if ($delivery === null && $run->cancellation_delivery_sequence !== null) { + $this->incomplete('delivery_history_unavailable', $run->id); + } + if ($delivery !== null && ($deliveries->count() !== 1 || $context === null + || $delivery->workflow_command_id !== $context->requestId + || ($delivery->payload['workflow_command_id'] ?? null) !== $context->requestId + || ! is_int($delivery->payload['sequence'] ?? null) + || $delivery->payload['sequence'] !== $run->cancellation_delivery_sequence)) { + $this->incomplete('delivery_history_invalid', $run->id); + $delivery = null; + } + $terminals = $events->filter(static fn (WorkflowHistoryEvent $event): bool => in_array($event->event_type, [ + HistoryEventType::WorkflowCancelled, HistoryEventType::WorkflowTerminated, + HistoryEventType::WorkflowFailed, HistoryEventType::WorkflowCompleted, HistoryEventType::WorkflowTimedOut, + ], true)); + $terminal = $terminals->last(); + if ($terminals->count() > 1 || ($run->status->isTerminal() && $terminal === null)) { + $this->incomplete('terminal_history_unavailable', $run->id); + } + $cleanup = $terminal?->payload['cancellation_cleanup'] ?? null; + $cleanup = is_array($cleanup) && $context !== null ? $cleanup : null; + if ($terminal?->event_type === HistoryEventType::WorkflowCancelled && $context !== null + && (! is_array($cleanup) || ! $this->validCleanup($run, $context, $cleanup, $delivery))) { + $this->incomplete('cleanup_outcome_unavailable', $run->id); + $cleanup = null; + } + if ($terminal !== null && $run->status->value !== match ($terminal->event_type) { + HistoryEventType::WorkflowCancelled => 'cancelled', + HistoryEventType::WorkflowTerminated => 'terminated', + HistoryEventType::WorkflowFailed => 'failed', + HistoryEventType::WorkflowTimedOut => 'failed', + HistoryEventType::WorkflowCompleted => 'completed', + default => null, + }) { + $this->incomplete('terminal_projection_mismatch', $run->id); + } + + $stops = []; + $recovery = []; + $propagation = []; + foreach ($events as $event) { + if (in_array($event->event_type, [HistoryEventType::ChildCancellationRequested, + HistoryEventType::ChildCancellationResolved], true) && $context !== null) { + if ($event->workflow_command_id !== $context->requestId + || ($event->payload['parent_request_id'] ?? null) !== $context->requestId + || ($event->payload['root_request_id'] ?? null) !== $context->rootRequestId + || ($event->payload['cleanup_deadline_at'] ?? null) !== $context->deadline()->toISOString()) { + $this->incomplete('propagation_history_invalid', $run->id); + continue; + } + $targetId = $event->payload['child_workflow_run_id'] ?? null; + $target = is_string($targetId) ? $this->visibleRun($targetId) : null; + if ($target === null) { + $this->incomplete('related_run_unavailable', $run->id); + } + $propagation[] = [ + 'history_event_id' => $event->id, + 'event_type' => $event->event_type->value, + 'child_run_id' => $target?->id, + 'policy' => $event->payload['policy'] ?? null, + 'request_outcome' => $event->payload['request_outcome'] ?? null, + 'rejection_reason' => $event->payload['rejection_reason'] ?? null, + 'child_terminal_history_event_id' => $target === null ? null + : ($event->payload['child_terminal_history_event_id'] ?? null), + ]; + } + if ($event->event_type === HistoryEventType::ActivityCancelled && $context !== null + && $event->workflow_command_id === $context->requestId) { + $receipt = ActivityCancellationCompletion::stopReceipt($run, $event); + $stops[] = [ + 'activity_execution_id' => $event->payload['activity_execution_id'] ?? null, + 'activity_attempt_id' => $event->payload['activity_attempt_id'] ?? null, + 'activity_type' => $event->payload['activity_type'] ?? null, + 'execution_mode' => ($event->payload['local_activity'] ?? false) === true ? 'local' : 'remote', + 'fence_history_event_id' => $event->id, + 'callback_state' => $receipt === null ? 'unknown' : 'reported_stopped', + 'stop_history_event_id' => $receipt?->id, + 'evidence_source' => $receipt?->payload['evidence_source'] ?? null, + 'acknowledged_at' => $receipt?->payload['acknowledged_at'] ?? null, + 'received_after_deadline' => $receipt?->payload['received_after_deadline'] ?? null, + ]; + } + if ($event->event_type === HistoryEventType::ActivityRetryScheduled + && is_array($event->payload['local_recovery'] ?? null)) { + $recovery[] = [ + 'history_event_id' => $event->id, + 'activity_execution_id' => $event->payload['activity_execution_id'] ?? null, + 'recorded_at' => $event->recorded_at?->toISOString(), + 'attempt' => array_intersect_key($event->payload['local_recovery'], array_flip([ + 'original_workflow_task_id', 'original_workflow_task_attempt', + 'original_lease_owner', 'original_lease_expires_at', + 'workflow_task_id', 'workflow_task_attempt', 'lease_owner', 'callback_stop_state', + ])), + ]; + } + } + + return [ + 'run_id' => $run->id, + 'workflow_id' => $run->workflow_instance_id, + 'workflow_type' => $run->workflow_type ?? $run->workflow_class, + 'projected_status' => $run->status->value, + 'lifecycle' => $this->lifecycle($run, $context, $events, $delivery, $terminal, $cleanup), + 'terminal_event_type' => $terminal?->event_type->value, + 'terminal_history_event_id' => $terminal?->id, + 'request' => $context === null ? null : $this->metadata($context), + 'same_root_budget' => $context !== null && $origin !== null && $this->sameOrigin($origin, $context), + 'delivery' => $delivery === null ? null : [ + 'history_event_id' => $delivery->id, + 'sequence' => $delivery->payload['sequence'], + 'sequence_span' => $delivery->payload['sequence_span'] ?? 1, + 'call_kind' => $delivery->payload['call_kind'] ?? null, + 'recorded_at' => $delivery->recorded_at?->toISOString(), + ], + 'cleanup' => is_array($cleanup) ? array_intersect_key($cleanup, array_flip([ + 'request_id', 'outcome', 'cleanup_deadline_at', 'finished_at', + 'delivery_history_event_id', 'delivery_sequence', + ])) : null, + 'activity_stops' => $stops, + 'cleanup_recovery' => $recovery, + 'child_propagation' => $propagation, + ]; + } + + /** + * @return list + */ + private function references(WorkflowRun $run, int $remaining): array + { + $calls = WorkflowChildCall::query()->where('parent_workflow_run_id', $run->id) + ->orderBy('sequence') + ->limit($remaining + 1) + ->get(); + $references = []; + foreach ($calls->take($remaining) as $call) { + $references[] = [ + 'id' => $call->id, + 'source' => 'child_call', + 'kind' => 'child_workflow', + 'target_run_id' => $call->resolved_child_run_id, + ]; + } + if ($calls->count() > $remaining) { + $this->truncate('edge_limit', $run->id); + } + $remaining -= count($references); + $links = WorkflowLink::query()->where('parent_workflow_run_id', $run->id) + ->whereIn('link_type', ['child_workflow', 'continue_as_new']) + ->whereNotIn('child_workflow_run_id', array_filter(array_column($references, 'target_run_id'))) + ->orderBy('sequence') + ->orderBy('id') + ->limit($remaining + 1) + ->get(); + foreach ($links->take($remaining) as $link) { + $references[] = [ + 'id' => $link->id, + 'source' => 'workflow_link', + 'kind' => $link->link_type, + 'target_run_id' => $link->child_workflow_run_id, + ]; + } + if ($links->count() > $remaining) { + $this->truncate('edge_limit', $run->id); + } + return $references; + } + + private function sameOrigin(CancellationContext $root, CancellationContext $child): bool + { + $child = $child->scopeOrigin?->rootContext ?? $child; + return $root->rootRequestId === $child->rootRequestId + && $root->rootWorkflowRunId === $child->rootWorkflowRunId + && $root->rootWorkflowInstanceId === $child->rootWorkflowInstanceId + && $root->requestedAt() + ->equalTo($child->requestedAt()) + && $root->deadline() + ->equalTo($child->deadline()) + && $root->reason === $child->reason && $root->requester === $child->requester + && $root->source === $child->source; + } + + /** + * @return array + */ + private function metadata(CancellationContext $context): array + { + // Edges explain lineage without exposing unresolvable snapshot targets. + $metadata = array_diff_key($context->toArray(), [ + 'lineage' => true, + ]); + if ($context->scopeOrigin !== null) { + $metadata['scope_origin'] = null; + $lineage = $context->scopeOrigin->lineage; + $available = count($lineage) <= self::EDGE_LIMIT; + if (! $available) { + $this->truncate( + 'scope_origin_limit', + $context->lineage[count($context->lineage) - 1]['workflow_run_id'] + ); + } else { + foreach ($lineage as $entry) { + $run = $this->visibleRun($entry['workflow_run_id']); + if ($run === null || $run->workflow_instance_id !== $entry['workflow_instance_id']) { + $available = false; + $this->incomplete( + 'scope_origin_unavailable', + $context->lineage[count($context->lineage) - 1]['workflow_run_id'] + ); + break; + } + } + } + if ($available) { + $metadata['scope_origin'] = [ + 'schema' => $context->scopeOrigin::SCHEMA, + 'root_context' => $this->metadata($context->scopeOrigin->rootContext), + 'lineage' => $lineage, + ]; + } else { + $metadata['parent_request_id'] = null; + } + } + foreach ($metadata as $key => $value) { + if (is_string($value) && strlen($value) > self::REQUEST_TEXT_BYTES) { + $metadata[$key] = mb_strcut($value, 0, self::REQUEST_TEXT_BYTES, 'UTF-8'); + $this->truncate( + 'request_text_limit', + $context->lineage[count($context->lineage) - 1]['workflow_run_id'] + ); + } + } + foreach ($context->requester as $key => $value) { + if (strlen($value) > self::REQUEST_TEXT_BYTES) { + $metadata['requester'][$key] = mb_strcut($value, 0, self::REQUEST_TEXT_BYTES, 'UTF-8'); + $this->truncate( + 'request_text_limit', + $context->lineage[count($context->lineage) - 1]['workflow_run_id'] + ); + } + } + return $metadata; + } + + /** + * @param array $cleanup + */ + private function validCleanup( + WorkflowRun $run, + CancellationContext $context, + array $cleanup, + ?WorkflowHistoryEvent $delivery, + ): bool { + if (($cleanup['request_id'] ?? null) !== $context->requestId + || ($cleanup['cleanup_deadline_at'] ?? null) !== $context->deadline()->toISOString() + || $run->closed_at === null + || ($cleanup['finished_at'] ?? null) !== $run->closed_at->toISOString()) { + return false; + } + $expired = $run->closed_at->gte($context->deadline()); + return match ($cleanup['outcome'] ?? null) { + 'deadline_expired' => $expired, + 'completed' => ! $expired && $delivery !== null + && ($cleanup['delivery_history_event_id'] ?? null) === $delivery->id + && ($cleanup['delivery_sequence'] ?? null) === $delivery->payload['sequence'], + 'not_delivered' => ! $expired && $delivery === null && $run->cancellation_delivery_sequence === null, + 'unavailable' => ! $expired, + default => false, + }; + } + + /** + * @param Collection $events + * @param array|null $cleanup + */ + private function lifecycle( + WorkflowRun $run, + ?CancellationContext $context, + Collection $events, + ?WorkflowHistoryEvent $delivery, + ?WorkflowHistoryEvent $terminal, + ?array $cleanup, + ): ?string { + if ($terminal !== null) { + return match ($terminal->event_type) { + HistoryEventType::WorkflowCancelled => ($cleanup['outcome'] ?? null) === 'deadline_expired' + ? 'deadline_expired' : 'cancelled', + HistoryEventType::WorkflowTerminated => 'terminated', + HistoryEventType::WorkflowFailed => 'failed', + HistoryEventType::WorkflowTimedOut => 'timed_out', + HistoryEventType::WorkflowCompleted => 'completed', + default => 'unknown', + }; + } + if ($run->status->isTerminal()) { + return 'unknown'; + } + if ($delivery !== null) { + $lastSequence = $delivery->payload['sequence'] + ($delivery->payload['sequence_span'] ?? 1) - 1; + $started = $events->contains(static fn (WorkflowHistoryEvent $event): bool => + in_array( + $event->event_type, + [HistoryEventType::ActivityScheduled, HistoryEventType::TimerScheduled], + true + ) + && $event->sequence > $delivery->sequence + && is_int($event->payload['sequence'] ?? null) && $event->payload['sequence'] > $lastSequence); + return $started ? 'cleaning_up' : 'delivered'; + } + return $context === null ? null : 'requested'; + } + + private function truncate(string $code, string $runId): void + { + $this->truncated = true; + $this->incomplete($code, $runId); + } + + private function incomplete(string $code, string $runId): void + { + $message = match ($code) { + 'run_limit' => 'Some related runs are omitted because this view reached its run limit.', + 'edge_limit' => 'Some relations are omitted because this view reached its relation limit.', + 'history_limit' => 'Some history is omitted. Open the run history for additional evidence.', + 'request_text_limit' => 'Some request text is omitted because it exceeds the inspection text limit.', + 'scope_origin_limit' => 'The originating scope path exceeds this view\'s inspection limit. Its details are omitted.', + 'scope_origin_unavailable' => 'An originating scope run is unavailable in this namespace. Its path is omitted.', + 'root_unavailable', 'request_root_unavailable' => 'The root run is unavailable. Its original cascade budget cannot be verified.', + 'related_run_unavailable' => 'A related run is unavailable in this namespace. Its details are omitted.', + 'selected_run_not_reachable' => 'The selected run has no retained path from the root. It is shown separately.', + 'request_history_unavailable' => 'The recorded cancellation request is missing. Its identity and deadline cannot be verified.', + 'request_context_unavailable' => 'The recorded request has no usable cancellation context.', + 'request_projection_mismatch' => 'Saved run state disagrees with the original cancellation request or budget. The original request remains in history.', + 'request_history_conflict', 'request_context_invalid', 'root_request_mismatch' => 'The recorded request has inconsistent cancellation metadata.', + 'delivery_history_invalid', 'delivery_history_unavailable' => 'The original cancellation delivery cannot be verified.', + 'terminal_history_unavailable' => 'The run has missing or conflicting terminal history.', + 'terminal_projection_mismatch' => 'Run status disagrees with its terminal history. Inspect the recorded terminal event before recovery.', + 'cleanup_outcome_unavailable' => 'Cleanup completion cannot be verified from the recorded cancellation outcome.', + 'propagation_history_invalid' => 'Child propagation does not match the original cancellation request and budget.', + default => 'Some cancellation evidence cannot be verified.', + }; + $finding = [ + 'code' => $code, + 'run_id' => $runId, + 'message' => $message, + ]; + if (! in_array($finding, $this->findings, true)) { + $this->findings[] = $finding; + } + } +} diff --git a/src/V2/Support/CancellationCleanupLease.php b/src/V2/Support/CancellationCleanupLease.php new file mode 100644 index 000000000..7877e62bb --- /dev/null +++ b/src/V2/Support/CancellationCleanupLease.php @@ -0,0 +1,73 @@ +cancellation_request_command_id !== null + ? $run->cancellation_deadline_at : null; + } + + public static function expiresAt(?WorkflowRun $run): CarbonInterface + { + $deadline = self::deadline($run); + + if ($deadline !== null) { + return self::forDeadline($deadline); + } + + return self::forScopes($run); + } + + public static function forScopes(?WorkflowRun $run): CarbonInterface + { + $deadline = self::deadline($run); + // A scoped budget bounds callback authority, not the lifetime of its + // unaffected siblings. Keep shared ownership recoverable while that + // original budget is live without ending the whole claim at its deadline. + if ($run?->cancellation_scope_recovery_until !== null + && now() + ->lt($run->cancellation_scope_recovery_until)) { + return $deadline === null ? self::renewableExpiry() : self::forDeadline($deadline); + } + + // Synchronous embedded callbacks cannot renew during execution. Their + // configured ownership interval still ends at the original run budget. + $expiry = WorkflowTaskLease::expiresAt(); + + return $deadline !== null && $deadline->lt($expiry) ? $deadline->copy() : $expiry; + } + + public static function forDeadline(CarbonInterface $deadline): CarbonInterface + { + $expiry = self::renewableExpiry(); + + return $deadline->lt($expiry) ? $deadline->copy() : $expiry; + } + + /** + * A renewable ownership interval, never another cancellation budget. + */ + public static function renewableExpiry(): CarbonInterface + { + $now = now(); + $expiry = WorkflowTaskLease::expiresAt($now); + $maximum = $now->copy() + ->addSeconds(self::MAXIMUM_SECONDS); + if ($maximum->lt($expiry)) { + $expiry = $maximum; + } + + return $expiry; + } +} diff --git a/src/V2/Support/CancellationDeliveryState.php b/src/V2/Support/CancellationDeliveryState.php new file mode 100644 index 000000000..878b8a590 --- /dev/null +++ b/src/V2/Support/CancellationDeliveryState.php @@ -0,0 +1,13 @@ +getConnection()->transactionLevel() !== 0) { + throw new LogicException('cancellation_scope_preparation_requires_own_transaction'); + } + return self::writeBoundary( + $run, + $task, + $scopeId, + $requestId, + $sequence, + $callKind, + $protocolVersion, + $sequenceSpan, + $operationSequence, + $operationSequenceSpan, + true, + $renewWorkflowLease + ); + } + + public static function record( + WorkflowRun $run, + WorkflowTask $task, + string $scopeId, + string $requestId, + int $sequence, + string $callKind, + string $protocolVersion, + int $sequenceSpan = 1, + ?int $operationSequence = null, + int $operationSequenceSpan = 1, + ): WorkflowHistoryEvent { + return self::writeBoundary( + $run, + $task, + $scopeId, + $requestId, + $sequence, + $callKind, + $protocolVersion, + $sequenceSpan, + $operationSequence, + $operationSequenceSpan, + false + ); + } + + /** + * Cold replay reads the original boundary and clock, even after authority ends. + * The deadline snapshot is an observation, never permission to admit effects. + */ + public static function recorded(WorkflowRun $run, string $scopeId): ?WorkflowHistoryEvent + { + return self::readBoundary($run, $scopeId, false); + } + + /** + * The preparation is replay data, not renewed effect authority. + */ + public static function prepared(WorkflowRun $run, string $scopeId): ?WorkflowHistoryEvent + { + return self::readBoundary($run, $scopeId, true); + } + + /** + * Preserve recorded-command replay while denying new work in a prepared scope. + * @param array|null $cleanupProof + */ + public static function admissionRefusal( + WorkflowRun $run, + string $scopeId, + int $sequence, + ?array $cleanupProof = null + ): ?string { + if ($scopeId === CancellationScopeHistory::ROOT_SCOPE_ID) { + return self::positionReserved( + $run, + $sequence, + $scopeId + ) ? 'operation_cancellation_delivery_reserved' : null; + } + $scopes = CancellationScopeHistory::forRun($run); + $address = $scopeId; + while (isset($scopes[$address])) { + if ($run->historyEvents()->where('event_type', HistoryEventType::CancellationScopeDeliveryPrepared) + ->where('payload->scope_id', $address) + ->exists()) { + $preparation = self::prepared($run, $address); + if (($cleanupProof['scope_id'] ?? null) === $address + && PortableLocalActivityCleanup::snapshot($run, $cleanupProof, $sequence, $scopeId) !== null) { + return self::positionReserved( + $run, + $sequence, + $scopeId + ) ? 'operation_cancellation_delivery_reserved' : null; + } + $recorded = self::replaysOriginalOperation($run, $preparation, $scopeId, $sequence); + return $recorded ? null : 'operation_scope_cancellation_prepared'; + } + if ($scopes[$address]['shield_parent'] || $scopes[$address]['parent_scope_id'] === null) { + break; + } + $address = $scopes[$address]['parent_scope_id']; + } + // A preparation for an unscheduled await owns its exact command position. + // Existing unrelated operations keep running, but a new command cannot + // consume that position while delivery is still waiting on actor receipts. + return self::positionReserved($run, $sequence, $scopeId) ? 'operation_cancellation_delivery_reserved' : null; + } + + /** + * @return list + */ + public static function activityMembers(WorkflowRun $run, string $scopeId): array + { + $run->loadMissing('historyEvents'); + $members = []; + $ids = []; + $sequences = []; + foreach ($run->historyEvents as $event) { + if ($event->event_type !== HistoryEventType::ActivityScheduled) { + continue; + } + $payload = $event->payload; + $activity = $payload['activity'] ?? []; + $nested = $activity['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID; + $flat = $payload['cancellation_scope_id'] ?? $nested; + if ($flat !== $scopeId && $nested !== $scopeId) { + continue; + } + if ($flat !== $scopeId || $nested !== $scopeId) { + throw new LogicException('cancellation_scope_delivery_membership_mismatch'); + } + $id = $payload['activity_execution_id'] ?? null; + $sequence = $payload['sequence'] ?? null; + if (! is_string($id) || $id === '' || ($activity['id'] ?? null) !== $id + || ! is_int($sequence) || $sequence < 1 || isset($ids[$id]) || isset($sequences[$sequence])) { + throw new LogicException('cancellation_scope_activity_history_invalid'); + } + $ids[$id] = true; + $sequences[$sequence] = true; + if (PortableLocalActivityCleanup::isScopedScheduled($run, $event)) { + continue; + } + // Hash the cancellation-relevant scalar facts, without copying application payload bytes. + $members[] = [ + 'sequence' => $sequence, + 'activity_execution_id' => $id, + 'descriptor_hash' => hash('sha256', json_encode([ + $scopeId, $event->id, $activity['cancellation_policy'] ?? 'try_cancel', + $payload['local_activity'] ?? false, $payload['execution_mode'] ?? null, + $activity['schedule_to_close_deadline_at'] ?? null, + ], JSON_THROW_ON_ERROR)), + ]; + } + return $members; + } + + /** + * @return list + */ + public static function normalizeMembers(mixed $members): array + { + if (! is_array($members) || ! array_is_list($members)) { + throw new LogicException('cancellation_scope_preparation_history_invalid'); + } + $normalized = []; + foreach ($members as $member) { + if (! is_array($member) || count($member) !== 3 || ! is_int($member['sequence'] ?? null) + || ! is_string($member['activity_execution_id'] ?? null) + || ! is_string($member['descriptor_hash'] ?? null)) { + throw new LogicException('cancellation_scope_preparation_history_invalid'); + } + $normalized[] = [ + 'sequence' => $member['sequence'], + 'activity_execution_id' => $member['activity_execution_id'], + 'descriptor_hash' => $member['descriptor_hash'], + ]; + } + return $normalized; + } + + private static function positionReserved(WorkflowRun $run, int $sequence, ?string $scopeId = null): bool + { + foreach ($run->historyEvents()->where( + 'event_type', + HistoryEventType::CancellationScopeDeliveryPrepared + )->get() as $event) { + $preparedScope = $event->payload['scope_id'] ?? null; + if (! is_string($preparedScope) || $preparedScope === '') { + throw new LogicException('cancellation_scope_preparation_history_invalid'); + } + $preparation = self::prepared($run, $preparedScope); + if ($preparation === null) { + throw new LogicException('cancellation_scope_preparation_history_invalid'); + } + if ($preparation->payload['sequence'] === $sequence && self::recorded($run, $preparedScope) === null + && ($scopeId === null || ! self::replaysOriginalOperation($run, $preparation, $scopeId, $sequence))) { + return true; + } + } + return false; + } + + private static function replaysOriginalOperation( + WorkflowRun $run, + WorkflowHistoryEvent $preparation, + string $scopeId, + int $sequence + ): bool { + return $run->historyEvents() + ->where('sequence', '<', $preparation->sequence) + ->whereIn('event_type', [HistoryEventType::ActivityScheduled, HistoryEventType::TimerScheduled, + HistoryEventType::ChildWorkflowScheduled, HistoryEventType::SignalWaitOpened, + HistoryEventType::ConditionWaitOpened]) + ->where('payload->sequence', $sequence) + ->get() + ->contains(static function (WorkflowHistoryEvent $row) use ($scopeId): bool { + $payload = $row->payload; + $descriptor = match ($row->event_type) { + HistoryEventType::ActivityScheduled => $payload['activity'] ?? [], + HistoryEventType::TimerScheduled => $payload['timer'] ?? [], + HistoryEventType::ChildWorkflowScheduled => $payload['child_workflow'] ?? [], + default => [], + }; + $membership = array_key_exists('cancellation_scope_id', $payload) + ? $payload['cancellation_scope_id'] + : ($descriptor['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID); + return $membership === $scopeId + && (! array_key_exists('cancellation_scope_id', $descriptor) + || $descriptor['cancellation_scope_id'] === $scopeId); + }); + } + + /** + * Preserve the first acknowledged boundary across response loss and replacement. + * Re-read the authenticated claim and authority under the configured run lock. + */ + private static function writeBoundary( + WorkflowRun $run, + WorkflowTask $task, + string $scopeId, + string $requestId, + int $sequence, + string $callKind, + string $protocolVersion, + int $sequenceSpan = 1, + ?int $operationSequence = null, + int $operationSequenceSpan = 1, + bool $preparing = false, + bool $renewWorkflowLease = true, + ): WorkflowHistoryEvent { + if (! WorkerProtocolVersion::supportsCancellationScopeMembership($protocolVersion)) { + throw new LogicException('cancellation_scope_requires_protocol_1_20'); + } + $invalid = CooperativeCancellationDelivery::validateBoundarySyntax( + $sequence, + $callKind, + $sequenceSpan, + $operationSequence, + $operationSequenceSpan + ); + if (! $run->exists || ! $task->exists || $invalid !== null) { + throw new LogicException($invalid ?? 'invalid_cancellation_scope_delivery'); + } + + $event = ConfiguredV2Models::query('run_model', WorkflowRun::class)->getModel()->getConnection() + ->transaction(static function () use ( + $run, + $task, + $scopeId, + $requestId, + $sequence, + $callKind, + $sequenceSpan, + $operationSequence, + $operationSequenceSpan, + $preparing, + $renewWorkflowLease + ): WorkflowHistoryEvent { + // Match workflow claims, heartbeat and repair: task before run. + /** @var WorkflowTask|null $claim */ + $claim = ConfiguredV2Models::query('task_model', WorkflowTask::class)->lockForUpdate()->find($task->id); + /** @var WorkflowRun $locked */ + $locked = ConfiguredV2Models::query('run_model', WorkflowRun::class)->lockForUpdate()->findOrFail( + $run->id + ); + if (! self::matchesClaim($claim, $locked, $task)) { + throw new LogicException('cancellation_scope_workflow_claim_mismatch'); + } + $context = CancellationScopeRequests::context($locked, $scopeId); + if ($context === null || $requestId !== $context->requestId) { + throw new LogicException('cancellation_scope_request_mismatch'); + } + $authority = CancellationScopeRequests::authority($locked, $scopeId); + if (! $authority['active'] || $authority['deadline_at'] === null) { + throw new LogicException('cancellation_scope_authority_expired'); + } + $existing = $preparing ? self::prepared($locked, $scopeId) : self::recorded($locked, $scopeId); + if ($existing !== null) { + $payload = $existing->payload; + if ($payload['sequence'] !== $sequence || $payload['call_kind'] !== $callKind + || $payload['sequence_span'] !== $sequenceSpan + || $payload['operation_sequence'] !== $operationSequence + || $payload['operation_sequence_span'] !== $operationSequenceSpan) { + throw new LogicException('cancellation_scope_delivery_mismatch'); + } + if ($preparing && self::recorded($locked, $scopeId) === null) { + $request = $locked->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeRequested) + ->where('payload->scope_id', $scopeId) + ->sole(); + $invalid = CooperativeCancellationDelivery::validateCallBoundary( + $locked, + $request, + $sequence, + $callKind, + $sequenceSpan, + $operationSequence, + $operationSequenceSpan + ); + if ($invalid !== null) { + throw new LogicException($invalid); + } + } + self::renewHosting($locked, $claim, $existing, $renewWorkflowLease); + return $existing; + } + if ($preparing && self::positionReserved($locked, $sequence)) { + throw new LogicException('cancellation_scope_command_sequence_reserved'); + } + $request = $locked->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeRequested) + ->where('payload->scope_id', $scopeId) + ->sole(); + $scope = CancellationScopeHistory::forRun($locked)[$scopeId]; + if ($scope['shield_parent'] && $request->payload['parent_scope_id'] !== null) { + throw new LogicException('cancellation_scope_parent_shielded'); + } + $invalid = CooperativeCancellationDelivery::validateCallBoundary( + $locked, + $request, + $sequence, + $callKind, + $sequenceSpan, + $operationSequence, + $operationSequenceSpan + ); + if ($invalid !== null) { + throw new LogicException($invalid); + } + $operationStart = $operationSequence ?? $sequence; + $operationSpan = $operationSequence === null ? $sequenceSpan : $operationSequenceSpan; + if (! CancellationScopeHistory::isRecordedBefore($locked, $scopeId, $operationStart) + || ! self::matchesMembership($locked, $scopeId, $operationStart, $operationSpan)) { + throw new LogicException('cancellation_scope_delivery_membership_mismatch'); + } + $preparation = $preparing ? null : self::prepared($locked, $scopeId); + if (! $preparing && ($preparation === null + || ! self::sameBoundary( + $preparation->payload, + $sequence, + $callKind, + $sequenceSpan, + $operationSequence, + $operationSequenceSpan + ))) { + throw new LogicException($preparation === null + ? 'cancellation_scope_delivery_not_prepared' : 'cancellation_scope_delivery_mismatch'); + } + if ($preparation !== null && now()->gte( + CarbonImmutable::parse($preparation->payload['authority_deadline_at']) + )) { + throw new LogicException('cancellation_scope_authority_expired'); + } + if ($preparing) { + CancellationScopeDescendants::request($locked, $scopeId); + // Requests and preparation commit together. Recheck after every nested + // request has completed so a lost claim/budget cannot publish that tree. + $authority = CancellationScopeRequests::authority($locked, $scopeId); + /** @var WorkflowTask|null $claim */ + $claim = ConfiguredV2Models::query('task_model', WorkflowTask::class) + ->lockForUpdate() + ->find($task->id); + if (! self::matchesClaim($claim, $locked, $task)) { + throw new LogicException('cancellation_scope_workflow_claim_mismatch'); + } + if (! $authority['active'] || $authority['deadline_at'] === null) { + throw new LogicException('cancellation_scope_authority_expired'); + } + } + $members = $preparing ? self::activityMembers( + $locked, + $scopeId + ) : $preparation->payload['activity_members']; + $timerMembers = $preparing ? ScopedTimerCancellation::members($locked, $scopeId) + : $preparation->payload['timer_members']; + $waitMembers = $preparing ? ScopedWaitCancellation::members($locked, $scopeId) + : $preparation->payload['wait_members']; + $childMembers = $preparing ? ScopedChildCancellation::members($locked, $scopeId) + : $preparation->payload['child_members']; + $descendantMembers = $preparing ? CancellationScopeDescendants::members( + $locked, + $scopeId, + $authority['deadline_at'], + ) : $preparation->payload['descendant_members']; + if ($preparing && $locked->historyEvents->contains(static fn (WorkflowHistoryEvent $event): bool => + in_array( + $event->event_type, + [HistoryEventType::ActivityCancelled, HistoryEventType::TimerCancelled, + HistoryEventType::SignalWaitCancelled, HistoryEventType::ConditionWaitCancelled], + true + ) + && ($event->payload['cancellation_scope']['scope_id'] ?? null) === $scopeId)) { + throw new LogicException('cancellation_scope_preparation_after_effect'); + } + if (! $preparing) { + // The await range can include shields and siblings. Stop + // proof belongs to each original frozen scope inventory, + // including descendants with their own accepted context. + foreach ($members as $member) { + ScopedActivityDeliveryPolicy::assertReady($locked, $context, $member['sequence'], 1); + } + ScopedTimerCancellation::assertReady($locked, $preparation); + ScopedWaitCancellation::assertReady($locked, $preparation); + ScopedChildCancellation::assertReady($preparation, $locked); + CancellationScopeDescendants::assertReady( + $locked, + $descendantMembers, + preparationHistoryEventId: $preparation->id + ); + } + /** @var WorkflowTask|null $currentClaim */ + $currentClaim = ConfiguredV2Models::query('task_model', WorkflowTask::class) + ->lockForUpdate() + ->find($task->id); + if (! self::matchesClaim($currentClaim, $locked, $task)) { + throw new LogicException('cancellation_scope_workflow_claim_mismatch'); + } + $currentAuthority = CancellationScopeRequests::authority($locked, $scopeId); + if (! $currentAuthority['active'] || $currentAuthority['deadline_at'] === null + || ($preparation !== null && now()->gte( + CarbonImmutable::parse($preparation->payload['authority_deadline_at']) + ))) { + throw new LogicException('cancellation_scope_authority_expired'); + } + if ($currentAuthority['deadline_at'] !== $authority['deadline_at']) { + throw new LogicException('cancellation_scope_authority_changed'); + } + $boundary = WorkflowHistoryEvent::record($locked, $preparing + ? HistoryEventType::CancellationScopeDeliveryPrepared : HistoryEventType::CancellationScopeDelivered, [ + 'schema' => $preparing ? self::PREPARATION_SCHEMA : self::SCHEMA, + 'workflow_run_id' => $locked->id, + 'scope_id' => $scopeId, + 'request_id' => $context->requestId, + 'cancellation' => $context->toArray(), + 'sequence' => $sequence, + 'call_kind' => $callKind, + 'sequence_span' => $sequenceSpan, + 'operation_sequence' => $operationSequence, + 'operation_sequence_span' => $operationSequenceSpan, + 'authority_deadline_at' => $preparation?->payload['authority_deadline_at'] ?? $authority['deadline_at'], + ...($preparing ? [ + 'activity_members' => $members, + 'timer_members' => $timerMembers, + 'wait_members' => $waitMembers, + 'child_members' => $childMembers, + 'descendant_members' => $descendantMembers, + ] + : [ + 'preparation_history_event_id' => $preparation->id, + ]), + ], $claim); + self::renewHosting($locked, $claim, $boundary, $renewWorkflowLease); + return $boundary; + }, 3); + $run->refresh(); + return $event; + } + + private static function renewHosting( + WorkflowRun $run, + WorkflowTask $task, + WorkflowHistoryEvent $boundary, + bool $renewWorkflowLease + ): void { + $captured = CarbonImmutable::parse($boundary->payload['authority_deadline_at']); + if ($run->cancellation_scope_recovery_until === null || $captured->gt( + $run->cancellation_scope_recovery_until + )) { + $run->forceFill([ + 'cancellation_scope_recovery_until' => $captured, + ])->save(); + } + if ($renewWorkflowLease) { + LocalActivityRuntime::renewWorkflowTask($task, CancellationCleanupLease::deadline($run), true, $run); + } + } + + private static function readBoundary(WorkflowRun $run, string $scopeId, bool $preparing): ?WorkflowHistoryEvent + { + $context = CancellationScopeRequests::context($run, $scopeId); + $events = $run->historyEvents() + ->where('event_type', $preparing ? HistoryEventType::CancellationScopeDeliveryPrepared + : HistoryEventType::CancellationScopeDelivered) + ->where('payload->scope_id', $scopeId) + ->get(); + if ($events->isEmpty()) { + return null; + } + if ($events->count() !== 1 || $context === null) { + throw new LogicException('cancellation_scope_delivery_history_invalid'); + } + $event = $events->sole(); + $payload = $event->payload; + if (($payload['schema'] ?? null) !== ($preparing ? self::PREPARATION_SCHEMA : self::SCHEMA) + || ($payload['workflow_run_id'] ?? null) !== $run->id + || ($payload['scope_id'] ?? null) !== $scopeId || ($payload['request_id'] ?? null) !== $context->requestId + || ! is_array($payload['cancellation'] ?? null) + || ! is_int($payload['sequence'] ?? null) || ! is_string($payload['call_kind'] ?? null) + || ! is_int($payload['sequence_span'] ?? null) || ! array_key_exists('operation_sequence', $payload) + || ($payload['operation_sequence'] !== null && ! is_int($payload['operation_sequence'])) + || ! is_int($payload['operation_sequence_span'] ?? null) + || ! is_string($payload['authority_deadline_at'] ?? null)) { + throw new LogicException('cancellation_scope_delivery_history_invalid'); + } + try { + $snapshot = ScopedCancellationContext::fromArray($payload['cancellation']); + $deadline = CarbonImmutable::parse($payload['authority_deadline_at']); + } catch (InvalidArgumentException $error) { + throw new LogicException('cancellation_scope_delivery_history_invalid', previous: $error); + } + $request = $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeRequested) + ->where('payload->scope_id', $scopeId) + ->sole(); + $run->loadMissing('historyEvents'); + $operationStart = $payload['operation_sequence'] ?? $payload['sequence']; + $operationSpan = $payload['operation_sequence'] === null ? $payload['sequence_span'] : $payload['operation_sequence_span']; + $recordedKind = CooperativeCancellationDelivery::callKindAt($run, $payload['sequence']); + if ($snapshot->toArray() !== $context->toArray() || $deadline->toISOString() !== $payload['authority_deadline_at'] + || $deadline->greaterThan($context->deadline()) || $deadline->lessThan($context->requestedAt()) + || $event->sequence <= $request->sequence + || ! CancellationScopeHistory::isRecordedBefore($run, $scopeId, $operationStart) + || ! self::matchesMembership($run, $scopeId, $operationStart, $operationSpan, $event->sequence) + || ($recordedKind !== null && ! in_array($payload['call_kind'], ['parallel', 'selection_handle'], true) + && $payload['call_kind'] !== $recordedKind) + || CooperativeCancellationDelivery::validateBoundarySyntax( + $payload['sequence'], + $payload['call_kind'], + $payload['sequence_span'], + $payload['operation_sequence'], + $payload['operation_sequence_span'] + ) !== null) { + throw new LogicException('cancellation_scope_delivery_history_invalid'); + } + try { + if ($preparing) { + $history = $run->historyEvents; + $run->setRelation('historyEvents', $history->filter( + static fn (WorkflowHistoryEvent $row): bool => $row->sequence < $event->sequence + )); + try { + if (self::normalizeMembers($payload['activity_members'] ?? null) !== self::activityMembers( + $run, + $scopeId + ) + || ScopedTimerCancellation::normalizeMembers($payload['timer_members'] ?? null) + !== ScopedTimerCancellation::members($run, $scopeId) + || ScopedWaitCancellation::normalizeMembers($payload['wait_members'] ?? null) + !== ScopedWaitCancellation::members($run, $scopeId) + || ScopedChildCancellation::normalizeMembers($payload['child_members'] ?? null) + !== ScopedChildCancellation::members($run, $scopeId) + || CancellationScopeDescendants::normalizeMembers($payload['descendant_members'] ?? null) + !== CancellationScopeDescendants::members($run, $scopeId, $payload['authority_deadline_at']) + || CooperativeCancellationDelivery::validateCallBoundary( + $run, + $request, + $payload['sequence'], + $payload['call_kind'], + $payload['sequence_span'], + $payload['operation_sequence'], + $payload['operation_sequence_span'] + ) !== null + || $run->historyEvents->contains(static fn (WorkflowHistoryEvent $row): bool => + in_array( + $row->event_type, + [HistoryEventType::ActivityCancelled, HistoryEventType::TimerCancelled, + HistoryEventType::SignalWaitCancelled, HistoryEventType::ConditionWaitCancelled], + true + ) + && ($row->payload['cancellation_scope']['scope_id'] ?? null) === $scopeId)) { + throw new LogicException('cancellation_scope_preparation_history_invalid'); + } + } finally { + $run->setRelation('historyEvents', $history); + } + } else { + $preparation = self::prepared($run, $scopeId); + if ($preparation === null || $preparation->sequence >= $event->sequence + || ($payload['preparation_history_event_id'] ?? null) !== $preparation->id + || $payload['authority_deadline_at'] !== $preparation->payload['authority_deadline_at'] + || ! self::sameBoundary( + $preparation->payload, + $payload['sequence'], + $payload['call_kind'], + $payload['sequence_span'], + $payload['operation_sequence'], + $payload['operation_sequence_span'] + )) { + throw new LogicException('cancellation_scope_preparation_history_invalid'); + } + foreach ($preparation->payload['activity_members'] as $member) { + ScopedActivityDeliveryPolicy::assertReady($run, $context, $member['sequence'], 1, $event->sequence); + } + ScopedTimerCancellation::assertReady($run, $preparation, $event->sequence); + ScopedWaitCancellation::assertReady($run, $preparation, $event->sequence); + ScopedChildCancellation::assertReady($preparation, $run, $event->sequence); + CancellationScopeDescendants::assertReady( + $run, + $preparation->payload['descendant_members'], + $event->sequence, + $preparation->id, + ); + } + } catch (LogicException $error) { + throw new LogicException('cancellation_scope_delivery_history_invalid', previous: $error); + } + return $event; + } + + /** + * @param array $payload + */ + private static function sameBoundary( + array $payload, + int $sequence, + string $kind, + int $span, + ?int $operationSequence, + int $operationSpan + ): bool { + return $payload['sequence'] === $sequence && $payload['call_kind'] === $kind + && $payload['sequence_span'] === $span && $payload['operation_sequence'] === $operationSequence + && $payload['operation_sequence_span'] === $operationSpan; + } + + private static function matchesMembership( + WorkflowRun $run, + string $scopeId, + int $start, + int $span, + ?int $beforeHistorySequence = null, + ): bool { + $scopes = CancellationScopeHistory::forRun($run); + $openings = $run->historyEvents->filter(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::CancellationScopeOpened + && ($beforeHistorySequence === null || $event->sequence < $beforeHistorySequence))->keyBy('id'); + $included = [ + $scopeId => true, + ]; + foreach ($scopes as $id => $scope) { + if ($openings->has($scope['history_event_id']) && ! $scope['shield_parent'] + && isset($included[$scope['parent_scope_id']])) { + $included[$id] = true; + } + } + $scheduled = false; + $affected = false; + foreach ($run->historyEvents as $event) { + $payload = $event->payload; + $sequence = $payload['sequence'] ?? null; + if (($beforeHistorySequence !== null && $event->sequence >= $beforeHistorySequence) + || ! is_int($sequence) || $sequence < $start || $sequence - $start >= $span + || ! in_array( + $event->event_type, + [HistoryEventType::ActivityScheduled, HistoryEventType::TimerScheduled, + HistoryEventType::ConditionWaitOpened, HistoryEventType::SignalWaitOpened, + HistoryEventType::ChildWorkflowScheduled], + true + )) { + continue; + } + $nested = match ($event->event_type) { + HistoryEventType::ActivityScheduled => 'activity', + HistoryEventType::TimerScheduled => 'timer', + HistoryEventType::ChildWorkflowScheduled => 'child_workflow', + default => null, + }; + if ($nested !== null && array_key_exists($nested, $payload) && ! is_array($payload[$nested])) { + return false; + } + $descriptor = $nested !== null ? ($payload[$nested] ?? []) : []; + $hasFlat = array_key_exists('cancellation_scope_id', $payload); + $hasNested = array_key_exists('cancellation_scope_id', $descriptor); + $membership = $hasFlat ? $payload['cancellation_scope_id'] + : ($hasNested ? $descriptor['cancellation_scope_id'] : CancellationScopeHistory::ROOT_SCOPE_ID); + if (! is_string($membership) + || ($hasNested && $descriptor['cancellation_scope_id'] !== $membership)) { + return false; + } + if ($membership !== CancellationScopeHistory::ROOT_SCOPE_ID) { + $member = $scopes[$membership] ?? null; + $opening = $member === null ? null : $openings->get($member['history_event_id']); + if ($member === null || $opening === null || $member['sequence'] >= $sequence + || $opening->sequence >= $event->sequence) { + return false; + } + } + $scheduled = true; + $affected = $affected || isset($included[$membership]); + } + // An unscheduled await keeps its authenticated scope address. A group + // receives cancellation when any leaf is in the unshielded subtree. + // Actors still reconcile only the frozen subtree inventory. + return ! $scheduled || $affected; + } + + private static function matchesClaim(?WorkflowTask $claim, WorkflowRun $run, WorkflowTask $original): bool + { + return $claim !== null && $claim->workflow_run_id === $run->id && $claim->namespace === $run->namespace + && $claim->task_type === TaskType::Workflow && $claim->status === TaskStatus::Leased + && $claim->lease_owner !== null && $claim->lease_owner !== '' && $claim->attempt_count > 0 + && $claim->lease_owner === $original->lease_owner && $claim->attempt_count === $original->attempt_count + && $claim->lease_expires_at !== null && now() + ->lt($claim->lease_expires_at); + } +} diff --git a/src/V2/Support/CancellationScopeDescendants.php b/src/V2/Support/CancellationScopeDescendants.php new file mode 100644 index 000000000..3ae7d374a --- /dev/null +++ b/src/V2/Support/CancellationScopeDescendants.php @@ -0,0 +1,252 @@ +getConnection()->transactionLevel() === 0) { + throw new LogicException('cancellation_scope_descendants_require_preparation_transaction'); + } + foreach (self::scopes($run, $scopeId) as $scope) { + CancellationScopeRequests::request( + $run, + $scope['scope_id'], + CancellationScopeHistory::MINIMUM_PROTOCOL_VERSION, + parentScopeId: $scope['parent_scope_id'], + ); + } + $run->unsetRelation('historyEvents'); + } + + /** + * Read only the caller's history prefix. Later scope openings or requests cannot change this inventory. + * + * @return list> + */ + public static function members(WorkflowRun $run, string $scopeId, string $authorityDeadline): array + { + $run->loadMissing('historyEvents'); + $members = []; + foreach (self::scopes($run, $scopeId) as $scope) { + $request = self::accepted($run, $scope['scope_id']); + $context = CancellationScopeRequests::context($run, $scope['scope_id']); + $parent = CancellationScopeRequests::context($run, $scope['parent_scope_id']); + if ($request === null || $context === null || $parent === null) { + throw new LogicException('cancellation_scope_descendant_request_history_invalid'); + } + $propagation = self::propagation($run, $scope, $request, $context, $parent); + $deadline = CarbonImmutable::parse($authorityDeadline); + foreach (CancellationScopeHistory::ancestry($run, $scope['scope_id']) as $ancestor) { + $accepted = self::accepted($run, $ancestor); + if ($accepted !== null) { + $ceiling = ScopedCancellationContext::fromArray($accepted->payload['cancellation'])->deadline(); + if ($ceiling->lt($deadline)) { + $deadline = $ceiling; + } + } + } + $members[] = [ + 'scope_id' => $scope['scope_id'], + 'parent_scope_id' => $scope['parent_scope_id'], + 'scope_history_event_id' => $scope['history_event_id'], + 'request_history_event_id' => $request->id, + 'request_id' => $context->requestId, + 'propagation_history_event_id' => $propagation->id, + 'cancellation' => $context->toArray(), + 'authority_deadline_at' => $deadline->toISOString(), + 'activity_members' => CancellationScopeDelivery::activityMembers($run, $scope['scope_id']), + 'timer_members' => ScopedTimerCancellation::members($run, $scope['scope_id']), + 'wait_members' => ScopedWaitCancellation::members($run, $scope['scope_id']), + 'child_members' => ScopedChildCancellation::members($run, $scope['scope_id']), + ]; + } + return self::normalizeMembers($members); + } + + /** + * @return list> + */ + public static function normalizeMembers(mixed $members): array + { + if (! is_array($members) || ! array_is_list($members)) { + throw new LogicException('cancellation_scope_preparation_history_invalid'); + } + $normalized = []; + $keys = ['scope_id', 'parent_scope_id', 'scope_history_event_id', 'request_history_event_id', + 'request_id', 'propagation_history_event_id', 'authority_deadline_at']; + foreach ($members as $member) { + if (! is_array($member) || count($member) !== 12 || ! is_array($member['cancellation'] ?? null)) { + throw new LogicException('cancellation_scope_preparation_history_invalid'); + } + $value = []; + foreach ($keys as $key) { + if (! is_string($member[$key] ?? null) || $member[$key] === '') { + throw new LogicException('cancellation_scope_preparation_history_invalid'); + } + $value[$key] = $member[$key]; + } + try { + $context = ScopedCancellationContext::fromArray($member['cancellation']); + } catch (InvalidArgumentException $error) { + throw new LogicException('cancellation_scope_preparation_history_invalid', previous: $error); + } + $normalized[] = [ + ...$value, + 'cancellation' => $context->toArray(), + 'activity_members' => CancellationScopeDelivery::normalizeMembers($member['activity_members'] ?? null), + 'timer_members' => ScopedTimerCancellation::normalizeMembers($member['timer_members'] ?? null), + 'wait_members' => ScopedWaitCancellation::normalizeMembers($member['wait_members'] ?? null), + 'child_members' => ScopedChildCancellation::normalizeMembers($member['child_members'] ?? null), + ]; + } + return $normalized; + } + + /** + * A parent marker cannot skip unresolved original descendants. Operation dispatch remains separate. + * + * @param list> $members + */ + public static function assertReady( + WorkflowRun $run, + array $members, + ?int $beforeHistorySequence = null, + ?string $preparationHistoryEventId = null, + ): void { + foreach (self::normalizeMembers($members) as $member) { + $preparation = ScopedCancellationPreparation::forScope( + $run, + $member['scope_id'], + $preparationHistoryEventId + ); + if ($preparation === null) { + throw new LogicException('cancellation_scope_descendant_delivery_not_prepared'); + } + foreach (['activity_members', 'timer_members', 'wait_members', 'child_members'] as $field) { + $normalized = match ($field) { + 'activity_members' => $preparation->activityMembers, + 'timer_members' => $preparation->timerMembers, + 'wait_members' => $preparation->waitMembers, + 'child_members' => $preparation->childMembers, + }; + if ($normalized !== $member[$field] || $preparation->requestId !== $member['request_id'] + || $preparation->context->toArray() !== $member['cancellation'] + || $preparation->authorityDeadlineAt !== $member['authority_deadline_at']) { + throw new LogicException('cancellation_scope_descendant_delivery_membership_mismatch'); + } + } + if (ScopedCancellationReconciliation::completed($run, $preparation, $beforeHistorySequence) !== null) { + continue; + } + if (ScopedCancellationReconciliation::pending($run, $preparation, $beforeHistorySequence)) { + throw new LogicException('cancellation_scope_descendant_delivery_pending'); + } + $context = ScopedCancellationContext::fromArray($member['cancellation']); + foreach ($member['activity_members'] as $activity) { + ScopedActivityDeliveryPolicy::assertReady( + $run, + $context, + $activity['sequence'], + 1, + $beforeHistorySequence + ); + } + ScopedTimerCancellation::assertReady($run, $preparation, $beforeHistorySequence); + ScopedWaitCancellation::assertReady($run, $preparation, $beforeHistorySequence); + ScopedChildCancellation::assertReady($preparation, $run, $beforeHistorySequence); + } + } + + /** + * @return list + */ + private static function scopes(WorkflowRun $run, string $scopeId): array + { + $run->loadMissing('historyEvents'); + $prefix = $run->historyEvents->keyBy('id'); + $scopes = CancellationScopeHistory::forRun($run); + if ($scopeId !== CancellationScopeHistory::ROOT_SCOPE_ID && ! isset($scopes[$scopeId])) { + throw new LogicException('cancellation_scope_not_recorded'); + } + $included = [ + $scopeId => true, + ]; + $descendants = []; + foreach ($scopes as $id => $scope) { + if ($id === $scopeId || ! $prefix->has($scope['history_event_id']) + || $scope['shield_parent'] || ! isset($included[$scope['parent_scope_id']])) { + continue; + } + $included[$id] = true; + $descendants[] = $scope; + } + return $descendants; + } + + private static function accepted(WorkflowRun $run, string $scopeId): ?WorkflowHistoryEvent + { + $events = $run->historyEvents->filter(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::CancellationScopeRequested + && ($event->payload['scope_id'] ?? null) === $scopeId); + if ($events->count() > 1) { + throw new LogicException('cancellation_scope_descendant_request_history_invalid'); + } + return $events->first(); + } + + /** + * @param array{scope_id: string, parent_scope_id: string, shield_parent: bool, sequence: int, history_event_id: string} $scope + */ + private static function propagation( + WorkflowRun $run, + array $scope, + WorkflowHistoryEvent $request, + ScopedCancellationContext $accepted, + ScopedCancellationContext $parent, + ): WorkflowHistoryEvent { + if ($accepted->rootContext->rootRequestId === $parent->rootContext->rootRequestId) { + if (($request->payload['parent_scope_id'] ?? null) !== $scope['parent_scope_id']) { + throw new LogicException('cancellation_scope_descendant_request_history_invalid'); + } + return $request; + } + foreach ($run->historyEvents as $event) { + $payload = $event->payload; + if ($event->event_type !== HistoryEventType::CancellationScopeRequestConflicted + || ($payload['scope_id'] ?? null) !== $scope['scope_id'] + || ($payload['parent_scope_id'] ?? null) !== $scope['parent_scope_id']) { + continue; + } + $incoming = ScopedCancellationContext::fromArray($payload['incoming_cancellation']); + $priorAccepted = ScopedCancellationContext::fromArray($payload['accepted_cancellation']); + if ($incoming->rootContext->rootRequestId === $parent->rootContext->rootRequestId + && $priorAccepted->toArray() === $accepted->toArray() + && $incoming->toArray() === $parent->forDescendant( + $incoming->requestId, + $run->workflow_instance_id, + $run->id, + $scope['scope_id'], + )->toArray()) { + return $event; + } + } + throw new LogicException('cancellation_scope_descendant_request_history_invalid'); + } +} diff --git a/src/V2/Support/CancellationScopeHistory.php b/src/V2/Support/CancellationScopeHistory.php new file mode 100644 index 000000000..ca37a7903 --- /dev/null +++ b/src/V2/Support/CancellationScopeHistory.php @@ -0,0 +1,194 @@ +exists || ! $task->exists) { + throw new LogicException('invalid_cancellation_scope_open'); + } + + $event = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->getModel() + ->getConnection() + ->transaction(static function () use ( + $run, + $task, + $sequence, + $parentScopeId, + $shieldParent + ): WorkflowHistoryEvent { + /** @var WorkflowRun $lockedRun */ + $lockedRun = ConfiguredV2Models::query('run_model', WorkflowRun::class)->lockForUpdate()->findOrFail( + $run->id + ); + /** @var WorkflowTask|null $claim */ + $claim = ConfiguredV2Models::query('task_model', WorkflowTask::class)->lockForUpdate()->find($task->id); + if (! $claim instanceof WorkflowTask || $claim->workflow_run_id !== $lockedRun->id + || $claim->namespace !== $lockedRun->namespace || $claim->task_type !== TaskType::Workflow + || $claim->status !== TaskStatus::Leased || $claim->lease_owner === null || $claim->lease_owner === '' + || $claim->attempt_count < 1 + || $claim->lease_owner !== $task->lease_owner || $claim->attempt_count !== $task->attempt_count + || $claim->lease_expires_at === null || now() + ->gte($claim->lease_expires_at)) { + throw new LogicException('cancellation_scope_workflow_claim_mismatch'); + } + if ($lockedRun->status->isTerminal()) { + throw new LogicException('cancellation_scope_run_not_active'); + } + $scopes = self::forRun($lockedRun); + foreach ($scopes as $scope) { + if ($scope['sequence'] === $sequence) { + if ($scope['parent_scope_id'] !== $parentScopeId || $scope['shield_parent'] !== $shieldParent) { + throw new HistoryEventShapeMismatchException( + $sequence, + WorkflowStepHistory::CANCELLATION_SCOPE, + [HistoryEventType::CancellationScopeOpened->value], + 'Recorded cancellation scope parent or shield mode differs from replay.', + ); + } + return $lockedRun->historyEvents() + ->whereKey($scope['history_event_id'])->firstOrFail(); + } + } + if ($lockedRun->cancellation_request_command_id !== null + || ($lockedRun->execution_deadline_at !== null && now()->gte($lockedRun->execution_deadline_at)) + || ($lockedRun->run_deadline_at !== null && now()->gte($lockedRun->run_deadline_at))) { + throw new LogicException('cancellation_scope_run_not_active'); + } + if ($parentScopeId !== self::ROOT_SCOPE_ID && ! isset($scopes[$parentScopeId])) { + throw new LogicException('cancellation_scope_parent_not_recorded'); + } + // The original subtree has already been frozen. A new shield must + // not let later work escape that preparation; existing opens replay above. + $refusal = CancellationScopeDelivery::admissionRefusal($lockedRun, $parentScopeId, $sequence); + if ($refusal !== null) { + throw new LogicException($refusal === 'operation_cancellation_delivery_reserved' + ? 'cancellation_scope_command_sequence_reserved' : 'cancellation_scope_parent_delivery_prepared'); + } + WorkflowStepHistory::assertCompatible($lockedRun, $sequence, WorkflowStepHistory::CANCELLATION_SCOPE); + if ($sequence !== WorkflowStepHistory::nextDurableCommandSequence($lockedRun)) { + throw new LogicException('cancellation_scope_sequence_mismatch'); + } + return WorkflowHistoryEvent::record($lockedRun, HistoryEventType::CancellationScopeOpened, [ + 'schema' => self::SCHEMA, + 'workflow_run_id' => $lockedRun->id, + 'sequence' => $sequence, + 'scope_id' => (string) Str::ulid(), + 'parent_scope_id' => $parentScopeId, + 'shield_parent' => $shieldParent, + ], $claim); + }, 3); + + $run->refresh(); + return $event; + } + + /** + * Read exact-run canonical addresses without following an instance's current run. + * Parent-before-child validation makes malformed forward edges and cycles invalid. + * + * @return array + */ + public static function forRun(WorkflowRun $run): array + { + $scopes = []; + $sequences = []; + foreach ($run->historyEvents()->where('event_type', HistoryEventType::CancellationScopeOpened) + ->orderBy('sequence') + ->cursor() as $event) { + $payload = $event->payload; + $id = $payload['scope_id'] ?? null; + $parent = $payload['parent_scope_id'] ?? null; + $shield = $payload['shield_parent'] ?? null; + $sequence = $payload['sequence'] ?? null; + if (($payload['schema'] ?? null) !== self::SCHEMA || ($payload['workflow_run_id'] ?? null) !== $run->id + || ! is_string($id) || $id === '' || $id === self::ROOT_SCOPE_ID || isset($scopes[$id]) + || ! is_string($parent) || $parent === '' || ! is_bool($shield) + || ! is_int($sequence) || $sequence < 1 || isset($sequences[$sequence]) + || ($parent !== self::ROOT_SCOPE_ID && (! isset($scopes[$parent]) || $scopes[$parent]['sequence'] >= $sequence))) { + throw new LogicException('cancellation_scope_history_invalid'); + } + $scopes[$id] = [ + 'scope_id' => $id, + 'parent_scope_id' => $parent, + 'shield_parent' => $shield, + 'sequence' => $sequence, + 'history_event_id' => $event->id, + ]; + $sequences[$sequence] = true; + } + return $scopes; + } + + /** + * @return list + */ + public static function ancestry(WorkflowRun $run, string $scopeId): array + { + $scopes = self::forRun($run); + $ancestry = []; + while ($scopeId !== self::ROOT_SCOPE_ID) { + if (! isset($scopes[$scopeId])) { + throw new LogicException('cancellation_scope_not_recorded'); + } + $ancestry[] = $scopeId; + $scopeId = $scopes[$scopeId]['parent_scope_id']; + } + $ancestry[] = self::ROOT_SCOPE_ID; + return array_reverse($ancestry); + } + + /** + * The admission caller holds the run lock. A scope created later cannot + * retroactively own an earlier operation. Historical omission means root. + */ + public static function isRecordedBefore(WorkflowRun $run, string $scopeId, int $operationSequence): bool + { + if ($operationSequence < 1) { + return false; + } + if ($scopeId === self::ROOT_SCOPE_ID) { + return true; + } + $scope = self::forRun($run)[$scopeId] ?? null; + return $scope !== null && $scope['sequence'] < $operationSequence; + } +} diff --git a/src/V2/Support/CancellationScopeRequests.php b/src/V2/Support/CancellationScopeRequests.php new file mode 100644 index 000000000..34b37fdd5 --- /dev/null +++ b/src/V2/Support/CancellationScopeRequests.php @@ -0,0 +1,391 @@ + 3600 || ! $run->exists) { + throw new LogicException('invalid_scope_cancellation_request'); + } + + return $run->getConnection() + ->transaction(static function () use ( + $run, + $scopeId, + $cleanupTimeoutSeconds, + $reason, + $caller, + $parentScopeId + ): WorkflowHistoryEvent { + // Claims and heartbeat lock task before run. Acceptance must + // establish recoverable ownership before a worker prepares delivery. + $openTasks = ConfiguredV2Models::query('task_model', WorkflowTask::class) + ->where('workflow_run_id', $run->id) + ->where('task_type', TaskType::Workflow) + ->whereIn('status', [TaskStatus::Ready, TaskStatus::Leased]) + ->orderBy('id') + ->lockForUpdate() + ->get(); + /** @var WorkflowRun $locked */ + $locked = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->lockForUpdate() + ->findOrFail($run->id); + $scopes = CancellationScopeHistory::forRun($locked); + self::assertScope($scopes, $scopeId); + $parent = $parentScopeId === null ? null : self::parentContext($locked, $scopeId, $parentScopeId); + $existing = self::acceptedEvent($locked, $scopeId); + $accepted = $existing === null ? null : self::context($locked, $scopeId); + if ($existing !== null && ($parent === null + || $accepted?->rootContext->rootRequestId === $parent->rootContext->rootRequestId)) { + return $existing; + } + if ($accepted !== null && $parent !== null) { + foreach ($locked->historyEvents()->where( + 'event_type', + HistoryEventType::CancellationScopeRequestConflicted + ) + ->get() as $conflict) { + $payload = $conflict->payload; + if (($payload['scope_id'] ?? null) === $scopeId && ($payload['parent_scope_id'] ?? null) === $parentScopeId + && ($payload['incoming_cancellation']['root_context']['root_request_id'] ?? null) === $parent->rootContext->rootRequestId) { + $priorAccepted = ScopedCancellationContext::fromArray($payload['accepted_cancellation']); + $priorIncoming = ScopedCancellationContext::fromArray($payload['incoming_cancellation']); + if ($priorAccepted->toArray() !== $accepted->toArray() + || $priorIncoming->toArray() !== $parent->forDescendant( + $priorIncoming->requestId, + $locked->workflow_instance_id, + $locked->id, + $scopeId + )->toArray()) { + throw new LogicException('cancellation_scope_request_history_invalid'); + } + return $conflict; + } + } + } + + $requestId = (string) Str::ulid(); + $incoming = $parent?->forDescendant($requestId, $locked->workflow_instance_id, $locked->id, $scopeId) + ?? self::directContext($locked, $scopeId, $requestId, $cleanupTimeoutSeconds, $reason, $caller); + if ($accepted !== null) { + return WorkflowHistoryEvent::record($locked, HistoryEventType::CancellationScopeRequestConflicted, [ + 'schema' => self::SCHEMA, + 'workflow_run_id' => $locked->id, + 'scope_id' => $scopeId, + 'parent_scope_id' => $parentScopeId, + 'reason' => 'cancellation_root_conflict', + 'accepted_cancellation' => $accepted->toArray(), + 'incoming_cancellation' => $incoming->toArray(), + ]); + } + + $authority = self::authoritySnapshot($locked, $scopeId); + if (! $authority['active'] || now()->gte($incoming->deadline())) { + throw new LogicException('cancellation_scope_authority_expired'); + } + $event = WorkflowHistoryEvent::record($locked, HistoryEventType::CancellationScopeRequested, [ + 'schema' => self::SCHEMA, + 'workflow_run_id' => $locked->id, + 'scope_id' => $scopeId, + 'parent_scope_id' => $parentScopeId, + 'request_id' => $requestId, + 'cancellation' => $incoming->toArray(), + ]); + $ceiling = $incoming->deadline(); + if ($authority['deadline_at'] !== null) { + $authorityDeadline = CarbonImmutable::parse($authority['deadline_at']); + if ($authorityDeadline->lt($ceiling)) { + $ceiling = $authorityDeadline; + } + } + if ($locked->cancellation_scope_recovery_until === null || $ceiling->gt( + $locked->cancellation_scope_recovery_until + )) { + $locked->forceFill([ + 'cancellation_scope_recovery_until' => $ceiling, + ])->save(); + } + $expiry = CancellationCleanupLease::expiresAt($locked); + /** @var WorkflowTask $claim */ + foreach ($openTasks->where('status', TaskStatus::Leased) as $claim) { + if ($claim->lease_expires_at === null || now()->gte($claim->lease_expires_at) + || ! $expiry->lt($claim->lease_expires_at)) { + continue; + } + $claim->forceFill([ + 'lease_expires_at' => $expiry, + ])->save(); + WorkflowRunSummary::query()->whereKey($locked->id)->where('next_task_id', $claim->id)->update([ + 'next_task_lease_expires_at' => $expiry, + ]); + } + if ($openTasks->isEmpty()) { + // A workflow waiting on a scoped timer/activity still needs + // a claim to prepare delivery. Acceptance alone must not + // leave it asleep past the original cleanup deadline. + /** @var WorkflowTask $task */ + $task = ConfiguredV2Models::query('task_model', WorkflowTask::class)->create([ + 'workflow_run_id' => $locked->id, + 'namespace' => $locked->namespace, + 'task_type' => TaskType::Workflow, + 'status' => TaskStatus::Ready, + 'available_at' => now(), + 'payload' => [ + 'resume_source_kind' => 'cancellation_scope_request', + 'resume_source_id' => $event->id, + 'workflow_command_id' => $requestId, + 'scope_id' => $scopeId, + ], + 'connection' => $locked->connection, + 'queue' => $locked->queue, + 'compatibility' => $locked->compatibility, + ]); + app(HistoryProjectionRole::class)->projectRun( + $locked->fresh( + ['instance', 'tasks', 'activityExecutions', 'timers', 'failures', 'historyEvents'] + ) + ); + TaskDispatcher::dispatch($task); + } + return $event; + }, 3); + } + + /** + * Read the accepted canonical address, including after a cold reload. + */ + public static function context(WorkflowRun $run, string $scopeId): ?ScopedCancellationContext + { + self::assertScope(CancellationScopeHistory::forRun($run), $scopeId); + $event = self::acceptedEvent($run, $scopeId); + if ($event === null) { + return null; + } + $payload = $event->payload; + if (($payload['schema'] ?? null) !== self::SCHEMA || ($payload['workflow_run_id'] ?? null) !== $run->id + || ! is_array($payload['cancellation'] ?? null) || ! array_key_exists('parent_scope_id', $payload)) { + throw new LogicException('cancellation_scope_request_history_invalid'); + } + $context = ScopedCancellationContext::fromArray($payload['cancellation']); + if ($context->scopeId !== $scopeId || $context->workflowRunId !== $run->id + || $context->workflowInstanceId !== $run->workflow_instance_id + || $context->requestId !== ($payload['request_id'] ?? null)) { + throw new LogicException('cancellation_scope_request_history_invalid'); + } + $parentScopeId = $payload['parent_scope_id']; + if ($parentScopeId === null) { + if (count($context->lineage) !== 1 || $context->rootContext->requestId !== $context->requestId) { + throw new LogicException('cancellation_scope_request_history_invalid'); + } + } else { + if (! is_string($parentScopeId)) { + throw new LogicException('cancellation_scope_request_history_invalid'); + } + $parent = self::parentContext($run, $scopeId, $parentScopeId); + $expected = $parent->forDescendant($context->requestId, $run->workflow_instance_id, $run->id, $scopeId); + if ($context->toArray() !== $expected->toArray()) { + throw new LogicException('cancellation_scope_request_history_invalid'); + } + } + return $context; + } + + /** + * Inspect authority separately from the immutable accepted request budget. + * Shields defer delivery; they do not remove ancestor/run authority ceilings. + * + * @return array{active: bool, deadline_at: string|null} + */ + public static function authority(WorkflowRun $run, string $scopeId): array + { + return $run->getConnection() + ->transaction(static function () use ($run, $scopeId): array { + /** @var WorkflowRun $locked */ + $locked = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->lockForUpdate() + ->findOrFail($run->id); + return self::authoritySnapshot($locked, $scopeId); + }); + } + + /** + * @return array{active: bool, deadline_at: string|null} + */ + private static function authoritySnapshot(WorkflowRun $run, string $scopeId): array + { + $ancestry = CancellationScopeHistory::ancestry($run, $scopeId); + self::assertScope(CancellationScopeHistory::forRun($run), $scopeId); + $deadline = null; + $ceilings = [$run->execution_deadline_at, $run->run_deadline_at, self::runContext($run)?->deadline()]; + foreach ($ancestry as $ancestor) { + if ($ancestor !== CancellationScopeHistory::ROOT_SCOPE_ID) { + $ceilings[] = self::context($run, $ancestor)?->deadline(); + } + } + foreach ($ceilings as $ceiling) { + if ($ceiling !== null && ($deadline === null || $ceiling->lt($deadline))) { + $deadline = CarbonImmutable::instance($ceiling); + } + } + return [ + 'active' => ! $run->status->isTerminal() && ($deadline === null || now()->lt($deadline)), + 'deadline_at' => $deadline?->toISOString(), + ]; + } + + /** + * @param array $scopes + */ + private static function assertScope(array $scopes, string $scopeId): void + { + if ($scopeId === CancellationScopeHistory::ROOT_SCOPE_ID) { + throw new LogicException('root_scope_requires_run_cancellation'); + } + if (! isset($scopes[$scopeId])) { + throw new LogicException('cancellation_scope_not_recorded'); + } + } + + private static function acceptedEvent(WorkflowRun $run, string $scopeId): ?WorkflowHistoryEvent + { + $matching = $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeRequested) + ->get() + ->filter( + static fn (WorkflowHistoryEvent $event): bool => ($event->payload['scope_id'] ?? null) === $scopeId + ); + if ($matching->count() > 1) { + throw new LogicException('cancellation_scope_request_history_invalid'); + } + return $matching->first(); + } + + private static function parentContext( + WorkflowRun $run, + string $scopeId, + string $parentScopeId + ): ScopedCancellationContext { + $scope = CancellationScopeHistory::forRun($run)[$scopeId] ?? null; + if ($scope === null || $scope['parent_scope_id'] !== $parentScopeId) { + throw new LogicException('cancellation_scope_parent_mismatch'); + } + $parent = $parentScopeId === CancellationScopeHistory::ROOT_SCOPE_ID + ? (($context = self::runContext($run)) === null ? null : ScopedCancellationContext::fromRunContext( + $context + )) + : self::context($run, $parentScopeId); + if ($parent === null) { + throw new LogicException('cancellation_scope_parent_request_unavailable'); + } + return $parent; + } + + private static function runContext(WorkflowRun $run): ?CancellationContext + { + if ($run->cancellation_request_command_id === null) { + return null; + } + /** @var WorkflowCommand|null $command */ + $command = ConfiguredV2Models::query('command_model', WorkflowCommand::class) + ->find($run->cancellation_request_command_id); + $context = $command === null ? null : (new CommandResult($command))->cancellationContext(); + $last = $context === null ? null : $context->lineage[count($context->lineage) - 1]; + if ($command === null || $command->status !== CommandStatus::Accepted + || $command->command_type !== CommandType::RequestCancellation || $command->workflow_run_id !== $run->id + || $context === null || $context->requestId !== $command->id + || $last['workflow_run_id'] !== $run->id || $last['workflow_instance_id'] !== $run->workflow_instance_id + || $run->cancellation_deadline_at === null || ! $context->deadline() + ->equalTo($run->cancellation_deadline_at)) { + throw new LogicException('cancellation_scope_run_context_mismatch'); + } + return $context; + } + + private static function directContext( + WorkflowRun $run, + string $scopeId, + string $requestId, + int $timeout, + ?string $reason, + ?CommandContext $caller + ): ScopedCancellationContext { + $attributes = ($caller ?? CommandContext::phpApi())->attributes(); + $requestedAt = CarbonImmutable::instance(now()); + $root = CancellationContext::fromArray([ + 'schema' => 'durable-workflow.cancellation-context/v1', + 'request_id' => $requestId, + 'root_request_id' => $requestId, + 'root_workflow_instance_id' => $run->workflow_instance_id, + 'root_workflow_run_id' => $run->id, + 'parent_request_id' => null, + 'reason' => $reason, + 'requester' => array_intersect_key( + $attributes['context']['principal'] ?? $attributes['context']['caller'], + array_flip(['type', 'id', 'label']), + ), + 'source' => $attributes['source'], + 'requested_at' => $requestedAt->toISOString(), + 'cleanup_deadline_at' => $requestedAt->addSeconds($timeout) + ->toISOString(), + 'lineage' => [[ + 'request_id' => $requestId, + 'workflow_instance_id' => $run->workflow_instance_id, + 'workflow_run_id' => $run->id, + ]], + ]); + return ScopedCancellationContext::fromArray([ + 'schema' => ScopedCancellationContext::SCHEMA, + 'root_context' => $root->toArray(), + 'lineage' => [[ + ...$root->lineage[0], + 'scope_id' => $scopeId, + 'cleanup_deadline_at' => $root->deadline() + ->toISOString(), + ]], + ]); + } +} diff --git a/src/V2/Support/ChildCallService.php b/src/V2/Support/ChildCallService.php index a8a5c8903..076147cef 100644 --- a/src/V2/Support/ChildCallService.php +++ b/src/V2/Support/ChildCallService.php @@ -5,6 +5,7 @@ namespace Workflow\V2\Support; use Illuminate\Support\Facades\DB; +use Workflow\V2\Enums\CancellationPolicy; use Workflow\V2\Enums\ChildCallStatus; use Workflow\V2\Enums\ParentClosePolicy; use Workflow\V2\Models\WorkflowChildCall; @@ -43,7 +44,10 @@ public function scheduleChild( 'connection' => $options->connection ?? $parentRun->connection, 'queue' => $options->queue ?? $parentRun->queue, 'compatibility' => $parentRun->compatibility, - 'cancellation_propagation' => false, // Future expansion + 'cancellation_propagation' => $options->cancellationPolicy !== CancellationPolicy::Abandon, + 'metadata' => [ + 'cancellation_policy' => $options->cancellationPolicy->value, + ], 'retry_policy' => null, // Future expansion 'timeout_policy' => null, // Future expansion 'arguments' => $call->arguments, @@ -204,7 +208,10 @@ public function enforceParentClosePolicy(WorkflowRun $parentRun): array foreach ($openChildren as $childCall) { match ($childCall->parent_close_policy) { ParentClosePolicy::Abandon => $this->handleAbandon($childCall, $stats), - ParentClosePolicy::RequestCancel => $this->handleRequestCancel($childCall, $stats), + ParentClosePolicy::RequestCancel, ParentClosePolicy::RequestCancellation => $this->handleRequestCancel( + $childCall, + $stats, + ), ParentClosePolicy::Terminate => $this->handleTerminate($childCall, $stats), }; } diff --git a/src/V2/Support/ChildCancellation.php b/src/V2/Support/ChildCancellation.php new file mode 100644 index 000000000..9c2a8cdec --- /dev/null +++ b/src/V2/Support/ChildCancellation.php @@ -0,0 +1,108 @@ +payload['cancellation_policy'] ?? CancellationPolicy::Abandon->value; + + return is_string($value) + ? CancellationPolicy::from($value) + : throw new LogicException('Recorded child cancellation policy is invalid.'); + } + + /** + * True means wait for canonical child terminal history before delivering to parent code. + */ + public static function prepare(WorkflowRun $run, WorkflowTask $task, int $sequence): bool + { + $policy = self::policy($run, $sequence); + if ($policy === CancellationPolicy::Abandon) { + return false; + } + $child = ChildRunHistory::childRunForSequence($run, $sequence); + if (! $child instanceof WorkflowRun) { + return false; + } + $context = CooperativeCancellationDelivery::context($run); + if ($context === null) { + throw new LogicException('Child cancellation propagation requires a canonical cancellation context.'); + } + $run->loadMissing('historyEvents'); + $recorded = $run->historyEvents->first(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::ChildCancellationRequested + && ($event->payload['sequence'] ?? null) === $sequence + && ($event->payload['child_workflow_run_id'] ?? null) === $child->id + && $event->workflow_command_id === $run->cancellation_request_command_id); + if (! $recorded instanceof WorkflowHistoryEvent && ! $child->status->isTerminal()) { + $request = WorkflowStub::loadRun($child->id)->attemptRequestCancellationFromParent($run->id); + if ($request->rejected() && ! in_array($request->rejectionReason(), [ + 'cancellation_root_conflict', 'run_not_active', + ], true)) { + throw new LogicException('Child cooperative cancellation refused: ' . $request->rejectionReason()); + } + $child->refresh(); + $childContext = CooperativeCancellationDelivery::context($child); + $event = WorkflowHistoryEvent::record($run, HistoryEventType::ChildCancellationRequested, [ + 'sequence' => $sequence, + 'policy' => $policy->value, + 'parent_request_id' => $context->requestId, + 'root_request_id' => $context->rootRequestId, + 'cleanup_deadline_at' => $context->deadline() + ->toISOString(), + 'child_workflow_instance_id' => $child->workflow_instance_id, + 'child_workflow_run_id' => $child->id, + 'child_request_id' => $childContext?->requestId, + 'child_root_request_id' => $childContext?->rootRequestId, + 'child_cleanup_deadline_at' => $childContext?->deadline() + ->toISOString(), + 'request_outcome' => $request->accepted() ? 'accepted' : 'rejected', + 'rejection_reason' => $request->rejectionReason(), + ], $task, $run->cancellation_request_command_id); + $run->historyEvents->push($event); + } + $terminal = ChildRunHistory::terminalEventForRun($child); + if ($terminal !== null && ! $run->historyEvents->contains(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::ChildCancellationResolved + && ($event->payload['sequence'] ?? null) === $sequence + && ($event->payload['child_workflow_run_id'] ?? null) === $child->id + && $event->workflow_command_id === $run->cancellation_request_command_id)) { + $childContext = CooperativeCancellationDelivery::context($child); + $event = WorkflowHistoryEvent::record($run, HistoryEventType::ChildCancellationResolved, [ + 'sequence' => $sequence, + 'policy' => $policy->value, + 'parent_request_id' => $context->requestId, + 'root_request_id' => $context->rootRequestId, + 'cleanup_deadline_at' => $context->deadline() + ->toISOString(), + 'child_workflow_instance_id' => $child->workflow_instance_id, + 'child_workflow_run_id' => $child->id, + 'child_request_id' => $childContext?->requestId, + 'child_root_request_id' => $childContext?->rootRequestId, + 'child_cleanup_deadline_at' => $childContext?->deadline() + ->toISOString(), + 'child_status' => ChildRunHistory::resolvedStatus(null, $child)?->value, + 'child_terminal_history_event_id' => $terminal->id, + 'child_terminal_event_type' => $terminal->event_type->value, + 'reason' => $terminal->payload['reason'] ?? null, + ], $task, $run->cancellation_request_command_id); + $run->historyEvents->push($event); + } + + return $policy === CancellationPolicy::WaitCancellationCompleted && $terminal === null; + } +} diff --git a/src/V2/Support/ChildRunHistory.php b/src/V2/Support/ChildRunHistory.php index 642316c25..dc509a1bf 100644 --- a/src/V2/Support/ChildRunHistory.php +++ b/src/V2/Support/ChildRunHistory.php @@ -157,7 +157,7 @@ public static function markChildCallClosedByParentPolicy( match ($policy) { ParentClosePolicy::RequestCancel => $childCall->markCancelled(), ParentClosePolicy::Terminate => $childCall->markTerminated(), - ParentClosePolicy::Abandon => null, + ParentClosePolicy::Abandon, ParentClosePolicy::RequestCancellation => null, }; }); } @@ -666,6 +666,30 @@ public static function exceptionForChildRun(?WorkflowRun $childRun): Throwable ); } + public static function terminalEventForRun(?WorkflowRun $childRun): ?WorkflowHistoryEvent + { + if (! $childRun instanceof WorkflowRun) { + return null; + } + + $childRun->loadMissing('historyEvents'); + + /** @var WorkflowHistoryEvent|null $event */ + $event = $childRun->historyEvents + ->filter( + static fn (WorkflowHistoryEvent $event): bool => in_array($event->event_type, [ + HistoryEventType::WorkflowCompleted, + HistoryEventType::WorkflowFailed, + HistoryEventType::WorkflowCancelled, + HistoryEventType::WorkflowTerminated, + ], true) + ) + ->sortByDesc('sequence') + ->first(); + + return $event; + } + private static function afterAuthoritativeCommit(callable $projection): void { $callback = static function () use ($projection): void { @@ -716,30 +740,6 @@ private static function parentPerspectiveMessage( return sprintf('Child workflow %s closed as %s.', $childIdentity, $statusLabel); } - private static function terminalEventForRun(?WorkflowRun $childRun): ?WorkflowHistoryEvent - { - if (! $childRun instanceof WorkflowRun) { - return null; - } - - $childRun->loadMissing('historyEvents'); - - /** @var WorkflowHistoryEvent|null $event */ - $event = $childRun->historyEvents - ->filter( - static fn (WorkflowHistoryEvent $event): bool => in_array($event->event_type, [ - HistoryEventType::WorkflowCompleted, - HistoryEventType::WorkflowFailed, - HistoryEventType::WorkflowCancelled, - HistoryEventType::WorkflowTerminated, - ], true) - ) - ->sortByDesc('sequence') - ->first(); - - return $event; - } - private static function workflowStartedEvent(WorkflowRun $run): ?WorkflowHistoryEvent { $run->loadMissing('historyEvents'); diff --git a/src/V2/Support/ChildWorkflowOptions.php b/src/V2/Support/ChildWorkflowOptions.php index b4b6f1cc3..4c94cbc98 100644 --- a/src/V2/Support/ChildWorkflowOptions.php +++ b/src/V2/Support/ChildWorkflowOptions.php @@ -4,6 +4,7 @@ namespace Workflow\V2\Support; +use Workflow\V2\Enums\CancellationPolicy; use Workflow\V2\Enums\ParentClosePolicy; /** @@ -19,6 +20,7 @@ public function __construct( public readonly ParentClosePolicy $parentClosePolicy = ParentClosePolicy::Abandon, public readonly ?string $connection = null, public readonly ?string $queue = null, + public readonly CancellationPolicy $cancellationPolicy = CancellationPolicy::Abandon, ) { } @@ -27,6 +29,7 @@ public function __construct( * parent_close_policy: string, * connection: string|null, * queue: string|null, + * cancellation_policy: string, * } */ public function toSnapshot(): array @@ -35,6 +38,7 @@ public function toSnapshot(): array 'parent_close_policy' => $this->parentClosePolicy->value, 'connection' => $this->connection, 'queue' => $this->queue, + 'cancellation_policy' => $this->cancellationPolicy->value, ]; } diff --git a/src/V2/Support/ConditionWaits.php b/src/V2/Support/ConditionWaits.php index 7fbcbaf2c..5284a33b1 100644 --- a/src/V2/Support/ConditionWaits.php +++ b/src/V2/Support/ConditionWaits.php @@ -21,6 +21,7 @@ final class ConditionWaits HistoryEventType::TimerFired, HistoryEventType::ConditionWaitSatisfied, HistoryEventType::ConditionWaitTimedOut, + HistoryEventType::ConditionWaitCancelled, HistoryEventType::SelectionOperationCancelled, HistoryEventType::WorkflowCompleted, HistoryEventType::WorkflowFailed, @@ -140,6 +141,10 @@ public static function forRun(WorkflowRun $run): array $timerId = self::stringValue($event->payload['timer_id'] ?? null); + if (($waits[$waitId]['status'] ?? null) === 'cancelled') { + continue; + } + $waits[$waitId]['source_status'] = 'timeout_fired'; $waits[$waitId]['sequence'] = self::intValue($waits[$waitId]['sequence'] ?? null) ?? self::intValue($event->payload['sequence'] ?? null); @@ -169,6 +174,24 @@ public static function forRun(WorkflowRun $run): array continue; } + if ($event->event_type === HistoryEventType::ConditionWaitCancelled) { + $waitId = self::waitIdForEvent($event); + if ($waitId !== null && isset($waits[$waitId]) && $waits[$waitId]['status'] === 'open') { + $waits[$waitId]['status'] = 'cancelled'; + $waits[$waitId]['source_status'] = 'scope_cancelled'; + $waits[$waitId]['resolved_at'] = $event->recorded_at ?? $event->created_at; + $waits[$waitId]['resume_source_kind'] = 'scope_cancellation'; + $waits[$waitId]['resume_source_id'] = self::stringValue( + $event->payload['cancellation_scope']['request_id'] ?? null + ); + $index = array_search($waitId, $openWaitIds, true); + if ($index !== false) { + array_splice($openWaitIds, $index, 1); + } + } + continue; + } + if (! in_array($event->event_type, [ HistoryEventType::ConditionWaitSatisfied, HistoryEventType::ConditionWaitTimedOut, @@ -200,6 +223,10 @@ public static function forRun(WorkflowRun $run): array $waits[$waitId] = self::wait($waitId, $event); } + if (($waits[$waitId]['status'] ?? null) === 'cancelled') { + continue; + } + $waits[$waitId]['status'] = 'resolved'; $waits[$waitId]['source_status'] = $event->event_type === HistoryEventType::ConditionWaitTimedOut ? 'timed_out' diff --git a/src/V2/Support/CooperativeCancellationDelivery.php b/src/V2/Support/CooperativeCancellationDelivery.php index edcdbb753..b5600ee86 100644 --- a/src/V2/Support/CooperativeCancellationDelivery.php +++ b/src/V2/Support/CooperativeCancellationDelivery.php @@ -4,6 +4,7 @@ namespace Workflow\V2\Support; +use Workflow\V2\CancellationContext; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Exceptions\HistoryEventShapeMismatchException; use Workflow\V2\Models\WorkflowHistoryEvent; @@ -56,16 +57,15 @@ public static function validate( return 'cancellation_request_mismatch'; } - $limit = StructuralLimits::commandBatchSizeLimit(); - if ($sequence < 1 || $sequenceSpan < 1 || $operationSequenceSpan < 1 - || $sequence > PHP_INT_MAX - $sequenceSpan - || ($limit > 0 && $sequenceSpan > $limit) - || ($limit > 0 && $operationSequenceSpan > $limit) - || (! isset(self::CALL_SHAPES[$callKind]) && ! in_array($callKind, ['parallel', 'selection_handle'], true)) - || ($callKind !== 'parallel' && $sequenceSpan !== 1) - || ($callKind === 'selection_handle') !== ($operationSequence !== null) - || ($callKind !== 'selection_handle' && $operationSequenceSpan !== 1)) { - return 'invalid_cancellation_delivery'; + $invalid = self::validateBoundarySyntax( + $sequence, + $callKind, + $sequenceSpan, + $operationSequence, + $operationSequenceSpan + ); + if ($invalid !== null) { + return $invalid; } if (! $run->relationLoaded('historyEvents')) { @@ -81,6 +81,15 @@ public static function validate( return 'cancellation_request_history_missing'; } + $membership = self::validateRootOperationMembership( + $run, + $operationSequence ?? $sequence, + $operationSequence === null ? $sequenceSpan : $operationSequenceSpan, + ); + if ($membership !== null) { + return $membership; + } + $existing = self::recorded($run); if ($existing instanceof WorkflowHistoryEvent) { return ($existing->workflow_command_id === $requestId @@ -96,6 +105,67 @@ public static function validate( return 'cancellation_delivery_history_missing'; } + $waitingBoundary = ActivityCancellationWait::validateBoundary( + $run, + $sequence, + $callKind, + $sequenceSpan, + $operationSequence, + $operationSequenceSpan + ); + if ($waitingBoundary !== null) { + return $waitingBoundary; + } + + return self::validateCallBoundary( + $run, + $request, + $sequence, + $callKind, + $sequenceSpan, + $operationSequence, + $operationSequenceSpan + ); + } + + /** + * @internal Shared syntax gate for run and scoped delivery. + */ + public static function validateBoundarySyntax( + int $sequence, + string $callKind, + int $sequenceSpan = 1, + ?int $operationSequence = null, + int $operationSequenceSpan = 1, + ): ?string { + $limit = StructuralLimits::commandBatchSizeLimit(); + return ($sequence < 1 || $sequenceSpan < 1 || $operationSequenceSpan < 1 + || $sequence > PHP_INT_MAX - $sequenceSpan + || ($limit > 0 && $sequenceSpan > $limit) + || ($limit > 0 && $operationSequenceSpan > $limit) + || (! isset(self::CALL_SHAPES[$callKind]) && ! in_array($callKind, ['parallel', 'selection_handle'], true)) + || ($callKind !== 'parallel' && $sequenceSpan !== 1) + || ($callKind === 'selection_handle') !== ($operationSequence !== null) + || ($callKind !== 'selection_handle' && $operationSequenceSpan !== 1)) + ? 'invalid_cancellation_delivery' : null; + } + + /** + * @internal Validate durable call ordering against the selected canonical request. + * Identity, syntax, scope membership, claim and authority are separate gates. + */ + public static function validateCallBoundary( + WorkflowRun $run, + WorkflowHistoryEvent $request, + int $sequence, + string $callKind, + int $sequenceSpan = 1, + ?int $operationSequence = null, + int $operationSequenceSpan = 1, + ): ?string { + if (! $run->relationLoaded('historyEvents')) { + $run->loadMissing('historyEvents'); + } $nextSequence = WorkflowStepHistory::nextDurableCommandSequence($run); if ($sequence > $nextSequence) { return 'cancellation_delivery_sequence_mismatch'; @@ -151,6 +221,21 @@ public static function recorded(WorkflowRun $run): ?WorkflowHistoryEvent ); } + public static function context(WorkflowRun $run): ?CancellationContext + { + if (! $run->relationLoaded('historyEvents')) { + $run->loadMissing('historyEvents'); + } + $request = $run->historyEvents->first( + static fn (WorkflowHistoryEvent $event): bool => $event->event_type + === HistoryEventType::CooperativeCancellationRequested + && $event->workflow_command_id === $run->cancellation_request_command_id, + ); + $snapshot = $request?->payload['cancellation'] ?? null; + + return is_array($snapshot) ? CancellationContext::fromArray($snapshot) : null; + } + public static function record( WorkflowRun $run, WorkflowTask $task, @@ -177,6 +262,10 @@ public static function record( 'sequence' => $sequence, 'call_kind' => $callKind, ]; + $context = self::context($run); + if ($context !== null) { + $payload['cancellation'] = $context->toArray(); + } if ($sequenceSpan !== 1) { $payload['sequence_span'] = $sequenceSpan; } @@ -228,6 +317,52 @@ public static function callKindAt(WorkflowRun $run, int $sequence): ?string return null; } + /** + * Root delivery actors have no frozen scoped inventory. Recorded scoped + * operations must use scope preparation and delivery, including a group + * containing both root and scoped leaves. Check before receipt replay so + * a changed membership cannot reuse an earlier root delivery marker. + */ + private static function validateRootOperationMembership(WorkflowRun $run, int $start, int $span): ?string + { + foreach ($run->historyEvents as $event) { + $payload = $event->payload; + $sequence = $payload['sequence'] ?? null; + if (! is_int($sequence) || $sequence < $start || $sequence - $start >= $span) { + continue; + } + $descriptorKey = match ($event->event_type) { + HistoryEventType::ActivityScheduled => 'activity', + HistoryEventType::TimerScheduled => 'timer', + HistoryEventType::ChildWorkflowScheduled => 'child_workflow', + HistoryEventType::ConditionWaitOpened, HistoryEventType::SignalWaitOpened => '', + default => null, + }; + if ($descriptorKey === null) { + continue; + } + $descriptor = []; + if ($descriptorKey !== '' && array_key_exists($descriptorKey, $payload)) { + $descriptor = $payload[$descriptorKey]; + if (! is_array($descriptor) || ($descriptor !== [] && array_is_list($descriptor))) { + return 'cancellation_delivery_scope_membership_invalid'; + } + } + $hasFlat = array_key_exists('cancellation_scope_id', $payload); + $hasNested = array_key_exists('cancellation_scope_id', $descriptor); + $scope = $hasFlat ? $payload['cancellation_scope_id'] + : ($hasNested ? $descriptor['cancellation_scope_id'] : CancellationScopeHistory::ROOT_SCOPE_ID); + if (! is_string($scope) || $scope === '' + || ($hasNested && $descriptor['cancellation_scope_id'] !== $scope)) { + return 'cancellation_delivery_scope_membership_invalid'; + } + if ($scope !== CancellationScopeHistory::ROOT_SCOPE_ID) { + return 'cancellation_delivery_requires_scope'; + } + } + return null; + } + private static function resolvedBeforeRequest( WorkflowRun $run, int $sequence, diff --git a/src/V2/Support/DefaultActivityTaskBridge.php b/src/V2/Support/DefaultActivityTaskBridge.php index d39f872cc..3704b61ba 100644 --- a/src/V2/Support/DefaultActivityTaskBridge.php +++ b/src/V2/Support/DefaultActivityTaskBridge.php @@ -455,8 +455,10 @@ private static function attemptContinuationState( ?WorkflowRun $run, ?WorkflowTask $task, ): array { + $abandoned = $run instanceof WorkflowRun && $execution instanceof ActivityExecution + && ActivityAbandonment::allows($run, $execution); if ($attempt->status !== ActivityAttemptStatus::Running) { - if ($run instanceof WorkflowRun && $run->status === RunStatus::Cancelled) { + if ($run instanceof WorkflowRun && $run->status === RunStatus::Cancelled && ! $abandoned) { return [false, true, 'run_cancelled']; } @@ -475,7 +477,7 @@ private static function attemptContinuationState( return [false, false, 'workflow_run_missing']; } - if ($run->status === RunStatus::Cancelled) { + if ($run->status === RunStatus::Cancelled && ! $abandoned) { return [false, true, 'run_cancelled']; } @@ -491,10 +493,14 @@ private static function attemptContinuationState( return [false, true, 'task_cancelled']; } - if ($run->status->isTerminal()) { + if ($run->status->isTerminal() && ! $abandoned) { return [false, false, 'run_closed']; } + if ($abandoned && ActivityTimeoutEnforcer::hasExpiredDeadline($execution, now())) { + return [false, false, 'activity_deadline_elapsed']; + } + if ($execution->status !== ActivityStatus::Running) { return [false, false, 'activity_not_running']; } diff --git a/src/V2/Support/DefaultWorkflowTaskBridge.php b/src/V2/Support/DefaultWorkflowTaskBridge.php index 9ce1459c2..f22fc0388 100644 --- a/src/V2/Support/DefaultWorkflowTaskBridge.php +++ b/src/V2/Support/DefaultWorkflowTaskBridge.php @@ -9,18 +9,23 @@ use Illuminate\Support\Str; use Illuminate\Validation\ValidationException; use InvalidArgumentException; +use JsonException; use LogicException; use RuntimeException; use Throwable; use Workflow\Serializers\CodecDecodeException; use Workflow\Serializers\CodecRegistry; use Workflow\Serializers\Serializer; +use Workflow\V2\Contracts\CancellationScopeAdmission; use Workflow\V2\Contracts\CooperativeWorkflowTaskBridge; use Workflow\V2\Contracts\HistoryProjectionRole; +use Workflow\V2\Contracts\PreparedCancellationScopeTaskBridge; +use Workflow\V2\Contracts\PreparedLocalActivityGroupTaskBridge; use Workflow\V2\Contracts\ServiceControlPlane; use Workflow\V2\Contracts\WorkflowControlPlane; use Workflow\V2\Enums\ActivityAttemptStatus; use Workflow\V2\Enums\ActivityStatus; +use Workflow\V2\Enums\CancellationPolicy; use Workflow\V2\Enums\ChildCallStatus; use Workflow\V2\Enums\CommandOutcome; use Workflow\V2\Enums\FailureCategory; @@ -50,7 +55,7 @@ use Workflow\V2\Models\WorkflowTimer; use Workflow\V2\Models\WorkflowUpdate; -final class DefaultWorkflowTaskBridge implements CooperativeWorkflowTaskBridge +final class DefaultWorkflowTaskBridge implements CooperativeWorkflowTaskBridge, PreparedLocalActivityGroupTaskBridge, CancellationScopeAdmission, PreparedCancellationScopeTaskBridge { public const POLL_BATCH_CAP = 100; @@ -316,7 +321,7 @@ public function claimStatus(string $taskId, ?string $leaseOwner = null): array } $resolvedLeaseOwner = $leaseOwner ?? $taskId; - $leaseExpiresAt = WorkflowTaskLease::expiresAt(); + $leaseExpiresAt = CancellationCleanupLease::expiresAt($run); $stickyReplayMode = StickyExecution::claimReplayMode($task, $resolvedLeaseOwner); $task->forceFill([ @@ -688,6 +693,23 @@ public function fail(string $taskId, Throwable|array|string $failure, ?string $c }); } + /** + * @internal Portable remote policy admission marker for an installed bridge. + */ + public function supportsRemoteActivityCancellationPolicies(): bool + { + return true; + } + + /** + * @internal Policy admission supported by this installed prepared-local bridge. + * @return list + */ + public function supportedLocalActivityCancellationPolicies(): array + { + return [CancellationPolicy::TryCancel->value, CancellationPolicy::WaitCancellationCompleted->value]; + } + public function deliverCancellation( string $taskId, string $requestId, @@ -719,6 +741,9 @@ public function deliverCancellation( 'operation_sequence' => $event?->payload['operation_sequence'] ?? null, 'operation_sequence_span' => $event === null ? null : ($event->payload['operation_sequence_span'] ?? 1), 'reason' => $reason, + ...(in_array($reason, ['cancellation_waiting_for_child', 'cancellation_waiting_for_activity'], true) ? [ + 'claim_released' => true, + ] : []), ]; if ($task === null || $task->task_type !== TaskType::Workflow) { return $response($task === null ? 'task_not_found' : 'task_not_workflow'); @@ -763,9 +788,15 @@ public function deliverCancellation( $operationSequence, $operationSequenceSpan, ); + if ($event instanceof WorkflowHistoryEvent) { + LocalActivityRuntime::renewWorkflowTask($task, CancellationCleanupLease::deadline($run)); + } self::projectRun($run, self::PROJECTION_RUN_RELATIONS); - return $response(null, $run, $event); + return $response($event === null ? ( + isset($task->payload['cancellation_activity_wait']) + ? 'cancellation_waiting_for_activity' : 'cancellation_waiting_for_child' + ) : null, $run, $event); }); } @@ -825,7 +856,7 @@ public function heartbeat(string $taskId): array ]; } - $leaseExpiresAt = WorkflowTaskLease::expiresAt(); + $leaseExpiresAt = CancellationCleanupLease::expiresAt($run); $task->forceFill([ 'lease_expires_at' => $leaseExpiresAt, @@ -914,6 +945,276 @@ public function status(string $taskId): array ]; } + /** + * @internal Candidate scope registration before workflow body execution. + * @return array + */ + public function openCancellationScope( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + string $parentScopeId = 'root', + bool $shieldParent = false, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return PortableCancellationScopeOpening::open( + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $sequence, + $parentScopeId, + $shieldParent, + $protocolVersion + ); + } + + public function prepareCancellationScopeDelivery( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + string $scopeId, + string $requestId, + int $sequence, + string $callKind, + int $sequenceSpan = 1, + ?int $operationSequence = null, + int $operationSequenceSpan = 1, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return PortableCancellationScopeDelivery::mutate( + true, + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $scopeId, + $requestId, + $sequence, + $callKind, + $sequenceSpan, + $operationSequence, + $operationSequenceSpan, + $protocolVersion + ); + } + + public function deliverCancellationScope( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + string $scopeId, + string $requestId, + int $sequence, + string $callKind, + int $sequenceSpan = 1, + ?int $operationSequence = null, + int $operationSequenceSpan = 1, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return PortableCancellationScopeDelivery::mutate( + false, + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $scopeId, + $requestId, + $sequence, + $callKind, + $sequenceSpan, + $operationSequence, + $operationSequenceSpan, + $protocolVersion + ); + } + + /** + * @internal Candidate prepared local callback admission. + * @param array $descriptor + * @return array + */ + public function prepareLocalActivity( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + string $workerAttemptId, + array $descriptor, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return PortableLocalActivityPreparation::prepare( + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $sequence, + $workerAttemptId, + $descriptor, + $protocolVersion + ); + } + + public function recordLocalActivityOutcome( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + array $report, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return app(LocalActivityExecutor::class)->recordPortableOutcome( + $attemptId, + $leaseOwner, + $workflowTaskAttempt, + $report, + $protocolVersion + ); + } + + public function controlLocalActivity( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + bool $renewLease = false, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return PortableLocalActivityControl::poll( + $attemptId, + $leaseOwner, + $workflowTaskAttempt, + $renewLease, + $protocolVersion, + ); + } + + public function recoverLocalActivity( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + array $descriptor, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return PortableLocalActivityPreparation::recover( + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $sequence, + $descriptor, + $protocolVersion + ); + } + + public function heartbeatLocalActivity( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + array $progress = [], + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return PortableLocalActivityControl::heartbeat( + $attemptId, + $leaseOwner, + $workflowTaskAttempt, + $progress, + $protocolVersion, + ); + } + + public function acknowledgeLocalActivityCancellation( + string $attemptId, + string $leaseOwner, + string $requestId, + int $workflowTaskAttempt, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + if (preg_match('/^[0-9]+\.[0-9]+$/D', $protocolVersion) !== 1 + || version_compare($protocolVersion, PortableLocalActivityPreparation::MINIMUM_PROTOCOL_VERSION, '<')) { + return [ + 'acknowledged' => false, + 'duplicate' => false, + 'reason' => 'local_activity_stop_receipt_requires_protocol_1_20', + ]; + } + return ActivityCancellationAcknowledgement::recordLocalStopped( + $attemptId, + $leaseOwner, + $requestId, + $workflowTaskAttempt + ); + } + + /** + * @internal Candidate local callback prefix checkpoint. This retains the + * claim and is not callback admission. Preparation must follow separately. + * The SDK must replay committed history before submitting later commands. + * + * Only the latest sequential checkpoint receipt is retained on the claim. + * An SDK must acknowledge it before sending a subsequent checkpoint. + * + * @param list $commands + * @return array + */ + public function checkpointLocalActivityPrefix( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + string $checkpointId, + int $startSequence, + array $commands, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return $this->checkpointCommands( + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $checkpointId, + $startSequence, + $commands, + $protocolVersion, + 'local' + ); + } + + public function checkpointLocalActivityGroup( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + string $checkpointId, + int $startSequence, + array $commands, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return $this->checkpointCommands( + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $checkpointId, + $startSequence, + $commands, + $protocolVersion, + 'local_group' + ); + } + + public function checkpointCancellationScopePrefix( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + string $checkpointId, + int $startSequence, + array $commands, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return $this->checkpointCommands( + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $checkpointId, + $startSequence, + $commands, + $protocolVersion, + 'cancellation_scope' + ); + } + public function complete(string $taskId, array $commands): array { $parsed = self::parseCommands($commands); @@ -1029,6 +1330,17 @@ public function complete(string $taskId, array $commands): array $sequence = WorkflowStepHistory::nextDurableCommandSequence($run); $createdTaskIds = []; + $scopeError = $this->validateCancellationScopeMembership($run, $parsed['non_terminal'], $sequence); + if ($scopeError !== null) { + return [ + 'completed' => false, + 'task_id' => $taskId, + 'workflow_run_id' => $run->id, + 'run_status' => $run->status->value, + 'created_task_ids' => [], + 'reason' => $scopeError, + ]; + } $invalidUpdateCommands = $this->validateUpdateCommands($run, $task, $parsed['non_terminal']); if ($invalidUpdateCommands !== null) { @@ -1106,6 +1418,381 @@ public function complete(string $taskId, array $commands): array }); } + /** + * @param list $commands + */ + public function validateCancellationScopeMembership(WorkflowRun $run, array $commands, int $sequence): ?string + { + if (! self::operationScopesAreRecorded($run, $commands, $sequence)) { + return 'operation_scope_not_recorded'; + } + foreach ($commands as $offset => $command) { + if ($command['type'] === 'start_timer') { + $cleanupError = ScopedCancellationCleanup::timerRefusal($run, $command, $sequence + $offset); + if ($cleanupError !== null) { + return $cleanupError; + } + } + // Implicit-root commands also share the authored cursor. A pending + // scoped delivery must consume its reserved position before any + // unrelated operation or metadata write can take that position. + $refusal = CancellationScopeDelivery::admissionRefusal( + $run, + $command['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, + $sequence + $offset, + in_array($command['type'], ['prepare_local_activity', 'start_timer'], true) + ? ($command['cancellation_cleanup'] ?? null) : null, + ); + if ($refusal !== null) { + return $refusal; + } + if ($command['type'] === 'prepare_local_activity' + && ! CancellationScopeHistory::isRecordedBefore( + $run, + $command['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, + $sequence + $offset, + )) { + return 'local_activity_scope_not_recorded'; + } + } + return null; + } + + /** @param list $commands + * @return array + */ + private function checkpointCommands( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + string $checkpointId, + int $startSequence, + array $commands, + string $protocolVersion, + string $kind, + ): array { + $group = $kind === 'local_group'; + $scopePrefix = $kind === 'cancellation_scope'; + $checkpointName = $scopePrefix ? 'cancellation_scope' : 'local_activity'; + $refused = static fn (string $reason): array => [ + 'checkpointed' => false, + 'duplicate' => false, + 'task_id' => $taskId, + 'reason' => $reason, + ]; + if (($scopePrefix && ! WorkerProtocolVersion::supportsCancellationScopeMembership($protocolVersion)) + || preg_match('/^[0-9]+\.[0-9]+$/D', $protocolVersion) !== 1 + || version_compare($protocolVersion, PortableLocalActivityPreparation::MINIMUM_PROTOCOL_VERSION, '<')) { + return $refused($checkpointName . '_checkpoint_requires_protocol_1_20'); + } + if ($startSequence < 1 || $workflowTaskAttempt < 1 || $leaseOwner === '' + || trim($checkpointId) === '' || strlen($checkpointId) > 255 + || preg_match('//u', $checkpointId) !== 1 || ! array_is_list($commands) + || count($commands) > 100) { + return $refused('invalid_' . $checkpointName . '_checkpoint'); + } + $parsed = $group ? self::parsePreparedLocalGroupCommands($commands) : ($commands === [] ? [ + 'non_terminal' => [], + 'terminal' => null, + ] : self::parseCommands($commands)); + if ($parsed === null || $parsed['terminal'] !== null) { + return $refused('invalid_' . $checkpointName . '_checkpoint_commands'); + } + foreach ($parsed['non_terminal'] as $command) { + if (isset($command['cancellation_scope_id']) + && ! WorkerProtocolVersion::supportsCancellationScopeMembership($protocolVersion)) { + return $refused('operation_scope_requires_protocol_1_20'); + } + // A wait closes a turn, a local report reconstructs an executed + // callback, and selection cancellation requires activity fencing. + // None belongs in this retained-claim preparation prefix. + if (in_array($command['type'], ['record_local_activity', 'open_condition_wait', + 'open_signal_wait', 'cancel_selection_operation'], true)) { + return $refused('invalid_' . $checkpointName . '_checkpoint_commands'); + } + } + try { + $fingerprint = hash('sha256', json_encode($parsed['non_terminal'], JSON_THROW_ON_ERROR)); + } catch (JsonException) { + return $refused('invalid_' . $checkpointName . '_checkpoint_commands'); + } + /** @var WorkflowTask|null $snapshot */ + $snapshot = ConfiguredV2Models::query('task_model', WorkflowTask::class)->find($taskId); + if ($snapshot === null) { + return $refused('task_not_found'); + } + + return DB::transaction(function () use ( + $snapshot, + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $checkpointId, + $startSequence, + $parsed, + $fingerprint, + $refused, + $group, + $scopePrefix, + $checkpointName, + ): array { + /** @var WorkflowRun|null $run */ + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->lockForUpdate() + ->find($snapshot->workflow_run_id); + /** @var WorkflowTask|null $task */ + $task = ConfiguredV2Models::query('task_model', WorkflowTask::class)->lockForUpdate()->find($taskId); + if ($run === null || $task === null || $task->workflow_run_id !== $run->id) { + return $refused('workflow_claim_not_found'); + } + if ($task->task_type !== TaskType::Workflow || $task->status !== TaskStatus::Leased + || $task->lease_owner !== $leaseOwner || $task->attempt_count !== $workflowTaskAttempt) { + return $refused('workflow_claim_mismatch'); + } + if ($task->lease_expires_at === null || now()->gte($task->lease_expires_at)) { + return $refused('workflow_claim_expired'); + } + if ($run->status->isTerminal()) { + return $refused('run_closed'); + } + if (($run->execution_deadline_at !== null && now()->gte($run->execution_deadline_at)) + || ($run->run_deadline_at !== null && now()->gte($run->run_deadline_at))) { + return $refused('run_deadline_expired'); + } + $payload = is_array($task->payload) ? $task->payload : []; + $receiptKey = $scopePrefix ? 'portable_scope_checkpoint' + : ($group ? 'portable_local_group_checkpoint' : 'portable_local_checkpoint'); + $receipt = $payload[$receiptKey] ?? null; + if (is_array($receipt) && ($receipt['checkpoint_id'] ?? null) === $checkpointId) { + if (($receipt['workflow_task_attempt'] ?? null) !== $workflowTaskAttempt + || ($receipt['lease_owner'] ?? null) !== $leaseOwner + || ($receipt['start_sequence'] ?? null) !== $startSequence + || ($receipt['fingerprint'] ?? null) !== $fingerprint) { + return $refused($checkpointName . '_checkpoint_mismatch'); + } + // This receipt proves prefix commit, not permission to start + // application code. Preparation still checks cancellation. + return [ + ...$receipt, + 'checkpointed' => true, + 'duplicate' => true, + 'reason' => null, + ]; + } + if ($run->cancellation_request_command_id !== null) { + $delivery = $run->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered) + ->where('workflow_command_id', $run->cancellation_request_command_id) + ->first(); + $cleanup = PortableLocalActivityCleanup::snapshot($run, [ + 'request_id' => $run->cancellation_request_command_id, + 'delivery_history_event_id' => $delivery?->id, + ], $startSequence); + if ($cleanup === null) { + // An atomic local group can prove scoped cleanup on every + // member. Ordinary prefixes still need root delivery. + if (! $group || $parsed['non_terminal'] === []) { + return $refused('cancellation_requested'); + } + foreach ($parsed['non_terminal'] as $offset => $command) { + if ($command['type'] !== 'prepare_local_activity') { + return $refused('cancellation_requested'); + } + $memberCleanup = PortableLocalActivityCleanup::snapshot( + $run, + $command['cancellation_cleanup'] ?? null, + $startSequence + $offset, + $command['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, + ); + if (! PortableLocalActivityCleanup::belongsToRunRequest($run, $memberCleanup)) { + return $refused('cancellation_requested'); + } + } + } + if (now()->gte($run->cancellation_deadline_at)) { + return $refused('cancellation_deadline_expired'); + } + } + $sequence = WorkflowStepHistory::nextDurableCommandSequence($run); + if ($sequence !== $startSequence) { + return $refused($checkpointName . '_checkpoint_sequence_mismatch'); + } + $scopeError = $this->validateCancellationScopeMembership($run, $parsed['non_terminal'], $sequence); + if ($scopeError !== null) { + return $refused($scopeError); + } + $invalidUpdate = $this->validateUpdateCommands($run, $task, $parsed['non_terminal']); + if ($invalidUpdate !== null) { + return $refused($invalidUpdate); + } + if (! self::parallelCommandsMatchSequences($parsed['non_terminal'], $sequence, $run)) { + return $refused('invalid_' . $checkpointName . '_checkpoint_commands'); + } + // Validate every local scope/cleanup member before creating any sibling. + foreach ($parsed['non_terminal'] as $offset => $command) { + if ($command['type'] !== 'prepare_local_activity') { + continue; + } + if (! CancellationScopeHistory::isRecordedBefore( + $run, + $command['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, + $sequence + $offset, + )) { + return $refused('local_activity_scope_not_recorded'); + } + $cleanup = PortableLocalActivityCleanup::snapshot( + $run, + $command['cancellation_cleanup'] ?? null, + $sequence + $offset, + $command['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, + ); + if ((isset($command['cancellation_cleanup']) && $cleanup === null) + || ($run->cancellation_request_command_id !== null + && ! PortableLocalActivityCleanup::belongsToRunRequest($run, $cleanup)) + || ($run->cancellation_request_command_id === null && isset($command['cancellation_cleanup']) && ! isset($cleanup['scope_id']))) { + return $refused('local_activity_cleanup_authority_mismatch'); + } + if ($cleanup !== null && now()->gte(PortableLocalActivityCleanup::snapshotDeadline($cleanup))) { + return $refused(isset($cleanup['scope_id']) ? 'local_activity_cleanup_deadline_expired' : 'cancellation_deadline_expired'); + } + if (isset($cleanup['scope_id']) && now()->gte( + PortableLocalActivityCleanup::currentDeadline($run, $cleanup) + )) { + return $refused('local_activity_cleanup_authority_expired'); + } + } + $this->recordAppliedSignalForSignalResume($run, $task); + $this->recordSatisfiedConditionWaitForSignalResume($run, $task, $parsed['non_terminal']); + $createdTaskIds = []; + $localActivities = []; + foreach ($parsed['non_terminal'] as $command) { + if ($command['type'] === 'prepare_local_activity') { + $execution = PortableLocalActivityPreparation::admitGroupMember( + $run, + $task, + $sequence, + [ + ...$command, + 'type' => 'record_local_activity', + ], + $checkpointId, + $fingerprint + ); + $localActivities[] = [ + 'sequence' => $sequence, + 'activity_execution_id' => $execution->id, + ]; + ++$sequence; + continue; + } + $sequence = $this->applyNonTerminalCommand($run, $task, $command, $sequence, $createdTaskIds); + } + // Command application can update task payload (for example signal + // consumption). Preserve that state rather than the old snapshot. + $payload = is_array($task->payload) ? $task->payload : []; + $receipt = [ + 'checkpoint_id' => $checkpointId, + 'task_id' => $taskId, + 'workflow_run_id' => $run->id, + 'workflow_task_attempt' => $workflowTaskAttempt, + 'lease_owner' => $leaseOwner, + 'lease_expires_at' => $task->lease_expires_at->toISOString(), + 'start_sequence' => $startSequence, + 'next_sequence' => $sequence, + 'fingerprint' => $fingerprint, + 'created_task_ids' => $createdTaskIds, + 'recorded_at' => now() + ->toISOString(), + ...($group ? [ + 'local_activities' => $localActivities, + ] : []), + ]; + $task->forceFill([ + 'payload' => [ + ...$payload, + $receiptKey => $receipt, + ], + ])->save(); + + return [ + ...$receipt, + 'checkpointed' => true, + 'duplicate' => false, + 'reason' => null, + ]; + }, 5); + } + + /** + * Local descriptors have no fabricated outcomes. Validate their common + * activity grammar with a scheduling shadow, then restore admission-only + * descriptors. Ordinary task completion does not recognize this command. + * Selection groups remain refused until their stop-policy path is qualified. + * @param list $commands + * @return array{non_terminal: list, terminal: null}|null + */ + private static function parsePreparedLocalGroupCommands(array $commands): ?array + { + $shadows = []; + $locals = []; + try { + foreach ($commands as $offset => $command) { + if (! is_array($command)) { + return null; + } + if (($command['type'] ?? null) === 'prepare_local_activity') { + $local = PortableLocalActivityPreparation::normalizeDescriptor([ + ...$command, + 'type' => 'record_local_activity', + ]); + if (($local['parallel_group_path'] ?? []) === []) { + return null; + } + foreach ($local['parallel_group_path'] as $entry) { + if (($entry['parallel_group_mode'] ?? 'all') !== 'all') { + return null; + } + } + $locals[$offset] = [ + ...$local, + 'type' => 'prepare_local_activity', + ]; + unset($local['execution_mode'], $local['cancellation_cleanup'], $local['cancellation_scope_id']); + $command = [ + ...$local, + 'type' => 'schedule_activity', + ]; + } + $shadows[] = $command; + } + if ($locals === []) { + return null; + } + $normalized = WorkflowCommandNormalizer::normalize( + $shadows, + PortableLocalActivityPreparation::MINIMUM_PROTOCOL_VERSION + ); + foreach ($normalized as $command) { + foreach ($command['parallel_group_path'] ?? [] as $entry) { + if (($entry['parallel_group_mode'] ?? 'all') !== 'all') { + return null; + } + } + } + } catch (ValidationException|InvalidArgumentException) { + return null; + } + $parsed = self::parseCommands($normalized); + if ($parsed === null || $parsed['terminal'] !== null + || count($parsed['non_terminal']) !== count($commands)) { + return null; + } + foreach ($locals as $offset => $local) { + $parsed['non_terminal'][$offset] = $local; + } + return $parsed; + } + /** * @param array{type: string, result?: string|null, payload_codec?: string|null} $command */ @@ -1478,6 +2165,11 @@ private function recordAppliedSignalForSignalResume(WorkflowRun $run, WorkflowTa return; } + if ($signalWaitId !== null && ScopedWaitCancellation::cancelled($run, 'signal', $signalWaitId)) { + // Keep the received signal bytes, but deny this cancelled occurrence. + return; + } + $alreadyApplied = ConfiguredV2Models::query('history_event_model', WorkflowHistoryEvent::class) ->where('workflow_run_id', $run->id) ->where('event_type', HistoryEventType::SignalApplied->value) @@ -2679,6 +3371,18 @@ private function applyScheduleActivity( is_array($command['retry_policy'] ?? null) ? $command['retry_policy'] : null, $options, ); + $activityOptions = $options?->toSnapshot(); + if (isset($command['cancellation_policy']) || isset($command['cancellation_scope_id'])) { + $activityOptions = [ + ...($activityOptions ?? []), + ...(isset($command['cancellation_policy']) ? [ + 'cancellation_policy' => $command['cancellation_policy'], + ] : []), + ...(isset($command['cancellation_scope_id']) ? [ + 'cancellation_scope_id' => $command['cancellation_scope_id'], + ] : []), + ]; + } /** @var ActivityExecution $execution */ $execution = ActivityExecution::query()->create([ @@ -2693,7 +3397,7 @@ private function applyScheduleActivity( 'connection' => $connection, 'queue' => $queue, 'retry_policy' => $retryPolicy, - 'activity_options' => $options?->toSnapshot(), + 'activity_options' => $activityOptions, 'schedule_deadline_at' => $scheduleDeadlineAt, 'schedule_to_close_deadline_at' => $scheduleToCloseDeadlineAt, ]); @@ -2757,6 +3461,15 @@ private function applyStartTimer( 'sequence' => $sequence, 'delay_seconds' => $delaySeconds, 'fire_at' => $fireAt->toJSON(), + ...(isset($command['cancellation_cleanup']) ? [ + 'cancellation_cleanup' => PortableLocalActivityCleanup::snapshot( + $run, + $command['cancellation_cleanup'], + $sequence, + $command['cancellation_scope_id'], + ), + ] : []), + ...self::operationScopeMetadata($command), ...self::parallelMetadataForCommand($command), ], $task); @@ -2769,6 +3482,7 @@ private function applyStartTimer( 'available_at' => $fireAt, 'payload' => [ 'timer_id' => $timer->id, + ...self::operationScopeMetadata($command), ...self::parallelMetadataForCommand($command), ], 'connection' => $run->connection, @@ -3115,7 +3829,7 @@ private function applyOpenConditionWait( $timeoutSeconds = is_int($command['timeout_seconds'] ?? null) && $command['timeout_seconds'] >= 0 ? (int) $command['timeout_seconds'] : null; - $parallelMetadata = self::parallelMetadataForCommand($command); + $parallelMetadata = [...self::parallelMetadataForCommand($command), ...self::operationScopeMetadata($command)]; $parallelPath = ParallelChildGroup::metadataPathFromPayload($parallelMetadata); $waitId = (string) Str::ulid(); @@ -3229,7 +3943,7 @@ private function applyOpenSignalWait( $timeoutSeconds = is_int($command['timeout_seconds'] ?? null) && $command['timeout_seconds'] >= 0 ? (int) $command['timeout_seconds'] : null; - $parallelMetadata = self::parallelMetadataForCommand($command); + $parallelMetadata = [...self::parallelMetadataForCommand($command), ...self::operationScopeMetadata($command)]; $parallelPath = ParallelChildGroup::metadataPathFromPayload($parallelMetadata); $pendingSignalWaitId = $this->pendingSignalWaitIdForOpenSignalWait($run, $signalName); $waitId = $pendingSignalWaitId ?? (string) Str::ulid(); @@ -3351,6 +4065,9 @@ private function fireImmediateSignalTimeout( 'sequence' => $sequence, 'delay_seconds' => $timer->delay_seconds, 'fired_at' => $timer->fired_at?->toJSON(), + ...(isset($parallelMetadata['cancellation_scope_id']) ? [ + 'fire_at' => $recordedAt->toJSON(), + ] : []), 'timer_kind' => 'signal_timeout', 'signal_wait_id' => $waitId, 'signal_name' => $signalName, @@ -3401,6 +4118,9 @@ private function fireImmediateConditionTimeout( return WorkflowHistoryEvent::record($run, HistoryEventType::TimerFired, [ ...$payload, 'fired_at' => $recordedAt->toJSON(), + ...(isset($parallelMetadata['cancellation_scope_id']) ? [ + 'fire_at' => $recordedAt->toJSON(), + ] : []), ], $task); } @@ -3452,6 +4172,7 @@ private function pendingSignalWaitIdForOpenSignalWait(WorkflowRun $run, string $ * connection?: string|null, * queue?: string|null, * parent_close_policy?: string|null, + * cancellation_policy?: string, * retry_policy?: array, * execution_timeout_seconds?: int, * run_timeout_seconds?: int @@ -3538,6 +4259,9 @@ private function applyStartChildWorkflow( $childCallId = (string) Str::ulid(); $parentClosePolicy = $command['parent_close_policy'] ?? ParentClosePolicy::Abandon->value; + $cancellationPolicy = CancellationPolicy::from( + $command['cancellation_policy'] ?? CancellationPolicy::Abandon->value + ); ChildRunHistory::recordChildCallStarted([ 'parent_workflow_run_id' => $run->id, @@ -3551,7 +4275,7 @@ private function applyStartChildWorkflow( 'compatibility' => $childRun->compatibility, 'retry_policy' => $retryPolicy, 'timeout_policy' => $timeoutPolicy, - 'cancellation_propagation' => false, + 'cancellation_propagation' => $cancellationPolicy !== CancellationPolicy::Abandon, 'status' => ChildCallStatus::Started, 'scheduled_at' => $now, 'started_at' => $now, @@ -3561,6 +4285,8 @@ private function applyStartChildWorkflow( 'metadata' => [ 'child_call_id' => $childCallId, 'attempt_count' => 1, + 'cancellation_policy' => $cancellationPolicy->value, + ...self::operationScopeMetadata($command), ], 'resolved_child_instance_id' => $childInstance->id, 'resolved_child_run_id' => $childRun->id, @@ -3589,8 +4315,10 @@ private function applyStartChildWorkflow( 'child_workflow_class' => $workflowType, 'child_workflow_type' => $workflowType, 'parent_close_policy' => $parentClosePolicy, + 'cancellation_policy' => $cancellationPolicy->value, 'retry_policy' => $retryPolicy, 'timeout_policy' => $timeoutPolicy, + ...self::operationScopeMetadata($command), ...self::parallelMetadataForCommand($command), ], $task); @@ -3604,6 +4332,8 @@ private function applyStartChildWorkflow( 'child_workflow_type' => $workflowType, 'child_run_number' => 1, 'parent_close_policy' => $parentClosePolicy, + 'cancellation_policy' => $cancellationPolicy->value, + ...self::operationScopeMetadata($command), 'retry_policy' => $retryPolicy, 'timeout_policy' => $timeoutPolicy, 'execution_timeout_seconds' => $executionTimeoutSeconds, @@ -4981,7 +5711,20 @@ private static function normalizeCommand(array $command): ?array return null; } - return match ($type) { + $scopeMetadata = self::operationScopeMetadata($command); + $cleanupMetadata = ScopedCancellationCleanup::metadata($command); + if ($cleanupMetadata === null || ($cleanupMetadata !== [] && $type !== 'start_timer')) { + return null; + } + if (array_key_exists('cancellation_scope_id', $command) + && ($scopeMetadata === [] || ! in_array( + $type, + ['schedule_activity', 'start_timer', 'start_child_workflow', 'open_signal_wait', 'open_condition_wait'], + true + ))) { + return null; + } + $normalized = match ($type) { 'cancel_selection_operation' => self::normalizeCancelSelectionOperationCommand($command), 'complete_workflow' => self::normalizeCompleteWorkflowCommand($command), 'fail_workflow' => self::normalizeFailWorkflowCommand($command), @@ -5001,6 +5744,38 @@ private static function normalizeCommand(array $command): ?array 'open_signal_wait' => self::normalizeOpenSignalWaitCommand($command), default => null, }; + return $normalized === null ? null : [...$normalized, ...$scopeMetadata, ...$cleanupMetadata]; + } + + /** @param array $command + * @return array{cancellation_scope_id?: string} + */ + private static function operationScopeMetadata(array $command): array + { + $scopeId = $command['cancellation_scope_id'] ?? null; + return is_string($scopeId) && trim($scopeId) !== '' && strlen($scopeId) <= 255 + && preg_match('//u', $scopeId) === 1 ? [ + 'cancellation_scope_id' => $scopeId, + ] : []; + } + + /** + * Scope opening is checkpointed separately. Every operation in this batch + * must already have its exact-run scope before the first admission. + * @param list $commands + */ + private static function operationScopesAreRecorded(WorkflowRun $run, array $commands, int $sequence): bool + { + foreach ($commands as $command) { + if (isset($command['cancellation_scope_id']) && in_array( + $command['type'], + ['schedule_activity', 'start_timer', 'start_child_workflow', 'open_signal_wait', 'open_condition_wait'], + true, + ) && ! CancellationScopeHistory::isRecordedBefore($run, $command['cancellation_scope_id'], $sequence)) { + return false; + } + } + return true; } /** @@ -5199,6 +5974,7 @@ private static function normalizeFailWorkflowCommand(array $command): ?array * schedule_to_start_timeout?: int, * schedule_to_close_timeout?: int, * heartbeat_timeout?: int, + * cancellation_policy?: string, * worker_session?: array * }|null */ @@ -5210,6 +5986,7 @@ private static function normalizeScheduleActivityCommand(array $command): ?array $scheduleToStartTimeout = self::normalizePositiveInt($command['schedule_to_start_timeout'] ?? null); $scheduleToCloseTimeout = self::normalizePositiveInt($command['schedule_to_close_timeout'] ?? null); $heartbeatTimeout = self::normalizePositiveInt($command['heartbeat_timeout'] ?? null); + $cancellationPolicy = self::normalizeOptionalString($command['cancellation_policy'] ?? null); $arguments = self::normalizeCommandPayloadString($command, 'arguments'); if ($activityType === null) { @@ -5224,6 +6001,12 @@ private static function normalizeScheduleActivityCommand(array $command): ?array return null; } + if (($command['cancellation_policy'] ?? null) !== null + && ($cancellationPolicy === null || CancellationPolicy::tryFrom($cancellationPolicy) === null + || ($cancellationPolicy === CancellationPolicy::Abandon->value && $scheduleToCloseTimeout === null))) { + return null; + } + foreach ( [ 'start_to_close_timeout' => $startToCloseTimeout, @@ -5254,6 +6037,7 @@ private static function normalizeScheduleActivityCommand(array $command): ?array 'schedule_to_start_timeout' => $scheduleToStartTimeout, 'schedule_to_close_timeout' => $scheduleToCloseTimeout, 'heartbeat_timeout' => $heartbeatTimeout, + 'cancellation_policy' => $cancellationPolicy, 'worker_session' => self::normalizeWorkerSessionCommand($command['worker_session'] ?? null), ...$parallelMetadata, ], static fn (mixed $value): bool => $value !== null); @@ -5351,6 +6135,7 @@ private static function normalizeStartTimerCommand(array $command): ?array * connection?: string|null, * queue?: string|null, * parent_close_policy?: string|null, + * cancellation_policy?: string, * retry_policy?: array, * execution_timeout_seconds?: int, * run_timeout_seconds?: int @@ -5363,6 +6148,12 @@ private static function normalizeStartChildWorkflowCommand(array $command): ?arr $executionTimeoutSeconds = self::normalizePositiveInt($command['execution_timeout_seconds'] ?? null); $runTimeoutSeconds = self::normalizePositiveInt($command['run_timeout_seconds'] ?? null); $arguments = self::normalizeCommandPayloadString($command, 'arguments'); + $cancellationPolicy = $command['cancellation_policy'] ?? null; + if ($cancellationPolicy !== null && (! is_string($cancellationPolicy) || CancellationPolicy::tryFrom( + $cancellationPolicy + ) === null)) { + return null; + } if ($workflowType === null) { return null; @@ -5400,6 +6191,7 @@ private static function normalizeStartChildWorkflowCommand(array $command): ?arr 'connection' => self::normalizeOptionalString($command['connection'] ?? null), 'queue' => self::normalizeOptionalString($command['queue'] ?? null), 'parent_close_policy' => self::normalizeOptionalString($command['parent_close_policy'] ?? null), + 'cancellation_policy' => $cancellationPolicy, 'retry_policy' => $retryPolicy, 'execution_timeout_seconds' => $executionTimeoutSeconds, 'run_timeout_seconds' => $runTimeoutSeconds, @@ -6348,7 +7140,7 @@ private function claimIfReady(string $taskId): bool 'status' => TaskStatus::Leased, 'leased_at' => now(), 'lease_owner' => $taskId, - 'lease_expires_at' => WorkflowTaskLease::expiresAt(), + 'lease_expires_at' => CancellationCleanupLease::forScopes($run), 'attempt_count' => $task->attempt_count + 1, 'sticky_replay_mode' => StickyExecution::claimReplayMode($task, $taskId), 'sticky_claimed_at' => now(), diff --git a/src/V2/Support/HistoryEventPayloadContract.php b/src/V2/Support/HistoryEventPayloadContract.php index c23ed96f3..b5c384217 100644 --- a/src/V2/Support/HistoryEventPayloadContract.php +++ b/src/V2/Support/HistoryEventPayloadContract.php @@ -25,6 +25,39 @@ final class HistoryEventPayloadContract * @var array> */ private const PAYLOAD_KEYS = [ + 'CancellationScopeRequested' => [ + 'schema', 'workflow_run_id', 'scope_id', 'parent_scope_id', 'request_id', 'cancellation', + ], + 'CancellationScopeDeliveryPrepared' => [ + 'schema', 'workflow_run_id', 'scope_id', 'request_id', 'cancellation', + 'sequence', 'call_kind', 'sequence_span', 'operation_sequence', 'operation_sequence_span', + 'authority_deadline_at', 'activity_members', 'timer_members', 'wait_members', 'child_members', + 'descendant_members', + ], + 'SignalWaitCancelled' => [ + 'signal_name', 'signal_wait_id', 'sequence', 'timeout_seconds', 'timer_id', 'cancelled_at', 'cancellation_scope', + ], + 'ConditionWaitCancelled' => [ + 'condition_wait_id', 'condition_wait_occurrence_id', 'condition_key', 'condition_definition_fingerprint', + 'sequence', 'timeout_seconds', 'timer_id', 'cancelled_at', 'cancellation_scope', + ], + 'CancellationScopeDelivered' => [ + 'schema', 'workflow_run_id', 'scope_id', 'request_id', 'cancellation', + 'sequence', 'call_kind', 'sequence_span', 'operation_sequence', 'operation_sequence_span', + 'authority_deadline_at', 'preparation_history_event_id', + ], + 'CancellationScopeRequestConflicted' => [ + 'schema', 'workflow_run_id', 'scope_id', 'parent_scope_id', 'reason', + 'accepted_cancellation', 'incoming_cancellation', + ], + 'CancellationScopeOpened' => [ + 'schema', + 'workflow_run_id', + 'sequence', + 'scope_id', + 'parent_scope_id', + 'shield_parent', + ], 'StartAccepted' => [ 'workflow_command_id', 'workflow_instance_id', @@ -105,6 +138,8 @@ final class HistoryEventPayloadContract 'execution_mode', 'local_activity', 'workflow_task_id', + 'local_preparation', + 'local_group_admission', 'activity', 'parallel_group_id', 'parallel_group_kind', @@ -124,6 +159,7 @@ final class HistoryEventPayloadContract 'execution_mode', 'local_activity', 'workflow_task_id', + 'local_preparation', 'lease_expires_at', 'activity', 'activity_attempt', @@ -150,8 +186,16 @@ final class HistoryEventPayloadContract 'activity', 'activity_attempt', 'progress', + 'parallel_group_id', + 'parallel_group_kind', + 'parallel_group_base_sequence', + 'parallel_group_size', + 'parallel_group_index', + 'parallel_group_path', ], 'ActivityRetryScheduled' => [ + 'local_outcome', + 'local_recovery', 'activity_execution_id', 'activity_attempt_id', 'worker_attempt_id', @@ -186,6 +230,7 @@ final class HistoryEventPayloadContract 'parallel_group_path', ], 'ActivityCompleted' => [ + 'local_outcome', 'activity_execution_id', 'activity_attempt_id', 'worker_attempt_id', @@ -211,6 +256,8 @@ final class HistoryEventPayloadContract 'parallel_group_path', ], 'ActivityFailed' => [ + 'local_outcome', + 'local_recovery', 'activity_execution_id', 'activity_attempt_id', 'worker_attempt_id', @@ -242,6 +289,7 @@ final class HistoryEventPayloadContract 'structural_limit_configured', ], 'ActivityCancelled' => [ + 'cancellation_scope', 'workflow_command_id', 'activity_execution_id', 'activity_attempt_id', @@ -256,8 +304,35 @@ final class HistoryEventPayloadContract 'workflow_task_id', 'activity', 'activity_attempt', + 'parallel_group_id', + 'parallel_group_kind', + 'parallel_group_base_sequence', + 'parallel_group_size', + 'parallel_group_index', + 'parallel_group_path', + ], + 'ActivityCancellationAcknowledged' => [ + 'sequence', + 'activity_execution_id', + 'activity_attempt_id', + 'worker_attempt_id', + 'execution_mode', + 'local_activity', + 'workflow_task_id', + 'task', + 'lease_owner', + 'cancellation_history_event_id', + 'request_id', + 'root_request_id', + 'cleanup_deadline_at', + 'callback_state', + 'evidence_source', + 'acknowledged_at', + 'received_after_deadline', ], 'ActivityTimedOut' => [ + 'local_outcome', + 'local_recovery', 'activity_execution_id', 'activity_attempt_id', 'worker_attempt_id', @@ -287,10 +362,12 @@ final class HistoryEventPayloadContract 'parallel_group_path', ], 'TimerScheduled' => [ + 'cancellation_cleanup', 'timer_id', 'sequence', 'delay_seconds', 'fire_at', + 'cancellation_scope_id', 'parallel_group_id', 'parallel_group_kind', 'parallel_group_base_sequence', @@ -311,6 +388,7 @@ final class HistoryEventPayloadContract 'delay_seconds', 'fire_at', 'fired_at', + 'cancellation_scope_id', 'parallel_group_id', 'parallel_group_kind', 'parallel_group_base_sequence', @@ -351,6 +429,7 @@ final class HistoryEventPayloadContract ], 'TimerCancelled' => [ 'timer_id', + 'cancellation_scope', 'sequence', 'delay_seconds', 'fire_at', @@ -398,6 +477,7 @@ final class HistoryEventPayloadContract 'signal_wait_id', 'sequence', 'timeout_seconds', + 'cancellation_scope_id', 'parallel_group_id', 'parallel_group_kind', 'parallel_group_base_sequence', @@ -460,6 +540,7 @@ final class HistoryEventPayloadContract 'condition_definition_fingerprint', 'sequence', 'timeout_seconds', + 'cancellation_scope_id', 'parallel_group_id', 'parallel_group_kind', 'parallel_group_base_sequence', @@ -512,6 +593,8 @@ final class HistoryEventPayloadContract 'child_workflow_type', 'child_run_number', 'parent_close_policy', + 'cancellation_policy', + 'cancellation_scope_id', 'retry_policy', 'timeout_policy', 'parallel_group_id', @@ -532,6 +615,8 @@ final class HistoryEventPayloadContract 'child_run_number', 'child_status', 'parent_close_policy', + 'cancellation_policy', + 'cancellation_scope_id', 'retry_policy', 'timeout_policy', 'execution_timeout_seconds', @@ -787,10 +872,12 @@ final class HistoryEventPayloadContract 'command_type', 'reason', 'cleanup_deadline_at', + 'cancellation', ], 'CooperativeCancellationDelivered' => [ 'workflow_command_id', 'workflow_run_id', 'sequence', 'call_kind', 'sequence_span', 'operation_sequence', 'operation_sequence_span', + 'cancellation', ], 'WorkflowCancelled' => [ 'workflow_command_id', @@ -802,6 +889,7 @@ final class HistoryEventPayloadContract 'exception_class', 'message', 'reason', + 'cancellation_cleanup', ], 'TerminateRequested' => [ 'workflow_command_id', @@ -874,8 +962,29 @@ final class HistoryEventPayloadContract 'message', 'handled', ], - 'ParentClosePolicyApplied' => ['child_instance_id', 'child_run_id', 'policy', 'reason'], - 'ParentClosePolicyFailed' => ['child_instance_id', 'child_run_id', 'policy', 'reason', 'error'], + 'ChildCancellationRequested' => [ + 'sequence', 'policy', 'parent_request_id', 'root_request_id', 'cleanup_deadline_at', + 'child_workflow_instance_id', 'child_workflow_run_id', 'child_request_id', + 'child_root_request_id', 'child_cleanup_deadline_at', 'request_outcome', 'rejection_reason', + 'cancellation_scope', 'child_cancellation', + ], + 'ChildCancellationResolved' => [ + 'sequence', 'policy', 'parent_request_id', 'root_request_id', 'cleanup_deadline_at', + 'child_workflow_instance_id', 'child_workflow_run_id', 'child_status', + 'child_request_id', 'child_root_request_id', 'child_cleanup_deadline_at', + 'child_terminal_history_event_id', 'child_terminal_event_type', 'reason', + 'cancellation_scope', + ], + 'ParentCloseCancellationRequested' => [ + 'parent_terminal_history_event_id', 'parent_terminal_event_type', 'cancellation', + ], + 'ParentClosePolicyApplied' => [ + 'child_instance_id', 'child_run_id', 'policy', 'reason', 'request_id', 'cancellation', + ], + 'ParentClosePolicyFailed' => [ + 'child_instance_id', 'child_run_id', 'policy', 'reason', 'error', 'request_id', 'cancellation', + 'request_diagnostics', + ], 'MessageCursorAdvanced' => ['stream_key', 'previous_position', 'new_position'], 'ScheduleCreated' => ['spec', 'action', 'overlap_policy', 'next_fire_at', 'command_context'], 'SchedulePaused' => ['reason', 'paused_at', 'command_context'], diff --git a/src/V2/Support/HistoryTimeline.php b/src/V2/Support/HistoryTimeline.php index beac34eef..dc4bbf5be 100644 --- a/src/V2/Support/HistoryTimeline.php +++ b/src/V2/Support/HistoryTimeline.php @@ -211,7 +211,37 @@ private static function mapEvent( 'command_outcome' => $commandMetadata['outcome'] ?? null, 'command_rejection_reason' => $commandMetadata['rejection_reason'] ?? null, 'workflow_sequence' => self::intValue($payload['sequence'] ?? null), + ...($event->event_type === HistoryEventType::CancellationScopeOpened ? [ + 'cancellation_scope' => [ + 'schema' => self::stringValue($payload['schema'] ?? null), + 'scope_id' => self::stringValue($payload['scope_id'] ?? null), + 'parent_scope_id' => self::stringValue($payload['parent_scope_id'] ?? null), + 'shield_parent' => is_bool($payload['shield_parent'] ?? null) ? $payload['shield_parent'] : null, + ], + ] : []), + ...(in_array( + $event->event_type, + [HistoryEventType::CancellationScopeRequested, HistoryEventType::CancellationScopeDeliveryPrepared, + HistoryEventType::CancellationScopeDelivered, + HistoryEventType::CancellationScopeRequestConflicted], + true + ) ? [ + 'cancellation_scope' => array_intersect_key($payload, array_flip([ + 'schema', 'scope_id', 'parent_scope_id', 'request_id', 'cancellation', + 'reason', 'accepted_cancellation', 'incoming_cancellation', + 'sequence', 'call_kind', 'sequence_span', 'operation_sequence', 'operation_sequence_span', + 'authority_deadline_at', 'activity_members', 'timer_members', 'wait_members', 'child_members', 'preparation_history_event_id', + ])), + ] : []), 'service_call_id' => self::stringValue($payload['service_call_id'] ?? null), + ...(in_array( + $event->event_type, + [HistoryEventType::SignalWaitCancelled, HistoryEventType::ConditionWaitCancelled], + true + ) + ? [ + 'cancellation_scope' => $payload['cancellation_scope'] ?? null, + ] : []), 'signal_id' => self::stringValue($payload['signal_id'] ?? null), 'signal_wait_id' => self::stringValue($payload['signal_wait_id'] ?? null), 'condition_wait_id' => self::stringValue($payload['condition_wait_id'] ?? null), @@ -265,6 +295,67 @@ private static function mapEvent( 'activity' => $activityMetadata, 'timer' => $timerMetadata, 'child' => $childMetadata, + ...(is_array($payload['local_recovery'] ?? null) ? [ + 'local_recovery' => [ + 'original_workflow_task_id' => self::stringValue( + $payload['local_recovery']['original_workflow_task_id'] ?? null + ), + 'original_workflow_task_attempt' => self::intValue( + $payload['local_recovery']['original_workflow_task_attempt'] ?? null + ), + 'original_lease_owner' => self::stringValue( + $payload['local_recovery']['original_lease_owner'] ?? null + ), + 'original_lease_expires_at' => self::timestamp( + $payload['local_recovery']['original_lease_expires_at'] ?? null + ), + 'workflow_task_id' => self::stringValue($payload['local_recovery']['workflow_task_id'] ?? null), + 'workflow_task_attempt' => self::intValue( + $payload['local_recovery']['workflow_task_attempt'] ?? null + ), + 'lease_owner' => self::stringValue($payload['local_recovery']['lease_owner'] ?? null), + 'callback_stop_state' => self::stringValue( + $payload['local_recovery']['callback_stop_state'] ?? null + ), + ], + ] : []), + ...($event->event_type === HistoryEventType::ActivityCancellationAcknowledged ? [ + 'cancellation_acknowledgement' => [ + 'activity_attempt_id' => self::stringValue($payload['activity_attempt_id'] ?? null), + 'cancellation_history_event_id' => self::stringValue( + $payload['cancellation_history_event_id'] ?? null + ), + 'request_id' => self::stringValue($payload['request_id'] ?? null), + 'root_request_id' => self::stringValue($payload['root_request_id'] ?? null), + 'cleanup_deadline_at' => self::timestamp($payload['cleanup_deadline_at'] ?? null), + 'callback_state' => self::stringValue($payload['callback_state'] ?? null), + 'evidence_source' => self::stringValue($payload['evidence_source'] ?? null), + 'acknowledged_at' => self::timestamp($payload['acknowledged_at'] ?? null), + 'received_after_deadline' => $payload['received_after_deadline'] ?? null, + ], + ] : []), + ...(in_array($event->event_type, [ + HistoryEventType::ParentCloseCancellationRequested, + HistoryEventType::ParentClosePolicyApplied, + HistoryEventType::ParentClosePolicyFailed, + ], true) ? [ + 'parent_close' => [ + 'policy' => self::stringValue($payload['policy'] ?? null), + 'child_instance_id' => self::stringValue($payload['child_instance_id'] ?? null), + 'child_run_id' => self::stringValue($payload['child_run_id'] ?? null), + 'request_id' => self::stringValue($payload['request_id'] ?? null), + 'cancellation' => $payload['cancellation'] ?? null, + 'error' => self::stringValue($payload['error'] ?? null), + 'request_diagnostics' => $payload['request_diagnostics'] ?? null, + 'parent_terminal_history_event_id' => self::stringValue( + $payload['parent_terminal_history_event_id'] ?? null, + ), + ], + ] : []), + ...($event->event_type === HistoryEventType::WorkflowCancelled + && is_array($payload['cancellation_cleanup'] ?? null) ? [ + 'cancellation_cleanup' => $payload['cancellation_cleanup'], + ] : []), 'failure' => $failureMetadata, ]; } @@ -283,6 +374,7 @@ private static function kindFor(HistoryEventType $eventType): string HistoryEventType::TerminateRequested, HistoryEventType::ArchiveRequested => 'command', HistoryEventType::SignalWaitOpened, + HistoryEventType::SignalWaitCancelled, HistoryEventType::SignalApplied => 'signal', HistoryEventType::UpdateApplied, HistoryEventType::UpdateCompleted => 'update', @@ -291,6 +383,8 @@ private static function kindFor(HistoryEventType $eventType): string HistoryEventType::ChildRunCompleted, HistoryEventType::ChildRunFailed, HistoryEventType::ChildRunCancelled, + HistoryEventType::ChildCancellationRequested, + HistoryEventType::ChildCancellationResolved, HistoryEventType::ChildRunTerminated => 'child', HistoryEventType::ServiceCallStarted, HistoryEventType::ServiceCallCompleted, @@ -298,7 +392,8 @@ private static function kindFor(HistoryEventType $eventType): string HistoryEventType::ServiceCallCancelled => 'service_call', HistoryEventType::ConditionWaitOpened, HistoryEventType::ConditionWaitSatisfied, - HistoryEventType::ConditionWaitTimedOut => 'condition', + HistoryEventType::ConditionWaitTimedOut, + HistoryEventType::ConditionWaitCancelled => 'condition', HistoryEventType::ActivityScheduled, HistoryEventType::ActivityStarted, HistoryEventType::ActivityHeartbeatRecorded, @@ -306,12 +401,18 @@ private static function kindFor(HistoryEventType $eventType): string HistoryEventType::ActivityCompleted, HistoryEventType::ActivityFailed, HistoryEventType::ActivityCancelled, + HistoryEventType::ActivityCancellationAcknowledged, HistoryEventType::ActivityTimedOut => 'activity', HistoryEventType::FailureHandled => 'failure', HistoryEventType::SideEffectRecorded => 'side_effect', HistoryEventType::VersionMarkerRecorded => 'version', HistoryEventType::SelectionResolved => 'selection', HistoryEventType::SelectionOperationCancelled => 'selection', + HistoryEventType::CancellationScopeOpened, + HistoryEventType::CancellationScopeRequested, + HistoryEventType::CancellationScopeDeliveryPrepared, + HistoryEventType::CancellationScopeDelivered, + HistoryEventType::CancellationScopeRequestConflicted => 'cancellation_scope', HistoryEventType::TimerScheduled, HistoryEventType::TimerFired, HistoryEventType::TimerCancelled => 'timer', @@ -341,7 +442,7 @@ private static function summaryFor( $timerKind = self::stringValue($payload['timer_kind'] ?? null); $childLabel = self::displayLabel($child['type'] ?? $child['class'] ?? $child['run_id'] ?? 'child workflow'); - return match ($event->event_type) { + $summary = match ($event->event_type) { HistoryEventType::StartAccepted => $outcome === null ? 'Start accepted.' : sprintf('Start accepted as %s.', $outcome), @@ -349,6 +450,18 @@ private static function summaryFor( ? 'Start rejected.' : sprintf('Start rejected: %s.', $rejectionReason), HistoryEventType::WorkflowStarted => 'Workflow run started.', + HistoryEventType::CancellationScopeRequested => 'Cooperative operation scope cancellation requested.', + HistoryEventType::CancellationScopeDeliveryPrepared => 'Operation scope cancellation preparation retained.', + HistoryEventType::CancellationScopeDelivered => 'Operation scope cancellation delivery boundary recorded.', + HistoryEventType::CancellationScopeRequestConflicted => 'Operation scope cancellation root conflicts with its accepted request.', + HistoryEventType::CancellationScopeOpened => sprintf( + 'Cancellation scope %s opened under %s%s.', + self::stringValue($payload['scope_id'] ?? null) ?? 'unknown', + self::stringValue($payload['parent_scope_id'] ?? null) ?? 'unknown', + ($payload['shield_parent'] ?? null) === true ? ' with parent shielding' : '', + ), + HistoryEventType::SignalWaitCancelled => 'Signal wait cancelled by its operation scope.', + HistoryEventType::ConditionWaitCancelled => 'Condition wait cancelled by its operation scope.', HistoryEventType::WorkflowContinuedAsNew => sprintf( 'Continued as new on run %s.', self::stringValue($payload['continued_to_run_id'] ?? null) ?? 'unknown' @@ -448,7 +561,13 @@ private static function summaryFor( HistoryEventType::CancelRequested => 'Cancel requested.', HistoryEventType::CooperativeCancellationRequested => 'Cooperative cancellation requested.', HistoryEventType::CooperativeCancellationDelivered => 'Cancellation delivered to workflow code.', - HistoryEventType::WorkflowCancelled => 'Workflow cancelled.', + HistoryEventType::WorkflowCancelled => match ($payload['cancellation_cleanup']['outcome'] ?? null) { + 'completed' => 'Workflow cancelled after cleanup completed.', + 'deadline_expired' => 'Workflow cancelled at the cleanup deadline.', + 'not_delivered' => 'Workflow cancelled without cancellation delivery to workflow code.', + 'unavailable' => 'Workflow cancelled. Cleanup delivery evidence is unavailable.', + default => 'Workflow cancelled.', + }, HistoryEventType::TerminateRequested => 'Terminate requested.', HistoryEventType::WorkflowTerminated => 'Workflow terminated.', HistoryEventType::ArchiveRequested => match ($outcome) { @@ -470,6 +589,11 @@ private static function summaryFor( ? sprintf('Failed %s.', $activityLabel) : sprintf('Failed %s: %s.', $activityLabel, $message), HistoryEventType::ActivityCancelled => sprintf('Cancelled %s.', $activityLabel), + HistoryEventType::ActivityCancellationAcknowledged => sprintf( + 'Worker reported stopped callback for %s%s.', + $activityLabel, + ($payload['received_after_deadline'] ?? false) === true ? ' after the cleanup deadline' : '', + ), HistoryEventType::ActivityTimedOut => $message === null ? sprintf('Timed out %s.', $activityLabel) : sprintf('Timed out %s: %s.', $activityLabel, $message), @@ -516,6 +640,21 @@ private static function summaryFor( 'signal_timeout' => 'Signal timeout cancelled.', default => 'Timer cancelled.', }, + HistoryEventType::ChildCancellationRequested => sprintf( + 'Child cancellation %s under policy %s%s.', + self::stringValue($payload['request_outcome'] ?? null) ?? 'requested', + self::stringValue($payload['policy'] ?? null) ?? 'unknown', + self::stringValue($payload['rejection_reason'] ?? null) !== null + ? ': ' . $payload['rejection_reason'] : '', + ), + HistoryEventType::ChildCancellationResolved => sprintf( + 'Child cancellation resolved with durable outcome %s.', + self::stringValue($payload['child_status'] ?? null) ?? 'unknown', + ), + HistoryEventType::ParentCloseCancellationRequested => sprintf( + 'Parent-close policy requested cooperative child cleanup, deadline %s.', + self::stringValue($payload['cancellation']['cleanup_deadline_at'] ?? null) ?? 'unknown', + ), HistoryEventType::ParentClosePolicyApplied => sprintf( 'Applied parent-close policy %s to child %s.', self::stringValue($payload['policy'] ?? null) ?? 'unknown', @@ -573,6 +712,10 @@ private static function summaryFor( self::stringValue($payload['reason'] ?? null) ?? 'unknown reason', ), }; + + return ($payload['local_recovery']['callback_stop_state'] ?? null) === 'unknown' + ? $summary . ' The previous attempt lost authority. Callback stop is unknown.' + : $summary; } /** @@ -616,6 +759,26 @@ private static function childMetadata(WorkflowHistoryEvent $event, array $payloa }, 'run_number' => self::intValue($payload['child_run_number'] ?? null), 'parallel_group_path' => ParallelChildGroup::metadataPathFromPayload($payload), + ...(in_array($event->event_type, [ + HistoryEventType::ChildCancellationRequested, + HistoryEventType::ChildCancellationResolved, + ], true) ? [ + 'cancellation' => [ + 'policy' => self::stringValue($payload['policy'] ?? null), + 'parent_request_id' => self::stringValue($payload['parent_request_id'] ?? null), + 'root_request_id' => self::stringValue($payload['root_request_id'] ?? null), + 'cleanup_deadline_at' => self::timestamp($payload['cleanup_deadline_at'] ?? null), + 'child_request_id' => self::stringValue($payload['child_request_id'] ?? null), + 'child_root_request_id' => self::stringValue($payload['child_root_request_id'] ?? null), + 'child_cleanup_deadline_at' => self::timestamp($payload['child_cleanup_deadline_at'] ?? null), + 'request_outcome' => self::stringValue($payload['request_outcome'] ?? null), + 'rejection_reason' => self::stringValue($payload['rejection_reason'] ?? null), + 'terminal_history_event_id' => self::stringValue( + $payload['child_terminal_history_event_id'] ?? null + ), + 'terminal_event_type' => self::stringValue($payload['child_terminal_event_type'] ?? null), + ], + ] : []), ]; } @@ -839,6 +1002,8 @@ private static function activityMetadata( ?? self::stringValue($payload['activity_class'] ?? null), 'parallel_group_path' => ParallelChildGroup::metadataPathFromPayload($payload), 'attempt_id' => self::stringValue($snapshot['attempt_id'] ?? null) + ?? ($event->event_type === HistoryEventType::ActivityCancellationAcknowledged + ? self::stringValue($payload['activity_attempt_id'] ?? null) : null) ?? ($event->event_type === HistoryEventType::ActivityScheduled ? null : self::stringValue($activity?->current_attempt_id)), @@ -851,7 +1016,8 @@ private static function activityMetadata( HistoryEventType::ActivityCompleted => 'completed', HistoryEventType::ActivityFailed => 'failed', HistoryEventType::ActivityTimedOut => 'failed', - HistoryEventType::ActivityCancelled => 'cancelled', + HistoryEventType::ActivityCancelled, + HistoryEventType::ActivityCancellationAcknowledged => 'cancelled', default => $activity?->status?->value, }, 'attempt_count' => self::intValue($snapshot['attempt_count'] ?? null) @@ -1058,6 +1224,7 @@ private static function sourceKindFor(WorkflowHistoryEvent $event): string HistoryEventType::ActivityCompleted, HistoryEventType::ActivityFailed, HistoryEventType::ActivityCancelled, + HistoryEventType::ActivityCancellationAcknowledged, HistoryEventType::ActivityTimedOut => 'activity_execution', HistoryEventType::FailureHandled => 'workflow_failure', HistoryEventType::VersionMarkerRecorded => 'version_marker', @@ -1162,6 +1329,10 @@ private static function historicalTaskType(WorkflowHistoryEvent $event, ?string if ($taskId === null) { return null; } + if (($event->payload['local_activity'] ?? null) === true + || ($event->payload['execution_mode'] ?? null) === LocalActivityRuntime::EXECUTION_MODE) { + return 'workflow'; + } return match ($event->event_type) { HistoryEventType::ActivityStarted, @@ -1170,6 +1341,7 @@ private static function historicalTaskType(WorkflowHistoryEvent $event, ?string HistoryEventType::ActivityCompleted, HistoryEventType::ActivityFailed, HistoryEventType::ActivityCancelled, + HistoryEventType::ActivityCancellationAcknowledged, HistoryEventType::ActivityTimedOut => 'activity', HistoryEventType::TimerFired => 'timer', default => 'workflow', @@ -1181,11 +1353,16 @@ private static function historicalTaskStatus(WorkflowHistoryEvent $event, ?strin if ($taskId === null) { return null; } + if (($event->payload['local_activity'] ?? null) === true + || ($event->payload['execution_mode'] ?? null) === LocalActivityRuntime::EXECUTION_MODE) { + return self::stringValue($event->payload['task']['status'] ?? null); + } return match ($event->event_type) { HistoryEventType::ActivityStarted, HistoryEventType::ActivityHeartbeatRecorded => 'leased', - HistoryEventType::ActivityCancelled => 'cancelled', + HistoryEventType::ActivityCancelled, + HistoryEventType::ActivityCancellationAcknowledged => 'cancelled', HistoryEventType::WorkflowFailed => 'failed', HistoryEventType::WorkflowTimedOut => 'completed', default => 'completed', diff --git a/src/V2/Support/LocalActivityExecutor.php b/src/V2/Support/LocalActivityExecutor.php index fb578774a..0824664a3 100644 --- a/src/V2/Support/LocalActivityExecutor.php +++ b/src/V2/Support/LocalActivityExecutor.php @@ -6,6 +6,8 @@ use Illuminate\Support\Facades\DB; use Illuminate\Support\Str; +use Illuminate\Validation\ValidationException; +use JsonException; use RuntimeException; use Throwable; use Workflow\Serializers\CodecRegistry; @@ -17,6 +19,7 @@ use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Enums\TaskType; use Workflow\V2\Exceptions\ActivityTimeoutException; +use Workflow\V2\Exceptions\RestoredWorkflowException; use Workflow\V2\Exceptions\StructuralLimitExceededException; use Workflow\V2\Models\ActivityAttempt; use Workflow\V2\Models\ActivityExecution; @@ -29,6 +32,210 @@ final class LocalActivityExecutor { + /** + * @internal Candidate portable completion under an already prepared local + * attempt. The SDK reports only after its callback has returned or stopped. + * Result recording does not itself acknowledge physical callback stop. + * + * @param array $report + * @return array + */ + public function recordPortableOutcome( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + array $report, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + $refused = static fn (string $reason): array => [ + 'recorded' => false, + 'duplicate' => false, + 'reason' => $reason, + 'activity_attempt_id' => $attemptId, + 'claim_released' => false, + 'created_task_ids' => [], + ]; + if (preg_match('/^[0-9]+\.[0-9]+$/D', $protocolVersion) !== 1 + || version_compare($protocolVersion, PortableLocalActivityPreparation::MINIMUM_PROTOCOL_VERSION, '<')) { + return $refused('local_activity_outcome_requires_protocol_1_20'); + } + try { + $normalized = self::normalizePortableReport($report); + $fingerprint = hash('sha256', json_encode($normalized, JSON_THROW_ON_ERROR)); + } catch (ValidationException|JsonException) { + return $refused('invalid_local_activity_outcome'); + } + + return DB::transaction(function () use ( + $attemptId, + $leaseOwner, + $workflowTaskAttempt, + $normalized, + $fingerprint, + $refused, + ): array { + $rows = ActivityRowLockOrder::lockForAttempt($attemptId); + $attempt = $rows['attempt']; + $execution = $rows['execution']; + if (! $attempt instanceof ActivityAttempt || ! $execution instanceof ActivityExecution) { + return $refused('activity_attempt_not_found'); + } + /** @var WorkflowRun|null $run */ + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->lockForUpdate() + ->find($execution->workflow_run_id); + /** @var WorkflowTask|null $task */ + $task = ConfiguredV2Models::query('task_model', WorkflowTask::class) + ->lockForUpdate() + ->find($attempt->workflow_task_id); + if ($run === null || $task === null || $task->workflow_run_id !== $run->id) { + return $refused('workflow_claim_not_found'); + } + $started = PortableLocalActivityPreparation::originalStart( + $run, + $execution, + $attempt, + $leaseOwner, + $workflowTaskAttempt, + ); + if ($started === null) { + return $refused('local_activity_preparation_mismatch'); + } + $receipt = $run->historyEvents() + ->whereIn('event_type', [HistoryEventType::ActivityCompleted, HistoryEventType::ActivityFailed, + HistoryEventType::ActivityTimedOut, HistoryEventType::ActivityRetryScheduled]) + ->where('payload->activity_attempt_id', $attemptId) + ->whereNotNull('payload->local_outcome') + ->orderBy('sequence') + ->first(); + if ($receipt instanceof WorkflowHistoryEvent) { + if ($receipt->sequence <= $started->sequence || $receipt->workflow_task_id !== $task->id + || ($receipt->payload['activity_execution_id'] ?? null) !== $execution->id + || ($receipt->payload['local_outcome']['version'] ?? null) !== 1 + || ($receipt->payload['local_outcome']['report_fingerprint'] ?? null) !== $fingerprint + || ($receipt->payload['local_outcome']['workflow_task_attempt'] ?? null) !== $workflowTaskAttempt + || ($receipt->payload['task']['lease_owner'] ?? null) !== $leaseOwner + || ($receipt->payload['task']['attempt_count'] ?? null) !== $workflowTaskAttempt) { + return $refused('local_activity_outcome_mismatch'); + } + return self::portableReceipt($receipt, true); + } + if ($execution->current_attempt_id !== $attemptId || $execution->attempt_count !== $attempt->attempt_number) { + return $refused('stale_activity_attempt'); + } + // An original owner may fence its own still-running attempt after + // takeover. This grants no result authority and leaves the hosting + // workflow claim untouched. A separate joined-callback receipt follows. + $cleanup = PortableLocalActivityCleanup::isExecution($run, $execution, $started); + $runCleanup = $cleanup && PortableLocalActivityCleanup::belongsToRunRequest( + $run, + $execution->activity_options['cancellation_cleanup'], + ); + if ($run->cancellation_request_command_id !== null + && (! $runCleanup || now()->gte($run->cancellation_deadline_at))) { + $cancelled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancelled) + ->where('payload->activity_execution_id', $execution->id) + ->where('payload->activity_attempt_id', $attemptId) + ->where('payload->workflow_command_id', $run->cancellation_request_command_id) + ->first(); + if (! $cancelled instanceof WorkflowHistoryEvent) { + if ($attempt->status !== ActivityAttemptStatus::Running || $execution->status !== ActivityStatus::Running) { + return $refused('stale_activity_attempt'); + } + $cancelled = ActivityCancellation::record( + $run, + $execution, + command: $run->cancellation_request_command_id + ); + } + return [ + ...$refused($runCleanup + ? 'cancellation_deadline_expired' : 'cancellation_requested'), + 'fenced' => true, + 'cancellation_history_event_id' => $cancelled?->id, + ]; + } + if ($attempt->status !== ActivityAttemptStatus::Running || $execution->status !== ActivityStatus::Running) { + return $refused('stale_activity_attempt'); + } + if ($task->task_type !== TaskType::Workflow || $task->status !== TaskStatus::Leased + || $task->lease_owner !== $leaseOwner || $task->attempt_count !== $workflowTaskAttempt) { + return $refused('workflow_claim_mismatch'); + } + if (PortableLocalActivityCleanup::isScoped($execution)) { + if ($run->status->isTerminal()) { + return $refused('run_closed'); + } + if (($run->execution_deadline_at !== null && now()->gte($run->execution_deadline_at)) + || ($run->run_deadline_at !== null && now()->gte($run->run_deadline_at))) { + return $refused('run_deadline_expired'); + } + if (PortableLocalActivityCleanup::currentAuthorityExpired($run, $execution)) { + return $refused('local_activity_cleanup_authority_expired'); + } + } + if ($task->lease_expires_at === null || now()->gte($task->lease_expires_at) + || $attempt->lease_expires_at === null || now() + ->gte($attempt->lease_expires_at)) { + return $refused('workflow_claim_expired'); + } + if ($run->status->isTerminal()) { + return $refused('run_closed'); + } + if (($run->execution_deadline_at !== null && now()->gte($run->execution_deadline_at)) + || ($run->run_deadline_at !== null && now()->gte($run->run_deadline_at))) { + return $refused('run_deadline_expired'); + } + $metadata = [ + 'worker_attempt_id' => $attempt->worker_attempt_id, + 'local_outcome' => [ + 'version' => 1, + 'report_fingerprint' => $fingerprint, + 'workflow_task_attempt' => $workflowTaskAttempt, + 'submitted_outcome' => $normalized['outcome'], + ], + ]; + $timeoutKind = self::timeoutKind($execution); + if ($timeoutKind !== null) { + $outcome = $this->recordTimeoutOutcome($run, $task, $execution, $timeoutKind, $metadata); + } elseif ($normalized['outcome'] === 'timed_out') { + return $refused('local_activity_timeout_not_due'); + } elseif ($normalized['outcome'] === 'completed') { + $outcome = $this->recordSuccess($run, $task, $execution, $attempt, null, [ + 'blob' => $normalized['result'], + 'codec' => $normalized['payload_codec'], + ], $metadata); + } else { + $failure = new RestoredWorkflowException([ + 'class' => $normalized['exception_type'], + 'type' => $normalized['exception_type'], + 'message' => $normalized['message'], + 'non_retryable' => $normalized['non_retryable'], + 'code' => 0, + ]); + $outcome = $this->recordFailureOrRetry($run, $task, $execution, $attempt, $failure, $metadata); + } + if ($outcome['next_task'] instanceof WorkflowTask) { + $task->forceFill([ + 'status' => TaskStatus::Completed, + 'lease_expires_at' => null, + ])->save(); + self::projectRun($run); + } + $receipt = $outcome['event'] ?? $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityRetryScheduled) + ->where('payload->activity_attempt_id', $attemptId) + ->orderByDesc('sequence') + ->first(); + if (! $receipt instanceof WorkflowHistoryEvent) { + throw new RuntimeException('Portable local outcome did not record its canonical receipt.'); + } + + return self::portableReceipt($receipt, false); + }, 5); + } + /** * @return array{status: 'completed'|'failed'|'waiting', event: WorkflowHistoryEvent|null, next_task: WorkflowTask|null} */ @@ -102,6 +309,161 @@ public function execute( return $this->recordAttemptOutcome($task, $execution, $attempt, $result, $throwable); } + /** + * @internal Portable preparation validates and locks the execution, run and + * hosting claim before calling this shared attempt recorder. + * @param array $preparation + */ + public function startPortableAttempt( + WorkflowRun $run, + WorkflowTask $task, + ActivityExecution $execution, + string $workerAttemptId, + array $preparation, + ): ActivityAttempt { + $preparation['schedule_to_close_deadline_at'] = $execution->schedule_to_close_deadline_at?->toISOString(); + return $this->startAttempt($run, $task, $execution, $workerAttemptId, $preparation); + } + + /** + * @internal The caller holds canonical locks and has validated the retry + * authority and the expired original total deadline. No callback is admitted. + */ + public function expirePortableRetry( + WorkflowRun $run, + WorkflowTask $task, + ActivityExecution $execution, + ): WorkflowHistoryEvent { + $outcome = $this->recordTimeoutOutcome($run, $task, $execution, 'schedule_to_close'); + + return $outcome['event']; + } + + /** + * @internal The caller holds canonical locks, has validated original start + * history and has proved loss of the previous claim. Lease expiry fences + * publication. It does not establish physical callback stop. + * @param array $recovery + * @return array{event: WorkflowHistoryEvent, next_task: WorkflowTask|null} + */ + public function recoverPortableAttempt( + WorkflowRun $run, + WorkflowTask $task, + ActivityExecution $execution, + ActivityAttempt $attempt, + array $recovery, + ): array { + $metadata = [ + 'worker_attempt_id' => $attempt->worker_attempt_id, + 'local_recovery' => $recovery, + ]; + self::closeAttempt($attempt, ActivityAttemptStatus::Expired); + $timeoutKind = self::timeoutKind($execution); + $outcome = $timeoutKind === null + ? $this->recordInterruptedAttempt($run, $task, $execution, $metadata) + : $this->recordTimeoutOutcome($run, $task, $execution, $timeoutKind, $metadata); + if ($outcome['next_task'] instanceof WorkflowTask) { + $task->forceFill([ + 'status' => TaskStatus::Completed, + 'lease_expires_at' => null, + ])->save(); + self::projectRun($run); + } + $event = $outcome['event'] ?? $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityRetryScheduled) + ->where('payload->activity_attempt_id', $attempt->id) + ->orderByDesc('sequence') + ->first(); + if (! $event instanceof WorkflowHistoryEvent) { + throw new RuntimeException('Portable local recovery did not record its canonical receipt.'); + } + + return [ + 'event' => $event, + 'next_task' => $outcome['next_task'], + ]; + } + + /** @param array $report + * @return array + */ + private static function normalizePortableReport(array $report): array + { + $outcome = $report['outcome'] ?? null; + $fields = match ($outcome) { + 'completed' => ['outcome', 'result', 'payload_codec'], + 'failed' => ['outcome', 'message', 'exception_type', 'non_retryable'], + 'timed_out' => ['outcome'], + default => [], + }; + if ($fields === [] || array_diff(array_keys($report), $fields) !== []) { + throw ValidationException::withMessages([ + 'local_activity' => ['Invalid prepared outcome report.'], + ]); + } + if ($outcome === 'completed') { + $result = PayloadEnvelopeResolver::resolveCommandPayloadWithCodec($report['result'] ?? null); + $codec = $result['codec'] ?? ($report['payload_codec'] ?? null); + if (! is_string($result['payload']) || $result['payload'] === '' || $codec !== 'avro' + || (array_key_exists('payload_codec', $report) && $report['payload_codec'] !== $codec)) { + throw ValidationException::withMessages([ + 'local_activity.result' => ['Expected an encoded Avro result.'], + ]); + } + return [ + 'outcome' => $outcome, + 'result' => $result['payload'], + 'payload_codec' => $codec, + ]; + } + if ($outcome === 'failed') { + $message = $report['message'] ?? null; + $type = $report['exception_type'] ?? RuntimeException::class; + if (! is_string($message) || ! is_string($type) || trim($type) === '' || strlen($type) > 255 + || (array_key_exists('non_retryable', $report) && ! is_bool($report['non_retryable']))) { + throw ValidationException::withMessages([ + 'local_activity.failure' => ['Expected a typed failure report.'], + ]); + } + return [ + 'outcome' => $outcome, + 'message' => $message, + 'exception_type' => $type, + 'non_retryable' => $report['non_retryable'] ?? false, + ]; + } + + return [ + 'outcome' => $outcome, + ]; + } + + /** + * @return array + */ + private static function portableReceipt(WorkflowHistoryEvent $event, bool $duplicate): array + { + $payload = $event->payload; + $retryTaskId = $payload['retry_task_id'] ?? null; + + return [ + 'recorded' => true, + 'duplicate' => $duplicate, + 'reason' => null, + 'event_id' => $event->id, + 'event_type' => $event->event_type->value, + 'workflow_run_id' => $event->workflow_run_id, + 'workflow_task_id' => $event->workflow_task_id, + 'workflow_task_attempt' => $payload['local_outcome']['workflow_task_attempt'], + 'activity_execution_id' => $payload['activity_execution_id'], + 'activity_attempt_id' => $payload['activity_attempt_id'], + 'worker_attempt_id' => $payload['worker_attempt_id'], + 'recorded_at' => $event->recorded_at->toISOString(), + 'claim_released' => $event->event_type === HistoryEventType::ActivityRetryScheduled, + 'created_task_ids' => is_string($retryTaskId) ? [$retryTaskId] : [], + ]; + } + /** * @return array{execution: ActivityExecution|null, attempt: ActivityAttempt|null, event: WorkflowHistoryEvent|null, next_task: WorkflowTask|null} */ @@ -250,20 +612,31 @@ private function startAttempt( WorkflowRun $run, WorkflowTask $task, ActivityExecution $execution, + ?string $workerAttemptId = null, + array $preparation = [], ): ActivityAttempt { $now = now(); $attemptId = (string) Str::ulid(); $attemptNumber = ((int) $execution->attempt_count) + 1; $retryPolicy = is_array($execution->retry_policy) ? $execution->retry_policy : []; - $leaseExpiresAt = LocalActivityRuntime::renewWorkflowTask($task) - ?? $task->lease_expires_at - ?? LocalActivityRuntime::workflowTaskLeaseExpiresAt(); + $leaseExpiresAt = $workerAttemptId === null + ? LocalActivityRuntime::renewWorkflowTask($task, run: $run) + ?? $task->lease_expires_at + ?? LocalActivityRuntime::workflowTaskLeaseExpiresAt() + : $task->lease_expires_at; $startToCloseTimeout = is_int($retryPolicy['start_to_close_timeout'] ?? null) ? $retryPolicy['start_to_close_timeout'] : null; $heartbeatTimeout = is_int($retryPolicy['heartbeat_timeout'] ?? null) ? $retryPolicy['heartbeat_timeout'] : null; + $cleanupDeadline = PortableLocalActivityCleanup::currentDeadline( + $run, + $execution->activity_options['cancellation_cleanup'] ?? null + ); + if ($cleanupDeadline !== null && $leaseExpiresAt !== null && $cleanupDeadline->lt($leaseExpiresAt)) { + $leaseExpiresAt = $cleanupDeadline; + } $execution->forceFill([ 'status' => ActivityStatus::Running, @@ -271,15 +644,24 @@ private function startAttempt( 'current_attempt_id' => $attemptId, 'started_at' => $now, 'last_heartbeat_at' => $now, - 'close_deadline_at' => $startToCloseTimeout === null ? null : $now->copy() - ->addSeconds($startToCloseTimeout), - 'heartbeat_deadline_at' => $heartbeatTimeout === null ? null : $now->copy() - ->addSeconds($heartbeatTimeout), + 'close_deadline_at' => PortableLocalActivityCleanup::bound( + $execution, + $startToCloseTimeout === null ? null : $now->copy() + ->addSeconds($startToCloseTimeout), + $run, + ), + 'heartbeat_deadline_at' => PortableLocalActivityCleanup::bound( + $execution, + $heartbeatTimeout === null ? null : $now->copy() + ->addSeconds($heartbeatTimeout), + $run, + ), ])->save(); /** @var ActivityAttempt $attempt */ $attempt = ActivityAttempt::query()->create([ 'id' => $attemptId, + 'worker_attempt_id' => $workerAttemptId, 'workflow_run_id' => $run->id, 'activity_execution_id' => $execution->id, 'workflow_task_id' => $task->id, @@ -291,6 +673,10 @@ private function startAttempt( 'lease_expires_at' => $leaseExpiresAt, ]); + $metadata = $workerAttemptId === null ? [] : [ + 'worker_attempt_id' => $workerAttemptId, + 'local_preparation' => $preparation, + ]; WorkflowHistoryEvent::record($run, HistoryEventType::ActivityStarted, LocalActivityRuntime::eventPayload([ 'activity_execution_id' => $execution->id, 'activity_attempt_id' => $attempt->id, @@ -302,6 +688,7 @@ private function startAttempt( 'lease_expires_at' => $leaseExpiresAt?->toJSON(), 'activity' => ActivitySnapshot::fromExecution($execution), 'activity_attempt' => self::attemptSnapshot($attempt), + ...$metadata, ]), $task); LifecycleEventDispatcher::activityStarted( @@ -390,8 +777,10 @@ private function recordSuccess( ActivityExecution $execution, ActivityAttempt $attempt, mixed $result, + ?array $encodedResult = null, + array $historyMetadata = [], ): array { - $encoded = self::serializeWithCodec($result, self::preferredPayloadCodec($execution, $run)); + $encoded = $encodedResult ?? self::serializeWithCodec($result, self::preferredPayloadCodec($execution, $run)); $encoded['blob'] = ExternalPayloads::externalizeForNamespace( $encoded['blob'], $encoded['codec'], @@ -402,7 +791,7 @@ private function recordSuccess( StructuralLimits::guardPayloadSize($encoded['blob']); } catch (StructuralLimitExceededException $limitExceeded) { /** @var array{status: 'failed'|'waiting', event: WorkflowHistoryEvent|null, next_task: WorkflowTask|null} $failure */ - $failure = $this->recordFailureOrRetry($run, $task, $execution, $attempt, $limitExceeded); + $failure = $this->recordFailureOrRetry($run, $task, $execution, $attempt, $limitExceeded, $historyMetadata); return $failure; } @@ -438,6 +827,7 @@ private function recordSuccess( 'workflow_task_id' => $task->id, 'activity' => ActivitySnapshot::fromExecution($execution), 'activity_attempt' => self::attemptSnapshot($attempt->fresh() ?? $attempt), + ...$historyMetadata, ]), $task ); @@ -469,6 +859,7 @@ private function recordFailureOrRetry( ActivityExecution $execution, ActivityAttempt $attempt, Throwable $throwable, + array $historyMetadata = [], ): array { $attemptNumber = max(1, (int) $attempt->attempt_number); $maxAttempts = ActivityRetryPolicy::maxAttemptsFromSnapshot($execution); @@ -481,6 +872,7 @@ private function recordFailureOrRetry( $attempt, $throwable, LocalActivityRuntime::RETRY_REASON_FAILURE, + historyMetadata: $historyMetadata, ); return [ @@ -490,7 +882,7 @@ private function recordFailureOrRetry( ]; } - $event = $this->recordTerminalFailure($run, $task, $execution, $attempt, $throwable); + $event = $this->recordTerminalFailure($run, $task, $execution, $attempt, $throwable, $historyMetadata); return [ 'status' => 'failed', @@ -507,6 +899,7 @@ private function scheduleRetry( Throwable $throwable, string $retryReason, ?string $timeoutKind = null, + array $historyMetadata = [], ): WorkflowTask { $attemptNumber = max(1, (int) ($attempt?->attempt_number ?? $execution->attempt_count)); $maxAttempts = ActivityRetryPolicy::maxAttemptsFromSnapshot($execution); @@ -516,7 +909,7 @@ private function scheduleRetry( $exceptionPayload = FailureFactory::payload($throwable); $runCodec = is_string($run->payload_codec) && $run->payload_codec !== '' ? $run->payload_codec : null; - if ($attempt instanceof ActivityAttempt) { + if ($attempt instanceof ActivityAttempt && ! isset($historyMetadata['local_recovery'])) { self::closeAttempt($attempt, ActivityAttemptStatus::Failed); } @@ -576,6 +969,11 @@ private function scheduleRetry( 'exception' => $exceptionPayload, 'workflow_task_id' => $task->id, 'activity' => ActivitySnapshot::fromExecution($execution), + ...($historyMetadata !== [] && $attempt instanceof ActivityAttempt + ? [ + 'activity_attempt' => self::attemptSnapshot($attempt), + ] : []), + ...$historyMetadata, ]), $task ); @@ -591,6 +989,7 @@ private function recordTerminalFailure( ActivityExecution $execution, ?ActivityAttempt $attempt, Throwable $throwable, + array $historyMetadata = [], ): WorkflowHistoryEvent { $exceptionPayload = FailureFactory::payload($throwable); $failureCategory = $throwable instanceof StructuralLimitExceededException @@ -624,7 +1023,7 @@ private function recordTerminalFailure( 'heartbeat_deadline_at' => null, ])->save(); - if ($attempt instanceof ActivityAttempt) { + if ($attempt instanceof ActivityAttempt && ! isset($historyMetadata['local_recovery'])) { self::closeAttempt($attempt, ActivityAttemptStatus::Failed); } @@ -651,6 +1050,7 @@ private function recordTerminalFailure( 'activity_attempt' => $attempt instanceof ActivityAttempt ? self::attemptSnapshot($attempt->fresh() ?? $attempt) : null, + ...$historyMetadata, ], self::structuralLimitPayload($throwable))), $task ); @@ -687,6 +1087,7 @@ private function recordTimeoutOutcome( WorkflowTask $task, ActivityExecution $execution, string $timeoutKind, + array $historyMetadata = [], ): array { $attempt = self::currentAttempt($execution); $attemptNumber = max(1, (int) ($attempt?->attempt_number ?? $execution->attempt_count)); @@ -705,11 +1106,12 @@ private function recordTimeoutOutcome( $throwable, LocalActivityRuntime::RETRY_REASON_TIMEOUT, $timeoutKind, + $historyMetadata, ), ]; } - if ($attempt instanceof ActivityAttempt) { + if ($attempt instanceof ActivityAttempt && ! isset($historyMetadata['local_recovery'])) { self::closeAttempt($attempt, ActivityAttemptStatus::Failed); } @@ -763,6 +1165,7 @@ private function recordTimeoutOutcome( 'activity_attempt' => $attempt instanceof ActivityAttempt ? self::attemptSnapshot($attempt->fresh() ?? $attempt) : null, + ...$historyMetadata, ]), $task ); @@ -801,6 +1204,7 @@ private function recordInterruptedAttempt( WorkflowRun $run, WorkflowTask $task, ActivityExecution $execution, + array $historyMetadata = [], ): array { $attempt = self::currentAttempt($execution); $attemptNumber = max(1, (int) ($attempt?->attempt_number ?? $execution->attempt_count)); @@ -810,13 +1214,18 @@ private function recordInterruptedAttempt( } $maxAttempts = ActivityRetryPolicy::maxAttemptsFromSnapshot($execution); - $throwable = new RuntimeException( - 'Local activity attempt was interrupted before a terminal event and will be replayed from durable history.' - ); + $throwable = new RuntimeException('Local activity attempt was interrupted before a terminal event.'); if ($attemptNumber >= $maxAttempts) { return [ - 'event' => $this->recordTerminalFailure($run, $task, $execution, $attempt, $throwable), + 'event' => $this->recordTerminalFailure( + $run, + $task, + $execution, + $attempt, + $throwable, + $historyMetadata + ), 'next_task' => null, ]; } @@ -830,6 +1239,7 @@ private function recordInterruptedAttempt( $attempt, $throwable, LocalActivityRuntime::RETRY_REASON_COLD_REPLAY, + historyMetadata: $historyMetadata, ), ]; } @@ -976,6 +1386,7 @@ private static function attemptSnapshot(ActivityAttempt $attempt): array { return array_filter([ 'id' => $attempt->id, + 'worker_attempt_id' => $attempt->worker_attempt_id, 'activity_execution_id' => $attempt->activity_execution_id, 'task_id' => $attempt->workflow_task_id, 'attempt_number' => $attempt->attempt_number, diff --git a/src/V2/Support/LocalActivityRuntime.php b/src/V2/Support/LocalActivityRuntime.php index 2e3be7c70..49e7e3d75 100644 --- a/src/V2/Support/LocalActivityRuntime.php +++ b/src/V2/Support/LocalActivityRuntime.php @@ -8,6 +8,7 @@ use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Enums\TaskType; use Workflow\V2\Models\ActivityExecution; +use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowRunSummary; use Workflow\V2\Models\WorkflowTask; @@ -34,9 +35,11 @@ public static function isExecution(ActivityExecution $execution): bool */ public static function eventPayload(array $payload = []): array { + $path = $payload['activity']['parallel_group_path'] ?? []; return [ 'execution_mode' => self::EXECUTION_MODE, 'local_activity' => true, + ...ParallelChildGroup::payloadForPath(is_array($path) ? $path : []), ...$payload, ]; } @@ -56,6 +59,7 @@ public static function workflowTaskPayload(ActivityExecution $execution, array $ 'activity_type' => $execution->activity_type ?? $execution->activity_class, 'workflow_sequence' => $execution->sequence, 'execution_mode' => self::EXECUTION_MODE, + ...ParallelChildGroup::payloadForPath($execution->parallel_group_path ?? []), ...$payload, ]; } @@ -65,13 +69,23 @@ public static function workflowTaskLeaseExpiresAt(): CarbonInterface return WorkflowTaskLease::expiresAt(); } - public static function renewWorkflowTask(WorkflowTask $task): ?CarbonInterface - { + public static function renewWorkflowTask( + WorkflowTask $task, + ?CarbonInterface $deadline = null, + bool $cleanup = false, + ?WorkflowRun $run = null, + ): ?CarbonInterface { if ($task->task_type !== TaskType::Workflow || $task->status !== TaskStatus::Leased) { return null; } - $leaseExpiresAt = self::workflowTaskLeaseExpiresAt(); + $run ??= ConfiguredV2Models::query('run_model', WorkflowRun::class)->find($task->workflow_run_id); + if ($run !== null && $run->id !== $task->workflow_run_id) { + throw new \LogicException('Workflow hosting renewal requires its original run.'); + } + $leaseExpiresAt = $deadline === null + ? ($cleanup ? CancellationCleanupLease::renewableExpiry() : CancellationCleanupLease::forScopes($run)) + : CancellationCleanupLease::forDeadline($deadline); $task->forceFill([ 'lease_expires_at' => $leaseExpiresAt, diff --git a/src/V2/Support/ParentCloseCancellation.php b/src/V2/Support/ParentCloseCancellation.php new file mode 100644 index 000000000..2b11120e4 --- /dev/null +++ b/src/V2/Support/ParentCloseCancellation.php @@ -0,0 +1,114 @@ +cancellation_request_command_id) || self::context($parent) !== null) { + return; + } + + $closed = self::closure($parent); + if ($closed === null) { + throw new LogicException('cancellation_parent_closure_unavailable'); + } + $requestId = (string) Str::ulid(); + $requestedAt = $closed->recorded_at->toImmutable(); + $context = CancellationContext::fromArray([ + 'schema' => 'durable-workflow.cancellation-context/v1', + 'request_id' => $requestId, + 'root_request_id' => $requestId, + 'root_workflow_instance_id' => $parent->workflow_instance_id, + 'root_workflow_run_id' => $parent->id, + 'parent_request_id' => null, + 'reason' => $reason, + 'requester' => [ + 'type' => 'workflow', + 'id' => $parent->workflow_instance_id, + 'label' => 'Parent-close policy', + ], + 'source' => 'parent_close_policy', + 'requested_at' => $requestedAt->toISOString(), + 'cleanup_deadline_at' => $requestedAt->addSeconds(self::CLEANUP_TIMEOUT_SECONDS)->toISOString(), + 'lineage' => [[ + 'request_id' => $requestId, + 'workflow_instance_id' => $parent->workflow_instance_id, + 'workflow_run_id' => $parent->id, + ]], + ]); + WorkflowHistoryEvent::record($parent, HistoryEventType::ParentCloseCancellationRequested, [ + 'parent_terminal_history_event_id' => $closed->id, + 'parent_terminal_event_type' => $closed->event_type->value, + 'cancellation' => $context->toArray(), + ]); + } + + public static function context(WorkflowRun $parent): ?CancellationContext + { + /** @var WorkflowHistoryEvent|null $origin */ + $origin = $parent->historyEvents() + ->where('event_type', HistoryEventType::ParentCloseCancellationRequested->value) + ->orderBy('sequence') + ->first(); + if ($origin === null) { + return null; + } + $snapshot = $origin->payload['cancellation'] ?? null; + if (! is_array($snapshot)) { + throw new LogicException('cancellation_parent_context_mismatch'); + } + $context = CancellationContext::fromArray($snapshot); + $closed = self::closure($parent); + if ($closed === null || $origin->sequence <= $closed->sequence + || ($origin->payload['parent_terminal_history_event_id'] ?? null) !== $closed->id + || ($origin->payload['parent_terminal_event_type'] ?? null) !== $closed->event_type->value + || $context->rootWorkflowRunId !== $parent->id + || $context->rootWorkflowInstanceId !== $parent->workflow_instance_id + || $context->requestId !== $context->rootRequestId + || count($context->lineage) !== 1 + || $context->source !== 'parent_close_policy' + || ! $context->requestedAt() + ->equalTo($closed->recorded_at) + || ! $context->deadline() + ->equalTo($context->requestedAt()->addSeconds(self::CLEANUP_TIMEOUT_SECONDS))) { + throw new LogicException('cancellation_parent_context_mismatch'); + } + + return $context; + } + + private static function closure(WorkflowRun $parent): ?WorkflowHistoryEvent + { + /** @var WorkflowHistoryEvent|null $event */ + $event = $parent->historyEvents() + ->whereIn('event_type', [ + HistoryEventType::WorkflowCompleted->value, + HistoryEventType::WorkflowFailed->value, + HistoryEventType::WorkflowCancelled->value, + HistoryEventType::WorkflowTerminated->value, + HistoryEventType::WorkflowTimedOut->value, + ])->orderByDesc('sequence') + ->first(); + + return $event; + } +} diff --git a/src/V2/Support/ParentClosePolicyEnforcer.php b/src/V2/Support/ParentClosePolicyEnforcer.php index 865c84bdc..3fa4dec78 100644 --- a/src/V2/Support/ParentClosePolicyEnforcer.php +++ b/src/V2/Support/ParentClosePolicyEnforcer.php @@ -5,6 +5,8 @@ namespace Workflow\V2\Support; use Illuminate\Support\Facades\Log; +use LogicException; +use Workflow\V2\CommandContext; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\ParentClosePolicy; use Workflow\V2\Enums\RunStatus; @@ -32,6 +34,22 @@ final class ParentClosePolicyEnforcer * @return list Instance IDs of children that had policy applied */ public static function enforce(WorkflowRun $run): array + { + return $run->getConnection() + ->transaction(static function () use ($run): array { + /** @var WorkflowRun $lockedRun */ + $lockedRun = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->lockForUpdate() + ->findOrFail($run->id); + + return self::enforceLocked($lockedRun); + }, 3); + } + + /** + * @return list + */ + private static function enforceLocked(WorkflowRun $run): array { $appliedTo = []; @@ -77,6 +95,49 @@ public static function enforce(WorkflowRun $run): array try { $stub = WorkflowStub::load($childInstanceId); + $cancellation = []; + + if ($policy === ParentClosePolicy::RequestCancellation) { + // A previously accepted request is already durable. Replaying + // enforcement must not append another receipt or grant time. + if ($run->historyEvents() + ->where('event_type', HistoryEventType::ParentClosePolicyApplied->value) + ->where('payload->child_run_id', $childRun->id) + ->where('payload->policy', $policy->value) + ->exists()) { + continue; + } + ParentCloseCancellation::ensureOrigin($run, $reason); + $result = $stub->withCommandContext(CommandContext::workflow( + $run->workflow_instance_id, + $run->id, + $link->sequence ?? 0, + $link->id, + )) + ->attemptRequestCancellationFromParent($run->id); + $cancellation = [ + 'request_id' => $result->commandId(), + 'cancellation' => $result->cancellationContext()?->toArray(), + ]; + if (! $result->accepted()) { + WorkflowHistoryEvent::record($run, HistoryEventType::ParentClosePolicyFailed, array_merge([ + 'child_instance_id' => $childInstanceId, + 'child_run_id' => $childRun->id, + 'policy' => $policy->value, + 'reason' => $reason, + 'error' => $result->rejectionReason() ?? 'cancellation_request_rejected', + 'request_diagnostics' => $result->payloadValues([ + 'existing_request_id', 'existing_root_request_id', 'existing_cleanup_deadline_at', + 'incoming_root_request_id', 'incoming_cleanup_deadline_at', + ]), + ], $cancellation)); + + continue; + } + if ($result->cancellationContext() === null) { + throw new LogicException('cancellation_child_context_unavailable'); + } + } match ($policy) { ParentClosePolicy::RequestCancel => $stub->attemptCancel($reason), @@ -86,12 +147,12 @@ public static function enforce(WorkflowRun $run): array self::markChildCallForPolicy($run, $link, $policy); - WorkflowHistoryEvent::record($run, HistoryEventType::ParentClosePolicyApplied, [ + WorkflowHistoryEvent::record($run, HistoryEventType::ParentClosePolicyApplied, array_merge([ 'child_instance_id' => $childInstanceId, 'child_run_id' => $childRun->id, 'policy' => $policy->value, 'reason' => $reason, - ]); + ], $cancellation)); $appliedTo[] = $childInstanceId; } catch (\Throwable $throwable) { diff --git a/src/V2/Support/PlatformConformanceSuite.php b/src/V2/Support/PlatformConformanceSuite.php index 6173c60a8..cd100d83e 100644 --- a/src/V2/Support/PlatformConformanceSuite.php +++ b/src/V2/Support/PlatformConformanceSuite.php @@ -147,8 +147,11 @@ public static function workflowSourceRelease(): string } $release = is_array($composer) ? ($composer['extra']['durable-workflow']['product-train'] ?? null) : null; - if (! is_string($release) || preg_match('/\A2\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\z/D', $release) !== 1) { - throw new RuntimeException('Workflow package metadata must declare one exact stable 2.x release.'); + if (! is_string($release) || preg_match( + '/\A2\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)(?:-(?:alpha|beta|rc)\.(?:0|[1-9][0-9]*))?\z/D', + $release + ) !== 1) { + throw new RuntimeException('Workflow package metadata must declare one exact 2.x release.'); } return $release; diff --git a/src/V2/Support/PortableCancellationScopeDelivery.php b/src/V2/Support/PortableCancellationScopeDelivery.php new file mode 100644 index 000000000..ff408d163 --- /dev/null +++ b/src/V2/Support/PortableCancellationScopeDelivery.php @@ -0,0 +1,304 @@ + + */ + public static function mutate( + bool $preparing, + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + string $scopeId, + string $requestId, + int $sequence, + string $callKind, + int $sequenceSpan, + ?int $operationSequence, + int $operationSequenceSpan, + string $protocolVersion, + bool $renewWorkflowLease = true, + ): array { + $response = [ + 'prepared' => false, + 'delivered' => false, + 'claim_released' => false, + 'task_id' => $taskId, + 'created_task_ids' => [], + 'reason' => null, + ]; + $refused = static fn (string $reason): array => [ + ...$response, + 'reason' => $reason, + ]; + if (! WorkerProtocolVersion::supportsCancellationScopeMembership($protocolVersion)) { + return $refused('cancellation_scope_requires_protocol_1_20'); + } + if ($leaseOwner === '' || $workflowTaskAttempt < 1 || $scopeId === '' || $requestId === '') { + return $refused('invalid_cancellation_scope_delivery'); + } + /** @var WorkflowTask|null $claim */ + $claim = ConfiguredV2Models::query('task_model', WorkflowTask::class)->find($taskId); + if ($claim === null) { + return $refused('task_not_found'); + } + /** @var WorkflowRun|null $run */ + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class)->find($claim->workflow_run_id); + if ($run === null) { + return $refused('run_not_found'); + } + if ($claim->namespace !== $run->namespace || $claim->task_type !== TaskType::Workflow + || $claim->status !== TaskStatus::Leased || $claim->lease_owner !== $leaseOwner + || $claim->attempt_count !== $workflowTaskAttempt || $claim->lease_expires_at === null + || now() + ->gte($claim->lease_expires_at)) { + return $refused('cancellation_scope_workflow_claim_mismatch'); + } + if ($run->getConnection()->transactionLevel() !== 0) { + return $refused('cancellation_scope_preparation_requires_own_transaction'); + } + try { + $preparation = $preparing ? null : CancellationScopeDelivery::prepared($run, $scopeId); + if (! $preparing && $preparation === null) { + return $refused('cancellation_scope_delivery_not_prepared'); + } + // Replay validates the exact original boundary before any effects. + // Its run/task locks are released before an operation actor starts. + $event = CancellationScopeDelivery::prepare( + $run, + $claim, + $scopeId, + $requestId, + $sequence, + $callKind, + $protocolVersion, + $sequenceSpan, + $operationSequence, + $operationSequenceSpan, + $renewWorkflowLease, + ); + $response = [...$response, ...self::frame($run, $event->payload, $event->id)]; + if (! $preparing) { + $run->unsetRelation('historyEvents'); + self::assertOperationAddresses($run); + $references = [ScopedCancellationPreparation::fromEvent($run, $event)]; + foreach ($event->payload['descendant_members'] as $descendant) { + $references[] = ScopedCancellationPreparation::forScope($run, $descendant['scope_id'], $event->id) + ?? throw new LogicException('cancellation_scope_descendant_not_prepared'); + } + // Check the whole subtree before dispatching even the parent's + // first actor. An older prepared boundary retains its effects. + foreach ($references as $reference) { + if (ScopedCancellationReconciliation::pending($run, $reference)) { + return [ + ...$response, + 'reason' => 'cancellation_scope_descendant_delivery_pending', + ]; + } + } + $response['activity_cancellations'] = []; + $response['timer_cancellations'] = []; + $response['wait_cancellations'] = []; + $response['child_cancellations'] = []; + foreach ($references as $reference) { + $completed = ScopedCancellationReconciliation::completed($run, $reference); + if ($completed !== null) { + foreach ($completed->receipts($run) as $field => $receipts) { + array_push($response[$field], ...$receipts); + } + continue; + } + foreach ($reference->waitMembers as $member) { + $receipt = ScopedWaitCancellation::fence( + $run, + $claim, + $member['wait_id'], + $reference->scopeId, + $reference->requestId, + $protocolVersion, + $reference->historyEventId, + ); + unset($receipt['timer_cancellation']); + $response['wait_cancellations'][] = $receipt; + } + foreach ($reference->timerMembers as $member) { + $response['timer_cancellations'][] = ScopedTimerCancellation::fence( + $run, + $claim, + $member['timer_id'], + $reference->scopeId, + $reference->requestId, + $protocolVersion, + $reference->historyEventId, + ); + } + foreach ($reference->activityMembers as $member) { + $receipt = ScopedActivityCancellation::fence( + $run, + $claim, + $member['activity_execution_id'], + $reference->scopeId, + $reference->requestId, + $protocolVersion, + $reference->historyEventId, + ); + // Database JSON key order must not change a retried receipt. + if (isset($receipt['cancellation_scope'])) { + $snapshot = $receipt['cancellation_scope']; + $receipt['cancellation_scope'] = [ + 'schema' => $snapshot['schema'], + 'workflow_run_id' => $snapshot['workflow_run_id'], + 'scope_id' => $snapshot['scope_id'], + 'request_id' => $snapshot['request_id'], + 'request_history_event_id' => $snapshot['request_history_event_id'], + 'cancellation' => ScopedCancellationContext::fromArray( + $snapshot['cancellation'] + )->toArray(), + 'authority_deadline_at' => $snapshot['authority_deadline_at'], + ]; + } + $response['activity_cancellations'][] = [ + 'sequence' => $member['sequence'], + 'activity_execution_id' => $member['activity_execution_id'], + ...$receipt, + ]; + } + foreach ($reference->childMembers as $member) { + $response['child_cancellations'][] = ScopedChildCancellationDelivery::request( + $run, + $claim, + $member['child_call_id'], + $reference->scopeId, + $reference->requestId, + $protocolVersion, + $reference->historyEventId, + ); + } + } + // Recording remains a barrier over every original member's policy. + $event = CancellationScopeDelivery::record( + $run, + $claim, + $scopeId, + $requestId, + $sequence, + $callKind, + $protocolVersion, + $sequenceSpan, + $operationSequence, + $operationSequenceSpan, + ); + } + return [ + ...$response, + ...self::frame( + $run, + $event->payload, + $preparing ? $event->id : $event->payload['preparation_history_event_id'] + ), + 'delivered' => ! $preparing, + 'history_event_id' => $event->id, + 'reason' => null, + ]; + } catch (LogicException $error) { + $reason = $error->getMessage(); + if (! str_starts_with($reason, 'cancellation_scope_') + && ! str_starts_with($reason, 'cancellation_delivery_') + && ! in_array( + $reason, + ['invalid_cancellation_scope_delivery', 'invalid_cancellation_delivery', + 'activity_cancellation_owned_by_another_request'], + true + )) { + throw $error; + } + return [ + ...$response, + 'reason' => $reason, + ]; + } + } + + /** + * @param array $payload + * @return array + */ + private static function frame(WorkflowRun $run, array $payload, string $preparationId): array + { + return [ + 'prepared' => true, + 'workflow_run_id' => $run->id, + 'preparation_history_event_id' => $preparationId, + 'scope_id' => $payload['scope_id'], + 'request_id' => $payload['request_id'], + 'cancellation' => ScopedCancellationContext::fromArray($payload['cancellation'])->toArray(), + 'sequence' => $payload['sequence'], + 'call_kind' => $payload['call_kind'], + 'sequence_span' => $payload['sequence_span'], + 'operation_sequence' => $payload['operation_sequence'], + 'operation_sequence_span' => $payload['operation_sequence_span'], + 'authority_deadline_at' => $payload['authority_deadline_at'], + ...(array_key_exists('activity_members', $payload) ? [ + 'activity_members' => array_map(static fn (array $member): array => [ + 'sequence' => $member['sequence'], + 'activity_execution_id' => $member['activity_execution_id'], + 'descriptor_hash' => $member['descriptor_hash'], + ], $payload['activity_members']), + ] : []), + ...(array_key_exists('timer_members', $payload) ? [ + 'timer_members' => ScopedTimerCancellation::normalizeMembers($payload['timer_members']), + ] : []), + ...(array_key_exists('wait_members', $payload) ? [ + 'wait_members' => ScopedWaitCancellation::normalizeMembers($payload['wait_members']), + ] : []), + ...(array_key_exists('child_members', $payload) ? [ + 'child_members' => ScopedChildCancellation::normalizeMembers($payload['child_members']), + ] : []), + ]; + } + + private static function assertOperationAddresses(WorkflowRun $run): void + { + $scopes = CancellationScopeHistory::forRun($run); + foreach ($run->historyEvents as $event) { + $descriptor = match ($event->event_type) { + HistoryEventType::ActivityScheduled => $event->payload['activity'] ?? [], + HistoryEventType::TimerScheduled => $event->payload['timer'] ?? [], + HistoryEventType::ChildWorkflowScheduled => $event->payload['child_workflow'] ?? [], + HistoryEventType::ConditionWaitOpened, HistoryEventType::SignalWaitOpened => [], + default => null, + }; + if ($descriptor === null) { + continue; + } + if (! is_array($descriptor)) { + throw new LogicException('cancellation_scope_delivery_history_invalid'); + } + $address = $event->payload['cancellation_scope_id'] ?? $descriptor['cancellation_scope_id'] + ?? CancellationScopeHistory::ROOT_SCOPE_ID; + if (! is_string($address) + || ($address !== CancellationScopeHistory::ROOT_SCOPE_ID && ! isset($scopes[$address])) + || (array_key_exists('cancellation_scope_id', $descriptor) + && $descriptor['cancellation_scope_id'] !== $address)) { + throw new LogicException('cancellation_scope_delivery_history_invalid'); + } + } + } +} diff --git a/src/V2/Support/PortableCancellationScopeOpening.php b/src/V2/Support/PortableCancellationScopeOpening.php new file mode 100644 index 000000000..dadf45b2c --- /dev/null +++ b/src/V2/Support/PortableCancellationScopeOpening.php @@ -0,0 +1,122 @@ + + */ + public static function open( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + string $parentScopeId, + bool $shieldParent, + string $protocolVersion, + ): array { + $refused = static fn (string $reason): array => [ + 'opened' => false, + 'duplicate' => false, + 'claim_released' => false, + 'task_id' => $taskId, + 'created_task_ids' => [], + 'reason' => $reason, + ]; + if (! WorkerProtocolVersion::supportsCancellationScopeMembership($protocolVersion)) { + return $refused('cancellation_scope_requires_protocol_1_20'); + } + if ($workflowTaskAttempt < 1 || $sequence < 1 || $leaseOwner === '' || trim($parentScopeId) === '' + || strlen($parentScopeId) > 255 || preg_match('//u', $parentScopeId) !== 1) { + return $refused('invalid_cancellation_scope_open'); + } + /** @var WorkflowTask|null $snapshot */ + $snapshot = ConfiguredV2Models::query('task_model', WorkflowTask::class)->find($taskId); + if ($snapshot === null) { + return $refused('task_not_found'); + } + try { + return $snapshot->getConnection() + ->transaction(static function () use ( + $snapshot, + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $sequence, + $parentScopeId, + $shieldParent, + $protocolVersion, + $refused + ): array { + // Match the canonical opening kernel's run-before-task order. + /** @var WorkflowRun|null $run */ + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class)->lockForUpdate() + ->find($snapshot->workflow_run_id); + if ($run === null) { + return $refused('run_not_found'); + } + /** @var WorkflowTask|null $claim */ + $claim = ConfiguredV2Models::query('task_model', WorkflowTask::class)->lockForUpdate()->find( + $taskId + ); + if ($claim === null || $claim->workflow_run_id !== $run->id || $claim->namespace !== $run->namespace + || $claim->task_type !== TaskType::Workflow || $claim->status !== TaskStatus::Leased + || $claim->lease_owner !== $leaseOwner || $claim->attempt_count !== $workflowTaskAttempt + || $claim->lease_expires_at === null || now() + ->gte($claim->lease_expires_at)) { + return $refused('cancellation_scope_workflow_claim_mismatch'); + } + $duplicate = $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeOpened) + ->where('payload->sequence', $sequence) + ->exists(); + $event = CancellationScopeHistory::open( + $run, + $claim, + $sequence, + $protocolVersion, + $parentScopeId, + $shieldParent + ); + + return [ + 'opened' => true, + 'duplicate' => $duplicate, + 'claim_released' => false, + 'task_id' => $taskId, + 'workflow_run_id' => $run->id, + 'history_event_id' => $event->id, + 'scope_id' => $event->payload['scope_id'], + 'parent_scope_id' => $event->payload['parent_scope_id'], + 'shield_parent' => $event->payload['shield_parent'], + 'sequence' => $event->payload['sequence'], + 'created_task_ids' => [], + 'reason' => null, + ]; + }, 3); + } catch (HistoryEventShapeMismatchException) { + return $refused('cancellation_scope_replay_mismatch'); + } catch (LogicException $exception) { + if (! in_array($exception->getMessage(), [ + 'cancellation_scope_workflow_claim_mismatch', 'cancellation_scope_run_not_active', + 'cancellation_scope_parent_not_recorded', 'cancellation_scope_sequence_mismatch', + 'cancellation_scope_history_invalid', + ], true)) { + throw $exception; + } + return $refused($exception->getMessage()); + } + } +} diff --git a/src/V2/Support/PortableLocalActivityCleanup.php b/src/V2/Support/PortableLocalActivityCleanup.php new file mode 100644 index 000000000..8a5609f13 --- /dev/null +++ b/src/V2/Support/PortableLocalActivityCleanup.php @@ -0,0 +1,328 @@ +|null $proof + * @return array|null + */ + public static function snapshot( + WorkflowRun $run, + ?array $proof, + int $sequence, + string $operationScope = CancellationScopeHistory::ROOT_SCOPE_ID, + ): ?array { + if ($proof !== null && array_key_exists('scope_id', $proof)) { + return self::scopedSnapshot($run, $proof, $sequence, $operationScope); + } + if ($proof === null || array_diff(array_keys($proof), ['request_id', 'delivery_history_event_id']) !== [] + || ($proof['request_id'] ?? null) !== $run->cancellation_request_command_id + || ! is_string($proof['delivery_history_event_id'] ?? null)) { + return null; + } + $request = $run->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationRequested) + ->where('workflow_command_id', $run->cancellation_request_command_id) + ->first(); + $delivery = $run->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered) + ->where('workflow_command_id', $run->cancellation_request_command_id) + ->whereKey($proof['delivery_history_event_id'])->first(); + if (! $request instanceof WorkflowHistoryEvent || ! $delivery instanceof WorkflowHistoryEvent + || $delivery->sequence <= $request->sequence + || ($delivery->payload['workflow_command_id'] ?? null) !== $run->cancellation_request_command_id + || ($delivery->payload['workflow_run_id'] ?? null) !== $run->id + || ! is_int($delivery->payload['sequence'] ?? null) + || ($delivery->payload['sequence'] ?? null) !== $run->cancellation_delivery_sequence + || $run->cancellation_delivered_at === null + || $delivery->recorded_at->lt($run->cancellation_delivered_at)) { + return null; + } + $span = $delivery->payload['sequence_span'] ?? 1; + if (! is_int($span) || $span < 1 || $span > PHP_INT_MAX - $delivery->payload['sequence'] + || $sequence < $delivery->payload['sequence'] + $span) { + return null; + } + try { + $requested = $request->payload['cancellation'] ?? null; + $delivered = $delivery->payload['cancellation'] ?? null; + if (! is_array($requested) || ! is_array($delivered)) { + return null; + } + $context = CancellationContext::fromArray($requested); + $deliveryContext = CancellationContext::fromArray($delivered); + } catch (InvalidArgumentException) { + return null; + } + if ($context->requestId !== $run->cancellation_request_command_id + || $context->rootRequestId !== $deliveryContext->rootRequestId + || $context->requestId !== $deliveryContext->requestId + || ! $context->requestedAt() + ->equalTo($deliveryContext->requestedAt()) + || ! $context->deadline() + ->equalTo($deliveryContext->deadline()) + || $run->cancellation_deadline_at === null + || ! $context->deadline() + ->equalTo($run->cancellation_deadline_at)) { + return null; + } + + return [ + 'request_id' => $context->requestId, + 'root_request_id' => $context->rootRequestId, + 'delivery_history_event_id' => $delivery->id, + 'cleanup_deadline_at' => $context->deadline() + ->toISOString(), + ]; + } + + public static function isExecution( + WorkflowRun $run, + ActivityExecution $execution, + WorkflowHistoryEvent $started, + ): bool { + $stored = $execution->activity_options['cancellation_cleanup'] ?? null; + if (! is_array($stored)) { + return false; + } + $snapshot = self::snapshot($run, [ + 'request_id' => $stored['request_id'] ?? null, + 'delivery_history_event_id' => $stored['delivery_history_event_id'] ?? null, + ...(array_key_exists('scope_id', $stored) ? [ + 'scope_id' => $stored['scope_id'], + ] : []), + ], $execution->sequence, $execution->activity_options['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID); + $recorded = $started->payload['local_preparation']['cancellation_cleanup'] ?? null; + if ($snapshot === null || ! is_array($recorded) + || $started->sequence <= $run->historyEvents() + ->whereKey($snapshot['delivery_history_event_id'])->value('sequence')) { + return false; + } + // JSON object order differs across supported databases. Preserve exact + // fields and scalar types while comparing the immutable snapshots. + ksort($stored); + ksort($recorded); + ksort($snapshot); + + return $stored === $snapshot && $recorded === $snapshot; + } + + public static function deadline(ActivityExecution $execution): ?CarbonInterface + { + return self::snapshotDeadline($execution->activity_options['cancellation_cleanup'] ?? null); + } + + /** + * A later preparation must not recancel an original, explicitly shielded cleanup call. + */ + public static function isScopedScheduled(WorkflowRun $run, WorkflowHistoryEvent $event): bool + { + $stored = $event->payload['local_preparation']['cancellation_cleanup'] + ?? $event->payload['local_group_admission']['cancellation_cleanup'] ?? null; + if (! is_array($stored) || ! array_key_exists('scope_id', $stored)) { + return false; + } + $sequence = $event->payload['sequence'] ?? null; + $scope = $event->payload['activity']['cancellation_scope_id'] ?? null; + $deliverySequence = $run->historyEvents() + ->whereKey($stored['delivery_history_event_id'] ?? null) + ->where('event_type', HistoryEventType::CancellationScopeDelivered)->value('sequence'); + if ($event->event_type !== HistoryEventType::ActivityScheduled + || ($event->payload['local_activity'] ?? null) !== true + || ! is_int($sequence) || ! is_string($scope) + || ! is_int($deliverySequence) || $deliverySequence >= $event->sequence) { + throw new LogicException('cancellation_scope_cleanup_history_invalid'); + } + $run->loadMissing('historyEvents'); + $history = $run->historyEvents; + $run->setRelation('historyEvents', $history->filter( + static fn (WorkflowHistoryEvent $row): bool => $row->sequence < $event->sequence + )); + try { + $snapshot = self::snapshot($run, [ + 'scope_id' => $stored['scope_id'], + 'request_id' => $stored['request_id'] ?? null, + 'delivery_history_event_id' => $stored['delivery_history_event_id'] ?? null, + ], $sequence, $scope); + } finally { + $run->setRelation('historyEvents', $history); + } + if ($snapshot === null) { + throw new LogicException('cancellation_scope_cleanup_history_invalid'); + } + ksort($stored); + ksort($snapshot); + if ($stored !== $snapshot) { + throw new LogicException('cancellation_scope_cleanup_history_invalid'); + } + return true; + } + + public static function isScoped(ActivityExecution $execution): bool + { + return isset($execution->activity_options['cancellation_cleanup']['scope_id']); + } + + /** + * A validated cleanup snapshot may continue under the run request from + * which it inherited. A shared root ID alone does not identify that hop. + * Callers must first validate the snapshot or its recorded execution. + * + * @param array|null $snapshot + */ + public static function belongsToRunRequest(WorkflowRun $run, ?array $snapshot): bool + { + if ($snapshot === null || ! is_string($run->cancellation_request_command_id)) { + return false; + } + try { + $event = $run->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationRequested) + ->where('workflow_command_id', $run->cancellation_request_command_id) + ->first(); + $context = $event?->payload['cancellation'] ?? null; + $request = is_array($context) ? CancellationContext::fromArray($context) : null; + if ($request === null || $request->requestId !== $run->cancellation_request_command_id + || $run->cancellation_deadline_at === null + || ! $request->deadline() + ->equalTo($run->cancellation_deadline_at)) { + return false; + } + if (! isset($snapshot['scope_id'])) { + return ($snapshot['request_id'] ?? null) === $request->requestId + && ($snapshot['root_request_id'] ?? null) === $request->rootRequestId; + } + $preparation = ScopedCancellationPreparation::forScope( + $run, + $snapshot['operation_scope_id'], + $snapshot['preparation_history_event_id'], + ); + $runRoot = ScopedCancellationContext::fromRunContext($request); + return $preparation !== null + && $preparation->context->rootContext->toArray() === $runRoot->rootContext->toArray() + && array_slice($preparation->context->lineage, 0, count($runRoot->lineage)) === $runRoot->lineage; + } catch (InvalidArgumentException|LogicException) { + return false; + } + } + + /** + * @param array|null $snapshot + */ + public static function snapshotDeadline(?array $snapshot): ?CarbonInterface + { + $deadline = $snapshot['cleanup_deadline_at'] ?? null; + if (! is_string($deadline)) { + return null; + } + $deadline = \Illuminate\Support\Carbon::parse($deadline); + $ceiling = $snapshot['authority_deadline_at'] ?? null; + + return is_string($ceiling) ? $deadline->min(\Illuminate\Support\Carbon::parse($ceiling)) : $deadline; + } + + public static function currentDeadline(WorkflowRun $run, ?array $snapshot): ?CarbonInterface + { + $deadline = self::snapshotDeadline($snapshot); + if ($deadline === null || ! isset($snapshot['scope_id'])) { + return $deadline; + } + $authority = CancellationScopeRequests::authority($run, $snapshot['operation_scope_id']); + $current = $authority['deadline_at']; + + return $current === null ? $deadline : $deadline->min(\Illuminate\Support\Carbon::parse($current)); + } + + public static function currentAuthorityExpired(WorkflowRun $run, ActivityExecution $execution): bool + { + if (! self::isScoped($execution)) { + return false; + } + $deadline = self::currentDeadline($run, $execution->activity_options['cancellation_cleanup']); + + return $deadline !== null && $deadline->lt(self::deadline($execution)) && now()->gte($deadline); + } + + public static function bound( + ActivityExecution $execution, + ?CarbonInterface $deadline, + ?WorkflowRun $run = null + ): ?CarbonInterface { + $cleanup = $run === null ? self::deadline($execution) + : self::currentDeadline($run, $execution->activity_options['cancellation_cleanup'] ?? null); + + return $cleanup !== null && ($deadline === null || $cleanup->lt($deadline)) ? $cleanup : $deadline; + } + + /** @param array $proof + * @return array|null + */ + private static function scopedSnapshot( + WorkflowRun $run, + array $proof, + int $sequence, + string $operationScope + ): ?array { + if (count($proof) !== 3 || array_diff( + array_keys($proof), + ['scope_id', 'request_id', 'delivery_history_event_id'] + ) !== [] + || ! is_string($proof['scope_id'] ?? null) || $proof['scope_id'] === CancellationScopeHistory::ROOT_SCOPE_ID + || ! is_string($proof['request_id'] ?? null) || ! is_string($proof['delivery_history_event_id'] ?? null)) { + return null; + } + try { + $delivery = CancellationScopeDelivery::recorded($run, $proof['scope_id']); + if ($delivery === null || $delivery->id !== $proof['delivery_history_event_id'] + || $delivery->payload['request_id'] !== $proof['request_id']) { + return null; + } + $preparation = ScopedCancellationPreparation::forScope( + $run, + $operationScope, + $delivery->payload['preparation_history_event_id'], + ); + $context = \Workflow\V2\ScopedCancellationContext::fromArray($delivery->payload['cancellation']); + if ($preparation === null || $preparation->context->rootContext->toArray() !== $context->rootContext->toArray()) { + return null; + } + } catch (InvalidArgumentException|LogicException) { + return null; + } + $span = $delivery->payload['sequence_span']; + if ($span > PHP_INT_MAX - $delivery->payload['sequence'] + || $sequence < $delivery->payload['sequence'] + $span) { + return null; + } + + return [ + 'scope_id' => $proof['scope_id'], + 'operation_scope_id' => $operationScope, + 'request_id' => $context->requestId, + 'root_request_id' => $context->rootContext->rootRequestId, + 'delivery_history_event_id' => $delivery->id, + 'preparation_history_event_id' => $preparation->historyEventId, + 'cleanup_deadline_at' => $context->deadline() + ->toISOString(), + 'authority_deadline_at' => $preparation->authorityDeadlineAt, + ]; + } +} diff --git a/src/V2/Support/PortableLocalActivityControl.php b/src/V2/Support/PortableLocalActivityControl.php new file mode 100644 index 000000000..e664553cf --- /dev/null +++ b/src/V2/Support/PortableLocalActivityControl.php @@ -0,0 +1,482 @@ + + */ + public static function poll( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + bool $renewLease = false, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return self::observe($attemptId, $leaseOwner, $workflowTaskAttempt, $renewLease, false, null, $protocolVersion); + } + + /** @param array $progress + * @return array + */ + public static function heartbeat( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + array $progress = [], + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + try { + $normalized = HeartbeatProgress::normalizeForWrite($progress); + } catch (LogicException) { + return self::response('invalid_local_activity_heartbeat'); + } + + return self::observe($attemptId, $leaseOwner, $workflowTaskAttempt, false, true, $normalized, $protocolVersion); + } + + /** @param array|null $progress + * @return array + */ + private static function observe( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + bool $renewLease, + bool $applicationHeartbeat, + ?array $progress, + string $protocolVersion, + ): array { + $reply = self::lockedObservation( + $attemptId, + $leaseOwner, + $workflowTaskAttempt, + $renewLease, + $applicationHeartbeat, + $progress, + $protocolVersion + ); + $pending = $reply['_scope_control'] ?? null; + if (! is_array($pending)) { + return $reply; + } + unset($reply['_scope_control']); + // Release Activity/run/task locks before canonical preparation. Its + // original scalar call and ancestor tree commit before the fence actor. + $prepared = PortableCancellationScopeDelivery::mutate( + true, + $reply['workflow_task_id'], + $leaseOwner, + $workflowTaskAttempt, + $pending['delivery_scope_id'], + $pending['delivery_request_id'], + $pending['sequence'], + 'local_activity', + 1, + null, + 1, + $protocolVersion, + renewWorkflowLease: false + ); + if (! ($prepared['prepared'] ?? false)) { + return [ + ...$reply, + 'reason' => $prepared['reason'] ?? 'cancellation_scope_control_preparation_refused', + ]; + } + /** @var WorkflowRun|null $run */ + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class)->find($pending['workflow_run_id']); + /** @var WorkflowTask|null $task */ + $task = ConfiguredV2Models::query('task_model', WorkflowTask::class)->find($reply['workflow_task_id']); + if ($run === null || $task === null || $task->lease_owner !== $leaseOwner + || $task->attempt_count !== $workflowTaskAttempt) { + return [ + ...$reply, + 'reason' => 'cancellation_scope_workflow_claim_mismatch', + ]; + } + try { + ScopedActivityCancellation::fence( + $run, + $task, + $reply['activity_execution_id'], + $pending['scope_id'], + $pending['request_id'], + $protocolVersion, + $prepared['preparation_history_event_id'] + ); + } catch (LogicException $error) { + return [ + ...$reply, + 'reason' => $error->getMessage(), + ]; + } + return self::lockedObservation( + $attemptId, + $leaseOwner, + $workflowTaskAttempt, + $renewLease, + $applicationHeartbeat, + $progress, + $protocolVersion + ); + } + + /** @param array|null $progress + * @return array + */ + private static function lockedObservation( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + bool $renewLease, + bool $applicationHeartbeat, + ?array $progress, + string $protocolVersion, + ): array { + if (preg_match('/^[0-9]+\.[0-9]+$/D', $protocolVersion) !== 1 + || version_compare($protocolVersion, PortableLocalActivityPreparation::MINIMUM_PROTOCOL_VERSION, '<')) { + return self::response('local_activity_control_requires_protocol_1_20'); + } + if ($attemptId === '' || $leaseOwner === '' || $workflowTaskAttempt < 1) { + return self::response('invalid_local_activity_control'); + } + + return DB::transaction(static function () use ( + $attemptId, + $leaseOwner, + $workflowTaskAttempt, + $renewLease, + $applicationHeartbeat, + $progress, + ): array { + $rows = ActivityRowLockOrder::lockForAttempt($attemptId); + $attempt = $rows['attempt']; + $execution = $rows['execution']; + if (! $attempt instanceof ActivityAttempt || ! $execution instanceof ActivityExecution) { + return self::response('activity_attempt_not_found'); + } + /** @var WorkflowRun|null $run */ + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->lockForUpdate() + ->find($execution->workflow_run_id); + /** @var WorkflowTask|null $task */ + $task = ConfiguredV2Models::query('task_model', WorkflowTask::class) + ->lockForUpdate() + ->find($attempt->workflow_task_id); + if ($run === null || $task === null || $task->workflow_run_id !== $run->id) { + return self::response('workflow_claim_not_found'); + } + $started = PortableLocalActivityPreparation::originalStart( + $run, + $execution, + $attempt, + $leaseOwner, + $workflowTaskAttempt, + ); + if ($started === null) { + return self::response('local_activity_preparation_mismatch'); + } + $reply = static fn (?string $reason, bool $renewed = false): array => self::response( + $reason, + $run, + $task, + $execution, + $attempt, + $workflowTaskAttempt, + $renewed, + ); + if ($execution->current_attempt_id !== $attemptId || $execution->attempt_count !== $attempt->attempt_number) { + return $reply('stale_activity_attempt'); + } + // Preserve an already accepted scope fence even if a later run + // request or replacement claim exists. Reading this fact cannot + // renew the hosting claim or prove that the callback has stopped. + $scopeFence = $execution->status === ActivityStatus::Cancelled && $attempt->status === ActivityAttemptStatus::Cancelled + ? $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancelled) + ->where('payload->activity_attempt_id', $attemptId) + ->first() + : null; + if ($scopeFence !== null && array_key_exists('cancellation_scope', $scopeFence->payload)) { + $context = ActivityCancellationContext::forEvent($run, $scopeFence); + if (! $context instanceof ScopedCancellationContext) { + return $reply('cancellation_scope_context_not_recorded'); + } + $metadata = $scopeFence->payload['cancellation_scope']; + return [ + ...$reply(now()->gte(\Carbon\CarbonImmutable::parse( + $metadata['authority_deadline_at'] + )) ? 'cancellation_scope_deadline_expired' : 'cancellation_scope_requested'), + 'cancellation_scope' => $metadata, + 'fenced' => true, + 'cancellation_history_event_id' => $scopeFence->id, + ]; + } + // The original callback supervisor must see accepted cancellation + // even after takeover. This never renews the replacement claim. + $cleanup = PortableLocalActivityCleanup::isExecution($run, $execution, $started); + $runCleanup = $cleanup && PortableLocalActivityCleanup::belongsToRunRequest( + $run, + $execution->activity_options['cancellation_cleanup'], + ); + $scopeId = $execution->activity_options['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID; + $scope = ! $cleanup && $scopeId !== CancellationScopeHistory::ROOT_SCOPE_ID + ? CancellationScopeRequests::context($run, $scopeId) : null; + if ($scope !== null) { + if (($execution->parallel_group_path ?? []) !== []) { + return $reply('cancellation_scope_local_group_control_not_supported'); + } + if ($task->task_type !== TaskType::Workflow || $task->status !== TaskStatus::Leased + || $task->lease_owner !== $leaseOwner || $task->attempt_count !== $workflowTaskAttempt) { + return $reply('workflow_claim_mismatch'); + } + if ($task->lease_expires_at === null || now()->gte($task->lease_expires_at) + || $attempt->lease_expires_at === null || now() + ->gte($attempt->lease_expires_at)) { + return $reply('workflow_claim_expired'); + } + $delivery = $scope; + $scopes = CancellationScopeHistory::forRun($run); + $address = $scopeId; + while (isset($scopes[$address]) && ! $scopes[$address]['shield_parent']) { + $address = $scopes[$address]['parent_scope_id']; + $ancestor = $address === CancellationScopeHistory::ROOT_SCOPE_ID ? null + : CancellationScopeRequests::context($run, $address); + if ($ancestor === null || CancellationScopeDelivery::recorded($run, $address) !== null) { + continue; + } + if ($ancestor->rootContext->toArray() !== $scope->rootContext->toArray() + || array_slice($scope->lineage, 0, count($ancestor->lineage)) !== $ancestor->lineage) { + return $reply('cancellation_scope_control_lineage_mismatch'); + } + $delivery = $ancestor; + } + return [ + ...$reply('cancellation_scope_control_preparation_pending'), + '_scope_control' => [ + 'workflow_run_id' => $run->id, + 'sequence' => $execution->sequence, + 'scope_id' => $scope->scopeId, + 'request_id' => $scope->requestId, + 'delivery_scope_id' => $delivery->scopeId, + 'delivery_request_id' => $delivery->requestId, + ], + ]; + } + if ($run->cancellation_request_command_id !== null + && (! $runCleanup || now()->gte($run->cancellation_deadline_at))) { + // A supervisor needs one request, not the run's entire history. + $requested = $run->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationRequested) + ->where('workflow_command_id', $run->cancellation_request_command_id) + ->first(); + $snapshot = $requested?->payload['cancellation'] ?? null; + if (! is_array($snapshot)) { + return $reply('cancellation_context_not_recorded'); + } + try { + $context = CancellationContext::fromArray($snapshot); + } catch (InvalidArgumentException) { + return $reply('cancellation_context_not_recorded'); + } + if ($context->requestId !== $run->cancellation_request_command_id + || ! $requested instanceof WorkflowHistoryEvent) { + return $reply('cancellation_context_not_recorded'); + } + if (! $runCleanup && ($started->sequence >= $requested->sequence + || $started->recorded_at->gt($requested->recorded_at))) { + return $reply('local_activity_preparation_mismatch'); + } + $cancelled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancelled) + ->where('payload->activity_attempt_id', $attemptId) + ->where('workflow_command_id', $context->requestId) + ->first(); + if (! $cancelled instanceof WorkflowHistoryEvent + && $execution->status === ActivityStatus::Running && $attempt->status === ActivityAttemptStatus::Running) { + // No hosting task argument. Its cleanup claim stays intact. + $cancelled = ActivityCancellation::record($run, $execution, command: $context->requestId); + } + return [ + ...$reply(now()->gte( + $context->deadline() + ) ? 'cancellation_deadline_expired' : 'cancellation_requested'), + 'cancellation_request' => $context->toArray(), + 'fenced' => $cancelled instanceof WorkflowHistoryEvent, + 'cancellation_history_event_id' => $cancelled?->id, + ]; + } + if ($run->status->isTerminal()) { + return $reply('run_closed'); + } + if (($run->execution_deadline_at !== null && now()->gte($run->execution_deadline_at)) + || ($run->run_deadline_at !== null && now()->gte($run->run_deadline_at))) { + return $reply('run_deadline_expired'); + } + if ($attempt->status !== ActivityAttemptStatus::Running || $execution->status !== ActivityStatus::Running) { + return $reply('stale_activity_attempt'); + } + if (PortableLocalActivityCleanup::currentAuthorityExpired($run, $execution)) { + return $reply('local_activity_cleanup_authority_expired'); + } + if ($task->task_type !== TaskType::Workflow || $task->status !== TaskStatus::Leased + || $task->lease_owner !== $leaseOwner || $task->attempt_count !== $workflowTaskAttempt) { + return $reply('workflow_claim_mismatch'); + } + if ($task->lease_expires_at === null || now()->gte($task->lease_expires_at)) { + return $reply('workflow_claim_expired'); + } + if ($attempt->lease_expires_at === null || now()->gte($attempt->lease_expires_at)) { + return $reply('local_activity_lease_expired'); + } + foreach ([ + $execution->close_deadline_at, + $execution->schedule_to_close_deadline_at, + $execution->heartbeat_deadline_at, + ] as $deadline) { + if ($deadline !== null && now()->gte($deadline)) { + return $reply('local_activity_deadline_expired'); + } + } + if ($renewLease) { + // Both rows and the task summary commit in one transaction. + // No application heartbeat or recorded execution deadline moves. + $expiry = LocalActivityRuntime::renewWorkflowTask( + $task, + ! $runCleanup && PortableLocalActivityCleanup::isScoped( + $execution + ) ? null : PortableLocalActivityCleanup::deadline($execution), + $cleanup, + $run, + ); + $deadline = PortableLocalActivityCleanup::currentDeadline( + $run, + $execution->activity_options['cancellation_cleanup'] ?? null + ); + if ($deadline !== null && $expiry !== null && $deadline->lt($expiry)) { + $expiry = $deadline; + } + $attempt->forceFill([ + 'lease_expires_at' => $expiry, + ])->save(); + } + $heartbeatEvent = null; + if ($applicationHeartbeat) { + $heartbeatAt = now(); + $timeout = $execution->retry_policy['heartbeat_timeout'] ?? null; + $execution->forceFill([ + 'last_heartbeat_at' => $heartbeatAt, + 'heartbeat_deadline_at' => PortableLocalActivityCleanup::bound( + $execution, + is_int($timeout) && $timeout > 0 ? $heartbeatAt->copy() + ->addSeconds($timeout) : null, + $run, + ), + ])->save(); + $attempt->forceFill([ + 'last_heartbeat_at' => $heartbeatAt, + ])->save(); + $heartbeatEvent = WorkflowHistoryEvent::record( + $run, + HistoryEventType::ActivityHeartbeatRecorded, + LocalActivityRuntime::eventPayload([ + 'activity_execution_id' => $execution->id, + 'activity_attempt_id' => $attempt->id, + 'activity_class' => $execution->activity_class, + 'activity_type' => $execution->activity_type, + 'sequence' => $execution->sequence, + 'attempt_number' => $attempt->attempt_number, + 'heartbeat_at' => $heartbeatAt->toISOString(), + 'activity' => ActivitySnapshot::fromExecution($execution), + 'activity_attempt' => [ + 'id' => $attempt->id, + 'activity_execution_id' => $execution->id, + 'task_id' => $task->id, + 'attempt_number' => $attempt->attempt_number, + 'status' => $attempt->status->value, + 'lease_owner' => $attempt->lease_owner, + 'worker_attempt_id' => $attempt->worker_attempt_id, + 'started_at' => $attempt->started_at?->toISOString(), + 'last_heartbeat_at' => $heartbeatAt->toISOString(), + 'lease_expires_at' => $attempt->lease_expires_at?->toISOString(), + ], + ...($progress === null ? [] : [ + 'progress' => $progress, + ]), + ]), + $task + ); + app(\Workflow\V2\Contracts\HistoryProjectionRole::class)->projectRun( + $run->fresh(['instance', 'tasks', 'activityExecutions', 'failures', 'historyEvents']) ?? $run + ); + } + return [ + ...$reply(null, $renewLease), + 'heartbeat_recorded' => $heartbeatEvent !== null, + 'heartbeat_history_event_id' => $heartbeatEvent?->id, + ]; + }); + } + + /** + * @return array + */ + private static function response( + ?string $reason, + ?WorkflowRun $run = null, + ?WorkflowTask $task = null, + ?ActivityExecution $execution = null, + ?ActivityAttempt $attempt = null, + ?int $workflowTaskAttempt = null, + bool $renewed = false, + ): array { + return [ + 'active' => $reason === null, + 'renewed' => $renewed, + 'stop_required' => $reason !== null, + 'reason' => $reason, + 'activity_execution_id' => $execution?->id, + 'activity_attempt_id' => $attempt?->id, + 'workflow_task_id' => $attempt?->workflow_task_id, + 'workflow_task_attempt' => $workflowTaskAttempt, + 'lease_owner' => $attempt?->lease_owner, + 'lease_expires_at' => $attempt?->lease_expires_at?->toISOString(), + 'workflow_lease_expires_at' => $task?->lease_expires_at?->toISOString(), + 'start_to_close_deadline_at' => $execution?->close_deadline_at?->toISOString(), + 'schedule_to_close_deadline_at' => $execution?->schedule_to_close_deadline_at?->toISOString(), + 'heartbeat_deadline_at' => $execution?->heartbeat_deadline_at?->toISOString(), + 'cancellation_cleanup' => $execution?->activity_options['cancellation_cleanup'] ?? null, + 'heartbeat_recorded' => false, + 'heartbeat_history_event_id' => null, + 'run_status' => $run?->status->value, + 'task_status' => $task?->status->value, + 'server_time' => now() + ->toISOString(), + ]; + } +} diff --git a/src/V2/Support/PortableLocalActivityPreparation.php b/src/V2/Support/PortableLocalActivityPreparation.php new file mode 100644 index 000000000..5d4ce94eb --- /dev/null +++ b/src/V2/Support/PortableLocalActivityPreparation.php @@ -0,0 +1,1131 @@ + $descriptor + * @return array + */ + public static function prepare( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + string $workerAttemptId, + array $descriptor, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + if (preg_match('/^[0-9]+\.[0-9]+$/D', $protocolVersion) !== 1 + || version_compare($protocolVersion, self::MINIMUM_PROTOCOL_VERSION, '<')) { + return self::response('local_activity_preparation_requires_protocol_1_20'); + } + if ($sequence < 1 || $workflowTaskAttempt < 1 || $leaseOwner === '' + || trim($workerAttemptId) === '' || strlen($workerAttemptId) > 255 + || preg_match('//u', $workerAttemptId) !== 1) { + return self::response('invalid_local_activity_preparation'); + } + try { + $normalized = self::normalizeDescriptor($descriptor); + $fingerprint = hash('sha256', json_encode($normalized, JSON_THROW_ON_ERROR)); + } catch (ValidationException|JsonException) { + return self::response('invalid_local_activity_preparation'); + } + /** @var WorkflowTask|null $snapshotTask */ + $snapshotTask = ConfiguredV2Models::query('task_model', WorkflowTask::class)->find($taskId); + if ($snapshotTask === null) { + return self::response('task_not_found'); + } + /** @var ActivityExecution|null $snapshotExecution */ + $snapshotExecution = ActivityExecution::query() + ->where('workflow_run_id', $snapshotTask->workflow_run_id) + ->where('sequence', $sequence) + ->first(); + + return DB::transaction(static function () use ( + $snapshotTask, + $snapshotExecution, + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $sequence, + $workerAttemptId, + $normalized, + $fingerprint, + ): array { + // Follow the shared attempt/execution/run/task lock order whenever + // an execution exists. An unseen concurrent creation is retried. + $rows = $snapshotExecution === null ? null : ( + is_string($snapshotExecution->current_attempt_id) + ? ActivityRowLockOrder::lockForAttempt($snapshotExecution->current_attempt_id) + : ActivityRowLockOrder::lockForExecution($snapshotExecution->id) + ); + /** @var WorkflowRun|null $run */ + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->lockForUpdate() + ->find($snapshotTask->workflow_run_id); + /** @var WorkflowTask|null $task */ + $task = ConfiguredV2Models::query('task_model', WorkflowTask::class) + ->lockForUpdate() + ->find($taskId); + if ($run === null || $task === null || $task->workflow_run_id !== $run->id) { + return self::response('workflow_claim_not_found'); + } + if ($task->task_type !== TaskType::Workflow || $task->status !== TaskStatus::Leased + || $task->lease_owner !== $leaseOwner || $task->attempt_count !== $workflowTaskAttempt) { + return self::response('workflow_claim_mismatch'); + } + if ($task->lease_expires_at === null || now()->gte($task->lease_expires_at)) { + return self::response('workflow_claim_expired'); + } + if ($run->status->isTerminal()) { + return self::response('run_closed'); + } + if (! CancellationScopeHistory::isRecordedBefore( + $run, + $normalized['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, + $sequence, + )) { + return self::response('local_activity_scope_not_recorded'); + } + $cleanup = PortableLocalActivityCleanup::snapshot( + $run, + $normalized['cancellation_cleanup'] ?? null, + $sequence, + $normalized['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, + ); + if (isset($normalized['cancellation_cleanup']) && $cleanup === null) { + return self::response('local_activity_cleanup_authority_mismatch'); + } + $scopeRefusal = CancellationScopeDelivery::admissionRefusal( + $run, + $normalized['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, + $sequence, + $normalized['cancellation_cleanup'] ?? null, + ); + if ($scopeRefusal !== null) { + return self::response($scopeRefusal); + } + $scopedCleanup = isset($cleanup['scope_id']); + if ($run->cancellation_request_command_id !== null + && ! PortableLocalActivityCleanup::belongsToRunRequest($run, $cleanup)) { + return self::response(isset($normalized['cancellation_cleanup']) + && ! $scopedCleanup ? 'local_activity_cleanup_authority_mismatch' : 'cancellation_requested'); + } + if ($run->cancellation_request_command_id === null && isset($normalized['cancellation_cleanup']) && ! $scopedCleanup) { + return self::response('local_activity_cleanup_authority_mismatch'); + } + if ($cleanup !== null && now()->gte(PortableLocalActivityCleanup::snapshotDeadline($cleanup))) { + return self::response( + $scopedCleanup ? 'local_activity_cleanup_deadline_expired' : 'cancellation_deadline_expired' + ); + } + if (($run->execution_deadline_at !== null && now()->gte($run->execution_deadline_at)) + || ($run->run_deadline_at !== null && now()->gte($run->run_deadline_at))) { + return self::response('run_deadline_expired'); + } + if ($scopedCleanup && now()->gte(PortableLocalActivityCleanup::currentDeadline($run, $cleanup))) { + return self::response('local_activity_cleanup_authority_expired'); + } + $execution = $rows['execution'] ?? null; + $attempt = $rows['attempt'] ?? null; + if ($snapshotExecution !== null) { + if (! $execution instanceof ActivityExecution + || $execution->id !== $snapshotExecution->id || $execution->sequence !== $sequence + || $execution->current_attempt_id !== ($rows['snapshot_attempt_id'] ?? null)) { + return self::response('local_activity_previous_attempt_unresolved'); + } + if ($attempt === null) { + return self::prepareAdmittedMember( + $run, + $task, + $execution, + $normalized, + $workerAttemptId, + $fingerprint + ); + } + if (! $attempt instanceof ActivityAttempt) { + return self::response('local_activity_previous_attempt_unresolved'); + } + if ($execution->status === ActivityStatus::Pending) { + return self::prepareRetry($run, $task, $execution, $attempt, $workerAttemptId, $fingerprint); + } + return self::duplicate($run, $task, $execution, $attempt, $workerAttemptId, $fingerprint); + } + if (ActivityExecution::query()->where('workflow_run_id', $run->id)->where( + 'sequence', + $sequence + )->exists()) { + return self::response('local_activity_preparation_retry'); + } + if (WorkflowStepHistory::nextDurableCommandSequence($run) !== $sequence) { + return self::response('local_activity_command_prefix_not_recorded'); + } + if (($normalized['parallel_group_path'] ?? []) !== []) { + return self::response('local_activity_group_not_admitted'); + } + $preparation = [ + 'version' => 1, + 'descriptor_fingerprint' => $fingerprint, + 'worker_attempt_id' => $workerAttemptId, + 'workflow_task_attempt' => $workflowTaskAttempt, + ...($cleanup === null ? [] : [ + 'cancellation_cleanup' => $cleanup, + ]), + ]; + $execution = self::createExecution( + $run, + $task, + $sequence, + $normalized, + $cleanup, + [ + 'local_preparation' => $preparation, + ] + ); + $attempt = app(LocalActivityExecutor::class)->startPortableAttempt( + $run, + $task, + $execution, + $workerAttemptId, + $preparation + ); + + return self::response(null, $execution, $attempt, task: $task); + }, 5); + } + + /** + * @internal The complete group was validated and the caller holds run/task + * locks inside its atomic checkpoint transaction. This creates no attempt. + * @param array $descriptor + */ + public static function admitGroupMember( + WorkflowRun $run, + WorkflowTask $task, + int $sequence, + array $descriptor, + string $checkpointId, + string $batchFingerprint + ): ActivityExecution { + $normalized = self::normalizeDescriptor($descriptor); + if (DB::transactionLevel() < 1 || ($normalized['parallel_group_path'] ?? []) === []) { + throw ValidationException::withMessages([ + 'local_activity' => ['Expected an atomic group transaction.'], + ]); + } + if (! CancellationScopeHistory::isRecordedBefore( + $run, + $normalized['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, + $sequence, + )) { + throw ValidationException::withMessages([ + 'local_activity.cancellation_scope_id' => ['Scope must be recorded in this run before its operation.'], + ]); + } + $cleanup = PortableLocalActivityCleanup::snapshot( + $run, + $normalized['cancellation_cleanup'] ?? null, + $sequence, + $normalized['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, + ); + $admission = [ + 'version' => 1, + 'descriptor_fingerprint' => hash('sha256', json_encode($normalized, JSON_THROW_ON_ERROR)), + 'checkpoint_id' => $checkpointId, + 'batch_fingerprint' => $batchFingerprint, + 'workflow_task_attempt' => $task->attempt_count, + ...($cleanup === null ? [] : [ + 'cancellation_cleanup' => $cleanup, + ]), + ]; + return self::createExecution( + $run, + $task, + $sequence, + $normalized, + $cleanup, + [ + 'local_group_admission' => $admission, + ] + ); + } + + /** + * @internal Recover an interrupted prepared call. This never admits an + * application callback. A scheduled retry releases this claim, and the SDK + * returns to polling. A terminal outcome is replayed on the retained claim. + * @param array $descriptor + * @return array + */ + public static function recover( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + array $descriptor, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + $refused = static fn (string $reason): array => [ + 'recovered' => false, + 'duplicate' => false, + 'reason' => $reason, + 'claim_released' => false, + 'created_task_ids' => [], + ]; + if (preg_match('/^[0-9]+\.[0-9]+$/D', $protocolVersion) !== 1 + || version_compare($protocolVersion, self::MINIMUM_PROTOCOL_VERSION, '<')) { + return $refused('local_activity_recovery_requires_protocol_1_20'); + } + if ($workflowTaskAttempt < 1 || $sequence < 1 || $leaseOwner === '') { + return $refused('invalid_local_activity_recovery'); + } + try { + $fingerprint = hash('sha256', json_encode(self::normalizeDescriptor($descriptor), JSON_THROW_ON_ERROR)); + } catch (ValidationException|JsonException) { + return $refused('invalid_local_activity_recovery'); + } + /** @var WorkflowTask|null $snapshotTask */ + $snapshotTask = ConfiguredV2Models::query('task_model', WorkflowTask::class)->find($taskId); + /** @var ActivityExecution|null $snapshotExecution */ + $snapshotExecution = $snapshotTask === null ? null : ActivityExecution::query() + ->where('workflow_run_id', $snapshotTask->workflow_run_id) + ->where('sequence', $sequence) + ->first(); + if ($snapshotTask === null || $snapshotExecution === null + || ! is_string($snapshotExecution->current_attempt_id)) { + return $refused('local_activity_preparation_not_found'); + } + return DB::transaction(static function () use ( + $snapshotTask, + $snapshotExecution, + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $sequence, + $fingerprint, + $refused, + ): array { + $rows = ActivityRowLockOrder::lockForAttempt($snapshotExecution->current_attempt_id); + $execution = $rows['execution']; + $attempt = $rows['attempt']; + /** @var WorkflowRun|null $run */ + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->lockForUpdate() + ->find($snapshotTask->workflow_run_id); + /** @var WorkflowTask|null $task */ + $task = ConfiguredV2Models::query('task_model', WorkflowTask::class) + ->lockForUpdate() + ->find($taskId); + if (! $execution instanceof ActivityExecution || ! $attempt instanceof ActivityAttempt + || $execution->id !== $snapshotExecution->id || $execution->sequence !== $sequence + || $run === null || $execution->workflow_run_id !== $run->id + || $task === null || $task->workflow_run_id !== $run->id) { + return $refused('local_activity_preparation_not_found'); + } + $receipt = $run->historyEvents() + ->whereIn('event_type', [HistoryEventType::ActivityRetryScheduled, HistoryEventType::ActivityFailed, + HistoryEventType::ActivityTimedOut]) + ->where('payload->sequence', $sequence) + ->where('payload->local_recovery->workflow_task_id', $taskId) + ->where('payload->local_recovery->workflow_task_attempt', $workflowTaskAttempt) + ->where('payload->local_recovery->lease_owner', $leaseOwner) + ->orderBy('sequence') + ->first(); + if ($receipt instanceof WorkflowHistoryEvent) { + if (($receipt->payload['local_recovery']['version'] ?? null) !== 1 + || ($receipt->payload['local_recovery']['descriptor_fingerprint'] ?? null) !== $fingerprint + || ($receipt->payload['activity_execution_id'] ?? null) !== $execution->id + || $receipt->workflow_task_id !== $taskId + || ($receipt->payload['task']['id'] ?? null) !== $taskId + || ($receipt->payload['task']['lease_owner'] ?? null) !== $leaseOwner + || ($receipt->payload['task']['attempt_count'] ?? null) !== $workflowTaskAttempt + || ($receipt->payload['local_recovery']['callback_stop_state'] ?? null) !== 'unknown') { + return $refused('local_activity_recovery_mismatch'); + } + return self::recoveryReceipt($receipt, true); + } + if ($task->task_type !== TaskType::Workflow || $task->status !== TaskStatus::Leased + || $task->lease_owner !== $leaseOwner || $task->attempt_count !== $workflowTaskAttempt) { + return $refused('workflow_claim_mismatch'); + } + if ($task->lease_expires_at === null || now()->gte($task->lease_expires_at)) { + return $refused('workflow_claim_expired'); + } + $started = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted) + ->where('payload->activity_attempt_id', $attempt->id) + ->first(); + $originalTaskAttempt = $started?->payload['local_preparation']['workflow_task_attempt'] ?? null; + if (! is_int($originalTaskAttempt) + || self::originalStart($run, $execution, $attempt, $attempt->lease_owner, $originalTaskAttempt) === null + || ($started->payload['local_preparation']['descriptor_fingerprint'] ?? null) !== $fingerprint + || $execution->current_attempt_id !== $attempt->id || $execution->attempt_count !== $attempt->attempt_number) { + return $refused('local_activity_preparation_mismatch'); + } + // Accepted cancellation may fence immediately, without waiting for + // lease expiry. The replacement claim and root budget stay intact. + $cleanup = PortableLocalActivityCleanup::isExecution($run, $execution, $started); + if ($cleanup && now()->gte(PortableLocalActivityCleanup::deadline($execution))) { + return $refused(PortableLocalActivityCleanup::isScoped($execution) + ? 'local_activity_cleanup_deadline_expired' : 'cancellation_deadline_expired'); + } + if ($run->cancellation_request_command_id !== null && (! $cleanup + || ! PortableLocalActivityCleanup::belongsToRunRequest( + $run, + $execution->activity_options['cancellation_cleanup'] + ))) { + $cancelled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancelled) + ->where('payload->activity_attempt_id', $attempt->id) + ->where('payload->workflow_command_id', $run->cancellation_request_command_id) + ->first(); + if (! $cancelled instanceof WorkflowHistoryEvent) { + if ($execution->status !== ActivityStatus::Running || $attempt->status !== ActivityAttemptStatus::Running) { + return $refused('stale_activity_attempt'); + } + $cancelled = ActivityCancellation::record( + $run, + $execution, + command: $run->cancellation_request_command_id + ); + } + return [ + ...$refused('cancellation_requested'), + 'fenced' => true, + 'cancellation_history_event_id' => $cancelled?->id, + ]; + } + if ($run->status->isTerminal()) { + return $refused('run_closed'); + } + if (($run->execution_deadline_at !== null && now()->gte($run->execution_deadline_at)) + || ($run->run_deadline_at !== null && now()->gte($run->run_deadline_at))) { + return $refused('run_deadline_expired'); + } + if ($execution->status !== ActivityStatus::Running || $attempt->status !== ActivityAttemptStatus::Running) { + return $refused('local_activity_previous_attempt_unresolved'); + } + if (PortableLocalActivityCleanup::currentAuthorityExpired($run, $execution)) { + return $refused('local_activity_cleanup_authority_expired'); + } + if ($attempt->lease_expires_at === null || now()->lt($attempt->lease_expires_at)) { + return $refused('local_activity_previous_attempt_live'); + } + /** @var WorkflowTask|null $originalTask */ + $originalTask = ConfiguredV2Models::query('task_model', WorkflowTask::class) + ->lockForUpdate() + ->find($attempt->workflow_task_id); + if ($originalTask === null || $originalTask->workflow_run_id !== $run->id + || $originalTask->task_type !== TaskType::Workflow) { + return $refused('original_workflow_claim_not_found'); + } + if ($originalTask->id === $task->id) { + if ($task->attempt_count <= $originalTaskAttempt) { + return $refused('local_activity_original_claim_not_reclaimed'); + } + } elseif ($originalTask->status === TaskStatus::Leased && $originalTask->lease_expires_at !== null + && now() + ->lt($originalTask->lease_expires_at)) { + return $refused('local_activity_original_claim_live'); + } + $recovery = [ + 'version' => 1, + 'descriptor_fingerprint' => $fingerprint, + 'workflow_task_id' => $task->id, + 'workflow_task_attempt' => $workflowTaskAttempt, + 'lease_owner' => $leaseOwner, + 'original_workflow_task_id' => $originalTask->id, + 'original_workflow_task_attempt' => $originalTaskAttempt, + 'original_lease_owner' => $attempt->lease_owner, + 'original_lease_expires_at' => $attempt->lease_expires_at->toISOString(), + 'callback_stop_state' => 'unknown', + ]; + if ($originalTask->id !== $task->id && in_array( + $originalTask->status, + [TaskStatus::Ready, TaskStatus::Leased], + true + )) { + $originalTask->forceFill([ + 'status' => TaskStatus::Completed, + 'lease_expires_at' => null, + ])->save(); + } + $outcome = app(LocalActivityExecutor::class)->recoverPortableAttempt( + $run, + $task, + $execution, + $attempt, + $recovery + ); + + return self::recoveryReceipt($outcome['event'], false); + }, 5); + } + + /** @param array $descriptor + * @return array + */ + public static function normalizeDescriptor(array $descriptor): array + { + $allowed = ['type', 'activity_type', 'arguments', 'payload_codec', 'retry_policy', + 'start_to_close_timeout', 'schedule_to_close_timeout', 'heartbeat_timeout', 'execution_mode', + 'cancellation_cleanup', 'cancellation_policy', 'cancellation_scope_id', 'parallel_group_id', 'parallel_group_kind', 'parallel_group_mode', + 'parallel_group_base_sequence', 'parallel_group_size', 'parallel_group_index', 'parallel_group_path']; + if (($descriptor['type'] ?? null) !== 'record_local_activity' + || array_diff(array_keys($descriptor), $allowed) !== [] + || (isset($descriptor['execution_mode']) && $descriptor['execution_mode'] !== LocalActivityRuntime::EXECUTION_MODE)) { + throw ValidationException::withMessages([ + 'local_activity' => ['Expected a local activity preparation descriptor.'], + ]); + } + if (array_key_exists('cancellation_policy', $descriptor) + && ! in_array($descriptor['cancellation_policy'], ['try_cancel', 'wait_cancellation_completed'], true)) { + throw ValidationException::withMessages([ + 'local_activity.cancellation_policy' => [ + 'Prepared local activities support try_cancel and wait_cancellation_completed. Abandon requires an independent callback lifetime.', + ], + ]); + } + if (array_key_exists('cancellation_scope_id', $descriptor) + && (! is_string($descriptor['cancellation_scope_id']) + || trim($descriptor['cancellation_scope_id']) === '' + || strlen($descriptor['cancellation_scope_id']) > 255 + || preg_match('//u', $descriptor['cancellation_scope_id']) !== 1)) { + throw ValidationException::withMessages([ + 'local_activity.cancellation_scope_id' => ['Expected a nonempty canonical scope identity.'], + ]); + } + // Common activity input/retry/timeout validation does not need a + // fabricated outcome or a claim that an application attempt ran. + $input = $descriptor; + unset($input['execution_mode'], $input['cancellation_cleanup'], $input['cancellation_scope_id']); + $input['type'] = 'schedule_activity'; + $normalized = WorkflowCommandNormalizer::normalize([$input], self::MINIMUM_PROTOCOL_VERSION)[0]; + if (! is_string($normalized['arguments'] ?? null) || ! is_string($normalized['payload_codec'] ?? null)) { + throw ValidationException::withMessages([ + 'local_activity.arguments' => ['Preparation requires encoded arguments.'], + ]); + } + $normalized['type'] = 'record_local_activity'; + $normalized['execution_mode'] = LocalActivityRuntime::EXECUTION_MODE; + if (array_key_exists('cancellation_scope_id', $descriptor)) { + $normalized['cancellation_scope_id'] = $descriptor['cancellation_scope_id']; + } + if (array_key_exists('cancellation_cleanup', $descriptor)) { + $proof = $descriptor['cancellation_cleanup']; + if (! is_array($proof) || array_diff( + array_keys($proof), + ['request_id', 'delivery_history_event_id', 'scope_id'] + ) !== [] + || ! is_string($proof['request_id'] ?? null) || trim($proof['request_id']) === '' + || ! is_string($proof['delivery_history_event_id'] ?? null) + || trim($proof['delivery_history_event_id']) === '' + || (array_key_exists('scope_id', $proof) && (! is_string($proof['scope_id']) + || trim( + $proof['scope_id'] + ) === '' || $proof['scope_id'] === CancellationScopeHistory::ROOT_SCOPE_ID))) { + throw ValidationException::withMessages([ + 'local_activity.cancellation_cleanup' => ['Expected canonical request and delivery identities.'], + ]); + } + $normalized['cancellation_cleanup'] = [ + 'request_id' => $proof['request_id'], + 'delivery_history_event_id' => $proof['delivery_history_event_id'], + ...(array_key_exists('scope_id', $proof) ? [ + 'scope_id' => $proof['scope_id'], + ] : []), + ]; + } + + return $normalized; + } + + /** + * @internal Read the original immutable preparation authority. The caller + * holds attempt/execution/run locks. Current task ownership is separate. + */ + public static function originalStart( + WorkflowRun $run, + ActivityExecution $execution, + ActivityAttempt $attempt, + string $leaseOwner, + int $workflowTaskAttempt, + ): ?WorkflowHistoryEvent { + if (! LocalActivityRuntime::isExecution($execution) || $execution->workflow_run_id !== $run->id + || $attempt->workflow_run_id !== $run->id || $attempt->activity_execution_id !== $execution->id + || $attempt->lease_owner !== $leaseOwner || $workflowTaskAttempt < 1) { + return null; + } + $started = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted) + ->where('payload->activity_attempt_id', $attempt->id) + ->first(); + if (! $started instanceof WorkflowHistoryEvent + || $started->workflow_task_id !== $attempt->workflow_task_id + || ($started->payload['activity_execution_id'] ?? null) !== $execution->id + || ($started->payload['activity_type'] ?? null) !== $execution->activity_type + || ($started->payload['activity_class'] ?? null) !== $execution->activity_class + || ($started->payload['sequence'] ?? null) !== $execution->sequence + || ($started->payload['workflow_task_id'] ?? null) !== $attempt->workflow_task_id + || ($started->payload['local_activity'] ?? null) !== true + || ($started->payload['execution_mode'] ?? null) !== LocalActivityRuntime::EXECUTION_MODE + || ($started->payload['local_preparation']['version'] ?? null) !== 1 + || ! is_string($started->payload['local_preparation']['descriptor_fingerprint'] ?? null) + || ($started->payload['local_preparation']['worker_attempt_id'] ?? null) !== $attempt->worker_attempt_id + || ($started->payload['local_preparation']['workflow_task_attempt'] ?? null) !== $workflowTaskAttempt + || ($started->payload['task']['id'] ?? null) !== $attempt->workflow_task_id + || ($started->payload['task']['type'] ?? null) !== TaskType::Workflow->value + || ($started->payload['task']['status'] ?? null) !== TaskStatus::Leased->value + || ($started->payload['task']['attempt_count'] ?? null) !== $workflowTaskAttempt + || ($started->payload['task']['lease_owner'] ?? null) !== $leaseOwner + || ($started->payload['activity_attempt']['id'] ?? null) !== $attempt->id + || ($started->payload['activity_attempt']['activity_execution_id'] ?? null) !== $execution->id + || ($started->payload['activity_attempt']['task_id'] ?? null) !== $attempt->workflow_task_id + || ($started->payload['activity_attempt']['attempt_number'] ?? null) !== $attempt->attempt_number + || ($started->payload['activity_attempt']['lease_owner'] ?? null) !== $leaseOwner + || ($started->payload['activity_attempt']['worker_attempt_id'] ?? null) !== $attempt->worker_attempt_id) { + return null; + } + $scopeId = $execution->activity_options['cancellation_scope_id'] ?? null; + if (($started->payload['activity']['cancellation_scope_id'] ?? null) !== $scopeId) { + return null; + } + if ($scopeId !== null) { + $scheduled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityScheduled) + ->where('payload->activity_execution_id', $execution->id) + ->first(); + if (! is_string($scopeId) + || ! CancellationScopeHistory::isRecordedBefore($run, $scopeId, $execution->sequence) + || ($scheduled?->payload['activity']['cancellation_scope_id'] ?? null) !== $scopeId) { + return null; + } + } + if (array_key_exists('schedule_to_close_deadline_at', $started->payload['local_preparation']) + && $started->payload['local_preparation']['schedule_to_close_deadline_at'] + !== $execution->schedule_to_close_deadline_at?->toISOString()) { + return null; + } + if ((array_key_exists('cancellation_cleanup', $execution->activity_options ?? []) + || array_key_exists('cancellation_cleanup', $started->payload['local_preparation'])) + && ! PortableLocalActivityCleanup::isExecution($run, $execution, $started)) { + return null; + } + + return $started; + } + + /** @param array $normalized + * @param array|null $cleanup + * @param array $opening + */ + private static function createExecution( + WorkflowRun $run, + WorkflowTask $task, + int $sequence, + array $normalized, + ?array $cleanup, + array $opening + ): ActivityExecution { + StructuralLimits::guardPendingActivities($run); + $codec = $normalized['payload_codec']; + $arguments = ExternalPayloads::externalizeForNamespace( + $normalized['arguments'], + $codec, + is_string($run->namespace) ? $run->namespace : null + ); + StructuralLimits::guardPayloadSize($arguments); + $now = now(); + $options = new ActivityOptions( + startToCloseTimeout: $normalized['start_to_close_timeout'] ?? null, + scheduleToCloseTimeout: $normalized['schedule_to_close_timeout'] ?? null, + heartbeatTimeout: $normalized['heartbeat_timeout'] ?? null, + ); + /** @var ActivityExecution $execution */ + $execution = ActivityExecution::query()->create([ + 'workflow_run_id' => $run->id, + 'sequence' => $sequence, + 'activity_class' => $normalized['activity_type'], + 'activity_type' => $normalized['activity_type'], + 'status' => ActivityStatus::Pending, + 'attempt_count' => 0, + 'payload_codec' => $codec, + 'arguments' => $arguments, + 'connection' => $run->connection, + 'queue' => $run->queue, + 'parallel_group_path' => $normalized['parallel_group_path'] ?? null, + 'retry_policy' => ActivityRetryPolicy::snapshotExternal($normalized['retry_policy'] ?? null, $options), + 'activity_options' => [ + 'execution_mode' => LocalActivityRuntime::EXECUTION_MODE, + 'queue_bypassed' => true, + 'routing' => 'workflow_worker_process', + ...(isset($normalized['cancellation_policy']) ? [ + 'cancellation_policy' => $normalized['cancellation_policy'], + ] : []), + ...(isset($normalized['cancellation_scope_id']) ? [ + 'cancellation_scope_id' => $normalized['cancellation_scope_id'], + ] : []), + ...($cleanup === null ? [] : [ + 'cancellation_cleanup' => $cleanup, + ]), + ], + 'schedule_to_close_deadline_at' => $cleanup === null ? ($options->scheduleToCloseTimeout === null + ? null : $now->copy() + ->addSeconds($options->scheduleToCloseTimeout)) + : ($options->scheduleToCloseTimeout === null ? PortableLocalActivityCleanup::currentDeadline( + $run, + $cleanup + ) + : $now->copy() + ->addSeconds($options->scheduleToCloseTimeout) + ->min(PortableLocalActivityCleanup::currentDeadline($run, $cleanup))), + ]); + WorkflowHistoryEvent::record($run, HistoryEventType::ActivityScheduled, LocalActivityRuntime::eventPayload([ + 'activity_execution_id' => $execution->id, + 'activity_class' => $execution->activity_class, + 'activity_type' => $execution->activity_type, + 'sequence' => $sequence, + 'workflow_task_id' => $task->id, + 'activity' => ActivitySnapshot::fromExecution($execution), + ...$opening, + ]), $task); + return $execution; + } + + /** @param array $normalized + * @return array + */ + private static function prepareAdmittedMember( + WorkflowRun $run, + WorkflowTask $task, + ActivityExecution $execution, + array $normalized, + string $workerAttemptId, + string $fingerprint + ): array { + $scheduled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityScheduled) + ->where('payload->activity_execution_id', $execution->id) + ->first(); + $admission = $scheduled?->payload['local_group_admission'] ?? []; + $originalTask = $scheduled === null ? null + : ConfiguredV2Models::query('task_model', WorkflowTask::class)->find($scheduled->workflow_task_id); + $receipt = $originalTask?->payload['portable_local_group_checkpoint'] ?? []; + $members = $receipt['local_activities'] ?? []; + $memberMatches = false; + foreach ($members as $member) { + if (is_array($member) && ($member['sequence'] ?? null) === $execution->sequence + && ($member['activity_execution_id'] ?? null) === $execution->id) { + $memberMatches = true; + } + } + $path = $normalized['parallel_group_path'] ?? []; + $cleanup = PortableLocalActivityCleanup::snapshot( + $run, + $normalized['cancellation_cleanup'] ?? null, + $execution->sequence, + $normalized['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, + ); + $storedCleanup = $execution->activity_options['cancellation_cleanup'] ?? null; + if (is_array($cleanup)) { + ksort($cleanup); + } + if (is_array($storedCleanup)) { + ksort($storedCleanup); + } + if (! LocalActivityRuntime::isExecution($execution) || $execution->status !== ActivityStatus::Pending + || $execution->attempt_count !== 0 || $execution->current_attempt_id !== null + || ! $scheduled instanceof WorkflowHistoryEvent || ($admission['version'] ?? null) !== 1 + || ($admission['descriptor_fingerprint'] ?? null) !== $fingerprint + || ($scheduled->payload['activity']['cancellation_scope_id'] ?? null) + !== ($execution->activity_options['cancellation_scope_id'] ?? null) + || ($admission['checkpoint_id'] ?? null) !== ($receipt['checkpoint_id'] ?? null) + || ($admission['batch_fingerprint'] ?? null) !== ($receipt['fingerprint'] ?? null) + || ($admission['workflow_task_attempt'] ?? null) !== ($receipt['workflow_task_attempt'] ?? null) + || ($scheduled->payload['task']['attempt_count'] ?? null) !== ($receipt['workflow_task_attempt'] ?? null) + || ($scheduled->payload['task']['lease_owner'] ?? null) !== ($receipt['lease_owner'] ?? null) + || ($receipt['task_id'] ?? null) !== $scheduled->workflow_task_id + || ($receipt['workflow_run_id'] ?? null) !== $run->id + || ! $memberMatches + || ($scheduled->payload['sequence'] ?? null) !== $execution->sequence + || ($scheduled->payload['local_activity'] ?? null) !== true + || $storedCleanup !== $cleanup + || $path === [] || ! self::sameGroupPath($execution->parallel_group_path, $path) + || ! self::sameGroupPath($scheduled->payload['parallel_group_path'] ?? null, $path)) { + return self::response('local_activity_group_admission_mismatch'); + } + if ($execution->schedule_to_close_deadline_at !== null && now()->gte( + $execution->schedule_to_close_deadline_at + )) { + $event = app(LocalActivityExecutor::class)->expirePortableRetry($run, $task, $execution); + return [ + ...self::response('local_activity_deadline_expired'), + 'event_id' => $event->id, + 'event_type' => $event->event_type->value, + ]; + } + $attempt = app(LocalActivityExecutor::class)->startPortableAttempt($run, $task, $execution, $workerAttemptId, [ + 'version' => 1, + 'descriptor_fingerprint' => $fingerprint, + 'worker_attempt_id' => $workerAttemptId, + 'workflow_task_attempt' => $task->attempt_count, + ...(isset($execution->activity_options['cancellation_cleanup']) + ? [ + 'cancellation_cleanup' => $execution->activity_options['cancellation_cleanup'], + ] : []), + ]); + return self::response(null, $execution, $attempt, task: $task); + } + + /** Object member order can change in database JSON. List order, values and + * scalar types retain authority and must match exactly. + * @param list> $expected + */ + private static function sameGroupPath(mixed $actual, array $expected): bool + { + if (! is_array($actual) || ! array_is_list($actual) || count($actual) !== count($expected)) { + return false; + } + foreach ($actual as $offset => $entry) { + if (! is_array($entry)) { + return false; + } + ksort($entry); + $other = $expected[$offset]; + ksort($other); + if ($entry !== $other) { + return false; + } + } + return true; + } + + /** + * @return array + */ + private static function prepareRetry( + WorkflowRun $run, + WorkflowTask $task, + ActivityExecution $execution, + ActivityAttempt $previous, + string $workerAttemptId, + string $fingerprint, + ): array { + $started = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted) + ->where('payload->activity_attempt_id', $previous->id) + ->first(); + $originalTaskAttempt = $started?->payload['local_preparation']['workflow_task_attempt'] ?? null; + if (! LocalActivityRuntime::isExecution($execution) || $execution->workflow_run_id !== $run->id + || ! in_array($previous->status, [ActivityAttemptStatus::Failed, ActivityAttemptStatus::Expired], true) + || $previous->closed_at === null + || $execution->attempt_count !== $previous->attempt_number + || ! is_int($originalTaskAttempt) + || self::originalStart($run, $execution, $previous, $previous->lease_owner, $originalTaskAttempt) === null + || ($started->payload['local_preparation']['descriptor_fingerprint'] ?? null) !== $fingerprint) { + return self::response('local_activity_retry_preparation_mismatch'); + } + $retry = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityRetryScheduled) + ->where('payload->activity_attempt_id', $previous->id) + ->first(); + if (! $retry instanceof WorkflowHistoryEvent || $retry->sequence <= $started->sequence + || ($retry->payload['activity_execution_id'] ?? null) !== $execution->id + || ! self::retryTaskMatches($run, $task, $execution, $retry) + || ($retry->payload['retry_of_task_id'] ?? null) !== $retry->workflow_task_id + || ($retry->payload['retry_after_attempt_id'] ?? null) !== $previous->id + || ($retry->payload['retry_after_attempt'] ?? null) !== $previous->attempt_number + || ($retry->payload['retry_policy'] ?? null) !== $execution->retry_policy + || ! self::retryAuthorityMatches($retry, $previous, $originalTaskAttempt, $fingerprint) + || ! is_string($retry->payload['retry_available_at'] ?? null) + || $execution->attempt_count >= ActivityRetryPolicy::maxAttemptsFromSnapshot($execution)) { + return self::response('local_activity_retry_preparation_mismatch'); + } + if ($execution->attempts()->where('worker_attempt_id', $workerAttemptId)->exists()) { + return self::response('local_activity_retry_worker_attempt_reused'); + } + // The durable receipt owns backoff. Mutable task availability cannot + // admit a callback early. Total timeout owns the entire retry chain. + if ($execution->schedule_to_close_deadline_at !== null && now()->gte( + $execution->schedule_to_close_deadline_at + )) { + $event = app(LocalActivityExecutor::class)->expirePortableRetry($run, $task, $execution); + return [ + ...self::response('local_activity_deadline_expired'), + 'event_id' => $event->id, + 'event_type' => $event->event_type->value, + ]; + } + if (now()->lt(Carbon::parse($retry->payload['retry_available_at']))) { + return self::response('local_activity_retry_not_due'); + } + $attempt = app(LocalActivityExecutor::class)->startPortableAttempt($run, $task, $execution, $workerAttemptId, [ + 'version' => 1, + 'descriptor_fingerprint' => $fingerprint, + 'worker_attempt_id' => $workerAttemptId, + 'workflow_task_attempt' => $task->attempt_count, + ...(isset($execution->activity_options['cancellation_cleanup']) + ? [ + 'cancellation_cleanup' => $execution->activity_options['cancellation_cleanup'], + ] : []), + ]); + + return self::response(null, $execution, $attempt, task: $task); + } + + /** A sibling's cold recovery releases the shared hosting claim. Only its + * immutable retry chain within the same admitted batch can transfer the + * earlier member's retry authority to the final replacement claim. + */ + private static function retryTaskMatches( + WorkflowRun $run, + WorkflowTask $task, + ActivityExecution $execution, + WorkflowHistoryEvent $retry + ): bool { + $cursor = $retry->payload['retry_task_id'] ?? null; + if ($cursor === $task->id) { + return true; + } + $admission = self::groupAdmission($run, $execution); + if (! is_string($cursor) || $cursor === '' || $admission === null + || ($retry->payload['local_recovery']['version'] ?? null) !== 1) { + return false; + } + $after = $retry->sequence; + $visited = []; + for ($count = 0; $count < 100; ++$count) { + if (isset($visited[$cursor])) { + return false; + } + $visited[$cursor] = true; + $previousTask = ConfiguredV2Models::query('task_model', WorkflowTask::class)->find($cursor); + $links = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityRetryScheduled) + ->where('workflow_task_id', $cursor) + ->where('sequence', '>', $after) + ->get(); + if ($previousTask === null || $previousTask->workflow_run_id !== $run->id + || $previousTask->task_type !== TaskType::Workflow || $previousTask->status !== TaskStatus::Completed + || $links->count() !== 1) { + return false; + } + $link = $links->sole(); + $payload = $link->payload; + $sibling = ActivityExecution::query()->find($payload['activity_execution_id'] ?? null); + $attempt = ActivityAttempt::query()->find($payload['activity_attempt_id'] ?? null); + $started = $attempt === null ? null : $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted) + ->where('payload->activity_attempt_id', $attempt->id) + ->first(); + $epoch = $started?->payload['local_preparation']['workflow_task_attempt'] ?? null; + $fingerprint = $started?->payload['local_preparation']['descriptor_fingerprint'] ?? null; + $next = $payload['retry_task_id'] ?? null; + if (! $sibling instanceof ActivityExecution || ! $attempt instanceof ActivityAttempt + || self::groupAdmission($run, $sibling) !== $admission + || $attempt->status !== ActivityAttemptStatus::Expired || $attempt->closed_at === null + || ! is_int($epoch) || ! is_string($fingerprint) + || self::originalStart($run, $sibling, $attempt, $attempt->lease_owner, $epoch) === null + || ($payload['local_recovery']['version'] ?? null) !== 1 + || ! self::retryAuthorityMatches($link, $attempt, $epoch, $fingerprint) + || ($payload['retry_of_task_id'] ?? null) !== $cursor + || ($payload['retry_after_attempt_id'] ?? null) !== $attempt->id + || ($payload['retry_after_attempt'] ?? null) !== $attempt->attempt_number + || ! is_string($next) || $next === '' || isset($visited[$next])) { + return false; + } + if ($next === $task->id) { + return true; + } + $cursor = $next; + $after = $link->sequence; + } + return false; + } + + /** @return array{task_id: string, checkpoint_id: string, batch_fingerprint: string, workflow_task_attempt: int}|null + */ + private static function groupAdmission(WorkflowRun $run, ActivityExecution $execution): ?array + { + if ($execution->workflow_run_id !== $run->id || ($execution->parallel_group_path ?? []) === []) { + return null; + } + $scheduled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityScheduled) + ->where('payload->activity_execution_id', $execution->id) + ->first(); + $admission = $scheduled?->payload['local_group_admission'] ?? []; + if (! $scheduled instanceof WorkflowHistoryEvent || ! is_string($scheduled->workflow_task_id) + || ($admission['version'] ?? null) !== 1 + || ! is_string($admission['checkpoint_id'] ?? null) || $admission['checkpoint_id'] === '' + || ! is_string($admission['batch_fingerprint'] ?? null) || $admission['batch_fingerprint'] === '' + || ! is_int($admission['workflow_task_attempt'] ?? null) + || $admission['workflow_task_attempt'] < 1) { + return null; + } + return [ + 'task_id' => $scheduled->workflow_task_id, + 'checkpoint_id' => $admission['checkpoint_id'], + 'batch_fingerprint' => $admission['batch_fingerprint'], + 'workflow_task_attempt' => $admission['workflow_task_attempt'], + ]; + } + + /** + * @return array + */ + private static function retryAuthorityMatches( + WorkflowHistoryEvent $retry, + ActivityAttempt $previous, + int $originalTaskAttempt, + string $fingerprint, + ): bool { + $payload = $retry->payload; + if (($payload['local_outcome']['version'] ?? null) === 1) { + return $previous->status === ActivityAttemptStatus::Failed + && $retry->workflow_task_id === $previous->workflow_task_id + && ($payload['local_outcome']['workflow_task_attempt'] ?? null) === $originalTaskAttempt + && ($payload['task']['lease_owner'] ?? null) === $previous->lease_owner + && ($payload['task']['attempt_count'] ?? null) === $originalTaskAttempt; + } + $recovery = $payload['local_recovery'] ?? []; + return $previous->status === ActivityAttemptStatus::Expired + && ($recovery['version'] ?? null) === 1 + && ($recovery['descriptor_fingerprint'] ?? null) === $fingerprint + && ($recovery['original_workflow_task_id'] ?? null) === $previous->workflow_task_id + && ($recovery['original_workflow_task_attempt'] ?? null) === $originalTaskAttempt + && ($recovery['original_lease_owner'] ?? null) === $previous->lease_owner + && ($recovery['callback_stop_state'] ?? null) === 'unknown' + && ($recovery['workflow_task_id'] ?? null) === $retry->workflow_task_id + && ($payload['task']['id'] ?? null) === $retry->workflow_task_id + && is_int($recovery['workflow_task_attempt'] ?? null) + && ($payload['task']['attempt_count'] ?? null) === $recovery['workflow_task_attempt'] + && is_string($recovery['lease_owner'] ?? null) + && ($payload['task']['lease_owner'] ?? null) === $recovery['lease_owner']; + } + + /** + * @return array + */ + private static function duplicate( + WorkflowRun $run, + WorkflowTask $task, + ActivityExecution $execution, + ActivityAttempt $attempt, + string $workerAttemptId, + string $fingerprint, + ): array { + if (! LocalActivityRuntime::isExecution($execution) || $execution->workflow_run_id !== $run->id + || $execution->status !== ActivityStatus::Running + || $attempt->workflow_task_id !== $task->id || $attempt->lease_owner !== $task->lease_owner + || $attempt->worker_attempt_id !== $workerAttemptId || $attempt->status !== ActivityAttemptStatus::Running) { + return self::response('local_activity_preparation_mismatch'); + } + $started = self::originalStart($run, $execution, $attempt, $task->lease_owner, $task->attempt_count); + if (! $started instanceof WorkflowHistoryEvent + || ($started->payload['local_preparation']['descriptor_fingerprint'] ?? null) !== $fingerprint) { + return self::response('local_activity_preparation_mismatch'); + } + if ($attempt->lease_expires_at === null || now()->gte($attempt->lease_expires_at)) { + return self::response('local_activity_preparation_lease_expired'); + } + foreach ([ + $execution->close_deadline_at, + $execution->schedule_to_close_deadline_at, + $execution->heartbeat_deadline_at, + ] as $deadline) { + if ($deadline !== null && now()->gte($deadline)) { + return self::response('local_activity_deadline_expired'); + } + } + + return self::response(null, $execution, $attempt, true, $task); + } + + /** + * @return array + */ + private static function recoveryReceipt(WorkflowHistoryEvent $event, bool $duplicate): array + { + $retryTaskId = $event->payload['retry_task_id'] ?? null; + + return [ + 'recovered' => true, + 'duplicate' => $duplicate, + 'reason' => null, + 'event_id' => $event->id, + 'event_type' => $event->event_type->value, + 'activity_execution_id' => $event->payload['activity_execution_id'], + 'activity_attempt_id' => $event->payload['activity_attempt_id'], + 'workflow_task_id' => $event->workflow_task_id, + 'recorded_at' => $event->recorded_at->toISOString(), + 'callback_stop_state' => 'unknown', + 'claim_released' => $event->event_type === HistoryEventType::ActivityRetryScheduled, + 'created_task_ids' => is_string($retryTaskId) ? [$retryTaskId] : [], + ]; + } + + /** + * @return array + */ + private static function response( + ?string $reason, + ?ActivityExecution $execution = null, + ?ActivityAttempt $attempt = null, + bool $duplicate = false, + ?WorkflowTask $task = null, + ): array { + return [ + 'prepared' => $reason === null, + 'duplicate' => $duplicate, + 'reason' => $reason, + 'activity_execution_id' => $execution?->id, + 'activity_attempt_id' => $attempt?->id, + 'worker_attempt_id' => $attempt?->worker_attempt_id, + 'attempt_number' => $attempt?->attempt_number, + 'workflow_task_id' => $attempt?->workflow_task_id, + 'workflow_task_attempt' => $task?->attempt_count, + 'lease_owner' => $attempt?->lease_owner, + 'lease_expires_at' => $attempt?->lease_expires_at?->toISOString(), + 'start_to_close_deadline_at' => $execution?->close_deadline_at?->toISOString(), + 'schedule_to_close_deadline_at' => $execution?->schedule_to_close_deadline_at?->toISOString(), + 'heartbeat_deadline_at' => $execution?->heartbeat_deadline_at?->toISOString(), + 'cancellation_cleanup' => $execution?->activity_options['cancellation_cleanup'] ?? null, + ...(isset($execution?->activity_options['cancellation_scope_id']) ? [ + 'cancellation_scope_id' => $execution->activity_options['cancellation_scope_id'], + ] : []), + 'server_time' => now() + ->toISOString(), + ]; + } +} diff --git a/src/V2/Support/RunLineageView.php b/src/V2/Support/RunLineageView.php index a4a898aaf..1f44ee9f2 100644 --- a/src/V2/Support/RunLineageView.php +++ b/src/V2/Support/RunLineageView.php @@ -344,7 +344,9 @@ private static function resolveRun(WorkflowRun $selectedRun, string $runId): ?Wo * parent_close_policy: ?string, * parent_close_policy_outcome: ?string, * parent_close_policy_reason: ?string, - * parent_close_policy_error: ?string + * parent_close_policy_error: ?string, + * parent_close_cancellation: ?array, + * parent_close_request_diagnostics: ?array * }|null $parentClosePolicy * @return array */ @@ -400,6 +402,11 @@ private static function entry( 'parent_close_policy_outcome' => $parentClosePolicy['parent_close_policy_outcome'] ?? null, 'parent_close_policy_reason' => $parentClosePolicy['parent_close_policy_reason'] ?? null, 'parent_close_policy_error' => $parentClosePolicy['parent_close_policy_error'] ?? null, + ...(isset($parentClosePolicy['parent_close_cancellation']) + || isset($parentClosePolicy['parent_close_request_diagnostics']) ? [ + 'parent_close_cancellation' => $parentClosePolicy['parent_close_cancellation'] ?? null, + 'parent_close_request_diagnostics' => $parentClosePolicy['parent_close_request_diagnostics'] ?? null, + ] : []), ]; } @@ -415,7 +422,9 @@ private static function entry( * parent_close_policy: ?string, * parent_close_policy_outcome: ?string, * parent_close_policy_reason: ?string, - * parent_close_policy_error: ?string + * parent_close_policy_error: ?string, + * parent_close_cancellation: ?array, + * parent_close_request_diagnostics: ?array * }|null */ private static function parentClosePolicyForChild( @@ -447,6 +456,10 @@ private static function parentClosePolicyForChild( 'parent_close_policy_outcome' => $outcome, 'parent_close_policy_reason' => $reason, 'parent_close_policy_error' => $error, + 'parent_close_cancellation' => is_array($eventPayload['cancellation'] ?? null) + ? $eventPayload['cancellation'] : null, + 'parent_close_request_diagnostics' => is_array($eventPayload['request_diagnostics'] ?? null) + ? $eventPayload['request_diagnostics'] : null, ]; } diff --git a/src/V2/Support/ScopedActivityCancellation.php b/src/V2/Support/ScopedActivityCancellation.php new file mode 100644 index 000000000..1ec04d455 --- /dev/null +++ b/src/V2/Support/ScopedActivityCancellation.php @@ -0,0 +1,222 @@ + + */ + public static function fence( + WorkflowRun $run, + WorkflowTask $workflowTask, + string $executionId, + string $scopeId, + string $requestId, + string $protocolVersion, + ?string $preparationHistoryEventId = null, + ): array { + if (! WorkerProtocolVersion::supportsCancellationScopeMembership($protocolVersion)) { + throw new LogicException('cancellation_scope_requires_protocol_1_20'); + } + return $run->getConnection() + ->transaction(static function () use ( + $run, + $workflowTask, + $executionId, + $scopeId, + $requestId, + $preparationHistoryEventId, + ): array { + // Preserve the worker's activity attempt/execution lock prefix. + $rows = ActivityRowLockOrder::lockForExecution($executionId, true); + $execution = $rows['execution']; + if (! $execution instanceof ActivityExecution || $execution->workflow_run_id !== $run->id) { + throw new LogicException('cancellation_scope_activity_not_found'); + } + if ($execution->current_attempt_id !== $rows['snapshot_attempt_id'] + || ($execution->current_attempt_id !== null && $rows['attempt'] === null) + || ($rows['attempt'] !== null && ($rows['attempt']->workflow_run_id !== $run->id + || $rows['attempt']->activity_execution_id !== $executionId)) + || ($execution->status === ActivityStatus::Running && $rows['attempt'] === null)) { + throw new LogicException('cancellation_scope_activity_attempt_changed'); + } + /** @var WorkflowRun $locked */ + $locked = ConfiguredV2Models::query('run_model', WorkflowRun::class)->lockForUpdate()->findOrFail( + $run->id + ); + /** @var WorkflowTask|null $claim */ + $claim = ConfiguredV2Models::query('task_model', WorkflowTask::class)->lockForUpdate()->find( + $workflowTask->id + ); + if ($claim === null || $claim->workflow_run_id !== $locked->id || $claim->namespace !== $locked->namespace + || $claim->task_type !== TaskType::Workflow || $claim->status !== TaskStatus::Leased + || $claim->lease_owner === null || $claim->lease_owner === '' || $claim->attempt_count < 1 + || $claim->lease_owner !== $workflowTask->lease_owner || $claim->attempt_count !== $workflowTask->attempt_count + || $claim->lease_expires_at === null || now() + ->gte($claim->lease_expires_at)) { + throw new LogicException('cancellation_scope_workflow_claim_mismatch'); + } + $context = CancellationScopeRequests::context($locked, $scopeId); + if ($context === null || $context->requestId !== $requestId) { + throw new LogicException('cancellation_scope_request_mismatch'); + } + $authority = CancellationScopeRequests::authority($locked, $scopeId); + if (! $authority['active'] || $authority['deadline_at'] === null) { + throw new LogicException('cancellation_scope_authority_expired'); + } + $request = $locked->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeRequested) + ->where('payload->scope_id', $scopeId) + ->sole(); + $scope = CancellationScopeHistory::forRun($locked)[$scopeId]; + if ($scope['shield_parent'] && $request->payload['parent_scope_id'] !== null) { + throw new LogicException('cancellation_scope_parent_shielded'); + } + $metadata = [ + 'schema' => ActivityCancellationContext::SCOPE_SCHEMA, + 'workflow_run_id' => $locked->id, + 'scope_id' => $scopeId, + 'request_id' => $requestId, + 'request_history_event_id' => $request->id, + 'cancellation' => $context->toArray(), + 'authority_deadline_at' => $authority['deadline_at'], + ]; + if (ActivityCancellationContext::forScopeSnapshot( + $locked, + $metadata, + $executionId, + $execution->sequence + ) === null + || ($execution->activity_options['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID) !== $scopeId) { + throw new LogicException('cancellation_scope_activity_membership_mismatch'); + } + $preparation = ScopedCancellationPreparation::forScope($locked, $scopeId, $preparationHistoryEventId); + if ($preparation !== null) { + ScopedCancellationReconciliation::assertDispatchable($locked, $preparation); + } + $policy = ActivityCancellationWait::policy($locked, $execution->sequence); + $local = LocalActivityRuntime::isExecution($execution); + if ($policy === CancellationPolicy::Abandon) { + if ($local || $execution->schedule_to_close_deadline_at === null) { + throw new LogicException('cancellation_scope_activity_abandon_not_supported'); + } + return [ + 'fenced' => false, + 'abandoned' => true, + 'waiting_for_stop' => false, + 'history_event_id' => null, + ]; + } + if (in_array($execution->status, [ActivityStatus::Completed, ActivityStatus::Failed], true)) { + if (! ActivityCancellationCompletion::resolved($locked, $executionId, $context)) { + throw new LogicException('cancellation_scope_activity_terminal_history_not_recorded'); + } + return [ + 'fenced' => false, + 'abandoned' => false, + 'waiting_for_stop' => false, + 'history_event_id' => null, + ]; + } + if ($preparation === null) { + throw new LogicException('cancellation_scope_delivery_not_prepared'); + } + if (now()->gte(\Carbon\CarbonImmutable::parse($preparation->authorityDeadlineAt))) { + throw new LogicException('cancellation_scope_authority_expired'); + } + if (! collect($preparation->activityMembers)->contains(static fn (array $member): bool => + $member['activity_execution_id'] === $executionId && $member['sequence'] === $execution->sequence)) { + throw new LogicException('cancellation_scope_activity_not_prepared'); + } + $metadata['authority_deadline_at'] = $preparation->authorityDeadlineAt; + $existing = $locked->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancelled) + ->where('payload->activity_execution_id', $executionId) + ->first(); + if ($existing !== null) { + $original = ActivityCancellationContext::forEvent($locked, $existing); + if ($original === null || $original->requestId !== $requestId) { + throw new LogicException('activity_cancellation_owned_by_another_request'); + } + $event = $existing; + } elseif (in_array($execution->status, [ActivityStatus::Pending, ActivityStatus::Running], true)) { + $activityTask = $local ? $claim : null; + if (! $local) { + $tasks = $locked->tasks() + ->where('task_type', TaskType::Activity) + ->where('payload->activity_execution_id', $executionId); + $activityTask = $execution->status === ActivityStatus::Running + ? $tasks->whereKey($rows['attempt']->workflow_task_id)->lockForUpdate()->sole() + : $tasks->whereIn('status', [TaskStatus::Ready, TaskStatus::Leased])->lockForUpdate() + ->sole(); + } + // The remote task lock must not extend the hosting claim. + if (now()->gte($claim->lease_expires_at)) { + throw new LogicException('cancellation_scope_workflow_claim_mismatch'); + } + if (now()->gte(\Carbon\CarbonImmutable::parse($authority['deadline_at'])) + || now() + ->gte(\Carbon\CarbonImmutable::parse($preparation->authorityDeadlineAt))) { + throw new LogicException('cancellation_scope_authority_expired'); + } + $event = ActivityCancellation::record( + $locked, + $execution, + $activityTask, + $requestId, + $metadata, + $preparation + ); + if (! $event instanceof WorkflowHistoryEvent) { + throw new LogicException('cancellation_scope_activity_fence_not_recorded'); + } + } else { + // A naturally completed operation is not converted into cancellation. + $locked->unsetRelation('historyEvents'); + if (! in_array($execution->status, [ActivityStatus::Completed, ActivityStatus::Failed], true) + || ! ActivityCancellationCompletion::resolved($locked, $executionId, $context)) { + throw new LogicException('cancellation_scope_activity_terminal_history_not_recorded'); + } + return [ + 'fenced' => false, + 'abandoned' => false, + 'waiting_for_stop' => false, + 'history_event_id' => null, + ]; + } + $locked->unsetRelation('historyEvents'); + return [ + 'fenced' => true, + 'abandoned' => false, + 'history_event_id' => $event->id, + 'request_id' => $requestId, + 'root_request_id' => $context->rootContext->rootRequestId, + 'scope_id' => $scopeId, + 'cleanup_deadline_at' => $context->deadline() + ->toISOString(), + 'cancellation_scope' => $event->payload['cancellation_scope'], + 'waiting_for_stop' => $policy === CancellationPolicy::WaitCancellationCompleted + && ! ActivityCancellationCompletion::resolved($locked, $executionId, $context), + ]; + }, 5); + } +} diff --git a/src/V2/Support/ScopedActivityDeliveryPolicy.php b/src/V2/Support/ScopedActivityDeliveryPolicy.php new file mode 100644 index 000000000..bd53ff828 --- /dev/null +++ b/src/V2/Support/ScopedActivityDeliveryPolicy.php @@ -0,0 +1,111 @@ +loadMissing('historyEvents'); + // Cold replay must prove readiness before the original delivery marker. + // A later stop receipt cannot retroactively make an early marker valid. + $history = $run->historyEvents; + if ($beforeHistorySequence !== null) { + $run->setRelation('historyEvents', $history->filter( + static fn (WorkflowHistoryEvent $event): bool => $event->sequence < $beforeHistorySequence + )); + } + try { + foreach ($run->historyEvents as $scheduled) { + $sequence = $scheduled->payload['sequence'] ?? null; + if ($scheduled->event_type !== HistoryEventType::ActivityScheduled + || ! is_int($sequence) || $sequence < $start || $sequence - $start >= $span) { + continue; + } + $executionId = $scheduled->payload['activity_execution_id'] ?? null; + if (! is_string($executionId) || $executionId === '' + || ($scheduled->payload['activity']['id'] ?? null) !== $executionId + || $run->historyEvents->filter(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::ActivityScheduled + && (($event->payload['activity_execution_id'] ?? null) === $executionId + || ($event->payload['sequence'] ?? null) === $sequence))->count() !== 1) { + throw new LogicException('cancellation_scope_activity_history_invalid'); + } + $value = $scheduled->payload['activity']['cancellation_policy'] ?? CancellationPolicy::TryCancel->value; + $policy = is_string($value) ? CancellationPolicy::tryFrom($value) : null; + if ($policy === null) { + throw new LogicException('cancellation_scope_activity_history_invalid'); + } + $cancelled = $run->historyEvents->first(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::ActivityCancelled + && ($event->payload['activity_execution_id'] ?? null) === $executionId); + if ($policy === CancellationPolicy::Abandon) { + $deadline = $scheduled->payload['activity']['schedule_to_close_deadline_at'] ?? null; + if (($scheduled->payload['local_activity'] ?? false) === true + || ($scheduled->payload['execution_mode'] ?? null) === 'local' + || ! is_string($deadline) || $deadline === '' || $cancelled !== null) { + throw new LogicException('cancellation_scope_activity_abandon_not_supported'); + } + try { + if (CarbonImmutable::parse($deadline)->toISOString() !== $deadline) { + throw new LogicException('cancellation_scope_activity_abandon_not_supported'); + } + } catch (\InvalidArgumentException $error) { + throw new LogicException('cancellation_scope_activity_abandon_not_supported', previous: $error); + } + if ($beforeHistorySequence === null) { + $execution = ActivityExecution::query()->find($executionId); + if ($execution === null || $execution->workflow_run_id !== $run->id + || $execution->sequence !== $sequence || LocalActivityRuntime::isExecution($execution) + || $execution->schedule_to_close_deadline_at === null + || $execution->schedule_to_close_deadline_at->toISOString() !== $deadline) { + throw new LogicException('cancellation_scope_activity_abandon_not_supported'); + } + } + continue; + } + if ($cancelled === null && ActivityCancellationCompletion::resolved($run, $executionId, $context)) { + continue; + } + $original = $cancelled === null ? null : ActivityCancellationContext::forEvent($run, $cancelled); + if (! $original instanceof ScopedCancellationContext || $original->toArray() !== $context->toArray()) { + throw new LogicException('cancellation_scope_activity_fence_not_established'); + } + if ($beforeHistorySequence === null) { + // Mutation is serialized by the caller's run lock. Projection + // drift must not turn a history-only fence into permission. + $execution = ActivityExecution::query()->find($executionId); + if ($execution === null || $execution->workflow_run_id !== $run->id + || $execution->status !== ActivityStatus::Cancelled + || $execution->current_attempt_id !== ($cancelled->payload['activity_attempt_id'] ?? null)) { + throw new LogicException('cancellation_scope_activity_fence_not_established'); + } + } + if ($policy === CancellationPolicy::WaitCancellationCompleted + && ! ActivityCancellationCompletion::resolved($run, $executionId, $context)) { + throw new LogicException('cancellation_scope_activity_stop_not_acknowledged'); + } + } + } finally { + $run->setRelation('historyEvents', $history); + } + } +} diff --git a/src/V2/Support/ScopedCancellationCleanup.php b/src/V2/Support/ScopedCancellationCleanup.php new file mode 100644 index 000000000..8098308c2 --- /dev/null +++ b/src/V2/Support/ScopedCancellationCleanup.php @@ -0,0 +1,128 @@ + $command + * @return array>|null + */ + public static function metadata(array $command): ?array + { + if (! array_key_exists('cancellation_cleanup', $command)) { + return []; + } + $proof = $command['cancellation_cleanup']; + if (! is_array($proof) || count($proof) !== 3 + || array_diff(array_keys($proof), ['scope_id', 'request_id', 'delivery_history_event_id']) !== [] + || ! is_string($command['cancellation_scope_id'] ?? null) + || $command['cancellation_scope_id'] === CancellationScopeHistory::ROOT_SCOPE_ID) { + return null; + } + foreach (['scope_id', 'request_id', 'delivery_history_event_id'] as $field) { + if (! is_string($proof[$field] ?? null) || trim($proof[$field]) === '' + || strlen($proof[$field]) > 255 || preg_match('//u', $proof[$field]) !== 1) { + return null; + } + } + if ($proof['scope_id'] === CancellationScopeHistory::ROOT_SCOPE_ID) { + return null; + } + return [ + 'cancellation_cleanup' => $proof, + ]; + } + + /** + * @param array $command + */ + public static function timerRefusal(WorkflowRun $run, array $command, int $sequence): ?string + { + if (! isset($command['cancellation_cleanup'])) { + return null; + } + $snapshot = PortableLocalActivityCleanup::snapshot( + $run, + $command['cancellation_cleanup'], + $sequence, + $command['cancellation_scope_id'], + ); + if ($snapshot === null) { + return 'cancellation_scope_cleanup_authority_mismatch'; + } + $authority = CancellationScopeRequests::authority($run, $command['cancellation_scope_id']); + $deadline = CarbonImmutable::parse($snapshot['authority_deadline_at']); + if ($authority['deadline_at'] !== null) { + $deadline = $deadline->min(CarbonImmutable::parse($authority['deadline_at'])); + } + if (! $authority['active'] || now()->gte($deadline)) { + return 'cancellation_scope_cleanup_authority_expired'; + } + // Preserve the authored delay. Do not turn a too-long wait into an + // early TimerFired event or let it extend the original cleanup budget. + return now()->addSeconds($command['delay_seconds'])->gte($deadline) + ? 'cancellation_scope_cleanup_timer_exceeds_deadline' : null; + } + + /** + * A later ancestor preparation must preserve already shielded cleanup. + */ + public static function isScheduledTimer(WorkflowRun $run, WorkflowHistoryEvent $event): bool + { + $stored = $event->payload['cancellation_cleanup'] ?? null; + if (! array_key_exists('cancellation_cleanup', $event->payload)) { + return false; + } + $sequence = $event->payload['sequence'] ?? null; + $scope = $event->payload['cancellation_scope_id'] ?? null; + if ($event->event_type !== HistoryEventType::TimerScheduled || ! is_array($stored) + || ! is_int($sequence) || ! is_string($scope)) { + throw new LogicException('cancellation_scope_cleanup_history_invalid'); + } + $deliverySequence = $run->historyEvents() + ->whereKey($stored['delivery_history_event_id'] ?? null) + ->where('event_type', HistoryEventType::CancellationScopeDelivered)->value('sequence'); + if (! is_int($deliverySequence) || $deliverySequence >= $event->sequence) { + throw new LogicException('cancellation_scope_cleanup_history_invalid'); + } + $run->loadMissing('historyEvents'); + $history = $run->historyEvents; + $run->setRelation('historyEvents', $history->filter( + static fn (WorkflowHistoryEvent $row): bool => $row->sequence < $event->sequence, + )); + try { + $snapshot = PortableLocalActivityCleanup::snapshot($run, [ + 'scope_id' => $stored['scope_id'] ?? null, + 'request_id' => $stored['request_id'] ?? null, + 'delivery_history_event_id' => $stored['delivery_history_event_id'] ?? null, + ], $sequence, $scope); + } finally { + $run->setRelation('historyEvents', $history); + } + if ($snapshot === null) { + throw new LogicException('cancellation_scope_cleanup_history_invalid'); + } + ksort($snapshot); + ksort($stored); + if ($snapshot !== $stored) { + throw new LogicException('cancellation_scope_cleanup_history_invalid'); + } + $deadline = CarbonImmutable::parse($snapshot['authority_deadline_at']); + if (! is_string($event->payload['fire_at'] ?? null) + || CarbonImmutable::parse($event->payload['fire_at'])->gte($deadline) + || CarbonImmutable::parse($event->recorded_at ?? $event->created_at)->gte($deadline) + || ($event->payload['timer_kind'] ?? null) !== null) { + throw new LogicException('cancellation_scope_cleanup_history_invalid'); + } + return true; + } +} diff --git a/src/V2/Support/ScopedCancellationPreparation.php b/src/V2/Support/ScopedCancellationPreparation.php new file mode 100644 index 000000000..a0c460810 --- /dev/null +++ b/src/V2/Support/ScopedCancellationPreparation.php @@ -0,0 +1,125 @@ +> $activityMembers + * @param list> $timerMembers + * @param list> $waitMembers + * @param list> $childMembers + */ + private function __construct( + public readonly string $workflowRunId, + public readonly string $historyEventId, + public readonly int $historySequence, + public readonly string $preparedScopeId, + public readonly string $scopeId, + public readonly string $requestId, + public readonly string $requestHistoryEventId, + public readonly ScopedCancellationContext $context, + public readonly string $authorityDeadlineAt, + public readonly array $activityMembers, + public readonly array $timerMembers, + public readonly array $waitMembers, + public readonly array $childMembers, + public readonly CarbonImmutable $recordedAt, + ) { + } + + public static function forScope( + WorkflowRun $run, + string $scopeId, + ?string $preparationHistoryEventId = null, + ): ?self { + if ($preparationHistoryEventId === null) { + $event = CancellationScopeDelivery::prepared($run, $scopeId); + if ($event === null) { + return null; + } + } else { + $original = $run->historyEvents() + ->whereKey($preparationHistoryEventId) + ->where('event_type', HistoryEventType::CancellationScopeDeliveryPrepared) + ->first(); + $preparedScope = $original?->payload['scope_id'] ?? null; + if (! is_string($preparedScope) || $preparedScope === '') { + throw new LogicException('cancellation_scope_preparation_reference_invalid'); + } + // Use the canonical producer read. An ID cannot bypass its prefix validation. + $event = CancellationScopeDelivery::prepared($run, $preparedScope); + if ($event === null || $event->id !== $preparationHistoryEventId) { + throw new LogicException('cancellation_scope_preparation_reference_invalid'); + } + } + $payload = $event->payload; + $snapshot = $payload; + if ($payload['scope_id'] !== $scopeId) { + $members = array_values(array_filter( + CancellationScopeDescendants::normalizeMembers($payload['descendant_members'] ?? []), + static fn (array $member): bool => $member['scope_id'] === $scopeId, + )); + if (count($members) !== 1) { + throw new LogicException('cancellation_scope_descendant_not_prepared'); + } + $snapshot = $members[0]; + } + $context = ScopedCancellationContext::fromArray($snapshot['cancellation']); + $request = $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeRequested) + ->where('payload->scope_id', $scopeId) + ->sole(); + if ($context->scopeId !== $scopeId || $context->workflowRunId !== $run->id + || $context->requestId !== $snapshot['request_id'] + || ($request->payload['request_id'] ?? null) !== $context->requestId + || $request->sequence >= $event->sequence + || ($payload['scope_id'] !== $scopeId && $request->id !== $snapshot['request_history_event_id'])) { + throw new LogicException('cancellation_scope_preparation_reference_invalid'); + } + $deadline = CarbonImmutable::parse($snapshot['authority_deadline_at']); + if ($deadline->gt(CarbonImmutable::parse($payload['authority_deadline_at'])) + || $deadline->gt($context->deadline())) { + throw new LogicException('cancellation_scope_preparation_reference_invalid'); + } + return new self( + $run->id, + $event->id, + $event->sequence, + $payload['scope_id'], + $scopeId, + $context->requestId, + $request->id, + $context, + $snapshot['authority_deadline_at'], + CancellationScopeDelivery::normalizeMembers($snapshot['activity_members'] ?? []), + ScopedTimerCancellation::normalizeMembers($snapshot['timer_members'] ?? []), + ScopedWaitCancellation::normalizeMembers($snapshot['wait_members'] ?? []), + ScopedChildCancellation::normalizeMembers($snapshot['child_members'] ?? []), + CarbonImmutable::parse($event->recorded_at ?? $event->created_at), + ); + } + + public static function fromEvent(WorkflowRun $run, WorkflowHistoryEvent $event): self + { + $scopeId = $event->payload['scope_id'] ?? null; + if ($event->workflow_run_id !== $run->id || ! is_string($scopeId)) { + throw new LogicException('cancellation_scope_preparation_reference_invalid'); + } + return self::forScope($run, $scopeId, $event->id) + ?? throw new LogicException('cancellation_scope_preparation_reference_invalid'); + } +} diff --git a/src/V2/Support/ScopedCancellationReconciliation.php b/src/V2/Support/ScopedCancellationReconciliation.php new file mode 100644 index 000000000..81e4ad683 --- /dev/null +++ b/src/V2/Support/ScopedCancellationReconciliation.php @@ -0,0 +1,244 @@ +>> + */ + public function receipts(WorkflowRun $run): array + { + if ($run->id !== $this->preparation->workflowRunId) { + throw new LogicException('cancellation_scope_preparation_reference_invalid'); + } + $run->loadMissing('historyEvents'); + $history = $run->historyEvents; + $run->setRelation('historyEvents', $history->filter( + fn (WorkflowHistoryEvent $event): bool => $event->sequence < $this->delivery->sequence + )); + try { + $result = [ + 'activity_cancellations' => [], + 'timer_cancellations' => [], + 'wait_cancellations' => [], + 'child_cancellations' => [], + ]; + foreach ($this->preparation->timerMembers as $member) { + $terminal = ScopedTimerCancellation::terminal($run, $member['timer_id'], $member['sequence']); + if ($terminal === null) { + throw new LogicException('cancellation_scope_timer_fence_not_established'); + } + $cancelled = $terminal->event_type === HistoryEventType::TimerCancelled; + $result['timer_cancellations'][] = [ + 'fenced' => $cancelled, + 'history_event_id' => $cancelled ? $terminal->id : null, + 'timer_id' => $member['timer_id'], + 'sequence' => $member['sequence'], + ]; + } + foreach ($this->preparation->waitMembers as $member) { + $terminal = ScopedWaitCancellation::terminal($run, $member); + if ($terminal === null) { + throw new LogicException('cancellation_scope_wait_fence_not_established'); + } + $result['wait_cancellations'][] = [ + 'kind' => $member['kind'], + 'wait_id' => $member['wait_id'], + 'sequence' => $member['sequence'], + 'cancelled' => in_array($terminal->event_type, [HistoryEventType::SignalWaitCancelled, + HistoryEventType::ConditionWaitCancelled], true), + 'history_event_id' => $terminal->id, + ]; + } + foreach ($this->preparation->activityMembers as $member) { + $scheduled = $run->historyEvents->first(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::ActivityScheduled + && ($event->payload['activity_execution_id'] ?? null) === $member['activity_execution_id']); + $cancelled = $run->historyEvents->first(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::ActivityCancelled + && ($event->payload['activity_execution_id'] ?? null) === $member['activity_execution_id']); + $receipt = [ + 'fenced' => false, + 'abandoned' => ($scheduled?->payload['activity']['cancellation_policy'] ?? null) + === CancellationPolicy::Abandon->value, + 'waiting_for_stop' => false, + 'history_event_id' => null, + ]; + if ($cancelled !== null) { + $snapshot = $cancelled->payload['cancellation_scope']; + $context = $this->preparation->context; + $receipt = [ + 'fenced' => true, + 'abandoned' => false, + 'history_event_id' => $cancelled->id, + 'request_id' => $context->requestId, + 'root_request_id' => $context->rootContext->rootRequestId, + 'scope_id' => $context->scopeId, + 'cleanup_deadline_at' => $context->deadline() + ->toISOString(), + 'cancellation_scope' => [ + 'schema' => $snapshot['schema'], + 'workflow_run_id' => $snapshot['workflow_run_id'], + 'scope_id' => $snapshot['scope_id'], + 'request_id' => $snapshot['request_id'], + 'request_history_event_id' => $snapshot['request_history_event_id'], + 'cancellation' => ScopedCancellationContext::fromArray( + $snapshot['cancellation'] + )->toArray(), + 'authority_deadline_at' => $snapshot['authority_deadline_at'], + ], + 'waiting_for_stop' => ($scheduled?->payload['activity']['cancellation_policy'] ?? null) + === CancellationPolicy::WaitCancellationCompleted->value + && ! ActivityCancellationCompletion::resolved( + $run, + $member['activity_execution_id'], + $context + ), + ]; + } + $result['activity_cancellations'][] = [ + 'sequence' => $member['sequence'], + 'activity_execution_id' => $member['activity_execution_id'], + ...$receipt, + ]; + } + foreach ($this->preparation->childMembers as $member) { + $matches = $run->historyEvents->filter(static fn (WorkflowHistoryEvent $event): bool => + ($event->payload['sequence'] ?? null) === $member['sequence'] && $event->workflow_command_id === null); + $receipt = $matches->firstWhere('event_type', HistoryEventType::ChildCancellationRequested); + $resolved = $matches->firstWhere('event_type', HistoryEventType::ChildCancellationResolved); + if ($receipt === null) { + throw new LogicException('cancellation_scope_child_delivery_not_established'); + } + $result['child_cancellations'][] = [ + 'child_call_id' => $member['child_call_id'], + 'child_workflow_run_id' => $member['child_workflow_run_id'], + 'history_event_id' => $receipt->id, + 'resolution_history_event_id' => $resolved?->id, + 'request_id' => $receipt->payload['child_request_id'], + 'policy' => $member['cancellation_policy'], + 'ready' => $member['cancellation_policy'] !== CancellationPolicy::WaitCancellationCompleted->value + || $resolved !== null, + ]; + } + return $result; + } finally { + $run->setRelation('historyEvents', $history); + } + } + + /** + * @param list $candidates + */ + private static function completedFrom( + WorkflowRun $run, + array $candidates, + ?int $beforeHistorySequence, + ): ?self { + foreach ($candidates as $candidate) { + $event = $run->historyEvents->first(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::CancellationScopeDelivered + && ($event->payload['scope_id'] ?? null) === $candidate['scope_id']); + if ($event === null || ($beforeHistorySequence !== null && $event->sequence >= $beforeHistorySequence)) { + continue; + } + // Delivery may finish after ancestor preparation, but must precede + // the consuming marker. Strictly earlier preparations keep reads finite. + $delivery = CancellationScopeDelivery::recorded($run, $candidate['scope_id']); + if ($delivery === null || $delivery->id !== $event->id) { + throw new LogicException('cancellation_scope_delivery_history_invalid'); + } + return new self($candidate['reference'], $delivery); + } + return null; + } + + /** + * @return list + */ + private static function earlier(WorkflowRun $run, ScopedCancellationPreparation $current): array + { + if ($current->workflowRunId !== $run->id) { + throw new LogicException('cancellation_scope_preparation_reference_invalid'); + } + $run->loadMissing('historyEvents'); + $candidates = []; + foreach ($run->historyEvents->sortBy('sequence') as $event) { + if ($event->event_type !== HistoryEventType::CancellationScopeDeliveryPrepared + || $event->sequence >= $current->historySequence) { + continue; + } + $scopeId = $event->payload['scope_id'] ?? null; + if (! is_string($scopeId)) { + throw new LogicException('cancellation_scope_preparation_history_invalid'); + } + if ($scopeId !== $current->scopeId && ! collect($event->payload['descendant_members'] ?? []) + ->contains('scope_id', $current->scopeId)) { + continue; + } + $prepared = CancellationScopeDelivery::prepared($run, $scopeId); + if ($prepared === null || $prepared->id !== $event->id) { + throw new LogicException('cancellation_scope_preparation_history_invalid'); + } + $original = ScopedCancellationPreparation::forScope($run, $current->scopeId, $prepared->id); + if ($original === null || $original->requestHistoryEventId !== $current->requestHistoryEventId + || $original->context->toArray() !== $current->context->toArray() + || $original->activityMembers !== $current->activityMembers + || $original->timerMembers !== $current->timerMembers + || $original->waitMembers !== $current->waitMembers + || $original->childMembers !== $current->childMembers) { + throw new LogicException('cancellation_scope_descendant_delivery_membership_mismatch'); + } + // Captured ceilings can differ. Neither a later preparation nor a + // completed proof transfers or renews the original actor authority. + $candidates[] = [ + 'scope_id' => $scopeId, + 'reference' => $original, + ]; + } + return $candidates; + } +} diff --git a/src/V2/Support/ScopedChildCancellation.php b/src/V2/Support/ScopedChildCancellation.php new file mode 100644 index 000000000..7e610c030 --- /dev/null +++ b/src/V2/Support/ScopedChildCancellation.php @@ -0,0 +1,143 @@ + + */ + public static function members(WorkflowRun $run, string $scopeId): array + { + $run->loadMissing('historyEvents'); + $members = []; + $calls = []; + $sequences = []; + foreach ($run->historyEvents->sortBy('sequence') as $event) { + if ($event->event_type !== HistoryEventType::ChildWorkflowScheduled) { + continue; + } + $payload = $event->payload; + $descriptor = $payload['child_workflow'] ?? []; + if (! is_array($descriptor)) { + throw new LogicException('cancellation_scope_child_history_invalid'); + } + $nested = $descriptor['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID; + $address = $payload['cancellation_scope_id'] ?? $nested; + if ($address !== $scopeId && $nested !== $scopeId) { + continue; + } + if ($address !== $scopeId || (array_key_exists('cancellation_scope_id', $descriptor) + && $nested !== $scopeId)) { + throw new LogicException('cancellation_scope_delivery_membership_mismatch'); + } + $sequence = $payload['sequence'] ?? null; + $callId = $payload['child_call_id'] ?? null; + $instanceId = $payload['child_workflow_instance_id'] ?? null; + $runId = $payload['child_workflow_run_id'] ?? null; + $policy = $payload['cancellation_policy'] ?? CancellationPolicy::Abandon->value; + if (! is_int($sequence) || $sequence < 1 || ! self::text($callId) || ! self::text($instanceId) + || ! self::text($runId) || ! is_string($policy) || CancellationPolicy::tryFrom($policy) === null + || $runId === $run->id || isset($calls[$callId]) || isset($sequences[$sequence])) { + throw new LogicException('cancellation_scope_child_history_invalid'); + } + // A continued/retried child is selected only from history that was + // committed before preparation. Mutable links and current_run_id + // must never substitute a newer target during cold replay. + $started = $run->historyEvents->filter(static fn (WorkflowHistoryEvent $row): bool => + $row->event_type === HistoryEventType::ChildRunStarted + && ($row->payload['sequence'] ?? null) === $sequence)->sortBy('sequence'); + foreach ($started as $row) { + $next = $row->payload; + if ($row->sequence <= $event->sequence || ($next['child_call_id'] ?? null) !== $callId + || ($next['child_workflow_instance_id'] ?? null) !== $instanceId + || ! self::text($next['child_workflow_run_id'] ?? null) + || $next['child_workflow_run_id'] === $run->id + || ($next['cancellation_scope_id'] ?? $address) !== $address + || ($next['cancellation_policy'] ?? $policy) !== $policy) { + throw new LogicException('cancellation_scope_child_history_invalid'); + } + $runId = $next['child_workflow_run_id']; + } + $calls[$callId] = true; + $sequences[$sequence] = true; + $members[] = [ + 'sequence' => $sequence, + 'child_call_id' => $callId, + 'child_workflow_instance_id' => $instanceId, + 'child_workflow_run_id' => $runId, + 'cancellation_policy' => $policy, + 'descriptor_hash' => hash('sha256', json_encode([ + $scopeId, $event->id, $sequence, $callId, $instanceId, $runId, $policy, + $payload['parent_close_policy'] ?? null, $payload['child_workflow_type'] ?? null, + $started->last()?->id, ParallelChildGroup::metadataPathFromPayload($payload), + ], JSON_THROW_ON_ERROR)), + ]; + } + return $members; + } + + /** + * @return list + */ + public static function normalizeMembers(mixed $members): array + { + if (! is_array($members) || ! array_is_list($members)) { + throw new LogicException('cancellation_scope_preparation_history_invalid'); + } + $normalized = []; + foreach ($members as $member) { + if (! is_array($member) || count($member) !== 6 || ! is_int($member['sequence'] ?? null) + || $member['sequence'] < 1 || ! self::text($member['child_call_id'] ?? null) + || ! self::text($member['child_workflow_instance_id'] ?? null) + || ! self::text($member['child_workflow_run_id'] ?? null) + || ! is_string($member['cancellation_policy'] ?? null) + || CancellationPolicy::tryFrom($member['cancellation_policy']) === null + || ! is_string($member['descriptor_hash'] ?? null) + || preg_match('/^[a-f0-9]{64}$/D', $member['descriptor_hash']) !== 1) { + throw new LogicException('cancellation_scope_preparation_history_invalid'); + } + $normalized[] = [ + 'sequence' => $member['sequence'], + 'child_call_id' => $member['child_call_id'], + 'child_workflow_instance_id' => $member['child_workflow_instance_id'], + 'child_workflow_run_id' => $member['child_workflow_run_id'], + 'cancellation_policy' => $member['cancellation_policy'], + 'descriptor_hash' => $member['descriptor_hash'], + ]; + } + return $normalized; + } + + /** + * A preparation snapshot is not proof that the child actor committed. + */ + public static function assertReady( + WorkflowHistoryEvent|ScopedCancellationPreparation $preparation, + ?WorkflowRun $run = null, + ?int $beforeHistorySequence = null, + ): void { + $members = $preparation instanceof ScopedCancellationPreparation + ? $preparation->childMembers : self::normalizeMembers($preparation->payload['child_members']); + if ($members !== []) { + if ($run === null) { + throw new LogicException('cancellation_scope_child_delivery_not_established'); + } + ScopedChildCancellationDelivery::assertReady($run, $preparation, $beforeHistorySequence); + } + } + + private static function text(mixed $value): bool + { + return is_string($value) && trim($value) !== ''; + } +} diff --git a/src/V2/Support/ScopedChildCancellationDelivery.php b/src/V2/Support/ScopedChildCancellationDelivery.php new file mode 100644 index 000000000..8bcce1dce --- /dev/null +++ b/src/V2/Support/ScopedChildCancellationDelivery.php @@ -0,0 +1,425 @@ + + */ + public static function request( + WorkflowRun $run, + WorkflowTask $workflowTask, + string $childCallId, + string $scopeId, + string $requestId, + string $protocolVersion, + ?string $preparationHistoryEventId = null, + ): array { + if (! WorkerProtocolVersion::supportsCancellationScopeMembership($protocolVersion)) { + throw new LogicException('cancellation_scope_requires_protocol_1_20'); + } + if ($run->getConnection()->transactionLevel() !== 0) { + throw new LogicException('cancellation_scope_child_requires_own_transaction'); + } + return $run->getConnection() + ->transaction(static function () use ( + $run, + $workflowTask, + $childCallId, + $scopeId, + $requestId, + $preparationHistoryEventId + ): array { + /** @var WorkflowRun $parent */ + $parent = ConfiguredV2Models::query('run_model', WorkflowRun::class)->lockForUpdate()->findOrFail( + $run->id + ); + /** @var WorkflowTask|null $claim */ + $claim = ConfiguredV2Models::query('task_model', WorkflowTask::class)->lockForUpdate()->find( + $workflowTask->id + ); + if ($claim === null || $claim->workflow_run_id !== $parent->id || $claim->namespace !== $parent->namespace + || $claim->task_type !== TaskType::Workflow || $claim->status !== TaskStatus::Leased + || $claim->lease_owner === null || $claim->lease_owner === '' || $claim->attempt_count < 1 + || $claim->lease_owner !== $workflowTask->lease_owner || $claim->attempt_count !== $workflowTask->attempt_count + || $claim->lease_expires_at === null || now() + ->gte($claim->lease_expires_at)) { + throw new LogicException('cancellation_scope_workflow_claim_mismatch'); + } + $preparation = ScopedCancellationPreparation::forScope($parent, $scopeId, $preparationHistoryEventId); + $origin = CancellationScopeRequests::context($parent, $scopeId); + $authority = CancellationScopeRequests::authority($parent, $scopeId); + if ($origin === null || $origin->requestId !== $requestId) { + throw new LogicException('cancellation_scope_request_mismatch'); + } + if ($preparation === null) { + throw new LogicException('cancellation_scope_delivery_not_prepared'); + } + ScopedCancellationReconciliation::assertDispatchable($parent, $preparation); + if (! $authority['active'] || now()->gte($origin->deadline()) + || now() + ->gte(CarbonImmutable::parse($preparation->authorityDeadlineAt))) { + throw new LogicException('cancellation_scope_authority_expired'); + } + $member = collect($preparation->childMembers) + ->firstWhere('child_call_id', $childCallId); + if ($member === null) { + throw new LogicException('cancellation_scope_child_target_mismatch'); + } + $policy = CancellationPolicy::from($member['cancellation_policy']); + $child = ConfiguredV2Models::query('run_model', WorkflowRun::class)->find( + $member['child_workflow_run_id'] + ); + if ($child === null || $child->workflow_instance_id !== $member['child_workflow_instance_id'] + || $child->namespace !== $parent->namespace) { + throw new LogicException('cancellation_scope_child_target_mismatch'); + } + $base = self::base($parent, $preparation, $member); + $receipt = self::receipt( + $parent, + $preparation, + $childCallId, + HistoryEventType::ChildCancellationRequested + ); + $terminal = ChildRunHistory::terminalEventForRun($child); + if ($receipt === null) { + $context = null; + $outcome = $policy === CancellationPolicy::Abandon ? 'abandoned' : 'already_terminal'; + if ($policy !== CancellationPolicy::Abandon && $terminal === null) { + $request = WorkflowStub::loadRun($child->id)->attemptRequestCancellationFromScope( + $parent->id, + $scopeId, + $preparation->historyEventId, + $childCallId + ); + if ($request->rejected()) { + throw new LogicException( + 'cancellation_scope_child_request_refused:' . $request->rejectionReason() + ); + } + $context = $request->cancellationContext(); + $outcome = 'accepted'; + // The child request rechecks parent authority after taking child locks. + $child->refresh(); + $terminal = ChildRunHistory::terminalEventForRun($child); + } + // A child-instance lock can outlast the hosting lease or budget. + $authority = CancellationScopeRequests::authority($parent, $scopeId); + if (! $authority['active'] || now()->gte($origin->deadline()) + || now() + ->gte($claim->lease_expires_at) + || now() + ->gte(CarbonImmutable::parse($preparation->authorityDeadlineAt))) { + throw new LogicException('cancellation_scope_authority_expired'); + } + $receipt = WorkflowHistoryEvent::record($parent, HistoryEventType::ChildCancellationRequested, [ + ...$base, + 'child_request_id' => $context?->requestId, + 'child_root_request_id' => $context?->rootRequestId, + 'child_cleanup_deadline_at' => $context?->deadline() + ->toISOString(), + 'child_cancellation' => $context?->toArray(), + 'request_outcome' => $outcome, + 'rejection_reason' => null, + ], $claim); + $parent->historyEvents->push($receipt); + } + self::assertRequest($parent, $preparation, $member, $receipt); + $resolved = self::receipt( + $parent, + $preparation, + $childCallId, + HistoryEventType::ChildCancellationResolved + ); + if ($policy !== CancellationPolicy::Abandon && $terminal !== null && $resolved === null) { + $authority = CancellationScopeRequests::authority($parent, $scopeId); + if (! $authority['active'] || now()->gte($origin->deadline()) + || now() + ->gte($claim->lease_expires_at) + || now() + ->gte(CarbonImmutable::parse($preparation->authorityDeadlineAt))) { + throw new LogicException('cancellation_scope_authority_expired'); + } + $resolved = WorkflowHistoryEvent::record($parent, HistoryEventType::ChildCancellationResolved, [ + ...$base, + 'child_request_id' => $receipt->payload['child_request_id'], + 'child_root_request_id' => $receipt->payload['child_root_request_id'], + 'child_cleanup_deadline_at' => $receipt->payload['child_cleanup_deadline_at'], + 'child_status' => ChildRunHistory::resolvedStatus(null, $child)?->value, + 'child_terminal_history_event_id' => $terminal->id, + 'child_terminal_event_type' => $terminal->event_type->value, + 'reason' => $terminal->payload['reason'] ?? null, + ], $claim); + $parent->historyEvents->push($resolved); + } + return [ + 'child_call_id' => $childCallId, + 'child_workflow_run_id' => $child->id, + 'history_event_id' => $receipt->id, + 'resolution_history_event_id' => $resolved?->id, + 'request_id' => $receipt->payload['child_request_id'], + 'policy' => $policy->value, + 'ready' => $policy !== CancellationPolicy::WaitCancellationCompleted || $resolved !== null, + ]; + }, 5); + } + + public static function assertReady( + WorkflowRun $run, + WorkflowHistoryEvent|ScopedCancellationPreparation $preparation, + ?int $beforeHistorySequence = null, + ): void { + $preparation = $preparation instanceof WorkflowHistoryEvent + ? ScopedCancellationPreparation::fromEvent($run, $preparation) : $preparation; + foreach ($preparation->childMembers as $member) { + $receipt = self::receipt( + $run, + $preparation, + $member['child_call_id'], + HistoryEventType::ChildCancellationRequested, + $beforeHistorySequence + ); + if ($receipt === null) { + throw new LogicException('cancellation_scope_child_delivery_not_established'); + } + self::assertRequest($run, $preparation, $member, $receipt); + if ($member['cancellation_policy'] !== CancellationPolicy::WaitCancellationCompleted->value + && $receipt->payload['request_outcome'] !== 'already_terminal') { + continue; + } + $resolved = self::receipt( + $run, + $preparation, + $member['child_call_id'], + HistoryEventType::ChildCancellationResolved, + $beforeHistorySequence + ); + if ($resolved === null || $resolved->sequence <= $receipt->sequence) { + throw new LogicException('cancellation_scope_child_completion_not_established'); + } + self::assertBase($run, $preparation, $member, $resolved); + $terminal = ConfiguredV2Models::query('history_event_model', WorkflowHistoryEvent::class) + ->find($resolved->payload['child_terminal_history_event_id'] ?? null); + if ($terminal === null || $terminal->workflow_run_id !== $member['child_workflow_run_id'] + || ! in_array( + $terminal->event_type, + [HistoryEventType::WorkflowCompleted, HistoryEventType::WorkflowFailed, + HistoryEventType::WorkflowCancelled, HistoryEventType::WorkflowTerminated], + true + ) + || $terminal->event_type->value !== ($resolved->payload['child_terminal_event_type'] ?? null) + || ($resolved->payload['child_status'] ?? null) !== match ($terminal->event_type) { + HistoryEventType::WorkflowCompleted => RunStatus::Completed->value, + HistoryEventType::WorkflowCancelled => RunStatus::Cancelled->value, + HistoryEventType::WorkflowTerminated => RunStatus::Terminated->value, + default => RunStatus::Failed->value, + } + || ConfiguredV2Models::query('history_event_model', WorkflowHistoryEvent::class) + ->where('workflow_run_id', $member['child_workflow_run_id']) + ->whereIn('event_type', [HistoryEventType::WorkflowCompleted, HistoryEventType::WorkflowFailed, + HistoryEventType::WorkflowCancelled, HistoryEventType::WorkflowTerminated])->count() !== 1 + || ($resolved->payload['child_request_id'] ?? null) !== $receipt->payload['child_request_id'] + || ($resolved->payload['child_root_request_id'] ?? null) !== $receipt->payload['child_root_request_id'] + || ($resolved->payload['child_cleanup_deadline_at'] ?? null) !== $receipt->payload['child_cleanup_deadline_at']) { + throw new LogicException('cancellation_scope_child_completion_not_established'); + } + } + } + + /** + * @param array $member + */ + private static function assertRequest( + WorkflowRun $run, + ScopedCancellationPreparation $preparation, + array $member, + WorkflowHistoryEvent $receipt, + ): void { + self::assertBase($run, $preparation, $member, $receipt); + $payload = $receipt->payload; + $abandoned = $member['cancellation_policy'] === CancellationPolicy::Abandon->value; + if ($abandoned || ($payload['request_outcome'] ?? null) === 'already_terminal') { + if (($payload['request_outcome'] ?? null) !== ($abandoned ? 'abandoned' : 'already_terminal') + || $payload['child_request_id'] !== null || $payload['child_cancellation'] !== null + || $payload['child_root_request_id'] !== null || $payload['child_cleanup_deadline_at'] !== null) { + throw new LogicException('cancellation_scope_child_delivery_not_established'); + } + return; + } + if (($payload['request_outcome'] ?? null) !== 'accepted' || ! is_array( + $payload['child_cancellation'] ?? null + )) { + throw new LogicException('cancellation_scope_child_delivery_not_established'); + } + $context = self::context($payload['child_cancellation']); + $origin = $preparation->context; + $expected = CancellationContext::fromScopeContext( + $origin, + $context->requestId, + $member['child_workflow_instance_id'], + $member['child_workflow_run_id'], + $context->deadline() + ); + $child = ConfiguredV2Models::query('run_model', WorkflowRun::class)->find($member['child_workflow_run_id']); + $canonical = $child === null ? null : CooperativeCancellationDelivery::context($child); + $command = ConfiguredV2Models::query('command_model', WorkflowCommand::class)->find($context->requestId); + $requested = $child?->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationRequested)->get(); + if (self::canonical($context->toArray()) !== self::canonical($expected->toArray()) + || $context->deadline() + ->greaterThan(CarbonImmutable::parse($preparation->authorityDeadlineAt)) + || self::canonical($canonical?->toArray()) !== self::canonical($context->toArray()) + || $command === null || $command->status !== CommandStatus::Accepted + || $command->command_type !== CommandType::RequestCancellation + || $command->workflow_run_id !== $member['child_workflow_run_id'] + || $command->workflow_instance_id !== $member['child_workflow_instance_id'] + || self::canonical((new CommandResult($command))->cancellationContext()?->toArray()) !== self::canonical( + $context->toArray() + ) + || $context->requestId !== $payload['child_request_id'] + || $context->rootRequestId !== $payload['child_root_request_id'] + || $context->deadline() + ->toISOString() !== $payload['child_cleanup_deadline_at'] + || $requested?->count() !== 1 || $requested->sole() + ->workflow_command_id !== $context->requestId + || self::canonical(self::context( + $requested->sole() + ->payload['cancellation'] + )->toArray()) !== self::canonical($context->toArray())) { + throw new LogicException('cancellation_scope_child_delivery_not_established'); + } + } + + /** + * @param array $member + */ + private static function assertBase( + WorkflowRun $run, + ScopedCancellationPreparation $preparation, + array $member, + WorkflowHistoryEvent $receipt, + ): void { + $expected = self::base($run, $preparation, $member); + $payload = $receipt->payload; + if (! is_array($payload['cancellation_scope'] ?? null) + || ! is_array($payload['cancellation_scope']['cancellation'] ?? null)) { + throw new LogicException('cancellation_scope_child_delivery_not_established'); + } + try { + $payload['cancellation_scope']['cancellation'] = ScopedCancellationContext::fromArray( + $payload['cancellation_scope']['cancellation'] + )->toArray(); + } catch (InvalidArgumentException $error) { + throw new LogicException('cancellation_scope_child_delivery_not_established', previous: $error); + } + $payload['cancellation_scope']['member'] = ScopedChildCancellation::normalizeMembers([ + $payload['cancellation_scope']['member'] ?? null, + ])[0]; + foreach ($expected as $key => $value) { + if (self::canonical($payload[$key] ?? null) !== self::canonical($value) + || $receipt->sequence <= $preparation->historySequence) { + throw new LogicException('cancellation_scope_child_delivery_not_established'); + } + } + } + + /** @param array $member + * @return array + */ + private static function base(WorkflowRun $run, ScopedCancellationPreparation $preparation, array $member): array + { + $origin = $preparation->context; + return [ + 'sequence' => $member['sequence'], + 'policy' => $member['cancellation_policy'], + 'parent_request_id' => $origin->requestId, + 'root_request_id' => $origin->rootContext->rootRequestId, + 'cleanup_deadline_at' => $origin->deadline() + ->toISOString(), + 'child_workflow_instance_id' => $member['child_workflow_instance_id'], + 'child_workflow_run_id' => $member['child_workflow_run_id'], + 'cancellation_scope' => [ + 'schema' => self::SCHEMA, + 'workflow_run_id' => $run->id, + 'scope_id' => $origin->scopeId, + 'request_id' => $origin->requestId, + 'preparation_history_event_id' => $preparation->historyEventId, + 'authority_deadline_at' => $preparation->authorityDeadlineAt, + 'cancellation' => $origin->toArray(), + 'member' => $member, + ], + ]; + } + + private static function receipt( + WorkflowRun $run, + ScopedCancellationPreparation $preparation, + string $callId, + HistoryEventType $type, + ?int $beforeHistorySequence = null, + ): ?WorkflowHistoryEvent { + $run->loadMissing('historyEvents'); + $member = collect($preparation->childMembers) + ->firstWhere('child_call_id', $callId); + $matches = $run->historyEvents->filter(static fn (WorkflowHistoryEvent $row): bool => + $row->event_type === $type + && $row->workflow_command_id === null + && ($row->payload['sequence'] ?? null) === $member['sequence'] + && ($beforeHistorySequence === null || $row->sequence < $beforeHistorySequence)); + if ($matches->count() > 1) { + throw new LogicException('cancellation_scope_child_delivery_not_established'); + } + return $matches->first(); + } + + /** + * @param array $snapshot + */ + private static function context(array $snapshot): CancellationContext + { + try { + return CancellationContext::fromArray($snapshot); + } catch (InvalidArgumentException $error) { + throw new LogicException('cancellation_scope_child_delivery_not_established', previous: $error); + } + } + + private static function canonical(mixed $value): mixed + { + if (! is_array($value)) { + return $value; + } + if (! array_is_list($value)) { + ksort($value, SORT_STRING); + } + foreach ($value as $key => $entry) { + $value[$key] = self::canonical($entry); + } + return $value; + } +} diff --git a/src/V2/Support/ScopedTimerCancellation.php b/src/V2/Support/ScopedTimerCancellation.php new file mode 100644 index 000000000..b82930c2c --- /dev/null +++ b/src/V2/Support/ScopedTimerCancellation.php @@ -0,0 +1,374 @@ + + */ + public static function members(WorkflowRun $run, string $scopeId): array + { + $run->loadMissing('historyEvents'); + $members = []; + $ids = []; + $sequences = []; + foreach ($run->historyEvents->sortBy('sequence') as $event) { + if ($event->event_type !== HistoryEventType::TimerScheduled) { + continue; + } + $payload = $event->payload; + $descriptor = $payload['timer'] ?? []; + if (! is_array($descriptor)) { + throw new LogicException('cancellation_scope_timer_history_invalid'); + } + $nested = $descriptor['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID; + $address = $payload['cancellation_scope_id'] ?? $nested; + if ($address !== $scopeId && $nested !== $scopeId) { + continue; + } + if ($address !== $scopeId || (array_key_exists('cancellation_scope_id', $descriptor) + && $nested !== $scopeId)) { + throw new LogicException('cancellation_scope_delivery_membership_mismatch'); + } + $id = $payload['timer_id'] ?? null; + $sequence = $payload['sequence'] ?? null; + $delay = $payload['delay_seconds'] ?? null; + $fireAt = $payload['fire_at'] ?? null; + if (! is_string($id) || $id === '' || ! is_int($sequence) || $sequence < 1 + || ! is_int($delay) || $delay < 0 || ! is_string($fireAt) || $fireAt === '' + || isset($ids[$id]) || (array_key_exists($sequence, $sequences) + && (! in_array($payload['timer_kind'] ?? null, ['signal_timeout', 'condition_timeout'], true) + || $sequences[$sequence] !== $payload['timer_kind']))) { + throw new LogicException('cancellation_scope_timer_history_invalid'); + } + try { + if (CarbonImmutable::parse($fireAt)->toISOString() !== $fireAt) { + throw new LogicException('cancellation_scope_timer_history_invalid'); + } + } catch (\InvalidArgumentException $error) { + throw new LogicException('cancellation_scope_timer_history_invalid', previous: $error); + } + $ids[$id] = true; + $sequences[$sequence] = $payload['timer_kind'] ?? null; + if (ScopedCancellationCleanup::isScheduledTimer($run, $event)) { + continue; + } + $members[] = [ + 'sequence' => $sequence, + 'timer_id' => $id, + 'descriptor_hash' => hash('sha256', json_encode([ + $scopeId, $event->id, $sequence, $id, $delay, $fireAt, + $payload['timer_kind'] ?? null, $payload['condition_wait_id'] ?? null, + $payload['condition_wait_occurrence_id'] ?? null, $payload['signal_wait_id'] ?? null, + ParallelChildGroup::metadataPathFromPayload($payload), + ], JSON_THROW_ON_ERROR)), + ]; + } + return $members; + } + + /** + * @return list + */ + public static function normalizeMembers(mixed $members): array + { + if (! is_array($members) || ! array_is_list($members)) { + throw new LogicException('cancellation_scope_preparation_history_invalid'); + } + $normalized = []; + foreach ($members as $member) { + if (! is_array($member) || count($member) !== 3 || ! is_int($member['sequence'] ?? null) + || ! is_string($member['timer_id'] ?? null) || ! is_string($member['descriptor_hash'] ?? null)) { + throw new LogicException('cancellation_scope_preparation_history_invalid'); + } + $normalized[] = [ + 'sequence' => $member['sequence'], + 'timer_id' => $member['timer_id'], + 'descriptor_hash' => $member['descriptor_hash'], + ]; + } + return $normalized; + } + + /** + * @return array{fenced: bool, history_event_id: string|null, timer_id: string, sequence: int} + */ + public static function fence( + WorkflowRun $run, + WorkflowTask $workflowTask, + string $timerId, + string $scopeId, + string $requestId, + string $protocolVersion, + ?string $preparationHistoryEventId = null, + ): array { + if (! WorkerProtocolVersion::supportsCancellationScopeMembership($protocolVersion)) { + throw new LogicException('cancellation_scope_requires_protocol_1_20'); + } + if ($run->getConnection()->transactionLevel() !== 0) { + throw new LogicException('cancellation_scope_timer_requires_own_transaction'); + } + $preparation = ScopedCancellationPreparation::forScope($run->fresh(), $scopeId, $preparationHistoryEventId); + $wait = $preparation === null ? null : ScopedWaitCancellation::forTimer($preparation, $timerId); + if ($wait !== null) { + // Every entry point preserves the wait/timer atomic commit. + return ScopedWaitCancellation::fence( + $run, + $workflowTask, + $wait['wait_id'], + $scopeId, + $requestId, + $protocolVersion, + $preparationHistoryEventId, + )['timer_cancellation']; + } + $tasks = ConfiguredV2Models::query('task_model', WorkflowTask::class) + ->where('workflow_run_id', $run->id) + ->where('task_type', TaskType::Timer) + ->where('payload->timer_id', $timerId) + ->get(); + if ($tasks->count() !== 1) { + throw new LogicException('cancellation_scope_timer_task_mismatch'); + } + $timerTaskSnapshot = $tasks->sole(); + $timerTaskId = $timerTaskSnapshot->id; + return $run->getConnection() + ->transaction(static function () use ( + $run, + $workflowTask, + $timerId, + $scopeId, + $requestId, + $timerTaskId, + $preparationHistoryEventId, + ): array { + /** @var WorkflowTask|null $timerTask */ + $timerTask = ConfiguredV2Models::query('task_model', WorkflowTask::class)->lockForUpdate()->find( + $timerTaskId + ); + /** @var WorkflowRun $locked */ + $locked = ConfiguredV2Models::query('run_model', WorkflowRun::class)->lockForUpdate()->findOrFail( + $run->id + ); + if ($timerTask === null || $timerTask->workflow_run_id !== $locked->id + || $timerTask->namespace !== $locked->namespace || $timerTask->task_type !== TaskType::Timer + || ($timerTask->payload['timer_id'] ?? null) !== $timerId + || $locked->tasks() + ->where('task_type', TaskType::Timer)->where('payload->timer_id', $timerId)->count() !== 1) { + throw new LogicException('cancellation_scope_timer_task_mismatch'); + } + /** @var WorkflowTask|null $claim */ + $claim = ConfiguredV2Models::query('task_model', WorkflowTask::class)->lockForUpdate()->find( + $workflowTask->id + ); + if ($claim === null || $claim->workflow_run_id !== $locked->id || $claim->namespace !== $locked->namespace + || $claim->task_type !== TaskType::Workflow || $claim->status !== TaskStatus::Leased + || $claim->lease_owner === null || $claim->lease_owner === '' || $claim->attempt_count < 1 + || $claim->lease_owner !== $workflowTask->lease_owner || $claim->attempt_count !== $workflowTask->attempt_count + || $claim->lease_expires_at === null || now() + ->gte($claim->lease_expires_at)) { + throw new LogicException('cancellation_scope_workflow_claim_mismatch'); + } + $context = CancellationScopeRequests::context($locked, $scopeId); + if ($context === null || $context->requestId !== $requestId) { + throw new LogicException('cancellation_scope_request_mismatch'); + } + $authority = CancellationScopeRequests::authority($locked, $scopeId); + $preparation = ScopedCancellationPreparation::forScope($locked, $scopeId, $preparationHistoryEventId); + if ($preparation === null) { + throw new LogicException('cancellation_scope_delivery_not_prepared'); + } + ScopedCancellationReconciliation::assertDispatchable($locked, $preparation); + if (! $authority['active'] || $authority['deadline_at'] === null + || now() + ->gte(CarbonImmutable::parse($preparation->authorityDeadlineAt))) { + throw new LogicException('cancellation_scope_authority_expired'); + } + $member = collect($preparation->timerMembers) + ->firstWhere('timer_id', $timerId); + if (! is_array($member)) { + throw new LogicException('cancellation_scope_timer_not_prepared'); + } + $terminal = self::terminal($locked, $timerId, $member['sequence']); + /** @var WorkflowTimer|null $timer */ + $timer = ConfiguredV2Models::query('timer_model', WorkflowTimer::class)->lockForUpdate()->find( + $timerId + ); + $timer ??= TimerRecovery::restore($locked, $timerId); + if ($timer === null || $timer->workflow_run_id !== $locked->id || $timer->sequence !== $member['sequence']) { + throw new LogicException('cancellation_scope_timer_projection_mismatch'); + } + // The final row lock may have waited beyond either authority. + // Holding the rows does not extend the original claim or budget. + if (now()->gte($claim->lease_expires_at)) { + throw new LogicException('cancellation_scope_workflow_claim_mismatch'); + } + if (now()->gte(CarbonImmutable::parse($authority['deadline_at'])) + || now() + ->gte(CarbonImmutable::parse($preparation->authorityDeadlineAt))) { + throw new LogicException('cancellation_scope_authority_expired'); + } + if ($terminal?->event_type === HistoryEventType::TimerFired) { + return [ + 'fenced' => false, + 'history_event_id' => null, + 'timer_id' => $timerId, + 'sequence' => $member['sequence'], + ]; + } + if ($terminal !== null) { + self::assertReceipt($locked, $preparation, $terminal); + } else { + if ($timer->status !== TimerStatus::Pending + || ! in_array($timerTask->status, [TaskStatus::Ready, TaskStatus::Leased], true)) { + throw new LogicException('cancellation_scope_timer_terminal_history_not_recorded'); + } + $request = $locked->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeRequested) + ->where('payload->scope_id', $scopeId) + ->sole(); + $terminal = TimerCancellation::record($locked, $timer, $claim, cancellationScope: [ + 'schema' => self::SCHEMA, + 'workflow_run_id' => $locked->id, + 'scope_id' => $scopeId, + 'request_id' => $requestId, + 'request_history_event_id' => $request->id, + 'preparation_history_event_id' => $preparation->historyEventId, + 'cancellation' => $context->toArray(), + 'authority_deadline_at' => $preparation->authorityDeadlineAt, + ]); + } + $timer->forceFill([ + 'status' => TimerStatus::Cancelled, + 'fired_at' => null, + ])->save(); + $timerTask->forceFill([ + 'status' => TaskStatus::Cancelled, + 'lease_owner' => null, + 'lease_expires_at' => null, + ])->save(); + return [ + 'fenced' => true, + 'history_event_id' => $terminal->id, + 'timer_id' => $timerId, + 'sequence' => $member['sequence'], + ]; + }, 5); + } + + public static function assertReady( + WorkflowRun $run, + WorkflowHistoryEvent|ScopedCancellationPreparation $preparation, + ?int $beforeHistorySequence = null, + ): void { + $preparation = $preparation instanceof WorkflowHistoryEvent + ? ScopedCancellationPreparation::fromEvent($run, $preparation) : $preparation; + $run->loadMissing('historyEvents'); + $history = $run->historyEvents; + if ($beforeHistorySequence !== null) { + $run->setRelation('historyEvents', $history->filter( + static fn (WorkflowHistoryEvent $event): bool => $event->sequence < $beforeHistorySequence + )); + } + try { + foreach ($preparation->timerMembers as $member) { + $terminal = self::terminal($run, $member['timer_id'], $member['sequence']); + if ($terminal === null) { + throw new LogicException('cancellation_scope_timer_fence_not_established'); + } + if ($terminal->event_type === HistoryEventType::TimerFired) { + continue; + } + if (isset($terminal->payload['cancellation_scope']) + || ! ScopedWaitCancellation::hasTimerWaitProof($run, $preparation, $member['timer_id'])) { + self::assertReceipt($run, $preparation, $terminal); + } + if ($beforeHistorySequence === null) { + $timer = ConfiguredV2Models::query('timer_model', WorkflowTimer::class)->find($member['timer_id']); + $tasks = $run->tasks() + ->where('task_type', TaskType::Timer) + ->where('payload->timer_id', $member['timer_id'])->get(); + $timerTask = $tasks->count() === 1 ? $tasks->first() : null; + if ($timer === null || $timer->workflow_run_id !== $run->id + || $timer->sequence !== $member['sequence'] || $timer->status !== TimerStatus::Cancelled + || $timerTask === null || $timerTask->namespace !== $run->namespace + || $timerTask->status !== TaskStatus::Cancelled || $timerTask->lease_expires_at !== null) { + throw new LogicException('cancellation_scope_timer_fence_not_established'); + } + } + } + } finally { + $run->setRelation('historyEvents', $history); + } + } + + public static function terminal(WorkflowRun $run, string $timerId, int $sequence): ?WorkflowHistoryEvent + { + $run->loadMissing('historyEvents'); + $scheduled = $run->historyEvents->filter(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::TimerScheduled && ($event->payload['timer_id'] ?? null) === $timerId); + $terminals = $run->historyEvents->filter(static fn (WorkflowHistoryEvent $event): bool => + in_array($event->event_type, [HistoryEventType::TimerCancelled, HistoryEventType::TimerFired], true) + && ($event->payload['timer_id'] ?? null) === $timerId); + if ($scheduled->count() !== 1 || ($scheduled->sole()->payload['sequence'] ?? null) !== $sequence + || $terminals->count() > 1) { + throw new LogicException('cancellation_scope_timer_history_invalid'); + } + $scheduledEvent = $scheduled->sole(); + $terminal = $terminals->first(); + if ($terminal !== null && (($terminal->payload['sequence'] ?? null) !== $sequence + || $terminal->sequence <= $scheduledEvent->sequence + || ($terminal->payload['delay_seconds'] ?? null) !== $scheduledEvent->payload['delay_seconds'] + || ($terminal->payload['fire_at'] ?? null) !== $scheduledEvent->payload['fire_at'])) { + throw new LogicException('cancellation_scope_timer_history_invalid'); + } + return $terminal; + } + + private static function assertReceipt( + WorkflowRun $run, + ScopedCancellationPreparation $preparation, + WorkflowHistoryEvent $terminal, + ): void { + $snapshot = $terminal->payload['cancellation_scope'] ?? null; + $request = $run->historyEvents->first(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::CancellationScopeRequested + && ($event->payload['scope_id'] ?? null) === $preparation->scopeId); + if (! is_array($snapshot) || ($snapshot['schema'] ?? null) !== self::SCHEMA + || ($snapshot['workflow_run_id'] ?? null) !== $run->id + || ($snapshot['scope_id'] ?? null) !== $preparation->scopeId + || ($snapshot['request_id'] ?? null) !== $preparation->requestId + || ($snapshot['request_history_event_id'] ?? null) !== $request?->id + || ($snapshot['preparation_history_event_id'] ?? null) !== $preparation->historyEventId + || ($snapshot['authority_deadline_at'] ?? null) !== $preparation->authorityDeadlineAt + || ! is_array($snapshot['cancellation'] ?? null) || $terminal->sequence <= $preparation->historySequence) { + throw new LogicException('cancellation_scope_timer_fence_not_established'); + } + try { + if (ScopedCancellationContext::fromArray($snapshot['cancellation'])->toArray() + !== $preparation->context->toArray()) { + throw new LogicException('cancellation_scope_timer_fence_not_established'); + } + } catch (\InvalidArgumentException $error) { + throw new LogicException('cancellation_scope_timer_fence_not_established', previous: $error); + } + } +} diff --git a/src/V2/Support/ScopedWaitCancellation.php b/src/V2/Support/ScopedWaitCancellation.php new file mode 100644 index 000000000..14ccfcd3a --- /dev/null +++ b/src/V2/Support/ScopedWaitCancellation.php @@ -0,0 +1,503 @@ + + */ + public static function members(WorkflowRun $run, string $scopeId): array + { + $run->loadMissing('historyEvents'); + $members = []; + $ids = []; + foreach ($run->historyEvents->sortBy('sequence') as $event) { + $kind = match ($event->event_type) { + HistoryEventType::SignalWaitOpened => 'signal', + HistoryEventType::ConditionWaitOpened => 'condition', + default => null, + }; + if ($kind === null || ($event->payload['cancellation_scope_id'] ?? 'root') !== $scopeId) { + continue; + } + $payload = $event->payload; + $id = $payload[$kind . '_wait_id'] ?? null; + $sequence = $payload['sequence'] ?? null; + if (! is_string($id) || $id === '' || ! is_int($sequence) || $sequence < 1 || isset($ids[$id])) { + throw new LogicException('cancellation_scope_wait_history_invalid'); + } + $timers = $run->historyEvents->filter(static fn (WorkflowHistoryEvent $row): bool => + $row->event_type === HistoryEventType::TimerScheduled + && ($row->payload[$kind . '_wait_id'] ?? null) === $id); + if ($timers->count() > 1) { + throw new LogicException('cancellation_scope_wait_history_invalid'); + } + $timer = $timers->first(); + if ($timer !== null && (($timer->payload['timer_kind'] ?? null) !== $kind . '_timeout' + || ($timer->payload['sequence'] ?? null) !== $sequence || $timer->sequence <= $event->sequence + || ($timer->payload['cancellation_scope_id'] ?? 'root') !== $scopeId + || ! is_string($timer->payload['timer_id'] ?? null) || $timer->payload['timer_id'] === '')) { + throw new LogicException('cancellation_scope_wait_history_invalid'); + } + $ids[$id] = true; + $members[] = [ + 'kind' => $kind, + 'sequence' => $sequence, + 'wait_id' => $id, + 'timer_id' => $timer?->payload['timer_id'], + 'descriptor_hash' => hash('sha256', json_encode([ + $scopeId, $event->id, $kind, $sequence, $id, $timer?->id, $timer?->payload['timer_id'], + $payload['timeout_seconds'] ?? null, $payload['signal_name'] ?? null, + $payload['condition_wait_occurrence_id'] ?? null, $payload['condition_key'] ?? null, + $payload['condition_definition_fingerprint'] ?? null, + ParallelChildGroup::metadataPathFromPayload($payload), + ], JSON_THROW_ON_ERROR)), + ]; + } + return $members; + } + + /** + * @return list + */ + public static function normalizeMembers(mixed $members): array + { + if (! is_array($members) || ! array_is_list($members)) { + throw new LogicException('cancellation_scope_preparation_history_invalid'); + } + $normalized = []; + foreach ($members as $member) { + if (! is_array($member) || count($member) !== 5 || ! in_array( + $member['kind'] ?? null, + ['signal', 'condition'], + true + ) + || ! is_int($member['sequence'] ?? null) || $member['sequence'] < 1 + || ! is_string($member['wait_id'] ?? null) || $member['wait_id'] === '' + || ! array_key_exists('timer_id', $member) + || ($member['timer_id'] !== null && (! is_string($member['timer_id']) || $member['timer_id'] === '')) + || ! is_string($member['descriptor_hash'] ?? null) + || preg_match('/^[a-f0-9]{64}$/D', $member['descriptor_hash']) !== 1) { + throw new LogicException('cancellation_scope_preparation_history_invalid'); + } + $normalized[] = [ + 'kind' => $member['kind'], + 'sequence' => $member['sequence'], + 'wait_id' => $member['wait_id'], + 'timer_id' => $member['timer_id'], + 'descriptor_hash' => $member['descriptor_hash'], + ]; + } + return $normalized; + } + + /** + * @return array|null + */ + public static function forTimer( + WorkflowHistoryEvent|ScopedCancellationPreparation $preparation, + string $timerId + ): ?array { + $members = $preparation instanceof ScopedCancellationPreparation + ? $preparation->waitMembers : self::normalizeMembers($preparation->payload['wait_members']); + return collect($members)->firstWhere('timer_id', $timerId); + } + + /** + * @return array + */ + public static function fence( + WorkflowRun $run, + WorkflowTask $workflowTask, + string $waitId, + string $scopeId, + string $requestId, + string $protocolVersion, + ?string $preparationHistoryEventId = null, + ): array { + if (! WorkerProtocolVersion::supportsCancellationScopeMembership($protocolVersion)) { + throw new LogicException('cancellation_scope_requires_protocol_1_20'); + } + if ($run->getConnection()->transactionLevel() !== 0) { + throw new LogicException('cancellation_scope_wait_requires_own_transaction'); + } + $original = ScopedCancellationPreparation::forScope($run->fresh(), $scopeId, $preparationHistoryEventId); + $member = $original === null ? null : collect($original->waitMembers) + ->firstWhere('wait_id', $waitId); + if (! is_array($member)) { + throw new LogicException('cancellation_scope_wait_not_prepared'); + } + $timerId = $member['timer_id']; + $timerTasks = $timerId === null ? collect() : $run->tasks() + ->where('task_type', TaskType::Timer) + ->where('payload->timer_id', $timerId) + ->get(); + if ($timerTasks->count() > 1) { + throw new LogicException('cancellation_scope_timer_task_mismatch'); + } + $timerTaskId = $timerTasks->first()?->id; + return $run->getConnection() + ->transaction(static function () use ( + $run, + $workflowTask, + $waitId, + $scopeId, + $requestId, + $timerId, + $timerTaskId, + $preparationHistoryEventId, + ): array { + // Match RunTimerTask's task/run lock prefix before the hosting claim. + $timerTask = $timerTaskId === null ? null + : ConfiguredV2Models::query('task_model', WorkflowTask::class)->lockForUpdate()->find($timerTaskId); + /** @var WorkflowRun $locked */ + $locked = ConfiguredV2Models::query('run_model', WorkflowRun::class)->lockForUpdate()->findOrFail( + $run->id + ); + /** @var WorkflowTask|null $claim */ + $claim = ConfiguredV2Models::query('task_model', WorkflowTask::class)->lockForUpdate()->find( + $workflowTask->id + ); + if ($claim === null || $claim->workflow_run_id !== $locked->id || $claim->namespace !== $locked->namespace + || $claim->task_type !== TaskType::Workflow || $claim->status !== TaskStatus::Leased + || $claim->lease_owner === null || $claim->lease_owner === '' || $claim->attempt_count < 1 + || $claim->lease_owner !== $workflowTask->lease_owner || $claim->attempt_count !== $workflowTask->attempt_count + || $claim->lease_expires_at === null || now() + ->gte($claim->lease_expires_at)) { + throw new LogicException('cancellation_scope_workflow_claim_mismatch'); + } + $context = CancellationScopeRequests::context($locked, $scopeId); + $preparation = ScopedCancellationPreparation::forScope($locked, $scopeId, $preparationHistoryEventId); + $authority = CancellationScopeRequests::authority($locked, $scopeId); + if ($context === null || $context->requestId !== $requestId) { + throw new LogicException('cancellation_scope_request_mismatch'); + } + $member = $preparation === null ? null : collect($preparation->waitMembers) + ->firstWhere('wait_id', $waitId); + if (! is_array($member) || $member['timer_id'] !== $timerId) { + throw new LogicException('cancellation_scope_wait_not_prepared'); + } + ScopedCancellationReconciliation::assertDispatchable($locked, $preparation); + $timer = $timerId === null ? null + : ConfiguredV2Models::query('timer_model', WorkflowTimer::class)->lockForUpdate()->find($timerId); + if ($timerId !== null) { + $timer ??= TimerRecovery::restore($locked, $timerId); + if ($timer === null || $timer->workflow_run_id !== $locked->id || $timer->sequence !== $member['sequence']) { + throw new LogicException('cancellation_scope_timer_projection_mismatch'); + } + } + // Waiting on the last row never renews either original authority. + if (now()->gte($claim->lease_expires_at)) { + throw new LogicException('cancellation_scope_workflow_claim_mismatch'); + } + if (! $authority['active'] || $authority['deadline_at'] === null + || now() + ->gte(CarbonImmutable::parse($authority['deadline_at'])) + || now() + ->gte(CarbonImmutable::parse($preparation->authorityDeadlineAt))) { + throw new LogicException('cancellation_scope_authority_expired'); + } + $timerTerminal = $timerId === null ? null : ScopedTimerCancellation::terminal( + $locked, + $timerId, + $member['sequence'] + ); + if ($timerId !== null && ($timerTask === null + ? $timerTerminal?->event_type !== HistoryEventType::TimerFired + : ($timerTask->workflow_run_id !== $locked->id || $timerTask->namespace !== $locked->namespace + || $timerTask->task_type !== TaskType::Timer || ($timerTask->payload['timer_id'] ?? null) !== $timerId + || $locked->tasks() + ->where('task_type', TaskType::Timer)->where( + 'payload->timer_id', + $timerId + )->count() !== 1))) { + throw new LogicException('cancellation_scope_timer_task_mismatch'); + } + $terminal = self::terminal($locked, $member); + if ($terminal !== null && self::isCancelled($terminal)) { + self::assertReceipt($locked, $preparation, $terminal, $member); + } elseif ($terminal === null) { + $opened = $locked->historyEvents->first(static fn (WorkflowHistoryEvent $event): bool => + ($event->payload[$member['kind'] . '_wait_id'] ?? null) === $waitId + && $event->event_type === ($member['kind'] === 'signal' + ? HistoryEventType::SignalWaitOpened : HistoryEventType::ConditionWaitOpened)); + $request = $locked->historyEvents->first(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::CancellationScopeRequested + && ($event->payload['scope_id'] ?? null) === $scopeId); + $cancelledEventType = $member['kind'] === 'signal' + ? HistoryEventType::SignalWaitCancelled : HistoryEventType::ConditionWaitCancelled; + $terminal = WorkflowHistoryEvent::record($locked, $cancelledEventType, array_filter( + [ + ...array_intersect_key($opened->payload, array_flip([ + 'signal_name', 'signal_wait_id', 'condition_wait_id', 'condition_wait_occurrence_id', + 'condition_key', 'condition_definition_fingerprint', 'sequence', 'timeout_seconds', + ])), + 'timer_id' => $timerId, + 'cancelled_at' => now() + ->toISOString(), + 'cancellation_scope' => [ + 'schema' => self::SCHEMA, + 'workflow_run_id' => $locked->id, + 'scope_id' => $scopeId, + 'request_id' => $requestId, + 'request_history_event_id' => $request->id, + 'preparation_history_event_id' => $preparation->historyEventId, + 'cancellation' => $context->toArray(), + 'authority_deadline_at' => $preparation->authorityDeadlineAt, + ], + ...ParallelChildGroup::payloadForPath( + ParallelChildGroup::metadataPathFromPayload($opened->payload) + ), + ], + static fn (mixed $value): bool => $value !== null + ), $claim); + $locked->historyEvents->push($terminal); + } + $timerReceipt = null; + if ($timer !== null) { + if ($timerTerminal === null) { + if ($timer->status !== TimerStatus::Pending + || ! in_array($timerTask->status, [TaskStatus::Ready, TaskStatus::Leased], true)) { + throw new LogicException('cancellation_scope_timer_terminal_history_not_recorded'); + } + $request = $locked->historyEvents->first(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::CancellationScopeRequested + && ($event->payload['scope_id'] ?? null) === $scopeId); + $timerTerminal = TimerCancellation::record($locked, $timer, $claim, cancellationScope: [ + 'schema' => ScopedTimerCancellation::SCHEMA, + 'workflow_run_id' => $locked->id, + 'scope_id' => $scopeId, + 'request_id' => $requestId, + 'request_history_event_id' => $request->id, + 'preparation_history_event_id' => $preparation->historyEventId, + 'cancellation' => $context->toArray(), + 'authority_deadline_at' => $preparation->authorityDeadlineAt, + ]); + } + $cancelled = $timerTerminal->event_type === HistoryEventType::TimerCancelled; + if ($cancelled) { + $timer->forceFill([ + 'status' => TimerStatus::Cancelled, + 'fired_at' => null, + ])->save(); + $timerTask->forceFill([ + 'status' => TaskStatus::Cancelled, + 'lease_owner' => null, + 'lease_expires_at' => null, + ])->save(); + } + $timerReceipt = [ + 'fenced' => $cancelled, + 'history_event_id' => $cancelled ? $timerTerminal->id : null, + 'timer_id' => $timerId, + 'sequence' => $member['sequence'], + ]; + } + return [ + 'kind' => $member['kind'], + 'wait_id' => $waitId, + 'sequence' => $member['sequence'], + 'cancelled' => self::isCancelled($terminal), + 'history_event_id' => $terminal->id, + 'timer_cancellation' => $timerReceipt, + ]; + }, 5); + } + + public static function assertReady( + WorkflowRun $run, + WorkflowHistoryEvent|ScopedCancellationPreparation $preparation, + ?int $beforeHistorySequence = null + ): void { + $preparation = $preparation instanceof WorkflowHistoryEvent + ? ScopedCancellationPreparation::fromEvent($run, $preparation) : $preparation; + $run->loadMissing('historyEvents'); + $history = $run->historyEvents; + if ($beforeHistorySequence !== null) { + $run->setRelation('historyEvents', $history->filter(static fn (WorkflowHistoryEvent $event): bool => + $event->sequence < $beforeHistorySequence)); + } + try { + foreach ($preparation->waitMembers as $member) { + $terminal = self::terminal($run, $member); + if ($terminal === null) { + throw new LogicException('cancellation_scope_wait_fence_not_established'); + } + if (self::isCancelled($terminal)) { + self::assertReceipt($run, $preparation, $terminal, $member); + } + } + } finally { + $run->setRelation('historyEvents', $history); + } + } + + public static function hasTimerWaitProof( + WorkflowRun $run, + WorkflowHistoryEvent|ScopedCancellationPreparation $preparation, + string $timerId, + ): bool { + $preparation = $preparation instanceof WorkflowHistoryEvent + ? ScopedCancellationPreparation::fromEvent($run, $preparation) : $preparation; + $member = self::forTimer($preparation, $timerId); + if ($member === null) { + return false; + } + $terminal = self::terminal($run, $member); + if ($terminal === null) { + return false; + } + if (self::isCancelled($terminal)) { + self::assertReceipt($run, $preparation, $terminal, $member); + } + return true; + } + + public static function cancelled(WorkflowRun $run, string $kind, string $waitId): bool + { + $run->loadMissing('historyEvents'); + return $run->historyEvents->contains(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === ($kind === 'signal' ? HistoryEventType::SignalWaitCancelled : HistoryEventType::ConditionWaitCancelled) + && ($event->payload[$kind . '_wait_id'] ?? null) === $waitId); + } + + /** + * @param array $member + */ + public static function terminal(WorkflowRun $run, array $member): ?WorkflowHistoryEvent + { + $run->loadMissing('historyEvents'); + $cancelled = $run->historyEvents->filter(static fn (WorkflowHistoryEvent $event): bool => + self::isCancelled( + $event + ) && ($event->payload[$member['kind'] . '_wait_id'] ?? null) === $member['wait_id']); + if ($cancelled->count() > 1) { + throw new LogicException('cancellation_scope_wait_history_invalid'); + } + if ($cancelled->isNotEmpty() && $run->historyEvents->contains(static fn (WorkflowHistoryEvent $event): bool => + (($event->payload[$member['kind'] . '_wait_id'] ?? null) === $member['wait_id'] + && in_array($event->event_type, [HistoryEventType::SignalReceived, HistoryEventType::SignalApplied, + HistoryEventType::ConditionWaitSatisfied, HistoryEventType::ConditionWaitTimedOut], true)) + || ($member['timer_id'] !== null && ($event->payload['timer_id'] ?? null) === $member['timer_id'] + && $event->event_type === HistoryEventType::TimerFired))) { + throw new LogicException('cancellation_scope_wait_history_invalid'); + } + foreach ($run->historyEvents->sortBy('sequence') as $event) { + $id = $event->payload[$member['kind'] . '_wait_id'] ?? null; + if ($id === $member['wait_id'] && in_array($event->event_type, $member['kind'] === 'signal' + ? [ + HistoryEventType::SignalReceived, + HistoryEventType::SignalApplied, + HistoryEventType::SignalWaitCancelled, + ] + : [ + HistoryEventType::ConditionWaitSatisfied, + HistoryEventType::ConditionWaitTimedOut, + HistoryEventType::ConditionWaitCancelled, + ], true)) { + return $event; + } + if ($member['timer_id'] !== null && ($event->payload['timer_id'] ?? null) === $member['timer_id'] + && ($event->event_type === HistoryEventType::TimerFired + || ($member['kind'] === 'signal' && $event->event_type === HistoryEventType::TimerCancelled))) { + return $event; + } + if ($event->event_type === HistoryEventType::SelectionOperationCancelled + && is_int($event->payload['member_base_sequence'] ?? null) && is_int( + $event->payload['member_size'] ?? null + ) + && $member['sequence'] >= $event->payload['member_base_sequence'] + && $event->payload['member_size'] > $member['sequence'] - $event->payload['member_base_sequence']) { + return $event; + } + } + return null; + } + + private static function isCancelled(WorkflowHistoryEvent $event): bool + { + return in_array( + $event->event_type, + [HistoryEventType::SignalWaitCancelled, HistoryEventType::ConditionWaitCancelled], + true + ); + } + + /** + * @param array $member + */ + private static function assertReceipt( + WorkflowRun $run, + ScopedCancellationPreparation $preparation, + WorkflowHistoryEvent $terminal, + array $member + ): void { + $snapshot = $terminal->payload['cancellation_scope'] ?? null; + $request = $run->historyEvents->first(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === HistoryEventType::CancellationScopeRequested && ($event->payload['scope_id'] ?? null) === $preparation->scopeId); + if (! is_array($snapshot) || ($snapshot['schema'] ?? null) !== self::SCHEMA + || ($snapshot['workflow_run_id'] ?? null) !== $run->id || ($snapshot['scope_id'] ?? null) !== $preparation->scopeId + || ($snapshot['request_id'] ?? null) !== $preparation->requestId + || ($snapshot['request_history_event_id'] ?? null) !== $request?->id + || ($snapshot['preparation_history_event_id'] ?? null) !== $preparation->historyEventId + || ($snapshot['authority_deadline_at'] ?? null) !== $preparation->authorityDeadlineAt + || ($terminal->payload['sequence'] ?? null) !== $member['sequence'] + || ($terminal->payload['timer_id'] ?? null) !== $member['timer_id'] || $terminal->sequence <= $preparation->historySequence + || ! is_array($snapshot['cancellation'] ?? null)) { + throw new LogicException('cancellation_scope_wait_fence_not_established'); + } + $opened = $run->historyEvents->first(static fn (WorkflowHistoryEvent $event): bool => + $event->event_type === ($member['kind'] === 'signal' ? HistoryEventType::SignalWaitOpened : HistoryEventType::ConditionWaitOpened) + && ($event->payload[$member['kind'] . '_wait_id'] ?? null) === $member['wait_id']); + foreach ([ + 'signal_name', + 'condition_wait_occurrence_id', + 'condition_key', + 'condition_definition_fingerprint', + 'timeout_seconds', + ] as $field) { + if (($terminal->payload[$field] ?? null) !== ($opened?->payload[$field] ?? null)) { + throw new LogicException('cancellation_scope_wait_fence_not_established'); + } + } + if (ParallelChildGroup::metadataPathFromPayload( + $terminal->payload + ) !== ParallelChildGroup::metadataPathFromPayload($opened->payload)) { + throw new LogicException('cancellation_scope_wait_fence_not_established'); + } + try { + $cancelledAt = $terminal->payload['cancelled_at'] ?? null; + if (! is_string($cancelledAt) || CarbonImmutable::parse($cancelledAt)->toISOString() !== $cancelledAt + || CarbonImmutable::parse($cancelledAt)->lt($preparation->recordedAt) + || CarbonImmutable::parse($cancelledAt)->gte( + CarbonImmutable::parse($preparation->authorityDeadlineAt) + )) { + throw new LogicException('cancellation_scope_wait_fence_not_established'); + } + if (ScopedCancellationContext::fromArray($snapshot['cancellation'])->toArray() + !== $preparation->context->toArray()) { + throw new LogicException('cancellation_scope_wait_fence_not_established'); + } + } catch (\InvalidArgumentException $error) { + throw new LogicException('cancellation_scope_wait_fence_not_established', previous: $error); + } + } +} diff --git a/src/V2/Support/SignalWaits.php b/src/V2/Support/SignalWaits.php index b883d26cf..d1e3e9d19 100644 --- a/src/V2/Support/SignalWaits.php +++ b/src/V2/Support/SignalWaits.php @@ -141,6 +141,17 @@ public static function forRun(WorkflowRun $run): array continue; } + if ($event->event_type === HistoryEventType::SignalWaitCancelled) { + $waitId = self::stringValue($event->payload['signal_wait_id'] ?? null); + if ($waitId !== null && isset($waits[$waitId]) && $waits[$waitId]['status'] === 'open') { + $waits[$waitId]['status'] = 'cancelled'; + $waits[$waitId]['source_status'] = 'scope_cancelled'; + $waits[$waitId]['resolved_at'] = $event->recorded_at ?? $event->created_at; + self::consumeOpenWaitId($openWaitIdsByName, $waits[$waitId]['signal_name'], $waitId); + } + continue; + } + if ($event->event_type === HistoryEventType::SelectionOperationCancelled) { self::closeCancelledSelectionWaits($waits, $openWaitIdsByName, $event); diff --git a/src/V2/Support/SurfaceStabilityContract.php b/src/V2/Support/SurfaceStabilityContract.php index 83060f646..431bf8485 100644 --- a/src/V2/Support/SurfaceStabilityContract.php +++ b/src/V2/Support/SurfaceStabilityContract.php @@ -28,7 +28,7 @@ final class SurfaceStabilityContract { public const SCHEMA = 'durable-workflow.v2.surface-stability.contract'; - public const VERSION = 4; + public const VERSION = 5; public const AUTHORITY_URL = 'https://durable-workflow.github.io/docs/2.0/compatibility'; diff --git a/src/V2/Support/TaskRepair.php b/src/V2/Support/TaskRepair.php index 52d2ac819..c4da6534e 100644 --- a/src/V2/Support/TaskRepair.php +++ b/src/V2/Support/TaskRepair.php @@ -35,16 +35,23 @@ public static function repairRun(WorkflowRun $run, WorkflowRunSummary $summary): public static function recoverExistingTask(WorkflowTask $task, WorkflowRun $run): ?WorkflowTask { + $execution = $run->status->isTerminal() && $task->task_type === TaskType::Activity + ? self::activityExecutionForTask($task) : null; + $abandoned = $execution instanceof ActivityExecution && ActivityAbandonment::allows($run, $execution); if (in_array($run->status, [ RunStatus::Completed, RunStatus::Failed, RunStatus::Cancelled, RunStatus::Terminated, - ], true)) { + ], true) && ! $abandoned) { self::settleTerminalTask($task, $run); return null; } + if ($abandoned && ActivityTimeoutEnforcer::hasExpiredDeadline($execution, now())) { + // Timeout enforcement owns the canonical terminal outcome. + return null; + } if ($task->status === TaskStatus::Ready) { if (! TaskRepairPolicy::readyTaskNeedsRedispatch($task)) { diff --git a/src/V2/Support/TimerCancellation.php b/src/V2/Support/TimerCancellation.php index f6c9723ca..dcd44a52d 100644 --- a/src/V2/Support/TimerCancellation.php +++ b/src/V2/Support/TimerCancellation.php @@ -14,12 +14,16 @@ final class TimerCancellation { + /** + * @param array|null $cancellationScope + */ public static function record( WorkflowRun $run, WorkflowTimer $timer, WorkflowTask|string|null $task = null, WorkflowCommand|string|null $command = null, ?CarbonInterface $cancelledAt = null, + ?array $cancellationScope = null, ): WorkflowHistoryEvent { $run->loadMissing('historyEvents'); @@ -53,6 +57,7 @@ public static function record( 'signal_wait_id' => self::stringValue($scheduledPayload['signal_wait_id'] ?? null), 'signal_name' => self::stringValue($scheduledPayload['signal_name'] ?? null), 'cancelled_at' => $cancelledAt->toJSON(), + 'cancellation_scope' => $cancellationScope, ...ParallelChildGroup::payloadForPath(ParallelChildGroup::metadataPathFromPayload($scheduledPayload)), ], static fn (mixed $value): bool => $value !== null), $task, $command); diff --git a/src/V2/Support/WorkerProtocolVersion.php b/src/V2/Support/WorkerProtocolVersion.php index 185880e82..a42706222 100644 --- a/src/V2/Support/WorkerProtocolVersion.php +++ b/src/V2/Support/WorkerProtocolVersion.php @@ -30,7 +30,7 @@ final class WorkerProtocolVersion * pagination semantics). Bump the minor for additive changes (new * optional fields, new non-terminal command types). */ - public const VERSION = '1.19'; + public const VERSION = '1.20'; /** * Worker registration capability for server-routed workflow query @@ -287,6 +287,21 @@ public static function workerCapabilitiesForVersion(string $protocolVersion): ar return $capabilities; } + public static function supportsChildCancellationPolicies(string $protocolVersion): bool + { + return self::supportsFeatureVersion($protocolVersion, '1.20'); + } + + public static function supportsActivityCancellationPolicies(string $protocolVersion): bool + { + return self::supportsFeatureVersion($protocolVersion, '1.20'); + } + + public static function supportsCancellationScopeMembership(string $protocolVersion): bool + { + return self::supportsFeatureVersion($protocolVersion, CancellationScopeHistory::MINIMUM_PROTOCOL_VERSION); + } + public static function supportsMessageStreams(string $protocolVersion): bool { return self::supportsFeatureVersion($protocolVersion, self::MESSAGE_STREAMS_MINIMUM_PROTOCOL_VERSION); diff --git a/src/V2/Support/WorkflowCommandNormalizer.php b/src/V2/Support/WorkflowCommandNormalizer.php index 0a6a7acc1..218cdc6e0 100644 --- a/src/V2/Support/WorkflowCommandNormalizer.php +++ b/src/V2/Support/WorkflowCommandNormalizer.php @@ -146,6 +146,24 @@ final class WorkflowCommandNormalizer 'allowed' => ['start_child_workflow'], 'guidance' => 'parent_close_policy declares how a child workflow reacts when its parent closes and only applies to a start_child_workflow command.', ], + 'cancellation_policy' => [ + 'allowed' => ['start_child_workflow', 'schedule_activity'], + 'guidance' => 'cancellation_policy declares how an awaiting workflow handles child or remote activity cancellation. Explicit activity policies require protocol 1.20 and remote abandon requires a finite schedule_to_close_timeout.', + ], + 'cancellation_scope_id' => [ + 'allowed' => [ + 'schedule_activity', + 'start_timer', + 'start_child_workflow', + 'open_signal_wait', + 'open_condition_wait', + ], + 'guidance' => 'cancellation_scope_id records activity, timer, child or wait membership and requires candidate protocol 1.20.', + ], + 'cancellation_cleanup' => [ + 'allowed' => ['start_timer'], + 'guidance' => 'Shielded scoped timers require the original scope, request and delivery identities under candidate protocol 1.20.', + ], 'delay_seconds' => [ 'allowed' => ['start_timer'], 'guidance' => 'delay_seconds is the timer delay and only applies to a start_timer command.', @@ -396,6 +414,7 @@ public static function normalize(array $commands, ?string $protocolVersion = nul self::assertCommandFieldScope($type, is_array($command) ? $command : [], $index, $errors); self::assertParallelMetadataScope($type, is_array($command) ? $command : [], $index, $errors); + $scopeMetadata = self::optionalOperationScopeMetadata($command, $index, $errors, $protocolVersion); if ($type === 'cancel_selection_operation') { $groupId = $command['selection_group_id'] ?? null; @@ -515,10 +534,33 @@ public static function normalize(array $commands, ?string $protocolVersion = nul $scheduleToClose = self::optionalPositiveInt($command, 'schedule_to_close_timeout', $index, $errors); $heartbeat = self::optionalPositiveInt($command, 'heartbeat_timeout', $index, $errors); $workerSession = self::optionalWorkerSession($command, $index, $errors); + $cancellationPolicy = self::optionalCommandString($command, 'cancellation_policy', $index, $errors); $parallelMetadata = self::optionalParallelMetadataForCommand($command, $type, $index, $errors); self::assertActivityTimeoutOrdering($startToClose, $scheduleToClose, $heartbeat, $index, $errors); + if ($cancellationPolicy !== null) { + if (! in_array( + $cancellationPolicy, + ['try_cancel', 'wait_cancellation_completed', 'abandon'], + true + )) { + $errors["commands.{$index}.cancellation_policy"] = [ + 'The cancellation_policy must be one of: try_cancel, wait_cancellation_completed, abandon.', + ]; + } + if (! WorkerProtocolVersion::supportsActivityCancellationPolicies($protocolVersion)) { + $errors["commands.{$index}.cancellation_policy"] = [ + 'Explicit activity cancellation policies require worker protocol 1.20 or newer in the same major.', + ]; + } + if ($cancellationPolicy === 'abandon' && $scheduleToClose === null) { + $errors["commands.{$index}.schedule_to_close_timeout"] = [ + 'An abandoned remote activity requires a finite schedule_to_close_timeout.', + ]; + } + } + $arguments = self::resolveCommandArgumentsWithCodec($command, $index, $errors); $payloadCodec = self::payloadCodecForResolvedPayload( $command, @@ -541,6 +583,8 @@ public static function normalize(array $commands, ?string $protocolVersion = nul 'schedule_to_close_timeout' => $scheduleToClose, 'heartbeat_timeout' => $heartbeat, 'worker_session' => $workerSession, + 'cancellation_policy' => $cancellationPolicy, + ...$scopeMetadata, ...$parallelMetadata, ], static fn (mixed $value): bool => $value !== null); @@ -665,6 +709,16 @@ public static function normalize(array $commands, ?string $protocolVersion = nul } if ($type === 'start_timer') { + $cleanupMetadata = ScopedCancellationCleanup::metadata($command); + if ($cleanupMetadata === null + || ($cleanupMetadata !== [] && ! WorkerProtocolVersion::supportsCancellationScopeMembership( + $protocolVersion + ))) { + $errors["commands.{$index}.cancellation_cleanup"] = [ + 'Scoped cleanup timers require exactly the original scope, request and delivery identities under protocol 1.20.', + ]; + continue; + } if (! is_int($command['delay_seconds'] ?? null) || (int) $command['delay_seconds'] < 0) { $errors["commands.{$index}.delay_seconds"] = [ 'Start timer commands require a non-negative integer delay_seconds.', @@ -676,6 +730,8 @@ public static function normalize(array $commands, ?string $protocolVersion = nul $normalized[] = [ 'type' => $type, 'delay_seconds' => (int) $command['delay_seconds'], + ...$scopeMetadata, + ...$cleanupMetadata, ...self::optionalParallelMetadataForCommand($command, $type, $index, $errors), ]; @@ -692,21 +748,46 @@ public static function normalize(array $commands, ?string $protocolVersion = nul } $parentClosePolicy = self::optionalCommandString($command, 'parent_close_policy', $index, $errors); + $cancellationPolicy = self::optionalCommandString($command, 'cancellation_policy', $index, $errors); $retryPolicy = self::optionalRetryPolicy($command, $index, $errors, 'Child workflow'); $parallelMetadata = self::optionalParallelMetadataForCommand($command, $type, $index, $errors); if ($parentClosePolicy !== null && ! in_array( $parentClosePolicy, - ['abandon', 'request_cancel', 'terminate'], + ['abandon', 'request_cancel', 'request_cancellation', 'terminate'], true )) { $errors["commands.{$index}.parent_close_policy"] = [ - 'The parent_close_policy must be one of: abandon, request_cancel, terminate.', + 'The parent_close_policy must be one of: abandon, request_cancel, request_cancellation, terminate.', ]; continue; } + if ($cancellationPolicy !== null && ! in_array( + $cancellationPolicy, + ['try_cancel', 'wait_cancellation_completed', 'abandon'], + true, + )) { + $errors["commands.{$index}.cancellation_policy"] = [ + 'The cancellation_policy must be one of: try_cancel, wait_cancellation_completed, abandon.', + ]; + + continue; + } + if (! WorkerProtocolVersion::supportsChildCancellationPolicies($protocolVersion)) { + if ($parentClosePolicy === 'request_cancellation') { + $errors["commands.{$index}.parent_close_policy"] = [ + 'Cooperative parent-close cancellation requires worker protocol 1.20 or newer in the same major.', + ]; + } + if (in_array($cancellationPolicy, ['try_cancel', 'wait_cancellation_completed'], true)) { + $errors["commands.{$index}.cancellation_policy"] = [ + 'Cooperative child cancellation policies require worker protocol 1.20 or newer in the same major.', + ]; + } + } + $executionTimeout = self::optionalPositiveInt($command, 'execution_timeout_seconds', $index, $errors); $runTimeout = self::optionalPositiveInt($command, 'run_timeout_seconds', $index, $errors); @@ -729,9 +810,11 @@ public static function normalize(array $commands, ?string $protocolVersion = nul 'connection' => self::optionalCommandString($command, 'connection', $index, $errors), 'queue' => self::optionalCommandString($command, 'queue', $index, $errors), 'parent_close_policy' => $parentClosePolicy, + 'cancellation_policy' => $cancellationPolicy, 'retry_policy' => $retryPolicy, 'execution_timeout_seconds' => $executionTimeout, 'run_timeout_seconds' => $runTimeout, + ...$scopeMetadata, ...$parallelMetadata, ], static fn (mixed $value): bool => $value !== null); @@ -1130,6 +1213,7 @@ public static function normalize(array $commands, ?string $protocolVersion = nul 'condition_definition_fingerprint' => $conditionDefinitionFingerprint, 'condition_wait_occurrence_id' => $conditionWaitOccurrenceId, 'timeout_seconds' => $timeoutSeconds, + ...$scopeMetadata, ...$parallelMetadata, ], static fn (mixed $value): bool => $value !== null); @@ -1164,6 +1248,7 @@ public static function normalize(array $commands, ?string $protocolVersion = nul 'type' => $type, 'signal_name' => trim((string) $command['signal_name']), 'timeout_seconds' => $timeoutSeconds, + ...$scopeMetadata, ...$parallelMetadata, ], static fn (mixed $value): bool => $value !== null); @@ -2106,6 +2191,37 @@ private static function requiredCommandString(array $command, string $field, int /** * @param array $command * @param array> $errors + * @return array{cancellation_scope_id?: string} + */ + private static function optionalOperationScopeMetadata( + array $command, + int $index, + array &$errors, + string $protocolVersion, + ): array { + if (! array_key_exists('cancellation_scope_id', $command)) { + return []; + } + $scopeId = $command['cancellation_scope_id']; + if (! is_string($scopeId) || trim($scopeId) === '' || strlen($scopeId) > 255 + || preg_match('//u', $scopeId) !== 1) { + $errors["commands.{$index}.cancellation_scope_id"] = ['Expected a nonempty canonical scope identity.']; + return []; + } + if (! WorkerProtocolVersion::supportsCancellationScopeMembership($protocolVersion)) { + $errors["commands.{$index}.cancellation_scope_id"] = [ + 'Operation scope membership requires candidate protocol 1.20.', + ]; + return []; + } + return [ + 'cancellation_scope_id' => $scopeId, + ]; + } + + /** + * @param array $command + * @param array> $errors */ private static function optionalCommandString(array $command, string $field, int $index, array &$errors): ?string { diff --git a/src/V2/Support/WorkflowExecution.php b/src/V2/Support/WorkflowExecution.php index 33c6539a3..23c7fdd7e 100644 --- a/src/V2/Support/WorkflowExecution.php +++ b/src/V2/Support/WorkflowExecution.php @@ -8,6 +8,7 @@ use Fiber; use Throwable; use Workflow\V2\Exceptions\StraightLineWorkflowRequiredException; +use Workflow\V2\Exceptions\WorkflowCancellationRequestedException; use Workflow\V2\Exceptions\WorkflowFiberDiscardedException; use Workflow\V2\Workflow; @@ -94,14 +95,19 @@ public function valid(): bool return false; } - public function send(mixed $value, ?CarbonInterface $eventTime = null): mixed - { + public function send( + mixed $value, + ?CarbonInterface $eventTime = null, + bool $advanceCancellationTime = true, + ): mixed { if (! $this->fiber instanceof Fiber) { return null; } if ($eventTime !== null) { - WorkflowFiberContext::setTime($eventTime, $this->fiber); + WorkflowFiberContext::setTime($eventTime, $this->fiber, $advanceCancellationTime); + } elseif ($advanceCancellationTime) { + WorkflowFiberContext::observeCancellationTime(null, $this->fiber); } $result = $this->fiber->resume($value); @@ -118,14 +124,24 @@ public function send(mixed $value, ?CarbonInterface $eventTime = null): mixed return null; } - public function throw(Throwable $throwable, ?CarbonInterface $eventTime = null): mixed - { + public function throw( + Throwable $throwable, + ?CarbonInterface $eventTime = null, + bool $advanceCancellationTime = true, + ): mixed { if (! $this->fiber instanceof Fiber) { throw $throwable; } + if ($throwable instanceof WorkflowCancellationRequestedException && $throwable->cancellation !== null) { + $throwable->cancellation->bindToReplayFiber($this->fiber); + WorkflowFiberContext::startCancellationTime($eventTime, $this->fiber); + } + if ($eventTime !== null) { - WorkflowFiberContext::setTime($eventTime, $this->fiber); + WorkflowFiberContext::setTime($eventTime, $this->fiber, $advanceCancellationTime); + } elseif ($advanceCancellationTime) { + WorkflowFiberContext::observeCancellationTime(null, $this->fiber); } $result = $this->fiber->throw($throwable); diff --git a/src/V2/Support/WorkflowExecutor.php b/src/V2/Support/WorkflowExecutor.php index 8fd613a71..39fa4011f 100644 --- a/src/V2/Support/WorkflowExecutor.php +++ b/src/V2/Support/WorkflowExecutor.php @@ -18,6 +18,7 @@ use Workflow\V2\Contracts\ServiceControlPlane; use Workflow\V2\Contracts\WorkflowControlPlane; use Workflow\V2\Enums\ActivityStatus; +use Workflow\V2\Enums\CancellationPolicy; use Workflow\V2\Enums\ChildCallStatus; use Workflow\V2\Enums\CommandOutcome; use Workflow\V2\Enums\CommandStatus; @@ -152,7 +153,7 @@ public function run(WorkflowRun $run, WorkflowTask $task): ?WorkflowTask : $this->waitResolutionEvent($run, $current->call, $current->baseSequence); } - if ($this->deliverCancellationAtCall( + $cancellationDelivery = $this->deliverCancellationAtCall( $run, $task, $sequence, @@ -162,33 +163,17 @@ public function run(WorkflowRun $run, WorkflowTask $task): ?WorkflowTask null, $current->baseSequence, $current->size, - )) { - if ($current->call instanceof AllCall) { - foreach ($current->call->leafDescriptors($current->baseSequence) as $descriptor) { - $this->cancelOpenWaitAtSequence( - $run, - $task, - $current->baseSequence + $descriptor['offset'], - $descriptor['call'] instanceof ActivityCall ? 'activity' : ( - $descriptor['call'] instanceof ChildWorkflowCall ? 'child' : 'timer' - ), - ); - } - } else { - $this->cancelOpenWaitAtSequence( - $run, - $task, - $current->baseSequence, - $current->call instanceof ActivityCall ? 'activity' : ( - $current->call instanceof ChildWorkflowCall ? 'child' : 'timer' - ), - ); - } - + ); + if ($cancellationDelivery === CancellationDeliveryState::Waiting) { + return $this->waitForNextResumeSource($run, $task); + } + if ($cancellationDelivery === CancellationDeliveryState::Delivered) { try { $this->syncWorkflowCursor($workflow, $sequence + 1); $current = $workflowExecution->throw( - new WorkflowCancellationRequestedException('Cooperative cancellation requested.'), + new WorkflowCancellationRequestedException( + cancellation: CooperativeCancellationDelivery::context($run), + ), $run->cancellation_delivered_at, ); } catch (Throwable $throwable) { @@ -324,18 +309,24 @@ public function run(WorkflowRun $run, WorkflowTask $task): ?WorkflowTask $activityCompletion = $this->activityCompletionEvent($run, $sequence); - if ($this->deliverCancellationAtCall( + $cancellationDelivery = $this->deliverCancellationAtCall( $run, $task, $sequence, 'activity', $activityCompletion, $workflowExecution - )) { + ); + if ($cancellationDelivery === CancellationDeliveryState::Waiting) { + return $this->waitForNextResumeSource($run, $task); + } + if ($cancellationDelivery === CancellationDeliveryState::Delivered) { try { $this->syncWorkflowCursor($workflow, $sequence + 1); $current = $workflowExecution->throw( - new WorkflowCancellationRequestedException('Cooperative cancellation requested.'), + new WorkflowCancellationRequestedException( + cancellation: CooperativeCancellationDelivery::context($run), + ), $run->cancellation_delivered_at, ); } catch (Throwable $throwable) { @@ -485,18 +476,24 @@ public function run(WorkflowRun $run, WorkflowTask $task): ?WorkflowTask $resolutionEvent = $this->conditionWaitResolutionEvent($run, $sequence); - if ($this->deliverCancellationAtCall( + $cancellationDelivery = $this->deliverCancellationAtCall( $run, $task, $sequence, 'condition', $resolutionEvent, $workflowExecution - )) { + ); + if ($cancellationDelivery === CancellationDeliveryState::Waiting) { + return $this->waitForNextResumeSource($run, $task); + } + if ($cancellationDelivery === CancellationDeliveryState::Delivered) { try { $this->syncWorkflowCursor($workflow, $sequence + 1); $current = $workflowExecution->throw( - new WorkflowCancellationRequestedException('Cooperative cancellation requested.'), + new WorkflowCancellationRequestedException( + cancellation: CooperativeCancellationDelivery::context($run), + ), $run->cancellation_delivered_at, ); } catch (Throwable $throwable) { @@ -685,6 +682,7 @@ public function run(WorkflowRun $run, WorkflowTask $task): ?WorkflowTask $current = $workflowExecution->send( $this->sideEffectResult($sideEffectEvent, $run), $sideEffectEvent->recorded_at, + advanceCancellationTime: false, ); } catch (Throwable $throwable) { $this->failRun($run, $task, $throwable, 'workflow_run', $run->id); @@ -745,7 +743,8 @@ public function run(WorkflowRun $run, WorkflowTask $task): ?WorkflowTask $this->syncWorkflowCursor($workflow, $sequence + ($resolution->advancesSequence ? 1 : 0)); $current = $workflowExecution->send( $current->resolveValue($version), - $versionMarkerEvent?->recorded_at + $versionMarkerEvent?->recorded_at, + advanceCancellationTime: false, ); } catch (Throwable $throwable) { $this->failRun($run, $task, $throwable, 'workflow_run', $run->id); @@ -790,7 +789,11 @@ public function run(WorkflowRun $run, WorkflowTask $task): ?WorkflowTask } $this->syncWorkflowCursor($workflow, $sequence + 1); - $current = $workflowExecution->send(null, $upsertEvent?->recorded_at); + $current = $workflowExecution->send( + null, + $upsertEvent?->recorded_at, + advanceCancellationTime: false + ); } catch (Throwable $throwable) { $this->failRun($run, $task, $throwable, 'workflow_run', $run->id); @@ -826,7 +829,7 @@ public function run(WorkflowRun $run, WorkflowTask $task): ?WorkflowTask } $this->syncWorkflowCursor($workflow, $sequence + 1); - $current = $workflowExecution->send(null, $memoEvent?->recorded_at); + $current = $workflowExecution->send(null, $memoEvent?->recorded_at, advanceCancellationTime: false); } catch (Throwable $throwable) { $this->failRun($run, $task, $throwable, 'workflow_run', $run->id); @@ -850,18 +853,24 @@ public function run(WorkflowRun $run, WorkflowTask $task): ?WorkflowTask $timerFired = $this->timerFiredEvent($run, $sequence); - if ($this->deliverCancellationAtCall( + $cancellationDelivery = $this->deliverCancellationAtCall( $run, $task, $sequence, 'timer', $timerFired, $workflowExecution - )) { + ); + if ($cancellationDelivery === CancellationDeliveryState::Waiting) { + return $this->waitForNextResumeSource($run, $task); + } + if ($cancellationDelivery === CancellationDeliveryState::Delivered) { try { $this->syncWorkflowCursor($workflow, $sequence + 1); $current = $workflowExecution->throw( - new WorkflowCancellationRequestedException('Cooperative cancellation requested.'), + new WorkflowCancellationRequestedException( + cancellation: CooperativeCancellationDelivery::context($run), + ), $run->cancellation_delivered_at, ); } catch (Throwable $throwable) { @@ -966,18 +975,24 @@ public function run(WorkflowRun $run, WorkflowTask $task): ?WorkflowTask $signalEvent = $this->appliedSignalEvent($run, $sequence, $current); - if ($this->deliverCancellationAtCall( + $cancellationDelivery = $this->deliverCancellationAtCall( $run, $task, $sequence, 'signal', $signalEvent, $workflowExecution - )) { + ); + if ($cancellationDelivery === CancellationDeliveryState::Waiting) { + return $this->waitForNextResumeSource($run, $task); + } + if ($cancellationDelivery === CancellationDeliveryState::Delivered) { try { $this->syncWorkflowCursor($workflow, $sequence + 1); $current = $workflowExecution->throw( - new WorkflowCancellationRequestedException('Cooperative cancellation requested.'), + new WorkflowCancellationRequestedException( + cancellation: CooperativeCancellationDelivery::context($run), + ), $run->cancellation_delivered_at, ); } catch (Throwable $throwable) { @@ -1218,18 +1233,24 @@ public function run(WorkflowRun $run, WorkflowTask $task): ?WorkflowTask $resolutionEvent = ChildRunHistory::resolutionEventForSequence($run, $sequence); $childRun = ChildRunHistory::childRunForSequence($run, $sequence); - if ($this->deliverCancellationAtCall( + $cancellationDelivery = $this->deliverCancellationAtCall( $run, $task, $sequence, 'child', $resolutionEvent, $workflowExecution - )) { + ); + if ($cancellationDelivery === CancellationDeliveryState::Waiting) { + return $this->waitForNextResumeSource($run, $task); + } + if ($cancellationDelivery === CancellationDeliveryState::Delivered) { try { $this->syncWorkflowCursor($workflow, $sequence + 1); $current = $workflowExecution->throw( - new WorkflowCancellationRequestedException('Cooperative cancellation requested.'), + new WorkflowCancellationRequestedException( + cancellation: CooperativeCancellationDelivery::context($run), + ), $run->cancellation_delivered_at, ); } catch (Throwable $throwable) { @@ -1410,7 +1431,7 @@ public function run(WorkflowRun $run, WorkflowTask $task): ?WorkflowTask $groupResolution = $this->parallelResolutionEvent($run, $current, $sequence, $leafDescriptors); - if ($this->deliverCancellationAtCall( + $cancellationDelivery = $this->deliverCancellationAtCall( $run, $task, $sequence, @@ -1418,11 +1439,17 @@ public function run(WorkflowRun $run, WorkflowTask $task): ?WorkflowTask $groupResolution, $workflowExecution, $current - )) { + ); + if ($cancellationDelivery === CancellationDeliveryState::Waiting) { + return $this->waitForNextResumeSource($run, $task); + } + if ($cancellationDelivery === CancellationDeliveryState::Delivered) { try { $this->syncWorkflowCursor($workflow, $sequence + $groupSize); $current = $workflowExecution->throw( - new WorkflowCancellationRequestedException('Cooperative cancellation requested.'), + new WorkflowCancellationRequestedException( + cancellation: CooperativeCancellationDelivery::context($run), + ), $run->cancellation_delivered_at, ); } catch (Throwable $throwable) { @@ -2344,7 +2371,7 @@ public function deliverPortableCancellation( int $sequenceSpan = 1, ?int $operationSequence = null, int $operationSequenceSpan = 1, - ): WorkflowHistoryEvent { + ): ?WorkflowHistoryEvent { $existing = CooperativeCancellationDelivery::recorded($run); if ($existing instanceof WorkflowHistoryEvent) { return $existing; @@ -2352,14 +2379,33 @@ public function deliverPortableCancellation( $run->loadMissing(['tasks', 'timers', 'activityExecutions']); $targetSequence = $operationSequence ?? $sequence; $targetSpan = $operationSequence === null ? $sequenceSpan : $operationSequenceSpan; + $waiting = false; for ($offset = 0; $offset < $targetSpan; ++$offset) { $targetKind = in_array($callKind, ['parallel', 'selection_handle'], true) ? CooperativeCancellationDelivery::callKindAt($run, $targetSequence + $offset) : $callKind; if ($targetKind !== null) { - $this->cancelOpenWaitAtSequence($run, $task, $targetSequence + $offset, $targetKind); + $waiting = $this->cancelOpenWaitAtSequence( + $run, + $task, + $targetSequence + $offset, + $targetKind + ) || $waiting; } } + if ($waiting) { + ActivityCancellationWait::bindBoundary( + $task, + $sequence, + $callKind, + $sequenceSpan, + $operationSequence, + $operationSequenceSpan + ); + $this->waitForNextResumeSource($run, $task); + + return null; + } return CooperativeCancellationDelivery::record( $run, @@ -2690,6 +2736,7 @@ private function scheduleChildWorkflow( $parentClosePolicy = $childWorkflowCall->options?->parentClosePolicy ?? ParentClosePolicy::Abandon; + $cancellationPolicy = $childWorkflowCall->options?->cancellationPolicy ?? CancellationPolicy::Abandon; ChildRunHistory::recordChildCallStarted([ 'parent_workflow_run_id' => $run->id, @@ -2701,7 +2748,7 @@ private function scheduleChildWorkflow( 'connection' => $childRun->connection, 'queue' => $childRun->queue, 'compatibility' => $childRun->compatibility, - 'cancellation_propagation' => false, + 'cancellation_propagation' => $cancellationPolicy !== CancellationPolicy::Abandon, 'status' => ChildCallStatus::Started, 'scheduled_at' => $now, 'started_at' => $now, @@ -2712,6 +2759,7 @@ private function scheduleChildWorkflow( 'metadata' => [ 'child_call_id' => $childCallId, 'attempt_count' => 1, + 'cancellation_policy' => $cancellationPolicy->value, ], 'resolved_child_instance_id' => $childInstance->id, 'resolved_child_run_id' => $childRun->id, @@ -2740,6 +2788,7 @@ private function scheduleChildWorkflow( 'child_workflow_class' => $childRun->workflow_class, 'child_workflow_type' => $childRun->workflow_type, 'parent_close_policy' => $parentClosePolicy->value, + 'cancellation_policy' => $cancellationPolicy->value, ], $parallelMetadata ?? []), $task); WorkflowHistoryEvent::record($run, HistoryEventType::ChildRunStarted, array_merge([ @@ -2752,6 +2801,7 @@ private function scheduleChildWorkflow( 'child_workflow_type' => $childRun->workflow_type, 'child_run_number' => $childRun->run_number, 'parent_close_policy' => $parentClosePolicy->value, + 'cancellation_policy' => $cancellationPolicy->value, ], $parallelMetadata ?? []), $task); WorkflowHistoryEvent::record($childRun, HistoryEventType::StartAccepted, [ @@ -3852,12 +3902,21 @@ private function finishCooperativeCancellation(WorkflowRun $run, WorkflowTask $t } // The caller holds the run lock; acquiring the instance lock here would invert command lock order. + $preservedActivityIds = $run->activityExecutions + ->filter(static fn (ActivityExecution $execution): bool => in_array( + $execution->status, + [ActivityStatus::Pending, ActivityStatus::Running], + true + ) && ActivityAbandonment::allows($run, $execution)) + ->pluck('id') + ->all(); $openTasks = $run->tasks ->filter(static fn (WorkflowTask $candidate): bool => in_array( $candidate->status, [TaskStatus::Ready, TaskStatus::Leased], true, - )); + ) && ! ($candidate->task_type === TaskType::Activity + && in_array($candidate->payload['activity_execution_id'] ?? null, $preservedActivityIds, true))); $tasksByActivityExecutionId = $openTasks ->filter(static fn (WorkflowTask $candidate): bool => is_string( $candidate->payload['activity_execution_id'] ?? null @@ -3873,7 +3932,8 @@ private function finishCooperativeCancellation(WorkflowRun $run, WorkflowTask $t } foreach ($run->activityExecutions as $execution) { - if (! in_array($execution->status, [ActivityStatus::Pending, ActivityStatus::Running], true)) { + if (in_array($execution->id, $preservedActivityIds, true) + || ! in_array($execution->status, [ActivityStatus::Pending, ActivityStatus::Running], true)) { continue; } @@ -3894,6 +3954,20 @@ private function finishCooperativeCancellation(WorkflowRun $run, WorkflowTask $t } $closedAt = now(); + $delivery = CooperativeCancellationDelivery::recorded($run); + $deadlineExpired = $run->cancellation_deadline_at !== null + && $closedAt->gte($run->cancellation_deadline_at); + if ($deadlineExpired) { + $reason = 'Cooperative cancellation cleanup deadline expired.'; + } + $deliveryMatches = $delivery !== null + && $delivery->workflow_command_id === $commandId + && ($delivery->payload['workflow_command_id'] ?? null) === $commandId + && is_int($delivery->payload['sequence'] ?? null) + && $run->cancellation_delivery_sequence === $delivery->payload['sequence']; + $cleanupOutcome = $deadlineExpired ? 'deadline_expired' + : ($deliveryMatches ? 'completed' + : ($delivery === null && $run->cancellation_delivery_sequence === null ? 'not_delivered' : 'unavailable')); $message = sprintf('Workflow cancelled: %s', $reason); /** @var WorkflowFailure $failure */ $failure = WorkflowFailure::query()->create([ @@ -3927,6 +4001,14 @@ private function finishCooperativeCancellation(WorkflowRun $run, WorkflowTask $t 'exception_class' => $failure->exception_class, 'message' => $message, 'reason' => $reason, + 'cancellation_cleanup' => [ + 'request_id' => $commandId, + 'outcome' => $cleanupOutcome, + 'cleanup_deadline_at' => $run->cancellation_deadline_at?->toISOString(), + 'delivery_history_event_id' => $deliveryMatches ? $delivery->id : null, + 'delivery_sequence' => $deliveryMatches ? $delivery->payload['sequence'] : null, + 'finished_at' => $closedAt->toISOString(), + ], ], $task, $commandId); PendingUpdateCloser::closeForTerminalRun($run, $task); @@ -5455,17 +5537,18 @@ private function deliverCancellationAtCall( ?AllCall $parallelCall = null, ?int $operationSequence = null, int $operationSequenceSpan = 1, - ): bool { + ): CancellationDeliveryState { if (! is_string($run->cancellation_request_command_id)) { - return false; + return CancellationDeliveryState::None; } if ($run->cancellation_delivery_sequence !== null) { - return $run->cancellation_delivery_sequence === $sequence; + return $run->cancellation_delivery_sequence === $sequence + ? CancellationDeliveryState::Delivered : CancellationDeliveryState::None; } if (WorkflowFiberContext::cancellationShielded($workflowExecution->fiber())) { - return false; + return CancellationDeliveryState::None; } /** @var WorkflowHistoryEvent|null $requested */ @@ -5480,23 +5563,59 @@ private function deliverCancellationAtCall( } if ($completionEvent instanceof WorkflowHistoryEvent && $completionEvent->sequence < $requested->sequence) { - return false; + return CancellationDeliveryState::None; + } + + $waitingBoundary = ActivityCancellationWait::validateBoundary( + $run, + $sequence, + $callKind, + $parallelCall instanceof AllCall ? count($parallelCall->leafDescriptors($sequence)) : 1, + $operationSequence, + $operationSequenceSpan, + ); + if ($waitingBoundary !== null) { + throw new LogicException($waitingBoundary); } + $waiting = false; if ($parallelCall instanceof AllCall) { foreach ($parallelCall->leafDescriptors($sequence) as $descriptor) { $call = $descriptor['call']; - $this->cancelOpenWaitAtSequence( + $waiting = $this->cancelOpenWaitAtSequence( $run, $task, $sequence + $descriptor['offset'], $call instanceof ActivityCall ? 'activity' : ( $call instanceof ChildWorkflowCall ? 'child' : 'timer' ), - ); + ) || $waiting; + } + } elseif ($operationSequence !== null) { + for ($offset = 0; $offset < $operationSequenceSpan; ++$offset) { + $kind = CooperativeCancellationDelivery::callKindAt($run, $operationSequence + $offset); + if ($kind !== null) { + $waiting = $this->cancelOpenWaitAtSequence( + $run, + $task, + $operationSequence + $offset, + $kind + ) || $waiting; + } } } else { - $this->cancelOpenWaitAtSequence($run, $task, $sequence, $callKind); + $waiting = $this->cancelOpenWaitAtSequence($run, $task, $sequence, $callKind); + } + if ($waiting) { + ActivityCancellationWait::bindBoundary( + $task, + $sequence, + $callKind, + $parallelCall instanceof AllCall ? count($parallelCall->leafDescriptors($sequence)) : 1, + $operationSequence, + $operationSequenceSpan, + ); + return CancellationDeliveryState::Waiting; } CooperativeCancellationDelivery::record( @@ -5509,7 +5628,7 @@ private function deliverCancellationAtCall( $operationSequenceSpan, ); - return true; + return CancellationDeliveryState::Delivered; } private function cancelOpenWaitAtSequence( @@ -5517,7 +5636,10 @@ private function cancelOpenWaitAtSequence( WorkflowTask $task, int $sequence, string $callKind, - ): void { + ): bool { + if ($callKind === 'child') { + return ChildCancellation::prepare($run, $task, $sequence); + } if (in_array($callKind, ['timer', 'condition', 'signal'], true)) { /** @var WorkflowTimer|null $timer */ $timer = $run->timers->firstWhere('sequence', $sequence); @@ -5542,6 +5664,20 @@ private function cancelOpenWaitAtSequence( /** @var ActivityExecution|null $execution */ $execution = $run->activityExecutions->firstWhere('sequence', $sequence); + if ($execution instanceof ActivityExecution + && ActivityCancellationWait::policy($run, $sequence) === CancellationPolicy::Abandon) { + ActivityAbandonment::prepare($run, $execution); + return false; + } + + if ($execution instanceof ActivityExecution + && ActivityCancellationWait::policy( + $run, + $sequence + ) === CancellationPolicy::WaitCancellationCompleted) { + return ActivityCancellationWait::prepare($run, $task, $execution); + } + if ($execution instanceof ActivityExecution && in_array( $execution->status, [ActivityStatus::Pending, ActivityStatus::Running], @@ -5560,6 +5696,8 @@ private function cancelOpenWaitAtSequence( ); } } + + return false; } private function timerFiredEvent(WorkflowRun $run, int $sequence): ?WorkflowHistoryEvent diff --git a/src/V2/Support/WorkflowFiberContext.php b/src/V2/Support/WorkflowFiberContext.php index 8aa8b2f57..2e196a63c 100644 --- a/src/V2/Support/WorkflowFiberContext.php +++ b/src/V2/Support/WorkflowFiberContext.php @@ -25,6 +25,16 @@ final class WorkflowFiberContext */ private static array $workflowTime = []; + /** + * @var array + */ + private static array $cancellationTime = []; + + /** + * @var array + */ + private static array $cancellationTimeAvailable = []; + /** * @var array */ @@ -51,6 +61,8 @@ public static function leave(): void unset(self::$activeFibers[spl_object_id($fiber)]); unset(self::$workflowTime[spl_object_id($fiber)]); + unset(self::$cancellationTime[spl_object_id($fiber)]); + unset(self::$cancellationTimeAvailable[spl_object_id($fiber)]); unset(self::$cancellationShields[spl_object_id($fiber)]); } @@ -101,13 +113,56 @@ public static function cancellationShielded(?Fiber $fiber): bool * reference must be supplied so the executor can seed the time before * resuming the workflow. */ - public static function setTime(CarbonInterface $time, ?Fiber $fiber = null): void - { + public static function setTime( + CarbonInterface $time, + ?Fiber $fiber = null, + bool $advanceCancellationTime = true, + ): void { $fiber ??= Fiber::getCurrent(); if ($fiber instanceof Fiber) { self::$workflowTime[spl_object_id($fiber)] = $time->copy(); + if ($advanceCancellationTime && array_key_exists(spl_object_id($fiber), self::$cancellationTimeAvailable)) { + self::observeCancellationTime($time, $fiber); + } + } + } + + public static function observeCancellationTime(?CarbonInterface $time, ?Fiber $fiber = null): void + { + $fiber ??= Fiber::getCurrent(); + if (! $fiber instanceof Fiber) { + return; + } + + $fiberId = spl_object_id($fiber); + if (! array_key_exists($fiberId, self::$cancellationTimeAvailable)) { + return; + } + self::$cancellationTimeAvailable[$fiberId] = $time !== null; + if ($time !== null && (! isset(self::$cancellationTime[$fiberId]) || $time->greaterThan( + self::$cancellationTime[$fiberId] + ))) { + self::$cancellationTime[$fiberId] = $time->copy(); + } + } + + public static function startCancellationTime(?CarbonInterface $time, Fiber $fiber): void + { + self::$cancellationTimeAvailable[spl_object_id($fiber)] = false; + self::observeCancellationTime($time, $fiber); + } + + public static function getCancellationTime(): CarbonInterface + { + $fiber = Fiber::getCurrent(); + if (! self::active() || ! $fiber instanceof Fiber + || ! (self::$cancellationTimeAvailable[spl_object_id($fiber)] ?? false) + || ! isset(self::$cancellationTime[spl_object_id($fiber)])) { + throw new LogicException('Cancellation remaining() requires a recorded blocking boundary timestamp.'); } + + return self::$cancellationTime[spl_object_id($fiber)]->copy(); } /** @@ -128,6 +183,16 @@ public static function getTime(): CarbonInterface return now(); } + public static function getRecordedTime(): CarbonInterface + { + $fiber = Fiber::getCurrent(); + if (! self::active() || ! $fiber instanceof Fiber || ! isset(self::$workflowTime[spl_object_id($fiber)])) { + throw new LogicException('Cancellation remaining() requires recorded workflow time.'); + } + + return self::$workflowTime[spl_object_id($fiber)]->copy(); + } + public static function suspend(mixed $call): mixed { if (! self::active()) { diff --git a/src/V2/Support/WorkflowFiberRunner.php b/src/V2/Support/WorkflowFiberRunner.php index 837d52ac6..d1c0d6c78 100644 --- a/src/V2/Support/WorkflowFiberRunner.php +++ b/src/V2/Support/WorkflowFiberRunner.php @@ -12,7 +12,9 @@ use RuntimeException; use Throwable; use Workflow\Serializers\Serializer; +use Workflow\V2\CancellationContext; use Workflow\V2\Contracts\YieldedCommand; +use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Exceptions\DurableOperationCancelledException; use Workflow\V2\Exceptions\HistoryEventShapeMismatchException; use Workflow\V2\Exceptions\UnresolvedWorkflowFailureException; @@ -90,10 +92,15 @@ final class WorkflowFiberRunner private array $recordedSignalOutcomes = []; /** - * @var array + * @var array */ private array $recordedCancellationDeliveries = []; + /** + * @var array + */ + private array $recordedScopeSequences = []; + /** * @var array */ @@ -321,13 +328,22 @@ private function nextObservableStep(): WorkflowStep $this->sequence += $current instanceof AllCall ? $current->leafCount() : 1; $this->execution->throw( - new WorkflowCancellationRequestedException('Cooperative cancellation requested.'), + new WorkflowCancellationRequestedException(cancellation: $cancellation['context']), $cancellation['recorded_at'], ); continue; } + if ($historySequence !== null && isset($this->recordedScopeSequences[$historySequence])) { + throw new HistoryEventShapeMismatchException( + $historySequence, + get_debug_type($current), + [HistoryEventType::CancellationScopeOpened->value], + 'A recorded cancellation scope must replay at its original creation boundary before operation admission.', + ); + } + if ($current instanceof CancelDurableOperationCall) { $handle = $current->handle; $cancelled = ParallelChildGroup::cancellationForHandle($this->workflow->run, $handle); @@ -478,7 +494,11 @@ private function nextObservableStep(): WorkflowStep if ($recorded !== null) { ++$this->sequence; - $this->execution->send($recorded['result'], $recorded['recorded_at']); + $this->execution->send( + $recorded['result'], + $recorded['recorded_at'], + advanceCancellationTime: false + ); continue; } @@ -488,7 +508,7 @@ private function nextObservableStep(): WorkflowStep $immediateCommands[] = self::singleCommand($step); ++$this->sequence; - $this->execution->send($result); + $this->execution->send($result, advanceCancellationTime: false); continue; } @@ -516,6 +536,7 @@ private function nextObservableStep(): WorkflowStep $this->execution->send( $current->resolveValue($resolution->version), $versionMarkerEvent?->recorded_at, + advanceCancellationTime: false, ); continue; @@ -535,7 +556,7 @@ private function nextObservableStep(): WorkflowStep ); ++$this->sequence; - $this->execution->send(null, $recorded['recorded_at']); + $this->execution->send(null, $recorded['recorded_at'], advanceCancellationTime: false); continue; } @@ -544,7 +565,7 @@ private function nextObservableStep(): WorkflowStep $immediateCommands[] = self::singleCommand($step); ++$this->sequence; - $this->execution->send(null); + $this->execution->send(null, advanceCancellationTime: false); continue; } @@ -563,7 +584,7 @@ private function nextObservableStep(): WorkflowStep ); ++$this->sequence; - $this->execution->send(null, $recorded['recorded_at']); + $this->execution->send(null, $recorded['recorded_at'], advanceCancellationTime: false); continue; } @@ -572,7 +593,7 @@ private function nextObservableStep(): WorkflowStep $immediateCommands[] = self::singleCommand($step); ++$this->sequence; - $this->execution->send(null); + $this->execution->send(null, advanceCancellationTime: false); continue; } @@ -1320,6 +1341,17 @@ private function loadHistoryEvents(array $historyEvents): void $this->namespace, ); $this->recordedCancellationDeliveries = self::indexRecordedCancellationDeliveries($this->historyEvents); + $this->recordedScopeSequences = []; + foreach ($this->historyEvents as $event) { + if (self::eventType($event) !== HistoryEventType::CancellationScopeOpened->value) { + continue; + } + $payload = is_array($event['payload'] ?? null) ? $event['payload'] : []; + $sequence = self::eventSequence($event, $payload); + if ($sequence !== null) { + $this->recordedScopeSequences[$sequence] = true; + } + } $this->openSignalWaits = array_diff_key( self::indexOpenSignalWaits($this->historyEvents), $this->recordedSignalOutcomes, @@ -1544,6 +1576,7 @@ private static function indexHistorySequencesByPosition(array $historyEvents): a private static function hasWorkflowCommandSequence(?string $type): bool { return in_array($type, [ + 'CancellationScopeOpened', 'ActivityScheduled', 'ActivityStarted', 'ActivityHeartbeatRecorded', @@ -1778,11 +1811,26 @@ private static function indexRecordedTimerOutcomes(array $historyEvents): array /** * @param list> $historyEvents - * @return array + * @return array */ private static function indexRecordedCancellationDeliveries(array $historyEvents): array { $deliveries = []; + $contexts = []; + + foreach ($historyEvents as $event) { + if (self::eventType($event) !== 'CooperativeCancellationRequested') { + continue; + } + $payload = is_array($event['payload'] ?? null) ? $event['payload'] : []; + if (is_array($payload['cancellation'] ?? null)) { + $context = CancellationContext::fromArray($payload['cancellation']); + if ($context->requestId !== ($payload['workflow_command_id'] ?? null)) { + throw new RuntimeException('Cancellation context changes its canonical request identity.'); + } + $contexts[$context->requestId] = $context; + } + } foreach ($historyEvents as $event) { if (self::eventType($event) !== 'CooperativeCancellationDelivered') { @@ -1797,6 +1845,7 @@ private static function indexRecordedCancellationDeliveries(array $historyEvents $deliveries[$sequence] = [ 'call_kind' => $callKind, 'recorded_at' => self::eventRecordedAt($event, $payload), + 'context' => $contexts[$payload['workflow_command_id'] ?? ''] ?? null, ]; } } diff --git a/src/V2/Support/WorkflowRunRetentionCleanup.php b/src/V2/Support/WorkflowRunRetentionCleanup.php index f07c1327f..f3902ab25 100644 --- a/src/V2/Support/WorkflowRunRetentionCleanup.php +++ b/src/V2/Support/WorkflowRunRetentionCleanup.php @@ -7,6 +7,8 @@ use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Model; use LogicException; +use Workflow\V2\Enums\CancellationPolicy; +use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\RunStatus; use Workflow\V2\Models\ActivityAttempt; use Workflow\V2\Models\ActivityExecution; @@ -172,6 +174,28 @@ public static function pruneRun(WorkflowRun|string $run): array }); } + /** + * @api Allows hosts to check the hold before reclaiming external payloads. + */ + public static function retentionHoldReason(WorkflowRun|string $run): ?string + { + $run = $run instanceof WorkflowRun ? $run + : ConfiguredV2Models::query('run_model', WorkflowRun::class)->findOrFail($run); + $schedules = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityScheduled->value) + ->where('payload->activity->cancellation_policy', CancellationPolicy::Abandon->value)->get(); + if ($schedules->isEmpty()) { + return null; + } + $run->setRelation('historyEvents', $run->historyEvents()->get()); + foreach ($schedules as $scheduled) { + if (! ActivityAbandonment::hasTerminalHistory($run, $scheduled)) { + return 'detached_activity_still_open'; + } + } + return null; + } + private static function assertPrunable(WorkflowRun $run): void { $status = $run->status instanceof RunStatus @@ -185,6 +209,10 @@ private static function assertPrunable(WorkflowRun $run): void if ($run->closed_at === null) { throw new LogicException(sprintf('Workflow run [%s] has not been closed.', $run->getKey())); } + $reason = self::retentionHoldReason($run); + if ($reason !== null) { + throw new LogicException(sprintf('Workflow run [%s] cannot be pruned: %s.', $run->getKey(), $reason)); + } } /** diff --git a/src/V2/Support/WorkflowStepHistory.php b/src/V2/Support/WorkflowStepHistory.php index 2c00df3ed..c4804de93 100644 --- a/src/V2/Support/WorkflowStepHistory.php +++ b/src/V2/Support/WorkflowStepHistory.php @@ -22,6 +22,8 @@ final class WorkflowStepHistory public const CHILD_WORKFLOW = 'child workflow'; + public const CANCELLATION_SCOPE = 'cancellation scope'; + public const CONDITION_WAIT = 'condition wait'; public const CONTINUE_AS_NEW = 'continue as new'; @@ -48,6 +50,7 @@ final class WorkflowStepHistory * @var list */ private const WORKFLOW_STEP_EVENT_TYPES = [ + HistoryEventType::CancellationScopeOpened, HistoryEventType::ActivityScheduled, HistoryEventType::ActivityStarted, HistoryEventType::ActivityHeartbeatRecorded, @@ -102,6 +105,7 @@ public static function nextDurableCommandSequence(WorkflowRun $run): int self::WORKFLOW_STEP_EVENT_TYPES, ); $stepTypes[] = HistoryEventType::CooperativeCancellationDelivered->value; + $stepTypes[] = HistoryEventType::CancellationScopeDelivered->value; return self::nextSequenceFromEvents( $run->historyEvents() @@ -251,7 +255,8 @@ private static function nextSequenceFromEvents(iterable $events): int foreach ($events as $event) { if (! $event instanceof WorkflowHistoryEvent || ( ! self::isWorkflowStepEvent($event) - && $event->event_type !== HistoryEventType::CooperativeCancellationDelivered + && ! in_array($event->event_type, [HistoryEventType::CooperativeCancellationDelivered, + HistoryEventType::CancellationScopeDelivered], true) )) { continue; } @@ -259,7 +264,8 @@ private static function nextSequenceFromEvents(iterable $events): int $sequence = self::intValue($event->payload['sequence'] ?? null); if ($sequence !== null) { - if ($event->event_type === HistoryEventType::CooperativeCancellationDelivered) { + if (in_array($event->event_type, [HistoryEventType::CooperativeCancellationDelivered, + HistoryEventType::CancellationScopeDelivered], true)) { $span = self::intValue($event->payload['sequence_span'] ?? null) ?? 1; if ($span > 1 && $sequence <= PHP_INT_MAX - $span) { $sequence += $span - 1; @@ -413,6 +419,7 @@ private static function diagnosticShape(string $expectedShape, array $expectedDe private static function eventMatchesShape(WorkflowHistoryEvent $event, string $expectedShape): bool { return match ($expectedShape) { + self::CANCELLATION_SCOPE => $event->event_type === HistoryEventType::CancellationScopeOpened, self::ACTIVITY => in_array($event->event_type, [ HistoryEventType::ActivityScheduled, HistoryEventType::ActivityStarted, diff --git a/src/V2/WorkflowStub.php b/src/V2/WorkflowStub.php index 0c4c9aa45..c53ec8ce1 100644 --- a/src/V2/WorkflowStub.php +++ b/src/V2/WorkflowStub.php @@ -5,6 +5,7 @@ namespace Workflow\V2; use BadMethodCallException; +use Carbon\CarbonImmutable; use Illuminate\Support\Collection; use Illuminate\Support\Facades\App; use Illuminate\Support\Facades\DB; @@ -42,6 +43,7 @@ use Workflow\V2\Models\WorkflowFailure; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowInstance; +use Workflow\V2\Models\WorkflowLink; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowRunSummary; use Workflow\V2\Models\WorkflowSignal; @@ -49,6 +51,9 @@ use Workflow\V2\Models\WorkflowTimer; use Workflow\V2\Models\WorkflowUpdate; use Workflow\V2\Support\ActivityCancellation; +use Workflow\V2\Support\CancellationScopeDescendants; +use Workflow\V2\Support\CancellationScopeHistory; +use Workflow\V2\Support\CancellationScopeRequests; use Workflow\V2\Support\ChildRunHistory; use Workflow\V2\Support\ConfiguredV2Models; use Workflow\V2\Support\CurrentRunResolver; @@ -56,6 +61,7 @@ use Workflow\V2\Support\LifecycleEventDispatcher; use Workflow\V2\Support\MemoUpsertService; use Workflow\V2\Support\ParallelChildGroup; +use Workflow\V2\Support\ParentCloseCancellation; use Workflow\V2\Support\ParentClosePolicyEnforcer; use Workflow\V2\Support\PendingMessageTask; use Workflow\V2\Support\PendingUpdateCloser; @@ -1701,144 +1707,36 @@ public function attemptCancel(?string $reason = null): CommandResult public function attemptRequestCancellation(?string $reason = null, int $cleanupTimeoutSeconds = 600): CommandResult { - if ($cleanupTimeoutSeconds < 1 || $cleanupTimeoutSeconds > 3600) { - throw new LogicException('Cancellation cleanup timeout must be between 1 and 3600 seconds.'); - } - - /** @var WorkflowCommand|null $command */ - $command = null; - $task = null; - - DB::transaction(function () use (&$command, &$task, $reason, $cleanupTimeoutSeconds): void { - /** @var WorkflowInstance $instance */ - $instance = self::instanceQuery()->lockForUpdate()->findOrFail($this->instance->id); - $currentRun = $this->currentRunForInstance($instance, true); - - if (! $currentRun instanceof WorkflowRun) { - $command = $this->rejectCommand( - $instance, - null, - CommandType::RequestCancellation, - 'instance_not_started', - $this->commandTargetScope(), - ); - - return; - } - - if ($this->runTargeted) { - /** @var WorkflowRun $run */ - $run = self::runQuery()->lockForUpdate()->findOrFail($this->selectedRunId); - - if ($run->id !== $currentRun->id) { - $command = $this->rejectCommand( - $instance, - $run, - CommandType::RequestCancellation, - 'selected_run_not_current', - $this->commandTargetScope(), - [ - 'resolved_workflow_run_id' => $currentRun->id, - ], - ); - - return; - } - } else { - $run = $currentRun; - } - - if (is_string($run->cancellation_request_command_id)) { - $command = WorkflowCommand::query()->findOrFail($run->cancellation_request_command_id); - - return; - } - - if (! $this->runIsActive($run)) { - $command = $this->rejectCommand( - $instance, - $run, - CommandType::RequestCancellation, - 'run_not_active', - $this->commandTargetScope(), - ); - - return; - } - - $requestedAt = now(); - $deadline = $requestedAt->copy() - ->addSeconds($cleanupTimeoutSeconds); - /** @var WorkflowCommand $command */ - $command = WorkflowCommand::record($instance, $run, $this->commandAttributes([ - 'command_type' => CommandType::RequestCancellation->value, - 'target_scope' => $this->commandTargetScope(), - 'status' => CommandStatus::Accepted->value, - 'outcome' => CommandOutcome::CancellationRequested->value, - 'payload_codec' => $run->payload_codec ?? CodecRegistry::defaultCodec(), - 'payload' => Serializer::serializeWithCodec( - $run->payload_codec ?? CodecRegistry::defaultCodec(), - [ - 'reason' => $reason, - 'cleanup_deadline_at' => $deadline->toISOString(), - ], - ), - 'accepted_at' => $requestedAt, - 'applied_at' => $requestedAt, - ])); - - $run->forceFill([ - 'cancellation_request_command_id' => $command->id, - 'cancellation_requested_at' => $requestedAt, - 'cancellation_deadline_at' => $deadline, - 'last_progress_at' => $requestedAt, - ])->save(); - - WorkflowHistoryEvent::record($run, HistoryEventType::CooperativeCancellationRequested, array_filter([ - 'workflow_command_id' => $command->id, - 'workflow_instance_id' => $instance->id, - 'workflow_run_id' => $run->id, - 'command_type' => CommandType::RequestCancellation->value, - 'reason' => $reason, - 'cleanup_deadline_at' => $deadline->toISOString(), - ], static fn (mixed $value): bool => $value !== null), null, $command); - - if (! $this->hasOpenWorkflowTask($run->id)) { - /** @var WorkflowTask $task */ - $task = self::taskQuery()->create([ - 'workflow_run_id' => $run->id, - 'namespace' => $run->namespace, - 'task_type' => TaskType::Workflow->value, - 'status' => TaskStatus::Ready->value, - 'available_at' => $requestedAt, - 'payload' => [ - 'resume_source_kind' => 'cancellation_request', - 'resume_source_id' => $command->id, - 'workflow_command_id' => $command->id, - ], - 'connection' => $run->connection, - 'queue' => $run->queue, - 'compatibility' => $run->compatibility, - ]); - } - - self::projectRun($run, self::PROJECTION_RUN_RELATIONS); - }); - - $this->refresh(); - - if ($task instanceof WorkflowTask) { - TaskDispatcher::dispatch($task); - } + return $this->recordCancellationRequest($reason, $cleanupTimeoutSeconds); + } - if (! $command instanceof WorkflowCommand) { - throw new LogicException(sprintf( - 'Workflow instance [%s] failed to record a cancellation request.', - $this->instance->id, - )); - } + /** + * Request cooperative cancellation from a recorded direct parent. + * + * Reads the parent's accepted context from storage. Callers cannot supply + * or replace its root identity, requester, lineage or original deadline. + * + * @internal Used by cancellation propagation and parent-close enforcement. + */ + public function attemptRequestCancellationFromParent(string $parentWorkflowRunId): CommandResult + { + return $this->recordCancellationRequest(null, 600, $parentWorkflowRunId); + } - return new CommandResult($command); + /** + * @internal Read the frozen scope origin from canonical parent history. + */ + public function attemptRequestCancellationFromScope( + string $parentWorkflowRunId, + string $scopeId, + string $preparationHistoryEventId, + string $childCallId, + ): CommandResult { + return $this->recordCancellationRequest(null, 600, $parentWorkflowRunId, [ + 'scope_id' => $scopeId, + 'preparation_history_event_id' => $preparationHistoryEventId, + 'child_call_id' => $childCallId, + ]); } public function terminate(?string $reason = null): CommandResult @@ -1989,6 +1887,343 @@ public function attemptArchive(?string $reason = null): CommandResult return new CommandResult($command); } + /** + * @param array{scope_id: string, preparation_history_event_id: string, child_call_id: string}|null $scopeTarget + */ + private function recordCancellationRequest( + ?string $reason, + int $cleanupTimeoutSeconds, + ?string $parentWorkflowRunId = null, + ?array $scopeTarget = null, + ): CommandResult { + if ($cleanupTimeoutSeconds < 1 || $cleanupTimeoutSeconds > 3600) { + throw new LogicException('Cancellation cleanup timeout must be between 1 and 3600 seconds.'); + } + + /** @var WorkflowCommand|null $command */ + $command = null; + $task = null; + + DB::transaction(function () use ( + &$command, + &$task, + $reason, + $cleanupTimeoutSeconds, + $parentWorkflowRunId, + $scopeTarget + ): void { + $command = null; + $task = null; + // Scoped actors hold the parent prefix before touching a child instance. + /** @var WorkflowRun|null $scopeParent */ + $scopeParent = $scopeTarget === null ? null : self::runQuery()->lockForUpdate()->findOrFail( + $parentWorkflowRunId + ); + /** @var WorkflowInstance $instance */ + $instance = self::instanceQuery()->lockForUpdate()->findOrFail($this->instance->id); + $currentRun = $this->currentRunForInstance($instance, true); + + if (! $currentRun instanceof WorkflowRun) { + $command = $this->rejectCommand( + $instance, + null, + CommandType::RequestCancellation, + 'instance_not_started', + $this->commandTargetScope(), + ); + + return; + } + + if ($this->runTargeted) { + /** @var WorkflowRun $run */ + $run = self::runQuery()->lockForUpdate()->findOrFail($this->selectedRunId); + + if ($run->id !== $currentRun->id) { + $command = $this->rejectCommand( + $instance, + $run, + CommandType::RequestCancellation, + 'selected_run_not_current', + $this->commandTargetScope(), + [ + 'resolved_workflow_run_id' => $currentRun->id, + ], + ); + + return; + } + } else { + $run = $currentRun; + } + + $parentContext = null; + $scopeOrigin = null; + $scopePreparation = null; + $scopeDeadline = null; + if ($scopeParent !== null && $scopeTarget !== null) { + try { + $scopePreparation = \Workflow\V2\Support\ScopedCancellationPreparation::forScope( + $scopeParent, + $scopeTarget['scope_id'], + $scopeTarget['preparation_history_event_id'] + ); + } catch (LogicException $error) { + if (! in_array($error->getMessage(), [ + 'cancellation_scope_preparation_reference_invalid', 'cancellation_scope_descendant_not_prepared', + ], true)) { + throw $error; + } + } + $member = $scopePreparation === null ? null : collect($scopePreparation->childMembers) + ->firstWhere('child_call_id', $scopeTarget['child_call_id']); + if ($scopePreparation === null || $scopePreparation->historyEventId !== $scopeTarget['preparation_history_event_id'] + || $member === null || $member['child_workflow_instance_id'] !== $instance->id + || $member['child_workflow_run_id'] !== $run->id || $member['cancellation_policy'] === 'abandon' + || $scopeParent->namespace !== $run->namespace) { + $command = $this->rejectCommand( + $instance, + $run, + CommandType::RequestCancellation, + 'cancellation_scope_child_target_mismatch', + $this->commandTargetScope() + ); + return; + } + $scopeOrigin = $scopePreparation->context; + $parentContext = $scopeOrigin->rootContext; + } elseif ($parentWorkflowRunId !== null) { + $linked = WorkflowLink::query() + ->where('parent_workflow_run_id', $parentWorkflowRunId) + ->where('child_workflow_run_id', $run->id) + ->where('child_workflow_instance_id', $instance->id) + ->where('link_type', 'child_workflow') + ->exists(); + /** @var WorkflowRun|null $parent */ + $parent = $linked ? self::runQuery()->find($parentWorkflowRunId) : null; + /** @var WorkflowCommand|null $parentCommand */ + $parentCommand = $parent instanceof WorkflowRun && is_string($parent->cancellation_request_command_id) + ? WorkflowCommand::query()->find($parent->cancellation_request_command_id) : null; + $parentContext = $parentCommand instanceof WorkflowCommand + ? (new CommandResult($parentCommand))->cancellationContext() + : ($parent instanceof WorkflowRun && $parent->cancellation_request_command_id === null + ? ParentCloseCancellation::context($parent) : null); + $parentEntry = $parentContext !== null ? $parentContext->lineage[count( + $parentContext->lineage + ) - 1] : null; + $failure = ! $linked ? 'cancellation_parent_not_linked' + : ($parentContext === null ? 'cancellation_parent_context_unavailable' + : (($parentCommand !== null && ($parentCommand->status !== CommandStatus::Accepted + || $parentCommand->command_type !== CommandType::RequestCancellation + || $parentContext->requestId !== $parentCommand->id)) + || $parentEntry['workflow_run_id'] !== $parent->id + || $parentEntry['workflow_instance_id'] !== $parent->workflow_instance_id + ? 'cancellation_parent_context_mismatch' + : (in_array($run->id, array_column($parentContext->lineage, 'workflow_run_id'), true) + ? 'cancellation_lineage_cycle' : null))); + if ($failure !== null) { + $command = $this->rejectCommand( + $instance, + $run, + CommandType::RequestCancellation, + $failure, + $this->commandTargetScope() + ); + + return; + } + } + + if (is_string($run->cancellation_request_command_id)) { + $existing = WorkflowCommand::query()->findOrFail($run->cancellation_request_command_id); + $existingContext = (new CommandResult($existing))->cancellationContext(); + if ($parentContext !== null && $existingContext?->rootRequestId !== $parentContext->rootRequestId) { + $codec = $run->payload_codec ?? CodecRegistry::defaultCodec(); + $command = $this->rejectCommand( + $instance, + $run, + CommandType::RequestCancellation, + 'cancellation_root_conflict', + $this->commandTargetScope(), + [ + 'payload' => Serializer::serializeWithCodec($codec, [ + 'existing_request_id' => $existing->id, + 'existing_root_request_id' => $existingContext?->rootRequestId, + 'existing_cleanup_deadline_at' => $run->cancellation_deadline_at?->toISOString(), + 'incoming_root_request_id' => $parentContext->rootRequestId, + 'incoming_cleanup_deadline_at' => ($scopeOrigin?->deadline() ?? $parentContext->deadline()) + ->toISOString(), + ]), + ] + ); + } else { + if ($scopeOrigin !== null && $existingContext?->scopeOrigin?->toArray() !== $scopeOrigin->toArray()) { + $command = $this->rejectCommand( + $instance, + $run, + CommandType::RequestCancellation, + 'cancellation_scope_child_origin_conflict', + $this->commandTargetScope() + ); + } else { + $command = $existing; + } + } + + return; + } + + if (! $this->runIsActive($run)) { + $command = $this->rejectCommand( + $instance, + $run, + CommandType::RequestCancellation, + 'run_not_active', + $this->commandTargetScope(), + ); + + return; + } + if ($scopeOrigin !== null && $scopeParent !== null && $scopeTarget !== null) { + $authority = CancellationScopeRequests::authority($scopeParent, $scopeTarget['scope_id']); + if (! $authority['active'] || now()->gte($scopeOrigin->deadline()) + || now() + ->gte(CarbonImmutable::parse($scopePreparation->authorityDeadlineAt))) { + $command = $this->rejectCommand( + $instance, + $run, + CommandType::RequestCancellation, + 'cancellation_scope_authority_expired', + $this->commandTargetScope() + ); + return; + } + $scopeDeadline = CarbonImmutable::parse($scopePreparation->authorityDeadlineAt); + if ($authority['deadline_at'] !== null && CarbonImmutable::parse($authority['deadline_at'])->lessThan( + $scopeDeadline + )) { + $scopeDeadline = CarbonImmutable::parse($authority['deadline_at']); + } + } + + $requestedAt = now(); + $deadline = $scopeDeadline ?? $scopeOrigin?->deadline() ?? $parentContext?->deadline() ?? $requestedAt->copy() + ->addSeconds($cleanupTimeoutSeconds); + $reason = $parentContext?->reason ?? $reason; + $requestId = (string) Str::ulid(); + $caller = $this->resolvedCommandContext() + ->attributes(); + $context = $scopeOrigin !== null + ? CancellationContext::fromScopeContext( + $scopeOrigin, + $requestId, + $instance->id, + $run->id, + $scopeDeadline + )->toArray() + : ($parentContext !== null + ? $parentContext->forDescendant($requestId, $instance->id, $run->id) + ->toArray() + : CancellationContext::fromArray([ + 'schema' => 'durable-workflow.cancellation-context/v1', + 'request_id' => $requestId, + 'root_request_id' => $requestId, + 'root_workflow_instance_id' => $instance->id, + 'root_workflow_run_id' => $run->id, + 'parent_request_id' => null, + 'reason' => $reason, + 'requester' => array_intersect_key( + $caller['context']['principal'] ?? $caller['context']['caller'], + array_flip(['type', 'id', 'label']), + ), + 'source' => $caller['source'], + 'requested_at' => $requestedAt->toISOString(), + 'cleanup_deadline_at' => $deadline->toISOString(), + 'lineage' => [[ + 'request_id' => $requestId, + 'workflow_instance_id' => $instance->id, + 'workflow_run_id' => $run->id, + ]], + ])->toArray()); + /** @var WorkflowCommand $command */ + $command = WorkflowCommand::record($instance, $run, $this->commandAttributes([ + 'id' => $requestId, + 'command_type' => CommandType::RequestCancellation->value, + 'target_scope' => $this->commandTargetScope(), + 'status' => CommandStatus::Accepted->value, + 'outcome' => CommandOutcome::CancellationRequested->value, + 'payload_codec' => $run->payload_codec ?? CodecRegistry::defaultCodec(), + 'payload' => Serializer::serializeWithCodec( + $run->payload_codec ?? CodecRegistry::defaultCodec(), + [ + 'reason' => $reason, + 'cleanup_deadline_at' => $deadline->toISOString(), + 'cancellation' => $context, + ], + ), + 'accepted_at' => $requestedAt, + 'applied_at' => $requestedAt, + ])); + + $run->forceFill([ + 'cancellation_request_command_id' => $command->id, + 'cancellation_requested_at' => $requestedAt, + 'cancellation_deadline_at' => $deadline, + 'last_progress_at' => $requestedAt, + ])->save(); + + WorkflowHistoryEvent::record($run, HistoryEventType::CooperativeCancellationRequested, array_filter([ + 'workflow_command_id' => $command->id, + 'workflow_instance_id' => $instance->id, + 'workflow_run_id' => $run->id, + 'command_type' => CommandType::RequestCancellation->value, + 'reason' => $reason, + 'cleanup_deadline_at' => $deadline->toISOString(), + 'cancellation' => $context, + ], static fn (mixed $value): bool => $value !== null), null, $command); + + // Accept the unshielded subtree in this same transaction. Replay + // and replacement workers inherit this request's original budget. + CancellationScopeDescendants::request($run, CancellationScopeHistory::ROOT_SCOPE_ID); + + if (! $this->hasOpenWorkflowTask($run->id)) { + /** @var WorkflowTask $task */ + $task = self::taskQuery()->create([ + 'workflow_run_id' => $run->id, + 'namespace' => $run->namespace, + 'task_type' => TaskType::Workflow->value, + 'status' => TaskStatus::Ready->value, + 'available_at' => $requestedAt, + 'payload' => [ + 'resume_source_kind' => 'cancellation_request', + 'resume_source_id' => $command->id, + 'workflow_command_id' => $command->id, + ], + 'connection' => $run->connection, + 'queue' => $run->queue, + 'compatibility' => $run->compatibility, + ]); + } + + self::projectRun($run, self::PROJECTION_RUN_RELATIONS); + }, 3); + + $this->refresh(); + + if ($task instanceof WorkflowTask) { + TaskDispatcher::dispatch($task); + } + + if (! $command instanceof WorkflowCommand) { + throw new LogicException(sprintf( + 'Workflow instance [%s] failed to record a cancellation request.', + $this->instance->id, + )); + } + + return new CommandResult($command); + } + /** * @param array $arguments * @return array{0: WorkflowCommand|null, 1: WorkflowUpdate|null, 2: WorkflowTask|null} diff --git a/src/migrations/2026_10_04_000100_add_scoped_cancellation_recovery_to_workflow_runs.php b/src/migrations/2026_10_04_000100_add_scoped_cancellation_recovery_to_workflow_runs.php new file mode 100644 index 000000000..9836c6dd1 --- /dev/null +++ b/src/migrations/2026_10_04_000100_add_scoped_cancellation_recovery_to_workflow_runs.php @@ -0,0 +1,24 @@ +timestamp('cancellation_scope_recovery_until', 6) + ->nullable(); + }); + } + + public function down(): void + { + Schema::table('workflow_runs', static function (Blueprint $table): void { + $table->dropColumn('cancellation_scope_recovery_until'); + }); + } +}; diff --git a/tests/Feature/V2/V2CancellationCascadeViewTest.php b/tests/Feature/V2/V2CancellationCascadeViewTest.php new file mode 100644 index 000000000..75a023bfa --- /dev/null +++ b/tests/Feature/V2/V2CancellationCascadeViewTest.php @@ -0,0 +1,500 @@ + 'database', + ]); + Queue::fake(); + Carbon::setTestNow('2026-10-02T00:00:00Z'); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + public function testOneViewRetainsTheRootBudgetThroughChildCleanupAndDuplicateRequests(): void + { + [$parent, $child] = $this->start(); + $request = $parent->requestCancellation('maintenance', 30); + $context = $request->cancellationContext(); + $this->assertNotNull($context); + $requested = CancellationCascadeView::forRun($parent->run()->fresh()); + $this->assertNotNull($requested); + $this->assertCount(2, $requested['runs']); + $this->assertTrue($requested['inspection_complete']); + $this->assertSame('requested', $requested['runs'][0]['lifecycle']); + $this->assertNull($requested['runs'][1]['request']); + $this->assertCount(1, $requested['edges']); + + $this->runTask($parent, TaskType::Workflow); + $this->runTask($child, TaskType::Workflow); + $cleaning = CancellationCascadeView::forRun($child->run()->fresh()); + $this->assertNotNull($cleaning); + $this->assertSame($child->runId(), $cleaning['selected_run_id']); + $this->assertSame($context->rootRequestId, $cleaning['root']['root_request_id']); + $this->assertSame($context->deadline()->toISOString(), $cleaning['root']['cleanup_deadline_at']); + $this->assertTrue($cleaning['inspection_complete']); + $this->assertSame('requested', $cleaning['runs'][0]['lifecycle']); + $this->assertSame('cleaning_up', $cleaning['runs'][1]['lifecycle']); + $this->assertTrue($cleaning['runs'][1]['same_root_budget']); + $this->assertNotSame( + $cleaning['runs'][0]['request']['request_id'], + $cleaning['runs'][1]['request']['request_id'] + ); + + Carbon::setTestNow(now()->addSeconds(2)); + $this->runTask($child, TaskType::Timer); + $this->runTask($child, TaskType::Workflow); + $this->runTask($parent, TaskType::Workflow); + $finished = CancellationCascadeView::forRun($parent->run()->fresh()); + $this->assertNotNull($finished); + $this->assertTrue($finished['inspection_complete']); + $this->assertCount(2, $finished['runs']); + foreach ($finished['runs'] as $node) { + $this->assertSame('cancelled', $node['lifecycle']); + $this->assertSame('WorkflowCancelled', $node['terminal_event_type']); + $this->assertSame('completed', $node['cleanup']['outcome']); + $this->assertSame($context->deadline()->toISOString(), $node['cleanup']['cleanup_deadline_at']); + $this->assertTrue($node['same_root_budget']); + } + $this->assertSame($request->commandId(), $parent->requestCancellation('duplicate', 300)->commandId()); + Carbon::setTestNow('2026-12-01T00:00:00Z'); + $this->assertSame($finished, CancellationCascadeView::forRun($parent->run()->fresh())); + } + + public function testForeignNamespaceReferencesCannotExposeTheirRunOrRequest(): void + { + [$parent] = $this->start(); + $parent->requestCancellation('public test reason', 30); + [$foreign] = $this->start(); + $foreign->run() + ->forceFill([ + 'namespace' => 'foreign', + ])->save(); + $foreign->run() + ->instance->forceFill([ + 'namespace' => 'foreign', + ])->save(); + $foreign->requestCancellation('DO_NOT_EXPOSE_PRIVATE_REQUEST', 30); + WorkflowLink::query()->create([ + 'parent_workflow_instance_id' => $parent->id(), + 'parent_workflow_run_id' => $parent->runId(), + 'child_workflow_instance_id' => $foreign->id(), + 'child_workflow_run_id' => $foreign->runId(), + 'link_type' => 'child_workflow', + 'sequence' => 99, + 'is_primary_parent' => false, + ]); + + $view = CancellationCascadeView::forRun($parent->run()->fresh()); + $this->assertNotNull($view); + $this->assertFalse($view['inspection_complete']); + $this->assertContains('related_run_unavailable', array_column($view['findings'], 'code')); + $encoded = json_encode($view, JSON_THROW_ON_ERROR); + $this->assertStringNotContainsString($foreign->runId(), $encoded); + $this->assertStringNotContainsString($foreign->id(), $encoded); + $this->assertStringNotContainsString('DO_NOT_EXPOSE_PRIVATE_REQUEST', $encoded); + $this->assertCount(2, $view['runs']); + } + + public function testScopedChildKeepsTheOriginalRootAndItsNarrowerAuthorityInOneView(): void + { + [$parent, $child] = $this->start(); + $root = $parent->requestCancellation('maintenance', 30) + ->cancellationContext(); + $this->assertNotNull($root); + $scope = ScopedCancellationContext::fromRunContext($root)->forDescendant( + 'inner-request', + $parent->id(), + $parent->runId(), + 'inner', + $root->requestedAt() + ->addSeconds(20) + ); + $context = $this->recordScopedChild($child, $scope); + + foreach ([$parent, $child] as $selected) { + $view = CancellationCascadeView::forRun($selected->run()->fresh()); + $this->assertNotNull($view); + $this->assertTrue($view['inspection_complete']); + $this->assertSame([], $view['findings']); + $this->assertSame($root->rootRequestId, $view['root']['root_request_id']); + $this->assertSame($root->deadline()->toISOString(), $view['root']['cleanup_deadline_at']); + $node = collect($view['runs'])->firstWhere('run_id', $child->runId()); + $this->assertTrue($node['same_root_budget']); + $this->assertSame($context->deadline()->toISOString(), $node['request']['cleanup_deadline_at']); + $this->assertSame( + $root->deadline() + ->toISOString(), + $node['request']['scope_origin']['root_context']['cleanup_deadline_at'] + ); + $this->assertSame(['root', 'inner'], array_column($node['request']['scope_origin']['lineage'], 'scope_id')); + } + } + + public function testScopedOriginCannotExposeAnUnavailableAncestorAddress(): void + { + [$parent, $child] = $this->start(); + $root = $parent->requestCancellation('maintenance', 30) + ->cancellationContext(); + $this->assertNotNull($root); + [$foreign] = $this->start(); + $foreign->run() + ->forceFill([ + 'namespace' => 'foreign', + ])->save(); + $foreign->run() + ->instance->forceFill([ + 'namespace' => 'foreign', + ])->save(); + $scope = ScopedCancellationContext::fromRunContext($root)->forDescendant( + 'DO_NOT_EXPOSE_PRIVATE_SCOPE_REQUEST', + $foreign->id(), + $foreign->runId(), + 'DO_NOT_EXPOSE_PRIVATE_SCOPE', + $root->requestedAt() + ->addSeconds(20) + ); + $this->recordScopedChild($child, $scope); + + $view = CancellationCascadeView::forRun($parent->run()->fresh()); + $this->assertNotNull($view); + $this->assertFalse($view['inspection_complete']); + $this->assertContains('scope_origin_unavailable', array_column($view['findings'], 'code')); + $node = collect($view['runs'])->firstWhere('run_id', $child->runId()); + $this->assertNull($node['request']['scope_origin']); + $encoded = json_encode($view, JSON_THROW_ON_ERROR); + foreach ([$foreign->id(), $foreign->runId(), 'DO_NOT_EXPOSE_PRIVATE_SCOPE'] as $hidden) { + $this->assertStringNotContainsString($hidden, $encoded); + } + } + + public function testScopedOriginUsesTheSameRequestTextLimitAsItsVisibleRoot(): void + { + [$parent, $child] = $this->start(); + $root = $parent->requestCancellation(str_repeat('x', 8193), 30) + ->cancellationContext(); + $this->assertNotNull($root); + $this->recordScopedChild($child, ScopedCancellationContext::fromRunContext($root)); + + $view = CancellationCascadeView::forRun($parent->run()->fresh()); + $this->assertNotNull($view); + $node = collect($view['runs'])->firstWhere('run_id', $child->runId()); + $this->assertSame(8192, strlen($node['request']['scope_origin']['root_context']['reason'])); + $this->assertFalse($view['inspection_complete']); + $this->assertTrue($view['truncated']); + $this->assertContains('request_text_limit', array_column($view['findings'], 'code')); + } + + public function testAnExceededScopePathLimitRemainsAnIncompleteInspection(): void + { + [$parent, $child] = $this->start(); + $root = $parent->requestCancellation('maintenance', 30) + ->cancellationContext(); + $this->assertNotNull($root); + $scope = ScopedCancellationContext::fromRunContext($root); + for ($index = 1; $index <= 100; ++$index) { + $scope = $scope->forDescendant( + 'scope-request-' . $index, + $parent->id(), + $parent->runId(), + 'scope-' . $index + ); + } + $this->recordScopedChild($child, $scope); + + $view = CancellationCascadeView::forRun($parent->run()->fresh()); + $this->assertNotNull($view); + $node = collect($view['runs'])->firstWhere('run_id', $child->runId()); + $this->assertNull($node['request']['scope_origin']); + $this->assertFalse($view['inspection_complete']); + $this->assertTrue($view['truncated']); + $this->assertContains('scope_origin_limit', array_column($view['findings'], 'code')); + $this->assertSame(100, $view['limits']['scope_origin_addresses']); + } + + public function testScopedOriginCannotSubstituteADifferentOriginalRootBudget(): void + { + [$parent, $child] = $this->start(); + $root = $parent->requestCancellation('maintenance', 30) + ->cancellationContext(); + $this->assertNotNull($root); + $snapshot = $root->toArray(); + $snapshot['cleanup_deadline_at'] = $root->requestedAt()->addSeconds(25)->toISOString(); + $changed = CancellationContext::fromArray($snapshot); + $this->recordScopedChild($child, ScopedCancellationContext::fromRunContext($changed)); + + $view = CancellationCascadeView::forRun($child->run()->fresh()); + $this->assertNotNull($view); + $this->assertNull($view['root']); + $this->assertFalse($view['inspection_complete']); + $this->assertContains('root_request_mismatch', array_column($view['findings'], 'code')); + $node = collect($view['runs'])->firstWhere('run_id', $child->runId()); + $this->assertFalse($node['same_root_budget']); + } + + public function testMissingRequestHistoryRemainsAnIncompleteInspection(): void + { + [$parent] = $this->start(); + $parent->requestCancellation('maintenance', 30); + WorkflowHistoryEvent::query()->where('workflow_run_id', $parent->runId()) + ->where('event_type', HistoryEventType::CooperativeCancellationRequested->value)->delete(); + + $view = CancellationCascadeView::forRun($parent->run()->fresh()); + $this->assertNotNull($view); + $this->assertFalse($view['inspection_complete']); + $this->assertNull($view['root']); + $this->assertNull($view['runs'][0]['request']); + $this->assertContains('request_history_unavailable', array_column($view['findings'], 'code')); + } + + public function testAnInaccessibleRootCannotBeExpandedFromAVisibleChildSnapshot(): void + { + [$parent, $child] = $this->start(); + $parent->requestCancellation('maintenance', 30); + $this->runTask($parent, TaskType::Workflow); + $parent->run() + ->forceFill([ + 'namespace' => 'foreign', + ])->save(); + $parent->run() + ->instance->forceFill([ + 'namespace' => 'foreign', + ])->save(); + + $view = CancellationCascadeView::forRun($child->run()->fresh()); + $this->assertNotNull($view); + $this->assertFalse($view['inspection_complete']); + $this->assertNull($view['root']); + $this->assertNull($view['runs'][0]['request']); + $this->assertContains('root_unavailable', array_column($view['findings'], 'code')); + $this->assertStringNotContainsString($parent->runId(), json_encode($view, JSON_THROW_ON_ERROR)); + $this->assertCount(1, $view['runs']); + } + + public function testADeletedRequestProjectionDoesNotHideTheOriginalCanonicalBudget(): void + { + [$parent] = $this->start(); + $context = $parent->requestCancellation('maintenance', 30) + ->cancellationContext(); + $this->assertNotNull($context); + $parent->run() + ->forceFill([ + 'cancellation_request_command_id' => null, + ])->save(); + + $view = CancellationCascadeView::forRun($parent->run()->fresh()); + $this->assertNotNull($view); + $this->assertFalse($view['inspection_complete']); + $this->assertSame($context->requestId, $view['root']['request_id']); + $this->assertSame($context->deadline()->toISOString(), $view['root']['cleanup_deadline_at']); + $this->assertContains('request_projection_mismatch', array_column($view['findings'], 'code')); + } + + public function testIndependentChildCancellationIsExplainedWithoutReplacingEitherBudget(): void + { + [$parent, $child] = $this->start(); + $childContext = $child->requestCancellation('independent child', 45) + ->cancellationContext(); + $parentContext = $parent->requestCancellation('parent request', 30) + ->cancellationContext(); + $this->assertNotNull($childContext); + $this->assertNotNull($parentContext); + $this->runTask($parent, TaskType::Workflow); + + $view = CancellationCascadeView::forRun($parent->run()->fresh()); + $this->assertNotNull($view); + $this->assertTrue($view['inspection_complete']); + $this->assertSame($parentContext->rootRequestId, $view['root']['root_request_id']); + $this->assertSame($childContext->rootRequestId, $view['runs'][1]['request']['root_request_id']); + $this->assertSame($childContext->deadline()->toISOString(), $view['runs'][1]['request']['cleanup_deadline_at']); + $this->assertFalse($view['runs'][1]['same_root_budget']); + $this->assertSame('cancellation_root_conflict', $view['runs'][0]['child_propagation'][0]['rejection_reason']); + $this->assertSame('rejected', $view['runs'][0]['child_propagation'][0]['request_outcome']); + } + + public function testParentCloseOriginKeepsItsCompletedParentAndOriginalSharedBudget(): void + { + $parent = WorkflowStub::make(TestParentCloseCooperativeWorkflow::class); + $parent->start(); + $this->runTask($parent, TaskType::Workflow); + foreach (WorkflowLink::query()->where('parent_workflow_run_id', $parent->runId())->get() as $link) { + $this->runTask(WorkflowStub::loadRun($link->child_workflow_run_id), TaskType::Workflow); + } + Carbon::setTestNow(now()->addSecond()); + $this->runTask($parent, TaskType::Timer); + $this->runTask($parent, TaskType::Workflow); + + $view = CancellationCascadeView::forRun($parent->run()->fresh()); + $this->assertNotNull($view); + $this->assertTrue($view['inspection_complete']); + $this->assertSame('parent_close_policy', $view['root']['source']); + $this->assertSame('completed', $view['runs'][0]['lifecycle']); + $this->assertNull($view['runs'][0]['cleanup']); + $this->assertCount(3, $view['runs']); + foreach (array_slice($view['runs'], 1) as $child) { + $this->assertSame('requested', $child['lifecycle']); + $this->assertTrue($child['same_root_budget']); + $this->assertSame($view['root']['cleanup_deadline_at'], $child['request']['cleanup_deadline_at']); + } + Carbon::setTestNow(now()->addSeconds(20)); + $this->assertSame($view, CancellationCascadeView::forRun($parent->run()->fresh())); + } + + public function testSelectingAnHistoricalRunDoesNotFollowTheCurrentRunPointer(): void + { + [$parent] = $this->start(); + $context = $parent->requestCancellation('historical request', 30) + ->cancellationContext(); + $this->assertNotNull($context); + $selected = $parent->run() + ->fresh(); + $replacement = $selected->replicate(['id']); + $replacement->run_number = $selected->run_number + 1; + $replacement->cancellation_request_command_id = null; + $replacement->save(); + $selected->instance->forceFill([ + 'current_run_id' => $replacement->id, + ])->save(); + + $view = CancellationCascadeView::forRun($selected); + $this->assertNotNull($view); + $this->assertSame($selected->id, $view['selected_run_id']); + $this->assertSame($context->requestId, $view['root']['request_id']); + $this->assertSame($selected->id, $view['runs'][0]['run_id']); + $this->assertStringNotContainsString($replacement->id, json_encode($view, JSON_THROW_ON_ERROR)); + } + + public function testTerminalStatusWithoutTerminalHistoryCannotClaimFinishedCleanup(): void + { + [$parent, $child] = $this->start(); + $parent->requestCancellation('maintenance', 30); + $this->runTask($parent, TaskType::Workflow); + $this->runTask($child, TaskType::Workflow); + Carbon::setTestNow(now()->addSeconds(2)); + $this->runTask($child, TaskType::Timer); + $this->runTask($child, TaskType::Workflow); + $this->runTask($parent, TaskType::Workflow); + WorkflowHistoryEvent::query()->where('workflow_run_id', $parent->runId()) + ->where('event_type', HistoryEventType::WorkflowCancelled->value)->delete(); + + $view = CancellationCascadeView::forRun($parent->run()->fresh()); + $this->assertNotNull($view); + $this->assertFalse($view['inspection_complete']); + $this->assertSame('unknown', $view['runs'][0]['lifecycle']); + $this->assertNull($view['runs'][0]['cleanup']); + $this->assertContains('terminal_history_unavailable', array_column($view['findings'], 'code')); + } + + public function testAnExceededHistoryWindowDoesNotClaimCompleteEvidence(): void + { + [$parent] = $this->start(); + $context = $parent->requestCancellation('maintenance', 30) + ->cancellationContext(); + $this->assertNotNull($context); + for ($index = 0; $index < 129; ++$index) { + WorkflowHistoryEvent::record($parent->run()->fresh(), HistoryEventType::ActivityRetryScheduled, []); + } + + $view = CancellationCascadeView::forRun($parent->run()->fresh()); + $this->assertNotNull($view); + $this->assertTrue($view['truncated']); + $this->assertFalse($view['inspection_complete']); + $this->assertContains('history_limit', array_column($view['findings'], 'code')); + $this->assertSame($context->deadline()->toISOString(), $view['root']['cleanup_deadline_at']); + $this->assertSame(128, $view['limits']['history_events_per_run']); + } + + /** + * @return array{WorkflowStub, WorkflowStub} + */ + private function start(): array + { + $parent = WorkflowStub::make(TestParentChildPolicyWorkflow::class); + $parent->start(CancellationPolicy::WaitCancellationCompleted->value); + $this->runTask($parent, TaskType::Workflow); + $link = WorkflowLink::query()->where('parent_workflow_run_id', $parent->runId())->sole(); + $child = WorkflowStub::loadRun($link->child_workflow_run_id); + $this->runTask($child, TaskType::Workflow); + return [$parent, $child]; + } + + private function recordScopedChild(WorkflowStub $child, ScopedCancellationContext $scope): CancellationContext + { + $context = CancellationContext::fromScopeContext( + $scope, + 'scoped-child-request', + $child->id(), + $child->runId(), + $scope->requestedAt() + ->addSeconds(15) + ); + $child->run() + ->historyEvents() + ->create([ + 'sequence' => $child->run() + ->historyEvents() + ->max('sequence') + 1, + 'event_type' => HistoryEventType::CooperativeCancellationRequested, + 'workflow_command_id' => $context->requestId, + 'recorded_at' => now(), + 'payload' => [ + 'workflow_command_id' => $context->requestId, + 'workflow_run_id' => $child->runId(), + 'workflow_instance_id' => $child->id(), + 'reason' => $context->reason, + 'cleanup_deadline_at' => $context->deadline() + ->toISOString(), + 'cancellation' => $context->toArray(), + ], + ]); + $child->run() + ->forceFill([ + 'cancellation_request_command_id' => $context->requestId, + 'cancellation_requested_at' => now(), + 'cancellation_deadline_at' => $context->deadline(), + ])->save(); + + return $context; + } + + private function runTask(WorkflowStub $workflow, TaskType $type): void + { + $task = WorkflowTask::query()->where('workflow_run_id', $workflow->runId()) + ->where('task_type', $type->value) + ->where('status', TaskStatus::Ready->value) + ->orderBy('created_at') + ->firstOrFail(); + $job = $type === TaskType::Timer ? new RunTimerTask($task->id) : new RunWorkflowTask($task->id); + $this->app->call([$job, 'handle']); + } +} diff --git a/tests/Feature/V2/V2CancellationCleanupOutcomeTest.php b/tests/Feature/V2/V2CancellationCleanupOutcomeTest.php new file mode 100644 index 000000000..26f3af31a --- /dev/null +++ b/tests/Feature/V2/V2CancellationCleanupOutcomeTest.php @@ -0,0 +1,226 @@ + 'database', + 'workflows.v2.workflow_task_lease_seconds' => 300, + ]); + Queue::fake(); + + try { + $workflow = WorkflowStub::make(TestElapsedLocalCleanupWorkflow::class); + $workflow->start(); + $runId = $workflow->runId(); + $this->assertIsString($runId); + $this->runTask($runId, TaskType::Workflow); + $request = $workflow->requestCancellation('embedded local cleanup', 30); + $deadline = $request->cancellationContext()?->deadline() + ->toISOString(); + $this->runTask($runId, TaskType::Workflow); + + $this->assertTrue($workflow->refresh()->cancelled()); + $this->assertSame([ + 'cleanup' => 'cleaned', + ], $workflow->memo()); + $outcome = $this->terminal($runId) +->payload['cancellation_cleanup']; + $this->assertSame('completed', $outcome['outcome']); + $this->assertSame($deadline, $outcome['cleanup_deadline_at']); + $started = WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::ActivityStarted)->sole(); + $this->assertSame($deadline, $started->payload['lease_expires_at']); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::ActivityHeartbeatRecorded)->count()); + $this->assertSame('2026-10-03T08:00:11.000000Z', now()->toISOString()); + } finally { + Carbon::setTestNow(); + } + } + + public function testLegacyTerminalCancellationDoesNotClaimCooperativeCleanup(): void + { + config([ + 'queue.default' => 'database', + ]); + Queue::fake(); + $workflow = WorkflowStub::make(TestCooperativeWaitCleanupWorkflow::class); + $workflow->start('signal'); + $runId = $workflow->runId(); + $this->assertIsString($runId); + $this->runTask($runId, TaskType::Workflow); + + $this->assertTrue($workflow->cancel('legacy terminal command')->accepted()); + $this->assertTrue($workflow->refresh()->cancelled()); + $this->assertArrayNotHasKey('cancellation_cleanup', $this->terminal($runId)->payload); + $this->assertSame([], $workflow->memo()); + } + + public function testRequestDuringNormalShieldedCleanupDoesNotInventCancellationDelivery(): void + { + config([ + 'queue.default' => 'database', + ]); + Queue::fake(); + $workflow = WorkflowStub::make(TestCooperativeNormalCleanupWorkflow::class); + $workflow->start(); + $runId = $workflow->runId(); + $this->assertIsString($runId); + $this->runTask($runId, TaskType::Workflow); + $this->runTask($runId, TaskType::Activity); + $this->runTask($runId, TaskType::Workflow); + $request = $workflow->requestCancellation('normal cleanup already started', 30); + $this->runTask($runId, TaskType::Workflow); + $this->runTask($runId, TaskType::Activity); + $this->runTask($runId, TaskType::Workflow); + + $terminal = $this->terminal($runId); + $outcome = $terminal->payload['cancellation_cleanup'] ?? null; + $this->assertIsArray($outcome); + $this->assertSame('not_delivered', $outcome['outcome']); + $this->assertSame($request->commandId(), $outcome['request_id']); + $this->assertSame($request->cancellationContext()?->deadline()->toISOString(), $outcome['cleanup_deadline_at']); + $this->assertNull($outcome['delivery_history_event_id']); + $this->assertNull($outcome['delivery_sequence']); + $workflow->refresh(); + $this->assertSame([ + 'cleanup' => 'Hello, cleanup!', + ], $workflow->memo()); + $this->assertTrue($workflow->cancelled()); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered->value)->count()); + } + + public function testCompletedCleanupRetainsItsCanonicalDeliveryAndOriginalBudgetAfterDuplicate(): void + { + config([ + 'queue.default' => 'database', + ]); + Queue::fake(); + $workflow = WorkflowStub::make(TestCooperativeWaitCleanupWorkflow::class); + $workflow->start('signal'); + $runId = $workflow->runId(); + $this->assertIsString($runId); + $this->runTask($runId, TaskType::Workflow); + $request = $workflow->requestCancellation('completed cleanup', 30); + $context = $request->cancellationContext(); + $this->assertNotNull($context); + $this->runTask($runId, TaskType::Workflow); + $this->runTask($runId, TaskType::Activity); + $this->runTask($runId, TaskType::Workflow); + + $terminal = $this->terminal($runId); + $delivery = WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered->value)->sole(); + $outcome = $terminal->payload['cancellation_cleanup'] ?? null; + $this->assertIsArray($outcome); + $this->assertSame('completed', $outcome['outcome']); + $this->assertSame($request->commandId(), $outcome['request_id']); + $this->assertSame($context->deadline()->toISOString(), $outcome['cleanup_deadline_at']); + $this->assertSame($delivery->id, $outcome['delivery_history_event_id']); + $this->assertSame($delivery->payload['sequence'], $outcome['delivery_sequence']); + $this->assertSame($workflow->run()->fresh()->closed_at->toISOString(), $outcome['finished_at']); + $this->assertTrue($terminal->recorded_at->lt($context->deadline())); + $workflow->refresh(); + $this->assertSame([ + 'cleanup' => 'Hello, cleanup!', + ], $workflow->memo()); + $this->assertTrue($workflow->cancelled()); + + $duplicate = $workflow->requestCancellation('must not replace the outcome', 300); + $this->assertSame($request->commandId(), $duplicate->commandId()); + $this->assertSame($context->toArray(), $duplicate->cancellationContext()?->toArray()); + $this->assertSame($terminal->payload, $this->terminal($runId)->payload); + } + + #[DataProvider('deliveryStates')] + public function testExpiryAtOriginalDeadlineNeverClaimsCleanupCompletion(bool $deliver): void + { + config([ + 'queue.default' => 'database', + ]); + Queue::fake(); + $workflow = WorkflowStub::make(TestCooperativeWaitCleanupWorkflow::class); + $workflow->start('signal'); + $runId = $workflow->runId(); + $this->assertIsString($runId); + $this->runTask($runId, TaskType::Workflow); + $request = $workflow->requestCancellation('expire cleanup', 30); + $context = $request->cancellationContext(); + $this->assertNotNull($context); + if ($deliver) { + $this->runTask($runId, TaskType::Workflow); + } + + Carbon::setTestNow($context->deadline()); + try { + $report = TaskWatchdog::runPass(respectThrottle: false, runIds: [$runId]); + $this->assertSame(1, $report['cancellation_deadlines_enforced']); + $terminal = $this->terminal($runId); + $outcome = $terminal->payload['cancellation_cleanup'] ?? null; + $this->assertIsArray($outcome); + $this->assertSame('deadline_expired', $outcome['outcome']); + $this->assertSame($request->commandId(), $outcome['request_id']); + $this->assertSame($context->deadline()->toISOString(), $outcome['cleanup_deadline_at']); + $this->assertSame($context->deadline()->toISOString(), $outcome['finished_at']); + $this->assertTrue($workflow->refresh()->cancelled()); + $this->assertSame([], $workflow->memo()); + $delivery = WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered->value)->first(); + $this->assertSame($delivery?->id, $outcome['delivery_history_event_id']); + $this->assertSame($delivery?->payload['sequence'] ?? null, $outcome['delivery_sequence']); + $this->assertSame($deliver, $delivery !== null); + $again = TaskWatchdog::runPass(respectThrottle: false, runIds: [$runId]); + $this->assertSame(0, $again['cancellation_deadlines_enforced']); + $this->assertSame($terminal->payload, $this->terminal($runId)->payload); + } finally { + Carbon::setTestNow(); + } + } + + public static function deliveryStates(): array + { + return [[false], [true]]; + } + + private function terminal(string $runId): WorkflowHistoryEvent + { + return WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::WorkflowCancelled->value)->sole(); + } + + private function runTask(string $runId, TaskType $type): void + { + $task = WorkflowTask::query()->where('workflow_run_id', $runId) + ->where('task_type', $type->value) + ->where('status', TaskStatus::Ready->value) + ->orderBy('created_at') + ->firstOrFail(); + $job = $type === TaskType::Activity ? new RunActivityTask($task->id) : new RunWorkflowTask($task->id); + $this->app->call([$job, 'handle']); + } +} diff --git a/tests/Feature/V2/V2CancellationRequestConcurrencyTest.php b/tests/Feature/V2/V2CancellationRequestConcurrencyTest.php new file mode 100644 index 000000000..c3688a981 --- /dev/null +++ b/tests/Feature/V2/V2CancellationRequestConcurrencyTest.php @@ -0,0 +1,310 @@ + 'poll', + 'queue.default' => 'database', + ]); + + $driver = DB::connection()->getDriverName(); + if (! in_array($driver, ['mysql', 'pgsql'], true) + || ($driver === 'mysql' && str_contains( + strtolower((string) DB::selectOne('SELECT VERSION() AS version')->version), + 'mariadb' + ))) { + $this->markTestSkipped('MySQL or PostgreSQL row-lock observation is required.'); + } + if (! function_exists('pcntl_fork') || ! function_exists('posix_kill') + || ! function_exists('stream_socket_pair')) { + $this->markTestSkipped('Process control and local sockets are required.'); + } + Carbon::setTestNow('2026-10-03T00:00:00Z'); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + public function testConcurrentDirectDuplicateKeepsOriginalIdentityAndBudget(): void + { + [$winner, $loser, $child] = $this->raceRequests(false, false); + $this->assertTrue($winner['accepted']); + $this->assertTrue($loser['accepted']); + $this->assertSame($winner['command_id'], $loser['command_id']); + $this->assertSameJsonObject($winner['context'], $loser['context']); + $this->assertSame('winner', $loser['context']['reason']); + $this->assertSame('2026-10-03T00:00:25.000000Z', $loser['context']['cleanup_deadline_at']); + $this->assertCanonicalRequest($child, $winner['context']); + } + + public function testConcurrentDirectDuplicateKeepsInheritedRootAndDeadline(): void + { + [$winner, $loser, $child, $parentContext] = $this->raceRequests(true, false); + $this->assertTrue($winner['accepted']); + $this->assertTrue($loser['accepted']); + $this->assertSame($winner['command_id'], $loser['command_id']); + $this->assertSameJsonObject($winner['context'], $loser['context']); + $this->assertSame($parentContext['root_request_id'], $loser['context']['root_request_id']); + $this->assertSame($parentContext['request_id'], $loser['context']['parent_request_id']); + $this->assertSame($parentContext['cleanup_deadline_at'], $loser['context']['cleanup_deadline_at']); + $this->assertSame($parentContext['requested_at'], $loser['context']['requested_at']); + $this->assertCanonicalRequest($child, $winner['context']); + } + + public function testConcurrentParentPropagationReportsConflictWithoutReplacingDirectRoot(): void + { + [$winner, $loser, $child, $parentContext] = $this->raceRequests(false, true); + $this->assertTrue($winner['accepted']); + $this->assertFalse($loser['accepted']); + $this->assertSame('cancellation_root_conflict', $loser['rejection_reason']); + $this->assertNull($loser['context']); + $this->assertNotSame($winner['context']['root_request_id'], $parentContext['root_request_id']); + $this->assertSameJsonObject([ + 'existing_request_id' => $winner['command_id'], + 'existing_root_request_id' => $winner['context']['root_request_id'], + 'existing_cleanup_deadline_at' => $winner['context']['cleanup_deadline_at'], + 'incoming_root_request_id' => $parentContext['root_request_id'], + 'incoming_cleanup_deadline_at' => $parentContext['cleanup_deadline_at'], + ], $loser['conflict']); + $this->assertSame(1, WorkflowCommand::query()->where('workflow_run_id', $child->runId()) + ->where('command_type', CommandType::RequestCancellation->value) + ->where('status', CommandStatus::Rejected->value)->count()); + $duplicate = $child->requestCancellation('later direct request', 300); + $this->assertSame($winner['command_id'], $duplicate->commandId()); + $this->assertSameJsonObject($winner['context'], $duplicate->cancellationContext()?->toArray()); + $this->assertCanonicalRequest($child, $winner['context']); + } + + /** + * @return array{array, array, WorkflowStub, array} + */ + private function raceRequests(bool $winnerInherited, bool $loserInherited): array + { + $parent = WorkflowStub::make(TestParentChildPolicyWorkflow::class); + $parent->start(CancellationPolicy::WaitCancellationCompleted->value); + $this->runWorkflowTask($parent); + $link = WorkflowLink::query()->where('parent_workflow_run_id', $parent->runId())->sole(); + $child = WorkflowStub::loadRun($link->child_workflow_run_id); + $this->runWorkflowTask($child); + $parentContext = $parent->requestCancellation('parent request', 30) + ->cancellationContext() + ->toArray(); + Carbon::setTestNow(now()->addSeconds(5)); + $childRunId = $child->runId(); + $parentRunId = $parent->runId(); + $operations = []; + DB::purge(); + + try { + $operations['winner'] = $this->forkRequest($childRunId, $parentRunId, $winnerInherited, true); + $this->assertIsArray($this->readMessage($operations['winner'])); + $operations['loser'] = $this->forkRequest($childRunId, $parentRunId, $loserInherited, false); + $loserReady = $this->readMessage($operations['loser']); + $this->assertIsInt($loserReady['connection_id']); + fwrite($operations['winner']['socket'], "go\n"); + $uncommitted = $this->readMessage($operations['winner']); + $this->assertTrue($uncommitted['accepted']); + fwrite($operations['loser']['socket'], "go\n"); + $this->awaitBlockedConnection($loserReady['connection_id']); + + // An actual database waiter has reached the locked request. Neither + // a sleep nor a second sequential call stands in for this race. + $this->assertNull($child->run()->fresh()->cancellation_request_command_id); + $this->assertSame(0, $child->run()->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationRequested)->count()); + fwrite($operations['winner']['socket'], "commit\n"); + $winner = $this->finishRequest($operations['winner']); + unset($operations['winner']); + $loser = $this->finishRequest($operations['loser']); + unset($operations['loser']); + } finally { + foreach ($operations as $operation) { + posix_kill($operation['pid'], SIGKILL); + pcntl_waitpid($operation['pid'], $status); + fclose($operation['socket']); + } + DB::purge(); + DB::reconnect(); + } + + $this->assertSameJsonObject( + $parentContext, + CooperativeCancellationDelivery::context($parent->run()->fresh())->toArray() + ); + return [$winner, $loser, $child, $parentContext]; + } + + /** + * @return array{pid: int, socket: resource} + */ + private function forkRequest(string $childRunId, string $parentRunId, bool $inherited, bool $holdCommit): array + { + $sockets = stream_socket_pair(STREAM_PF_UNIX, STREAM_SOCK_STREAM, STREAM_IPPROTO_IP); + $this->assertIsArray($sockets); + $pid = pcntl_fork(); + $this->assertNotSame(-1, $pid); + if ($pid === 0) { + fclose($sockets[0]); + stream_set_timeout($sockets[1], 10); + $ok = false; + try { + DB::reconnect(); + $query = DB::connection()->getDriverName() === 'mysql' + ? 'SELECT CONNECTION_ID() AS connection_id' : 'SELECT pg_backend_pid() AS connection_id'; + fwrite($sockets[1], json_encode([ + 'connection_id' => (int) DB::selectOne($query)->connection_id, + ], JSON_THROW_ON_ERROR) . PHP_EOL); + if (fgets($sockets[1]) !== "go\n") { + throw new RuntimeException('Cancellation actor was not released.'); + } + if ($holdCommit) { + DB::beginTransaction(); + } + $stub = WorkflowStub::loadRun($childRunId); + $result = $inherited ? $stub->attemptRequestCancellationFromParent($parentRunId) + : $stub->requestCancellation($holdCommit ? 'winner' : 'duplicate', $holdCommit ? 20 : 300); + $observation = [ + 'accepted' => $result->accepted(), + 'command_id' => $result->commandId(), + 'rejection_reason' => $result->rejectionReason(), + 'context' => $result->cancellationContext()?->toArray(), + 'conflict' => $result->payloadValues([ + 'existing_request_id', 'existing_root_request_id', 'existing_cleanup_deadline_at', + 'incoming_root_request_id', 'incoming_cleanup_deadline_at', + ]), + ]; + if ($holdCommit) { + fwrite($sockets[1], json_encode($observation, JSON_THROW_ON_ERROR) . PHP_EOL); + if (fgets($sockets[1]) !== "commit\n") { + throw new RuntimeException('Cancellation commit was not released.'); + } + DB::commit(); + } + fwrite($sockets[1], json_encode([ + 'result' => $observation, + ], JSON_THROW_ON_ERROR) . PHP_EOL); + $ok = true; + } catch (Throwable $error) { + fwrite($sockets[1], json_encode([ + 'error' => $error::class . ': ' . $error->getMessage(), + ], JSON_THROW_ON_ERROR) . PHP_EOL); + } finally { + DB::disconnect(); + fclose($sockets[1]); + } + exit($ok ? 0 : 1); + } + fclose($sockets[1]); + stream_set_timeout($sockets[0], 10); + return [ + 'pid' => $pid, + 'socket' => $sockets[0], + ]; + } + + private function awaitBlockedConnection(int $connectionId): void + { + $query = DB::connection()->getDriverName() === 'mysql' + ? 'SELECT COUNT(*) AS waiting FROM performance_schema.data_lock_waits w JOIN performance_schema.threads t ON t.THREAD_ID = w.REQUESTING_THREAD_ID WHERE t.PROCESSLIST_ID = ?' + : 'SELECT COUNT(*) AS waiting FROM pg_locks WHERE pid = ? AND NOT granted'; + $until = microtime(true) + 5; + do { + if ((int) DB::selectOne($query, [$connectionId])->waiting > 0) { + $this->addToAssertionCount(1); + return; + } + usleep(10_000); + } while (microtime(true) < $until); + $this->fail('The losing cancellation request did not wait on the database lock.'); + } + + /** @param array{pid: int, socket: resource} $operation + * @return array + */ + private function readMessage(array $operation): array + { + $line = fgets($operation['socket']); + $this->assertIsString($line, 'Cancellation actor did not respond.'); + $message = json_decode($line, true, flags: JSON_THROW_ON_ERROR); + $this->assertIsArray($message); + $this->assertArrayNotHasKey('error', $message, $message['error'] ?? ''); + return $message; + } + + /** @param array{pid: int, socket: resource} $operation + * @return array + */ + private function finishRequest(array $operation): array + { + $message = $this->readMessage($operation); + pcntl_waitpid($operation['pid'], $status); + $this->assertTrue(pcntl_wifexited($status)); + $this->assertSame(0, pcntl_wexitstatus($status)); + fclose($operation['socket']); + $this->assertIsArray($message['result']); + return $message['result']; + } + + /** + * @param array $context + */ + private function assertCanonicalRequest(WorkflowStub $child, array $context): void + { + $run = $child->run() + ->fresh(); + $this->assertSame($context['request_id'], $run->cancellation_request_command_id); + $this->assertSame($context['cleanup_deadline_at'], $run->cancellation_deadline_at->toISOString()); + $this->assertSame(RunStatus::Waiting, $run->status); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationRequested)->count() + ); + $this->assertSameJsonObject($context, CooperativeCancellationDelivery::context($run)->toArray()); + $this->assertSame(1, WorkflowCommand::query()->where('workflow_run_id', $run->id) + ->where('command_type', CommandType::RequestCancellation->value) + ->where('status', CommandStatus::Accepted->value)->count()); + $this->assertSame(1, WorkflowTask::query()->where('workflow_run_id', $run->id) + ->where('task_type', TaskType::Workflow->value)->where('status', TaskStatus::Ready->value)->count()); + } + + private function runWorkflowTask(WorkflowStub $workflow): void + { + $task = WorkflowTask::query()->where('workflow_run_id', $workflow->runId()) + ->where('task_type', TaskType::Workflow->value)->where('status', TaskStatus::Ready->value)->sole(); + $this->app->call([new RunWorkflowTask($task->id), 'handle']); + } +} diff --git a/tests/Feature/V2/V2CancellationScopeDeliveryTest.php b/tests/Feature/V2/V2CancellationScopeDeliveryTest.php new file mode 100644 index 000000000..4e4f7785f --- /dev/null +++ b/tests/Feature/V2/V2CancellationScopeDeliveryTest.php @@ -0,0 +1,2586 @@ + 'poll', + ]); + Carbon::setTestNow('2026-10-03T00:00:00Z'); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + #[DataProvider('callKinds')] + public function testUnscheduledCallRecordsScopeBoundaryWithoutRunCancellationOrClaimReplacement(string $kind): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30, 'release one scope'); + $claimBefore = $task->fresh() + ->getAttributes(); + $event = $this->deliver($run, $task, $scope, $kind); + $this->assertSame(HistoryEventType::CancellationScopeDelivered, $event->event_type); + $this->assertSame($request->payload['request_id'], $event->payload['request_id']); + $this->assertSame($scope, $event->payload['scope_id']); + $this->assertSame(3, $event->payload['sequence']); + $this->assertSame(4, WorkflowStepHistory::nextDurableCommandSequence($run->fresh())); + $this->assertSameJsonObject($request->payload['cancellation'], $event->payload['cancellation']); + $this->assertSame('2026-10-03T00:00:30.000000Z', $event->payload['authority_deadline_at']); + $this->assertHostingShortenedWithoutClaimReplacement($claimBefore, $task); + $this->assertSame(1, $run->tasks()->count()); + $this->assertNull($run->fresh()->cancellation_request_command_id); + $this->assertNull($run->fresh()->cancellation_delivery_sequence); + $this->assertNull($run->fresh()->cancellation_delivered_at); + $this->assertFalse($run->fresh()->status->isTerminal()); + $timeline = collect(HistoryTimeline::fromHistory($run->fresh()))->firstWhere('id', $event->id); + $this->assertSame('cancellation_scope', $timeline['kind']); + $this->assertSameJsonObject($event->payload['cancellation'], $timeline['cancellation_scope']['cancellation']); + $this->assertSame(3, $timeline['cancellation_scope']['sequence']); + } + + public static function callKinds(): iterable + { + foreach (['activity', 'local_activity', 'timer', 'condition', 'signal', 'child'] as $kind) { + yield $kind => [$kind]; + } + } + + #[DataProvider('rootDeliveryScopes')] + public function testRunDeliveryCannotBypassScopedOperationMembership( + bool $shield, + string $kind, + string $boundary + ): void { + [, $run, $task, , $scope] = $this->scopeTree($shield); + $type = $kind === 'activity' ? HistoryEventType::ActivityScheduled : HistoryEventType::TimerScheduled; + $this->schedule($run, $type, [ + 'sequence' => 3, + ...($kind === 'activity' ? [ + 'activity' => [ + 'cancellation_scope_id' => $scope, + ], + ] + : [ + 'cancellation_scope_id' => $scope, + ]), + ...ParallelChildGroup::itemMetadata(3, 2, 0, $kind), + ]); + $this->schedule($run, $type, [ + 'sequence' => 4, + ...ParallelChildGroup::itemMetadata(3, 2, 1, $kind), + ]); + $this->assertTrue(WorkflowStub::loadRun($run->id)->requestCancellation('stop the run', 30)->accepted()); + $run->refresh(); + $before = [ + $run->getAttributes(), $run->historyEvents() + ->get() + ->toArray(), + $run->tasks() + ->get() + ->toArray(), $run->activityExecutions() + ->get() + ->toArray(), + $run->timers() + ->get() + ->toArray(), + ]; + $result = app(DefaultWorkflowTaskBridge::class)->deliverCancellation( + $task->id, + $run->cancellation_request_command_id, + $boundary === 'selection_handle' ? 5 : 3, + $boundary === 'scalar' ? $kind : $boundary, + $boundary === 'parallel' ? 2 : 1, + $boundary === 'selection_handle' ? 3 : null, + $boundary === 'selection_handle' ? 2 : 1, + ); + $this->assertFalse($result['delivered']); + $this->assertSame('cancellation_delivery_requires_scope', $result['reason']); + $this->assertSame($before, [ + $run->fresh() + ->getAttributes(), $run->historyEvents() + ->get() + ->toArray(), + $run->tasks() + ->get() + ->toArray(), $run->activityExecutions() + ->get() + ->toArray(), + $run->timers() + ->get() + ->toArray(), + ]); + } + + public static function rootDeliveryScopes(): iterable + { + foreach ([false, true] as $shield) { + foreach (['activity', 'timer'] as $kind) { + foreach (['scalar', 'parallel', 'selection_handle'] as $boundary) { + yield ($shield ? 'shielded ' : 'unshielded ') . $kind . ' ' . $boundary => [ + $shield, + $kind, + $boundary, + ]; + } + } + } + } + + public function testPreparedPendingDeliveryRecoversThroughWatchdogWithoutReplacingOriginalBoundary(): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $claim = $task->fresh() + ->getAttributes(); + $prepared = CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + 3, + 'local_activity', + '1.20' + ); + $this->assertHostingShortenedWithoutClaimReplacement($claim, $task); + $this->assertNull(CancellationScopeDelivery::recorded($run->fresh(), $scope)); + $this->assertSame( + '2026-10-03T00:00:30.000000Z', + $run->fresh() + ->cancellation_scope_recovery_until->toISOString() + ); + Carbon::setTestNow('2026-10-03T00:00:10Z'); + $this->assertRefusedWithoutMutation($run, $task, 'cancellation_scope_workflow_claim_mismatch', static fn () => + CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + 3, + 'local_activity', + '1.20' + )); + Carbon::setTestNow('2026-10-03T00:00:11Z'); + $repair = TaskWatchdog::runPass(respectThrottle: false, runIds: [$run->id]); + $this->assertSame([], $repair['existing_task_failures']); + $this->assertSame(1, $repair['repaired_existing_tasks']); + $this->assertTrue(app(DefaultWorkflowTaskBridge::class)->claimStatus($task->id, 'replacement')['claimed']); + $replacement = $task->fresh(); + $this->assertSame(2, $replacement->attempt_count); + $this->assertSame('2026-10-03T00:00:21.000000Z', $replacement->lease_expires_at->toISOString()); + $replayed = CancellationScopeDelivery::prepare( + $run->fresh(), + $replacement, + $scope, + $request->payload['request_id'], + 3, + 'local_activity', + '1.20' + ); + $this->assertSame($prepared->id, $replayed->id); + $this->assertSameJsonObject($prepared->payload, $replayed->payload); + $this->assertSame($prepared->recorded_at->toISOString(), $replayed->recorded_at->toISOString()); + $this->assertNull(CancellationScopeDelivery::recorded($run->fresh(), $scope)); + $this->assertRefusedWithoutMutation($run, $task, 'cancellation_scope_workflow_claim_mismatch', static fn () => + CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + 3, + 'local_activity', + '1.20' + )); + $duplicate = CancellationScopeRequests::request($run->fresh(), $scope, '1.20', 300); + $this->assertSame($request->id, $duplicate->id); + $this->assertSameJsonObject($request->payload, $duplicate->payload); + $this->assertNull($run->fresh()->cancellation_request_command_id); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeDeliveryPrepared)->count() + ); + } + + #[DataProvider('preparationHostingIntervals')] + public function testPreparationUsesConfiguredRecoveryIntervalWithoutClampingSharedClaimToScopeBudget( + int $budget, + int $configuredLease, + int $expectedInterval, + ): void { + [, $run, $task, , $scope] = $this->scopeTree(); + config() + ->set('workflows.v2.workflow_task_lease_seconds', $configuredLease); + $request = CancellationScopeRequests::request($run, $scope, '1.20', $budget); + $before = $task->fresh() + ->getAttributes(); + $prepared = CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + 3, + 'local_activity', + '1.20' + ); + $claim = $task->fresh(); + $this->assertSame(now()->addSeconds($expectedInterval)->toISOString(), $claim->lease_expires_at->toISOString()); + $after = $claim->getAttributes(); + unset($before['lease_expires_at'], $after['lease_expires_at'], $before['updated_at'], $after['updated_at']); + $this->assertSame($before, $after); + $this->assertSame(now()->addSeconds($budget)->toISOString(), $prepared->payload['authority_deadline_at']); + $this->assertNull(CancellationScopeDelivery::recorded($run->fresh(), $scope)); + $this->assertNull($run->fresh()->cancellation_request_command_id); + } + + public static function preparationHostingIntervals(): iterable + { + yield 'ordinary lease' => [30, 60, 10]; + yield 'smaller configured lease' => [30, 5, 5]; + yield 'short subtree budget preserves shared ownership' => [3, 60, 10]; + } + + public function testResponseLossAndReplacementClaimReplayOriginalBoundaryAndClockWithoutNewBudget(): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + CancellationScopeRequests::request($run, $scope, '1.20', 30); + $event = $this->deliver($run, $task, $scope); + $original = $event->payload; + Carbon::setTestNow('2026-10-03T00:00:11Z'); + $replacement = $task->fresh(); + $replacement->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 2, + 'lease_expires_at' => now() + ->addSeconds(10), + ])->save(); + $replayed = $this->deliver($run->fresh(), $replacement, $scope); + $this->assertSame($event->id, $replayed->id); + $this->assertSameJsonObject($original, $replayed->payload); + $this->assertSame('2026-10-03T00:00:00.000000Z', $replayed->recorded_at->toISOString()); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeDelivered)->count() + ); + $this->expectExceptionMessage('cancellation_scope_workflow_claim_mismatch'); + $this->deliver($run->fresh(), $task, $scope); + } + + #[DataProvider('changedBoundaries')] + public function testDuplicateCannotMoveOrRelabelOriginalBoundary(int $sequence, string $kind, int $span): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + $request = CancellationScopeRequests::request($run, $scope, '1.20'); + $this->deliver($run, $task, $scope); + $this->assertRefusedWithoutMutation($run, $task, 'cancellation_scope_delivery_mismatch', static fn () => + CancellationScopeDelivery::record( + $run, + $task, + $scope, + $request->payload['request_id'], + $sequence, + $kind, + '1.20', + $span + )); + } + + public static function changedBoundaries(): iterable + { + yield 'move earlier' => [2, 'activity', 1]; + yield 'move later' => [4, 'activity', 1]; + yield 'change call' => [3, 'timer', 1]; + yield 'change range' => [3, 'parallel', 2]; + } + + public function testScopeAddressesHaveIndependentDeliveryRecordsAndIdentities(): void + { + [, $run, $task, $parent, $child] = $this->scopeTree(); + CancellationScopeRequests::request($run, $parent, '1.20', 30); + $first = $this->deliver($run, $task, $parent); + CancellationScopeRequests::request($run, $child, '1.20', 30); + $request = CancellationScopeRequests::context($run, $child); + CancellationScopeDelivery::prepare($run, $task, $child, $request->requestId, 4, 'timer', '1.20'); + $second = CancellationScopeDelivery::record($run, $task, $child, $request->requestId, 4, 'timer', '1.20'); + $this->assertNotSame($first->id, $second->id); + $this->assertNotSame($first->payload['request_id'], $second->payload['request_id']); + $this->assertSame($first->id, CancellationScopeDelivery::recorded($run->fresh(), $parent)->id); + $this->assertSame($second->id, CancellationScopeDelivery::recorded($run->fresh(), $child)->id); + $this->assertSame(5, WorkflowStepHistory::nextDurableCommandSequence($run->fresh())); + } + + #[DataProvider('inheritedShields')] + public function testInheritedRequestRetainsOriginalRootAndObeysParentShield(bool $shield): void + { + [, $run, $task, $parent, $child] = $this->scopeTree($shield); + $root = CancellationScopeRequests::request($run, $parent, '1.20', 30, 'original'); + Carbon::setTestNow('2026-10-03T00:00:07Z'); + CancellationScopeRequests::request($run, $child, '1.20', 300, parentScopeId: $parent); + if ($shield) { + $this->assertRefusedWithoutMutation($run, $task, 'cancellation_scope_parent_shielded', fn () => + $this->deliver($run, $task, $child)); + return; + } + $event = $this->deliver($run, $task, $child); + $this->assertSame( + $root->payload['request_id'], + $event->payload['cancellation']['root_context']['root_request_id'] + ); + $this->assertSame([$parent, $child], array_column($event->payload['cancellation']['lineage'], 'scope_id')); + $this->assertSame('2026-10-03T00:00:30.000000Z', $event->payload['authority_deadline_at']); + } + + public static function inheritedShields(): iterable + { + yield 'unshielded' => [false]; + yield 'shielded' => [true]; + } + + public function testDirectShieldRequestDeliversButCurrentAncestorCeilingStillFencesRetries(): void + { + [$workflow, $run, $task, , $scope] = $this->scopeTree(true); + $accepted = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $workflow->requestCancellation('run ceiling', 12); + $event = $this->deliver($run, $task, $scope); + $this->assertSameJsonObject($accepted->payload['cancellation'], $event->payload['cancellation']); + $this->assertSame('2026-10-03T00:00:12.000000Z', $event->payload['authority_deadline_at']); + Carbon::setTestNow('2026-10-03T00:00:09Z'); + $this->assertTrue(app(DefaultWorkflowTaskBridge::class)->heartbeat($task->id)['renewed']); + $this->assertSame('2026-10-03T00:00:12.000000Z', $task->fresh()->lease_expires_at->toISOString()); + Carbon::setTestNow('2026-10-03T00:00:12Z'); + $this->assertRefusedWithoutMutation($run, $task, 'cancellation_scope_workflow_claim_mismatch', fn () => + $this->deliver($run, $task, $scope)); + $this->assertSame($event->id, CancellationScopeDelivery::recorded($run->fresh(), $scope)->id); + } + + public function testLaterAncestorDeadlineDoesNotRewriteAnEarlierDeliveryReceipt(): void + { + [$workflow, $run, $task, , $scope] = $this->scopeTree(); + CancellationScopeRequests::request($run, $scope, '1.20', 30); + $event = $this->deliver($run, $task, $scope); + Carbon::setTestNow('2026-10-03T00:00:01Z'); + $workflow->requestCancellation('shorter run budget', 5); + $duplicate = $this->deliver($run, $task, $scope); + $this->assertSame($event->id, $duplicate->id); + $this->assertSame('2026-10-03T00:00:30.000000Z', $duplicate->payload['authority_deadline_at']); + $this->assertSame( + '2026-10-03T00:00:06.000000Z', + CancellationScopeRequests::authority($run, $scope)['deadline_at'] + ); + $this->assertSame('2026-10-03T00:00:06.000000Z', $task->fresh()->lease_expires_at->toISOString()); + Carbon::setTestNow('2026-10-03T00:00:06Z'); + $this->assertRefusedWithoutMutation($run, $task, 'cancellation_scope_workflow_claim_mismatch', fn () => + $this->deliver($run, $task, $scope)); + } + + #[DataProvider('memberships')] + public function testRecordedOperationMustBelongToTheExactScope(string $layout, bool $valid): void + { + [, $run, $task, $parent, $scope] = $this->scopeTree(); + $address = $valid ? $scope : $parent; + $payload = [ + 'sequence' => 3, + ]; + if ($layout !== 'nested') { + $payload['cancellation_scope_id'] = $address; + } + if ($layout !== 'flat') { + $payload['activity'] = [ + 'cancellation_scope_id' => $layout === 'contradictory' ? $parent : $address, + ]; + } + if ($layout === 'contradictory') { + // Deliberately inject contradictory stored metadata, bypassing the + // normal admission schema, to exercise the cold-history fence. + $event = WorkflowHistoryEvent::record($run, HistoryEventType::ActivityScheduled, [ + 'sequence' => 3, + 'activity' => [ + 'cancellation_scope_id' => $scope, + ], + ]); + $event->forceFill([ + 'payload' => $payload, + ])->save(); + $run->refresh(); + } else { + $this->schedule( + $run, + $layout === 'flat' ? HistoryEventType::TimerScheduled : HistoryEventType::ActivityScheduled, + $payload + ); + } + CancellationScopeRequests::request($run, $scope, '1.20'); + if (! $valid || $layout === 'contradictory') { + $this->assertRefusedWithoutMutation($run, $task, 'cancellation_scope_delivery_membership_mismatch', fn () => + $this->deliver($run, $task, $scope, $layout === 'flat' ? 'timer' : 'activity')); + return; + } + $this->assertSame( + 3, + $this->deliver($run, $task, $scope, $layout === 'flat' ? 'timer' : 'activity') +->payload['sequence'] + ); + } + + public static function memberships(): iterable + { + foreach (['flat', 'nested'] as $layout) { + yield $layout . ' matching' => [$layout, true]; + yield $layout . ' different' => [$layout, false]; + } + yield 'contradictory' => ['contradictory', true]; + } + + public function testHistoricalRootOperationCannotBeReparentedByDelivery(): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + $this->schedule($run, HistoryEventType::ActivityScheduled, [ + 'sequence' => 3, + ]); + CancellationScopeRequests::request($run, $scope, '1.20'); + $this->assertRefusedWithoutMutation($run, $task, 'cancellation_scope_delivery_membership_mismatch', fn () => + $this->deliver($run, $task, $scope)); + } + + #[DataProvider('resolutionOrdering')] + public function testCallResolutionUsesCanonicalHistoryOrderInsteadOfHostClock(bool $resolvedFirst): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + $this->schedule($run, HistoryEventType::ActivityScheduled, [ + 'sequence' => 3, + 'activity' => [ + 'cancellation_scope_id' => $scope, + ], + ]); + if ($resolvedFirst) { + $this->schedule($run, HistoryEventType::ActivityCompleted, [ + 'sequence' => 3, + ]); + } + CancellationScopeRequests::request($run, $scope, '1.20'); + if (! $resolvedFirst) { + $this->schedule($run, HistoryEventType::ActivityCompleted, [ + 'sequence' => 3, + ]); + $this->assertSame(3, $this->deliver($run, $task, $scope)->payload['sequence']); + return; + } + $this->assertRefusedWithoutMutation($run, $task, 'cancellation_delivery_not_eligible', fn () => + $this->deliver($run, $task, $scope)); + } + + public static function resolutionOrdering(): iterable + { + yield 'completion before request' => [true]; + yield 'completion after request' => [false]; + } + + #[DataProvider('parallelBoundaries')] + public function testParallelAndSelectionBoundariesAllowUnaffectedMembers(bool $selection, bool $mixed): void + { + [, $run, $task, $parent, $scope] = $this->scopeTree(); + for ($index = 0; $index < 2; ++$index) { + $this->schedule($run, HistoryEventType::ActivityScheduled, [ + 'sequence' => 3 + $index, + 'activity' => [ + 'cancellation_scope_id' => $mixed && $index === 1 ? $parent : $scope, + ], + ...ParallelChildGroup::itemMetadata(3, 2, $index, 'activity'), + ]); + } + $request = CancellationScopeRequests::request($run, $scope, '1.20'); + $delivery = static fn () => CancellationScopeDelivery::record( + $run, + $task, + $scope, + $request->payload['request_id'], + $selection ? 5 : 3, + $selection ? 'selection_handle' : 'parallel', + '1.20', + $selection ? 1 : 2, + $selection ? 3 : null, + $selection ? 2 : 1 + ); + CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + $selection ? 5 : 3, + $selection ? 'selection_handle' : 'parallel', + '1.20', + $selection ? 1 : 2, + $selection ? 3 : null, + $selection ? 2 : 1 + ); + $this->fenceActivities($run, $task, $scope); + $event = $delivery(); + $this->assertSame($selection ? 5 : 3, $event->payload['sequence']); + $this->assertSame($selection ? 6 : 5, WorkflowStepHistory::nextDurableCommandSequence($run->fresh())); + if ($mixed) { + $this->assertSame( + ActivityStatus::Pending, + $run->activityExecutions() + ->where('sequence', 4) + ->sole() +->status + ); + $this->assertNull(CancellationScopeRequests::context($run->fresh(), $parent)); + } + } + + public static function parallelBoundaries(): iterable + { + yield 'parallel same scope' => [false, false]; + yield 'parallel mixed scopes' => [false, true]; + yield 'selection same scope' => [true, false]; + yield 'selection mixed scopes' => [true, true]; + } + + #[DataProvider('descendantAwaits')] + public function testAncestorRequestDeliversAtAuthoredUnshieldedDescendantAwait(bool $deep): void + { + [, $run, $task, $parent, $child] = $this->scopeTree(); + $sequence = $deep ? 4 : 3; + $member = $deep + ? CancellationScopeHistory::open($run, $task, 3, '1.20', $child)->payload['scope_id'] : $child; + $bridge = app(DefaultWorkflowTaskBridge::class); + $checkpoint = $bridge->checkpointCancellationScopePrefix( + $task->id, + 'original', + 1, + 'descendant-await', + $sequence, + [[ + 'type' => 'start_timer', + 'delay_seconds' => 60, + 'cancellation_scope_id' => $member, + ]], + '1.20' + ); + $this->assertTrue($checkpoint['checkpointed'], $checkpoint['reason'] ?? ''); + $request = CancellationScopeRequests::request($run->fresh(), $parent, '1.20', 30); + $beforeClaim = $task->fresh() + ->getAttributes(); + $prepared = $bridge->prepareCancellationScopeDelivery( + $task->id, + 'original', + 1, + $parent, + $request->payload['request_id'], + $sequence, + 'timer', + protocolVersion: '1.20' + ); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $delivered = $bridge->deliverCancellationScope( + $task->id, + 'original', + 1, + $parent, + $request->payload['request_id'], + $sequence, + 'timer', + protocolVersion: '1.20' + ); + $this->assertTrue($delivered['delivered'], $delivered['reason'] ?? ''); + $this->assertSame(TimerStatus::Cancelled, $run->timers()->sole()->status); + $this->assertSame($sequence, $delivered['sequence']); + $this->assertSame('2026-10-03T00:00:30.000000Z', $delivered['authority_deadline_at']); + $context = CancellationScopeRequests::context($run->fresh(), $member); + $this->assertSame($request->payload['request_id'], $context->rootContext->rootRequestId); + $this->assertHostingShortenedWithoutClaimReplacement($beforeClaim, $task); + $this->assertFalse($run->fresh()->status->isTerminal()); + $this->assertNull($run->fresh()->cancellation_request_command_id); + $marker = CancellationScopeDelivery::recorded($run->fresh(), $parent); + Carbon::setTestNow('2026-10-03T00:00:31Z'); + $this->assertSame( + $marker->getAttributes(), + CancellationScopeDelivery::recorded($run->fresh(), $parent)->getAttributes() + ); + $this->assertSame( + $prepared['preparation_history_event_id'], + CancellationScopeDelivery::prepared($run->fresh(), $parent)->id + ); + } + + public static function descendantAwaits(): iterable + { + yield 'child' => [false]; + yield 'grandchild' => [true]; + } + + #[DataProvider('mixedShieldGroups')] + public function testMixedGroupCancelsOnlyRequestedSubtreeAndSurvivorsStillPublish( + bool $selection, + bool $directShield + ): void { + [, $run, $task, $parent, $child] = $this->scopeTree(); + $shield = CancellationScopeHistory::open($run, $task, 3, '1.20', $parent, true)->payload['scope_id']; + $sibling = CancellationScopeHistory::open($run, $task, 4, '1.20')->payload['scope_id']; + $scopes = [$child, $shield, $sibling]; + foreach ($scopes as $index => $scope) { + $this->schedule($run, HistoryEventType::ActivityScheduled, [ + 'sequence' => 5 + $index, + 'activity' => [ + 'cancellation_scope_id' => $scope, + ], + ...ParallelChildGroup::itemMetadata(5, 3, $index, 'activity'), + ]); + } + $target = $directShield ? $shield : $parent; + $request = CancellationScopeRequests::request($run, $target, '1.20', 30); + $bridge = app(DefaultWorkflowTaskBridge::class); + $arguments = [ + $task->id, 'original', 1, $target, $request->payload['request_id'], + $selection ? 8 : 5, $selection ? 'selection_handle' : 'parallel', + $selection ? 1 : 3, $selection ? 5 : null, $selection ? 3 : 1, '1.20', + ]; + $prepared = $bridge->prepareCancellationScopeDelivery(...$arguments); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $beforeClaim = $task->fresh() + ->getAttributes(); + $survivingExecutions = []; + $survivingTasks = []; + foreach ($run->activityExecutions()->orderBy('sequence')->get() as $execution) { + if ($execution->sequence !== ($directShield ? 6 : 5)) { + $survivingExecutions[$execution->id] = $execution->getAttributes(); + $survivorTask = $run->tasks() + ->where('payload->activity_execution_id', $execution->id) + ->sole(); + $survivingTasks[$survivorTask->id] = $survivorTask->getAttributes(); + } + } + $delivered = $bridge->deliverCancellationScope(...$arguments); + $this->assertTrue($delivered['delivered'], $delivered['reason'] ?? ''); + $this->assertHostingShortenedWithoutClaimReplacement($beforeClaim, $task); + $this->assertSame( + ActivityStatus::Cancelled, + $run->activityExecutions() + ->where('sequence', $directShield ? 6 : 5) + ->sole() +->status + ); + foreach ($survivingExecutions as $id => $attributes) { + $this->assertSame($attributes, ActivityExecution::query()->findOrFail($id)->getAttributes()); + } + foreach ($survivingTasks as $id => $attributes) { + $this->assertSame($attributes, WorkflowTask::query()->findOrFail($id)->getAttributes()); + } + $this->assertNull(CancellationScopeRequests::context($run->fresh(), $sibling)); + $this->assertNull(CancellationScopeRequests::context($run->fresh(), $directShield ? $child : $shield)); + $marker = CancellationScopeDelivery::recorded($run->fresh(), $target); + $beforeMarker = $marker->getAttributes(); + $beforePreparation = CancellationScopeDelivery::prepared($run->fresh(), $target)->getAttributes(); + $task->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 2, + ])->save(); + $arguments[1] = 'replacement'; + $arguments[2] = 2; + $replayed = $bridge->deliverCancellationScope(...$arguments); + $this->assertSame($delivered['history_event_id'], $replayed['history_event_id']); + $this->assertSame($delivered['cancellation'], $replayed['cancellation']); + foreach ($survivingExecutions as $attributes) { + $this->schedule($run, HistoryEventType::ActivityCompleted, [ + 'sequence' => $attributes['sequence'], + ]); + } + $this->assertSame(2, $run->activityExecutions()->where('status', ActivityStatus::Completed)->count()); + Carbon::setTestNow('2026-10-03T00:00:31Z'); + $this->assertSame($beforeMarker, CancellationScopeDelivery::recorded($run->fresh(), $target)->getAttributes()); + $this->assertSame( + $beforePreparation, + CancellationScopeDelivery::prepared($run->fresh(), $target)->getAttributes() + ); + $this->assertSame($selection ? 9 : 8, WorkflowStepHistory::nextDurableCommandSequence($run->fresh())); + $this->assertFalse($run->fresh()->status->isTerminal()); + } + + public static function mixedShieldGroups(): iterable + { + yield 'parallel ancestor request' => [false, false]; + yield 'selection ancestor request' => [true, false]; + yield 'parallel direct shield request' => [false, true]; + yield 'selection direct shield request' => [true, true]; + } + + public function testAncestorCannotDeliverAtShieldedDescendantAwaitButDirectShieldRequestCan(): void + { + [, $run, $task, $parent] = $this->scopeTree(); + $shield = CancellationScopeHistory::open($run, $task, 3, '1.20', $parent, true)->payload['scope_id']; + $child = CancellationScopeHistory::open($run, $task, 4, '1.20', $shield)->payload['scope_id']; + $this->schedule($run, HistoryEventType::TimerScheduled, [ + 'sequence' => 5, + 'cancellation_scope_id' => $child, + ]); + $ancestor = CancellationScopeRequests::request($run, $parent, '1.20', 30); + $this->assertRefusedWithoutMutation( + $run, + $task, + 'cancellation_scope_delivery_membership_mismatch', + static fn () => + CancellationScopeDelivery::prepare( + $run, + $task, + $parent, + $ancestor->payload['request_id'], + 5, + 'timer', + '1.20' + ) + ); + $this->assertSame(TimerStatus::Pending, $run->timers()->sole()->status); + $this->assertNull(CancellationScopeRequests::context($run->fresh(), $child)); + $direct = CancellationScopeRequests::request($run, $shield, '1.20', 30); + $bridge = app(DefaultWorkflowTaskBridge::class); + $arguments = [ + $task->id, + 'original', + 1, + $shield, + $direct->payload['request_id'], + 5, + 'timer', + 1, + null, + 1, + '1.20', + ]; + $prepared = $bridge->prepareCancellationScopeDelivery(...$arguments); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $delivered = $bridge->deliverCancellationScope(...$arguments); + $this->assertTrue($delivered['delivered'], $delivered['reason'] ?? ''); + $this->assertSame(TimerStatus::Cancelled, $run->timers()->sole()->status); + $this->assertSame( + $direct->payload['request_id'], + CancellationScopeRequests::context($run->fresh(), $child)->rootContext->rootRequestId + ); + $this->assertNotSame($ancestor->payload['request_id'], $direct->payload['request_id']); + } + + #[DataProvider('unaffectedGroupMembership')] + public function testMixedGroupRejectsInvalidUnaffectedMembershipBeforeAnyEffects(string $kind): void + { + [, $run, $task, $parent, $scope] = $this->scopeTree(); + for ($index = 0; $index < 2; ++$index) { + $this->schedule($run, HistoryEventType::ActivityScheduled, [ + 'sequence' => 3 + $index, + 'activity' => [ + 'cancellation_scope_id' => $index === 0 && $kind !== 'entirely unrelated' ? $scope : $parent, + ], + ...ParallelChildGroup::itemMetadata(3, 2, $index, 'activity'), + ]); + } + $event = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityScheduled)->where('payload->sequence', 4)->sole(); + $payload = $event->payload; + switch ($kind) { + case 'contradictory': + $payload['cancellation_scope_id'] = $scope; + break; + case 'null flat': + $payload['cancellation_scope_id'] = null; + break; + case 'null nested': + $payload['activity']['cancellation_scope_id'] = null; + break; + case 'unknown': + $payload['activity']['cancellation_scope_id'] = 'foreign-scope'; + break; + case 'future opening': + $payload['activity']['cancellation_scope_id'] = CancellationScopeHistory::open( + $run, + $task, + 5, + '1.20' + )->payload['scope_id']; + break; + } + $event->forceFill([ + 'payload' => $payload, + ])->save(); + $run->refresh(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $beforeExecutions = $run->activityExecutions() + ->orderBy('sequence') + ->get() + ->map->getAttributes() + ->all(); + $this->assertRefusedWithoutMutation( + $run, + $task, + 'cancellation_scope_delivery_membership_mismatch', + static fn () => + CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + 3, + 'parallel', + '1.20', + 2 + ) + ); + $this->assertSame( + $beforeExecutions, + $run->activityExecutions() + ->orderBy('sequence') + ->get() + ->map->getAttributes() + ->all() + ); + } + + public static function unaffectedGroupMembership(): iterable + { + foreach ([ + 'entirely unrelated', + 'contradictory', + 'null flat', + 'null nested', + 'unknown', + 'future opening', + ] as $kind) { + yield $kind => [$kind]; + } + } + + #[DataProvider('claimFailures')] + public function testClaimFencesApplyBeforeMutation(string $kind): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + CancellationScopeRequests::request($run, $scope, '1.20'); + $task->forceFill(match ($kind) { + 'expired' => [ + 'lease_expires_at' => now(), + ], + 'ready' => [ + 'status' => TaskStatus::Ready, + ], + 'activity' => [ + 'task_type' => TaskType::Activity, + ], + 'owner' => [ + 'lease_owner' => '', + ], + 'attempt' => [ + 'attempt_count' => 0, + ], + 'namespace' => [ + 'namespace' => 'foreign', + ], + })->save(); + $this->assertRefusedWithoutMutation($run, $task, 'cancellation_scope_workflow_claim_mismatch', fn () => + $this->deliver($run, $task, $scope)); + } + + public static function claimFailures(): iterable + { + foreach (['expired', 'ready', 'activity', 'owner', 'attempt', 'namespace'] as $kind) { + yield $kind => [$kind]; + } + } + + public function testRunClosureRefusesDeliveryButKeepsColdReceiptInspectable(): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + CancellationScopeRequests::request($run, $scope, '1.20'); + $event = $this->deliver($run, $task, $scope); + $run->forceFill([ + 'status' => RunStatus::Completed, + 'closed_at' => now(), + ])->save(); + $this->assertSame($event->id, CancellationScopeDelivery::recorded($run->fresh(), $scope)->id); + $this->assertRefusedWithoutMutation($run, $task, 'cancellation_scope_authority_expired', fn () => + $this->deliver($run, $task, $scope)); + } + + #[DataProvider('invalidRequests')] + public function testWrongIdentityAndPublishedProtocolCannotCreateDelivery(string $kind, string $reason): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + $request = CancellationScopeRequests::request($run, $scope, '1.20'); + $this->assertRefusedWithoutMutation($run, $task, $reason, static fn () => CancellationScopeDelivery::record( + $run, + $task, + $kind === 'unknown' ? 'foreign-scope' : $scope, + $kind === 'identity' ? 'foreign-request' : $request->payload['request_id'], + 3, + 'activity', + $kind === 'protocol' ? '1.19' : '1.20' + )); + } + + public static function invalidRequests(): iterable + { + yield 'foreign request' => ['identity', 'cancellation_scope_request_mismatch']; + yield 'foreign scope' => ['unknown', 'cancellation_scope_not_recorded']; + yield 'published protocol' => ['protocol', 'cancellation_scope_requires_protocol_1_20']; + } + + #[DataProvider('corruptReceipts')] + public function testColdReadRejectsContradictoryDelivery(string $field, mixed $value): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + CancellationScopeRequests::request($run, $scope, '1.20'); + $event = $this->deliver($run, $task, $scope); + $payload = $event->payload; + $payload[$field] = $value; + $event->forceFill([ + 'payload' => $payload, + ])->save(); + $this->expectExceptionMessage('cancellation_scope_delivery_history_invalid'); + CancellationScopeDelivery::recorded($run->fresh(), $scope); + } + + public static function corruptReceipts(): iterable + { + yield 'foreign run' => ['workflow_run_id', 'other-run']; + yield 'foreign request' => ['request_id', 'other-request']; + yield 'wrong schema' => ['schema', 'other-schema']; + yield 'invalid sequence' => ['sequence', 0]; + yield 'invalid kind' => ['call_kind', 'unknown']; + yield 'extended ceiling' => ['authority_deadline_at', '2026-10-03T00:05:00.000000Z']; + yield 'noncanonical clock' => ['authority_deadline_at', 'tomorrow']; + } + + #[DataProvider('reservedAdmissionPaths')] + public function testPreparedScopeBlocksEveryNewAdmissionPathBeforeMutation(string $path, string $reason): void + { + [, $run, $task, $parent, $scope] = $this->scopeTree(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + 3, + 'local_activity', + '1.20' + ); + $history = $run->historyEvents() + ->orderBy('sequence') + ->get() + ->toArray(); + $claim = $task->fresh() + ->getAttributes(); + $bridge = app(DefaultWorkflowTaskBridge::class); + $local = [ + 'type' => 'record_local_activity', + 'activity_type' => 'tests.scope-admission', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + ]; + $marker = [[ + 'type' => 'record_side_effect', + 'marker_id' => 'reserved-position', + 'result' => Serializer::serializeWithCodec('avro', 'must not commit'), + 'payload_codec' => 'avro', + ]]; + $reply = match ($path) { + 'scoped local' => $bridge->prepareLocalActivity( + $task->id, + 'original', + 1, + 3, + 'new-scoped-attempt', + [ + ...$local, + 'cancellation_scope_id' => $scope, + ], + '1.20' + ), + 'unscoped local' => $bridge->prepareLocalActivity( + $task->id, + 'original', + 1, + 3, + 'new-root-attempt', + $local, + '1.20' + ), + 'parent local' => $bridge->prepareLocalActivity( + $task->id, + 'original', + 1, + 3, + 'new-parent-attempt', + [ + ...$local, + 'cancellation_scope_id' => $parent, + ], + '1.20' + ), + 'completion' => $bridge->complete($task->id, [[ + ...$local, + 'type' => 'schedule_activity', + ]]), + 'local prefix' => $bridge->checkpointLocalActivityPrefix( + $task->id, + 'original', + 1, + 'reserved', + 3, + $marker, + '1.20' + ), + 'scope prefix' => $bridge->checkpointCancellationScopePrefix( + $task->id, + 'original', + 1, + 'reserved', + 3, + $marker, + '1.20' + ), + 'local group' => $bridge->checkpointLocalActivityGroup($task->id, 'original', 1, 'reserved', 3, [[ + ...$local, + 'type' => 'prepare_local_activity', + ...ParallelChildGroup::itemMetadata(3, 2, 0, 'mixed'), + ], [ + ...$local, + 'type' => 'schedule_activity', + 'cancellation_scope_id' => $parent, + ...ParallelChildGroup::itemMetadata(3, 2, 1, 'mixed'), + ]], '1.20'), + }; + $this->assertSame($reason, $reply['reason']); + $this->assertSame($history, $run->historyEvents()->orderBy('sequence')->get()->toArray()); + $this->assertSame($claim, $task->fresh()->getAttributes()); + $this->assertSame(0, $run->activityExecutions()->count()); + $this->assertSame(1, $run->tasks()->count()); + $this->assertNull($run->fresh()->cancellation_request_command_id); + } + + public static function reservedAdmissionPaths(): iterable + { + yield 'direct local scope admission' => ['scoped local', 'operation_scope_cancellation_prepared']; + foreach ([ + 'unscoped local', + 'parent local', + 'completion', + 'local prefix', + 'scope prefix', + 'local group', + ] as $path) { + yield $path => [$path, 'operation_cancellation_delivery_reserved']; + } + } + + public function testPreparedScopeRetainsOriginalLocalAdmissionAndAllowsParentAfterDelivery(): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + $bridge = app(DefaultWorkflowTaskBridge::class); + $descriptor = [ + 'type' => 'record_local_activity', + 'activity_type' => 'tests.scope-admission', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'cancellation_scope_id' => $scope, + ]; + $first = $bridge->prepareLocalActivity($task->id, 'original', 1, 3, 'original-attempt', $descriptor, '1.20'); + $this->assertTrue($first['prepared'], $first['reason'] ?? ''); + $request = CancellationScopeRequests::request($run->fresh(), $scope, '1.20', 30); + $preparation = CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + 3, + 'local_activity', + '1.20' + ); + $historyCount = $run->historyEvents() + ->count(); + $duplicate = $bridge->prepareLocalActivity( + $task->id, + 'original', + 1, + 3, + 'original-attempt', + $descriptor, + '1.20' + ); + $this->assertTrue($duplicate['prepared'], $duplicate['reason'] ?? ''); + $this->assertTrue($duplicate['duplicate']); + $this->assertSame($first['activity_attempt_id'], $duplicate['activity_attempt_id']); + $this->assertSame($historyCount, $run->historyEvents()->count()); + $this->assertSame('operation_scope_cancellation_prepared', $bridge->prepareLocalActivity( + $task->id, + 'original', + 1, + 4, + 'forbidden-attempt', + $descriptor, + '1.20' + )['reason']); + $this->assertSame(1, $run->activityExecutions()->count()); + $fence = ScopedActivityCancellation::fence( + $run, + $task, + $first['activity_execution_id'], + $scope, + $request->payload['request_id'], + '1.20' + ); + $this->assertTrue($fence['fenced']); + $delivery = CancellationScopeDelivery::record( + $run, + $task, + $scope, + $request->payload['request_id'], + 3, + 'local_activity', + '1.20' + ); + $this->assertSame($preparation->id, $delivery->payload['preparation_history_event_id']); + $this->assertSame('operation_scope_cancellation_prepared', $bridge->prepareLocalActivity( + $task->id, + 'original', + 1, + 4, + 'forbidden-after-delivery', + $descriptor, + '1.20' + )['reason']); + unset($descriptor['cancellation_scope_id']); + $parent = $bridge->prepareLocalActivity($task->id, 'original', 1, 4, 'parent-attempt', $descriptor, '1.20'); + $this->assertTrue($parent['prepared'], $parent['reason'] ?? ''); + $this->assertSame(2, $run->activityExecutions()->count()); + $this->assertSame(TaskStatus::Leased, $task->fresh()->status); + $this->assertNull($run->fresh()->cancellation_request_command_id); + } + + #[DataProvider('unaffectedLocalStates')] + public function testMixedPendingDeliveryPreservesPreviouslyAdmittedRootLocalWork(bool $startedBefore): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + $bridge = app(DefaultWorkflowTaskBridge::class); + $local = [ + 'type' => 'prepare_local_activity', + 'activity_type' => 'tests.unaffected-root', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + ...ParallelChildGroup::itemMetadata(3, 2, 0, 'mixed'), + ]; + $checkpoint = $bridge->checkpointLocalActivityGroup($task->id, 'original', 1, 'original-group', 3, [$local, [ + ...$local, + 'type' => 'schedule_activity', + 'activity_type' => 'tests.cancelled-member', + 'cancellation_scope_id' => $scope, + ...ParallelChildGroup::itemMetadata(3, 2, 1, 'mixed'), + ]], '1.20'); + $this->assertTrue($checkpoint['checkpointed'], $checkpoint['reason'] ?? ''); + $local['type'] = 'record_local_activity'; + $first = $startedBefore + ? $bridge->prepareLocalActivity($task->id, 'original', 1, 3, 'unaffected-attempt', $local, '1.20') : null; + if ($first !== null) { + $this->assertTrue($first['prepared'], $first['reason'] ?? ''); + } + $request = CancellationScopeRequests::request($run->fresh(), $scope, '1.20', 30); + $preparation = CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + 3, + 'parallel', + '1.20', + 2 + ); + $before = $task->fresh() + ->getAttributes(); + $reply = $bridge->prepareLocalActivity($task->id, 'original', 1, 3, 'unaffected-attempt', $local, '1.20'); + $this->assertTrue($reply['prepared'], $reply['reason'] ?? ''); + $this->assertSame($startedBefore, $reply['duplicate']); + if ($first !== null) { + $this->assertSame($first['activity_attempt_id'], $reply['activity_attempt_id']); + } + $this->assertSame(2, $run->activityExecutions()->count()); + $this->assertSame($before, $task->fresh()->getAttributes()); + $this->assertSame($preparation->id, CancellationScopeDelivery::prepared($run->fresh(), $scope)->id); + $this->assertNull($run->fresh()->cancellation_request_command_id); + } + + public static function unaffectedLocalStates(): iterable + { + yield 'previously started callback' => [true]; + yield 'admitted before preparation but not started' => [false]; + } + + /** + * @return array{WorkflowStub, WorkflowRun, WorkflowTask, string, string} + */ + #[DataProvider('scopedCleanupAddresses')] + public function testScopedCleanupRetainsOriginalAuthorityAndLetsItsParentFinish( + bool $ancestor, + bool $runRequest = false + ): void { + [, $run, $task, $parent, $scope] = $this->scopeTree(); + $run->forceFill([ + 'execution_deadline_at' => now() + ->addSeconds(15), + ])->save(); + if ($runRequest) { + $this->assertTrue(WorkflowStub::loadRun($run->id)->requestCancellation('clean up the run', 30)->accepted()); + $run->refresh(); + } + $address = $ancestor ? $parent : $scope; + $request = CancellationScopeRequests::request($run, $address, '1.20', 30, 'clean up this subtree'); + $delivery = $this->deliver($run, $task, $address, 'local_activity'); + $descriptor = $this->scopedCleanupDescriptor($address, $scope, $request, $delivery); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prepared = $bridge->prepareLocalActivity($task->id, 'original', 1, 4, 'cleanup-attempt', $descriptor, '1.20'); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $snapshot = $prepared['cancellation_cleanup']; + $this->assertSame($address, $snapshot['scope_id']); + $this->assertSame($scope, $snapshot['operation_scope_id']); + $this->assertSame($request->payload['request_id'], $snapshot['request_id']); + $this->assertSame( + $request->payload['cancellation']['root_context']['root_request_id'], + $snapshot['root_request_id'] + ); + $this->assertSame('2026-10-03T00:00:30.000000Z', $snapshot['cleanup_deadline_at']); + $this->assertSame('2026-10-03T00:00:15.000000Z', $snapshot['authority_deadline_at']); + $this->assertSame( + $delivery->payload['preparation_history_event_id'], + $snapshot['preparation_history_event_id'] + ); + $this->assertSame($snapshot['authority_deadline_at'], $prepared['start_to_close_deadline_at']); + $this->assertSame($snapshot['authority_deadline_at'], $prepared['schedule_to_close_deadline_at']); + $duplicate = $bridge->prepareLocalActivity($task->id, 'original', 1, 4, 'cleanup-attempt', $descriptor, '1.20'); + $this->assertTrue($duplicate['duplicate']); + $this->assertSame($prepared['activity_attempt_id'], $duplicate['activity_attempt_id']); + $this->assertSameJsonObject($snapshot, $duplicate['cancellation_cleanup']); + $run->forceFill([ + 'execution_deadline_at' => now() + ->addMinute(), + ])->save(); + $this->assertTrue( + $bridge->controlLocalActivity($prepared['activity_attempt_id'], 'original', 1, true, '1.20')['active'] + ); + $this->assertSame( + $snapshot['authority_deadline_at'], + ActivityExecution::query()->findOrFail($prepared['activity_execution_id'])->close_deadline_at->toISOString() + ); + $outcome = $bridge->recordLocalActivityOutcome($prepared['activity_attempt_id'], 'original', 1, [ + 'outcome' => 'completed', + 'result' => Serializer::serializeWithCodec('avro', 'cleaned'), + 'payload_codec' => 'avro', + ], '1.20'); + $this->assertTrue($outcome['recorded'], $outcome['reason'] ?? ''); + $this->assertFalse($outcome['claim_released']); + $run->refresh(); + if ($runRequest) { + $this->assertTrue( + $bridge->deliverCancellation($task->id, $run->cancellation_request_command_id, 5, 'timer')['delivered'] + ); + } else { + $this->assertNull($run->cancellation_request_command_id); + } + $this->assertTrue($bridge->complete($task->id, [[ + 'type' => 'complete_workflow', + 'output' => Serializer::serializeWithCodec('avro', 'parent survived'), + 'payload_codec' => 'avro', + ]])['completed']); + $this->assertSame($runRequest ? RunStatus::Cancelled : RunStatus::Completed, $run->refresh()->status); + } + + #[DataProvider('invalidScopedCleanupProofs')] + public function testScopedCleanupCannotInventOrMoveItsOriginalAuthority(string $mutation, string $reason): void + { + [, $run, $task, $parent, $scope] = $this->scopeTree(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $delivery = $this->deliver($run, $task, $scope, 'local_activity'); + $descriptor = $this->scopedCleanupDescriptor($scope, $scope, $request, $delivery); + $sequence = 4; + match ($mutation) { + 'missing proof' => $descriptor = array_diff_key($descriptor, [ + 'cancellation_cleanup' => true, + ]), + 'wrong request' => $descriptor['cancellation_cleanup']['request_id'] = 'another-request', + 'wrong delivery' => $descriptor['cancellation_cleanup']['delivery_history_event_id'] = 'another-delivery', + 'wrong address' => $descriptor['cancellation_cleanup']['scope_id'] = $parent, + 'unaffected parent' => $descriptor['cancellation_scope_id'] = $parent, + 'root operation' => $descriptor['cancellation_scope_id'] = CancellationScopeHistory::ROOT_SCOPE_ID, + 'invented budget' => $descriptor['cancellation_cleanup']['cleanup_deadline_at'] = now()->addHour()->toISOString(), + 'before delivery' => $sequence = 3, + }; + $history = $run->historyEvents() + ->get() + ->toArray(); + $claim = $task->fresh() + ->getAttributes(); + $reply = app(DefaultWorkflowTaskBridge::class)->prepareLocalActivity( + $task->id, + 'original', + 1, + $sequence, + 'forged-cleanup', + $descriptor, + '1.20', + ); + $this->assertFalse($reply['prepared']); + $this->assertSame($reason, $reply['reason']); + $this->assertSame($history, $run->historyEvents()->get()->toArray()); + $this->assertSame($claim, $task->fresh()->getAttributes()); + $this->assertSame(0, $run->activityExecutions()->count()); + } + + public static function invalidScopedCleanupProofs(): iterable + { + yield 'ordinary work' => ['missing proof', 'operation_scope_cancellation_prepared']; + foreach ([ + 'wrong request', + 'wrong delivery', + 'wrong address', + 'unaffected parent', + 'root operation', + 'before delivery', + ] as $mutation) { + yield $mutation => [$mutation, 'local_activity_cleanup_authority_mismatch']; + } + yield 'caller cannot extend the budget' => ['invented budget', 'invalid_local_activity_preparation']; + } + + public function testScopedCleanupDeadlineStopsRenewalAndCannotPublishALateResult(): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + $run->forceFill([ + 'execution_deadline_at' => now() + ->addSeconds(15), + ])->save(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $delivery = $this->deliver($run, $task, $scope, 'local_activity'); + $descriptor = $this->scopedCleanupDescriptor($scope, $scope, $request, $delivery); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prepared = $bridge->prepareLocalActivity($task->id, 'original', 1, 4, 'cleanup-attempt', $descriptor, '1.20'); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $run->forceFill([ + 'execution_deadline_at' => now() + ->addMinute(), + ])->save(); + Carbon::setTestNow(now()->addSeconds(16)); + $task->forceFill([ + 'lease_expires_at' => now() + ->addMinute(), + ])->save(); + $execution = ActivityExecution::query()->findOrFail($prepared['activity_execution_id']); + $execution->attempts() + ->whereKey($prepared['activity_attempt_id'])->firstOrFail()->forceFill([ + 'lease_expires_at' => now() + ->addMinute(), + ])->save(); + $history = $run->historyEvents() + ->count(); + $control = $bridge->controlLocalActivity($prepared['activity_attempt_id'], 'original', 1, true, '1.20'); + $this->assertFalse($control['active']); + $this->assertFalse($control['renewed']); + $this->assertSame('local_activity_deadline_expired', $control['reason']); + $this->assertSame($history, $run->historyEvents()->count()); + $late = $bridge->recordLocalActivityOutcome($prepared['activity_attempt_id'], 'original', 1, [ + 'outcome' => 'completed', + 'result' => Serializer::serializeWithCodec('avro', 'late'), + 'payload_codec' => 'avro', + ], '1.20'); + $this->assertTrue($late['recorded']); + $this->assertSame(HistoryEventType::ActivityTimedOut->value, $late['event_type']); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCompleted)->count()); + $this->assertSame('local_activity_cleanup_deadline_expired', $bridge->prepareLocalActivity( + $task->id, + 'original', + 1, + 5, + 'new-late-cleanup', + $descriptor, + '1.20', + )['reason']); + } + + public function testLaterWholeRunCancellationStillStopsScopedCleanup(): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $delivery = $this->deliver($run, $task, $scope, 'local_activity'); + $descriptor = $this->scopedCleanupDescriptor($scope, $scope, $request, $delivery); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prepared = $bridge->prepareLocalActivity($task->id, 'original', 1, 4, 'cleanup-attempt', $descriptor, '1.20'); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + Carbon::setTestNow(now()->addSecond()); + $wholeRun = WorkflowStub::loadRun($run->id)->requestCancellation('stop everything', 10); + $this->assertTrue($wholeRun->accepted()); + $control = $bridge->controlLocalActivity($prepared['activity_attempt_id'], 'original', 1, true, '1.20'); + $this->assertFalse($control['active']); + $this->assertFalse($control['renewed']); + $this->assertTrue($control['fenced']); + $this->assertSame('cancellation_requested', $control['reason']); + $this->assertSame( + $run->refresh() +->cancellation_request_command_id, + $control['cancellation_request']['request_id'] + ); + $this->assertSameJsonObject($prepared['cancellation_cleanup'], ActivityExecution::query()->findOrFail( + $prepared['activity_execution_id'], + )->activity_options['cancellation_cleanup']); + $this->assertSame('cancellation_requested', $bridge->prepareLocalActivity( + $task->id, + 'original', + 1, + 5, + 'root-cancelled-cleanup', + $descriptor, + '1.20', + )['reason']); + } + + public function testScopedCleanupUsesItsEarlierDescendantDeliveryEvenAfterParentDelivery(): void + { + [, $run, $task, $parent, $scope] = $this->scopeTree(); + $parentRequest = CancellationScopeRequests::request($run, $parent, '1.20', 30); + $request = CancellationScopeRequests::request($run, $scope, '1.20', parentScopeId: $parent); + $delivery = $this->deliver($run, $task, $scope, 'local_activity'); + CancellationScopeDelivery::prepare( + $run, + $task, + $parent, + $parentRequest->payload['request_id'], + 4, + 'local_activity', + '1.20' + ); + $parentDelivery = CancellationScopeDelivery::record( + $run, + $task, + $parent, + $parentRequest->payload['request_id'], + 4, + 'local_activity', + '1.20' + ); + $bridge = app(DefaultWorkflowTaskBridge::class); + $parentProof = $this->scopedCleanupDescriptor($parent, $scope, $parentRequest, $parentDelivery); + $this->assertSame('operation_scope_cancellation_prepared', $bridge->prepareLocalActivity( + $task->id, + 'original', + 1, + 5, + 'rebound-cleanup', + $parentProof, + '1.20', + )['reason']); + $this->assertSame(0, $run->activityExecutions()->count()); + $descriptor = $this->scopedCleanupDescriptor($scope, $scope, $request, $delivery); + $prepared = $bridge->prepareLocalActivity($task->id, 'original', 1, 5, 'original-cleanup', $descriptor, '1.20'); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $this->assertSame($delivery->id, $prepared['cancellation_cleanup']['delivery_history_event_id']); + $this->assertSame($request->payload['request_id'], $prepared['cancellation_cleanup']['request_id']); + $this->assertSame($parentRequest->payload['request_id'], $prepared['cancellation_cleanup']['root_request_id']); + } + + #[DataProvider('scopedCleanupRunRequests')] + public function testScopedCleanupReplacementRetainsTheOriginalDeliveryAndDeadline(bool $runRequest): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + config() + ->set('workflows.v2.workflow_task_lease_seconds', 60); + if ($runRequest) { + $this->assertTrue(WorkflowStub::loadRun($run->id)->requestCancellation('clean up the run', 30)->accepted()); + $run->refresh(); + } + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $delivery = $this->deliver($run, $task, $scope, 'local_activity'); + $descriptor = $this->scopedCleanupDescriptor($scope, $scope, $request, $delivery); + $descriptor['retry_policy'] = [ + 'max_attempts' => 2, + 'backoff_seconds' => [0], + ]; + $bridge = app(DefaultWorkflowTaskBridge::class); + $prepared = $bridge->prepareLocalActivity($task->id, 'original', 1, 4, 'cleanup-attempt', $descriptor, '1.20'); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $this->assertSame('2026-10-03T00:00:10.000000Z', $task->fresh()->lease_expires_at->toISOString()); + Carbon::setTestNow(now()->addSeconds(11)); + $repair = TaskWatchdog::runPass(respectThrottle: false, runIds: [$run->id]); + $this->assertSame([], $repair['existing_task_failures']); + $this->assertSame(1, $repair['repaired_existing_tasks']); + $this->assertSame(TaskStatus::Ready, $task->fresh()->status); + $this->assertTrue($bridge->claimStatus($task->id, 'replacement')['claimed']); + $this->assertSame(2, $task->fresh()->attempt_count); + $this->assertSame('2026-10-03T00:00:21.000000Z', $task->fresh()->lease_expires_at->toISOString()); + $this->assertSame( + $delivery->fresh() + ->getAttributes(), + CancellationScopeDelivery::recorded($run->fresh(), $scope)->getAttributes() + ); + $recovery = $bridge->recoverLocalActivity($task->id, 'replacement', 2, 4, $descriptor, '1.20'); + $this->assertTrue($recovery['recovered'], $recovery['reason'] ?? ''); + $this->assertSame('unknown', $recovery['callback_stop_state']); + $retryTask = WorkflowTask::query()->findOrFail($recovery['created_task_ids'][0]); + $this->assertTrue($bridge->claimStatus($retryTask->id, 'replacement')['claimed']); + $retry = $bridge->prepareLocalActivity( + $retryTask->id, + 'replacement', + 1, + 4, + 'replacement-cleanup', + $descriptor, + '1.20' + ); + $this->assertTrue($retry['prepared'], $retry['reason'] ?? ''); + $this->assertSame(2, $retry['attempt_number']); + $this->assertNotSame($prepared['activity_attempt_id'], $retry['activity_attempt_id']); + $this->assertSameJsonObject($prepared['cancellation_cleanup'], $retry['cancellation_cleanup']); + $this->assertSame($prepared['start_to_close_deadline_at'], $retry['start_to_close_deadline_at']); + $this->assertSame($prepared['schedule_to_close_deadline_at'], $retry['schedule_to_close_deadline_at']); + $stale = $bridge->recordLocalActivityOutcome($prepared['activity_attempt_id'], 'original', 1, [ + 'outcome' => 'completed', + 'result' => Serializer::serializeWithCodec('avro', 'stale'), + 'payload_codec' => 'avro', + ], '1.20'); + $this->assertFalse($stale['recorded']); + $this->assertSame('stale_activity_attempt', $stale['reason']); + $completed = $bridge->recordLocalActivityOutcome($retry['activity_attempt_id'], 'replacement', 1, [ + 'outcome' => 'completed', + 'result' => Serializer::serializeWithCodec('avro', 'resumed'), + 'payload_codec' => 'avro', + ], '1.20'); + $this->assertTrue($completed['recorded'], $completed['reason'] ?? ''); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeDelivered)->count() + ); + $this->assertSame( + $request->payload['request_id'], + CancellationScopeRequests::context($run->fresh(), $scope)->requestId + ); + $run->refresh(); + if ($runRequest) { + $requestId = $run->cancellation_request_command_id; + $deadline = $run->cancellation_deadline_at->toISOString(); + $this->assertTrue(WorkflowStub::loadRun($run->id)->requestCancellation('duplicate', 300)->accepted()); + $this->assertSame($requestId, $run->refresh()->cancellation_request_command_id); + $this->assertSame($deadline, $run->cancellation_deadline_at->toISOString()); + $this->assertTrue($bridge->deliverCancellation($retryTask->id, $requestId, 5, 'timer')['delivered']); + $this->assertTrue($bridge->complete($retryTask->id, [[ + 'type' => 'complete_workflow', + 'output' => Serializer::serializeWithCodec('avro', 'cleaned'), + 'payload_codec' => 'avro', + ]])['completed']); + $this->assertSame(RunStatus::Cancelled, $run->refresh()->status); + } else { + $this->assertNull($run->cancellation_request_command_id); + } + } + + public static function scopedCleanupRunRequests(): iterable + { + yield 'independent scope request' => [false]; + yield 'inherited run request' => [true]; + } + + #[DataProvider('scopedCleanupGroups')] + public function testScopedCleanupGroupAdmitsEveryValidMemberAtomically( + bool $invalidSecond, + bool $runRequest = false + ): void { + [, $run, $task, , $scope] = $this->scopeTree(); + if ($runRequest) { + $this->assertTrue(WorkflowStub::loadRun($run->id)->requestCancellation('clean up the run', 30)->accepted()); + $run->refresh(); + } + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $delivery = $this->deliver($run, $task, $scope, 'local_activity'); + $commands = []; + foreach ([0, 1] as $index) { + $commands[] = [ + ...$this->scopedCleanupDescriptor($scope, $scope, $request, $delivery), + 'type' => 'prepare_local_activity', + ...ParallelChildGroup::itemMetadata(4, 2, $index, 'activity'), + ]; + } + if ($invalidSecond) { + $commands[1]['cancellation_cleanup']['request_id'] = 'another-request'; + } + $history = $run->historyEvents() + ->count(); + $bridge = app(DefaultWorkflowTaskBridge::class); + $reply = $bridge->checkpointLocalActivityGroup($task->id, 'original', 1, 'cleanup-group', 4, $commands, '1.20'); + if ($invalidSecond) { + $this->assertFalse($reply['checkpointed']); + $this->assertSame( + $runRequest ? 'cancellation_requested' : 'operation_scope_cancellation_prepared', + $reply['reason'] + ); + $this->assertSame($history, $run->historyEvents()->count()); + $this->assertSame(0, $run->activityExecutions()->count()); + return; + } + $this->assertTrue($reply['checkpointed'], $reply['reason'] ?? ''); + $this->assertCount(2, $reply['local_activities']); + $snapshots = []; + foreach ($commands as $index => $command) { + $prepared = $bridge->prepareLocalActivity($task->id, 'original', 1, 4 + $index, 'cleanup-' . $index, [ + ...$command, + 'type' => 'record_local_activity', + ], '1.20'); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $snapshots[] = $prepared['cancellation_cleanup']; + $this->assertTrue( + $bridge->controlLocalActivity($prepared['activity_attempt_id'], 'original', 1, true, '1.20')['active'] + ); + $outcome = $bridge->recordLocalActivityOutcome($prepared['activity_attempt_id'], 'original', 1, [ + 'outcome' => 'completed', + 'result' => Serializer::serializeWithCodec('avro', $index), + 'payload_codec' => 'avro', + ], '1.20'); + $this->assertTrue($outcome['recorded'], $outcome['reason'] ?? ''); + } + $this->assertSameJsonObject($snapshots[0], $snapshots[1]); + $this->assertSame(2, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCompleted)->count()); + $run->refresh(); + if ($runRequest) { + $this->assertTrue( + $bridge->deliverCancellation($task->id, $run->cancellation_request_command_id, 6, 'timer')['delivered'] + ); + $this->assertTrue($bridge->complete($task->id, [[ + 'type' => 'complete_workflow', + 'output' => Serializer::serializeWithCodec('avro', 'group cleaned'), + 'payload_codec' => 'avro', + ]])['completed']); + $this->assertSame(RunStatus::Cancelled, $run->refresh()->status); + } else { + $this->assertNull($run->cancellation_request_command_id); + } + } + + public static function scopedCleanupGroups(): iterable + { + yield 'both original proofs' => [false, false]; + yield 'bad second member leaves no first effect' => [true, false]; + yield 'inherited run request' => [false, true]; + yield 'inherited run request with bad second member' => [true, true]; + } + + #[DataProvider('hostingRenewalPaths')] + public function testRunInheritedScopeCleanupCannotExtendTheOriginalDeadline(string $path): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + $this->assertTrue(WorkflowStub::loadRun($run->id)->requestCancellation('clean up the run', 30)->accepted()); + $run->refresh(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 300); + $delivery = $this->deliver($run, $task, $scope, 'local_activity'); + $descriptor = $this->scopedCleanupDescriptor($scope, $scope, $request, $delivery); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prepared = $bridge->prepareLocalActivity($task->id, 'original', 1, 4, 'bounded-cleanup', $descriptor, '1.20'); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $snapshot = $prepared['cancellation_cleanup']; + $this->assertSame('2026-10-03T00:00:30.000000Z', $snapshot['cleanup_deadline_at']); + foreach ([8, 8, 8] as $seconds) { + Carbon::setTestNow(now()->addSeconds($seconds)); + $control = $bridge->controlLocalActivity($prepared['activity_attempt_id'], 'original', 1, true, '1.20'); + $this->assertTrue($control['active'], $control['reason'] ?? ''); + } + $renewal = match ($path) { + 'workflow heartbeat' => $bridge->heartbeat($task->id)['renewed'], + 'surviving callback control' => $bridge->controlLocalActivity( + $prepared['activity_attempt_id'], + 'original', + 1, + true, + '1.20' + )['active'], + 'embedded local renewal' => LocalActivityRuntime::renewWorkflowTask($task->fresh()) !== null, + }; + $this->assertTrue($renewal); + $this->assertSame($snapshot['cleanup_deadline_at'], $task->fresh()->lease_expires_at->toISOString()); + Carbon::setTestNow(now()->addSeconds(7)); + $control = $bridge->controlLocalActivity($prepared['activity_attempt_id'], 'original', 1, true, '1.20'); + $this->assertFalse($control['active']); + $this->assertFalse($control['renewed']); + $this->assertTrue($control['fenced']); + $this->assertSame('cancellation_deadline_expired', $control['reason']); + $outcome = $bridge->recordLocalActivityOutcome($prepared['activity_attempt_id'], 'original', 1, [ + 'outcome' => 'completed', + 'result' => Serializer::serializeWithCodec('avro', 'late'), + 'payload_codec' => 'avro', + ], '1.20'); + $this->assertFalse($outcome['recorded']); + $this->assertSame('cancellation_deadline_expired', $outcome['reason']); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityHeartbeatRecorded)->count() + ); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCompleted)->count()); + $this->assertSameJsonObject($snapshot, ActivityExecution::query()->findOrFail( + $prepared['activity_execution_id'], + )->activity_options['cancellation_cleanup']); + } + + public function testScopedCleanupCannotBorrowAnOriginalShieldedDescendant(): void + { + [, $run, $task, $parent, $shield] = $this->scopeTree(true); + $request = CancellationScopeRequests::request($run, $parent, '1.20', 30); + $delivery = $this->deliver($run, $task, $parent, 'local_activity'); + $descriptor = $this->scopedCleanupDescriptor($parent, $shield, $request, $delivery); + $reply = app(DefaultWorkflowTaskBridge::class)->prepareLocalActivity( + $task->id, + 'original', + 1, + 4, + 'shield-cleanup', + $descriptor, + '1.20' + ); + $this->assertFalse($reply['prepared']); + $this->assertSame('local_activity_cleanup_authority_mismatch', $reply['reason']); + $this->assertSame(0, $run->activityExecutions()->count()); + } + + #[DataProvider('damagedScopedCleanupAuthority')] + public function testScopedCleanupLosingItsStoredAuthorityCannotRenewOrPublish(bool $rewriteBoth): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $delivery = $this->deliver($run, $task, $scope, 'local_activity'); + $descriptor = $this->scopedCleanupDescriptor($scope, $scope, $request, $delivery); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prepared = $bridge->prepareLocalActivity($task->id, 'original', 1, 4, 'cleanup-attempt', $descriptor, '1.20'); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $execution = ActivityExecution::query()->findOrFail($prepared['activity_execution_id']); + $options = $execution->activity_options; + if ($rewriteBoth) { + $options['cancellation_cleanup']['authority_deadline_at'] = now()->addHour()->toISOString(); + $started = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted)->sole(); + $payload = $started->payload; + $payload['local_preparation']['cancellation_cleanup'] = $options['cancellation_cleanup']; + $started->forceFill([ + 'payload' => $payload, + ])->save(); + } else { + unset($options['cancellation_cleanup']); + } + $execution->forceFill([ + 'activity_options' => $options, + ])->save(); + $history = $run->historyEvents() + ->count(); + $control = $bridge->controlLocalActivity($prepared['activity_attempt_id'], 'original', 1, true, '1.20'); + $this->assertFalse($control['active']); + $this->assertFalse($control['renewed']); + $this->assertSame('local_activity_preparation_mismatch', $control['reason']); + $outcome = $bridge->recordLocalActivityOutcome($prepared['activity_attempt_id'], 'original', 1, [ + 'outcome' => 'completed', + 'result' => Serializer::serializeWithCodec('avro', 'unfenced'), + 'payload_codec' => 'avro', + ], '1.20'); + $this->assertFalse($outcome['recorded']); + $this->assertSame('local_activity_preparation_mismatch', $outcome['reason']); + $this->assertSame($history, $run->historyEvents()->count()); + } + + public static function damagedScopedCleanupAuthority(): iterable + { + yield 'lost execution authority' => [false]; + yield 'matching rewritten snapshots cannot invent a budget' => [true]; + } + + #[DataProvider('currentScopedCleanupLimits')] + public function testShorterCurrentRunLimitsStillEndScopedCleanup(string $field): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $delivery = $this->deliver($run, $task, $scope, 'local_activity'); + $descriptor = $this->scopedCleanupDescriptor($scope, $scope, $request, $delivery); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prepared = $bridge->prepareLocalActivity($task->id, 'original', 1, 4, 'cleanup-attempt', $descriptor, '1.20'); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $run->forceFill([ + $field => now() + ->addSeconds(3), + ])->save(); + Carbon::setTestNow(now()->addSeconds(4)); + $history = $run->historyEvents() + ->count(); + $this->assertSame('run_deadline_expired', $bridge->controlLocalActivity( + $prepared['activity_attempt_id'], + 'original', + 1, + true, + '1.20', + )['reason']); + $outcome = $bridge->recordLocalActivityOutcome($prepared['activity_attempt_id'], 'original', 1, [ + 'outcome' => 'completed', + 'result' => Serializer::serializeWithCodec('avro', 'too late'), + 'payload_codec' => 'avro', + ], '1.20'); + $this->assertFalse($outcome['recorded']); + $this->assertSame('run_deadline_expired', $outcome['reason']); + $this->assertSame($history, $run->historyEvents()->count()); + $this->assertSameJsonObject($prepared['cancellation_cleanup'], ActivityExecution::query()->findOrFail( + $prepared['activity_execution_id'], + )->activity_options['cancellation_cleanup']); + } + + public static function currentScopedCleanupLimits(): iterable + { + yield 'execution budget' => ['execution_deadline_at']; + yield 'run budget' => ['run_deadline_at']; + } + + public static function scopedCleanupAddresses(): iterable + { + yield 'direct scope' => [false, false]; + yield 'original unshielded descendant' => [true, false]; + yield 'run request in direct scope' => [false, true]; + yield 'run request in original unshielded descendant' => [true, true]; + } + + public function testLaterAncestorBudgetStopsScopedCleanupWithoutChangingItsOriginalSnapshot(): void + { + [, $run, $task, $parent, $scope] = $this->scopeTree(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $delivery = $this->deliver($run, $task, $scope, 'local_activity'); + $descriptor = $this->scopedCleanupDescriptor($scope, $scope, $request, $delivery); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prepared = $bridge->prepareLocalActivity($task->id, 'original', 1, 4, 'cleanup-attempt', $descriptor, '1.20'); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + Carbon::setTestNow(now()->addSecond()); + CancellationScopeRequests::request($run->fresh(), $parent, '1.20', 10); + Carbon::setTestNow(now()->addSeconds(8)); + $this->assertTrue($bridge->heartbeat($task->id)['renewed']); + Carbon::setTestNow(now()->addSeconds(3)); + $history = $run->historyEvents() + ->count(); + $control = $bridge->controlLocalActivity($prepared['activity_attempt_id'], 'original', 1, true, '1.20'); + $this->assertFalse($control['active']); + $this->assertFalse($control['renewed']); + $this->assertSame('local_activity_cleanup_authority_expired', $control['reason']); + $this->assertSame('local_activity_cleanup_authority_expired', $bridge->prepareLocalActivity( + $task->id, + 'original', + 1, + 5, + 'expired-cleanup', + $descriptor, + '1.20', + )['reason']); + $outcome = $bridge->recordLocalActivityOutcome($prepared['activity_attempt_id'], 'original', 1, [ + 'outcome' => 'completed', + 'result' => Serializer::serializeWithCodec('avro', 'late'), + 'payload_codec' => 'avro', + ], '1.20'); + $this->assertFalse($outcome['recorded']); + $this->assertSame('local_activity_cleanup_authority_expired', $outcome['reason']); + $this->assertSame($history, $run->historyEvents()->count()); + $this->assertSameJsonObject($prepared['cancellation_cleanup'], ActivityExecution::query()->findOrFail( + $prepared['activity_execution_id'], + )->activity_options['cancellation_cleanup']); + $this->assertSame( + '2026-10-03T00:00:30.000000Z', + CancellationScopeRequests::context($run->fresh(), $scope)->deadline()->toISOString() + ); + } + + #[DataProvider('scopedCleanupBeforeAncestor')] + public function testScopedCleanupRemainsOriginalWhenALaterAncestorPreparesAndDelivers(string $mode): void + { + [, $run, $task, $parent, $scope] = $this->scopeTree(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $delivery = $this->deliver($run, $task, $scope, 'local_activity'); + $descriptor = $this->scopedCleanupDescriptor($scope, $scope, $request, $delivery); + $bridge = app(DefaultWorkflowTaskBridge::class); + $boundary = $mode === 'single' ? 5 : 6; + if ($mode !== 'single') { + $commands = []; + foreach ([0, 1] as $index) { + $commands[] = [ + ...$descriptor, + 'type' => 'prepare_local_activity', + ...ParallelChildGroup::itemMetadata(4, 2, $index, 'activity'), + ]; + } + $reply = $bridge->checkpointLocalActivityGroup( + $task->id, + 'original', + 1, + 'cleanup-group', + 4, + $commands, + '1.20' + ); + $this->assertTrue($reply['checkpointed'], $reply['reason'] ?? ''); + $descriptor = [ + ...$commands[0], + 'type' => 'record_local_activity', + ]; + } + if ($mode !== 'pending group') { + $cleanup = $bridge->prepareLocalActivity( + $task->id, + 'original', + 1, + 4, + 'cleanup-attempt', + $descriptor, + '1.20' + ); + $this->assertTrue($cleanup['prepared'], $cleanup['reason'] ?? ''); + } + Carbon::setTestNow(now()->addSecond()); + $parentRequest = CancellationScopeRequests::request($run->fresh(), $parent, '1.20', 10); + $preparation = CancellationScopeDelivery::prepare( + $run, + $task, + $parent, + $parentRequest->payload['request_id'], + $boundary, + 'local_activity', + '1.20', + ); + $this->assertSame([], $preparation->payload['descendant_members'][0]['activity_members']); + $parentDelivery = CancellationScopeDelivery::record( + $run, + $task, + $parent, + $parentRequest->payload['request_id'], + $boundary, + 'local_activity', + '1.20', + ); + $this->assertSame($delivery->id, CancellationScopeDelivery::recorded($run->fresh(), $scope)->id); + $this->assertSame($parentDelivery->id, CancellationScopeDelivery::recorded($run->fresh(), $parent)->id); + if ($mode === 'pending group') { + $cleanup = $bridge->prepareLocalActivity( + $task->id, + 'original', + 1, + 4, + 'cleanup-attempt', + $descriptor, + '1.20' + ); + $this->assertTrue($cleanup['prepared'], $cleanup['reason'] ?? ''); + } + $this->assertTrue( + $bridge->controlLocalActivity($cleanup['activity_attempt_id'], 'original', 1, true, '1.20')['active'] + ); + $this->assertSameJsonObject($cleanup['cancellation_cleanup'], ActivityExecution::query()->findOrFail( + $cleanup['activity_execution_id'], + )->activity_options['cancellation_cleanup']); + Carbon::setTestNow(now()->addSeconds(10)); + $this->assertSame('local_activity_cleanup_authority_expired', $bridge->controlLocalActivity( + $cleanup['activity_attempt_id'], + 'original', + 1, + true, + '1.20', + )['reason']); + } + + public static function scopedCleanupBeforeAncestor(): iterable + { + foreach (['single', 'running group', 'pending group'] as $mode) { + yield $mode => [$mode]; + } + } + + #[DataProvider('damagedScheduledCleanup')] + public function testScopedCleanupCannotHideInvalidHistoryFromALaterAncestor(string $mutation): void + { + [, $run, $task, $parent, $scope] = $this->scopeTree(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $delivery = $this->deliver($run, $task, $scope, 'local_activity'); + $descriptor = $this->scopedCleanupDescriptor($scope, $scope, $request, $delivery); + $cleanup = app(DefaultWorkflowTaskBridge::class)->prepareLocalActivity( + $task->id, + 'original', + 1, + 4, + 'cleanup-attempt', + $descriptor, + '1.20', + ); + $this->assertTrue($cleanup['prepared'], $cleanup['reason'] ?? ''); + $scheduled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityScheduled)->sole(); + $payload = $scheduled->payload; + $snapshot = &$payload['local_preparation']['cancellation_cleanup']; + match ($mutation) { + 'invented deadline' => $snapshot['cleanup_deadline_at'] = now()->addHour()->toISOString(), + 'wrong scope' => $snapshot['scope_id'] = $parent, + 'missing delivery' => $snapshot['delivery_history_event_id'] = 'missing-delivery', + }; + $scheduled->forceFill([ + 'payload' => $payload, + ])->save(); + $parentRequest = CancellationScopeRequests::request($run->fresh(), $parent, '1.20', 10); + $history = $run->historyEvents() + ->count(); + try { + CancellationScopeDelivery::prepare( + $run, + $task, + $parent, + $parentRequest->payload['request_id'], + 5, + 'local_activity', + '1.20', + ); + $this->fail('Invalid cleanup history was excluded from cancellation inventory.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_cleanup_history_invalid', $error->getMessage()); + } + $this->assertSame($history, $run->historyEvents()->count()); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeDeliveryPrepared)->count() + ); + } + + public static function damagedScheduledCleanup(): iterable + { + foreach (['invented deadline', 'wrong scope', 'missing delivery'] as $mutation) { + yield $mutation => [$mutation]; + } + } + + public function testScopedCleanupRenewalKeepsItsUnaffectedHostingClaimAndBoundsItsOwnLease(): void + { + [, $run, $task, $parent, $scope] = $this->scopeTree(); + config() + ->set('workflows.v2.workflow_task_lease_seconds', 60); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $delivery = $this->deliver($run, $task, $scope, 'local_activity'); + $descriptor = $this->scopedCleanupDescriptor($scope, $scope, $request, $delivery); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prepared = $bridge->prepareLocalActivity($task->id, 'original', 1, 4, 'cleanup-attempt', $descriptor, '1.20'); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $root = $descriptor; + unset($root['cancellation_cleanup']); + $root['cancellation_scope_id'] = CancellationScopeHistory::ROOT_SCOPE_ID; + $unaffected = $bridge->prepareLocalActivity($task->id, 'original', 1, 5, 'unaffected-attempt', $root, '1.20'); + $this->assertTrue($unaffected['prepared'], $unaffected['reason'] ?? ''); + Carbon::setTestNow(now()->addSecond()); + CancellationScopeRequests::request($run->fresh(), $parent, '1.20', 10); + Carbon::setTestNow(now()->addSecond()); + $control = $bridge->controlLocalActivity($prepared['activity_attempt_id'], 'original', 1, true, '1.20'); + $this->assertTrue($control['active'], $control['reason'] ?? ''); + $this->assertSame('2026-10-03T00:00:11.000000Z', $control['lease_expires_at']); + $this->assertSame('2026-10-03T00:00:12.000000Z', $control['workflow_lease_expires_at']); + Carbon::setTestNow(now()->addSeconds(7)); + $this->assertTrue( + $bridge->controlLocalActivity($unaffected['activity_attempt_id'], 'original', 1, true, '1.20')['active'] + ); + Carbon::setTestNow(now()->addSeconds(9)); + $this->assertTrue( + $bridge->controlLocalActivity($unaffected['activity_attempt_id'], 'original', 1, true, '1.20')['active'] + ); + Carbon::setTestNow(now()->addSeconds(2)); + $this->assertTrue( + $bridge->controlLocalActivity($unaffected['activity_attempt_id'], 'original', 1, false, '1.20')['active'] + ); + $outcome = $bridge->recordLocalActivityOutcome($unaffected['activity_attempt_id'], 'original', 1, [ + 'outcome' => 'completed', + 'result' => Serializer::serializeWithCodec('avro', 'survived'), + 'payload_codec' => 'avro', + ], '1.20'); + $this->assertTrue($outcome['recorded'], $outcome['reason'] ?? ''); + $this->assertFalse( + $bridge->controlLocalActivity($prepared['activity_attempt_id'], 'original', 1, true, '1.20')['active'] + ); + } + + public function testScopedDeliveryImmediatelyBoundsHostingWithoutEndingTheSiblingBudget(): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + config() + ->set('workflows.v2.workflow_task_lease_seconds', 60); + CancellationScopeRequests::request($run, $scope, '1.20', 5); + $delivery = $this->deliver($run, $task, $scope, 'local_activity'); + $claim = $task->fresh(); + $this->assertSame('original', $claim->lease_owner); + $this->assertSame(1, $claim->attempt_count); + $this->assertSame(TaskStatus::Leased, $claim->status); + $this->assertSame('2026-10-03T00:00:10.000000Z', $claim->lease_expires_at->toISOString()); + $this->assertSame('2026-10-03T00:00:05.000000Z', $delivery->payload['authority_deadline_at']); + $this->assertNull($run->fresh()->cancellation_request_command_id); + } + + #[DataProvider('hostingRenewalPaths')] + public function testScopedCleanupCannotLoseShortHostingOwnershipThroughAnUnaffectedRenewal(string $path): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + config() + ->set('workflows.v2.workflow_task_lease_seconds', 60); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $delivery = $this->deliver($run, $task, $scope, 'local_activity'); + $descriptor = $this->scopedCleanupDescriptor($scope, $scope, $request, $delivery); + $bridge = app(DefaultWorkflowTaskBridge::class); + $cleanup = $bridge->prepareLocalActivity($task->id, 'original', 1, 4, 'cleanup-attempt', $descriptor, '1.20'); + $this->assertTrue($cleanup['prepared'], $cleanup['reason'] ?? ''); + unset($descriptor['cancellation_cleanup']); + $descriptor['cancellation_scope_id'] = CancellationScopeHistory::ROOT_SCOPE_ID; + $sibling = $bridge->prepareLocalActivity($task->id, 'original', 1, 5, 'sibling-attempt', $descriptor, '1.20'); + $this->assertTrue($sibling['prepared'], $sibling['reason'] ?? ''); + Carbon::setTestNow(now()->addSeconds(2)); + $renewal = match ($path) { + 'workflow heartbeat' => $bridge->heartbeat($task->id)['renewed'], + 'surviving callback control' => $bridge->controlLocalActivity( + $sibling['activity_attempt_id'], + 'original', + 1, + true, + '1.20' + )['active'], + 'embedded local renewal' => LocalActivityRuntime::renewWorkflowTask($task->fresh()) !== null, + }; + $this->assertTrue($renewal); + $this->assertSame('2026-10-03T00:00:12.000000Z', $task->fresh()->lease_expires_at->toISOString()); + $this->assertTrue( + $bridge->controlLocalActivity($sibling['activity_attempt_id'], 'original', 1, false, '1.20')['active'] + ); + $this->assertSameJsonObject($cleanup['cancellation_cleanup'], ActivityExecution::query()->findOrFail( + $cleanup['activity_execution_id'], + )->activity_options['cancellation_cleanup']); + $this->assertSame('2026-10-03T00:00:30.000000Z', $cleanup['cancellation_cleanup']['cleanup_deadline_at']); + $this->assertSame('original', $task->fresh()->lease_owner); + $this->assertSame(1, $task->fresh()->attempt_count); + } + + public static function hostingRenewalPaths(): iterable + { + foreach (['workflow heartbeat', 'surviving callback control', 'embedded local renewal'] as $path) { + yield $path => [$path]; + } + } + + public function testExpiredScopeBudgetRestoresOrdinaryHostingForUnaffectedWorkWithoutRevivingCleanup(): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + config() + ->set('workflows.v2.workflow_task_lease_seconds', 60); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 5); + $delivery = $this->deliver($run, $task, $scope, 'local_activity'); + $descriptor = $this->scopedCleanupDescriptor($scope, $scope, $request, $delivery); + $bridge = app(DefaultWorkflowTaskBridge::class); + $cleanup = $bridge->prepareLocalActivity($task->id, 'original', 1, 4, 'cleanup-attempt', $descriptor, '1.20'); + $this->assertTrue($cleanup['prepared'], $cleanup['reason'] ?? ''); + unset($descriptor['cancellation_cleanup']); + $descriptor['cancellation_scope_id'] = CancellationScopeHistory::ROOT_SCOPE_ID; + $sibling = $bridge->prepareLocalActivity($task->id, 'original', 1, 5, 'sibling-attempt', $descriptor, '1.20'); + $this->assertTrue($sibling['prepared'], $sibling['reason'] ?? ''); + Carbon::setTestNow(now()->addSeconds(5)); + $control = $bridge->controlLocalActivity($sibling['activity_attempt_id'], 'original', 1, true, '1.20'); + $this->assertTrue($control['active'], $control['reason'] ?? ''); + $this->assertSame('2026-10-03T00:01:05.000000Z', $control['workflow_lease_expires_at']); + $this->assertFalse( + $bridge->controlLocalActivity($cleanup['activity_attempt_id'], 'original', 1, true, '1.20')['active'] + ); + $this->assertSame('2026-10-03T00:00:05.000000Z', $cleanup['cancellation_cleanup']['cleanup_deadline_at']); + $this->assertNull($run->fresh()->cancellation_request_command_id); + } + + #[DataProvider('independentHostingBudgets')] + public function testHostingRemainsRecoverableUntilEveryIndependentScopeBudgetExpires( + int $firstBudget, + int $secondBudget + ): void { + [, $run, $task, , $scope] = $this->scopeTree(); + config() + ->set('workflows.v2.workflow_task_lease_seconds', 60); + CancellationScopeRequests::request($run, $scope, '1.20', $firstBudget); + $first = $this->deliver($run, $task, $scope, 'local_activity'); + $sibling = CancellationScopeHistory::open($run, $task, 4, '1.20')->payload['scope_id']; + $request = CancellationScopeRequests::request($run, $sibling, '1.20', $secondBudget); + CancellationScopeDelivery::prepare( + $run, + $task, + $sibling, + $request->payload['request_id'], + 5, + 'local_activity', + '1.20' + ); + $second = CancellationScopeDelivery::record( + $run, + $task, + $sibling, + $request->payload['request_id'], + 5, + 'local_activity', + '1.20' + ); + $bridge = app(DefaultWorkflowTaskBridge::class); + $this->assertSame( + '2026-10-03T00:00:20.000000Z', + $run->fresh() + ->cancellation_scope_recovery_until->toISOString() + ); + Carbon::setTestNow('2026-10-03T00:00:05Z'); + $this->assertTrue($bridge->heartbeat($task->id)['renewed']); + $this->assertSame('2026-10-03T00:00:15.000000Z', $task->fresh()->lease_expires_at->toISOString()); + Carbon::setTestNow('2026-10-03T00:00:14Z'); + $this->assertTrue($bridge->heartbeat($task->id)['renewed']); + $this->assertSame('2026-10-03T00:00:24.000000Z', $task->fresh()->lease_expires_at->toISOString()); + Carbon::setTestNow('2026-10-03T00:00:20Z'); + $this->assertTrue($bridge->heartbeat($task->id)['renewed']); + $this->assertSame('2026-10-03T00:01:20.000000Z', $task->fresh()->lease_expires_at->toISOString()); + $this->assertSame( + $first->fresh() + ->getAttributes(), + CancellationScopeDelivery::recorded($run->fresh(), $scope)->getAttributes() + ); + $this->assertSame( + $second->fresh() + ->getAttributes(), + CancellationScopeDelivery::recorded($run->fresh(), $sibling)->getAttributes() + ); + $this->assertSame('original', $task->fresh()->lease_owner); + $this->assertSame(1, $task->fresh()->attempt_count); + $this->assertNull($run->fresh()->cancellation_request_command_id); + } + + public static function independentHostingBudgets(): iterable + { + yield 'later delivery has longer budget' => [5, 20]; + yield 'later delivery has shorter budget' => [20, 5]; + } + + public function testShieldedCleanupTimerUsesOriginalDeliveryAndRemainsOutsideLaterCancellationInventory(): void + { + [, $run, $task, , $scope] = $this->scopeTree(); + $run->forceFill([ + 'execution_deadline_at' => now() + ->addSeconds(15), + ])->save(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $delivery = $this->deliver($run, $task, $scope, 'timer'); + $proof = $this->scopedCleanupDescriptor($scope, $scope, $request, $delivery)['cancellation_cleanup']; + $command = [ + 'type' => 'start_timer', + 'delay_seconds' => 1, + 'cancellation_scope_id' => $scope, + 'cancellation_cleanup' => $proof, + ]; + $normalized = \Workflow\V2\Support\WorkflowCommandNormalizer::normalize([$command], '1.20'); + $this->assertSame($command, $normalized[0]); + $reply = app(DefaultWorkflowTaskBridge::class)->complete($task->id, $normalized); + $this->assertTrue($reply['completed'], $reply['reason'] ?? ''); + $timer = $run->timers() + ->sole(); + $this->assertSame(1, $timer->delay_seconds); + $this->assertSame('2026-10-03T00:00:01.000000Z', $timer->fire_at->toISOString()); + $scheduled = $run->historyEvents() + ->where('event_type', HistoryEventType::TimerScheduled)->sole(); + $snapshot = $scheduled->payload['cancellation_cleanup']; + $this->assertSame($delivery->id, $snapshot['delivery_history_event_id']); + $this->assertSame($request->payload['request_id'], $snapshot['request_id']); + $this->assertSame('2026-10-03T00:00:30.000000Z', $snapshot['cleanup_deadline_at']); + $this->assertSame('2026-10-03T00:00:15.000000Z', $snapshot['authority_deadline_at']); + $this->assertSame([], ScopedTimerCancellation::members($run->fresh(), $scope)); + $this->assertNull($run->fresh()->cancellation_request_command_id); + $this->assertFalse($run->fresh()->status->isTerminal()); + } + + #[DataProvider('invalidCleanupTimers')] + public function testCleanupTimerCannotInventAuthorityOrOutliveItsOriginalBudget( + string $mutation, + string $reason + ): void { + [, $run, $task, $parent, $scope] = $this->scopeTree(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $delivery = $this->deliver($run, $task, $scope, 'timer'); + $command = [ + 'type' => 'start_timer', + 'delay_seconds' => 1, + 'cancellation_scope_id' => $scope, + 'cancellation_cleanup' => $this->scopedCleanupDescriptor( + $scope, + $scope, + $request, + $delivery + )['cancellation_cleanup'], + ]; + match ($mutation) { + 'missing proof' => $command = array_diff_key($command, [ + 'cancellation_cleanup' => true, + ]), + 'wrong request' => $command['cancellation_cleanup']['request_id'] = 'another-request', + 'wrong delivery' => $command['cancellation_cleanup']['delivery_history_event_id'] = 'another-delivery', + 'unaffected parent' => $command['cancellation_scope_id'] = $parent, + 'invented budget' => $command['cancellation_cleanup']['cleanup_deadline_at'] = now()->addHour()->toISOString(), + 'timer at deadline' => $command['delay_seconds'] = 30, + 'timer after deadline' => $command['delay_seconds'] = 31, + }; + $before = $run->historyEvents() + ->get() + ->toArray(); + $claim = $task->fresh() + ->getAttributes(); + $reply = app(DefaultWorkflowTaskBridge::class)->complete($task->id, [$command]); + $this->assertFalse($reply['completed']); + $this->assertSame($reason, $reply['reason']); + $this->assertSame($before, $run->historyEvents()->get()->toArray()); + $this->assertSame($claim, $task->fresh()->getAttributes()); + $this->assertSame(0, $run->timers()->count()); + } + + public static function invalidCleanupTimers(): iterable + { + yield 'missing proof' => ['missing proof', 'operation_scope_cancellation_prepared']; + foreach (['wrong request', 'wrong delivery', 'unaffected parent'] as $mutation) { + yield $mutation => [$mutation, 'cancellation_scope_cleanup_authority_mismatch']; + } + yield 'caller cannot invent a budget' => ['invented budget', 'invalid_commands']; + foreach (['timer at deadline', 'timer after deadline'] as $mutation) { + yield $mutation => [$mutation, 'cancellation_scope_cleanup_timer_exceeds_deadline']; + } + } + + private function scopedCleanupDescriptor( + string $address, + string $operationScope, + WorkflowHistoryEvent $request, + WorkflowHistoryEvent $delivery, + ): array { + return [ + 'type' => 'record_local_activity', + 'activity_type' => 'tests.scoped-cleanup', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'cancellation_scope_id' => $operationScope, + 'cancellation_cleanup' => [ + 'scope_id' => $address, + 'request_id' => $request->payload['request_id'], + 'delivery_history_event_id' => $delivery->id, + ], + ]; + } + + private function scopeTree(bool $shield = false): array + { + $workflow = WorkflowStub::make(TestSignalWorkflow::class, 'scope-delivery'); + $workflow->start(); + $run = $workflow->run() + ->fresh(); + $task = $run->tasks() + ->where('task_type', TaskType::Workflow)->sole(); + $task->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'original', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addSeconds(60), + ])->save(); + $parent = CancellationScopeHistory::open($run, $task, 1, '1.20')->payload['scope_id']; + $child = CancellationScopeHistory::open($run, $task, 2, '1.20', $parent, $shield)->payload['scope_id']; + return [$workflow, $run, $task, $parent, $child]; + } + + private function deliver( + WorkflowRun $run, + WorkflowTask $task, + string $scope, + string $kind = 'activity' + ): WorkflowHistoryEvent { + CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + CancellationScopeRequests::context($run, $scope)->requestId, + 3, + $kind, + '1.20' + ); + $this->fenceActivities($run, $task, $scope); + return CancellationScopeDelivery::record( + $run, + $task, + $scope, + CancellationScopeRequests::context($run, $scope)->requestId, + 3, + $kind, + '1.20' + ); + } + + /** + * @param array $payload + */ + private function schedule(WorkflowRun $run, HistoryEventType $type, array $payload): void + { + if ($type === HistoryEventType::ActivityScheduled) { + $scope = $payload['activity']['cancellation_scope_id'] ?? $payload['cancellation_scope_id'] ?? 'root'; + $execution = ActivityExecution::query()->create([ + 'workflow_run_id' => $run->id, + 'sequence' => $payload['sequence'], + 'activity_type' => TestGreetingActivity::class, + 'activity_class' => TestGreetingActivity::class, + 'queue' => 'scope-delivery-activities', + 'status' => ActivityStatus::Pending, + 'payload_codec' => 'avro', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'activity_options' => [ + 'cancellation_scope_id' => $scope, + ], + ]); + $payload['activity_execution_id'] = $execution->id; + $payload['activity'] = [ + 'id' => $execution->id, + ...($payload['activity'] ?? []), + ]; + WorkflowTask::query()->create([ + 'workflow_run_id' => $run->id, + 'namespace' => $run->namespace, + 'task_type' => TaskType::Activity, + 'status' => TaskStatus::Ready, + 'queue' => 'scope-delivery-activities', + 'available_at' => now(), + 'payload' => [ + 'activity_execution_id' => $execution->id, + ], + ]); + } elseif ($type === HistoryEventType::TimerScheduled) { + $timer = WorkflowTimer::query()->create([ + 'workflow_run_id' => $run->id, + 'sequence' => $payload['sequence'], + 'status' => TimerStatus::Pending, + 'delay_seconds' => 60, + 'fire_at' => now() + ->addMinute(), + ]); + $payload = [ + ...$payload, + 'timer_id' => $timer->id, + 'delay_seconds' => 60, + 'fire_at' => $timer->fire_at->toISOString(), + ]; + WorkflowTask::query()->create([ + 'workflow_run_id' => $run->id, + 'namespace' => $run->namespace, + 'task_type' => TaskType::Timer, + 'status' => TaskStatus::Ready, + 'available_at' => $timer->fire_at, + 'payload' => [ + 'timer_id' => $timer->id, + ], + ]); + } elseif ($type === HistoryEventType::ActivityCompleted) { + $execution = $run->activityExecutions() + ->where('sequence', $payload['sequence'])->sole(); + $task = $run->tasks() + ->where('task_type', TaskType::Activity) + ->where('payload->activity_execution_id', $execution->id) + ->sole(); + $bridge = app(DefaultActivityTaskBridge::class); + $claim = $bridge->claimStatus($task->id, 'activity-owner'); + $this->assertTrue($claim['claimed'], $claim['reason'] ?? ''); + $result = $bridge->complete($claim['activity_attempt_id'], Serializer::serializeWithCodec('avro', 'done')); + $this->assertTrue($result['recorded'], $result['reason'] ?? ''); + $run->refresh(); + return; + } + WorkflowHistoryEvent::record($run, $type, $payload); + $run->refresh(); + } + + private function fenceActivities(WorkflowRun $run, WorkflowTask $task, string $scope): void + { + foreach (CancellationScopeDelivery::prepared($run->fresh(), $scope)->payload['timer_members'] as $member) { + ScopedTimerCancellation::fence( + $run, + $task, + $member['timer_id'], + $scope, + CancellationScopeRequests::context($run, $scope)->requestId, + '1.20' + ); + } + foreach ($run->activityExecutions()->get() as $execution) { + if ($execution->activity_options['cancellation_scope_id'] === $scope + && $execution->status === ActivityStatus::Pending) { + $this->assertTrue(ScopedActivityCancellation::fence( + $run, + $task, + $execution->id, + $scope, + CancellationScopeRequests::context($run, $scope)->requestId, + '1.20' + )['fenced']); + } + } + } + + private function assertRefusedWithoutMutation( + WorkflowRun $run, + WorkflowTask $task, + string $reason, + callable $action + ): void { + $history = $run->historyEvents() + ->count(); + $runBefore = $run->fresh() + ->getAttributes(); + $claim = $task->fresh() + ->getAttributes(); + try { + $action(); + $this->fail('Scoped delivery should have been refused.'); + } catch (LogicException $error) { + $this->assertSame($reason, $error->getMessage()); + } + $this->assertSame($history, $run->historyEvents()->count()); + $this->assertSame($runBefore, $run->fresh()->getAttributes()); + $this->assertSame($claim, $task->fresh()->getAttributes()); + } +} diff --git a/tests/Feature/V2/V2CancellationScopeDescendantsTest.php b/tests/Feature/V2/V2CancellationScopeDescendantsTest.php new file mode 100644 index 000000000..abc03969a --- /dev/null +++ b/tests/Feature/V2/V2CancellationScopeDescendantsTest.php @@ -0,0 +1,1660 @@ + 'poll', + ]); + Carbon::setTestNow('2026-10-04T00:00:00Z'); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + public function testPreparationFreezesUnshieldedDescendantsUnderTheOriginalRootAndBudget(): void + { + [$run, $task, $scopes] = $this->tree(); + $before = $task->fresh() + ->getAttributes(); + $request = CancellationScopeRequests::request($run, $scopes['parent'], '1.20', 30, 'original'); + Carbon::setTestNow('2026-10-04T00:00:09Z'); + $prepared = $this->prepare($run, $task, $scopes['parent']); + $members = $prepared->payload['descendant_members']; + $this->assertSame([$scopes['child'], $scopes['grandchild']], array_column($members, 'scope_id')); + foreach ($members as $index => $member) { + $this->assertSame( + $request->payload['request_id'], + $member['cancellation']['root_context']['root_request_id'] + ); + $this->assertSame('2026-10-04T00:00:00.000000Z', $member['cancellation']['root_context']['requested_at']); + $this->assertSame('2026-10-04T00:00:30.000000Z', $member['authority_deadline_at']); + $this->assertSame( + array_slice([$scopes['parent'], $scopes['child'], $scopes['grandchild']], 0, $index + 2), + array_column($member['cancellation']['lineage'], 'scope_id') + ); + $this->assertSame([], $member['activity_members']); + $this->assertSame([], $member['timer_members']); + $this->assertSame([], $member['wait_members']); + $this->assertSame([], $member['child_members']); + } + foreach (['shield', 'shield_child', 'sibling'] as $name) { + $this->assertNull(CancellationScopeRequests::context($run->fresh(), $scopes[$name])); + } + $this->assertHostingShortenedWithoutClaimReplacement($before, $task); + $this->assertNull($run->fresh()->cancellation_request_command_id); + $this->assertSame( + 3, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeRequested)->count() + ); + } + + public function testColdRetryAndReplacementKeepTheOriginalPreparationWithoutGrowingHistory(): void + { + [$run, $task, $scopes] = $this->tree(); + CancellationScopeRequests::request($run, $scopes['parent'], '1.20', 30); + $first = $this->prepare($run, $task, $scopes['parent']); + $count = $run->historyEvents() + ->count(); + Carbon::setTestNow('2026-10-04T00:00:20Z'); + $repair = TaskWatchdog::runPass(respectThrottle: false, runIds: [$run->id]); + $this->assertSame([], $repair['existing_task_failures']); + $this->assertSame(1, $repair['repaired_existing_tasks']); + $this->assertTrue(app(DefaultWorkflowTaskBridge::class)->claimStatus($task->id, 'replacement')['claimed']); + $replacement = $task->fresh(); + $this->assertSame(2, $replacement->attempt_count); + $retry = $this->prepare($run->fresh(), $replacement, $scopes['parent']); + $this->assertSame($first->id, $retry->id); + $this->assertSameJsonObject($first->payload, $retry->payload); + $this->assertSame($count, $run->historyEvents()->count()); + $run->forceFill([ + 'status' => RunStatus::Terminated, + ])->save(); + Carbon::setTestNow('2026-10-04T00:01:00Z'); + $this->assertSameJsonObject( + $first->payload, + CancellationScopeDelivery::prepared($run->fresh(), $scopes['parent'])->payload + ); + } + + public function testCompetingChildRootRemainsAcceptedAndItsOriginalConflictIsFrozen(): void + { + [$run, $task, $scopes] = $this->tree(); + $child = CancellationScopeRequests::request($run, $scopes['child'], '1.20', 20, 'independent'); + Carbon::setTestNow('2026-10-04T00:00:05Z'); + $parent = CancellationScopeRequests::request($run, $scopes['parent'], '1.20', 30, 'ancestor'); + $prepared = $this->prepare($run, $task, $scopes['parent']); + $members = $prepared->payload['descendant_members']; + $conflict = $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeRequestConflicted)->sole(); + $this->assertSame($child->id, $members[0]['request_history_event_id']); + $this->assertSame($conflict->id, $members[0]['propagation_history_event_id']); + $this->assertSame($child->payload['request_id'], $members[0]['request_id']); + foreach ($members as $member) { + $this->assertSame( + $child->payload['request_id'], + $member['cancellation']['root_context']['root_request_id'] + ); + $this->assertSame('2026-10-04T00:00:20.000000Z', $member['authority_deadline_at']); + } + $this->assertSame( + $parent->payload['request_id'], + $conflict->payload['incoming_cancellation']['root_context']['root_request_id'] + ); + $this->assertSame('2026-10-04T00:00:35.000000Z', $prepared->payload['authority_deadline_at']); + $reference = ScopedCancellationPreparation::forScope($run->fresh(), $scopes['child'], $prepared->id); + $this->assertSame($prepared->id, $reference->historyEventId); + $this->assertSame($child->id, $reference->requestHistoryEventId); + $this->assertSame($child->payload['request_id'], $reference->context->rootContext->rootRequestId); + $this->assertSame('2026-10-04T00:00:20.000000Z', $reference->authorityDeadlineAt); + $count = $run->historyEvents() + ->count(); + $this->assertSame($prepared->id, $this->prepare($run->fresh(), $task, $scopes['parent'])->id); + $this->assertSame($count, $run->historyEvents()->count()); + } + + #[DataProvider('completedChildBudgets')] + public function testParentReconcilesAnIndependentlyDeliveredChildWithoutRewritingItsReceipts( + string $timerScope, + int $parentRequestedAt, + int $parentGrace, + bool $timerFired = false, + ): void { + [$run, $task, $scopes] = $this->tree(); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prefix = $bridge->checkpointCancellationScopePrefix( + $task->id, + 'original', + 1, + 'independent-child-timer', + 7, + [[ + 'type' => 'start_timer', + 'delay_seconds' => $timerFired ? 1 : 3600, + 'cancellation_scope_id' => $scopes[$timerScope], + ]], + '1.20' + ); + $this->assertTrue($prefix['checkpointed'], $prefix['reason'] ?? ''); + if ($timerFired) { + Carbon::setTestNow('2026-10-04T00:00:02Z'); + $timerTask = $run->tasks() + ->where('task_type', TaskType::Timer)->sole(); + $this->app->call([new RunTimerTask($timerTask->id), 'handle']); + } + $child = CancellationScopeRequests::request($run, $scopes['child'], '1.20', 20, 'independent'); + $childDeadline = CancellationScopeRequests::context($run, $scopes['child'])->deadline()->toISOString(); + $this->prepare($run, $task, $scopes['child'], 8); + $childDelivery = $bridge->deliverCancellationScope( + $task->id, + 'original', + 1, + $scopes['child'], + $child->payload['request_id'], + 8, + 'timer', + protocolVersion: '1.20' + ); + $this->assertTrue($childDelivery['delivered'], $childDelivery['reason'] ?? ''); + $timer = $run->timers() + ->sole(); + $beforeTimer = $timer->getAttributes(); + $receipt = $run->historyEvents() + ->whereIn('event_type', [HistoryEventType::TimerCancelled, HistoryEventType::TimerFired])->sole(); + $beforeReceipt = $receipt->getAttributes(); + $boundary = CancellationScopeDelivery::recorded($run->fresh(), $scopes['child']); + $beforeBoundary = $boundary->getAttributes(); + $this->advanceWithWorkflowHeartbeats( + $task, + Carbon::parse('2026-10-04T00:00:00Z')->addSeconds($parentRequestedAt) + ); + $parent = CancellationScopeRequests::request($run, $scopes['parent'], '1.20', $parentGrace, 'ancestor'); + $this->prepare($run->fresh(), $task, $scopes['parent'], 9); + $parentDelivery = $bridge->deliverCancellationScope( + $task->id, + 'original', + 1, + $scopes['parent'], + $parent->payload['request_id'], + 9, + 'timer', + protocolVersion: '1.20' + ); + $this->assertTrue($parentDelivery['delivered'], $parentDelivery['reason'] ?? ''); + $this->assertSame($beforeTimer, $timer->fresh()->getAttributes()); + $this->assertSame($beforeReceipt, $receipt->fresh()->getAttributes()); + $this->assertSame($beforeBoundary, $boundary->fresh()->getAttributes()); + $this->assertSame($childDelivery['timer_cancellations'], $parentDelivery['timer_cancellations']); + $this->assertSame( + $child->payload['request_id'], + CancellationScopeRequests::context($run->fresh(), $scopes['child'])->requestId + ); + $this->assertSame( + $childDeadline, + CancellationScopeRequests::context($run->fresh(), $scopes['child'])->deadline()->toISOString() + ); + $parentBoundary = CancellationScopeDelivery::recorded($run->fresh(), $scopes['parent']); + $count = $run->historyEvents() + ->count(); + $retry = $bridge->deliverCancellationScope( + $task->id, + 'original', + 1, + $scopes['parent'], + $parent->payload['request_id'], + 9, + 'timer', + protocolVersion: '1.20' + ); + $this->assertSame($parentDelivery, $retry); + $replacement = $task->fresh(); + $replacement->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 2, + ])->save(); + $replayed = $bridge->deliverCancellationScope( + $task->id, + 'replacement', + 2, + $scopes['parent'], + $parent->payload['request_id'], + 9, + 'timer', + protocolVersion: '1.20' + ); + $this->assertSame($parentDelivery, $replayed); + $this->assertSame($count, $run->historyEvents()->count()); + Carbon::setTestNow('2026-10-04T00:02:00Z'); + $run->forceFill([ + 'status' => RunStatus::Terminated, + ])->save(); + $this->assertSame( + $parentBoundary->id, + CancellationScopeDelivery::recorded($run->fresh(), $scopes['parent'])->id + ); + $this->assertSame($count, $run->historyEvents()->count()); + } + + #[DataProvider('pendingChildOrderings')] + public function testAncestorPreservesAnEarlierPreparationUntilItsOriginalDeliveryCompletes( + string $timerScope, + bool $parentFirst, + int $parentGrace, + bool $corruptOrder = false, + ): void { + [$run, $task, $scopes] = $this->tree(); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prefix = $bridge->checkpointCancellationScopePrefix( + $task->id, + 'original', + 1, + 'pending-independent-child', + 7, + [[ + 'type' => 'start_timer', + 'delay_seconds' => 3600, + 'cancellation_scope_id' => $scopes['parent'], + ], [ + 'type' => 'start_timer', + 'delay_seconds' => 3600, + 'cancellation_scope_id' => $scopes[$timerScope], + ]], + '1.20' + ); + $this->assertTrue($prefix['checkpointed'], $prefix['reason'] ?? ''); + $child = CancellationScopeRequests::request($run->fresh(), $scopes['child'], '1.20', 20, 'independent'); + $childSequence = $timerScope === 'child' ? 8 : 9; + $childPrepared = $this->prepare($run->fresh(), $task, $scopes['child'], $childSequence); + $childContext = CancellationScopeRequests::context($run->fresh(), $scopes['child'])->toArray(); + Carbon::setTestNow('2026-10-04T00:00:05Z'); + $parent = CancellationScopeRequests::request( + $run->fresh(), + $scopes['parent'], + '1.20', + $parentGrace, + 'ancestor' + ); + $parentPrepared = $this->prepare($run->fresh(), $task, $scopes['parent'], 7); + $scheduled = $run->historyEvents() + ->where('event_type', HistoryEventType::TimerScheduled) + ->where('payload->sequence', 8) + ->sole(); + $timer = $run->timers() + ->whereKey($scheduled->payload['timer_id'])->sole(); + $beforeTimers = $run->timers() + ->orderBy('id') + ->get() + ->map->getAttributes() + ->all(); + if ($parentFirst) { + $before = $run->historyEvents() + ->count(); + $pending = $bridge->deliverCancellationScope( + $task->id, + 'original', + 1, + $scopes['parent'], + $parent->payload['request_id'], + 7, + 'timer', + protocolVersion: '1.20' + ); + $this->assertFalse($pending['delivered']); + $this->assertSame('cancellation_scope_descendant_delivery_pending', $pending['reason']); + $this->assertSame($beforeTimers, $run->timers()->orderBy('id')->get()->map->getAttributes()->all()); + $this->assertSame($before, $run->historyEvents()->count()); + } + if ($corruptOrder) { + // Establish the parent's own receipt before the child marker, so the + // only invalid future evidence below is the original child marker. + $parentTimer = $run->timers() + ->where('sequence', 7) + ->sole(); + $this->assertTrue(ScopedTimerCancellation::fence( + $run->fresh(), + $task, + $parentTimer->id, + $scopes['parent'], + $parent->payload['request_id'], + '1.20', + $parentPrepared->id, + )['fenced']); + } + $childDelivery = $bridge->deliverCancellationScope( + $task->id, + 'original', + 1, + $scopes['child'], + $child->payload['request_id'], + $childSequence, + 'timer', + protocolVersion: '1.20' + ); + $this->assertTrue($childDelivery['delivered'], $childDelivery['reason'] ?? ''); + $childMarker = CancellationScopeDelivery::recorded($run->fresh(), $scopes['child']); + $this->assertGreaterThan($parentPrepared->sequence, $childMarker->sequence); + $receipt = $run->historyEvents() + ->where('event_type', HistoryEventType::TimerCancelled) + ->where('payload->timer_id', $timer->id) + ->sole(); + $this->assertSame($childPrepared->id, $receipt->payload['cancellation_scope']['preparation_history_event_id']); + $beforeReceipt = $receipt->getAttributes(); + $afterTimer = $timer->fresh() + ->getAttributes(); + $parentDelivery = $bridge->deliverCancellationScope( + $task->id, + 'original', + 1, + $scopes['parent'], + $parent->payload['request_id'], + 7, + 'timer', + protocolVersion: '1.20' + ); + $this->assertTrue($parentDelivery['delivered'], $parentDelivery['reason'] ?? ''); + $this->assertSame($childDelivery['timer_cancellations'], array_values(array_filter( + $parentDelivery['timer_cancellations'], + static fn (array $receipt): bool => $receipt['timer_id'] === $timer->id, + ))); + $this->assertSame($beforeReceipt, $receipt->fresh()->getAttributes()); + $this->assertSame($afterTimer, $timer->fresh()->getAttributes()); + $this->assertSame( + $childContext, + CancellationScopeRequests::context($run->fresh(), $scopes['child'])->toArray() + ); + $parentMarker = CancellationScopeDelivery::recorded($run->fresh(), $scopes['parent']); + $this->assertNotNull($parentMarker); + if ($corruptOrder) { + $parentSequence = $parentMarker->sequence; + $childSequence = $childMarker->sequence; + $parentMarker->forceFill([ + 'sequence' => $run->historyEvents() + ->max('sequence') + 1, + ])->save(); + $childMarker->forceFill([ + 'sequence' => $parentSequence, + ])->save(); + $parentMarker->forceFill([ + 'sequence' => $childSequence, + ])->save(); + $this->assertNotNull(CancellationScopeDelivery::recorded($run->fresh(), $scopes['child'])); + try { + CancellationScopeDelivery::recorded($run->fresh(), $scopes['parent']); + $this->fail('A future child marker justified an earlier ancestor marker.'); + } catch (LogicException $exception) { + $this->assertSame('cancellation_scope_delivery_history_invalid', $exception->getMessage()); + $this->assertSame( + 'cancellation_scope_descendant_delivery_pending', + $exception->getPrevious()?->getMessage() + ); + } + return; + } + $count = $run->historyEvents() + ->count(); + $task->fresh() + ->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 2, + ])->save(); + $this->assertSame($parentDelivery, $bridge->deliverCancellationScope( + $task->id, + 'replacement', + 2, + $scopes['parent'], + $parent->payload['request_id'], + 7, + 'timer', + protocolVersion: '1.20', + )); + Carbon::setTestNow('2026-10-04T00:02:00Z'); + $run->forceFill([ + 'status' => RunStatus::Terminated, + ])->save(); + $this->assertSame($parentMarker->id, CancellationScopeDelivery::recorded($run->fresh(), $scopes['parent'])->id); + $this->assertSame($count, $run->historyEvents()->count()); + } + + public static function pendingChildOrderings(): iterable + { + foreach (['child', 'grandchild'] as $scope) { + yield $scope . ' finishes after ancestor preparation' => [$scope, false, 30]; + yield $scope . ' finishes after tighter ancestor preparation' => [$scope, false, 5]; + yield $scope . ' ancestor waits for original delivery' => [$scope, true, 30]; + yield $scope . ' tighter ancestor waits for original delivery' => [$scope, true, 5]; + yield $scope . ' future marker cannot justify ancestor delivery' => [$scope, true, 30, true]; + } + } + + public static function completedChildBudgets(): iterable + { + yield 'child original deadline' => ['child', 5, 30]; + yield 'child shorter parent budget' => ['child', 5, 5]; + yield 'child already expired' => ['child', 21, 30]; + yield 'grandchild original ancestor proof' => ['grandchild', 5, 30]; + yield 'grandchild shorter parent budget' => ['grandchild', 5, 5]; + yield 'grandchild already expired' => ['grandchild', 21, 30]; + yield 'child naturally fired' => ['child', 5, 30, true]; + yield 'grandchild naturally fired' => ['grandchild', 5, 30, true]; + } + + #[DataProvider('completedMixedPolicies')] + public function testCompletedMixedChildReconcilesAllFourFamiliesAfterItsDeadline( + string $kind, + string $activityPolicy, + string $childPolicy, + bool $activityCompleted, + bool $childCompleted, + bool $parentPreparedEarly = false, + ): void { + [$run, $task, $scopes] = $this->tree(); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prefix = $bridge->checkpointCancellationScopePrefix( + $task->id, + 'original', + 1, + 'completed-mixed-child', + 7, + [[ + 'type' => 'schedule_activity', + 'activity_type' => 'descendant-activity', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'cancellation_scope_id' => $scopes['child'], + 'cancellation_policy' => $activityPolicy, + 'schedule_to_close_timeout' => 120, + ], [ + 'type' => 'start_timer', + 'delay_seconds' => 3600, + 'cancellation_scope_id' => $scopes['grandchild'], + ], [ + 'type' => 'start_child_workflow', + 'workflow_type' => 'descendant-workflow', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'cancellation_policy' => $childPolicy, + 'cancellation_scope_id' => $scopes['child'], + ]], + '1.20' + ); + $this->assertTrue($prefix['checkpointed'], $prefix['reason'] ?? ''); + if ($activityCompleted) { + $activityTask = $run->tasks() + ->where('task_type', TaskType::Activity)->sole(); + $activityBridge = app(ActivityTaskBridge::class); + $activityClaim = $activityBridge->claimStatus($activityTask->id, 'completed-activity-owner'); + $this->assertTrue($activityBridge->complete( + $activityClaim['activity_attempt_id'], + Serializer::serializeWithCodec('avro', 'completed'), + 'avro' + )['recorded']); + } + if ($childCompleted) { + $scheduledChild = $run->historyEvents() + ->where('event_type', HistoryEventType::ChildWorkflowScheduled)->sole(); + WorkflowStub::loadRun($scheduledChild->payload['child_workflow_run_id'])->attemptCancel( + 'terminal child fixture' + ); + } + $completed = $bridge->complete($task->id, [[ + 'type' => 'open_' . $kind . '_wait', + 'cancellation_scope_id' => $scopes['child'], + 'timeout_seconds' => 3600, + ...($kind === 'signal' ? [ + 'signal_name' => 'ready', + ] : [ + 'condition_key' => 'ready', + ]), + ]]); + $this->assertTrue($completed['completed'], $completed['reason'] ?? ''); + $claim = $run->tasks() + ->create([ + 'namespace' => $run->namespace, + 'task_type' => TaskType::Workflow, + 'status' => TaskStatus::Leased, + 'lease_owner' => 'original', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addMinutes(5), + 'available_at' => now(), + 'connection' => $run->connection, + 'queue' => $run->queue, + 'compatibility' => $run->compatibility, + ]); + $parentSequence = $parentPreparedEarly ? 11 : 12; + $childSequence = $parentPreparedEarly ? 12 : 11; + if ($parentPreparedEarly) { + $parentPrefix = $bridge->checkpointCancellationScopePrefix( + $claim->id, + 'original', + 1, + 'pending-mixed-parent', + 11, + [[ + 'type' => 'start_timer', + 'delay_seconds' => 3600, + 'cancellation_scope_id' => $scopes['parent'], + ]], + '1.20' + ); + $this->assertTrue($parentPrefix['checkpointed'], $parentPrefix['reason'] ?? ''); + } + $child = CancellationScopeRequests::request($run->fresh(), $scopes['child'], '1.20', 20, 'independent'); + $this->prepare($run->fresh(), $claim, $scopes['child'], $childSequence); + if ($parentPreparedEarly) { + Carbon::setTestNow('2026-10-04T00:00:05Z'); + $parent = CancellationScopeRequests::request($run->fresh(), $scopes['parent'], '1.20', 30, 'ancestor'); + $parentPrepared = $this->prepare($run->fresh(), $claim, $scopes['parent'], $parentSequence); + $reference = ScopedCancellationPreparation::forScope($run->fresh(), $scopes['child'], $parentPrepared->id); + $before = [ + ActivityExecution::query()->orderBy('id')->get()->map->getAttributes()->all(), + WorkflowTimer::query()->orderBy('id')->get()->map->getAttributes()->all(), + WorkflowRun::query()->orderBy('id')->get()->map->getAttributes()->all(), + WorkflowTask::query()->orderBy('id')->get()->map->getAttributes()->all(), + $run->historyEvents() + ->count(), + ]; + $pending = $bridge->deliverCancellationScope( + $claim->id, + 'original', + 1, + $scopes['parent'], + $parent->payload['request_id'], + $parentSequence, + 'timer', + protocolVersion: '1.20', + ); + $this->assertFalse($pending['delivered']); + $this->assertSame('cancellation_scope_descendant_delivery_pending', $pending['reason']); + foreach ([ + static fn () => ScopedActivityCancellation::fence( + $run->fresh(), + $claim, + $reference->activityMembers[0]['activity_execution_id'], + $scopes['child'], + $child->payload['request_id'], + '1.20', + $parentPrepared->id + ), + static fn () => ScopedWaitCancellation::fence( + $run->fresh(), + $claim, + $reference->waitMembers[0]['wait_id'], + $scopes['child'], + $child->payload['request_id'], + '1.20', + $parentPrepared->id + ), + static fn () => ScopedTimerCancellation::fence( + $run->fresh(), + $claim, + $reference->waitMembers[0]['timer_id'], + $scopes['child'], + $child->payload['request_id'], + '1.20', + $parentPrepared->id + ), + static fn () => ScopedChildCancellationDelivery::request( + $run->fresh(), + $claim, + $reference->childMembers[0]['child_call_id'], + $scopes['child'], + $child->payload['request_id'], + '1.20', + $parentPrepared->id + ), + ] as $actor) { + try { + $actor(); + $this->fail('An ancestor actor replaced an unfinished original preparation.'); + } catch (LogicException $exception) { + $this->assertSame('cancellation_scope_descendant_delivery_pending', $exception->getMessage()); + } + } + $this->assertSame($before, [ + ActivityExecution::query()->orderBy('id')->get()->map->getAttributes()->all(), + WorkflowTimer::query()->orderBy('id')->get()->map->getAttributes()->all(), + WorkflowRun::query()->orderBy('id')->get()->map->getAttributes()->all(), + WorkflowTask::query()->orderBy('id')->get()->map->getAttributes()->all(), + $run->historyEvents() + ->count(), + ]); + } + $childDelivery = $bridge->deliverCancellationScope( + $claim->id, + 'original', + 1, + $scopes['child'], + $child->payload['request_id'], + $childSequence, + 'timer', + protocolVersion: '1.20' + ); + $this->assertTrue($childDelivery['delivered'], $childDelivery['reason'] ?? ''); + $beforeActivities = ActivityExecution::query()->orderBy('id')->get()->map->getAttributes()->all(); + $beforeTimers = WorkflowTimer::query()->where('sequence', '!=', 11)->orderBy('id')->get() + ->map->getAttributes() + ->all(); + $childRun = WorkflowRun::query()->findOrFail($childDelivery['child_cancellations'][0]['child_workflow_run_id']); + $beforeChildRun = $childRun->getAttributes(); + $beforeClaim = $claim->fresh() + ->getAttributes(); + $this->advanceWithWorkflowHeartbeats($claim, Carbon::parse('2026-10-04T00:00:21Z')); + if (! $parentPreparedEarly) { + $parent = CancellationScopeRequests::request($run->fresh(), $scopes['parent'], '1.20', 30, 'ancestor'); + $this->prepare($run->fresh(), $claim, $scopes['parent'], $parentSequence); + } + $parentDelivery = $bridge->deliverCancellationScope( + $claim->id, + 'original', + 1, + $scopes['parent'], + $parent->payload['request_id'], + $parentSequence, + 'timer', + protocolVersion: '1.20' + ); + $this->assertTrue($parentDelivery['delivered'], $parentDelivery['reason'] ?? ''); + foreach ([ + 'activity_cancellations', + 'timer_cancellations', + 'wait_cancellations', + 'child_cancellations', + ] as $field) { + $this->assertNotEmpty($childDelivery[$field]); + $actual = $parentDelivery[$field]; + if ($parentPreparedEarly && $field === 'timer_cancellations') { + $actual = array_values( + array_filter($actual, static fn (array $receipt): bool => $receipt['sequence'] !== 11) + ); + } + $this->assertSame($childDelivery[$field], $actual); + } + $this->assertSame( + $beforeActivities, + ActivityExecution::query()->orderBy('id')->get()->map->getAttributes()->all() + ); + $this->assertSame( + $beforeTimers, + WorkflowTimer::query()->where('sequence', '!=', 11)->orderBy('id')->get()->map->getAttributes()->all() + ); + $this->assertSame($beforeChildRun, $childRun->fresh()->getAttributes()); + $this->assertHostingShortenedWithoutClaimReplacement($beforeClaim, $claim); + $this->assertNotNull(CancellationScopeDelivery::recorded($run->fresh(), $scopes['parent'])); + } + + public static function completedMixedPolicies(): iterable + { + yield 'signal try policies' => ['signal', 'try_cancel', 'try_cancel', false, false]; + yield 'condition try policies' => ['condition', 'try_cancel', 'try_cancel', false, false]; + yield 'bounded abandoned operations' => ['signal', 'abandon', 'abandon', false, false]; + yield 'wait policies with resolved child' => [ + 'condition', + 'wait_cancellation_completed', + 'wait_cancellation_completed', + false, + true, + ]; + yield 'natural activity completion' => ['signal', 'try_cancel', 'try_cancel', true, false]; + yield 'pending signal try policies' => ['signal', 'try_cancel', 'try_cancel', false, false, true]; + yield 'pending condition try policies' => ['condition', 'try_cancel', 'try_cancel', false, false, true]; + yield 'pending bounded abandoned operations' => ['signal', 'abandon', 'abandon', false, false, true]; + yield 'pending wait policies with resolved child' => [ + 'condition', 'wait_cancellation_completed', 'wait_cancellation_completed', false, true, true, + ]; + yield 'pending natural activity completion' => ['signal', 'try_cancel', 'try_cancel', true, false, true]; + } + + #[DataProvider('invalidCompletedProofs')] + public function testParentCannotUseMissingOrCorruptEarlierDeliveryProof(string $change): void + { + [$run, $task, $scopes] = $this->tree(); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prefix = $bridge->checkpointCancellationScopePrefix( + $task->id, + 'original', + 1, + 'invalid-completed-proof', + 7, + [[ + 'type' => 'start_timer', + 'delay_seconds' => 3600, + 'cancellation_scope_id' => $scopes['child'], + ]], + '1.20' + ); + $this->assertTrue($prefix['checkpointed'], $prefix['reason'] ?? ''); + $child = CancellationScopeRequests::request($run, $scopes['child'], '1.20', 20); + $prepared = $this->prepare($run, $task, $scopes['child'], 8); + $delivery = $bridge->deliverCancellationScope( + $task->id, + 'original', + 1, + $scopes['child'], + $child->payload['request_id'], + 8, + 'timer', + protocolVersion: '1.20' + ); + $this->assertTrue($delivery['delivered'], $delivery['reason'] ?? ''); + $marker = CancellationScopeDelivery::recorded($run->fresh(), $scopes['child']); + Carbon::setTestNow('2026-10-04T00:00:05Z'); + $parent = CancellationScopeRequests::request($run->fresh(), $scopes['parent'], '1.20', 30); + $this->prepare($run->fresh(), $task, $scopes['parent'], 9); + if ($change === 'missing') { + $marker->delete(); + } else { + $event = $change === 'receipt' ? $run->historyEvents() + ->where('event_type', HistoryEventType::TimerCancelled)->sole() : $marker; + $payload = $event->payload; + if ($change === 'receipt') { + $payload['cancellation_scope']['authority_deadline_at'] = '2026-10-04T00:01:00.000000Z'; + } else { + $payload['preparation_history_event_id'] = $parent->id; + } + $event->forceFill([ + 'payload' => $payload, + ])->save(); + } + $before = $run->historyEvents() + ->count(); + $result = $bridge->deliverCancellationScope( + $task->id, + 'original', + 1, + $scopes['parent'], + $parent->payload['request_id'], + 9, + 'timer', + protocolVersion: '1.20' + ); + $this->assertFalse($result['delivered']); + $this->assertSame($before, $run->historyEvents()->count()); + $this->assertSame(1, $run->historyEvents()->where('event_type', HistoryEventType::TimerCancelled)->count()); + $this->assertSame($prepared->id, CancellationScopeDelivery::prepared($run->fresh(), $scopes['child'])->id); + } + + public static function invalidCompletedProofs(): iterable + { + yield 'no completed boundary' => ['missing']; + yield 'wrong original preparation' => ['marker']; + yield 'renewed original receipt deadline' => ['receipt']; + } + + public function testPreparationFreezesEveryDescendantOperationWithoutStartingCancellationEffects(): void + { + [$run, $task, $scopes] = $this->tree(); + $result = app(DefaultWorkflowTaskBridge::class)->checkpointCancellationScopePrefix( + $task->id, + 'original', + 1, + 'descendant-operations', + 7, + [ + [ + 'type' => 'schedule_activity', + 'activity_type' => 'descendant-activity', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'cancellation_scope_id' => $scopes['child'], + ], + [ + 'type' => 'start_timer', + 'delay_seconds' => 3600, + 'cancellation_scope_id' => $scopes['grandchild'], + ], + [ + 'type' => 'start_child_workflow', + 'workflow_type' => 'descendant-workflow', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'cancellation_policy' => 'try_cancel', + 'cancellation_scope_id' => $scopes['child'], + ], + ], + '1.20', + ); + $this->assertTrue($result['checkpointed'], $result['reason'] ?? ''); + WorkflowHistoryEvent::record($run, HistoryEventType::SignalWaitOpened, [ + 'sequence' => 10, + 'signal_wait_id' => 'descendant-wait', + 'signal_name' => 'ready', + 'cancellation_scope_id' => $scopes['child'], + ], $task); + $beforeActivities = ActivityExecution::query()->get()->map->getAttributes()->all(); + $beforeTimers = WorkflowTimer::query()->get()->map->getAttributes()->all(); + $beforeTask = $task->fresh() + ->getAttributes(); + CancellationScopeRequests::request($run, $scopes['parent'], '1.20', 30); + $prepared = $this->prepare($run, $task, $scopes['parent'], 11); + $members = $prepared->payload['descendant_members']; + $this->assertCount(1, $members[0]['activity_members']); + $this->assertCount(1, $members[0]['child_members']); + $this->assertCount(1, $members[0]['wait_members']); + $this->assertCount(1, $members[1]['timer_members']); + $this->assertSame($beforeActivities, ActivityExecution::query()->get()->map->getAttributes()->all()); + $this->assertSame($beforeTimers, WorkflowTimer::query()->get()->map->getAttributes()->all()); + $this->assertHostingShortenedWithoutClaimReplacement($beforeTask, $task); + $this->assertSameJsonObject( + $prepared->payload, + CancellationScopeDelivery::prepared($run->fresh(), $scopes['parent'])->payload + ); + $this->expectExceptionMessage('cancellation_scope_activity_fence_not_established'); + CancellationScopeDelivery::record( + $run, + $task, + $scopes['parent'], + $prepared->payload['request_id'], + 11, + 'timer', + '1.20' + ); + } + + public function testNewShieldCannotEscapePreparedParentButOriginalOpenStillReplays(): void + { + [$run, $task, $scopes] = $this->tree(); + CancellationScopeRequests::request($run, $scopes['parent'], '1.20', 30); + $prepared = $this->prepare($run, $task, $scopes['parent']); + $original = CancellationScopeHistory::open($run->fresh(), $task, 2, '1.20', $scopes['parent']); + $this->assertSame($scopes['child'], $original->payload['scope_id']); + $count = $run->historyEvents() + ->count(); + try { + CancellationScopeHistory::open($run->fresh(), $task, 7, '1.20', $scopes['parent'], true); + $this->fail('A late shield escaped a prepared subtree.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_parent_delivery_prepared', $error->getMessage()); + } + $this->assertSame($count, $run->historyEvents()->count()); + try { + CancellationScopeHistory::open($run->fresh(), $task, 7, '1.20', $scopes['sibling']); + $this->fail('An unrelated opening consumed the reserved delivery position.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_command_sequence_reserved', $error->getMessage()); + } + $this->assertSame($count, $run->historyEvents()->count()); + CancellationScopeDelivery::record( + $run->fresh(), + $task, + $scopes['parent'], + $prepared->payload['request_id'], + 7, + 'timer', + '1.20' + ); + CancellationScopeHistory::open($run->fresh(), $task, 8, '1.20', $scopes['sibling']); + $this->assertSameJsonObject( + $prepared->payload, + CancellationScopeDelivery::prepared($run->fresh(), $scopes['parent'])->payload + ); + } + + public static function inheritedActivityRoots(): iterable + { + yield 'inherited root' => [false]; + yield 'independently accepted child root' => [true]; + } + + public static function inheritedWaitKinds(): iterable + { + yield 'signal with timeout' => ['signal']; + yield 'condition with timeout' => ['condition']; + } + + public function testInheritedWaitPolicyRequiresOriginalActivityStopBeforeParentDelivery(): void + { + [$run, $task, $scopes] = $this->tree(); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prefix = $bridge->checkpointCancellationScopePrefix( + $task->id, + 'original', + 1, + 'inherited-wait-activity', + 7, + [[ + 'type' => 'schedule_activity', + 'activity_type' => 'descendant-activity', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'cancellation_policy' => 'wait_cancellation_completed', + 'cancellation_scope_id' => $scopes['child'], + ]], + '1.20' + ); + $this->assertTrue($prefix['checkpointed'], $prefix['reason'] ?? ''); + $activity = $run->activityExecutions() + ->sole(); + $activityTask = $run->tasks() + ->where('task_type', TaskType::Activity) + ->where('payload->activity_execution_id', $activity->id) + ->sole(); + $activityBridge = app(ActivityTaskBridge::class); + $attempt = $activityBridge->claimStatus($activityTask->id, 'activity-owner'); + $this->assertTrue($attempt['claimed'], $attempt['reason'] ?? ''); + $request = CancellationScopeRequests::request($run->fresh(), $scopes['parent'], '1.20', 30); + $prepared = $this->prepare($run->fresh(), $task, $scopes['parent'], 8); + $beforeClaim = $task->fresh() + ->getAttributes(); + $first = $bridge->deliverCancellationScope( + $task->id, + 'original', + 1, + $scopes['parent'], + $request->payload['request_id'], + 8, + 'timer', + protocolVersion: '1.20' + ); + $this->assertFalse($first['delivered']); + $this->assertSame('cancellation_scope_activity_stop_not_acknowledged', $first['reason']); + $this->assertTrue($first['activity_cancellations'][0]['waiting_for_stop']); + $this->assertSame(ActivityStatus::Cancelled, $activity->fresh()->status); + $this->assertNull(CancellationScopeDelivery::recorded($run->fresh(), $scopes['parent'])); + $this->assertFalse($activityBridge->complete( + $attempt['activity_attempt_id'], + Serializer::serializeWithCodec('avro', 'stale'), + 'avro' + )['recorded']); + $childRequest = CancellationScopeRequests::context($run->fresh(), $scopes['child'])->requestId; + $this->assertFalse(ActivityCancellationAcknowledgement::recordStopped( + $attempt['activity_attempt_id'], + 'activity-owner', + $request->payload['request_id'] + )['acknowledged']); + $this->assertTrue(ActivityCancellationAcknowledgement::recordStopped( + $attempt['activity_attempt_id'], + 'activity-owner', + $childRequest + )['acknowledged']); + $retry = $bridge->deliverCancellationScope( + $task->id, + 'original', + 1, + $scopes['parent'], + $request->payload['request_id'], + 8, + 'timer', + protocolVersion: '1.20' + ); + $this->assertTrue($retry['delivered'], $retry['reason'] ?? ''); + $this->assertFalse($retry['activity_cancellations'][0]['waiting_for_stop']); + $this->assertSame( + $first['activity_cancellations'][0]['history_event_id'], + $retry['activity_cancellations'][0]['history_event_id'] + ); + $this->assertHostingShortenedWithoutClaimReplacement($beforeClaim, $task); + $this->assertNull(CancellationScopeDelivery::prepared($run->fresh(), $scopes['child'])); + $this->assertSame($prepared->id, $retry['preparation_history_event_id']); + } + + #[DataProvider('inheritedWaitKinds')] + public function testMixedInheritedDeliveryResumesCommittedWaitAfterClaimReplacement(string $kind): void + { + [$run, $task, $scopes] = $this->tree(); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prefix = $bridge->checkpointCancellationScopePrefix( + $task->id, + 'original', + 1, + 'mixed-descendant-prefix', + 7, + [ + [ + 'type' => 'schedule_activity', + 'activity_type' => 'descendant-activity', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'cancellation_scope_id' => $scopes['child'], + ], [ + 'type' => 'start_timer', + 'delay_seconds' => 3600, + 'cancellation_scope_id' => $scopes['grandchild'], + ], [ + 'type' => 'start_child_workflow', + 'workflow_type' => 'descendant-workflow', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'cancellation_policy' => 'try_cancel', + 'cancellation_scope_id' => $scopes['child'], + ], + ], + '1.20' + ); + $this->assertTrue($prefix['checkpointed'], $prefix['reason'] ?? ''); + $completed = $bridge->complete($task->id, [[ + 'type' => 'open_' . $kind . '_wait', + 'cancellation_scope_id' => $scopes['child'], + 'timeout_seconds' => 3600, + ...($kind === 'signal' ? [ + 'signal_name' => 'ready', + ] : [ + 'condition_key' => 'ready', + ]), + ]]); + $this->assertTrue($completed['completed'], $completed['reason'] ?? ''); + $claim = $run->tasks() + ->create([ + 'namespace' => $run->namespace, + 'task_type' => TaskType::Workflow, + 'status' => TaskStatus::Leased, + 'lease_owner' => 'first-delivery-owner', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addMinutes(5), + 'available_at' => now(), + 'connection' => $run->connection, + 'queue' => $run->queue, + 'compatibility' => $run->compatibility, + ]); + $run->forceFill([ + 'run_deadline_at' => now() + ->addSeconds(25), + ])->save(); + $request = CancellationScopeRequests::request($run->fresh(), $scopes['parent'], '1.20', 30); + Carbon::setTestNow('2026-10-04T00:00:09Z'); + $prepared = CancellationScopeDelivery::prepare( + $run->fresh(), + $claim, + $scopes['parent'], + $request->payload['request_id'], + 10, + $kind, + '1.20' + ); + $reference = ScopedCancellationPreparation::forScope($run->fresh(), $scopes['child'], $prepared->id); + $wait = $reference->waitMembers[0]; + $receipt = ScopedWaitCancellation::fence( + $run->fresh(), + $claim, + $wait['wait_id'], + $reference->scopeId, + $reference->requestId, + '1.20', + $prepared->id + ); + $this->assertTrue($receipt['cancelled']); + $this->assertTrue($receipt['timer_cancellation']['fenced']); + $beforeReplacement = $run->historyEvents() + ->count(); + $originalClaim = $claim->fresh(); + $claim->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 2, + ])->save(); + $run->forceFill([ + 'run_deadline_at' => now() + ->addSeconds(100), + ])->save(); + Carbon::setTestNow('2026-10-04T00:00:10Z'); + try { + ScopedWaitCancellation::fence( + $run->fresh(), + $originalClaim, + $wait['wait_id'], + $reference->scopeId, + $reference->requestId, + '1.20', + $prepared->id + ); + $this->fail('The prior hosting attempt retained authority after replacement.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_workflow_claim_mismatch', $error->getMessage()); + } + $this->assertSame($beforeReplacement, $run->historyEvents()->count()); + $this->assertSame($receipt, ScopedWaitCancellation::fence( + $run->fresh(), + $claim, + $wait['wait_id'], + $reference->scopeId, + $reference->requestId, + '1.20', + $prepared->id + )); + $result = $bridge->deliverCancellationScope( + $claim->id, + 'replacement', + 2, + $scopes['parent'], + $request->payload['request_id'], + 10, + $kind, + protocolVersion: '1.20' + ); + $this->assertTrue($result['delivered'], $result['reason'] ?? ''); + $this->assertSame(10, $result['sequence']); + $this->assertSame($kind, $result['call_kind']); + $this->assertCount(1, $result['wait_cancellations']); + $this->assertCount(2, $result['timer_cancellations']); + $this->assertCount(1, $result['activity_cancellations']); + $this->assertCount(1, $result['child_cancellations']); + $this->assertSame($receipt['history_event_id'], $result['wait_cancellations'][0]['history_event_id']); + foreach ($run->historyEvents()->whereIn('event_type', [ + HistoryEventType::TimerCancelled, HistoryEventType::SignalWaitCancelled, + HistoryEventType::ConditionWaitCancelled, HistoryEventType::ActivityCancelled, + HistoryEventType::ChildCancellationRequested, + ])->get() as $event) { + $snapshot = $event->payload['cancellation_scope']; + $this->assertSame('2026-10-04T00:00:25.000000Z', $snapshot['authority_deadline_at']); + $this->assertSame( + $request->payload['request_id'], + $snapshot['cancellation']['root_context']['root_request_id'] + ); + if ($event->event_type !== HistoryEventType::ActivityCancelled) { + $this->assertSame($prepared->id, $snapshot['preparation_history_event_id']); + } + } + $child = WorkflowRun::query()->findOrFail($result['child_cancellations'][0]['child_workflow_run_id']); + $childContext = CooperativeCancellationDelivery::context($child); + $this->assertSame($request->payload['request_id'], $childContext->rootRequestId); + $this->assertSame('2026-10-04T00:00:25.000000Z', $childContext->deadline()->toISOString()); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeDeliveryPrepared)->count() + ); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeDelivered)->count() + ); + $recorded = CancellationScopeDelivery::recorded($run->fresh(), $scopes['parent']); + $this->assertNotNull($recorded); + $count = $run->historyEvents() + ->count(); + $this->assertSame($result, $bridge->deliverCancellationScope( + $claim->id, + 'replacement', + 2, + $scopes['parent'], + $request->payload['request_id'], + 10, + $kind, + protocolVersion: '1.20' + )); + $this->assertSame($count, $run->historyEvents()->count()); + $this->advanceWithWorkflowHeartbeats($claim, Carbon::parse('2026-10-04T00:00:40Z')); + $cold = CancellationScopeDelivery::recorded($run->fresh(), $scopes['parent']); + $this->assertSame($recorded->id, $cold->id); + $this->assertSameJsonObject($recorded->payload, $cold->payload); + $expired = $bridge->deliverCancellationScope( + $claim->id, + 'replacement', + 2, + $scopes['parent'], + $request->payload['request_id'], + 10, + $kind, + protocolVersion: '1.20' + ); + $this->assertFalse($expired['delivered']); + $this->assertSame('cancellation_scope_authority_expired', $expired['reason']); + $this->assertSame($count, $run->historyEvents()->count()); + } + + #[DataProvider('inheritedActivityRoots')] + public function testInheritedActivityFenceUsesOriginalPreparationAndPreservesExcludedWork(bool $independent): void + { + [$run, $task, $scopes] = $this->tree(); + $operations = []; + foreach (['child', 'grandchild', 'shield', 'sibling'] as $name) { + $operations[] = [ + 'type' => 'schedule_activity', + 'activity_type' => $name . '-activity', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'cancellation_scope_id' => $scopes[$name], + ]; + } + $checkpoint = app(DefaultWorkflowTaskBridge::class)->checkpointCancellationScopePrefix( + $task->id, + 'original', + 1, + 'inherited-activity-prefix', + 7, + $operations, + '1.20' + ); + $this->assertTrue($checkpoint['checkpointed'], $checkpoint['reason'] ?? ''); + $childRequest = $independent + ? CancellationScopeRequests::request($run->fresh(), $scopes['child'], '1.20', 20, 'independent') : null; + $run->forceFill([ + 'run_deadline_at' => now() + ->addSeconds(5), + ])->save(); + $parentRequest = CancellationScopeRequests::request($run->fresh(), $scopes['parent'], '1.20', 30); + $prepared = $this->prepare($run->fresh(), $task, $scopes['parent'], 11); + $run->forceFill([ + 'run_deadline_at' => now() + ->addSeconds(100), + ])->save(); + Carbon::setTestNow('2026-10-04T00:00:04Z'); + $beforeClaim = $task->fresh() + ->getAttributes(); + $receipts = []; + foreach (['child', 'grandchild'] as $name) { + $member = collect($prepared->payload['descendant_members'])->firstWhere('scope_id', $scopes[$name]); + $activity = ActivityExecution::query()->findOrFail($member['activity_members'][0]['activity_execution_id']); + $receipt = ScopedActivityCancellation::fence( + $run->fresh(), + $task, + $activity->id, + $scopes[$name], + $member['request_id'], + '1.20', + $prepared->id + ); + $this->assertTrue($receipt['fenced']); + $this->assertSame(ActivityStatus::Cancelled, $activity->fresh()->status); + $this->assertSame( + ($childRequest ?? $parentRequest) +->payload['request_id'], + $receipt['root_request_id'] + ); + $this->assertSame('2026-10-04T00:00:05.000000Z', $receipt['cancellation_scope']['authority_deadline_at']); + $this->assertNull(CancellationScopeDelivery::prepared($run->fresh(), $scopes[$name])); + $receipts[$name] = [$activity, $member, $receipt]; + } + $count = $run->historyEvents() + ->count(); + foreach ($receipts as $name => [$activity, $member, $receipt]) { + $retry = ScopedActivityCancellation::fence( + $run->fresh(), + $task, + $activity->id, + $scopes[$name], + $member['request_id'], + '1.20', + $prepared->id + ); + $this->assertSame($receipt['history_event_id'], $retry['history_event_id']); + } + $this->assertSame($count, $run->historyEvents()->count()); + $this->assertSame($beforeClaim, $task->fresh()->getAttributes()); + foreach (['shield', 'sibling'] as $name) { + $other = ActivityExecution::query()->where( + 'activity_options->cancellation_scope_id', + $scopes[$name] + )->sole(); + $this->assertSame(ActivityStatus::Pending, $other->status); + } + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeDeliveryPrepared)->count() + ); + } + + #[DataProvider('lostAuthority')] + public function testNestedRequestsRollBackWithPreparationIfOriginalAuthorityIsLost(string $change): void + { + [$run, $task, $scopes] = $this->tree(); + CancellationScopeRequests::request($run, $scopes['parent'], '1.20', 30); + $count = $run->historyEvents() + ->count(); + $eventName = 'eloquent.created: ' . WorkflowHistoryEvent::class; + Event::listen($eventName, static function (WorkflowHistoryEvent $event) use ( + $change, + $task, + $run, + $scopes + ): void { + if ($event->event_type !== HistoryEventType::CancellationScopeRequested + || ($event->payload['scope_id'] ?? null) !== $scopes['child']) { + return; + } + match ($change) { + 'owner' => $task->fresh() + ->forceFill([ + 'lease_owner' => 'another', + ])->save(), + 'attempt' => $task->fresh() + ->forceFill([ + 'attempt_count' => 2, + ])->save(), + 'lease' => $task->fresh() + ->forceFill([ + 'lease_expires_at' => now(), + ])->save(), + 'terminated' => $run->fresh() + ->forceFill([ + 'status' => RunStatus::Terminated, + ])->save(), + 'deadline' => Carbon::setTestNow('2026-10-04T00:00:30Z'), + }; + }); + try { + $this->prepare($run, $task, $scopes['parent']); + $this->fail('Preparation committed after authority loss.'); + } catch (LogicException $error) { + $this->assertSame(in_array($change, ['terminated', 'deadline'], true) + ? 'cancellation_scope_authority_expired' : 'cancellation_scope_workflow_claim_mismatch', $error->getMessage()); + } finally { + Event::forget($eventName); + } + $this->assertSame($count, $run->historyEvents()->count()); + $this->assertNull(CancellationScopeRequests::context($run->fresh(), $scopes['child'])); + $this->assertNull(CancellationScopeRequests::context($run->fresh(), $scopes['grandchild'])); + $this->assertNull(CancellationScopeDelivery::prepared($run->fresh(), $scopes['parent'])); + $this->assertSame('original', $task->fresh()->lease_owner); + } + + public static function lostAuthority(): iterable + { + foreach (['owner', 'attempt', 'lease', 'terminated', 'deadline'] as $change) { + yield $change => [$change]; + } + } + + #[DataProvider('changedInventory')] + public function testColdPreparationRejectsChangedDescendantInventory(string $field): void + { + [$run, $task, $scopes] = $this->tree(); + CancellationScopeRequests::request($run, $scopes['parent'], '1.20', 30); + $event = $this->prepare($run, $task, $scopes['parent']); + $payload = $event->payload; + if ($field === 'missing') { + unset($payload['descendant_members']); + } elseif ($field === 'extra') { + $payload['descendant_members'][0]['secret'] = 'not-for-history'; + } elseif ($field === 'cancellation') { + $payload['descendant_members'][0]['cancellation']['root_context']['cleanup_deadline_at'] = 'invalid'; + } elseif ($field === 'order') { + $payload['descendant_members'] = array_reverse($payload['descendant_members']); + } else { + $payload['descendant_members'][0][$field] = 'substituted'; + } + $event->forceFill([ + 'payload' => $payload, + ])->save(); + $this->expectExceptionMessage('cancellation_scope_delivery_history_invalid'); + CancellationScopeDelivery::prepared($run->fresh(), $scopes['parent']); + } + + public static function changedInventory(): iterable + { + foreach (['scope_id', 'parent_scope_id', 'scope_history_event_id', 'request_history_event_id', + 'request_id', 'propagation_history_event_id', 'authority_deadline_at', 'cancellation', 'order', 'missing', 'extra'] as $field) { + yield $field => [$field]; + } + } + + public function testDescendantPropagationCannotRunOutsidePreparationTransaction(): void + { + [$run, , $scopes] = $this->tree(); + CancellationScopeRequests::request($run, $scopes['parent'], '1.20', 30); + $this->expectExceptionMessage('cancellation_scope_descendants_require_preparation_transaction'); + CancellationScopeDescendants::request($run, $scopes['parent']); + } + + public function testOriginalRootOperationDoesNotInvalidateNestedDelivery(): void + { + [$run, $task, $scopes] = $this->tree(); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prefix = $bridge->checkpointCancellationScopePrefix( + $task->id, + 'original', + 1, + 'ordinary-root-timer', + 7, + [[ + 'type' => 'start_timer', + 'delay_seconds' => 3600, + ]], + '1.20' + ); + $this->assertTrue($prefix['checkpointed'], $prefix['reason'] ?? ''); + $timer = $run->timers() + ->sole(); + $before = $timer->getAttributes(); + $request = CancellationScopeRequests::request($run, $scopes['parent'], '1.20', 30); + $preparation = $bridge->prepareCancellationScopeDelivery( + $task->id, + 'original', + 1, + $scopes['parent'], + $request->payload['request_id'], + 8, + 'timer', + protocolVersion: '1.20' + ); + $this->assertTrue($preparation['prepared'], $preparation['reason'] ?? ''); + $delivery = $bridge->deliverCancellationScope( + $task->id, + 'original', + 1, + $scopes['parent'], + $request->payload['request_id'], + 8, + 'timer', + protocolVersion: '1.20' + ); + $this->assertTrue($delivery['delivered'], $delivery['reason'] ?? ''); + $this->assertSame($before, $timer->fresh()->getAttributes()); + $this->assertSame($preparation['preparation_history_event_id'], $delivery['preparation_history_event_id']); + } + + public function testColdPreparationKeepsItsRunCeilingAfterMutableLimitsChange(): void + { + [$run, $task, $scopes] = $this->tree(); + $run->forceFill([ + 'run_deadline_at' => now() + ->addSeconds(25), + ])->save(); + CancellationScopeRequests::request($run, $scopes['parent'], '1.20', 30); + $prepared = $this->prepare($run, $task, $scopes['parent']); + foreach ($prepared->payload['descendant_members'] as $member) { + $this->assertSame('2026-10-04T00:00:25.000000Z', $member['authority_deadline_at']); + $this->assertSame( + '2026-10-04T00:00:30.000000Z', + $member['cancellation']['root_context']['cleanup_deadline_at'] + ); + } + $run->forceFill([ + 'run_deadline_at' => now() + ->addSeconds(10), + ])->save(); + $this->assertSameJsonObject( + $prepared->payload, + CancellationScopeDelivery::prepared($run->fresh(), $scopes['parent'])->payload + ); + $this->advanceWithWorkflowHeartbeats($task, Carbon::parse('2026-10-04T00:00:10Z')); + $this->expectExceptionMessage('cancellation_scope_authority_expired'); + $this->prepare($run->fresh(), $task, $scopes['parent']); + } + + public function testInheritedReferenceUsesOriginalHistoryWithoutCreatingAnotherBoundary(): void + { + [$run, $task, $scopes] = $this->tree(); + $request = CancellationScopeRequests::request($run, $scopes['parent'], '1.20', 30); + $prepared = $this->prepare($run, $task, $scopes['parent']); + $count = $run->historyEvents() + ->count(); + foreach (['parent', 'child', 'grandchild'] as $name) { + $reference = ScopedCancellationPreparation::forScope($run->fresh(), $scopes[$name], $prepared->id); + $this->assertSame($prepared->id, $reference->historyEventId); + $this->assertSame($prepared->sequence, $reference->historySequence); + $this->assertSame($scopes['parent'], $reference->preparedScopeId); + $this->assertSame($scopes[$name], $reference->scopeId); + $this->assertSame($request->payload['request_id'], $reference->context->rootContext->rootRequestId); + } + $this->assertNull(ScopedCancellationPreparation::forScope($run->fresh(), $scopes['child'])); + $this->assertSame($count, $run->historyEvents()->count()); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeDeliveryPrepared)->count() + ); + } + + #[DataProvider('excludedReferenceScopes')] + public function testInheritedReferenceRejectsUnrelatedAndShieldedScopes(string $name): void + { + [$run, $task, $scopes] = $this->tree(); + CancellationScopeRequests::request($run, $scopes['parent'], '1.20', 30); + $prepared = $this->prepare($run, $task, $scopes['parent']); + $count = $run->historyEvents() + ->count(); + try { + ScopedCancellationPreparation::forScope($run->fresh(), $scopes[$name], $prepared->id); + $this->fail('An excluded scope obtained the original preparation.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_descendant_not_prepared', $error->getMessage()); + } + $this->assertSame($count, $run->historyEvents()->count()); + } + + public static function excludedReferenceScopes(): iterable + { + yield 'shield' => ['shield']; + yield 'shielded descendant' => ['shield_child']; + yield 'sibling' => ['sibling']; + } + + public function testColdInheritedReferenceKeepsOriginalBudgetAfterReplacementAndExpiry(): void + { + [$run, $task, $scopes] = $this->tree(); + $run->forceFill([ + 'run_deadline_at' => now() + ->addSeconds(25), + ])->save(); + CancellationScopeRequests::request($run, $scopes['parent'], '1.20', 30); + $prepared = $this->prepare($run, $task, $scopes['parent']); + $original = ScopedCancellationPreparation::forScope($run->fresh(), $scopes['child'], $prepared->id); + $count = $run->historyEvents() + ->count(); + $run->forceFill([ + 'run_deadline_at' => now() + ->addSeconds(100), + ])->save(); + $task->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 2, + ])->save(); + Carbon::setTestNow('2026-10-04T00:00:40Z'); + $replayed = ScopedCancellationPreparation::forScope($run->fresh(), $scopes['child'], $prepared->id); + $this->assertSame($original->historyEventId, $replayed->historyEventId); + $this->assertSame($original->requestHistoryEventId, $replayed->requestHistoryEventId); + $this->assertSame('2026-10-04T00:00:25.000000Z', $replayed->authorityDeadlineAt); + $this->assertSame($original->context->toArray(), $replayed->context->toArray()); + $this->assertFalse(CancellationScopeRequests::authority($run->fresh(), $scopes['child'])['active']); + $this->assertSame($count, $run->historyEvents()->count()); + } + + /** + * @return array{WorkflowRun, WorkflowTask, array} + */ + private function tree(): array + { + $workflow = WorkflowStub::make(TestSignalWorkflow::class, 'descendant-preparation'); + $workflow->start(); + $run = $workflow->run() + ->fresh(); + $task = $run->tasks() + ->where('task_type', TaskType::Workflow)->sole(); + $task->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'original', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addSeconds(60), + ])->save(); + $scopes = []; + $scopes['parent'] = CancellationScopeHistory::open($run, $task, 1, '1.20')->payload['scope_id']; + $scopes['child'] = CancellationScopeHistory::open( + $run, + $task, + 2, + '1.20', + $scopes['parent'] + )->payload['scope_id']; + $scopes['grandchild'] = CancellationScopeHistory::open( + $run, + $task, + 3, + '1.20', + $scopes['child'] + )->payload['scope_id']; + $scopes['shield'] = CancellationScopeHistory::open( + $run, + $task, + 4, + '1.20', + $scopes['parent'], + true + )->payload['scope_id']; + $scopes['shield_child'] = CancellationScopeHistory::open( + $run, + $task, + 5, + '1.20', + $scopes['shield'] + )->payload['scope_id']; + $scopes['sibling'] = CancellationScopeHistory::open($run, $task, 6, '1.20')->payload['scope_id']; + return [$run, $task, $scopes]; + } + + private function prepare( + WorkflowRun $run, + WorkflowTask $task, + string $scope, + int $sequence = 7 + ): WorkflowHistoryEvent { + return CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + CancellationScopeRequests::context($run, $scope)->requestId, + $sequence, + 'timer', + '1.20' + ); + } +} diff --git a/tests/Feature/V2/V2CancellationScopeHistoryTest.php b/tests/Feature/V2/V2CancellationScopeHistoryTest.php new file mode 100644 index 000000000..29a3eb610 --- /dev/null +++ b/tests/Feature/V2/V2CancellationScopeHistoryTest.php @@ -0,0 +1,1280 @@ + 'poll', + ]); + Carbon::setTestNow('2026-10-03T00:00:00Z'); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + public function testOptionalScopeOpeningPersistsBeforeTheBodyAndRetainsTheClaim(): void + { + [$workflow, $claim] = $this->workflowClaim('portable-open'); + $bridge = app(DefaultWorkflowTaskBridge::class); + $this->assertInstanceOf(CancellationScopeTaskBridge::class, $bridge); + $before = $claim->fresh() + ->getAttributes(); + $first = $bridge->openCancellationScope($claim->id, 'scope-owner', 1, 1, 'root', false, '1.20'); + $this->assertTrue($first['opened'], $first['reason'] ?? ''); + $this->assertFalse($first['duplicate']); + $this->assertFalse($first['claim_released']); + $this->assertSame([], $first['created_task_ids']); + $this->assertSame($workflow->runId(), $first['workflow_run_id']); + $event = $workflow->run() + ->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeOpened)->sole(); + $this->assertSame($event->id, $first['history_event_id']); + $this->assertSame($event->payload['scope_id'], $first['scope_id']); + $this->assertSame('root', $first['parent_scope_id']); + $this->assertFalse($first['shield_parent']); + $this->assertSame(1, $first['sequence']); + $this->assertSame($before, $claim->fresh()->getAttributes()); + $this->assertSame(0, ActivityExecution::query()->count()); + $this->assertSame(0, WorkflowTimer::query()->count()); + $this->assertSame(0, WorkflowLink::query()->count()); + $duplicate = $bridge->openCancellationScope($claim->id, 'scope-owner', 1, 1, 'root', false, '1.20'); + $this->assertTrue($duplicate['opened']); + $this->assertTrue($duplicate['duplicate']); + $this->assertSame($first['scope_id'], $duplicate['scope_id']); + $this->assertSame($first['history_event_id'], $duplicate['history_event_id']); + $this->assertSame( + 1, + $workflow->run() + ->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeOpened)->count() + ); + } + + public function testScopePrefixCommitsBeforeOpeningWithoutLocalActivityAdmission(): void + { + [$workflow, $claim] = $this->workflowClaim('scope-prefix'); + $bridge = app(DefaultWorkflowTaskBridge::class); + $originalLease = $claim->lease_expires_at->toISOString(); + $commands = [[ + 'type' => 'record_side_effect', + 'result' => Serializer::serializeWithCodec('avro', 'before-scope'), + ]]; + $first = $bridge->checkpointCancellationScopePrefix( + $claim->id, + 'scope-owner', + 1, + 'scope-prefix', + 1, + $commands, + '1.20' + ); + $this->assertTrue($first['checkpointed'], $first['reason'] ?? ''); + $this->assertFalse($first['duplicate']); + $this->assertSame(2, $first['next_sequence']); + $this->assertSame($originalLease, $claim->fresh()->lease_expires_at->toISOString()); + $this->assertArrayHasKey('portable_scope_checkpoint', $claim->fresh()->payload); + $this->assertArrayNotHasKey('portable_local_checkpoint', $claim->fresh()->payload); + $before = $workflow->run() + ->historyEvents() + ->count(); + $retry = $bridge->checkpointCancellationScopePrefix( + $claim->id, + 'scope-owner', + 1, + 'scope-prefix', + 1, + $commands, + '1.20' + ); + $this->assertTrue($retry['checkpointed']); + $this->assertTrue($retry['duplicate']); + $this->assertSame($first['fingerprint'], $retry['fingerprint']); + $this->assertSame($before, $workflow->run()->historyEvents()->count()); + $scope = $bridge->openCancellationScope($claim->id, 'scope-owner', 1, 2, 'root', false, '1.20'); + $this->assertTrue($scope['opened'], $scope['reason'] ?? ''); + $this->assertSame(3, WorkflowStepHistory::nextDurableCommandSequence($workflow->run()->fresh())); + $this->assertSame(TaskStatus::Leased, $claim->fresh()->status); + $this->assertSame(0, ActivityExecution::query()->count()); + } + + public static function refusedScopePrefixes(): iterable + { + yield 'legacy protocol' => [ + '1.19', + 1, + 'scope-owner', + 1, + [], + 'cancellation_scope_checkpoint_requires_protocol_1_20', + ]; + yield 'unknown protocol major' => [ + '2.0', + 1, + 'scope-owner', + 1, + [], + 'cancellation_scope_checkpoint_requires_protocol_1_20', + ]; + yield 'old attempt' => ['1.20', 2, 'scope-owner', 1, [], 'workflow_claim_mismatch']; + yield 'wrong owner' => ['1.20', 1, 'wrong-owner', 1, [], 'workflow_claim_mismatch']; + yield 'future sequence' => ['1.20', 1, 'scope-owner', 2, [], 'cancellation_scope_checkpoint_sequence_mismatch']; + yield 'terminal command' => ['1.20', 1, 'scope-owner', 1, [[ + 'type' => 'complete_workflow', + 'result' => 'ignored', + ]], 'invalid_cancellation_scope_checkpoint_commands']; + yield 'local callback report' => ['1.20', 1, 'scope-owner', 1, [[ + 'type' => 'record_local_activity', + ]], 'invalid_cancellation_scope_checkpoint_commands']; + yield 'local preparation' => ['1.20', 1, 'scope-owner', 1, [[ + 'type' => 'prepare_local_activity', + ]], 'invalid_cancellation_scope_checkpoint_commands']; + } + + #[DataProvider('refusedScopePrefixes')] + public function testScopePrefixRefusalLeavesHistoryAndClaimUnchanged( + string $protocol, + int $attempt, + string $owner, + int $sequence, + array $commands, + string $reason + ): void { + [$workflow, $claim] = $this->workflowClaim('refused-scope-prefix'); + $before = $claim->fresh() + ->getAttributes(); + $history = $workflow->run() + ->historyEvents() + ->count(); + $reply = app(DefaultWorkflowTaskBridge::class)->checkpointCancellationScopePrefix( + $claim->id, + $owner, + $attempt, + 'scope-prefix', + $sequence, + $commands, + $protocol + ); + $this->assertFalse($reply['checkpointed']); + $this->assertSame($reason, $reply['reason']); + $this->assertSame($before, $claim->fresh()->getAttributes()); + $this->assertSame($history, $workflow->run()->historyEvents()->count()); + } + + public function testScopePrefixCannotReplayChangedCommandsOrPublishFromAReplacedClaim(): void + { + [$workflow, $claim] = $this->workflowClaim('changed-scope-prefix'); + $bridge = app(DefaultWorkflowTaskBridge::class); + $commands = [[ + 'type' => 'record_side_effect', + 'result' => Serializer::serializeWithCodec('avro', 'first'), + ]]; + $this->assertTrue($bridge->checkpointCancellationScopePrefix( + $claim->id, + 'scope-owner', + 1, + 'scope-prefix', + 1, + $commands, + '1.20' + )['checkpointed']); + $history = $workflow->run() + ->historyEvents() + ->count(); + $commands[0]['result'] = Serializer::serializeWithCodec('avro', 'changed'); + $this->assertSame('cancellation_scope_checkpoint_mismatch', $bridge->checkpointCancellationScopePrefix( + $claim->id, + 'scope-owner', + 1, + 'scope-prefix', + 1, + $commands, + '1.20' + )['reason']); + $claim->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 2, + ])->save(); + $this->assertSame('workflow_claim_mismatch', $bridge->checkpointCancellationScopePrefix( + $claim->id, + 'scope-owner', + 1, + 'scope-prefix', + 2, + $commands, + '1.20' + )['reason']); + $scope = $bridge->openCancellationScope($claim->id, 'replacement', 2, 2, 'root', false, '1.20'); + $this->assertTrue($scope['opened'], $scope['reason'] ?? ''); + $this->assertSame($history + 1, $workflow->run()->historyEvents()->count()); + } + + public function testOptionalScopeOpeningReplaysTheSameNestedShieldAfterClaimTakeover(): void + { + [$workflow, $claim] = $this->workflowClaim('portable-shield'); + $bridge = app(DefaultWorkflowTaskBridge::class); + $parent = $bridge->openCancellationScope($claim->id, 'scope-owner', 1, 1, 'root', false, '1.20'); + $child = $bridge->openCancellationScope($claim->id, 'scope-owner', 1, 2, $parent['scope_id'], true, '1.20'); + $this->assertTrue($child['opened']); + $this->assertTrue($child['shield_parent']); + $claim->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 2, + ])->save(); + $before = $workflow->run() + ->historyEvents() + ->count(); + $stale = $bridge->openCancellationScope($claim->id, 'scope-owner', 1, 2, $parent['scope_id'], true, '1.20'); + $this->assertFalse($stale['opened']); + $this->assertSame('cancellation_scope_workflow_claim_mismatch', $stale['reason']); + $replay = $bridge->openCancellationScope($claim->id, 'replacement', 2, 2, $parent['scope_id'], true, '1.20'); + $this->assertTrue($replay['opened']); + $this->assertTrue($replay['duplicate']); + $this->assertSame($child['scope_id'], $replay['scope_id']); + $this->assertSame($child['history_event_id'], $replay['history_event_id']); + $this->assertSame($before, $workflow->run()->historyEvents()->count()); + } + + public function testOptionalScopeOpeningRefusesChangedReplayWithoutWritingHistory(): void + { + [$workflow, $claim] = $this->workflowClaim('portable-mismatch'); + $bridge = app(DefaultWorkflowTaskBridge::class); + $first = $bridge->openCancellationScope($claim->id, 'scope-owner', 1, 1, 'root', false, '1.20'); + $before = $workflow->run() + ->historyEvents() + ->count(); + foreach ([['root', true], [$first['scope_id'], false]] as [$parent, $shield]) { + $changed = $bridge->openCancellationScope($claim->id, 'scope-owner', 1, 1, $parent, $shield, '1.20'); + $this->assertFalse($changed['opened']); + $this->assertSame('cancellation_scope_replay_mismatch', $changed['reason']); + } + $this->assertSame($before, $workflow->run()->historyEvents()->count()); + } + + public static function invalidScopeOpenings(): array + { + return [ + ['1.19', 'scope-owner', 1, 1, 'root', 'cancellation_scope_requires_protocol_1_20'], + ['2.20', 'scope-owner', 1, 1, 'root', 'cancellation_scope_requires_protocol_1_20'], + ['invalid', 'scope-owner', 1, 1, 'root', 'cancellation_scope_requires_protocol_1_20'], + ['1.20', '', 1, 1, 'root', 'invalid_cancellation_scope_open'], + ['1.20', 'scope-owner', 0, 1, 'root', 'invalid_cancellation_scope_open'], + ['1.20', 'other-owner', 1, 1, 'root', 'cancellation_scope_workflow_claim_mismatch'], + ['1.20', 'scope-owner', 2, 1, 'root', 'cancellation_scope_workflow_claim_mismatch'], + ['1.20', 'scope-owner', 1, 0, 'root', 'invalid_cancellation_scope_open'], + ['1.20', 'scope-owner', 1, 1, '', 'invalid_cancellation_scope_open'], + ['1.20', 'scope-owner', 1, 1, ' ', 'invalid_cancellation_scope_open'], + ['1.20', 'scope-owner', 1, 1, "\xff", 'invalid_cancellation_scope_open'], + ['1.20', 'scope-owner', 1, 1, str_repeat('x', 256), 'invalid_cancellation_scope_open'], + ['1.20', 'scope-owner', 1, 1, 'unknown-parent', 'cancellation_scope_parent_not_recorded'], + ['1.20', 'scope-owner', 1, 2, 'root', 'cancellation_scope_sequence_mismatch'], + ]; + } + + #[DataProvider('invalidScopeOpenings')] + public function testOptionalScopeOpeningRefusesInvalidAuthorityBeforeAnyHistory( + string $version, + string $owner, + int $attempt, + int $sequence, + string $parent, + string $reason, + ): void { + [$workflow, $claim] = $this->workflowClaim('portable-refused'); + $before = $workflow->run() + ->historyEvents() + ->count(); + $attributes = $claim->fresh() + ->getAttributes(); + $reply = app(DefaultWorkflowTaskBridge::class)->openCancellationScope( + $claim->id, + $owner, + $attempt, + $sequence, + $parent, + false, + $version + ); + $this->assertFalse($reply['opened']); + $this->assertFalse($reply['claim_released']); + $this->assertSame($reason, $reply['reason']); + $this->assertSame($before, $workflow->run()->historyEvents()->count()); + $this->assertSame($attributes, $claim->fresh()->getAttributes()); + } + + public function testOptionalScopeOpeningRefusesExpiredClaimAndForeignParent(): void + { + [$other, $otherClaim] = $this->workflowClaim('portable-foreign'); + $bridge = app(DefaultWorkflowTaskBridge::class); + $foreign = $bridge->openCancellationScope($otherClaim->id, 'scope-owner', 1, 1, 'root', false, '1.20'); + [$workflow, $claim] = $this->workflowClaim('portable-current'); + $before = $workflow->run() + ->historyEvents() + ->count(); + $reply = $bridge->openCancellationScope($claim->id, 'scope-owner', 1, 1, $foreign['scope_id'], false, '1.20'); + $this->assertFalse($reply['opened']); + $this->assertSame('cancellation_scope_parent_not_recorded', $reply['reason']); + $claim->forceFill([ + 'lease_expires_at' => now(), + ])->save(); + $reply = $bridge->openCancellationScope($claim->id, 'scope-owner', 1, 1, 'root', false, '1.20'); + $this->assertFalse($reply['opened']); + $this->assertSame('cancellation_scope_workflow_claim_mismatch', $reply['reason']); + $this->assertSame($before, $workflow->run()->historyEvents()->count()); + } + + public function testOptionalScopeOpeningRefusesMissingTaskAndWrongNamespaceWithoutMutation(): void + { + $bridge = app(DefaultWorkflowTaskBridge::class); + $missing = $bridge->openCancellationScope('missing-task', 'scope-owner', 1, 1, 'root', false, '1.20'); + $this->assertFalse($missing['opened']); + $this->assertSame('task_not_found', $missing['reason']); + [$workflow, $claim] = $this->workflowClaim('portable-namespace'); + $before = $workflow->run() + ->historyEvents() + ->count(); + $claim->forceFill([ + 'namespace' => 'another-namespace', + ])->save(); + $reply = $bridge->openCancellationScope($claim->id, 'scope-owner', 1, 1, 'root', false, '1.20'); + $this->assertFalse($reply['opened']); + $this->assertSame('cancellation_scope_workflow_claim_mismatch', $reply['reason']); + $this->assertSame($before, $workflow->run()->historyEvents()->count()); + } + + public function testOptionalScopeOpeningRequiresAnActiveRunAndAnUnchangedCanonicalHistory(): void + { + [$workflow, $claim] = $this->workflowClaim('portable-history'); + $bridge = app(DefaultWorkflowTaskBridge::class); + $first = $bridge->openCancellationScope($claim->id, 'scope-owner', 1, 1, 'root', false, '1.20'); + $event = $workflow->run() + ->historyEvents() + ->findOrFail($first['history_event_id']); + $event->forceFill([ + 'payload' => [ + ...$event->payload, + 'workflow_run_id' => 'another-run', + ], + ])->save(); + $before = $workflow->run() + ->historyEvents() + ->count(); + $reply = $bridge->openCancellationScope($claim->id, 'scope-owner', 1, 1, 'root', false, '1.20'); + $this->assertFalse($reply['opened']); + $this->assertSame('cancellation_scope_history_invalid', $reply['reason']); + $this->assertSame($before, $workflow->run()->historyEvents()->count()); + $workflow->run() + ->forceFill([ + 'status' => \Workflow\V2\Enums\RunStatus::Cancelled, + ])->save(); + $reply = $bridge->openCancellationScope($claim->id, 'scope-owner', 1, 2, 'root', false, '1.20'); + $this->assertFalse($reply['opened']); + $this->assertSame('cancellation_scope_run_not_active', $reply['reason']); + $this->assertSame($before, $workflow->run()->historyEvents()->count()); + } + + public function testOptionalScopeAdmissionRoleValidatesBeforeEffectsWithoutMutatingTheClaim(): void + { + [$workflow, $claim] = $this->workflowClaim('admission-role'); + $run = $workflow->run() + ->fresh(); + $scope = CancellationScopeHistory::open($run, $claim, 1, '1.20')->payload['scope_id']; + $bridge = app(DefaultWorkflowTaskBridge::class); + $this->assertInstanceOf(CancellationScopeAdmission::class, $bridge); + $commands = $this->operationCommands(); + foreach ($commands as &$command) { + $command['cancellation_scope_id'] = $scope; + } + unset($command); + $history = $run->historyEvents() + ->count(); + $before = $claim->fresh() + ->getAttributes(); + $this->assertNull($bridge->validateCancellationScopeMembership($run, $commands, 2)); + foreach (array_keys($commands) as $index) { + $changed = $commands; + $changed[$index]['cancellation_scope_id'] = 'unknown-scope'; + $this->assertSame( + 'operation_scope_not_recorded', + $bridge->validateCancellationScopeMembership($run, $changed, 2) + ); + } + $this->assertSame( + 'operation_scope_not_recorded', + $bridge->validateCancellationScopeMembership($run, $commands, 1) + ); + $local = [ + 'type' => 'prepare_local_activity', + 'cancellation_scope_id' => $scope, + ]; + $this->assertNull($bridge->validateCancellationScopeMembership($run, [$local], 2)); + $local['cancellation_scope_id'] = 'unknown-scope'; + $this->assertSame( + 'local_activity_scope_not_recorded', + $bridge->validateCancellationScopeMembership($run, [$local], 2) + ); + $this->assertNull($bridge->validateCancellationScopeMembership($run, [[ + 'type' => 'prepare_local_activity', + ]], 2)); + $this->assertSame($history, $run->historyEvents()->count()); + $this->assertSame($before, $claim->fresh()->getAttributes()); + $this->assertSame(0, ActivityExecution::query()->count()); + $this->assertSame(0, WorkflowTimer::query()->count()); + $this->assertSame(0, WorkflowLink::query()->count()); + } + + public function testRemoteActivityTimerAndChildKeepCanonicalMembershipBeforeAdmission(): void + { + [$workflow, $claim] = $this->workflowClaim('operations'); + $run = $workflow->run() + ->fresh(); + $parent = CancellationScopeHistory::open($run, $claim, 1, '1.20')->payload['scope_id']; + $shield = CancellationScopeHistory::open($run, $claim, 2, '1.20', $parent, true)->payload['scope_id']; + $commands = $this->operationCommands(); + foreach ($commands as $index => &$command) { + $command['cancellation_scope_id'] = $index === 1 ? $shield : $parent; + } + unset($command); + $commands = WorkflowCommandNormalizer::normalize($commands, '1.20'); + $reply = app(DefaultWorkflowTaskBridge::class)->complete($claim->id, $commands); + $this->assertTrue($reply['completed'], $reply['reason'] ?? ''); + $activity = $run->activityExecutions() + ->sole(); + $this->assertSame($parent, $activity->activity_options['cancellation_scope_id']); + $scheduled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityScheduled)->sole(); + $this->assertSame($parent, $scheduled->payload['activity']['cancellation_scope_id']); + $timer = $run->historyEvents() + ->where('event_type', HistoryEventType::TimerScheduled)->sole(); + $this->assertSame($shield, $timer->payload['cancellation_scope_id']); + $timerTask = $run->tasks() + ->where('task_type', TaskType::Timer)->sole(); + $this->assertSame($shield, $timerTask->payload['cancellation_scope_id']); + foreach ($run->historyEvents()->whereIn('event_type', [HistoryEventType::ChildWorkflowScheduled, + HistoryEventType::ChildRunStarted])->get() as $event) { + $this->assertSame($parent, $event->payload['cancellation_scope_id']); + } + $projection = WorkflowChildCall::query()->where('parent_workflow_run_id', $run->id)->sole(); + $this->assertSame($parent, $projection->metadata['cancellation_scope_id']); + $projection->forceFill([ + 'metadata' => [ + 'cancellation_scope_id' => 'projection-only', + ], + ])->save(); + $this->assertSame($parent, $run->historyEvents()->where('event_type', HistoryEventType::ChildWorkflowScheduled) + ->sole() +->payload['cancellation_scope_id']); + $this->assertNull($run->fresh()->cancellation_request_command_id); + } + + public static function waitMemberships(): iterable + { + foreach (['signal', 'condition'] as $kind) { + foreach ([null, 0, 5] as $timeout) { + foreach ([false, true] as $scoped) { + yield $kind . ':' . ($timeout ?? 'untimed') . ':' . (int) $scoped => [$kind, $timeout, $scoped]; + } + } + } + } + + #[DataProvider('waitMemberships')] + public function testWaitOpeningAndItsTimeoutRetainCanonicalMembership( + string $kind, + ?int $timeout, + bool $scoped, + ): void { + [$workflow, $claim] = $this->workflowClaim('wait-membership'); + $run = $workflow->run() + ->fresh(); + $scope = CancellationScopeHistory::open($run, $claim, 1, '1.20', 'root', true)->payload['scope_id']; + $command = $this->waitCommand($kind, $timeout); + if ($scoped) { + $command['cancellation_scope_id'] = $scope; + } + $reply = app(DefaultWorkflowTaskBridge::class)->complete( + $claim->id, + WorkflowCommandNormalizer::normalize([$command], $scoped ? '1.20' : '1.19'), + ); + $this->assertTrue($reply['completed'], $reply['reason'] ?? ''); + $opened = $run->historyEvents() + ->where('event_type', $kind === 'signal' + ? HistoryEventType::SignalWaitOpened : HistoryEventType::ConditionWaitOpened)->sole(); + $this->assertSame(2, $opened->payload['sequence']); + $this->assertSame($timeout, $opened->payload['timeout_seconds'] ?? null); + $this->assertSame($scoped ? $scope : null, $opened->payload['cancellation_scope_id'] ?? null); + $this->assertSame($scoped, array_key_exists('cancellation_scope_id', $opened->payload)); + $this->assertTrue(CancellationScopeHistory::forRun($run->fresh())[$scope]['shield_parent']); + $waitId = $opened->payload[$kind . '_wait_id']; + $this->assertIsString($waitId); + $timers = $run->historyEvents() + ->where('event_type', HistoryEventType::TimerScheduled)->get(); + $this->assertCount($timeout === null ? 0 : 1, $timers); + foreach ($timers as $timer) { + $this->assertSame($waitId, $timer->payload[$kind . '_wait_id']); + $this->assertSame($scoped ? $scope : null, $timer->payload['cancellation_scope_id'] ?? null); + $this->assertSame($scoped, array_key_exists('cancellation_scope_id', $timer->payload)); + } + $timerTasks = $run->tasks() + ->where('task_type', TaskType::Timer)->get(); + $this->assertCount($timeout !== null && $timeout > 0 ? 1 : 0, $timerTasks); + foreach ($timerTasks as $timerTask) { + $this->assertSame($waitId, $timerTask->payload[$kind . '_wait_id']); + $this->assertSame($scoped ? $scope : null, $timerTask->payload['cancellation_scope_id'] ?? null); + } + foreach ($run->historyEvents()->where('event_type', HistoryEventType::TimerFired)->get() as $fired) { + $this->assertSame($waitId, $fired->payload[$kind . '_wait_id']); + $this->assertSame($scoped ? $scope : null, $fired->payload['cancellation_scope_id'] ?? null); + $this->assertSame($scoped, array_key_exists('fire_at', $fired->payload)); + if ($scoped) { + $this->assertSame($timers->sole()->payload['fire_at'], $fired->payload['fire_at']); + } + } + $this->assertNull($run->fresh()->cancellation_request_command_id); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::TimerCancelled)->count()); + } + + public static function foreignWaitScopes(): iterable + { + foreach (['signal', 'condition'] as $kind) { + foreach ([false, true] as $foreign) { + yield $kind . ':' . (int) $foreign => [$kind, $foreign]; + } + } + } + + #[DataProvider('foreignWaitScopes')] + public function testUnknownOrForeignWaitScopeRefusesTheWholeBatchBeforeAnyEffect(string $kind, bool $foreign): void + { + [$workflow, $claim] = $this->workflowClaim('foreign-wait'); + $run = $workflow->run() + ->fresh(); + CancellationScopeHistory::open($run, $claim, 1, '1.20'); + $scope = 'unknown-scope'; + if ($foreign) { + [$other, $otherClaim] = $this->workflowClaim('other-wait'); + $scope = CancellationScopeHistory::open( + $other->run() + ->fresh(), + $otherClaim, + 1, + '1.20' + )->payload['scope_id']; + } + $history = $run->historyEvents() + ->orderBy('sequence') + ->get() + ->toArray(); + $before = $claim->fresh() + ->getAttributes(); + $commands = [$this->operationCommands()[0], [ + ...$this->waitCommand($kind, 5), + 'cancellation_scope_id' => $scope, + ]]; + $reply = app(DefaultWorkflowTaskBridge::class)->complete( + $claim->id, + WorkflowCommandNormalizer::normalize($commands, '1.20'), + ); + $this->assertFalse($reply['completed']); + $this->assertSame('operation_scope_not_recorded', $reply['reason']); + $this->assertSame([], $reply['created_task_ids']); + $this->assertSame($history, $run->historyEvents()->orderBy('sequence')->get()->toArray()); + $this->assertSame($before, $claim->fresh()->getAttributes()); + $this->assertSame(0, ActivityExecution::query()->count()); + $this->assertSame(0, WorkflowTimer::query()->count()); + } + + public function testScopedWaitsStillCloseATurnAndCannotEnterARetainedClaimPrefix(): void + { + [$workflow, $claim] = $this->workflowClaim('wait-prefix'); + $run = $workflow->run() + ->fresh(); + $scope = CancellationScopeHistory::open($run, $claim, 1, '1.20')->payload['scope_id']; + $history = $run->historyEvents() + ->orderBy('sequence') + ->get() + ->toArray(); + $before = $claim->fresh() + ->getAttributes(); + $bridge = app(DefaultWorkflowTaskBridge::class); + foreach (['signal', 'condition'] as $kind) { + $commands = [[ + ...$this->waitCommand($kind, null), + 'cancellation_scope_id' => $scope, + ]]; + $this->assertSame( + 'invalid_cancellation_scope_checkpoint_commands', + $bridge->checkpointCancellationScopePrefix( + $claim->id, + 'scope-owner', + 1, + 'wait-prefix', + 2, + $commands, + '1.20', + )['reason'] + ); + $this->assertSame('invalid_local_activity_checkpoint_commands', $bridge->checkpointLocalActivityPrefix( + $claim->id, + 'scope-owner', + 1, + 'wait-prefix', + 2, + $commands, + '1.20', + )['reason']); + } + $this->assertSame($history, $run->historyEvents()->orderBy('sequence')->get()->toArray()); + $this->assertSame($before, $claim->fresh()->getAttributes()); + } + + public static function waitKinds(): iterable + { + yield 'signal' => ['signal']; + yield 'condition' => ['condition']; + } + + #[DataProvider('waitKinds')] + public function testPreparedWaitKeepsReplayMembershipButRefusesNewWorkBeforeEffects(string $kind): void + { + [$workflow, $claim] = $this->workflowClaim('prepared-wait'); + $run = $workflow->run() + ->fresh(); + $scope = CancellationScopeHistory::open($run, $claim, 1, '1.20')->payload['scope_id']; + $command = [ + ...$this->waitCommand($kind, 5), + 'cancellation_scope_id' => $scope, + ]; + $bridge = app(DefaultWorkflowTaskBridge::class); + $this->assertTrue($bridge->complete($claim->id, [$command])['completed']); + $hosting = $run->tasks() + ->create([ + 'namespace' => $run->namespace, + 'task_type' => TaskType::Workflow, + 'status' => TaskStatus::Leased, + 'lease_owner' => 'scope-replacement', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addMinutes(5), + 'available_at' => now(), + 'connection' => $run->connection, + 'queue' => $run->queue, + 'compatibility' => $run->compatibility, + ]); + $request = CancellationScopeRequests::request($run->fresh(), $scope, '1.20', 30); + $prepared = $bridge->prepareCancellationScopeDelivery( + $hosting->id, + 'scope-replacement', + 1, + $scope, + $request->payload['request_id'], + 2, + $kind, + protocolVersion: '1.20', + ); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $this->assertCount(1, $prepared['timer_members']); + $this->assertSame(2, $prepared['timer_members'][0]['sequence']); + $history = $run->historyEvents() + ->orderBy('sequence') + ->get() + ->toArray(); + $before = $hosting->fresh() + ->getAttributes(); + $this->assertNull($bridge->validateCancellationScopeMembership($run->fresh(), [$command], 2)); + $reply = $bridge->complete($hosting->id, [$command]); + $this->assertFalse($reply['completed']); + $this->assertSame('operation_scope_cancellation_prepared', $reply['reason']); + $this->assertSame($history, $run->historyEvents()->orderBy('sequence')->get()->toArray()); + $delivery = $bridge->deliverCancellationScope( + $hosting->id, + 'scope-replacement', + 1, + $scope, + $request->payload['request_id'], + 2, + $kind, + protocolVersion: '1.20', + ); + $this->assertTrue($delivery['delivered'], $delivery['reason'] ?? ''); + $this->assertCount(1, $delivery['wait_cancellations']); + $this->assertTrue($delivery['wait_cancellations'][0]['cancelled']); + $this->assertHostingShortenedWithoutClaimReplacement($before, $hosting); + $this->assertSame(1, $run->historyEvents()->where('event_type', HistoryEventType::TimerCancelled)->count()); + $this->assertSame( + 1, + $run->tasks() + ->where('task_type', TaskType::Timer)->where('status', TaskStatus::Cancelled)->count() + ); + } + + #[DataProvider('foreignOperationScopes')] + public function testUnrecordedOperationScopeRefusesTheWholeBatchBeforeAnySiblingIsCreated( + int $operation, + bool $foreign, + ): void { + [$workflow, $claim] = $this->workflowClaim('refused'); + $run = $workflow->run() + ->fresh(); + CancellationScopeHistory::open($run, $claim, 1, '1.20'); + $scope = 'unrecorded-scope'; + if ($foreign) { + [$other, $otherClaim] = $this->workflowClaim('foreign-operation'); + $scope = CancellationScopeHistory::open( + $other->run() + ->fresh(), + $otherClaim, + 1, + '1.20' + )->payload['scope_id']; + } + $commands = $this->operationCommands(); + $invalid = [ + ...$commands[$operation], + 'cancellation_scope_id' => $scope, + ]; + $before = $claim->fresh() + ->getAttributes(); + $historyBefore = $run->historyEvents() + ->count(); + $reply = app(DefaultWorkflowTaskBridge::class)->complete($claim->id, [$commands[0], $invalid]); + $this->assertFalse($reply['completed']); + $this->assertSame('operation_scope_not_recorded', $reply['reason']); + $this->assertSame([], $reply['created_task_ids']); + $this->assertSame(0, ActivityExecution::query()->count()); + $this->assertSame(0, WorkflowTimer::query()->count()); + $this->assertSame(0, WorkflowLink::query()->count()); + $this->assertSame($historyBefore, $run->historyEvents()->count()); + $this->assertSame($before, $claim->fresh()->getAttributes()); + $prefix = app(DefaultWorkflowTaskBridge::class)->checkpointLocalActivityPrefix( + $claim->id, + 'scope-owner', + 1, + 'refused-prefix', + 2, + [$commands[0], $invalid], + '1.20', + ); + $this->assertFalse($prefix['checkpointed']); + $this->assertSame('operation_scope_not_recorded', $prefix['reason']); + $this->assertSame(0, ActivityExecution::query()->count()); + $this->assertSame($historyBefore, $run->historyEvents()->count()); + $this->assertSame($before, $claim->fresh()->getAttributes()); + } + + public static function foreignOperationScopes(): iterable + { + foreach (['remote activity', 'timer', 'child'] as $index => $kind) { + yield $kind . ' unknown scope' => [$index, false]; + yield $kind . ' foreign run scope' => [$index, true]; + } + } + + #[DataProvider('operationKinds')] + public function testLostCheckpointResponseCannotReparentAnAdmittedOperation(int $operation): void + { + [$workflow, $claim] = $this->workflowClaim('lost-response'); + $run = $workflow->run() + ->fresh(); + $scope = CancellationScopeHistory::open($run, $claim, 1, '1.20')->payload['scope_id']; + $otherScope = CancellationScopeHistory::open($run, $claim, 2, '1.20')->payload['scope_id']; + $command = [ + ...$this->operationCommands()[$operation], + 'cancellation_scope_id' => $scope, + ]; + $checkpoint = static fn (array $commands): array => app(DefaultWorkflowTaskBridge::class) + ->checkpointLocalActivityPrefix($claim->id, 'scope-owner', 1, 'lost-response', 3, $commands, '1.20'); + $first = $checkpoint([$command]); + $this->assertTrue($first['checkpointed'], $first['reason'] ?? ''); + $historyBefore = $run->historyEvents() + ->orderBy('id') + ->get() + ->toArray(); + $claimBefore = $claim->fresh() + ->getAttributes(); + $duplicate = $checkpoint([$command]); + $this->assertTrue($duplicate['checkpointed']); + $this->assertTrue($duplicate['duplicate']); + $this->assertSame($first['created_task_ids'], $duplicate['created_task_ids']); + foreach ([$otherScope, CancellationScopeHistory::ROOT_SCOPE_ID, null] as $changed) { + $reparented = $command; + if ($changed === null) { + unset($reparented['cancellation_scope_id']); + } else { + $reparented['cancellation_scope_id'] = $changed; + } + $reply = $checkpoint([$reparented]); + $this->assertFalse($reply['checkpointed']); + $this->assertSame('local_activity_checkpoint_mismatch', $reply['reason']); + } + $this->assertSame($historyBefore, $run->historyEvents()->orderBy('id')->get()->toArray()); + $this->assertSame($claimBefore, $claim->fresh()->getAttributes()); + } + + public static function operationKinds(): iterable + { + foreach (['remote activity', 'timer', 'child'] as $index => $kind) { + yield $kind => [$index]; + } + } + + public function testRetainedOperationScopesRejectAnUnqualifiedMajorProtocolBeforeAdmission(): void + { + [$workflow, $claim] = $this->workflowClaim('unqualified-major'); + $run = $workflow->run() + ->fresh(); + $scope = CancellationScopeHistory::open($run, $claim, 1, '1.20')->payload['scope_id']; + $historyBefore = $run->historyEvents() + ->count(); + $claimBefore = $claim->fresh() + ->getAttributes(); + foreach ($this->operationCommands() as $command) { + $reply = app(DefaultWorkflowTaskBridge::class)->checkpointLocalActivityPrefix( + $claim->id, + 'scope-owner', + 1, + 'unqualified-major', + 2, + [[ + ...$command, + 'cancellation_scope_id' => $scope, + ]], + '2.0', + ); + $this->assertFalse($reply['checkpointed']); + $this->assertSame('operation_scope_requires_protocol_1_20', $reply['reason']); + $this->assertSame($historyBefore, $run->historyEvents()->count()); + $this->assertSame($claimBefore, $claim->fresh()->getAttributes()); + } + $this->assertSame(0, ActivityExecution::query()->count()); + $this->assertSame(0, WorkflowTimer::query()->count()); + $this->assertSame(0, WorkflowLink::query()->count()); + } + + public function testUnscopedActivityTimerAndChildKeepHistoricalSchedulingShapes(): void + { + [$workflow, $claim] = $this->workflowClaim('unscoped'); + $run = $workflow->run() + ->fresh(); + $reply = app(DefaultWorkflowTaskBridge::class)->complete( + $claim->id, + WorkflowCommandNormalizer::normalize($this->operationCommands(), '1.19'), + ); + $this->assertTrue($reply['completed'], $reply['reason'] ?? ''); + foreach ($run->historyEvents()->whereIn('event_type', [HistoryEventType::ActivityScheduled, + HistoryEventType::TimerScheduled, HistoryEventType::ChildWorkflowScheduled, + HistoryEventType::ChildRunStarted])->get() as $event) { + $this->assertArrayNotHasKey('cancellation_scope_id', $event->payload); + if ($event->event_type === HistoryEventType::ActivityScheduled) { + $this->assertArrayNotHasKey('cancellation_scope_id', $event->payload['activity']); + } + } + $this->assertNull($run->activityExecutions()->sole()->activity_options); + $this->assertArrayNotHasKey( + 'cancellation_scope_id', + $run->tasks() + ->where('task_type', TaskType::Timer)->sole()->payload + ); + $this->assertSame([], CancellationScopeHistory::forRun($run->fresh())); + } + + public function testRetainedClaimMixedGroupPreservesEveryLocalAndRemoteLeafScope(): void + { + [$workflow, $claim] = $this->workflowClaim('mixed-scoped'); + $run = $workflow->run() + ->fresh(); + $first = CancellationScopeHistory::open($run, $claim, 1, '1.20')->payload['scope_id']; + $second = CancellationScopeHistory::open($run, $claim, 2, '1.20')->payload['scope_id']; + $commands = $this->operationCommands(); + array_unshift($commands, [ + ...$commands[0], + 'type' => 'prepare_local_activity', + ]); + foreach ($commands as $index => &$command) { + $command = [ + ...$command, + 'cancellation_scope_id' => $index === 0 ? $first : $second, + ...ParallelChildGroup::itemMetadata(3, 4, $index, 'mixed'), + ]; + } + unset($command); + $before = $claim->fresh() + ->getAttributes(); + $bridge = app(DefaultWorkflowTaskBridge::class); + $reply = $bridge->checkpointLocalActivityGroup( + $claim->id, + 'scope-owner', + 1, + 'mixed-scoped', + 3, + $commands, + '1.20' + ); + $this->assertTrue($reply['checkpointed'], $reply['reason'] ?? ''); + $local = [ + ...$commands[0], + 'type' => 'record_local_activity', + ]; + $prepared = $bridge->prepareLocalActivity( + $claim->id, + 'scope-owner', + 1, + 3, + 'scoped-worker-attempt', + $local, + '1.20' + ); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $this->assertSame($first, $prepared['cancellation_scope_id']); + $this->assertSame($claim->id, $prepared['workflow_task_id']); + $remote = $run->activityExecutions() + ->where('sequence', 4) + ->sole(); + $this->assertSame($second, $remote->activity_options['cancellation_scope_id']); + $this->assertSame($second, $run->historyEvents()->where('event_type', HistoryEventType::TimerScheduled) + ->sole() +->payload['cancellation_scope_id']); + $this->assertSame($second, $run->historyEvents()->where('event_type', HistoryEventType::ChildWorkflowScheduled) + ->sole() +->payload['cancellation_scope_id']); + $this->assertSame(TaskStatus::Leased, $claim->fresh()->status); + $this->assertSame($before['lease_owner'], $claim->fresh()->lease_owner); + $this->assertSame($before['attempt_count'], $claim->fresh()->attempt_count); + $this->assertSame(1, $run->tasks()->where('task_type', TaskType::Activity)->count()); + } + + public function testNestedScopeIdentityAndShieldModeSurviveResponseLossAndFreshRead(): void + { + [$workflow, $claim] = $this->workflowClaim('nested'); + $run = $workflow->run() + ->fresh(); + $parent = CancellationScopeHistory::open($run, $claim, 1, '1.20'); + $child = CancellationScopeHistory::open($run, $claim, 2, '1.20', $parent->payload['scope_id'], true); + $duplicate = CancellationScopeHistory::open($run, $claim, 2, '1.20', $parent->payload['scope_id'], true); + + $this->assertSame($child->id, $duplicate->id); + $this->assertSame($child->fresh()->payload, $duplicate->payload); + $this->assertNotSame($parent->payload['scope_id'], $child->payload['scope_id']); + $scopes = CancellationScopeHistory::forRun($workflow->run()->fresh()); + $this->assertCount(2, $scopes); + $this->assertSame(true, $scopes[$child->payload['scope_id']]['shield_parent']); + $this->assertSame(false, $scopes[$parent->payload['scope_id']]['shield_parent']); + $this->assertSame([ + CancellationScopeHistory::ROOT_SCOPE_ID, + $parent->payload['scope_id'], + $child->payload['scope_id'], + ], CancellationScopeHistory::ancestry($workflow->run()->fresh(), $child->payload['scope_id'])); + $this->assertSame(3, WorkflowStepHistory::nextDurableCommandSequence($workflow->run()->fresh())); + $this->assertNull($workflow->run()->fresh()->cancellation_request_command_id); + $this->assertSame(TaskStatus::Leased, $claim->fresh()->status); + $this->assertSame('scope-owner', $claim->fresh()->lease_owner); + } + + #[DataProvider('changedDefinitionProvider')] + public function testReplayCannotChangeRecordedParentOrShield(bool $changeParent): void + { + [$workflow, $claim] = $this->workflowClaim('changed'); + $run = $workflow->run() + ->fresh(); + $parent = CancellationScopeHistory::open($run, $claim, 1, '1.20'); + $opened = CancellationScopeHistory::open($run, $claim, 2, '1.20', $parent->payload['scope_id']); + $recordedPayload = $opened->fresh() +->payload; + try { + CancellationScopeHistory::open( + $run, + $claim, + 2, + '1.20', + $changeParent ? CancellationScopeHistory::ROOT_SCOPE_ID : $parent->payload['scope_id'], + ! $changeParent, + ); + $this->fail('Changed cancellation scope definition was accepted.'); + } catch (HistoryEventShapeMismatchException $error) { + $this->assertSame(2, $error->workflowSequence); + } + $this->assertSame($recordedPayload, $opened->fresh()->payload); + $this->assertCount(2, CancellationScopeHistory::forRun($run->fresh())); + } + + public static function changedDefinitionProvider(): array + { + return [ + 'parent' => [true], + 'shield' => [false], + ]; + } + + public function testForeignScopeCannotBecomeParentInAnotherRun(): void + { + [$first, $firstClaim] = $this->workflowClaim('first'); + $foreign = CancellationScopeHistory::open($first->run()->fresh(), $firstClaim, 1, '1.20'); + [$second, $secondClaim] = $this->workflowClaim('second'); + try { + CancellationScopeHistory::open( + $second->run() + ->fresh(), + $secondClaim, + 1, + '1.20', + $foreign->payload['scope_id'] + ); + $this->fail('A foreign scope was accepted as parent.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_parent_not_recorded', $error->getMessage()); + } + $this->assertSame([], CancellationScopeHistory::forRun($second->run()->fresh())); + $this->expectExceptionMessage('cancellation_scope_not_recorded'); + CancellationScopeHistory::ancestry($second->run()->fresh(), $foreign->payload['scope_id']); + } + + public function testRecordedCreationCanReplayAfterRunRequestButNewScopeCannotOpen(): void + { + [$workflow, $claim] = $this->workflowClaim('requested'); + $run = $workflow->run() + ->fresh(); + $original = CancellationScopeHistory::open($run, $claim, 1, '1.20'); + $request = $workflow->requestCancellation('maintenance', 30); + $replayed = CancellationScopeHistory::open($run, $claim, 1, '1.20'); + $this->assertSame($original->id, $replayed->id); + $this->assertSame($request->commandId(), $run->fresh()->cancellation_request_command_id); + $this->expectExceptionMessage('cancellation_scope_run_not_active'); + CancellationScopeHistory::open($run, $claim, 2, '1.20'); + } + + #[DataProvider('unsupportedProtocolProvider')] + public function testUnsupportedProtocolCannotCreateHistory(string $protocol): void + { + [$workflow, $claim] = $this->workflowClaim('unsupported'); + try { + CancellationScopeHistory::open($workflow->run()->fresh(), $claim, 1, $protocol); + $this->fail('Unsupported protocol opened a scope.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_requires_protocol_1_20', $error->getMessage()); + } + $this->assertSame([], CancellationScopeHistory::forRun($workflow->run()->fresh())); + } + + public static function unsupportedProtocolProvider(): array + { + return [ + 'published default' => ['1.19'], + 'malformed' => ['1.20.extra'], + 'unknown protocol major' => ['2.0'], + ]; + } + + public function testExpiredAndReplacedClaimsCannotCreateScope(): void + { + [$workflow, $claim] = $this->workflowClaim('expired'); + $original = clone $claim; + $claim->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 2, + ])->save(); + foreach ([$original, $claim->fresh()] as $index => $candidate) { + if ($index === 1) { + Carbon::setTestNow(now()->addMinutes(6)); + } + try { + CancellationScopeHistory::open($workflow->run()->fresh(), $candidate, 1, '1.20'); + $this->fail('An invalid workflow claim opened a scope.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_workflow_claim_mismatch', $error->getMessage()); + } + } + $this->assertSame([], CancellationScopeHistory::forRun($workflow->run()->fresh())); + } + + public function testScopeCreationCannotReplaceAnExistingDurableOperation(): void + { + [$workflow, $claim] = $this->workflowClaim('occupied'); + $run = $workflow->run() + ->fresh(); + WorkflowHistoryEvent::record($run, HistoryEventType::TimerScheduled, [ + 'sequence' => 1, + ], $claim); + $this->expectException(HistoryEventShapeMismatchException::class); + CancellationScopeHistory::open($run, $claim, 1, '1.20'); + } + + public function testMalformedRecordedAncestryIsNotSilentlyIgnored(): void + { + [$workflow, $claim] = $this->workflowClaim('malformed'); + $event = CancellationScopeHistory::open($workflow->run()->fresh(), $claim, 1, '1.20'); + $payload = $event->payload; + $payload['parent_scope_id'] = $payload['scope_id']; + $event->forceFill([ + 'payload' => $payload, + ])->save(); + $this->expectExceptionMessage('cancellation_scope_history_invalid'); + CancellationScopeHistory::forRun($workflow->run()->fresh()); + } + + public function testScopeTransactionUsesConfiguredStorageConnection(): void + { + [$workflow, $claim] = $this->workflowClaim('storage'); + $run = $workflow->run() + ->fresh(); + $originalDefault = config('database.default'); + $originalStorage = config('workflows.storage.connection'); + config([ + 'workflows.storage.connection' => $run->getConnection() + ->getName(), + 'database.default' => 'scope-unconfigured-default', + ]); + try { + $scope = CancellationScopeHistory::open($run, $claim, 1, '1.20'); + $this->assertSame($run->id, $scope->workflow_run_id); + $this->assertCount(1, CancellationScopeHistory::forRun($run)); + } finally { + config([ + 'database.default' => $originalDefault, + 'workflows.storage.connection' => $originalStorage, + ]); + } + } + + public function testReplacementClaimReplaysTheOriginalScopeAndTaskSnapshot(): void + { + [$workflow, $claim] = $this->workflowClaim('cold-replay'); + $original = CancellationScopeHistory::open($workflow->run()->fresh(), $claim, 1, '1.20'); + $recorded = $original->fresh() +->payload; + $claim->forceFill([ + 'lease_owner' => 'replacement-owner', + 'attempt_count' => 2, + ])->save(); + $replayed = CancellationScopeHistory::open($workflow->run()->fresh(), $claim->fresh(), 1, '1.20'); + $this->assertSame($original->id, $replayed->id); + $this->assertSame($recorded, $replayed->payload); + $this->assertSame('scope-owner', $replayed->payload['task']['lease_owner']); + $this->assertSame(1, $replayed->payload['task']['attempt_count']); + $this->assertCount(1, CancellationScopeHistory::forRun($workflow->run()->fresh())); + } + + public function testForeignHostingClaimCannotMutateAnotherRun(): void + { + [$first, $claim] = $this->workflowClaim('claim-owner'); + [$second] = $this->workflowClaim('foreign-claim'); + try { + CancellationScopeHistory::open($second->run()->fresh(), $claim, 1, '1.20'); + $this->fail('A foreign hosting claim opened a scope.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_workflow_claim_mismatch', $error->getMessage()); + } + $this->assertSame([], CancellationScopeHistory::forRun($first->run()->fresh())); + $this->assertSame([], CancellationScopeHistory::forRun($second->run()->fresh())); + } + + public function testExpiredRunCanReplayRecordedScopeButCannotAdmitNewScope(): void + { + [$workflow, $claim] = $this->workflowClaim('run-deadline'); + $run = $workflow->run() + ->fresh(); + $original = CancellationScopeHistory::open($run, $claim, 1, '1.20'); + $run->forceFill([ + 'run_deadline_at' => now(), + ])->save(); + $this->assertSame($original->id, CancellationScopeHistory::open($run, $claim, 1, '1.20')->id); + $this->expectExceptionMessage('cancellation_scope_run_not_active'); + CancellationScopeHistory::open($run, $claim, 2, '1.20'); + } + + /** + * @return list + */ + private function operationCommands(): array + { + return [ + [ + 'type' => 'schedule_activity', + 'activity_type' => 'remote-scope-fixture', + 'arguments' => Serializer::serializeWithCodec('avro', ['value']), + 'payload_codec' => 'avro', + ], + [ + 'type' => 'start_timer', + 'delay_seconds' => 5, + ], + [ + 'type' => 'start_child_workflow', + 'workflow_type' => 'child-scope-fixture', + 'arguments' => Serializer::serializeWithCodec('avro', ['child']), + 'payload_codec' => 'avro', + ], + ]; + } + + /** + * @return array + */ + private function waitCommand(string $kind, ?int $timeout): array + { + return array_filter([ + 'type' => $kind === 'signal' ? 'open_signal_wait' : 'open_condition_wait', + ...($kind === 'signal' ? [ + 'signal_name' => 'ready', + ] : [ + 'condition_key' => 'ready', + ]), + 'timeout_seconds' => $timeout, + ], static fn (mixed $value): bool => $value !== null); + } + + /** + * @return array{WorkflowStub, WorkflowTask} + */ + private function workflowClaim(string $name): array + { + $workflow = WorkflowStub::make(TestSignalWorkflow::class, 'scope-' . $name); + $workflow->start(); + $task = WorkflowTask::query()->where('workflow_run_id', $workflow->runId()) + ->where('task_type', TaskType::Workflow->value)->sole(); + $task->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'scope-owner', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addMinutes(5), + ])->save(); + return [$workflow, $task]; + } +} diff --git a/tests/Feature/V2/V2CancellationScopeRequestsTest.php b/tests/Feature/V2/V2CancellationScopeRequestsTest.php new file mode 100644 index 000000000..e813494bb --- /dev/null +++ b/tests/Feature/V2/V2CancellationScopeRequestsTest.php @@ -0,0 +1,1049 @@ + 'poll', + ]); + Carbon::setTestNow('2026-10-03T00:00:00Z'); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + public function testAcceptedRequestRecoversBeforePreparationWithoutCreatingANewBudget(): void + { + [, $run, , $scope] = $this->scopeTree('request-recovery'); + $task = $run->tasks() + ->sole(); + WorkflowRunSummary::query()->whereKey($run->id)->update([ + 'next_task_id' => $task->id, + 'next_task_lease_expires_at' => $task->lease_expires_at, + ]); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $acceptedClaim = $task->fresh() + ->getAttributes(); + $acceptedSummary = WorkflowRunSummary::query()->findOrFail($run->id); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeDeliveryPrepared)->count() + ); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeDelivered)->count() + ); + + Carbon::setTestNow('2026-10-03T00:00:11Z'); + $repair = TaskWatchdog::runPass(respectThrottle: false, runIds: [$run->id]); + $this->assertSame([], $repair['existing_task_failures']); + $this->assertSame(1, $repair['repaired_existing_tasks']); + $this->assertTrue(app(DefaultWorkflowTaskBridge::class)->claimStatus($task->id, 'replacement')['claimed']); + $replacement = $task->fresh(); + $this->assertSame(2, $replacement->attempt_count); + $this->assertSame('2026-10-03T00:00:21.000000Z', $replacement->lease_expires_at->toISOString()); + $this->assertSame('2026-10-03T00:00:10.000000Z', $acceptedSummary->next_task_lease_expires_at->toISOString()); + $this->assertSame($acceptedClaim['lease_owner'], $task->lease_owner); + $this->assertSame($acceptedClaim['attempt_count'], $task->attempt_count); + $this->assertSame( + '2026-10-03T00:00:30.000000Z', + $run->fresh() + ->cancellation_scope_recovery_until->toISOString() + ); + + $beforeDuplicate = $replacement->getAttributes(); + $duplicate = CancellationScopeRequests::request($run->fresh(), $scope, '1.20', 300); + $this->assertSame($request->id, $duplicate->id); + $this->assertSameJsonObject($request->payload, $duplicate->payload); + $this->assertSame($beforeDuplicate, $replacement->fresh()->getAttributes()); + + $historyBefore = $run->historyEvents() + ->count(); + try { + CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + 3, + 'local_activity', + '1.20' + ); + $this->fail('Replaced owner must not prepare the cancellation.'); + } catch (LogicException $exception) { + $this->assertSame('cancellation_scope_workflow_claim_mismatch', $exception->getMessage()); + } + $this->assertSame($historyBefore, $run->historyEvents()->count()); + $this->assertSame($beforeDuplicate, $replacement->fresh()->getAttributes()); + $prepared = CancellationScopeDelivery::prepare( + $run->fresh(), + $replacement, + $scope, + $request->payload['request_id'], + 3, + 'local_activity', + '1.20' + ); + $this->assertSame($request->payload['request_id'], $prepared->payload['request_id']); + $this->assertSameJsonObject($request->payload['cancellation'], $prepared->payload['cancellation']); + $this->assertSame('2026-10-03T00:00:30.000000Z', $prepared->payload['authority_deadline_at']); + $this->assertSame('2026-10-03T00:00:11.000000Z', $prepared->recorded_at->toISOString()); + $this->assertNull(CancellationScopeDelivery::recorded($run->fresh(), $scope)); + $this->assertRunCancellationUntouched($run); + } + + public function testDirectRequestColdReadsAndDuplicatesPreserveOriginalMetadataWithoutCancellingRun(): void + { + [$workflow, $run, $parent, $child] = $this->scopeTree('direct'); + $claimBefore = $run->tasks() + ->sole() + ->getAttributes(); + $event = CancellationScopeRequests::request( + $run, + $child, + '1.20', + 30, + 'release resource', + CommandContext::phpApi()->withPrincipal('operator', 'operator-fixture', 'Operator'), + ); + $context = CancellationScopeRequests::context($run->fresh(), $child); + $this->assertSame($event->payload['request_id'], $context->requestId); + $this->assertSame($context->requestId, $context->rootContext->rootRequestId); + $this->assertSame($child, $context->rootScopeId); + $this->assertSame('release resource', $context->rootContext->reason); + $this->assertSameJsonObject([ + 'type' => 'operator', + 'id' => 'operator-fixture', + 'label' => 'Operator', + ], $context->rootContext->requester); + $this->assertSame('php', $context->rootContext->source); + $this->assertSame('2026-10-03T00:00:00.000000Z', $context->requestedAt()->toISOString()); + $this->assertSame('2026-10-03T00:00:30.000000Z', $context->deadline()->toISOString()); + Carbon::setTestNow('2026-10-03T00:00:10Z'); + $duplicate = CancellationScopeRequests::request($run->fresh(), $child, '1.20', 300, 'different reason'); + $this->assertSame($event->id, $duplicate->id); + $this->assertSame($context->toArray(), CancellationScopeRequests::context($run->fresh(), $child)->toArray()); + $this->assertNull(CancellationScopeRequests::context($run->fresh(), $parent)); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeRequested)->count() + ); + $claimAfter = $run->tasks() + ->sole() + ->getAttributes(); + $this->assertSame('2026-10-03T00:00:10.000000Z', $run->tasks()->sole()->lease_expires_at->toISOString()); + unset($claimBefore['lease_expires_at'], $claimAfter['lease_expires_at'], $claimBefore['updated_at'], $claimAfter['updated_at']); + $this->assertSame($claimBefore, $claimAfter); + $this->assertRunCancellationUntouched($run); + } + + #[DataProvider('requestHostingIntervals')] + public function testAcceptanceShortensLiveOwnershipWithoutExtendingAnExistingLease( + int $budget, + int $configuredLease, + int $existingLease, + int $expectedInterval, + ): void { + [, $run, , $scope] = $this->scopeTree('request-interval'); + config() + ->set('workflows.v2.workflow_task_lease_seconds', $configuredLease); + $task = $run->tasks() + ->sole(); + $task->forceFill([ + 'lease_expires_at' => now() + ->addSeconds($existingLease), + ])->save(); + $before = $task->getAttributes(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', $budget); + $after = $task->fresh() + ->getAttributes(); + $this->assertSame( + now() + ->addSeconds($expectedInterval) + ->toISOString(), + $task->fresh() + ->lease_expires_at->toISOString() + ); + unset($before['lease_expires_at'], $after['lease_expires_at'], $before['updated_at'], $after['updated_at']); + $this->assertSame($before, $after); + $this->assertSame( + now() + ->addSeconds($budget) + ->toISOString(), + $run->fresh() + ->cancellation_scope_recovery_until->toISOString() + ); + $this->assertSame( + now() + ->addSeconds($budget) + ->toISOString(), + CancellationScopeRequests::context($run->fresh(), $scope)->deadline()->toISOString() + ); + $this->assertSame($request->id, CancellationScopeRequests::request($run, $scope, '1.20', 3600)->id); + $this->assertRunCancellationUntouched($run); + } + + public static function requestHostingIntervals(): iterable + { + yield 'ordinary lease' => [30, 300, 300, 10]; + yield 'smaller configured lease' => [30, 5, 300, 5]; + yield 'short original lease is preserved' => [30, 300, 4, 4]; + yield 'short scope budget preserves shared hosting' => [3, 300, 300, 10]; + } + + public function testAcceptanceMakesPendingClaimRecoverableWithoutLeasingIt(): void + { + [, $run, , $scope] = $this->scopeTree('request-pending'); + $task = $run->tasks() + ->sole(); + $task->forceFill([ + 'status' => TaskStatus::Ready, + 'lease_owner' => null, + 'lease_expires_at' => null, + ])->save(); + $before = $task->fresh() + ->getAttributes(); + CancellationScopeRequests::request($run, $scope, '1.20', 30); + $this->assertSame($before, $task->fresh()->getAttributes()); + $this->assertSame( + '2026-10-03T00:00:30.000000Z', + $run->fresh() + ->cancellation_scope_recovery_until->toISOString() + ); + $this->assertTrue(app(DefaultWorkflowTaskBridge::class)->claimStatus($task->id, 'first-owner')['claimed']); + $this->assertSame('2026-10-03T00:00:10.000000Z', $task->fresh()->lease_expires_at->toISOString()); + $this->assertNull(CancellationScopeDelivery::recorded($run->fresh(), $scope)); + $this->assertRunCancellationUntouched($run); + } + + public function testAcceptanceWakesWaitingRunWithoutDeliveringOrDuplicatingAnOpenClaim(): void + { + [, $run, , $scope] = $this->scopeTree('request-sleeping'); + $original = $run->tasks() + ->sole(); + $original->forceFill([ + 'status' => TaskStatus::Completed, + 'lease_owner' => null, + 'lease_expires_at' => null, + ])->save(); + $run->forceFill([ + 'status' => RunStatus::Waiting, + ])->save(); + + $accepted = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $task = $run->tasks() + ->where('status', TaskStatus::Ready)->sole(); + $this->assertNotSame($original->id, $task->id); + $this->assertSame(TaskType::Workflow, $task->task_type); + $this->assertNull($task->lease_owner); + $this->assertNull($task->lease_expires_at); + $this->assertSame($run->namespace, $task->namespace); + $this->assertSame($run->connection, $task->connection); + $this->assertSame($run->queue, $task->queue); + $this->assertSame($run->compatibility, $task->compatibility); + $this->assertSame('2026-10-03T00:00:00.000000Z', $task->available_at->toISOString()); + $this->assertSameJsonObject([ + 'resume_source_kind' => 'cancellation_scope_request', + 'resume_source_id' => $accepted->id, + 'workflow_command_id' => $accepted->payload['request_id'], + 'scope_id' => $scope, + ], $task->payload); + $this->assertSame($task->id, WorkflowRunSummary::query()->findOrFail($run->id)->next_task_id); + $this->assertSame(RunStatus::Waiting, $run->fresh()->status); + $this->assertSame($accepted->id, CancellationScopeRequests::request($run, $scope, '1.20', 3600)->id); + $this->assertSame(2, $run->tasks()->count()); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeRequested)->count() + ); + $this->assertSame(0, $run->historyEvents()->whereIn('event_type', [ + HistoryEventType::CancellationScopeDeliveryPrepared, + HistoryEventType::CancellationScopeDelivered, + ])->count()); + $this->assertRunCancellationUntouched($run); + } + + public function testAcceptanceDoesNotReviveAnExpiredClaim(): void + { + [, $run, , $scope] = $this->scopeTree('request-expired'); + $task = $run->tasks() + ->sole(); + $task->forceFill([ + 'lease_expires_at' => now() + ->subSecond(), + ])->save(); + $before = $task->fresh() + ->getAttributes(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $this->assertSame($before, $task->fresh()->getAttributes()); + $this->assertSame( + '2026-10-03T00:00:30.000000Z', + $run->fresh() + ->cancellation_scope_recovery_until->toISOString() + ); + try { + CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + 3, + 'local_activity', + '1.20' + ); + $this->fail('An expired owner must not prepare the cancellation.'); + } catch (LogicException $exception) { + $this->assertSame('cancellation_scope_workflow_claim_mismatch', $exception->getMessage()); + } + $this->assertSame($before, $task->fresh()->getAttributes()); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeDeliveryPrepared)->count() + ); + $this->assertRunCancellationUntouched($run); + } + + public function testAcceptanceLeavesActivityOwnershipAndOtherRunsAlone(): void + { + [, $run, $parent, $scope] = $this->scopeTree('request-isolation'); + $activity = $run->tasks() + ->sole() + ->replicate(); + $activity->forceFill([ + 'task_type' => TaskType::Activity, + ])->save(); + [, $other] = $this->scopeTree('request-other-run'); + $activityBefore = $activity->fresh() + ->getAttributes(); + $otherClaimBefore = $other->tasks() + ->sole() + ->getAttributes(); + CancellationScopeRequests::request($run, $scope, '1.20', 30); + $this->assertSame($activityBefore, $activity->fresh()->getAttributes()); + $this->assertSame($otherClaimBefore, $other->tasks()->sole()->getAttributes()); + $this->assertNull($other->fresh()->cancellation_scope_recovery_until); + $this->assertNull(CancellationScopeRequests::context($run->fresh(), $parent)); + $this->assertSame( + '2026-10-03T00:00:10.000000Z', + $run->tasks() + ->where('task_type', TaskType::Workflow)->sole()->lease_expires_at->toISOString() + ); + $this->assertRunCancellationUntouched($run); + } + + public function testAcceptanceProjectsAuthorityCeilingWithoutChangingAcceptedDeadline(): void + { + [, $run, , $scope] = $this->scopeTree('request-ceiling'); + $run->forceFill([ + 'execution_deadline_at' => now() + ->addSeconds(5), + ])->save(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $this->assertSame( + '2026-10-03T00:00:05.000000Z', + $run->fresh() + ->cancellation_scope_recovery_until->toISOString() + ); + $this->assertSame( + '2026-10-03T00:00:30.000000Z', + CancellationScopeRequests::context($run->fresh(), $scope)->deadline()->toISOString() + ); + $this->assertSame('2026-10-03T00:00:10.000000Z', $run->tasks()->sole()->lease_expires_at->toISOString()); + $this->assertSame($request->id, CancellationScopeRequests::request($run->fresh(), $scope, '1.20', 300)->id); + $this->assertRunCancellationUntouched($run); + } + + public function testAcceptanceProjectionKeepsHeartbeatsRecoverableOnlyDuringOriginalScopeBudget(): void + { + [, $run, , $scope] = $this->scopeTree('request-heartbeat'); + config() + ->set('workflows.v2.workflow_task_lease_seconds', 300); + $task = $run->tasks() + ->sole(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 15); + Carbon::setTestNow('2026-10-03T00:00:05Z'); + $this->assertTrue(app(DefaultWorkflowTaskBridge::class)->heartbeat($task->id)['renewed']); + $this->assertSame('2026-10-03T00:00:15.000000Z', $task->fresh()->lease_expires_at->toISOString()); + $this->advanceWithWorkflowHeartbeats($task, now()->copy()->addSeconds(11)); + $this->assertTrue(app(DefaultWorkflowTaskBridge::class)->heartbeat($task->id)['renewed']); + $this->assertSame('2026-10-03T00:05:16.000000Z', $task->fresh()->lease_expires_at->toISOString()); + $this->assertFalse(CancellationScopeRequests::authority($run->fresh(), $scope)['active']); + $this->assertSame( + '2026-10-03T00:00:15.000000Z', + $run->fresh() + ->cancellation_scope_recovery_until->toISOString() + ); + $this->assertSame($request->id, CancellationScopeRequests::request($run->fresh(), $scope, '1.20', 300)->id); + $this->assertRunCancellationUntouched($run); + } + + public function testInheritanceHasDistinctAddressIdentityAndOneOriginalRootBudget(): void + { + [, $run, $parent, $child] = $this->scopeTree('inherit'); + $first = CancellationScopeRequests::request($run, $parent, '1.20', 30, 'original reason'); + $acceptedClaim = $run->tasks() + ->sole() + ->getAttributes(); + Carbon::setTestNow('2026-10-03T00:00:09Z'); + $inherited = CancellationScopeRequests::request( + $run, + $child, + '1.20', + 300, + 'ignored descendant reason', + parentScopeId: $parent + ); + $context = CancellationScopeRequests::context($run->fresh(), $child); + $this->assertNotSame($first->payload['request_id'], $context->requestId); + $this->assertSame($first->payload['request_id'], $context->parentRequestId); + $this->assertSame($first->payload['request_id'], $context->rootContext->rootRequestId); + $this->assertSame([$parent, $child], array_column($context->lineage, 'scope_id')); + $this->assertSame('original reason', $context->rootContext->reason); + $this->assertSame('2026-10-03T00:00:00.000000Z', $context->requestedAt()->toISOString()); + $this->assertSame('2026-10-03T00:00:30.000000Z', $context->deadline()->toISOString()); + $this->assertSame($context->rootDeadline()->toISOString(), $context->deadline()->toISOString()); + $this->assertSame($acceptedClaim, $run->tasks()->sole()->getAttributes()); + $this->assertSame( + '2026-10-03T00:00:30.000000Z', + $run->fresh() + ->cancellation_scope_recovery_until->toISOString() + ); + $this->assertSame( + $inherited->id, + CancellationScopeRequests::request($run, $child, '1.20', 3600, parentScopeId: $parent)->id + ); + $this->assertRunCancellationUntouched($run); + } + + public function testCompetingAncestorRootIsVisibleAndShortensAuthorityWithoutRewritingAcceptedBudget(): void + { + [, $run, $parent, $child] = $this->scopeTree('conflict', true); + CancellationScopeRequests::request($run, $child, '1.20', 30, 'independent scope'); + $accepted = CancellationScopeRequests::context($run, $child)->toArray(); + $ancestor = CancellationScopeRequests::request($run, $parent, '1.20', 10, 'parent budget'); + $conflict = CancellationScopeRequests::request($run, $child, '1.20', 300, parentScopeId: $parent); + $this->assertSame(HistoryEventType::CancellationScopeRequestConflicted, $conflict->event_type); + $this->assertSame('cancellation_root_conflict', $conflict->payload['reason']); + $this->assertSameJsonObject($accepted, $conflict->payload['accepted_cancellation']); + $incoming = $conflict->payload['incoming_cancellation']; + $this->assertSame($ancestor->payload['request_id'], $incoming['root_context']['root_request_id']); + $this->assertSame('2026-10-03T00:00:10.000000Z', $incoming['root_context']['cleanup_deadline_at']); + $this->assertSame($accepted, CancellationScopeRequests::context($run->fresh(), $child)->toArray()); + $this->assertSame([ + 'active' => true, + 'deadline_at' => '2026-10-03T00:00:10.000000Z', + ], CancellationScopeRequests::authority($run, $child)); + $timeline = collect(HistoryTimeline::fromHistory($run->fresh()))->firstWhere('id', $conflict->id); + $this->assertSame('cancellation_scope', $timeline['kind']); + $this->assertSameJsonObject($accepted, $timeline['cancellation_scope']['accepted_cancellation']); + $this->assertSameJsonObject($incoming, $timeline['cancellation_scope']['incoming_cancellation']); + $this->assertSame( + $conflict->id, + CancellationScopeRequests::request($run->fresh(), $child, '1.20', 3600, parentScopeId: $parent)->id + ); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeRequestConflicted)->count() + ); + Carbon::setTestNow('2026-10-03T00:00:10Z'); + $this->assertFalse(CancellationScopeRequests::authority($run, $child)['active']); + $this->assertSame($accepted, CancellationScopeRequests::context($run->fresh(), $child)->toArray()); + $this->assertRunCancellationUntouched($run); + } + + public function testRunRequestIsCanonicalParentAndOriginalRunFieldsKeepTheirMeaning(): void + { + [$workflow, $run, $parent] = $this->scopeTree('run-parent'); + $runRequest = $workflow->requestCancellation('whole run', 30) + ->cancellationContext(); + Carbon::setTestNow('2026-10-03T00:00:08Z'); + $event = CancellationScopeRequests::request($run, $parent, '1.20', 300, parentScopeId: 'root'); + $context = CancellationScopeRequests::context($run->fresh(), $parent); + $this->assertSame($runRequest->requestId, $context->rootContext->rootRequestId); + $this->assertSame($runRequest->requestId, $context->parentRequestId); + $this->assertSame(['root', $parent], array_column($context->lineage, 'scope_id')); + $this->assertSame($runRequest->deadline()->toISOString(), $context->deadline()->toISOString()); + $this->assertSame($runRequest->requestId, $run->fresh()->cancellation_request_command_id); + $this->assertNotSame($runRequest->requestId, $event->payload['request_id']); + $this->assertSame( + $runRequest->deadline() + ->toISOString(), + CancellationScopeRequests::authority($run, $parent)['deadline_at'] + ); + } + + public function testRunRequestPropagatesOriginalIdentityAndDeadlineToUnshieldedScopes(): void + { + [$workflow, $run, $parent, $child] = $this->scopeTree('run-propagation'); + $root = $workflow->withCommandContext( + CommandContext::phpApi()->withPrincipal('operator', 'operator-fixture', 'Operator') + )->requestCancellation('release the whole tree', 30) + ->cancellationContext(); + + $parentContext = CancellationScopeRequests::context($run->fresh(), $parent); + $childContext = CancellationScopeRequests::context($run->fresh(), $child); + $this->assertNotNull($parentContext); + $this->assertNotNull($childContext); + $this->assertSame(['root', $parent, $child], array_column($childContext->lineage, 'scope_id')); + $this->assertSame($parentContext->requestId, $childContext->parentRequestId); + foreach ([$parentContext, $childContext] as $context) { + $this->assertSame($root->toArray(), $context->rootContext->toArray()); + $this->assertSame($root->deadline()->toISOString(), $context->deadline()->toISOString()); + $this->assertSame($root->requestedAt()->toISOString(), $context->requestedAt()->toISOString()); + } + $this->assertSame( + 2, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeRequested)->count() + ); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeDelivered)->count() + ); + $this->assertFalse($run->fresh()->status->isTerminal()); + + $historyCount = $run->historyEvents() + ->count(); + Carbon::setTestNow(now()->addSeconds(10)); + $duplicate = $workflow->requestCancellation('replacement reason', 300) + ->cancellationContext(); + $this->assertSame($root->toArray(), $duplicate->toArray()); + $this->assertSame($historyCount, $run->historyEvents()->count()); + $this->assertSame( + $childContext->toArray(), + CancellationScopeRequests::context($run->fresh(), $child)->toArray() + ); + } + + public function testRunRequestDoesNotPropagateThroughAShield(): void + { + [$workflow, $run, $parent, $shield] = $this->scopeTree('run-shield', true); + $task = $run->tasks() + ->sole(); + $shieldChild = CancellationScopeHistory::open($run, $task, 3, '1.20', $shield)->payload['scope_id']; + $sibling = CancellationScopeHistory::open($run, $task, 4, '1.20')->payload['scope_id']; + $root = $workflow->requestCancellation('whole run', 30) + ->cancellationContext(); + + foreach ([$parent, $sibling] as $scope) { + $context = CancellationScopeRequests::context($run->fresh(), $scope); + $this->assertNotNull($context); + $this->assertSame($root->requestId, $context->rootContext->rootRequestId); + $this->assertSame($root->deadline()->toISOString(), $context->deadline()->toISOString()); + } + $this->assertNull(CancellationScopeRequests::context($run->fresh(), $shield)); + $this->assertNull(CancellationScopeRequests::context($run->fresh(), $shieldChild)); + $this->assertSame( + $root->deadline() + ->toISOString(), + CancellationScopeRequests::authority($run->fresh(), $shieldChild)['deadline_at'] + ); + $this->assertSame( + 2, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeRequested)->count() + ); + } + + public function testRunRequestPreservesAnIndependentScopeRequestAndCapsItsAuthority(): void + { + [$workflow, $run, $parent, $child] = $this->scopeTree('run-conflict'); + $accepted = CancellationScopeRequests::request($run, $child, '1.20', 300, 'independent cleanup'); + $original = CancellationScopeRequests::context($run->fresh(), $child)->toArray(); + $root = $workflow->requestCancellation('whole run', 30) + ->cancellationContext(); + + $this->assertSame($original, CancellationScopeRequests::context($run->fresh(), $child)->toArray()); + $this->assertSame($accepted->id, CancellationScopeRequests::request($run->fresh(), $child, '1.20', 3600)->id); + $conflict = $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeRequestConflicted)->sole(); + $this->assertSameJsonObject($original, $conflict->payload['accepted_cancellation']); + $this->assertSame($parent, $conflict->payload['parent_scope_id']); + $this->assertSame( + $root->requestId, + $conflict->payload['incoming_cancellation']['root_context']['root_request_id'] + ); + $this->assertSame( + $root->deadline() + ->toISOString(), + CancellationScopeRequests::authority($run->fresh(), $child)['deadline_at'] + ); + Carbon::setTestNow(now()->addSeconds(30)); + $this->assertFalse(CancellationScopeRequests::authority($run->fresh(), $child)['active']); + $this->assertSame($original, CancellationScopeRequests::context($run->fresh(), $child)->toArray()); + $duplicate = $workflow->requestCancellation('duplicate', 3600) + ->cancellationContext(); + $this->assertSame($root->requestId, $duplicate->requestId); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeRequestConflicted)->count() + ); + } + + public function testRunRequestRollsBackTheWholePropagationWhenADescendantIsInvalid(): void + { + [$workflow, $run, $parent, $child] = $this->scopeTree('run-atomicity'); + $accepted = CancellationScopeRequests::request($run, $child, '1.20', 300); + $payload = $accepted->payload; + $payload['cancellation']['lineage'][0]['scope_id'] = 'fabricated'; + $accepted->forceFill([ + 'payload' => $payload, + ])->save(); + $historyCount = $run->historyEvents() + ->count(); + $claimBefore = $run->tasks() + ->sole() + ->getAttributes(); + + try { + $workflow->requestCancellation('whole run', 30); + $this->fail('Invalid descendant authority must not leave a partial run request.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_request_history_invalid', $error->getMessage()); + } + $this->assertSame($historyCount, $run->historyEvents()->count()); + $this->assertNull(CancellationScopeRequests::context($run->fresh(), $parent)); + $this->assertSame($claimBefore, $run->tasks()->sole()->getAttributes()); + $this->assertRunCancellationUntouched($run); + } + + #[DataProvider('authorityCeilings')] + public function testShieldedIndependentScopeCannotOutliveRunAuthority(string $ceiling): void + { + [$workflow, $run, , $child] = $this->scopeTree('ceiling', true); + CancellationScopeRequests::request($run, $child, '1.20', 30); + $original = CancellationScopeRequests::context($run, $child)->toArray(); + if ($ceiling === 'cancellation') { + $workflow->requestCancellation('run ceiling', 12); + } else { + $run->forceFill([ + $ceiling => now() + ->addSeconds(12), + ])->save(); + } + $this->assertSame( + '2026-10-03T00:00:12.000000Z', + CancellationScopeRequests::authority($run, $child)['deadline_at'] + ); + Carbon::setTestNow('2026-10-03T00:00:12Z'); + $this->assertFalse(CancellationScopeRequests::authority($run, $child)['active']); + $this->assertSame($original, CancellationScopeRequests::context($run->fresh(), $child)->toArray()); + } + + public static function authorityCeilings(): iterable + { + yield 'run cancellation' => ['cancellation']; + yield 'execution deadline' => ['execution_deadline_at']; + yield 'run deadline' => ['run_deadline_at']; + } + + public function testTerminalRunReturnsAcceptedRetryButRejectsNewScopeRequest(): void + { + [, $run, $parent, $child] = $this->scopeTree('terminal'); + $event = CancellationScopeRequests::request($run, $child, '1.20'); + $run->forceFill([ + 'status' => RunStatus::Completed, + 'closed_at' => now(), + ])->save(); + $this->assertSame($event->id, CancellationScopeRequests::request($run, $child, '1.20', 300)->id); + $this->assertFalse(CancellationScopeRequests::authority($run, $child)['active']); + $this->expectExceptionMessage('cancellation_scope_authority_expired'); + CancellationScopeRequests::request($run, $parent, '1.20'); + } + + public function testExpiredParentCannotGrantNewDescendantBudget(): void + { + [, $run, $parent, $child] = $this->scopeTree('expired'); + CancellationScopeRequests::request($run, $parent, '1.20', 10); + Carbon::setTestNow('2026-10-03T00:00:10Z'); + $before = $run->historyEvents() + ->count(); + try { + CancellationScopeRequests::request($run, $child, '1.20', 300, parentScopeId: $parent); + $this->fail('An expired parent must not grant new authority.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_authority_expired', $error->getMessage()); + } + $this->assertNull(CancellationScopeRequests::context($run, $child)); + $this->assertSame($before, $run->historyEvents()->count()); + } + + #[DataProvider('invalidScopes')] + public function testUnknownForeignAndImplicitRootAddressesRefuseBeforeWriting(string $kind): void + { + [, $run] = $this->scopeTree('invalid'); + $scope = $kind === 'root' ? 'root' : 'missing'; + if ($kind === 'foreign') { + [, , $scope] = $this->scopeTree('foreign'); + } + $before = $run->historyEvents() + ->count(); + try { + CancellationScopeRequests::request($run, $scope, '1.20'); + $this->fail('An address outside the exact run must be refused.'); + } catch (LogicException $error) { + $this->assertSame( + $kind === 'root' ? 'root_scope_requires_run_cancellation' : 'cancellation_scope_not_recorded', + $error->getMessage() + ); + } + $this->assertSame($before, $run->historyEvents()->count()); + $this->assertRunCancellationUntouched($run); + } + + public static function invalidScopes(): iterable + { + yield 'unknown' => ['unknown']; + yield 'foreign' => ['foreign']; + yield 'implicit run root' => ['root']; + } + + #[DataProvider('invalidParents')] + public function testInheritanceRejectsUnrecordedRequestAndWrongParent(string $kind): void + { + [, $run, $parent, $child] = $this->scopeTree('invalid-parent'); + $requestedParent = $kind === 'unrequested' ? $parent : 'root'; + $before = $run->historyEvents() + ->count(); + try { + CancellationScopeRequests::request($run, $child, '1.20', parentScopeId: $requestedParent); + $this->fail('Inheritance must use its accepted canonical immediate parent.'); + } catch (LogicException $error) { + $this->assertSame( + $kind === 'unrequested' ? 'cancellation_scope_parent_request_unavailable' : 'cancellation_scope_parent_mismatch', + $error->getMessage() + ); + } + $this->assertSame($before, $run->historyEvents()->count()); + } + + public static function invalidParents(): iterable + { + yield 'parent has no request' => ['unrequested']; + yield 'skip canonical edge' => ['wrong']; + } + + public function testCorruptCanonicalRequestAddressIsRefusedOnFreshRead(): void + { + [, $run, , $child] = $this->scopeTree('corrupt'); + $event = CancellationScopeRequests::request($run, $child, '1.20'); + $payload = $event->payload; + $payload['cancellation']['lineage'][0]['scope_id'] = 'fabricated'; + $event->forceFill([ + 'payload' => $payload, + ])->save(); + $this->expectExceptionMessage('cancellation_scope_request_history_invalid'); + CancellationScopeRequests::context($run->fresh(), $child); + } + + #[DataProvider('invalidProtocols')] + public function testRequestsRequireKnownCandidateProtocol(string $version): void + { + [, $run, $parent] = $this->scopeTree('protocol'); + $this->expectExceptionMessage('cancellation_scope_requires_protocol_1_20'); + CancellationScopeRequests::request($run, $parent, $version); + } + + public static function invalidProtocols(): iterable + { + yield 'published default' => ['1.19']; + yield 'unknown major' => ['2.0']; + } + + #[DataProvider('invalidTimeouts')] + public function testRequestBudgetMustBeFiniteAndBounded(int $timeout): void + { + [, $run, $parent] = $this->scopeTree('invalid-budget'); + $before = $run->historyEvents() + ->count(); + try { + CancellationScopeRequests::request($run, $parent, '1.20', $timeout); + $this->fail('Unbounded or empty cleanup budgets must be refused.'); + } catch (LogicException $error) { + $this->assertSame('invalid_scope_cancellation_request', $error->getMessage()); + } + $this->assertSame($before, $run->historyEvents()->count()); + } + + public static function invalidTimeouts(): iterable + { + yield 'zero seconds' => [0]; + yield 'above maximum' => [3601]; + } + + public function testWholeRunProjectionCannotForgeTheInheritedRootBudget(): void + { + [$workflow, $run, $parent] = $this->scopeTree('corrupt-run'); + $workflow->requestCancellation('real request', 30); + $run->refresh() + ->forceFill([ + 'cancellation_deadline_at' => now() + ->addSeconds(300), + ])->save(); + $this->expectExceptionMessage('cancellation_scope_run_context_mismatch'); + CancellationScopeRequests::request($run, $parent, '1.20', parentScopeId: 'root'); + } + + #[DataProvider('requestRaces')] + public function testConcurrentRequestsKeepOneAcceptedAddressAndExplicitCompetingRoot( + bool $winnerInherited, + bool $loserInherited, + ): void { + $driver = DB::connection()->getDriverName(); + if (! in_array($driver, ['mysql', 'pgsql'], true) + || ($driver === 'mysql' && str_contains( + strtolower((string) DB::selectOne('SELECT VERSION() AS version')->version), + 'mariadb' + )) + || ! function_exists('pcntl_fork') || ! function_exists('posix_kill')) { + $this->markTestSkipped('MySQL or PostgreSQL row-lock observation and process control are required.'); + } + [, $run, $parent, $child] = $this->scopeTree('race'); + CancellationScopeRequests::request($run, $parent, '1.20', 30, 'ancestor'); + $parentContext = CancellationScopeRequests::context($run, $parent)->toArray(); + Carbon::setTestNow(now()->addSeconds(5)); + DB::purge(); + $actors = []; + try { + $actors['winner'] = $this->forkScopeRequest($run->id, $child, $winnerInherited ? $parent : null, true); + $this->readActor($actors['winner']); + $actors['loser'] = $this->forkScopeRequest($run->id, $child, $loserInherited ? $parent : null, false); + $ready = $this->readActor($actors['loser']); + fwrite($actors['winner']['socket'], "go\n"); + $uncommitted = $this->readActor($actors['winner']); + $this->assertSame(HistoryEventType::CancellationScopeRequested->value, $uncommitted['type']); + fwrite($actors['loser']['socket'], "go\n"); + $query = $driver === 'mysql' + ? 'SELECT COUNT(*) AS waiting FROM performance_schema.data_lock_waits w JOIN performance_schema.threads t ON t.THREAD_ID = w.REQUESTING_THREAD_ID WHERE t.PROCESSLIST_ID = ?' + : 'SELECT COUNT(*) AS waiting FROM pg_locks WHERE pid = ? AND NOT granted'; + $until = microtime(true) + 5; + do { + $waiting = (int) DB::selectOne($query, [$ready['connection_id']])->waiting; + if ($waiting > 0) { + break; + } + usleep(10_000); + } while (microtime(true) < $until); + $this->assertGreaterThan(0, $waiting, 'The second actor must actually wait on the database lock.'); + $this->assertNull(CancellationScopeRequests::context($run->fresh(), $child)); + fwrite($actors['winner']['socket'], "commit\n"); + $winner = $this->finishActor($actors['winner']); + unset($actors['winner']); + $loser = $this->finishActor($actors['loser']); + unset($actors['loser']); + } finally { + foreach ($actors as $actor) { + posix_kill($actor['pid'], SIGKILL); + pcntl_waitpid($actor['pid'], $status); + fclose($actor['socket']); + } + DB::purge(); + DB::reconnect(); + } + $accepted = CancellationScopeRequests::context($run->fresh(), $child)->toArray(); + $this->assertSameJsonObject($winner['payload']['cancellation'], $accepted); + $this->assertSame( + $winnerInherited ? '2026-10-03T00:00:30.000000Z' : '2026-10-03T00:00:25.000000Z', + $accepted['lineage'][count($accepted['lineage']) - 1]['cleanup_deadline_at'] + ); + if (! $winnerInherited && $loserInherited) { + $this->assertSame(HistoryEventType::CancellationScopeRequestConflicted->value, $loser['type']); + $this->assertSameJsonObject($accepted, $loser['payload']['accepted_cancellation']); + $this->assertSame( + $parentContext['root_context']['root_request_id'], + $loser['payload']['incoming_cancellation']['root_context']['root_request_id'] + ); + $this->assertSame('cancellation_root_conflict', $loser['payload']['reason']); + } else { + $this->assertSame($winner['id'], $loser['id']); + $this->assertSameJsonObject($winner['payload'], $loser['payload']); + } + $this->assertSame( + 2, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeRequested)->count() + ); + $this->assertSameJsonObject( + $parentContext, + CancellationScopeRequests::context($run->fresh(), $parent)->toArray() + ); + $this->assertRunCancellationUntouched($run); + } + + public static function requestRaces(): iterable + { + yield 'direct duplicate 20 versus 300 seconds' => [false, false]; + yield 'direct duplicate retains inherited root' => [true, false]; + yield 'ancestor conflicts with direct winner' => [false, true]; + } + + /** + * @return array{pid: int, socket: resource} + */ + private function forkScopeRequest(string $runId, string $scope, ?string $parent, bool $holdCommit): array + { + $sockets = stream_socket_pair(STREAM_PF_UNIX, STREAM_SOCK_STREAM, STREAM_IPPROTO_IP); + $this->assertIsArray($sockets); + $pid = pcntl_fork(); + $this->assertNotSame(-1, $pid); + if ($pid === 0) { + fclose($sockets[0]); + stream_set_timeout($sockets[1], 10); + $ok = false; + try { + DB::reconnect(); + $query = DB::connection()->getDriverName() === 'mysql' + ? 'SELECT CONNECTION_ID() AS connection_id' : 'SELECT pg_backend_pid() AS connection_id'; + fwrite($sockets[1], json_encode([ + 'connection_id' => (int) DB::selectOne($query)->connection_id, + ], JSON_THROW_ON_ERROR) . PHP_EOL); + if (fgets($sockets[1]) !== "go\n") { + throw new RuntimeException('Scope cancellation actor was not released.'); + } + if ($holdCommit) { + DB::beginTransaction(); + } + $event = CancellationScopeRequests::request( + WorkflowRun::query()->findOrFail($runId), + $scope, + '1.20', + $holdCommit ? 20 : 300, + $holdCommit ? 'winner' : 'duplicate', + parentScopeId: $parent, + ); + $result = [ + 'id' => $event->id, + 'type' => $event->event_type->value, + 'payload' => $event->payload, + ]; + if ($holdCommit) { + fwrite($sockets[1], json_encode($result, JSON_THROW_ON_ERROR) . PHP_EOL); + if (fgets($sockets[1]) !== "commit\n") { + throw new RuntimeException('Scope cancellation commit was not released.'); + } + DB::commit(); + } + fwrite($sockets[1], json_encode([ + 'result' => $result, + ], JSON_THROW_ON_ERROR) . PHP_EOL); + $ok = true; + } catch (Throwable $error) { + fwrite($sockets[1], json_encode([ + 'error' => $error::class . ': ' . $error->getMessage(), + ], JSON_THROW_ON_ERROR) . PHP_EOL); + } finally { + DB::disconnect(); + fclose($sockets[1]); + } + exit($ok ? 0 : 1); + } + fclose($sockets[1]); + stream_set_timeout($sockets[0], 10); + return [ + 'pid' => $pid, + 'socket' => $sockets[0], + ]; + } + + /** @param array{pid: int, socket: resource} $actor + * @return array + */ + private function readActor(array $actor): array + { + $line = fgets($actor['socket']); + $this->assertIsString($line, 'Scope cancellation actor did not respond.'); + $result = json_decode($line, true, flags: JSON_THROW_ON_ERROR); + $this->assertIsArray($result); + $this->assertArrayNotHasKey('error', $result, $result['error'] ?? ''); + return $result; + } + + /** @param array{pid: int, socket: resource} $actor + * @return array + */ + private function finishActor(array $actor): array + { + $message = $this->readActor($actor); + pcntl_waitpid($actor['pid'], $status); + $this->assertTrue(pcntl_wifexited($status)); + $this->assertSame(0, pcntl_wexitstatus($status)); + fclose($actor['socket']); + return $message['result']; + } + + /** + * @return array{WorkflowStub, WorkflowRun, string, string} + */ + private function scopeTree(string $name, bool $shield = false): array + { + $workflow = WorkflowStub::make(TestSignalWorkflow::class, 'scope-request-' . $name); + $workflow->start(); + $run = $workflow->run() + ->fresh(); + $task = $run->tasks() + ->where('task_type', TaskType::Workflow)->sole(); + $task->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'scope-request-fixture', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addMinutes(5), + ])->save(); + $parent = CancellationScopeHistory::open($run, $task, 1, '1.20')->payload['scope_id']; + $child = CancellationScopeHistory::open($run, $task, 2, '1.20', $parent, $shield)->payload['scope_id']; + return [$workflow, $run, $parent, $child]; + } + + private function assertRunCancellationUntouched(WorkflowRun $run): void + { + $run = $run->fresh(); + $this->assertFalse($run->status->isTerminal()); + $this->assertNull($run->cancellation_request_command_id); + $this->assertNull($run->cancellation_requested_at); + $this->assertNull($run->cancellation_deadline_at); + $this->assertSame(0, $run->commands()->where('command_type', 'request_cancellation')->count()); + } +} diff --git a/tests/Feature/V2/V2ChildCancellationPolicyTest.php b/tests/Feature/V2/V2ChildCancellationPolicyTest.php new file mode 100644 index 000000000..39c52c4de --- /dev/null +++ b/tests/Feature/V2/V2ChildCancellationPolicyTest.php @@ -0,0 +1,304 @@ + 'database', + ]); + Queue::fake(); + Carbon::setTestNow('2026-10-01T00:00:00Z'); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + public function testWaitDefersParentDeliveryUntilChildCleanupHasCanonicalTerminalHistory(): void + { + [$parent, $child] = $this->start(CancellationPolicy::WaitCancellationCompleted); + $root = $parent->requestCancellation('maintenance', 30) + ->cancellationContext(); + $this->runTask($parent, TaskType::Workflow); + + $this->assertSame(RunStatus::Waiting, $parent->run()->fresh()->status); + $this->assertSame([], $parent->refresh()->memo()); + $this->assertSame(0, $this->eventCount($parent, HistoryEventType::CooperativeCancellationDelivered)); + $context = CooperativeCancellationDelivery::context($child->run()->fresh()); + $this->assertNotNull($context); + $this->assertSame($root->rootRequestId, $context->rootRequestId); + $this->assertNotSame($root->requestId, $context->requestId); + $this->assertSame($root->deadline()->toISOString(), $context->deadline()->toISOString()); + $this->assertFalse($child->run()->fresh()->status->isTerminal()); + + $this->runTask($child, TaskType::Workflow); + $this->assertSame([], $child->refresh()->memo()); + $this->assertSame(1, $this->eventCount($child, HistoryEventType::CooperativeCancellationDelivered)); + $duplicate = $parent->requestCancellation('different reason', 90); + $this->assertSame($root->toArray(), $duplicate->cancellationContext()->toArray()); + $this->finishChild($child); + $this->runTask($parent, TaskType::Workflow); + + $this->assertTrue($parent->refresh()->cancelled()); + $this->assertSame([ + 'parent_cleanup' => 'complete', + ], $parent->memo()); + $this->assertSame(1, $this->eventCount($parent, HistoryEventType::ChildCancellationRequested)); + $this->assertSame(1, $this->eventCount($parent, HistoryEventType::ChildCancellationResolved)); + $this->assertSame(1, $this->eventCount($parent, HistoryEventType::CooperativeCancellationDelivered)); + $ack = $parent->run() + ->historyEvents() + ->where('event_type', HistoryEventType::ChildCancellationResolved)->sole(); + $terminal = $child->run() + ->historyEvents() + ->where('event_type', HistoryEventType::WorkflowCancelled)->sole(); + $this->assertSame($terminal->id, $ack->payload['child_terminal_history_event_id']); + $this->assertSame($root->rootRequestId, $ack->payload['root_request_id']); + $this->assertTrue($parent->run()->fresh()->closed_at->lessThan($root->deadline())); + $points = collect(HistoryTimeline::forRun($parent->run()->fresh())) + ->whereIn( + 'type', + [ + HistoryEventType::ChildCancellationRequested->value, + HistoryEventType::ChildCancellationResolved->value, + ] + ); + $this->assertCount(2, $points); + foreach ($points as $point) { + $this->assertSame('child', $point['kind']); + $this->assertSame($child->runId(), $point['child_workflow_run_id']); + $this->assertSame($root->rootRequestId, $point['child']['cancellation']['root_request_id']); + $this->assertSame( + CancellationPolicy::WaitCancellationCompleted->value, + $point['child']['cancellation']['policy'] + ); + } + } + + public function testTryCancelRequestsCooperationButDoesNotWaitForChildCleanup(): void + { + [$parent, $child] = $this->start(CancellationPolicy::TryCancel); + $root = $parent->requestCancellation('maintenance', 30) + ->cancellationContext(); + $this->runTask($parent, TaskType::Workflow); + + $this->assertTrue($parent->refresh()->cancelled()); + $this->assertSame([ + 'parent_cleanup' => 'complete', + ], $parent->memo()); + $this->assertFalse($child->run()->fresh()->status->isTerminal()); + $this->assertSame( + $root->rootRequestId, + CooperativeCancellationDelivery::context($child->run()->fresh())->rootRequestId + ); + $this->assertSame(0, $this->eventCount($parent, HistoryEventType::ChildCancellationResolved)); + $this->runTask($child, TaskType::Workflow); + $this->finishChild($child); + } + + public function testAbandonLeavesChildUntouched(): void + { + [$parent, $child] = $this->start(CancellationPolicy::Abandon); + $parent->requestCancellation('maintenance', 30); + $this->runTask($parent, TaskType::Workflow); + + $this->assertTrue($parent->refresh()->cancelled()); + $this->assertNull($child->run()->fresh()->cancellation_request_command_id); + $this->assertSame(0, $this->eventCount($parent, HistoryEventType::ChildCancellationRequested)); + $this->assertSame(RunStatus::Waiting, $child->run()->fresh()->status); + } + + public function testMixedParallelWaitFencesActivityButStillWaitsForChildCleanup(): void + { + [$parent, $child] = $this->start(CancellationPolicy::WaitCancellationCompleted, true); + $parent->requestCancellation('maintenance', 30); + $this->runTask($parent, TaskType::Workflow); + + $this->assertSame(ActivityStatus::Cancelled, $parent->run()->activityExecutions()->sole()->status); + $this->assertSame([], $parent->refresh()->memo()); + $this->assertSame(0, $this->eventCount($parent, HistoryEventType::CooperativeCancellationDelivered)); + $this->runTask($child, TaskType::Workflow); + $this->finishChild($child); + $this->runTask($parent, TaskType::Workflow); + $this->assertTrue($parent->refresh()->cancelled()); + $this->assertSame([ + 'parent_cleanup' => 'complete', + ], $parent->memo()); + } + + public function testHistoricalMissingPolicyRemainsAbandonOnColdExecution(): void + { + [$parent, $child] = $this->start(CancellationPolicy::WaitCancellationCompleted); + $event = $parent->run() + ->historyEvents() + ->where('event_type', HistoryEventType::ChildWorkflowScheduled)->sole(); + $payload = $event->payload; + unset($payload['cancellation_policy']); + $event->forceFill([ + 'payload' => $payload, + ])->save(); + $parent->requestCancellation('maintenance', 30); + $this->runTask($parent, TaskType::Workflow); + + $this->assertTrue($parent->refresh()->cancelled()); + $this->assertNull($child->run()->fresh()->cancellation_request_command_id); + } + + public function testSelectedChildHandleWaitsAtItsOriginalOperationRange(): void + { + [$parent, $child] = $this->start(CancellationPolicy::WaitCancellationCompleted, selection: true); + $parent->requestCancellation('maintenance', 30); + $this->runTask($parent, TaskType::Workflow); + $this->assertSame([], $parent->refresh()->memo()); + $this->assertSame(0, $this->eventCount($parent, HistoryEventType::CooperativeCancellationDelivered)); + $this->runTask($child, TaskType::Workflow); + $this->finishChild($child); + $this->runTask($parent, TaskType::Workflow); + + $this->assertTrue($parent->refresh()->cancelled()); + $delivery = $parent->run() + ->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered)->sole(); + $this->assertSame('selection_handle', $delivery->payload['call_kind']); + $this->assertSame(1, $delivery->payload['operation_sequence']); + $this->assertSame([ + 'parent_cleanup' => 'complete', + ], $parent->memo()); + } + + public function testIndependentChildRequestKeepsItsRootAndReportsPropagationConflict(): void + { + [$parent, $child] = $this->start(CancellationPolicy::WaitCancellationCompleted); + $childRoot = $child->requestCancellation('independent child request', 45) + ->cancellationContext(); + $parentRoot = $parent->requestCancellation('parent request', 30) + ->cancellationContext(); + $this->runTask($parent, TaskType::Workflow); + + $this->assertSame([], $parent->refresh()->memo()); + $this->assertSame( + $childRoot->toArray(), + CooperativeCancellationDelivery::context($child->run()->fresh())->toArray() + ); + $event = $parent->run() + ->historyEvents() + ->where('event_type', HistoryEventType::ChildCancellationRequested)->sole(); + $this->assertSame('rejected', $event->payload['request_outcome']); + $this->assertSame('cancellation_root_conflict', $event->payload['rejection_reason']); + $this->assertSame($parentRoot->rootRequestId, $event->payload['root_request_id']); + $this->assertSame($childRoot->rootRequestId, $event->payload['child_root_request_id']); + $this->runTask($child, TaskType::Workflow); + $this->finishChild($child); + $this->runTask($parent, TaskType::Workflow); + $this->assertTrue($parent->refresh()->cancelled()); + $this->assertSame( + $parentRoot->deadline() + ->toISOString(), + $parent->run() + ->fresh() + ->cancellation_deadline_at->toISOString() + ); + } + + public function testWaitingCannotExtendTheOriginalDeadlineWhenChildCleanupDoesNotFinish(): void + { + [$parent, $child] = $this->start(CancellationPolicy::WaitCancellationCompleted); + $root = $parent->requestCancellation('maintenance', 30) + ->cancellationContext(); + $this->runTask($parent, TaskType::Workflow); + Carbon::setTestNow($root->deadline()->addSecond()); + TaskWatchdog::runPass(respectThrottle: false, runIds: [$parent->runId(), $child->runId()]); + + foreach ([$parent, $child] as $workflow) { + $this->assertTrue($workflow->refresh()->cancelled()); + $this->assertSame([], $workflow->memo()); + $this->assertSame( + $root->deadline() + ->toISOString(), + $workflow->run() + ->fresh() + ->cancellation_deadline_at->toISOString() + ); + } + $this->assertSame(0, $this->eventCount($parent, HistoryEventType::CooperativeCancellationDelivered)); + } + + /** + * @return array{WorkflowStub, WorkflowStub} + */ + private function start(CancellationPolicy $policy, bool $parallel = false, bool $selection = false): array + { + $parent = WorkflowStub::make(TestParentChildPolicyWorkflow::class); + $parent->start($policy->value, $parallel, $selection); + $this->runTask($parent, TaskType::Workflow); + $link = WorkflowLink::query()->where('parent_workflow_run_id', $parent->runId())->sole(); + $child = WorkflowStub::loadRun($link->child_workflow_run_id); + $this->runTask($child, TaskType::Workflow); + + return [$parent, $child]; + } + + private function finishChild(WorkflowStub $child): void + { + Carbon::setTestNow(now()->addSeconds(2)); + $this->runTask($child, TaskType::Timer); + $this->runTask($child, TaskType::Workflow); + $this->assertTrue($child->refresh()->cancelled()); + $this->assertSame([ + 'child_cleanup' => 'complete', + ], $child->memo()); + } + + private function runTask(WorkflowStub $workflow, TaskType $type): void + { + $task = WorkflowTask::query()->where('workflow_run_id', $workflow->runId()) + ->where('task_type', $type->value) + ->where('status', TaskStatus::Ready->value) + ->orderBy('created_at') + ->firstOrFail(); + $this->assertFalse($task->available_at?->isFuture() ?? false); + $job = match ($type) { + TaskType::Workflow => new RunWorkflowTask($task->id), + TaskType::Timer => new RunTimerTask($task->id), + TaskType::Activity => new RunActivityTask($task->id), + default => throw new \LogicException('Unexpected test task type.'), + }; + $this->app->call([$job, 'handle']); + } + + private function eventCount(WorkflowStub $workflow, HistoryEventType $type): int + { + return WorkflowHistoryEvent::query()->where('workflow_run_id', $workflow->runId()) + ->where('event_type', $type->value) + ->count(); + } +} diff --git a/tests/Feature/V2/V2CooperativeParentCloseTest.php b/tests/Feature/V2/V2CooperativeParentCloseTest.php new file mode 100644 index 000000000..e3a11a1be --- /dev/null +++ b/tests/Feature/V2/V2CooperativeParentCloseTest.php @@ -0,0 +1,337 @@ + 'database', + ]); + Queue::fake(); + Carbon::setTestNow('2026-10-01T00:00:00Z'); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + public function testParentCloseInheritsOriginalRequestAndKeepsChildOpenDuringRealCleanup(): void + { + [$parent, $child] = $this->startWaitingParent(); + $root = $parent->requestCancellation('maintenance', 30) + ->cancellationContext(); + $this->runTask($parent, TaskType::Workflow); + + $this->assertTrue($parent->refresh()->cancelled()); + $context = CooperativeCancellationDelivery::context($child->run()->fresh()); + $this->assertNotNull($context); + $this->assertSame($root->rootRequestId, $context->rootRequestId); + $this->assertSame($root->requestId, $context->parentRequestId); + $this->assertSame($root->deadline()->toISOString(), $context->deadline()->toISOString()); + $this->assertSame('maintenance', $context->reason); + $this->assertSame($root->requester, $context->requester); + $this->assertCount(2, $context->lineage); + $this->assertFalse($child->run()->fresh()->status->isTerminal()); + $this->assertSame(ChildCallStatus::Started, $this->childCall($parent)->status); + $this->assertSame(0, $this->eventCount($parent, HistoryEventType::ParentCloseCancellationRequested)); + $this->assertSame(1, $this->eventCount($parent, HistoryEventType::ParentClosePolicyApplied)); + + $entry = collect(RunLineageView::continuedWorkflowsForRun($parent->run()->fresh())) + ->firstWhere('child_workflow_run_id', $child->runId()); + $this->assertNotNull($entry); + $this->assertSameJsonObject($context->toArray(), $entry['parent_close_cancellation']); + $point = collect(HistoryTimeline::forRun($parent->run()->fresh())) + ->firstWhere('type', HistoryEventType::ParentClosePolicyApplied->value); + $this->assertSameJsonObject($context->toArray(), $point['parent_close']['cancellation']); + + $this->runTask($child, TaskType::Workflow); + $this->assertSame([], $child->refresh()->memo()); + $this->assertFalse($child->run()->fresh()->status->isTerminal()); + $this->finishCleanup($child); + $this->assertTrue($child->run()->fresh()->closed_at->lessThan($root->deadline())); + } + + public function testNormallyCompletedParentCreatesOneCanonicalOriginForBothChildren(): void + { + $parent = WorkflowStub::make(TestParentCloseCooperativeWorkflow::class, 'normal-close'); + $parent->start(); + $this->runTask($parent, TaskType::Workflow); + $children = $this->children($parent); + $this->assertCount(2, $children); + foreach ($children as $child) { + $this->runTask($child, TaskType::Workflow); + } + Carbon::setTestNow(now()->addSecond()); + $this->runTask($parent, TaskType::Timer); + $this->runTask($parent, TaskType::Workflow); + + $this->assertTrue($parent->refresh()->completed()); + $this->assertNull($parent->run()->fresh()->cancellation_request_command_id); + $root = ParentCloseCancellation::context($parent->run()->fresh()); + $this->assertNotNull($root); + $this->assertSame($parent->runId(), $root->rootWorkflowRunId); + $this->assertSame('parent_close_policy', $root->source); + $this->assertSame(1, $this->eventCount($parent, HistoryEventType::ParentCloseCancellationRequested)); + $closed = $parent->run() + ->historyEvents() + ->where('event_type', HistoryEventType::WorkflowCompleted)->sole(); + $this->assertTrue($root->requestedAt()->equalTo($closed->recorded_at)); + $this->assertSame(600.0, (float) $root->requestedAt()->diffInSeconds($root->deadline())); + $localIds = []; + foreach ($children as $child) { + $context = CooperativeCancellationDelivery::context($child->run()->fresh()); + $this->assertSame($root->rootRequestId, $context->rootRequestId); + $this->assertSame($root->requestId, $context->parentRequestId); + $this->assertSame($root->deadline()->toISOString(), $context->deadline()->toISOString()); + $localIds[] = $context->requestId; + } + $this->assertCount(2, array_unique($localIds)); + Carbon::setTestNow(now()->addSeconds(20)); + $this->assertSame([], ParentClosePolicyEnforcer::enforce($parent->run()->fresh())); + $this->assertSame($root->toArray(), ParentCloseCancellation::context($parent->run()->fresh())->toArray()); + $this->assertSame(2, $this->eventCount($parent, HistoryEventType::ParentClosePolicyApplied)); + foreach ($children as $index => $child) { + $duplicate = $child->requestCancellation('later caller', 90) + ->cancellationContext(); + $this->assertSame($localIds[$index], $duplicate->requestId); + $this->assertSame($root->deadline()->toISOString(), $duplicate->deadline()->toISOString()); + $this->runTask($child, TaskType::Workflow); + $this->finishCleanup($child); + } + $this->assertTrue($parent->refresh()->completed()); + } + + public function testTerminalParentCloseAlsoRequestsCooperativeChildCleanup(): void + { + [$parent, $child] = $this->startWaitingParent(); + $this->assertTrue($parent->attemptTerminate('operator emergency')->accepted()); + $this->assertTrue($parent->refresh()->terminated()); + $root = ParentCloseCancellation::context($parent->run()->fresh()); + $this->assertNotNull($root); + $this->assertSame( + $root->rootRequestId, + CooperativeCancellationDelivery::context($child->run()->fresh())->rootRequestId + ); + $this->assertFalse($child->run()->fresh()->status->isTerminal()); + $this->runTask($child, TaskType::Workflow); + $this->finishCleanup($child); + } + + public function testExpiredOriginalBudgetIsInheritedWithoutAnotherTenMinutes(): void + { + [$parent, $child] = $this->startWaitingParent(); + $root = $parent->requestCancellation('maintenance', 30) + ->cancellationContext(); + Carbon::setTestNow(now()->addSeconds(31)); + $parent->attemptTerminate('cleanup worker absent'); + + $childContext = CooperativeCancellationDelivery::context($child->run()->fresh()); + $this->assertSame($root->deadline()->toISOString(), $childContext->deadline()->toISOString()); + $this->assertTrue($childContext->deadline()->isPast()); + $this->assertSame(0, $this->eventCount($parent, HistoryEventType::ParentCloseCancellationRequested)); + TaskWatchdog::runPass(respectThrottle: false, runIds: [$child->runId()]); + $this->assertTrue($child->refresh()->cancelled()); + $this->assertSame([], $child->memo()); + } + + public function testDifferentChildRootIsDiagnosedWithoutClaimingPolicyApplied(): void + { + [$parent, $child] = $this->startWaitingParent(); + $existing = $child->requestCancellation('independent request', 90) + ->cancellationContext(); + $incoming = $parent->requestCancellation('parent request', 30) + ->cancellationContext(); + $this->runTask($parent, TaskType::Workflow); + + $this->assertSame(0, $this->eventCount($parent, HistoryEventType::ParentClosePolicyApplied)); + $failed = $parent->run() + ->historyEvents() + ->where('event_type', HistoryEventType::ParentClosePolicyFailed)->sole(); + $this->assertSame('cancellation_root_conflict', $failed->payload['error']); + $this->assertSame( + $existing->rootRequestId, + $failed->payload['request_diagnostics']['existing_root_request_id'] + ); + $this->assertSame( + $incoming->rootRequestId, + $failed->payload['request_diagnostics']['incoming_root_request_id'] + ); + $this->assertSame( + $existing->toArray(), + CooperativeCancellationDelivery::context($child->run()->fresh())->toArray() + ); + $this->assertFalse($child->run()->fresh()->status->isTerminal()); + $this->runTask($child, TaskType::Workflow); + $this->finishCleanup($child); + } + + public function testMutableTerminalProjectionCannotCreateANewCancellationOrigin(): void + { + [$parent, $child] = $this->startWaitingParent(); + $parent->run() + ->forceFill([ + 'status' => RunStatus::Completed, + ])->save(); + $this->assertSame([], ParentClosePolicyEnforcer::enforce($parent->run()->fresh())); + $this->assertNull($child->run()->fresh()->cancellation_request_command_id); + $failed = $parent->run() + ->historyEvents() + ->where('event_type', HistoryEventType::ParentClosePolicyFailed)->sole(); + $this->assertSame('cancellation_parent_closure_unavailable', $failed->payload['error']); + $this->assertSame(0, $this->eventCount($parent, HistoryEventType::ParentCloseCancellationRequested)); + } + + public function testLegacyRequestCancelStillClosesChildImmediately(): void + { + [$parent, $child] = $this->startWaitingParent('request_cancel'); + $parent->attemptTerminate('operator emergency'); + $this->assertTrue($child->refresh()->cancelled()); + $this->assertSame([], $child->memo()); + $this->assertNull($child->run()->fresh()->cancellation_request_command_id); + $this->assertSame(0, $this->eventCount($parent, HistoryEventType::ParentCloseCancellationRequested)); + $this->assertSame(ChildCallStatus::Cancelled, $this->childCall($parent)->status); + } + + public function testDelayedEnforcementUsesRecordedClosureInsteadOfRepairTime(): void + { + [$parent, $child] = $this->startWaitingParent(); + $run = $parent->run() + ->fresh(); + // Reconstruct a closed history with no policy receipt, as seen by repair. + $closed = WorkflowHistoryEvent::record($run, HistoryEventType::WorkflowCompleted); + $run->forceFill([ + 'status' => RunStatus::Completed, + 'closed_at' => now(), + ])->save(); + Carbon::setTestNow(now()->addSeconds(601)); + $this->assertSame([$child->id()], ParentClosePolicyEnforcer::enforce($run->fresh())); + + $root = ParentCloseCancellation::context($run->fresh()); + $this->assertTrue($root->requestedAt()->equalTo($closed->recorded_at)); + $this->assertTrue($root->deadline()->isPast()); + $inherited = CooperativeCancellationDelivery::context($child->run()->fresh()); + $this->assertSame($root->deadline()->toISOString(), $inherited->deadline()->toISOString()); + TaskWatchdog::runPass(respectThrottle: false, runIds: [$child->runId()]); + $this->assertTrue($child->refresh()->cancelled()); + $this->assertSame([], $child->memo()); + } + + public function testLegacyMissingContextIsDiagnosedWithoutReplacingOriginalBudget(): void + { + [$parent, $child] = $this->startWaitingParent(); + $request = $parent->requestCancellation('legacy caller', 30); + $command = $parent->run() + ->commands() + ->findOrFail($request->commandId()); + $command->forceFill([ + 'payload' => \Workflow\Serializers\Serializer::serialize([ + 'reason' => 'legacy caller', + 'cleanup_deadline_at' => $request->cancellationContext() + ->deadline() + ->toISOString(), + ]), + ])->save(); + $parent->attemptTerminate('operator emergency'); + + $failed = $parent->run() + ->historyEvents() + ->where('event_type', HistoryEventType::ParentClosePolicyFailed)->sole(); + $this->assertSame('cancellation_parent_context_unavailable', $failed->payload['error']); + $this->assertNull($child->run()->fresh()->cancellation_request_command_id); + $this->assertSame($request->commandId(), $parent->run()->fresh()->cancellation_request_command_id); + $this->assertSame(0, $this->eventCount($parent, HistoryEventType::ParentCloseCancellationRequested)); + } + + /** + * @return array{WorkflowStub, WorkflowStub} + */ + private function startWaitingParent(string $parentClosePolicy = 'request_cancellation'): array + { + $parent = WorkflowStub::make(TestParentChildPolicyWorkflow::class, 'parent-close'); + $parent->start('abandon', false, false, $parentClosePolicy); + $this->runTask($parent, TaskType::Workflow); + $child = $this->children($parent)[0]; + $this->runTask($child, TaskType::Workflow); + + return [$parent, $child]; + } + + /** + * @return list + */ + private function children(WorkflowStub $parent): array + { + return WorkflowLink::query()->where('parent_workflow_run_id', $parent->runId()) + ->where('link_type', 'child_workflow') + ->orderBy('sequence') + ->get() + ->map(static fn (WorkflowLink $link): WorkflowStub => WorkflowStub::load($link->child_workflow_instance_id)) + ->all(); + } + + private function childCall(WorkflowStub $parent): WorkflowChildCall + { + return WorkflowChildCall::query()->where('parent_workflow_run_id', $parent->runId())->sole(); + } + + private function runTask(WorkflowStub $workflow, TaskType $type): void + { + $task = WorkflowTask::query()->where('workflow_run_id', $workflow->runId()) + ->where('task_type', $type->value) + ->where('status', TaskStatus::Ready->value) + ->orderBy('created_at') + ->firstOrFail(); + $this->assertFalse($task->available_at?->isFuture() ?? false); + $job = $type === TaskType::Workflow ? new RunWorkflowTask($task->id) : new RunTimerTask($task->id); + $this->app->call([$job, 'handle']); + } + + private function finishCleanup(WorkflowStub $child): void + { + Carbon::setTestNow(now()->addSeconds(2)); + $this->runTask($child, TaskType::Timer); + $this->runTask($child, TaskType::Workflow); + $this->assertTrue($child->refresh()->cancelled()); + $this->assertSame([ + 'child_cleanup' => 'complete', + ], $child->memo()); + } + + private function eventCount(WorkflowStub $workflow, HistoryEventType $type): int + { + return WorkflowHistoryEvent::query()->where('workflow_run_id', $workflow->runId()) + ->where('event_type', $type->value) + ->count(); + } +} diff --git a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php index 1589b2415..4955ddc1f 100644 --- a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php +++ b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php @@ -14,6 +14,7 @@ use Workflow\V2\Contracts\HistoryProjectionRole; use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Enums\HistoryEventType; +use Workflow\V2\Enums\RunStatus; use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Enums\TaskType; use Workflow\V2\Exceptions\HistoryEventShapeMismatchException; @@ -29,8 +30,12 @@ use Workflow\V2\Models\WorkflowSearchAttribute; use Workflow\V2\Models\WorkflowTask; use Workflow\V2\Models\WorkflowTimelineEntry; +use Workflow\V2\Support\CancellationScopeDelivery; +use Workflow\V2\Support\CancellationScopeHistory; +use Workflow\V2\Support\CancellationScopeRequests; use Workflow\V2\Support\ConditionWaits; use Workflow\V2\Support\DefaultHistoryProjectionRole; +use Workflow\V2\Support\DefaultWorkflowTaskBridge; use Workflow\V2\Support\EmbeddedV2HistoryImport; use Workflow\V2\Support\HistoryExport; use Workflow\V2\Support\QueryStateReplayer; @@ -112,6 +117,10 @@ public function testFixturesExecuteThroughDeclaredReplayConsumers(): void $this->assertPortableCleanupDeadlineAfterColdReload($fixture); } + if (($fixture['id'] ?? null) === 'run-inherited-scoped-cleanup-cold-reload') { + $this->assertRunInheritedScopedCleanupAfterColdReload($fixture); + } + if ($fixture['id'] === 'cooperative-delivery-before-wait-sequence') { $run = $this->createRunFromFixture($fixture); $this->assertSame(2, WorkflowStepHistory::nextDurableCommandSequence($run->fresh())); @@ -1350,6 +1359,152 @@ private function assertPortableCleanupDeadlineAfterColdReload(array $fixture): v } } + /** + * The portable bridge resumes the frozen scope prefix. Embedded scope + * authoring remains unavailable, as the fixture's cold replay asserts. + * + * @param array $fixture + */ + private function assertRunInheritedScopedCleanupAfterColdReload(array $fixture): void + { + $this->clearWorkflowState(); + Carbon::setTestNow(Carbon::parse($fixture['history'][0]['recorded_at'])); + try { + $stub = WorkflowStub::make( + $fixture['workflow']['type'], + sprintf('regression-corpus-inherited-cleanup-%d', ++$this->workflowNumber), + ); + $stub->start(...$fixture['workflow']['arguments']); + $run = WorkflowRun::query()->findOrFail($stub->runId()); + foreach (array_slice($fixture['history'], 1) as $event) { + // Only the synthetic run identity changes on import. + WorkflowHistoryEvent::record($run, HistoryEventType::CancellationScopeOpened, [ + ...$event['payload'], + 'workflow_run_id' => $run->id, + ]); + } + $scope = $fixture['history'][1]['payload']['scope_id']; + $child = $fixture['history'][2]['payload']['scope_id']; + DB::purge(); + DB::reconnect(); + $run = $run->fresh(); + $this->assertSame([$scope, $child], array_keys(CancellationScopeHistory::forRun($run))); + $task = $run->tasks() + ->where('task_type', TaskType::Workflow)->sole(); + $task->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'cold-scoped-cleanup', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addSeconds(60), + ])->save(); + $this->assertTrue($stub->requestCancellation('clean up the run', 30)->accepted()); + $run->refresh(); + $originalRequest = $run->cancellation_request_command_id; + $originalDeadline = $run->cancellation_deadline_at->toISOString(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 300); + CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + 3, + 'local_activity', + '1.20' + ); + $delivery = CancellationScopeDelivery::record( + $run, + $task, + $scope, + $request->payload['request_id'], + 3, + 'local_activity', + '1.20' + ); + $descriptor = [ + 'type' => 'record_local_activity', + 'activity_type' => 'tests.scoped-cleanup', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'cancellation_scope_id' => $child, + 'cancellation_cleanup' => [ + 'scope_id' => $scope, + 'request_id' => $request->payload['request_id'], + 'delivery_history_event_id' => $delivery->id, + ], + ]; + $bridge = $this->app->make(DefaultWorkflowTaskBridge::class); + $prepared = $bridge->prepareLocalActivity( + $task->id, + 'cold-scoped-cleanup', + 1, + 4, + 'inherited-cleanup', + $descriptor, + '1.20' + ); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $snapshot = $prepared['cancellation_cleanup']; + $this->assertSame($originalRequest, $snapshot['root_request_id']); + $this->assertSame($originalDeadline, $snapshot['cleanup_deadline_at']); + DB::purge(); + DB::reconnect(); + foreach ([8, 8, 8] as $seconds) { + Carbon::setTestNow(now()->addSeconds($seconds)); + $control = $bridge->controlLocalActivity( + $prepared['activity_attempt_id'], + 'cold-scoped-cleanup', + 1, + true, + '1.20' + ); + $this->assertTrue($control['active'], $control['reason'] ?? ''); + } + $this->assertSame($originalDeadline, $task->fresh()->lease_expires_at->toISOString()); + $completed = $bridge->recordLocalActivityOutcome( + $prepared['activity_attempt_id'], + 'cold-scoped-cleanup', + 1, + [ + 'outcome' => 'completed', + 'result' => Serializer::serializeWithCodec('avro', 'cleaned'), + 'payload_codec' => 'avro', + ], + '1.20' + ); + $this->assertTrue($completed['recorded'], $completed['reason'] ?? ''); + $this->assertTrue($stub->requestCancellation('duplicate with longer budget', 300)->accepted()); + $run->refresh(); + $this->assertSame($originalRequest, $run->cancellation_request_command_id); + $this->assertSame($originalDeadline, $run->cancellation_deadline_at->toISOString()); + $this->assertTrue($bridge->deliverCancellation($task->id, $originalRequest, 5, 'timer')['delivered']); + $this->assertTrue($bridge->complete($task->id, [[ + 'type' => 'complete_workflow', + 'output' => Serializer::serializeWithCodec('avro', 'cleaned'), + 'payload_codec' => 'avro', + ]])['completed']); + $this->assertSame(RunStatus::Cancelled, $run->refresh()->status); + $this->assertTrue(now()->lt($run->cancellation_deadline_at)); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeDelivered)->count() + ); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered)->count() + ); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityHeartbeatRecorded)->count() + ); + } finally { + Carbon::setTestNow(); + } + } + private function bindNoOpHistoryProjection(): void { $this->app->instance(HistoryProjectionRole::class, new class() implements HistoryProjectionRole { diff --git a/tests/Feature/V2/V2PortableCancellationDeliveryTest.php b/tests/Feature/V2/V2PortableCancellationDeliveryTest.php index 527bcd0b9..1ac8844ec 100644 --- a/tests/Feature/V2/V2PortableCancellationDeliveryTest.php +++ b/tests/Feature/V2/V2PortableCancellationDeliveryTest.php @@ -6,6 +6,7 @@ use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Queue; +use LogicException; use PHPUnit\Framework\Attributes\DataProvider; use Tests\Fixtures\V2\TestGreetingActivity; use Tests\Fixtures\V2\TestGreetingWorkflow; @@ -16,18 +17,34 @@ use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Enums\ActivityAttemptStatus; use Workflow\V2\Enums\ActivityStatus; +use Workflow\V2\Enums\CancellationPolicy; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\RunStatus; use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Enums\TaskType; use Workflow\V2\Enums\TimerStatus; +use Workflow\V2\Models\ActivityExecution; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowInstance; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; +use Workflow\V2\Support\ActivityAbandonment; +use Workflow\V2\Support\ActivityCancellation; +use Workflow\V2\Support\ActivityCancellationAcknowledgement; +use Workflow\V2\Support\ActivitySnapshot; +use Workflow\V2\Support\ActivityTimeoutEnforcer; use Workflow\V2\Support\DefaultWorkflowTaskBridge; +use Workflow\V2\Support\HistoryTimeline; +use Workflow\V2\Support\LocalActivityCall; +use Workflow\V2\Support\LocalActivityExecutor; +use Workflow\V2\Support\LocalActivityRuntime; use Workflow\V2\Support\ParallelChildGroup; use Workflow\V2\Support\RunSummaryProjector; +use Workflow\V2\Support\TaskRepair; +use Workflow\V2\Support\WorkflowCommandNormalizer; +use Workflow\V2\Support\WorkflowRunRetentionCleanup; +use Workflow\V2\TaskWatchdog; +use Workflow\V2\Testing\ActivityFakeContext; use Workflow\V2\WorkflowStub; final class V2PortableCancellationDeliveryTest extends TestCase @@ -50,6 +67,624 @@ public function testOptionalCooperativeBridgeUsesTheExistingBinding(): void $this->assertSame($this->bridge, $this->app->make(CooperativeWorkflowTaskBridge::class)); } + #[DataProvider('portableRemoteActivityPolicies')] + public function testPortableRemoteActivityPolicySurvivesAdmissionAndControlsCancellation(?string $policy): void + { + [$run, $initial] = $this->newRun(); + $command = [ + 'type' => 'schedule_activity', + 'activity_type' => 'tests.portable-remote', + 'arguments' => Serializer::serialize(['Taylor']), + 'schedule_to_close_timeout' => 180, + ]; + if ($policy !== null) { + $command['cancellation_policy'] = $policy; + } + $commands = WorkflowCommandNormalizer::normalize([$command], '1.20'); + $this->assertTrue($this->bridge->complete($initial->id, $commands)['completed']); + $execution = $run->activityExecutions() + ->sole(); + $scheduled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityScheduled)->sole(); + $this->assertSame($policy, $scheduled->payload['activity']['cancellation_policy'] ?? null); + $this->assertSame($policy, $execution->activity_options['cancellation_policy'] ?? null); + $originalActivityDeadline = $execution->schedule_to_close_deadline_at->toISOString(); + if ($policy === 'abandon') { + $this->assertSame( + $originalActivityDeadline, + $scheduled->payload['activity']['schedule_to_close_deadline_at'] + ); + } + $activityTask = $run->tasks() + ->where('task_type', TaskType::Activity)->sole(); + $activities = $this->app->make(ActivityTaskBridge::class); + $this->assertTrue($activities->claimStatus($activityTask->id, 'remote-owner')['claimed']); + $attempt = $execution->attempts() + ->sole(); + $this->request($run); + $originalCleanupDeadline = $run->cancellation_deadline_at->toISOString(); + $resume = $this->leaseReadyTask($run); + $delivery = $this->deliver($run, $resume); + if ($policy === 'wait_cancellation_completed') { + $this->assertFalse($delivery['delivered']); + $this->assertTrue($delivery['claim_released']); + $this->assertSame('cancellation_waiting_for_activity', $delivery['reason']); + $this->assertSame(0, $this->deliveryCount($run)); + $receipt = ActivityCancellationAcknowledgement::recordStopped( + $attempt->id, + 'remote-owner', + $run->cancellation_request_command_id, + ); + $this->assertTrue($receipt['acknowledged']); + $resume = $this->leaseReadyTask($run); + $delivery = $this->deliver($run, $resume); + } + $this->assertTrue($delivery['delivered']); + $this->assertSame(1, $this->deliveryCount($run)); + $this->assertSame($originalCleanupDeadline, $run->refresh()->cancellation_deadline_at->toISOString()); + $this->assertTrue($this->bridge->complete($resume->id, [[ + 'type' => 'complete_workflow', + ]])['completed']); + $this->assertSame(RunStatus::Cancelled, $run->refresh()->status); + $this->assertSame( + $originalActivityDeadline, + $execution->refresh() + ->schedule_to_close_deadline_at->toISOString() + ); + $completion = $activities->complete($attempt->id, 'late callback result'); + $this->assertSame($policy === 'abandon', $completion['recorded']); + $this->assertSame( + $policy === 'abandon' ? ActivityStatus::Completed : ActivityStatus::Cancelled, + $execution->refresh() +->status + ); + $this->assertSame(RunStatus::Cancelled, $run->refresh()->status); + $this->assertSame( + 0, + $run->tasks() + ->where('task_type', TaskType::Workflow)->where('status', TaskStatus::Ready)->count() + ); + } + + public static function portableRemoteActivityPolicies(): iterable + { + yield 'historical default' => [null]; + yield 'request and continue' => ['try_cancel']; + yield 'wait for original callback stop' => ['wait_cancellation_completed']; + yield 'bounded independent remote work' => ['abandon']; + } + + #[DataProvider('invalidPortableRemoteActivityPolicies')] + public function testPortableRemoteActivityPolicyRefusesInvalidBridgeInputBeforeScheduling(mixed $policy): void + { + [$run, $initial] = $this->newRun(); + $historyBefore = $run->historyEvents() + ->count(); + $outcome = $this->bridge->complete($initial->id, [[ + 'type' => 'schedule_activity', + 'activity_type' => 'tests.portable-remote', + 'cancellation_policy' => $policy, + ]]); + $this->assertFalse($outcome['completed']); + $this->assertSame(0, $run->activityExecutions()->count()); + $this->assertSame(0, $run->tasks()->where('task_type', TaskType::Activity)->count()); + $this->assertSame($historyBefore, $run->historyEvents()->count()); + $this->assertSame(TaskStatus::Leased, $initial->refresh()->status); + } + + public static function invalidPortableRemoteActivityPolicies(): iterable + { + yield 'unknown policy' => ['unknown']; + yield 'non-string policy' => [[]]; + yield 'unbounded abandon' => ['abandon']; + } + + public function testInternalActivityAbandonCompletesAfterParentClosureWithoutReopeningIt(): void + { + [$run, $initial] = $this->newRun(); + [$execution, $activityTask] = $this->scheduleInternalAbandonableActivity($run, $initial); + $activities = $this->app->make(ActivityTaskBridge::class); + $this->assertTrue($activities->claimStatus($activityTask->id, 'abandoned-owner')['claimed']); + $attempt = $execution->attempts() + ->sole(); + $activityDeadline = $execution->refresh() + ->schedule_to_close_deadline_at->toISOString(); + $this->closeCooperativeParent($run, $initial); + $closedAt = $run->closed_at->toISOString(); + $request = $run->cancellation_request_command_id; + $cleanupDeadline = $run->cancellation_deadline_at->toISOString(); + $this->assertSame(ActivityStatus::Running, $execution->refresh()->status); + $this->assertSame(TaskStatus::Leased, $activityTask->refresh()->status); + $this->assertSame('detached_activity_still_open', WorkflowRunRetentionCleanup::retentionHoldReason($run)); + $historyCount = $run->historyEvents() + ->count(); + try { + WorkflowRunRetentionCleanup::pruneRun($run); + $this->fail('An open detached activity must hold retention.'); + } catch (LogicException $exception) { + $this->assertStringContainsString('detached_activity_still_open', $exception->getMessage()); + } + $this->assertSame($historyCount, $run->historyEvents()->count()); + $this->assertNull($run->refresh()->details_pruned_at); + try { + Carbon::setTestNow($run->cancellation_deadline_at->copy()->addSecond()); + $control = $activities->heartbeat($attempt->id); + $this->assertTrue($control['can_continue']); + $this->assertFalse($control['cancel_requested']); + $outcome = $activities->complete($attempt->id, 'independent result'); + } finally { + Carbon::setTestNow(); + } + $this->assertTrue($outcome['recorded']); + $this->assertNull($outcome['next_task_id']); + $this->assertSame(ActivityStatus::Completed, $execution->refresh()->status); + $this->assertSame($activityDeadline, $execution->schedule_to_close_deadline_at->toISOString()); + $this->assertSame('independent result', Serializer::unserialize($execution->result)); + $this->assertSame(1, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCompleted)->count()); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCancelled)->count()); + $this->assertSame(RunStatus::Cancelled, $run->refresh()->status); + $this->assertSame($closedAt, $run->closed_at->toISOString()); + $this->assertSame($request, $run->cancellation_request_command_id); + $this->assertSame($cleanupDeadline, $run->cancellation_deadline_at->toISOString()); + $this->assertSame(0, $run->tasks()->where('task_type', TaskType::Workflow) + ->where('status', TaskStatus::Ready)->count()); + $this->assertNull(WorkflowRunRetentionCleanup::retentionHoldReason($run)); + $this->assertSame(1, WorkflowRunRetentionCleanup::pruneRun($run)['activity_executions_deleted']); + } + + public function testInternalActivityAbandonRetriesWithItsOriginalBudgetAndRefusesStalePublication(): void + { + [$run, $initial] = $this->newRun(); + [$execution, $activityTask] = $this->scheduleInternalAbandonableActivity($run, $initial, maxAttempts: 2); + $activities = $this->app->make(ActivityTaskBridge::class); + $this->assertTrue($activities->claimStatus($activityTask->id, 'first-owner')['claimed']); + $first = $execution->attempts() + ->sole(); + $deadline = $execution->refresh() + ->schedule_to_close_deadline_at->toISOString(); + $this->closeCooperativeParent($run, $initial); + $retry = $activities->fail($first->id, [ + 'class' => \RuntimeException::class, + 'message' => 'temporary failure', + ]); + $this->assertTrue($retry['recorded']); + $retryTask = WorkflowTask::query()->findOrFail($retry['next_task_id']); + $this->assertSame(TaskType::Activity, $retryTask->task_type); + $this->assertSame(ActivityStatus::Pending, $execution->refresh()->status); + $this->assertSame('detached_activity_still_open', WorkflowRunRetentionCleanup::retentionHoldReason($run)); + $this->assertTrue($activities->claimStatus($retryTask->id, 'replacement-owner')['claimed']); + $second = $execution->attempts() + ->where('attempt_number', 2) + ->sole(); + $this->assertSame('stale_attempt', $activities->complete($first->id, 'stale result')['reason']); + $this->assertTrue($activities->status($second->id)['can_continue']); + $outcome = $activities->complete($second->id, 'replacement result'); + $this->assertTrue($outcome['recorded']); + $this->assertNull($outcome['next_task_id']); + $this->assertSame($deadline, $execution->refresh()->schedule_to_close_deadline_at->toISOString()); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityRetryScheduled)->count() + ); + $this->assertSame(1, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCompleted)->count()); + $this->assertSame(RunStatus::Cancelled, $run->refresh()->status); + } + + public function testInternalActivityAbandonExpiresAtItsOriginalTotalDeadlineWithoutWakingParent(): void + { + [$run, $initial] = $this->newRun(); + [$execution, $activityTask] = $this->scheduleInternalAbandonableActivity($run, $initial, maxAttempts: 3); + $activities = $this->app->make(ActivityTaskBridge::class); + $this->assertTrue($activities->claimStatus($activityTask->id, 'deadline-owner')['claimed']); + $attempt = $execution->attempts() + ->sole(); + $this->closeCooperativeParent($run, $initial); + try { + Carbon::setTestNow($execution->refresh()->schedule_to_close_deadline_at); + $this->assertFalse($activities->status($attempt->id)['can_continue']); + $this->assertSame('activity_deadline_elapsed', $activities->complete($attempt->id, 'too late')['reason']); + $expired = ActivityTimeoutEnforcer::enforce($execution->id); + } finally { + Carbon::setTestNow(); + } + $this->assertTrue($expired['enforced']); + $this->assertNull($expired['next_task']); + $this->assertSame(ActivityStatus::Failed, $execution->refresh()->status); + $this->assertSame(1, $run->historyEvents()->where('event_type', HistoryEventType::ActivityTimedOut)->count()); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCompleted)->count()); + $this->assertSame(0, $run->tasks()->where('status', TaskStatus::Ready)->count()); + $this->assertNull(WorkflowRunRetentionCleanup::retentionHoldReason($run)); + $this->assertSame(RunStatus::Cancelled, $run->refresh()->status); + } + + public function testInternalActivityAbandonCanBeFirstClaimedAfterCooperativeParentClosure(): void + { + [$run, $initial] = $this->newRun(); + [$execution, $activityTask] = $this->scheduleInternalAbandonableActivity($run, $initial); + $this->closeCooperativeParent($run, $initial); + $this->assertSame(ActivityStatus::Pending, $execution->refresh()->status); + $this->assertSame(TaskStatus::Ready, $activityTask->refresh()->status); + $activities = $this->app->make(ActivityTaskBridge::class); + $this->assertTrue($activities->claimStatus($activityTask->id, 'late-first-owner')['claimed']); + $attempt = $execution->attempts() + ->sole(); + $this->assertTrue($activities->status($attempt->id)['can_continue']); + $this->assertTrue($activities->complete($attempt->id, 'late first claim')['recorded']); + $this->assertSame(RunStatus::Cancelled, $run->refresh()->status); + $this->assertSame( + 0, + $run->tasks() + ->where('task_type', TaskType::Workflow)->where('status', TaskStatus::Ready)->count() + ); + } + + public function testInternalActivityAbandonRecoversAnExpiredOwnerAfterParentClosure(): void + { + [$run, $initial] = $this->newRun(); + [$execution, $activityTask] = $this->scheduleInternalAbandonableActivity($run, $initial); + $activities = $this->app->make(ActivityTaskBridge::class); + $this->assertTrue($activities->claimStatus($activityTask->id, 'lost-owner')['claimed']); + $first = $execution->attempts() + ->sole(); + $deadline = $execution->refresh() + ->schedule_to_close_deadline_at->toISOString(); + $this->closeCooperativeParent($run, $initial); + $activityTask->refresh() + ->forceFill([ + 'lease_expires_at' => now() + ->subSecond(), + ])->save(); + $repaired = TaskRepair::recoverExistingTask($activityTask, $run->fresh()); + $this->assertNotNull($repaired); + $this->assertSame(TaskStatus::Ready, $repaired->status); + $this->assertSame(ActivityAttemptStatus::Expired, $first->refresh()->status); + $this->assertTrue($activities->claimStatus($repaired->id, 'fresh-owner')['claimed']); + $second = $execution->attempts() + ->where('attempt_number', 2) + ->sole(); + $this->assertSame('stale_attempt', $activities->complete($first->id, 'lost owner result')['reason']); + $this->assertTrue($activities->complete($second->id, 'recovered result')['recorded']); + $this->assertSame($deadline, $execution->refresh()->schedule_to_close_deadline_at->toISOString()); + $this->assertSame(RunStatus::Cancelled, $run->refresh()->status); + } + + public function testInternalActivityAbandonCannotBeEnabledByChangingOnlyAMutableOption(): void + { + [$run, $initial] = $this->newRun(); + [$execution, $activityTask] = $this->scheduleInternalAbandonableActivity( + $run, + $initial, + policy: CancellationPolicy::TryCancel + ); + $execution->forceFill([ + 'activity_options' => [ + 'cancellation_policy' => CancellationPolicy::Abandon->value, + ], + ])->save(); + $activities = $this->app->make(ActivityTaskBridge::class); + $this->assertTrue($activities->claimStatus($activityTask->id, 'mutable-owner')['claimed']); + $attempt = $execution->attempts() + ->sole(); + $this->closeCooperativeParent($run, $initial); + $this->assertSame(ActivityStatus::Cancelled, $execution->refresh()->status); + $this->assertFalse($activities->status($attempt->id)['can_continue']); + $this->assertFalse($activities->complete($attempt->id, 'not authorized')['recorded']); + } + + public function testInternalActivityAbandonDoesNotOverrideLegacyTerminalCancellation(): void + { + [$run, $initial] = $this->newRun(); + [$execution, $activityTask] = $this->scheduleInternalAbandonableActivity($run, $initial); + $activities = $this->app->make(ActivityTaskBridge::class); + $this->assertTrue($activities->claimStatus($activityTask->id, 'terminal-owner')['claimed']); + $attempt = $execution->attempts() + ->sole(); + $this->request($run); + $this->assertTrue( + WorkflowStub::load($run->workflow_instance_id)->cancel('terminal operator action')->accepted() + ); + $this->assertFalse($activities->status($attempt->id)['can_continue']); + $this->assertFalse($activities->complete($attempt->id, 'terminal late result')['recorded']); + $this->assertSame(RunStatus::Cancelled, $run->refresh()->status); + } + + public function testInternalActivityAbandonRefusesAnUnboundedDetachedLifetime(): void + { + [$run, $initial] = $this->newRun(); + [$execution] = $this->scheduleInternalAbandonableActivity($run, $initial, totalTimeout: null); + $this->request($run); + $this->expectException(LogicException::class); + $this->expectExceptionMessage('original finite total deadline'); + ActivityAbandonment::prepare($run->fresh(['historyEvents']), $execution); + } + + public function testInternalActivityWaitReleasesClaimAndResumesOnlyAfterOriginalCallbackStopReceipt(): void + { + [$run, $initial] = $this->newRun(); + [$execution, $activityTask, $attempt] = $this->scheduleInternalWaitingActivity($run, $initial, 1); + $initial->forceFill([ + 'status' => TaskStatus::Completed, + 'lease_expires_at' => null, + ])->save(); + $this->request($run); + $deadline = $run->cancellation_deadline_at->toISOString(); + $resume = $this->leaseReadyTask($run); + $waiting = $this->deliver($run, $resume); + $this->assertFalse($waiting['delivered']); + $this->assertTrue($waiting['claim_released']); + $this->assertSame('cancellation_waiting_for_activity', $waiting['reason']); + $this->assertSame(TaskStatus::Completed, $resume->refresh()->status); + $this->assertNull($resume->lease_expires_at); + $this->assertSame(ActivityStatus::Cancelled, $execution->refresh()->status); + $this->assertSame(TaskStatus::Cancelled, $activityTask->refresh()->status); + $this->assertSame(0, $this->deliveryCount($run)); + $this->assertSame(0, $run->tasks()->where('task_type', TaskType::Workflow->value) + ->where('status', TaskStatus::Ready->value)->count()); + + $stale = ActivityCancellationAcknowledgement::recordStopped( + $attempt->id, + 'replacement-owner', + $run->cancellation_request_command_id + ); + $this->assertFalse($stale['acknowledged']); + $this->assertSame(0, $run->tasks()->where('task_type', TaskType::Workflow->value) + ->where('status', TaskStatus::Ready->value)->count()); + $receipt = ActivityCancellationAcknowledgement::recordStopped( + $attempt->id, + $attempt->lease_owner, + $run->cancellation_request_command_id + ); + $this->assertTrue($receipt['acknowledged']); + $next = $this->leaseReadyTask($run); + $this->assertNotSame($resume->id, $next->id); + $this->assertSame('activity_cancellation_acknowledged', $next->payload['resume_source_kind']); + $this->assertSame($receipt['history_event_id'], $next->payload['resume_source_id']); + $changedBoundary = $this->deliver($run, $next, 2, 'timer'); + $this->assertFalse($changedBoundary['delivered']); + $this->assertSame('cancellation_wait_boundary_mismatch', $changedBoundary['reason']); + $this->assertSame(TaskStatus::Leased, $next->refresh()->status); + $delivery = $this->deliver($run, $next); + $this->assertTrue($delivery['delivered']); + $this->assertSame(1, $this->deliveryCount($run)); + $this->assertSame($deadline, $run->refresh()->cancellation_deadline_at->toISOString()); + $duplicate = ActivityCancellationAcknowledgement::recordStopped( + $attempt->id, + $attempt->lease_owner, + $run->cancellation_request_command_id + ); + $this->assertTrue($duplicate['duplicate']); + $this->assertSame($receipt['history_event_id'], $duplicate['history_event_id']); + $this->assertSame(0, $run->tasks()->where('task_type', TaskType::Workflow->value) + ->where('status', TaskStatus::Ready->value)->count()); + } + + public function testRepairBeforeOriginalStopReceiptKeepsCancellationResumeDiscoverable(): void + { + [$run, $initial] = $this->newRun(); + $run->forceFill([ + 'namespace' => 'default', + ])->save(); + $run->instance->forceFill([ + 'namespace' => 'default', + ])->save(); + [$execution, $activityTask, $attempt] = $this->scheduleInternalWaitingActivity($run, $initial, 1); + $run->tasks() + ->update([ + 'namespace' => 'default', + ]); + $initial->forceFill([ + 'status' => TaskStatus::Completed, + 'lease_expires_at' => null, + ])->save(); + $this->request($run); + $requestId = $run->cancellation_request_command_id; + $deadline = $run->cancellation_deadline_at->toISOString(); + $waiting = $this->deliver($run, $this->leaseReadyTask($run)); + $this->assertFalse($waiting['delivered']); + $this->assertTrue($waiting['claim_released']); + $this->assertSame(0, $this->stopReceiptCount($run)); + + // The scheduler can run after claim release and before physical stop is + // acknowledged. Force that order instead of depending on wall-clock luck. + $repair = TaskWatchdog::runPass(runIds: [$run->id]); + $this->assertSame(1, $repair['repaired_missing_tasks']); + $repaired = $run->tasks() + ->where('task_type', TaskType::Workflow) + ->where('status', TaskStatus::Ready)->sole(); + $this->assertSame('default', $repaired->namespace); + $this->assertSame([$repaired->id], array_column( + $this->bridge->poll(null, $run->queue, namespace: 'default', workflowTypes: [$run->workflow_type]), + 'task_id', + )); + $this->assertSame([], $this->bridge->poll(null, $run->queue, namespace: 'another-tenant')); + $this->assertSame(0, $this->deliveryCount($run)); + + $receipt = ActivityCancellationAcknowledgement::recordStopped($attempt->id, $attempt->lease_owner, $requestId); + $this->assertTrue($receipt['acknowledged']); + $this->assertSame(1, $run->tasks()->where('task_type', TaskType::Workflow) + ->where('status', TaskStatus::Ready)->count()); + $next = $this->leaseReadyTask($run); + $this->assertSame($repaired->id, $next->id); + $this->assertTrue($this->deliver($run, $next)['delivered']); + $duplicate = ActivityCancellationAcknowledgement::recordStopped( + $attempt->id, + $attempt->lease_owner, + $requestId + ); + $this->assertTrue($duplicate['duplicate']); + $this->assertSame($receipt['history_event_id'], $duplicate['history_event_id']); + $this->assertTrue($this->bridge->complete($next->id, [[ + 'type' => 'complete_workflow', + ]])['completed']); + $this->assertSame(RunStatus::Cancelled, $run->refresh()->status); + $this->assertSame($requestId, $run->cancellation_request_command_id); + $this->assertSame($deadline, $run->cancellation_deadline_at->toISOString()); + $this->assertTrue(now()->lt($run->cancellation_deadline_at)); + $this->assertSame(ActivityStatus::Cancelled, $execution->refresh()->status); + $this->assertSame(TaskStatus::Cancelled, $activityTask->refresh()->status); + $this->assertFalse($this->app->make(ActivityTaskBridge::class)->complete($attempt->id, 'late')['recorded']); + } + + public function testInternalParallelActivityWaitWakesOnlyAfterAllOriginalCallbacksAreAcknowledged(): void + { + [$run, $initial] = $this->newRun(); + [, , $first] = $this->scheduleInternalWaitingActivity($run, $initial, 1, 2); + [, , $second] = $this->scheduleInternalWaitingActivity($run, $initial, 2, 2); + $initial->forceFill([ + 'status' => TaskStatus::Completed, + 'lease_expires_at' => null, + ])->save(); + $this->request($run); + $waiting = $this->deliver($run, $this->leaseReadyTask($run), 1, 'parallel', 2); + $this->assertFalse($waiting['delivered']); + foreach ([$first, $second] as $index => $attempt) { + $receipt = ActivityCancellationAcknowledgement::recordStopped( + $attempt->id, + $attempt->lease_owner, + $run->cancellation_request_command_id + ); + $this->assertTrue($receipt['acknowledged']); + $this->assertSame($index, $run->tasks()->where('task_type', TaskType::Workflow->value) + ->where('status', TaskStatus::Ready->value)->count()); + } + $this->assertTrue($this->deliver($run, $this->leaseReadyTask($run), 1, 'parallel', 2)['delivered']); + $this->assertSame(1, $this->deliveryCount($run)); + } + + public function testInternalActivityWaitDoesNotTurnLateStopEvidenceIntoANewCleanupBudget(): void + { + [$run, $initial] = $this->newRun(); + [, , $attempt] = $this->scheduleInternalWaitingActivity($run, $initial, 1); + $initial->forceFill([ + 'status' => TaskStatus::Completed, + 'lease_expires_at' => null, + ])->save(); + $this->request($run); + $waiting = $this->deliver($run, $this->leaseReadyTask($run)); + $this->assertFalse($waiting['delivered']); + Carbon::setTestNow($run->cancellation_deadline_at->copy()->addSecond()); + try { + $receipt = ActivityCancellationAcknowledgement::recordStopped( + $attempt->id, + $attempt->lease_owner, + $run->cancellation_request_command_id + ); + $this->assertTrue($receipt['acknowledged']); + $event = $run->historyEvents() + ->findOrFail($receipt['history_event_id']); + $this->assertTrue($event->payload['received_after_deadline']); + $this->assertSame(0, $run->tasks()->where('task_type', TaskType::Workflow->value) + ->where('status', TaskStatus::Ready->value)->count()); + $this->assertSame(0, $this->deliveryCount($run)); + } finally { + Carbon::setTestNow(); + } + } + + public function testPortableChildWaitReleasesItsClaimUntilCanonicalChildCancellationCompletes(): void + { + [$run, $task] = $this->newRun(); + $scheduled = $this->bridge->complete($task->id, [[ + 'type' => 'start_child_workflow', + 'workflow_type' => 'portable-cleanup-child', + 'arguments' => Serializer::serialize([]), + 'cancellation_policy' => CancellationPolicy::WaitCancellationCompleted->value, + ]]); + $this->assertTrue($scheduled['completed']); + $child = $run->childLinks() + ->sole() +->childRun; + $this->assertInstanceOf(WorkflowRun::class, $child); + $this->request($run); + $resume = $this->leaseReadyTask($run); + $deadline = $run->cancellation_deadline_at->toISOString(); + + $waiting = $this->deliver($run, $resume, 1, 'child'); + $this->assertFalse($waiting['delivered']); + $this->assertTrue($waiting['claim_released']); + $this->assertSame('cancellation_waiting_for_child', $waiting['reason']); + $this->assertSame(TaskStatus::Completed, $resume->refresh()->status); + $this->assertNull($resume->lease_expires_at); + $this->assertSame(0, $this->deliveryCount($run)); + $this->assertSame('task_not_leased', $this->deliver($run, $resume, 1, 'child')['reason']); + $openTasks = $run->tasks() + ->where('task_type', TaskType::Workflow); + $openTasks->whereIn('status', [TaskStatus::Ready, TaskStatus::Leased]); + $this->assertSame(0, $openTasks->count()); + $this->assertSame($deadline, $child->refresh()->cancellation_deadline_at->toISOString()); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::ChildCancellationRequested)->count() + ); + $childTask = $this->leaseReadyTask($child); + $this->assertTrue($this->deliver($child, $childTask, 1, 'timer')['delivered']); + $this->assertTrue($this->bridge->complete($childTask->id, [[ + 'type' => 'complete_workflow', + 'output' => Serializer::serialize('cleanup complete'), + ]])['completed']); + $this->assertSame(RunStatus::Cancelled, $child->refresh()->status); + + $replacement = $this->leaseReadyTask($run); + $delivery = $this->deliver($run, $replacement, 1, 'child'); + $this->assertTrue($delivery['delivered']); + $this->assertSame($run->cancellation_request_command_id, $delivery['request_id']); + $this->assertSame(1, $this->deliveryCount($run)); + $this->assertTrue($this->bridge->complete($replacement->id, [[ + 'type' => 'complete_workflow', + 'output' => Serializer::serialize('parent cleanup complete'), + ]])['completed']); + $this->assertSame(RunStatus::Cancelled, $run->refresh()->status); + $this->assertSame($deadline, $run->cancellation_deadline_at->toISOString()); + } + + public function testPortableWaitDoesNotTreatATerminalProjectionAsCanonicalAcknowledgement(): void + { + [$run, $task] = $this->newRun(); + $this->assertTrue($this->bridge->complete($task->id, [[ + 'type' => 'start_child_workflow', + 'workflow_type' => 'portable-cleanup-child', + 'arguments' => Serializer::serialize([]), + 'cancellation_policy' => CancellationPolicy::WaitCancellationCompleted->value, + ]])['completed']); + $child = $run->childLinks() + ->sole() +->childRun; + $child->forceFill([ + 'status' => RunStatus::Cancelled, + ])->save(); + $this->request($run); + $resume = $this->leaseReadyTask($run); + + $result = $this->deliver($run, $resume, 1, 'child'); + $this->assertFalse($result['delivered']); + $this->assertSame('cancellation_waiting_for_child', $result['reason']); + $this->assertSame(0, $this->deliveryCount($run)); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ChildCancellationResolved)->count() + ); + } + + public function testPortableChildPolicyRejectsInvalidValuesBeforeCreatingAnyChild(): void + { + [$run, $task] = $this->newRun(); + foreach (['unknown', false, 1, []] as $invalid) { + $result = $this->bridge->complete($task->id, [[ + 'type' => 'start_child_workflow', + 'workflow_type' => 'portable-cleanup-child', + 'arguments' => Serializer::serialize([]), + 'cancellation_policy' => $invalid, + ]]); + $this->assertFalse($result['completed']); + $this->assertSame('invalid_commands', $result['reason']); + $this->assertSame(0, $run->childLinks()->count()); + $this->assertSame(TaskStatus::Leased, $task->refresh()->status); + } + } + public function testExistingCustomBridgeDoesNotAcquireCooperativeCapability(): void { $custom = \Mockery::mock(WorkflowTaskBridge::class); @@ -64,7 +699,7 @@ public function testDeliveryWithoutAScheduledCallReservesItsPositionAndPreserves [$run, $task] = $this->newRun(); $this->request($run); $deadline = $run->cancellation_deadline_at?->toISOString(); - $lease = $task->lease_expires_at?->toISOString(); + $lease = $task->lease_expires_at?->copy(); $result = $this->deliver($run, $task); @@ -72,7 +707,9 @@ public function testDeliveryWithoutAScheduledCallReservesItsPositionAndPreserves $this->assertSame($run->cancellation_request_command_id, $result['request_id']); $this->assertSame(1, $result['sequence']); $this->assertSame(TaskStatus::Leased, $task->refresh()->status); - $this->assertSame($lease, $task->lease_expires_at?->toISOString()); + $this->assertTrue($task->lease_expires_at->lt($lease)); + $this->assertTrue($task->lease_expires_at->gt(now())); + $this->assertTrue($task->lease_expires_at->lte(now()->addSeconds(10))); $this->assertSame($deadline, $run->refresh()->cancellation_deadline_at?->toISOString()); $this->assertTrue($this->bridge->heartbeat($task->id)['renewed']); $cleanup = $this->bridge->complete($task->id, [[ @@ -264,6 +901,401 @@ public static function workflowLeaseStates(): iterable yield 'fresh workflow lease' => [false]; } + public function testRemoteCallbackStopReceiptIsSeparateFromFencingAndKeepsTheOriginalBudget(): void + { + [$run, $activityTask, $attempt] = $this->cancelledRemoteAttempt(); + $deadline = $run->cancellation_deadline_at->toISOString(); + $taskBefore = $activityTask->getAttributes(); + $attemptBefore = $attempt->getAttributes(); + $this->assertSame(0, $this->stopReceiptCount($run)); + $accepted = ActivityCancellationAcknowledgement::recordStopped( + $attempt->id, + 'activity-owner', + $run->cancellation_request_command_id, + ); + $this->assertTrue($accepted['acknowledged']); + $this->assertFalse($accepted['duplicate']); + $event = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancellationAcknowledged)->sole(); + $this->assertSame($event->id, $accepted['history_event_id']); + $this->assertSame($run->cancellation_request_command_id, $event->workflow_command_id); + $this->assertSame($run->cancellation_request_command_id, $event->payload['root_request_id']); + $this->assertSame($deadline, $event->payload['cleanup_deadline_at']); + $this->assertSame('stopped', $event->payload['callback_state']); + $this->assertSame('activity_worker', $event->payload['evidence_source']); + $this->assertFalse($event->payload['received_after_deadline']); + $repeated = ActivityCancellationAcknowledgement::recordStopped( + $attempt->id, + 'activity-owner', + $run->cancellation_request_command_id, + ); + $this->assertTrue($repeated['acknowledged']); + $this->assertTrue($repeated['duplicate']); + $this->assertSame($accepted['history_event_id'], $repeated['history_event_id']); + $this->assertSame(1, $this->stopReceiptCount($run)); + $this->assertSame($taskBefore, $activityTask->refresh()->getAttributes()); + $this->assertSame($attemptBefore, $attempt->refresh()->getAttributes()); + $summary = RunSummaryProjector::project($run->fresh()); + $this->assertSame($run->historyEvents()->count(), $summary->history_event_count); + $receipt = collect(HistoryTimeline::forRun($run->fresh()))->firstWhere( + 'type', + 'ActivityCancellationAcknowledged' + ); + $this->assertSame($event->id, $receipt['id']); + $this->assertSame($attempt->id, $receipt['activity']['attempt_id']); + $this->assertSame('cancelled', $receipt['activity_status']); + $this->assertSame($deadline, $receipt['cancellation_acknowledgement']['cleanup_deadline_at']); + $this->assertSame( + $event->payload['root_request_id'], + $receipt['cancellation_acknowledgement']['root_request_id'] + ); + $this->assertSame($deadline, $run->refresh()->cancellation_deadline_at->toISOString()); + $this->assertFalse( + $this->app->make(ActivityTaskBridge::class)->complete($attempt->id, 'late result')['recorded'] + ); + } + + #[DataProvider('stopAcknowledgementFences')] + public function testRemoteStopReceiptRejectsAnotherOwnerAttemptOrRequest(string $field): void + { + [$run, , $attempt] = $this->cancelledRemoteAttempt(); + $arguments = [$attempt->id, 'activity-owner', $run->cancellation_request_command_id]; + $index = array_search($field, ['attempt', 'owner', 'request'], true); + $arguments[$index] = 'another-identity'; + $reply = ActivityCancellationAcknowledgement::recordStopped(...$arguments); + $this->assertFalse($reply['acknowledged']); + $this->assertFalse($reply['duplicate']); + $this->assertNotNull($reply['reason']); + $this->assertSame(0, $this->stopReceiptCount($run)); + } + + public static function stopAcknowledgementFences(): iterable + { + foreach (['attempt', 'owner', 'request'] as $field) { + yield $field => [$field]; + } + } + + public function testMutableCancelledRowsCannotSubstituteForCanonicalCancellationHistory(): void + { + [$run, $task, $attempt] = $this->cancelledRemoteAttempt(deliver: false); + $task->forceFill([ + 'status' => TaskStatus::Cancelled, + 'lease_expires_at' => null, + ])->save(); + $attempt->forceFill([ + 'status' => ActivityAttemptStatus::Cancelled, + 'lease_expires_at' => null, + ])->save(); + $attempt->execution->forceFill([ + 'status' => ActivityStatus::Cancelled, + ])->save(); + $reply = ActivityCancellationAcknowledgement::recordStopped( + $attempt->id, + 'activity-owner', + $run->cancellation_request_command_id, + ); + $this->assertFalse($reply['acknowledged']); + $this->assertSame('activity_cancellation_not_recorded', $reply['reason']); + $this->assertSame(0, $this->stopReceiptCount($run)); + } + + #[DataProvider('invalidStopSnapshots')] + public function testLegacyOrMalformedCancellationSnapshotCannotAuthorizeAWorkerStopReceipt(bool $malformed): void + { + [$run, , $attempt] = $this->cancelledRemoteAttempt(); + $event = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancelled)->sole(); + $payload = $event->payload; + unset($payload['activity_attempt']['id']); + if ($malformed) { + $payload['activity_attempt'] = 'not-an-attempt-snapshot'; + } + $event->forceFill([ + 'payload' => $payload, + ])->save(); + $reply = ActivityCancellationAcknowledgement::recordStopped( + $attempt->id, + 'activity-owner', + $run->cancellation_request_command_id, + ); + $this->assertFalse($reply['acknowledged']); + $this->assertSame('activity_cancellation_snapshot_mismatch', $reply['reason']); + $this->assertSame(0, $this->stopReceiptCount($run)); + } + + public static function invalidStopSnapshots(): iterable + { + yield 'attempt snapshot without identity' => [false]; + yield 'malformed attempt snapshot' => [true]; + } + + public function testLateStopReportRemainsLateAndCannotGrantAnotherCleanupBudget(): void + { + [$run, $task, $attempt] = $this->cancelledRemoteAttempt(); + $deadline = $run->cancellation_deadline_at->toISOString(); + Carbon::setTestNow($run->cancellation_deadline_at->copy()->addSecond()); + try { + $reply = ActivityCancellationAcknowledgement::recordStopped( + $attempt->id, + 'activity-owner', + $run->cancellation_request_command_id, + ); + $this->assertTrue($reply['acknowledged']); + $event = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancellationAcknowledged)->sole(); + $this->assertTrue($event->payload['received_after_deadline']); + $this->assertSame($deadline, $event->payload['cleanup_deadline_at']); + $this->assertSame($deadline, $run->refresh()->cancellation_deadline_at->toISOString()); + $this->assertNull($task->refresh()->lease_expires_at); + $this->assertNull($attempt->refresh()->lease_expires_at); + $this->assertSame(0, $run->tasks()->where('status', TaskStatus::Ready)->count()); + } finally { + Carbon::setTestNow(); + } + } + + public function testRemoteStopReceiptCannotAuthorizeALocalActivityCallback(): void + { + [$run, , $attempt] = $this->cancelledRemoteAttempt(); + $attempt->execution->forceFill([ + 'activity_options' => [ + 'execution_mode' => LocalActivityRuntime::EXECUTION_MODE, + ], + ])->save(); + $reply = ActivityCancellationAcknowledgement::recordStopped( + $attempt->id, + 'activity-owner', + $run->cancellation_request_command_id, + ); + $this->assertFalse($reply['acknowledged']); + $this->assertSame('local_cancellation_acknowledgement_requires_workflow_claim', $reply['reason']); + $this->assertSame(0, $this->stopReceiptCount($run)); + } + + public function testLocalStopReceiptKeepsTheOriginalWorkflowClaimAndDoesNotRenewCleanupAuthority(): void + { + [$run, $task, $attempt] = $this->cancelledLocalAttempt(); + $taskBefore = $task->getAttributes(); + $attemptBefore = $attempt->getAttributes(); + $deadline = $run->cancellation_deadline_at->toISOString(); + $reply = ActivityCancellationAcknowledgement::recordLocalStopped( + $attempt->id, + 'portable-worker', + $run->cancellation_request_command_id, + 1, + ); + $this->assertTrue($reply['acknowledged']); + $event = $run->historyEvents() + ->whereKey($reply['history_event_id'])->sole(); + $this->assertTrue($event->payload['local_activity']); + $this->assertSame(LocalActivityRuntime::EXECUTION_MODE, $event->payload['execution_mode']); + $this->assertSame('workflow_worker', $event->payload['evidence_source']); + $this->assertSame($task->id, $event->payload['workflow_task_id']); + $this->assertSame('portable-worker', $event->payload['task']['lease_owner']); + $this->assertSame(1, $event->payload['task']['attempt_count']); + $this->assertSame($deadline, $event->payload['cleanup_deadline_at']); + $this->assertSame($run->cancellation_request_command_id, $event->payload['root_request_id']); + $this->assertSame($taskBefore, $task->refresh()->getAttributes()); + $this->assertSame($attemptBefore, $attempt->refresh()->getAttributes()); + $this->assertSame(TaskStatus::Leased, $task->status); + $this->assertSame($deadline, $run->refresh()->cancellation_deadline_at->toISOString()); + $receipt = collect(HistoryTimeline::forRun($run->fresh()))->firstWhere( + 'type', + 'ActivityCancellationAcknowledged' + ); + $this->assertSame($event->id, $receipt['id']); + $this->assertSame($attempt->id, $receipt['activity']['attempt_id']); + } + + public function testLocalOriginalOwnerCanReportAfterWorkflowTakeoverButTheReplacementCannotImpersonateIt(): void + { + [$run, $task, $attempt] = $this->cancelledLocalAttempt(); + $task->forceFill([ + 'lease_owner' => 'replacement-owner', + 'attempt_count' => 2, + ])->save(); + $taskBefore = $task->refresh() + ->getAttributes(); + foreach ([['replacement-owner', 2], + ['portable-worker', 2], + ['portable-worker', 0], + ] as [$owner, $claimAttempt]) { + $reply = ActivityCancellationAcknowledgement::recordLocalStopped( + $attempt->id, + $owner, + $run->cancellation_request_command_id, + $claimAttempt, + ); + $this->assertFalse($reply['acknowledged']); + } + $this->assertSame(0, $this->stopReceiptCount($run)); + $original = ActivityCancellationAcknowledgement::recordLocalStopped( + $attempt->id, + 'portable-worker', + $run->cancellation_request_command_id, + 1, + ); + $this->assertTrue($original['acknowledged']); + $event = $run->historyEvents() + ->whereKey($original['history_event_id'])->sole(); + $this->assertSame('portable-worker', $event->payload['task']['lease_owner']); + $this->assertSame(1, $event->payload['task']['attempt_count']); + $this->assertSame($taskBefore, $task->refresh()->getAttributes()); + $receipt = collect(HistoryTimeline::forRun($run->fresh()))->firstWhere( + 'type', + 'ActivityCancellationAcknowledged' + ); + $this->assertSame('workflow', $receipt['task']['type']); + $this->assertSame('leased', $receipt['task']['status']); + $this->assertSame(1, $receipt['task']['attempt_count']); + $task->forceFill([ + 'status' => TaskStatus::Completed, + 'attempt_count' => 3, + ])->save(); + $taskBefore = $task->refresh() + ->getAttributes(); + $duplicate = ActivityCancellationAcknowledgement::recordLocalStopped( + $attempt->id, + 'portable-worker', + $run->cancellation_request_command_id, + 1, + ); + $this->assertTrue($duplicate['acknowledged']); + $this->assertTrue($duplicate['duplicate']); + $this->assertSame($original['history_event_id'], $duplicate['history_event_id']); + $this->assertSame(1, $this->stopReceiptCount($run)); + $this->assertSame($taskBefore, $task->refresh()->getAttributes()); + $receipt = collect(HistoryTimeline::forRun($run->fresh()))->firstWhere( + 'type', + 'ActivityCancellationAcknowledged' + ); + $this->assertSame('workflow', $receipt['task']['type']); + $this->assertSame('leased', $receipt['task']['status']); + $this->assertSame(1, $receipt['task']['attempt_count']); + } + + #[DataProvider('invalidLocalStopClaims')] + public function testLocalStopReceiptRequiresTheClaimSavedBeforeItsCallback(string $field, mixed $value): void + { + [$run, , $attempt] = $this->cancelledLocalAttempt(); + $event = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted)->sole(); + $payload = $event->payload; + if ($field === 'task') { + $payload['task'] = $value; + } else { + $payload['task'][$field] = $value; + } + $event->forceFill([ + 'payload' => $payload, + ])->save(); + $reply = ActivityCancellationAcknowledgement::recordLocalStopped( + $attempt->id, + 'portable-worker', + $run->cancellation_request_command_id, + 1, + ); + $this->assertFalse($reply['acknowledged']); + $this->assertSame('local_activity_workflow_claim_mismatch', $reply['reason']); + $this->assertSame(0, $this->stopReceiptCount($run)); + } + + public static function invalidLocalStopClaims(): iterable + { + yield 'legacy start without claim' => ['task', null]; + yield 'malformed claim' => ['task', 'not-a-claim']; + yield 'another task' => ['id', 'another-task']; + yield 'ordinary activity task' => ['type', TaskType::Activity->value]; + yield 'unclaimed task' => ['status', TaskStatus::Ready->value]; + yield 'another original owner' => ['lease_owner', 'another-owner']; + yield 'missing original attempt' => ['attempt_count', null]; + yield 'nonpositive original attempt' => ['attempt_count', 0]; + } + + public function testLocalStopReceiptCannotBorrowARemoteAttemptOrAnotherCancellationRequest(): void + { + [$run, , $attempt] = $this->cancelledRemoteAttempt(); + $reply = ActivityCancellationAcknowledgement::recordLocalStopped( + $attempt->id, + 'activity-owner', + $run->cancellation_request_command_id, + 1, + ); + $this->assertFalse($reply['acknowledged']); + $this->assertSame('activity_is_not_local', $reply['reason']); + [$run, , $attempt] = $this->cancelledLocalAttempt(); + $reply = ActivityCancellationAcknowledgement::recordLocalStopped( + $attempt->id, + 'portable-worker', + 'another-request', + 1, + ); + $this->assertFalse($reply['acknowledged']); + $this->assertSame('cancellation_request_mismatch', $reply['reason']); + $this->assertSame(0, $this->stopReceiptCount($run)); + } + + #[DataProvider('unpreparedLocalCallbacks')] + public function testLocalStopReceiptRejectsAnAbsentOrPostCancellationStart(bool $missing): void + { + [$run, , $attempt] = $this->cancelledLocalAttempt(); + $started = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted)->sole(); + if ($missing) { + $started->delete(); + } else { + $started->forceFill([ + 'sequence' => $run->last_history_sequence + 1, + ])->save(); + } + $reply = ActivityCancellationAcknowledgement::recordLocalStopped( + $attempt->id, + 'portable-worker', + $run->cancellation_request_command_id, + 1, + ); + $this->assertFalse($reply['acknowledged']); + $this->assertSame('local_activity_workflow_claim_mismatch', $reply['reason']); + $this->assertSame(0, $this->stopReceiptCount($run)); + } + + public static function unpreparedLocalCallbacks(): iterable + { + yield 'callback without durable preparation' => [true]; + yield 'post hoc callback start' => [false]; + } + + public function testLateLocalReceiptDoesNotRenewTheOriginalDeadlineOrReplacementLease(): void + { + [$run, $task, $attempt] = $this->cancelledLocalAttempt(); + $task->forceFill([ + 'lease_owner' => 'replacement-owner', + 'attempt_count' => 2, + ])->save(); + $taskBefore = $task->refresh() + ->getAttributes(); + $deadline = $run->cancellation_deadline_at->toISOString(); + Carbon::setTestNow($run->cancellation_deadline_at->copy()->addSecond()); + try { + $reply = ActivityCancellationAcknowledgement::recordLocalStopped( + $attempt->id, + 'portable-worker', + $run->cancellation_request_command_id, + 1, + ); + $this->assertTrue($reply['acknowledged']); + $event = $run->historyEvents() + ->whereKey($reply['history_event_id'])->sole(); + $this->assertTrue($event->payload['received_after_deadline']); + $this->assertSame($deadline, $event->payload['cleanup_deadline_at']); + $this->assertSame($taskBefore, $task->refresh()->getAttributes()); + $this->assertSame($deadline, $run->refresh()->cancellation_deadline_at->toISOString()); + } finally { + Carbon::setTestNow(); + } + } + public function testARecordedCallCannotBeRelabelledAsADifferentOperation(): void { [$run, $task] = $this->newRun(); @@ -554,6 +1586,209 @@ public static function parallelDeliveryCalls(): iterable yield 'selection member range' => [true]; } + /** + * Creates internal Source history before callback admission. Portable SDK + * policy admission and local Abandon lifetime are not exposed yet. + * + * @return array{ActivityExecution, WorkflowTask} + */ + private function scheduleInternalAbandonableActivity( + WorkflowRun $run, + WorkflowTask $task, + CancellationPolicy $policy = CancellationPolicy::Abandon, + ?int $totalTimeout = 180, + int $maxAttempts = 1, + ): array { + $execution = ActivityExecution::query()->create([ + 'workflow_run_id' => $run->id, + 'sequence' => 1, + 'activity_class' => TestGreetingActivity::class, + 'activity_type' => TestGreetingActivity::class, + 'status' => ActivityStatus::Pending, + 'arguments' => Serializer::serialize(['Taylor']), + 'connection' => 'database', + 'queue' => 'default', + 'activity_options' => [ + 'cancellation_policy' => $policy->value, + ], + 'retry_policy' => [ + 'snapshot_version' => 1, + 'max_attempts' => $maxAttempts, + 'backoff_seconds' => [0], + 'schedule_to_close_timeout' => $totalTimeout, + ], + 'schedule_to_close_deadline_at' => $totalTimeout === null ? null : now() + ->addSeconds($totalTimeout), + ]); + WorkflowHistoryEvent::record($run, HistoryEventType::ActivityScheduled, [ + 'activity_execution_id' => $execution->id, + 'activity_class' => $execution->activity_class, + 'activity_type' => $execution->activity_type, + 'sequence' => 1, + 'activity' => ActivitySnapshot::fromExecution($execution), + ], $task); + $activityTask = WorkflowTask::query()->create([ + 'workflow_run_id' => $run->id, + 'task_type' => TaskType::Activity, + 'status' => TaskStatus::Ready, + 'available_at' => now(), + 'connection' => 'database', + 'queue' => 'default', + 'compatibility' => 'build-a', + 'payload' => [ + 'activity_execution_id' => $execution->id, + ], + ]); + return [$execution, $activityTask]; + } + + private function closeCooperativeParent(WorkflowRun $run, WorkflowTask $initial): void + { + $initial->forceFill([ + 'status' => TaskStatus::Completed, + 'lease_expires_at' => null, + ])->save(); + $this->request($run); + $resume = $this->leaseReadyTask($run); + $this->assertTrue($this->deliver($run, $resume)['delivered']); + $this->assertTrue($this->bridge->complete($resume->id, [[ + 'type' => 'complete_workflow', + ]])['completed']); + $this->assertSame(RunStatus::Cancelled, $run->refresh()->status); + } + + /** + * @return array{ActivityExecution, WorkflowTask, \Workflow\V2\Models\ActivityAttempt} + */ + private function scheduleInternalWaitingActivity( + WorkflowRun $run, + WorkflowTask $task, + int $sequence, + ?int $groupSize = null, + ): array { + $metadata = $groupSize === null ? [] : ParallelChildGroup::itemMetadata( + 1, + $groupSize, + $sequence - 1, + 'activity' + ); + $execution = ActivityExecution::query()->create([ + 'workflow_run_id' => $run->id, + 'sequence' => $sequence, + 'activity_class' => TestGreetingActivity::class, + 'activity_type' => TestGreetingActivity::class, + 'status' => ActivityStatus::Pending, + 'arguments' => Serializer::serialize(['Taylor']), + 'connection' => 'database', + 'queue' => 'default', + 'activity_options' => [ + 'cancellation_policy' => CancellationPolicy::WaitCancellationCompleted->value, + ], + 'parallel_group_path' => $metadata['parallel_group_path'] ?? null, + ]); + WorkflowHistoryEvent::record($run, HistoryEventType::ActivityScheduled, [ + 'activity_execution_id' => $execution->id, + 'activity_class' => $execution->activity_class, + 'activity_type' => $execution->activity_type, + 'sequence' => $sequence, + 'activity' => ActivitySnapshot::fromExecution($execution), + ...$metadata, + ], $task); + $activityTask = WorkflowTask::query()->create([ + 'workflow_run_id' => $run->id, + 'task_type' => TaskType::Activity, + 'status' => TaskStatus::Ready, + 'available_at' => now(), + 'connection' => 'database', + 'queue' => 'default', + 'compatibility' => 'build-a', + 'payload' => [ + 'activity_execution_id' => $execution->id, + ], + ]); + $claimed = $this->app->make(ActivityTaskBridge::class)->claimStatus($activityTask->id, 'owner-' . $sequence); + $this->assertTrue($claimed['claimed']); + return [$execution, $activityTask, $execution->attempts()->sole()]; + } + + /** + * @return array{WorkflowRun, WorkflowTask, \Workflow\V2\Models\ActivityAttempt} + */ + private function cancelledRemoteAttempt(bool $deliver = true): array + { + [$run, $task] = $this->newRun(); + $scheduled = $this->bridge->complete($task->id, [[ + 'type' => 'schedule_activity', + 'activity_type' => TestGreetingActivity::class, + 'arguments' => Serializer::serialize(['Taylor']), + ]]); + $activityTask = WorkflowTask::query()->findOrFail($scheduled['created_task_ids'][0]); + $claim = $this->app->make(ActivityTaskBridge::class)->claimStatus($activityTask->id, 'activity-owner'); + $this->assertTrue($claim['claimed']); + $attempt = $run->activityExecutions() + ->sole() + ->attempts() + ->sole(); + $this->assertNull($attempt->worker_attempt_id); + $this->request($run); + if ($deliver) { + $this->assertTrue($this->deliver($run, $this->leaseReadyTask($run), 1, 'activity')['delivered']); + } + + return [$run, $activityTask->refresh(), $attempt->refresh()]; + } + + /** + * Exercises the actual embedded preparation path. The mock checks history + * before cancellation, but does not claim to prove an SDK process stopped. + * + * @return array{WorkflowRun, WorkflowTask, \Workflow\V2\Models\ActivityAttempt} + */ + private function cancelledLocalAttempt(): array + { + [$run, $task] = $this->newRun(); + $task->forceFill([ + 'attempt_count' => 1, + ])->save(); + WorkflowStub::mock(TestGreetingActivity::class, function (ActivityFakeContext $context): string { + $started = $context->run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted)->sole(); + $this->assertSame($context->taskId, $started->payload['task']['id']); + $this->assertSame('portable-worker', $started->payload['task']['lease_owner']); + $this->assertSame(1, $started->payload['task']['attempt_count']); + $this->assertSame(TaskStatus::Leased->value, $started->payload['task']['status']); + $this->request($context->run); + ActivityCancellation::record( + $context->run, + $context->execution, + command: $context->run->cancellation_request_command_id, + ); + + return 'a fenced local result'; + }); + $outcome = (new LocalActivityExecutor())->execute( + $run, + $task, + 1, + new LocalActivityCall(TestGreetingActivity::class, ['Taylor']), + ); + $this->assertSame('waiting', $outcome['status']); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCompleted)->count()); + $this->assertSame(0, $run->tasks()->where('task_type', TaskType::Activity)->count()); + $attempt = $run->activityExecutions() + ->sole() + ->attempts() + ->sole(); + + return [$run->refresh(), $task->refresh(), $attempt]; + } + + private function stopReceiptCount(WorkflowRun $run): int + { + return $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancellationAcknowledged)->count(); + } + /** * @return array{WorkflowRun, WorkflowTask} */ diff --git a/tests/Feature/V2/V2PortableLocalActivityCheckpointTest.php b/tests/Feature/V2/V2PortableLocalActivityCheckpointTest.php new file mode 100644 index 000000000..a8dc5055a --- /dev/null +++ b/tests/Feature/V2/V2PortableLocalActivityCheckpointTest.php @@ -0,0 +1,403 @@ +set('workflows.v2.compatibility.current', 'build-a'); + config() + ->set('workflows.v2.compatibility.supported', ['build-a']); + } + + public function testCommittedPrefixRetainsTheClaimAndAllowsTheNextPreparedLocalCall(): void + { + [$run, $task] = $this->newClaim(); + $before = $task->getAttributes(); + $reply = $this->checkpoint($task, $this->prefix()); + $this->assertTrue($reply['checkpointed']); + $this->assertFalse($reply['duplicate']); + $this->assertSame(4, $reply['next_sequence']); + $this->assertSame(4, WorkflowStepHistory::nextDurableCommandSequence($run->fresh())); + $memos = $run->fresh() + ->typedMemos(); + $this->assertSame([ + 'stage' => 'before-local', + ], $memos); + $this->assertSame(1, WorkflowLink::query()->where('parent_workflow_run_id', $run->id)->count()); + $this->assertCount(1, $reply['created_task_ids']); + $after = $task->refresh() + ->getAttributes(); + unset($before['payload'], $before['updated_at'], $after['payload'], $after['updated_at']); + $this->assertSame($before, $after); + $this->assertSame('keep-me', $task->payload['unrelated']); + $this->assertSame(TaskStatus::Leased, $task->status); + $this->assertSame($reply['lease_expires_at'], $task->lease_expires_at->toISOString()); + $this->assertTrue($this->prepare($task, 4)['prepared']); + $this->assertSame(1, ActivityExecution::query()->count()); + $this->assertSame(0, $run->tasks()->where('task_type', TaskType::Activity)->count()); + } + + public function testLostCheckpointResponseReturnsTheSameReceiptWithoutDuplicatingPrefixWork(): void + { + [$run, $task] = $this->newClaim(); + $first = $this->checkpoint($task, $this->prefix()); + $this->assertTrue($first['checkpointed']); + $count = WorkflowHistoryEvent::query()->count(); + $before = $task->refresh() + ->getAttributes(); + Carbon::setTestNow(now()->addSecond()); + try { + $second = $this->checkpoint($task, $this->prefix()); + $this->assertSameCheckpointReceipt($first, $second); + $this->assertSame($count, WorkflowHistoryEvent::query()->count()); + $this->assertSame(1, WorkflowLink::query()->where('parent_workflow_run_id', $run->id)->count()); + $this->assertSame($before, $task->refresh()->getAttributes()); + } finally { + Carbon::setTestNow(); + } + } + + public function testAChangedCheckpointCannotRelabelACommittedPrefix(): void + { + [, $task] = $this->newClaim(); + $this->assertTrue($this->checkpoint($task, $this->prefix())['checkpointed']); + $count = WorkflowHistoryEvent::query()->count(); + $changed = $this->prefix(); + $changed[0]['result'] = Serializer::serializeWithCodec('avro', 'different'); + $this->assertSame('local_activity_checkpoint_mismatch', $this->checkpoint($task, $changed)['reason']); + $this->assertSame('local_activity_checkpoint_mismatch', $this->checkpoint($task, $this->prefix(), 2)['reason']); + $this->assertSame($count, WorkflowHistoryEvent::query()->count()); + } + + public function testAnOldOwnerCannotCheckpointOrReadAnOriginalReceiptAfterClaimTakeover(): void + { + [, $task] = $this->newClaim(); + $this->assertTrue($this->checkpoint($task, $this->prefix())['checkpointed']); + $task->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 2, + ])->save(); + $before = $task->refresh() + ->getAttributes(); + $this->assertSame('workflow_claim_mismatch', $this->checkpoint($task, $this->prefix())['reason']); + $reply = app(DefaultWorkflowTaskBridge::class)->checkpointLocalActivityPrefix( + $task->id, + 'replacement', + 2, + 'checkpoint-one', + 1, + $this->prefix(), + '1.20', + ); + $this->assertSame('local_activity_checkpoint_mismatch', $reply['reason']); + $this->assertSame($before, $task->refresh()->getAttributes()); + } + + public function testAnExpiredClaimCannotCheckpointOrRenewItself(): void + { + [, $task] = $this->newClaim(); + $task->forceFill([ + 'lease_expires_at' => now() + ->subSecond(), + ])->save(); + $before = $task->refresh() + ->getAttributes(); + $this->assertSame('workflow_claim_expired', $this->checkpoint($task, $this->prefix())['reason']); + $this->assertSame($before, $task->refresh()->getAttributes()); + $this->assertSame(0, WorkflowHistoryEvent::query()->count()); + } + + public function testThePrefixMustStartAtTheCanonicalAuthoredCursor(): void + { + [, $task] = $this->newClaim(); + $this->assertSame( + 'local_activity_checkpoint_sequence_mismatch', + $this->checkpoint($task, $this->prefix(), 2)['reason'] + ); + $this->assertSame(0, WorkflowHistoryEvent::query()->count()); + $this->assertArrayNotHasKey('portable_local_checkpoint', $task->refresh()->payload); + } + + public function testCancellationStopsNewPrefixWorkButDoesNotEraseAnAlreadyCommittedReceipt(): void + { + [$run, $task] = $this->newClaim(); + $first = $this->checkpoint($task, $this->prefix()); + $this->assertTrue($first['checkpointed']); + $this->assertTrue( + WorkflowStub::load($run->workflow_instance_id)->requestCancellation('maintenance', 30)->accepted() + ); + $deadline = $run->refresh() + ->cancellation_deadline_at->toISOString(); + $count = WorkflowHistoryEvent::query()->count(); + $this->assertSameCheckpointReceipt($first, $this->checkpoint($task, $this->prefix())); + $this->assertSame('cancellation_requested', $this->checkpoint($task, [], 4, 'new-checkpoint')['reason']); + $this->assertSame('cancellation_requested', $this->prepare($task, 4)['reason']); + $this->assertSame($count, WorkflowHistoryEvent::query()->count()); + $this->assertSame($deadline, $run->refresh()->cancellation_deadline_at->toISOString()); + $this->assertSame(0, ActivityExecution::query()->count()); + } + + public function testSequentialCheckpointsKeepOneBoundedReceiptAndFinalCompletionUsesFreshHistory(): void + { + [$run, $task] = $this->newClaim(); + $this->assertTrue($this->checkpoint($task, [$this->prefix()[0]])['checkpointed']); + $second = $this->checkpoint($task, [$this->prefix()[2]], 2, 'checkpoint-two'); + $this->assertTrue($second['checkpointed']); + $this->assertSame(3, $second['next_sequence']); + $payload = $task->refresh() +->payload; + $this->assertSame(['unrelated', 'portable_local_checkpoint'], array_keys($payload)); + $this->assertSame('checkpoint-two', $payload['portable_local_checkpoint']['checkpoint_id']); + $this->assertSame( + 'local_activity_checkpoint_sequence_mismatch', + $this->checkpoint($task, [$this->prefix()[0]])['reason'] + ); + $completed = app(DefaultWorkflowTaskBridge::class)->complete($task->id, [[ + 'type' => 'complete_workflow', + 'result' => Serializer::serializeWithCodec('avro', 'done'), + 'payload_codec' => 'avro', + ]]); + $this->assertTrue($completed['completed']); + $this->assertSame(RunStatus::Completed, $run->refresh()->status); + $this->assertSame(1, $run->historyEvents()->where('event_type', HistoryEventType::SideEffectRecorded)->count()); + $this->assertSame(1, $run->historyEvents()->where('event_type', HistoryEventType::MemoUpserted)->count()); + } + + public function testAnApplicationFailureRollsBackTheWholePrefixAndItsReceipt(): void + { + [, $task] = $this->newClaim(); + $before = $task->getAttributes(); + $memos = []; + for ($i = 0; $i <= WorkflowMemo::MAX_MEMOS_PER_RUN; ++$i) { + $memos['memo-' . $i] = 'value'; + } + try { + $this->checkpoint($task, [ + $this->prefix()[0], [ + 'type' => 'upsert_memo', + 'entries' => MemoPayload::envelope($memos), + ]]); + $this->fail('The excessive memo count must abort the transaction.'); + } catch (InvalidArgumentException $exception) { + $this->assertStringContainsString('Memo count exceeds maximum', $exception->getMessage()); + } + $this->assertSame(0, WorkflowHistoryEvent::query()->count()); + $this->assertSame(0, WorkflowMemo::query()->count()); + $this->assertSame($before, $task->refresh()->getAttributes()); + } + + #[DataProvider('invalidCommands')] + public function testARejectedBatchCannotApplyEvenItsValidFirstCommand(array $invalid): void + { + [, $task] = $this->newClaim(); + $before = $task->getAttributes(); + $reply = $this->checkpoint($task, [$this->prefix()[0], $invalid]); + $this->assertFalse($reply['checkpointed']); + $this->assertSame('invalid_local_activity_checkpoint_commands', $reply['reason']); + $this->assertSame(0, WorkflowHistoryEvent::query()->count()); + $this->assertSame($before, $task->refresh()->getAttributes()); + } + + public static function invalidCommands(): iterable + { + yield 'terminal' => [[ + 'type' => 'complete_workflow', + ]]; + yield 'unknown' => [[ + 'type' => 'invented', + ]]; + yield 'wait closes a turn' => [[ + 'type' => 'open_signal_wait', + 'signal_name' => 'ready', + ]]; + yield 'posthoc local execution' => [[ + 'type' => 'record_local_activity', + ]]; + yield 'invalid encoding' => [[ + 'type' => 'record_version_marker', + 'change_id' => "bad-\xFF", + 'version' => 1, + ]]; + } + + public function testProtocol119AndOversizedBatchesDoNotEnterThePreparedPath(): void + { + [, $task] = $this->newClaim(); + $bridge = app(DefaultWorkflowTaskBridge::class); + $this->assertSame('local_activity_checkpoint_requires_protocol_1_20', $bridge->checkpointLocalActivityPrefix( + $task->id, + 'portable-worker', + 1, + 'checkpoint-one', + 1, + $this->prefix(), + '1.19', + )['reason']); + $this->assertSame( + 'invalid_local_activity_checkpoint', + $this->checkpoint($task, array_fill(0, 101, $this->prefix()[0]))['reason'] + ); + $this->assertSame(0, WorkflowHistoryEvent::query()->count()); + } + + /** @param array $first + * @param array $reply + */ + private function assertSameCheckpointReceipt(array $first, array $reply): void + { + $expected = [ + ...$first, + 'duplicate' => true, + ]; + // MySQL normalizes JSON object key order. Values and types must + // remain identical, while object member order has no authority. + ksort($expected); + ksort($reply); + $this->assertSame($expected, $reply); + } + + /** + * @return list + */ + private function prefix(): array + { + return [ + [ + 'type' => 'record_side_effect', + 'result' => Serializer::serializeWithCodec('avro', 'recorded-once'), + ], + [ + 'type' => 'start_child_workflow', + 'workflow_type' => 'python-child', + 'arguments' => Serializer::serializeWithCodec('avro', ['child']), + 'payload_codec' => 'avro', + ], + [ + 'type' => 'upsert_memo', + 'entries' => MemoPayload::envelope([ + 'stage' => 'before-local', + ]), + ], + ]; + } + + /** @param list $commands + * @return array + */ + private function checkpoint( + WorkflowTask $task, + array $commands, + int $sequence = 1, + string $id = 'checkpoint-one' + ): array { + return app(DefaultWorkflowTaskBridge::class)->checkpointLocalActivityPrefix( + $task->id, + 'portable-worker', + 1, + $id, + $sequence, + $commands, + '1.20' + ); + } + + /** + * @return array + */ + private function prepare(WorkflowTask $task, int $sequence): array + { + return PortableLocalActivityPreparation::prepare( + $task->id, + 'portable-worker', + 1, + $sequence, + 'sdk-local-attempt', + [ + 'type' => 'record_local_activity', + 'activity_type' => 'php-local', + 'arguments' => Serializer::serializeWithCodec('avro', ['local']), + 'payload_codec' => 'avro', + ], + '1.20' + ); + } + + /** + * @return array{WorkflowRun, WorkflowTask} + */ + private function newClaim(): array + { + $instance = WorkflowInstance::query()->create([ + 'workflow_class' => TestGreetingWorkflow::class, + 'workflow_type' => 'portable-parent', + 'run_count' => 1, + 'started_at' => now() + ->subMinute(), + ]); + $run = WorkflowRun::query()->create([ + 'workflow_instance_id' => $instance->id, + 'run_number' => 1, + 'workflow_class' => TestGreetingWorkflow::class, + 'workflow_type' => 'portable-parent', + 'status' => RunStatus::Waiting, + 'arguments' => Serializer::serializeWithCodec('avro', ['Taylor']), + 'payload_codec' => 'avro', + 'connection' => 'database', + 'queue' => 'default', + 'compatibility' => 'build-a', + 'started_at' => now() + ->subMinute(), + ]); + $instance->forceFill([ + 'current_run_id' => $run->id, + ])->save(); + $task = WorkflowTask::query()->create([ + 'workflow_run_id' => $run->id, + 'task_type' => TaskType::Workflow, + 'status' => TaskStatus::Leased, + 'attempt_count' => 1, + 'payload' => [ + 'unrelated' => 'keep-me', + ], + 'connection' => 'database', + 'queue' => 'default', + 'compatibility' => 'build-a', + 'lease_owner' => 'portable-worker', + 'lease_expires_at' => now() + ->addMinutes(5), + ]); + + return [$run, $task->refresh()]; + } +} diff --git a/tests/Feature/V2/V2PortableLocalActivityCleanupTest.php b/tests/Feature/V2/V2PortableLocalActivityCleanupTest.php new file mode 100644 index 000000000..90ec92acc --- /dev/null +++ b/tests/Feature/V2/V2PortableLocalActivityCleanupTest.php @@ -0,0 +1,686 @@ +set('workflows.v2.compatibility.current', 'build-a'); + config() + ->set('workflows.v2.compatibility.supported', ['build-a']); + config() + ->set('workflows.v2.workflow_task_lease_seconds', 10); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + public function testCleanupIsPreparedAfterDeliveryAndCompletesBeforeTheOriginalDeadline(): void + { + [$run, $task, $descriptor] = $this->cleanupClaim(); + $prepared = $this->prepare($task, $descriptor); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $deadline = $run->cancellation_deadline_at->toISOString(); + $this->assertSame($deadline, $prepared['start_to_close_deadline_at']); + $this->assertSame($deadline, $prepared['schedule_to_close_deadline_at']); + $this->assertSame($run->cancellation_request_command_id, $prepared['cancellation_cleanup']['root_request_id']); + $this->assertSame( + $descriptor['cancellation_cleanup']['delivery_history_event_id'], + $prepared['cancellation_cleanup']['delivery_history_event_id'] + ); + $started = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted)->sole(); + $this->assertCleanupSnapshot( + $prepared['cancellation_cleanup'], + $started->payload['local_preparation']['cancellation_cleanup'] + ); + $inspection = $run->fresh(); + $this->assertTrue(PortableLocalActivityCleanup::belongsToRunRequest( + $inspection, + $prepared['cancellation_cleanup'], + )); + $this->assertFalse($inspection->relationLoaded('historyEvents')); + $this->assertTrue( + $this->bridge() + ->controlLocalActivity($prepared['activity_attempt_id'], 'owner', 7, true, '1.20')['active'] + ); + $result = $this->outcome($prepared['activity_attempt_id']); + $this->assertTrue($result['recorded'], $result['reason'] ?? ''); + $this->assertFalse($result['claim_released']); + $this->assertTrue($this->bridge()->complete($task->id, [[ + 'type' => 'complete_workflow', + 'output' => Serializer::serializeWithCodec('avro', 'cleaned'), + 'payload_codec' => 'avro', + ]])['completed']); + $this->assertSame(RunStatus::Cancelled, $run->refresh()->status); + $this->assertSame($deadline, $run->cancellation_deadline_at->toISOString()); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered)->count() + ); + } + + public function testCancellationBeforeDeliveryCannotAdmitCleanupOrCheckpointItsPrefix(): void + { + [$run, $task] = $this->newClaim(); + WorkflowStub::loadRun($run->id)->requestCancellation('stop', 30); + $descriptor = $this->descriptor(); + $descriptor['cancellation_cleanup'] = [ + 'request_id' => $run->refresh() +->cancellation_request_command_id, + 'delivery_history_event_id' => 'not-delivered', + ]; + $this->assertSame('local_activity_cleanup_authority_mismatch', $this->prepare($task, $descriptor)['reason']); + $this->assertSame('cancellation_requested', $this->bridge()->checkpointLocalActivityPrefix( + $task->id, + 'owner', + 7, + 'before-delivery', + 1, + [], + '1.20' + )['reason']); + $this->assertSame(0, ActivityExecution::query()->count()); + } + + public function testHistoryRecordingMayFollowTheDeliveryStateTimestamp(): void + { + [$run, $task, $descriptor] = $this->cleanupClaim(); + $delivery = $run->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered)->sole(); + $delivery->forceFill([ + 'recorded_at' => $run->cancellation_delivered_at->copy() + ->addMicroseconds(200), + ])->save(); + Carbon::setTestNow(now()->addMillisecond()); + $prepared = $this->prepare($task, $descriptor); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $this->assertTrue( + $this->bridge() + ->controlLocalActivity($prepared['activity_attempt_id'], 'owner', 7, false, '1.20')['active'] + ); + $this->assertTrue($this->outcome($prepared['activity_attempt_id'])['recorded']); + } + + #[DataProvider('invalidProofs')] + public function testCleanupAdmissionRejectsChangedOrInventedAuthority(string $mutation, string $reason): void + { + [$run, $task, $descriptor] = $this->cleanupClaim(); + $sequence = 2; + match ($mutation) { + 'missing' => $descriptor = $this->descriptor(), + 'wrong request' => $descriptor['cancellation_cleanup']['request_id'] = 'another-request', + 'wrong delivery' => $descriptor['cancellation_cleanup']['delivery_history_event_id'] = 'another-delivery', + 'client deadline' => $descriptor['cancellation_cleanup']['cleanup_deadline_at'] = now()->addHour()->toISOString(), + 'before boundary' => $sequence = 1, + 'changed root budget' => $run->forceFill([ + 'cancellation_deadline_at' => now() + ->addHour(), + ])->save(), + 'expired root budget' => Carbon::setTestNow(now()->addSeconds(30)), + }; + if ($mutation === 'expired root budget') { + $task->forceFill([ + 'lease_expires_at' => now() + ->addSeconds(10), + ])->save(); + } + $reply = $this->prepare($task, $descriptor, sequence: $sequence); + $this->assertFalse($reply['prepared']); + $this->assertSame($reason, $reply['reason']); + $this->assertSame(0, ActivityExecution::query()->count()); + } + + public static function invalidProofs(): iterable + { + yield 'explicit shielding required' => ['missing', 'cancellation_requested']; + yield 'local request identity' => ['wrong request', 'local_activity_cleanup_authority_mismatch']; + yield 'canonical delivery identity' => ['wrong delivery', 'local_activity_cleanup_authority_mismatch']; + yield 'runtime owns deadline' => ['client deadline', 'invalid_local_activity_preparation']; + yield 'delivery consumes authored boundary' => ['before boundary', 'local_activity_cleanup_authority_mismatch']; + yield 'immutable root deadline' => ['changed root budget', 'local_activity_cleanup_authority_mismatch']; + yield 'no admission at deadline' => ['expired root budget', 'cancellation_deadline_expired']; + } + + public function testACommittedCleanupPrefixIsReplayedBeforeLocalAdmission(): void + { + [$run, $task, $descriptor] = $this->cleanupClaim(); + $prefix = [[ + 'type' => 'record_side_effect', + 'marker_id' => 'cleanup-marker', + 'result' => Serializer::serializeWithCodec('avro', 'once'), + 'payload_codec' => 'avro', + ]]; + $first = $this->bridge() + ->checkpointLocalActivityPrefix($task->id, 'owner', 7, 'cleanup-prefix', 2, $prefix, '1.20'); + $this->assertTrue($first['checkpointed'], $first['reason'] ?? ''); + $this->assertSame(3, $first['next_sequence']); + $this->assertTrue($this->bridge()->checkpointLocalActivityPrefix( + $task->id, + 'owner', + 7, + 'cleanup-prefix', + 2, + $prefix, + '1.20' + )['duplicate']); + $this->assertTrue($this->prepare($task, $descriptor, sequence: 3)['prepared']); + $this->assertSame(1, $run->historyEvents()->where('event_type', HistoryEventType::SideEffectRecorded)->count()); + $this->assertSame(3, ActivityExecution::query()->sole()->sequence); + } + + public function testOnlyAnApplicationHeartbeatMovesTheHeartbeatDeadline(): void + { + [$run, $task] = $this->newClaim(); + $prepared = $this->prepare($task, [ + ...$this->descriptor(), + 'heartbeat_timeout' => 3, + ], sequence: 1); + $this->assertTrue($prepared['prepared']); + $attempt = ActivityAttempt::query()->findOrFail($prepared['activity_attempt_id']); + $execution = ActivityExecution::query()->findOrFail($prepared['activity_execution_id']); + $lease = $attempt->lease_expires_at->toISOString(); + $fixed = [ + $execution->close_deadline_at->toISOString(), + $execution->schedule_to_close_deadline_at->toISOString(), + ]; + Carbon::setTestNow(now()->addSeconds(2)); + $heartbeat = $this->bridge() + ->heartbeatLocalActivity($attempt->id, 'owner', 7, [ + 'message' => 'step complete', + ], '1.20'); + $this->assertTrue($heartbeat['active']); + $this->assertTrue($heartbeat['heartbeat_recorded']); + $this->assertFalse($heartbeat['renewed']); + $this->assertSame($lease, $heartbeat['lease_expires_at']); + $this->assertSame($lease, $task->refresh()->lease_expires_at->toISOString()); + $this->assertSame( + now() + ->addSeconds(3) + ->toISOString(), + $execution->refresh() + ->heartbeat_deadline_at->toISOString() + ); + $this->assertSame( + $fixed, + [$execution->close_deadline_at->toISOString(), $execution->schedule_to_close_deadline_at->toISOString()] + ); + $event = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityHeartbeatRecorded)->sole(); + $this->assertSame([ + 'message' => 'step complete', + ], $event->payload['progress']); + $this->assertTrue($event->payload['local_activity']); + $this->assertSame($event->id, $heartbeat['heartbeat_history_event_id']); + Carbon::setTestNow(now()->addSeconds(2)); + $this->assertTrue($this->bridge()->controlLocalActivity($attempt->id, 'owner', 7, true, '1.20')['active']); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityHeartbeatRecorded)->count() + ); + } + + public function testAnApplicationHeartbeatCannotReviveAnExpiredTimeout(): void + { + [, $task] = $this->newClaim(); + $prepared = $this->prepare($task, [ + ...$this->descriptor(), + 'heartbeat_timeout' => 3, + ], sequence: 1); + Carbon::setTestNow(now()->addSeconds(3)); + $reply = $this->bridge() + ->heartbeatLocalActivity($prepared['activity_attempt_id'], 'owner', 7, [], '1.20'); + $this->assertSame('local_activity_deadline_expired', $reply['reason']); + $this->assertFalse($reply['heartbeat_recorded']); + } + + public function testCleanupHeartbeatsAndSupervisorRenewalsCannotExtendTheRootBudget(): void + { + [$run, $task, $descriptor] = $this->cleanupClaim(); + $descriptor['heartbeat_timeout'] = 60; + $prepared = $this->prepare($task, $descriptor); + $deadline = $run->cancellation_deadline_at->toISOString(); + for ($i = 0; $i < 3; ++$i) { + Carbon::setTestNow(now()->addSeconds(9)); + $control = $this->bridge() + ->controlLocalActivity($prepared['activity_attempt_id'], 'owner', 7, true, '1.20'); + $this->assertTrue($control['active']); + } + $this->assertSame($deadline, $control['lease_expires_at']); + $this->assertSame($deadline, $control['workflow_lease_expires_at']); + $heartbeat = $this->bridge() + ->heartbeatLocalActivity($prepared['activity_attempt_id'], 'owner', 7, [], '1.20'); + $this->assertTrue($heartbeat['heartbeat_recorded']); + foreach (['heartbeat_deadline_at', 'start_to_close_deadline_at', 'schedule_to_close_deadline_at'] as $field) { + $this->assertSame($deadline, $heartbeat[$field]); + } + $request = $run->cancellation_request_command_id; + $this->assertTrue(WorkflowStub::loadRun($run->id)->requestCancellation('duplicate', 600)->accepted()); + $this->assertSame($request, $run->refresh()->cancellation_request_command_id); + $this->assertSame($deadline, $run->cancellation_deadline_at->toISOString()); + Carbon::setTestNow(now()->addSeconds(3)); + $outcome = $this->outcome($prepared['activity_attempt_id']); + $this->assertSame('cancellation_deadline_expired', $outcome['reason']); + $this->assertTrue($outcome['fenced']); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCompleted)->count()); + $stop = $this->bridge() + ->controlLocalActivity($prepared['activity_attempt_id'], 'owner', 7, true, '1.20'); + $this->assertSame('cancellation_deadline_expired', $stop['reason']); + $this->assertTrue($stop['stop_required']); + $this->assertFalse($stop['renewed']); + $this->assertTrue($this->bridge()->acknowledgeLocalActivityCancellation( + $prepared['activity_attempt_id'], + 'owner', + $request, + 7, + '1.20' + )['acknowledged']); + } + + #[DataProvider('ordinaryLeaseDurations')] + public function testCleanupCanBeRecoveredBeforeItsDeadlineWithNormalRuntimeLeases(int $leaseSeconds): void + { + config()->set('workflows.v2.workflow_task_lease_seconds', $leaseSeconds); + [$run, $task, $descriptor] = $this->cleanupClaim($leaseSeconds); + $deadline = $run->cancellation_deadline_at->toISOString(); + $recoveryWindow = min(10, $leaseSeconds); + $this->assertSame(now()->addSeconds($recoveryWindow)->toISOString(), $task->lease_expires_at->toISOString()); + $this->assertSame($leaseSeconds, \Workflow\V2\Support\WorkflowTaskLease::seconds()); + $descriptor['retry_policy'] = [ + 'max_attempts' => 2, + 'backoff_seconds' => [0], + ]; + $prepared = $this->prepare($task, $descriptor); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $this->assertSame($task->lease_expires_at->toISOString(), $prepared['lease_expires_at']); + Carbon::setTestNow(now()->addSeconds(2)); + $renewed = $this->bridge() + ->controlLocalActivity($prepared['activity_attempt_id'], 'owner', 7, true, '1.20'); + $this->assertTrue($renewed['active'], $renewed['reason'] ?? ''); + $this->assertSame(now()->addSeconds($recoveryWindow)->toISOString(), $renewed['lease_expires_at']); + $this->assertSame($renewed['lease_expires_at'], $renewed['workflow_lease_expires_at']); + $heartbeat = app(\Workflow\V2\Contracts\WorkflowTaskBridge::class)->heartbeat($task->id); + $this->assertTrue($heartbeat['renewed']); + $this->assertSame($renewed['lease_expires_at'], $heartbeat['lease_expires_at']); + Carbon::setTestNow(now()->addSeconds($recoveryWindow + 1)); + $this->assertSame('workflow_claim_expired', $this->bridge()->controlLocalActivity( + $prepared['activity_attempt_id'], + 'owner', + 7, + true, + '1.20' + )['reason']); + $repaired = \Workflow\V2\Support\TaskRepair::recoverExistingTask($task->refresh(), $run->refresh()); + $this->assertInstanceOf(WorkflowTask::class, $repaired); + $claim = app(\Workflow\V2\Contracts\WorkflowTaskBridge::class)->claimStatus($task->id, 'replacement'); + $this->assertTrue($claim['claimed'], $claim['reason'] ?? ''); + $this->assertSame(now()->addSeconds($recoveryWindow)->toISOString(), $claim['lease_expires_at']); + $recovered = $this->bridge() + ->recoverLocalActivity($task->id, 'replacement', 8, 2, $descriptor, '1.20'); + $this->assertTrue($recovered['recovered'], $recovered['reason'] ?? ''); + $this->assertSame('unknown', $recovered['callback_stop_state']); + $retryTask = WorkflowTask::query()->findOrFail($recovered['created_task_ids'][0]); + $this->assertTrue(app(\Workflow\V2\Contracts\WorkflowTaskBridge::class) + ->claimStatus($retryTask->id, 'replacement')['claimed']); + $retry = $this->bridge() + ->prepareLocalActivity($retryTask->id, 'replacement', 1, 2, 'replacement-local', $descriptor, '1.20'); + $this->assertTrue($retry['prepared'], $retry['reason'] ?? ''); + $this->assertCleanupSnapshot($prepared['cancellation_cleanup'], $retry['cancellation_cleanup']); + $this->assertTrue($this->bridge()->recordLocalActivityOutcome($retry['activity_attempt_id'], 'replacement', 1, [ + 'outcome' => 'completed', + 'result' => Serializer::serializeWithCodec('avro', 'cleaned'), + 'payload_codec' => 'avro', + ], '1.20')['recorded']); + $this->assertTrue(app(\Workflow\V2\Contracts\WorkflowTaskBridge::class)->complete($retryTask->id, [[ + 'type' => 'complete_workflow', + 'output' => Serializer::serializeWithCodec('avro', 'cleaned'), + 'payload_codec' => 'avro', + ]])['completed']); + $this->assertSame(RunStatus::Cancelled, $run->refresh()->status); + $this->assertSame($deadline, $run->cancellation_deadline_at->toISOString()); + $this->assertTrue(now()->lt($run->cancellation_deadline_at)); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered)->count() + ); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityHeartbeatRecorded)->count() + ); + } + + public static function ordinaryLeaseDurations(): iterable + { + yield 'Server normal lease' => [60]; + yield 'longer configured lease' => [300]; + yield 'shorter configured lease' => [4]; + } + + public function testAReplacementRecoversCleanupWithoutRedeliveryOrAResetDeadline(): void + { + [$run, $task, $descriptor] = $this->cleanupClaim(); + $descriptor['retry_policy'] = [ + 'max_attempts' => 2, + 'backoff_seconds' => [0], + ]; + $prepared = $this->prepare($task, $descriptor); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $deadline = $run->cancellation_deadline_at->toISOString(); + Carbon::setTestNow(now()->addSeconds(11)); + $task->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 8, + 'lease_expires_at' => now() + ->addSeconds(10), + ])->save(); + $this->assertSame('workflow_claim_mismatch', $this->bridge()->controlLocalActivity( + $prepared['activity_attempt_id'], + 'owner', + 7, + true, + '1.20' + )['reason']); + $this->assertSame('workflow_claim_mismatch', $this->outcome($prepared['activity_attempt_id'])['reason']); + $recovered = $this->bridge() + ->recoverLocalActivity($task->id, 'replacement', 8, 2, $descriptor, '1.20'); + $this->assertTrue($recovered['recovered'], $recovered['reason'] ?? ''); + $this->assertSame('unknown', $recovered['callback_stop_state']); + $retryTask = WorkflowTask::query()->findOrFail($recovered['created_task_ids'][0]); + $retryTask->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'replacement', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addSeconds(10), + ])->save(); + $retry = $this->bridge() + ->prepareLocalActivity($retryTask->id, 'replacement', 1, 2, 'replacement-local', $descriptor, '1.20'); + $this->assertTrue($retry['prepared'], $retry['reason'] ?? ''); + $this->assertSame(2, $retry['attempt_number']); + $this->assertNotSame($prepared['activity_attempt_id'], $retry['activity_attempt_id']); + $this->assertCleanupSnapshot($prepared['cancellation_cleanup'], $retry['cancellation_cleanup']); + $this->assertSame($deadline, $retry['start_to_close_deadline_at']); + $this->assertSame($deadline, $retry['schedule_to_close_deadline_at']); + $completed = $this->bridge() + ->recordLocalActivityOutcome($retry['activity_attempt_id'], 'replacement', 1, [ + 'outcome' => 'completed', + 'result' => Serializer::serializeWithCodec('avro', 'resumed cleanup'), + 'payload_codec' => 'avro', + ], '1.20'); + $this->assertTrue($completed['recorded'], $completed['reason'] ?? ''); + $this->assertTrue($this->bridge()->complete($retryTask->id, [[ + 'type' => 'complete_workflow', + 'output' => Serializer::serializeWithCodec('avro', 'cleaned'), + 'payload_codec' => 'avro', + ]])['completed']); + $this->assertSame(RunStatus::Cancelled, $run->refresh()->status); + $this->assertTrue(now()->lt($run->cancellation_deadline_at)); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered)->count() + ); + $this->assertSame($deadline, $run->refresh()->cancellation_deadline_at->toISOString()); + } + + public function testRecoveryAndNewPrefixWorkStopAtTheOriginalDeadline(): void + { + [$run, $task, $descriptor] = $this->cleanupClaim(); + $prepared = $this->prepare($task, $descriptor); + $this->assertTrue($prepared['prepared']); + Carbon::setTestNow(now()->addSeconds(30)); + $task->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 8, + 'lease_expires_at' => now() + ->addSeconds(10), + ])->save(); + $before = $run->historyEvents() + ->count(); + $recovery = $this->bridge() + ->recoverLocalActivity($task->id, 'replacement', 8, 2, $descriptor, '1.20'); + $this->assertSame('cancellation_deadline_expired', $recovery['reason']); + $this->assertFalse($recovery['recovered']); + $this->assertSame([], $recovery['created_task_ids']); + $prefix = $this->bridge() + ->checkpointLocalActivityPrefix($task->id, 'replacement', 8, 'late-prefix', 3, [], '1.20'); + $this->assertSame('cancellation_deadline_expired', $prefix['reason']); + $this->assertSame($before, $run->historyEvents()->count()); + } + + public function testMalformedApplicationProgressCannotRenewOrRecordHeartbeat(): void + { + [, $task] = $this->newClaim(); + $prepared = $this->prepare($task, $this->descriptor(), sequence: 1); + $execution = ActivityExecution::query()->findOrFail($prepared['activity_execution_id']); + $before = $execution->getAttributes(); + $reply = $this->bridge() + ->heartbeatLocalActivity($prepared['activity_attempt_id'], 'owner', 7, [ + 'invented' => 'field', + ], '1.20'); + $this->assertSame('invalid_local_activity_heartbeat', $reply['reason']); + $this->assertFalse($reply['heartbeat_recorded']); + $this->assertFalse($reply['renewed']); + $this->assertSame($before, $execution->refresh()->getAttributes()); + } + + public function testACompletedReceiptRemainsReadableAfterTheRootDeadline(): void + { + [, $task, $descriptor] = $this->cleanupClaim(); + $prepared = $this->prepare($task, $descriptor); + $first = $this->outcome($prepared['activity_attempt_id']); + $this->assertTrue($first['recorded']); + Carbon::setTestNow(now()->addSeconds(31)); + $duplicate = $this->outcome($prepared['activity_attempt_id']); + $this->assertTrue($duplicate['recorded']); + $this->assertTrue($duplicate['duplicate']); + $this->assertSame($first['event_id'], $duplicate['event_id']); + } + + public function testAFailedHeartbeatWriteRollsBackTheDeadlineAndHistory(): void + { + [, $task] = $this->newClaim(); + $prepared = $this->prepare($task, [ + ...$this->descriptor(), + 'heartbeat_timeout' => 3, + ], sequence: 1); + $execution = ActivityExecution::query()->findOrFail($prepared['activity_execution_id']); + $before = $execution->getAttributes(); + $event = 'eloquent.saving: ' . ActivityAttempt::class; + Event::listen($event, static function (ActivityAttempt $saved): void { + if ($saved->isDirty('last_heartbeat_at')) { + throw new RuntimeException('synthetic heartbeat write failure'); + } + }); + Carbon::setTestNow(now()->addSecond()); + try { + $this->bridge() + ->heartbeatLocalActivity($prepared['activity_attempt_id'], 'owner', 7, [], '1.20'); + $this->fail('The write failure must propagate.'); + } catch (RuntimeException $error) { + $this->assertSame('synthetic heartbeat write failure', $error->getMessage()); + } finally { + Event::forget($event); + } + $this->assertSame($before, $execution->refresh()->getAttributes()); + $this->assertSame( + 0, + $execution->run->historyEvents() + ->where('event_type', HistoryEventType::ActivityHeartbeatRecorded)->count() + ); + } + + public function testCleanupGroupAdmitsPendingMembersUnderTheOriginalRootDeadline(): void + { + [$run, $task, $descriptor] = $this->cleanupClaim(); + $deadline = $run->cancellation_deadline_at; + $commands = []; + for ($index = 0; $index < 2; ++$index) { + $commands[] = [ + ...$descriptor, + 'type' => 'prepare_local_activity', + ...\Workflow\V2\Support\ParallelChildGroup::itemMetadata(2, 2, $index, 'activity'), + ]; + } + $reply = app(\Workflow\V2\Contracts\PreparedLocalActivityGroupTaskBridge::class) + ->checkpointLocalActivityGroup($task->id, 'owner', 7, 'cleanup-group', 2, $commands, '1.20'); + $this->assertTrue($reply['checkpointed'], $reply['reason'] ?? ''); + $this->assertCount(2, $reply['local_activities']); + $this->assertSame(0, ActivityAttempt::query()->count()); + Carbon::setTestNow(now()->addSeconds(5)); + foreach ($commands as $index => $command) { + $prepared = $this->prepare($task, [ + ...$command, + 'type' => 'record_local_activity', + ], sequence: $index + 2); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $this->assertEquals($deadline, Carbon::parse($prepared['schedule_to_close_deadline_at'])); + $this->assertSame($run->cancellation_request_command_id, $prepared['cancellation_cleanup']['request_id']); + $outcome = $this->outcome($prepared['activity_attempt_id']); + $this->assertTrue($outcome['recorded'], $outcome['reason'] ?? ''); + } + $this->assertSame(2, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCompleted)->count()); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered)->count() + ); + } + + private function assertCleanupSnapshot(array $expected, array $actual): void + { + ksort($expected); + ksort($actual); + $this->assertSame($expected, $actual); + } + + private function bridge(): PreparedLocalActivityTaskBridge + { + return app(PreparedLocalActivityTaskBridge::class); + } + + private function prepare(WorkflowTask $task, array $descriptor, int $sequence = 2): array + { + return $this->bridge() + ->prepareLocalActivity($task->id, 'owner', 7, $sequence, 'local-attempt', $descriptor, '1.20'); + } + + private function outcome(string $attemptId): array + { + return $this->bridge() + ->recordLocalActivityOutcome($attemptId, 'owner', 7, [ + 'outcome' => 'completed', + 'result' => Serializer::serializeWithCodec('avro', 'cleaned'), + 'payload_codec' => 'avro', + ], '1.20'); + } + + private function cleanupClaim(int $leaseSeconds = 10): array + { + [$run, $task] = $this->newClaim($leaseSeconds); + $this->assertTrue(WorkflowStub::loadRun($run->id)->requestCancellation('stop', 30)->accepted()); + $run->refresh(); + $delivery = app(CooperativeWorkflowTaskBridge::class)->deliverCancellation( + $task->id, + $run->cancellation_request_command_id, + 1, + 'timer' + ); + $this->assertTrue($delivery['delivered'], $delivery['reason'] ?? ''); + $descriptor = $this->descriptor(); + $descriptor['cancellation_cleanup'] = [ + 'request_id' => $run->cancellation_request_command_id, + 'delivery_history_event_id' => $run->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered)->sole()->id, + ]; + + return [$run->refresh(), $task->refresh(), $descriptor]; + } + + private function descriptor(): array + { + return [ + 'type' => 'record_local_activity', + 'activity_type' => 'opaque-cleanup', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'start_to_close_timeout' => 60, + 'schedule_to_close_timeout' => 120, + ]; + } + + private function newClaim(int $leaseSeconds = 10): array + { + $instance = WorkflowInstance::query()->create([ + 'workflow_class' => TestGreetingWorkflow::class, + 'workflow_type' => 'portable-parent', + ]); + $run = WorkflowRun::query()->create([ + 'workflow_instance_id' => $instance->id, + 'run_number' => 1, + 'workflow_class' => TestGreetingWorkflow::class, + 'workflow_type' => 'portable-parent', + 'status' => RunStatus::Waiting, + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'connection' => 'database', + 'queue' => 'default', + 'compatibility' => 'build-a', + 'started_at' => now(), + ]); + $instance->forceFill([ + 'current_run_id' => $run->id, + ])->save(); + $task = WorkflowTask::query()->create([ + 'workflow_run_id' => $run->id, + 'task_type' => TaskType::Workflow, + 'status' => TaskStatus::Leased, + 'attempt_count' => 7, + 'payload' => [], + 'connection' => 'database', + 'queue' => 'default', + 'compatibility' => 'build-a', + 'lease_owner' => 'owner', + 'lease_expires_at' => now() + ->addSeconds($leaseSeconds), + ]); + + return [$run, $task]; + } +} diff --git a/tests/Feature/V2/V2PortableLocalActivityControlTest.php b/tests/Feature/V2/V2PortableLocalActivityControlTest.php new file mode 100644 index 000000000..d9c95b56b --- /dev/null +++ b/tests/Feature/V2/V2PortableLocalActivityControlTest.php @@ -0,0 +1,601 @@ +set('workflows.v2.compatibility.current', 'build-a'); + config() + ->set('workflows.v2.compatibility.supported', ['build-a']); + config() + ->set('workflows.v2.workflow_task_lease_seconds', 10); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + public function testPollingAnActivePreparedCallbackDoesNotWriteOrRenew(): void + { + [$run, $task, $attempt, $execution] = $this->prepared(); + $before = $this->snapshot($run, $task, $attempt, $execution); + Carbon::setTestNow(now()->addSecond()); + $status = $this->control($attempt, renew: false); + $this->assertTrue($status['active']); + $this->assertFalse($status['stop_required']); + $this->assertFalse($status['renewed']); + $this->assertSame(7, $status['workflow_task_attempt']); + $this->assertSame($task->id, $status['workflow_task_id']); + $this->assertSame($before, $this->snapshot($run, $task, $attempt, $execution)); + } + + public function testSupervisorRenewalKeepsBothLeasesAliveWithoutAnApplicationHeartbeat(): void + { + [$run, $task, $attempt, $execution] = $this->prepared(); + $executionBefore = $execution->getAttributes(); + $heartbeat = $attempt->last_heartbeat_at->toISOString(); + $original = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted)->sole()->payload; + $this->assertSame($task->id, WorkflowRunSummary::query()->findOrFail($run->id)->next_task_id); + foreach ([4, 8, 8, 8] as $seconds) { + Carbon::setTestNow(now()->addSeconds($seconds)); + $status = $this->control($attempt); + $this->assertTrue($status['active']); + $this->assertTrue($status['renewed']); + $this->assertSame(now()->addSeconds(10)->toISOString(), $status['lease_expires_at']); + $this->assertSame($status['lease_expires_at'], $status['workflow_lease_expires_at']); + $this->assertSame( + $status['lease_expires_at'], + WorkflowRunSummary::query()->findOrFail($run->id)->next_task_lease_expires_at->toISOString() + ); + $this->assertSame($heartbeat, $attempt->refresh()->last_heartbeat_at->toISOString()); + $this->assertSame($executionBefore, $execution->refresh()->getAttributes()); + } + $this->assertSame(2, $run->historyEvents()->count()); + $this->assertSame( + $original, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted)->sole()->payload + ); + $this->assertSame(7, $task->refresh()->attempt_count); + $this->assertSame(1, $attempt->refresh()->attempt_number); + } + + public function testSupervisorPollingCannotResetAnApplicationHeartbeatTimeout(): void + { + [$run, $task, $attempt, $execution] = $this->prepared([ + 'heartbeat_timeout' => 3, + ]); + Carbon::setTestNow(now()->addSeconds(2)); + $this->assertTrue($this->control($attempt)['renewed']); + $before = $this->snapshot($run, $task, $attempt, $execution); + Carbon::setTestNow(now()->addSecond()); + $status = $this->control($attempt); + $this->assertSame('local_activity_deadline_expired', $status['reason']); + $this->assertTrue($status['stop_required']); + $this->assertFalse($status['renewed']); + $this->assertSame($before, $this->snapshot($run, $task, $attempt, $execution)); + } + + public function testSupervisorRenewalDoesNotMoveTheOriginalTotalDeadline(): void + { + [$run, $task, $attempt, $execution] = $this->prepared([ + 'start_to_close_timeout' => null, + 'schedule_to_close_timeout' => 3, + ]); + Carbon::setTestNow(now()->addSeconds(2)); + $this->assertTrue($this->control($attempt)['renewed']); + $before = $this->snapshot($run, $task, $attempt, $execution); + Carbon::setTestNow(now()->addSecond()); + $status = $this->control($attempt); + $this->assertSame('local_activity_deadline_expired', $status['reason']); + $this->assertTrue($status['stop_required']); + $this->assertFalse($status['renewed']); + $this->assertSame($before, $this->snapshot($run, $task, $attempt, $execution)); + } + + public function testFailedAttemptWriteRollsBackTheHostingClaimRenewal(): void + { + [$run, $task, $attempt, $execution] = $this->prepared(); + $before = $this->snapshot($run, $task, $attempt, $execution); + Carbon::setTestNow(now()->addSecond()); + $summaryBefore = WorkflowRunSummary::query()->findOrFail($run->id)->getAttributes(); + $event = 'eloquent.saving: ' . ActivityAttempt::class; + Event::listen($event, static function (ActivityAttempt $saved) use ($attempt): void { + if ($saved->id === $attempt->id && $saved->isDirty('lease_expires_at')) { + throw new RuntimeException('synthetic attempt write failure'); + } + }); + try { + $this->control($attempt); + $this->fail('The synthetic write failure must propagate.'); + } catch (RuntimeException $exception) { + $this->assertSame('synthetic attempt write failure', $exception->getMessage()); + } finally { + Event::forget($event); + } + $this->assertSame($before, $this->snapshot($run, $task, $attempt, $execution)); + $this->assertSame($summaryBefore, WorkflowRunSummary::query()->findOrFail($run->id)->getAttributes()); + } + + public function testCancellationPollingFencesWithoutRenewingOrInventingAStopReceipt(): void + { + [$run, $task, $attempt] = $this->prepared(); + $result = WorkflowStub::loadRun($run->id)->requestCancellation('stop both', 30); + $this->assertTrue($result->accepted()); + $context = $result->cancellationContext() + ->toArray(); + $taskBefore = $task->refresh() + ->getAttributes(); + $status = $this->control($attempt); + $this->assertSame('cancellation_requested', $status['reason']); + $this->assertSame($context, $status['cancellation_request']); + $this->assertFalse($status['renewed']); + $this->assertTrue($status['stop_required']); + $this->assertTrue($status['fenced']); + $this->assertSame(ActivityAttemptStatus::Cancelled, $attempt->refresh()->status); + $this->assertNull($attempt->lease_expires_at); + $this->assertSame($taskBefore, $task->refresh()->getAttributes()); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancellationAcknowledged)->count() + ); + $again = $this->control($attempt); + $this->assertSame($status['cancellation_history_event_id'], $again['cancellation_history_event_id']); + $this->assertSame(1, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCancelled)->count()); + $this->assertSame( + $context, + WorkflowStub::loadRun($run->id)->requestCancellation('another reason', 600)->cancellationContext() + ->toArray() + ); + } + + public function testOriginalSupervisorCanObserveAndAcknowledgeCancellationAfterTakeover(): void + { + [$run, $task, $attempt] = $this->prepared(); + $request = WorkflowStub::loadRun($run->id)->requestCancellation('stop', 30)->cancellationContext(); + $task->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'replacement-worker', + 'attempt_count' => 8, + 'lease_expires_at' => now() + ->addMinute(), + ])->save(); + $before = $task->refresh() + ->getAttributes(); + $status = $this->control($attempt); + $this->assertSame('cancellation_requested', $status['reason']); + $this->assertSame(7, $status['workflow_task_attempt']); + $this->assertSame($request->toArray(), $status['cancellation_request']); + $this->assertFalse($status['renewed']); + $this->assertSame($before, $task->refresh()->getAttributes()); + $ack = $this->bridge() + ->acknowledgeLocalActivityCancellation($attempt->id, 'original-worker', $request->requestId, 7, '1.20'); + $this->assertTrue($ack['acknowledged']); + $this->assertSame($before, $task->refresh()->getAttributes()); + $receipt = WorkflowHistoryEvent::query()->findOrFail($ack['history_event_id']); + $this->assertSame($request->rootRequestId, $receipt->payload['root_request_id']); + $this->assertSame($request->deadline()->toISOString(), $receipt->payload['cleanup_deadline_at']); + $this->assertFalse($receipt->payload['received_after_deadline']); + } + + #[DataProvider('localCancellationPolicies')] + public function testLocalPolicyDistinguishesImmediateDeliveryFromJoinedCallbackWaiting(?string $policy): void + { + [$run, $task, $attempt] = $this->prepared($policy === null ? [] : [ + 'cancellation_policy' => $policy, + ]); + $context = WorkflowStub::loadRun($run->id)->requestCancellation('stop local work', 30)->cancellationContext(); + $workflowBridge = app(CooperativeWorkflowTaskBridge::class); + $reply = $workflowBridge->deliverCancellation($task->id, $context->requestId, 1, 'local_activity'); + $this->assertSame(ActivityAttemptStatus::Cancelled, $attempt->refresh()->status); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancellationAcknowledged)->count() + ); + if ($policy === 'wait_cancellation_completed') { + $this->assertFalse($reply['delivered']); + $this->assertTrue($reply['claim_released']); + $this->assertSame('cancellation_waiting_for_activity', $reply['reason']); + $this->assertSame(TaskStatus::Completed, $task->refresh()->status); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered)->count() + ); + $wrong = $this->bridge() + ->acknowledgeLocalActivityCancellation($attempt->id, 'another-worker', $context->requestId, 7, '1.20'); + $this->assertFalse($wrong['acknowledged']); + $this->assertSame(0, $run->tasks()->where('status', TaskStatus::Ready)->count()); + Carbon::setTestNow(now()->addSecond()); + $stopped = $this->bridge() + ->acknowledgeLocalActivityCancellation($attempt->id, 'original-worker', $context->requestId, 7, '1.20'); + $this->assertTrue($stopped['acknowledged']); + $receipt = WorkflowHistoryEvent::query()->findOrFail($stopped['history_event_id']); + $this->assertSame($context->rootRequestId, $receipt->payload['root_request_id']); + $this->assertSame($context->deadline()->toISOString(), $receipt->payload['cleanup_deadline_at']); + $successor = $run->tasks() + ->where('status', TaskStatus::Ready)->sole(); + $successor->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'replacement-worker', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addSeconds(5), + ])->save(); + $wrongBoundary = $workflowBridge->deliverCancellation( + $successor->id, + $context->requestId, + 2, + 'local_activity' + ); + $this->assertFalse($wrongBoundary['delivered']); + $reply = $workflowBridge->deliverCancellation($successor->id, $context->requestId, 1, 'local_activity'); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancellationAcknowledged)->count() + ); + } + $this->assertTrue($reply['delivered']); + $this->assertSame(1, $reply['sequence']); + $this->assertSame('local_activity', $reply['call_kind']); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered)->count() + ); + $this->assertSame( + $context->toArray(), + WorkflowStub::loadRun($run->id)->requestCancellation('duplicate', 300)->cancellationContext()->toArray() + ); + $this->assertSame( + $context->deadline() + ->toISOString(), + $run->refresh() + ->cancellation_deadline_at->toISOString() + ); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityHeartbeatRecorded)->count() + ); + } + + public static function localCancellationPolicies(): iterable + { + yield 'historical default' => [null]; + yield 'try cancellation' => ['try_cancel']; + yield 'wait for stop receipt' => ['wait_cancellation_completed']; + } + + public function testLocalPolicyLateStopReceiptCannotResumeCleanupAfterTheOriginalDeadline(): void + { + [$run, $task, $attempt] = $this->prepared([ + 'cancellation_policy' => 'wait_cancellation_completed', + ]); + $context = WorkflowStub::loadRun($run->id)->requestCancellation( + 'bounded local wait', + 30 + )->cancellationContext(); + $delivery = app(CooperativeWorkflowTaskBridge::class) + ->deliverCancellation($task->id, $context->requestId, 1, 'local_activity'); + $this->assertFalse($delivery['delivered']); + $this->assertTrue($delivery['claim_released']); + Carbon::setTestNow($context->deadline()); + $stopped = $this->bridge() + ->acknowledgeLocalActivityCancellation($attempt->id, 'original-worker', $context->requestId, 7, '1.20'); + $this->assertTrue($stopped['acknowledged']); + $receipt = WorkflowHistoryEvent::query()->findOrFail($stopped['history_event_id']); + $this->assertTrue($receipt->payload['received_after_deadline']); + $this->assertSame($context->deadline()->toISOString(), $receipt->payload['cleanup_deadline_at']); + $this->assertSame(0, $run->tasks()->where('status', TaskStatus::Ready)->count()); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered)->count() + ); + } + + public function testADescendantCallbackStartedAfterTheRootRequestStillStopsOnLocalPropagation(): void + { + $parent = WorkflowStub::make(TestGreetingWorkflow::class, 'root'); + $parent->start('Taylor'); + $root = $parent->requestCancellation('stop tree', 30) + ->cancellationContext(); + Carbon::setTestNow(now()->addSecond()); + [$run, $task, $attempt] = $this->prepared(); + WorkflowLink::query()->create([ + 'parent_workflow_instance_id' => $parent->run() +->workflow_instance_id, + 'parent_workflow_run_id' => $parent->runId(), + 'child_workflow_instance_id' => $run->workflow_instance_id, + 'child_workflow_run_id' => $run->id, + 'sequence' => 1, + 'link_type' => 'child_workflow', + 'is_primary_parent' => true, + ]); + Carbon::setTestNow(now()->addSecond()); + $request = WorkflowStub::loadRun($run->id)->attemptRequestCancellationFromParent($parent->runId()); + $this->assertTrue($request->accepted()); + $context = $request->cancellationContext(); + $before = $task->refresh() + ->getAttributes(); + $status = $this->control($attempt); + $this->assertTrue($status['fenced']); + $this->assertTrue($status['stop_required']); + $this->assertSame('cancellation_requested', $status['reason']); + $this->assertSame($root->rootRequestId, $status['cancellation_request']['root_request_id']); + $this->assertNotSame($root->requestId, $context->requestId); + $this->assertSame($root->deadline()->toISOString(), $status['cancellation_request']['cleanup_deadline_at']); + $this->assertSame($before, $task->refresh()->getAttributes()); + $this->assertTrue( + $this->bridge() + ->acknowledgeLocalActivityCancellation( + $attempt->id, + 'original-worker', + $context->requestId, + 7, + '1.20' + )['acknowledged'] + ); + } + + public function testAnExpiredCancellationBudgetStillReturnsItsOriginalIdentityWithoutRenewal(): void + { + [$run, $task, $attempt] = $this->prepared(); + $context = WorkflowStub::loadRun($run->id)->requestCancellation('stop', 30)->cancellationContext()->toArray(); + $before = $task->refresh() + ->getAttributes(); + Carbon::setTestNow(now()->addSeconds(30)); + $status = $this->control($attempt); + $this->assertSame('cancellation_deadline_expired', $status['reason']); + $this->assertTrue($status['stop_required']); + $this->assertFalse($status['renewed']); + $this->assertSame($context, $status['cancellation_request']); + $this->assertSame($before, $task->refresh()->getAttributes()); + $this->assertSame($context['cleanup_deadline_at'], $run->refresh()->cancellation_deadline_at->toISOString()); + } + + #[DataProvider('invalidClaims')] + public function testInvalidOriginalClaimsCannotRenewOrReadCanonicalMetadata(string $owner, int $epoch): void + { + [$run, $task, $attempt, $execution] = $this->prepared(); + $before = $this->snapshot($run, $task, $attempt, $execution); + $status = $this->bridge() + ->controlLocalActivity($attempt->id, $owner, $epoch, true, '1.20'); + $this->assertTrue($status['stop_required']); + $this->assertFalse($status['renewed']); + $this->assertNull($status['activity_execution_id']); + $this->assertSame($before, $this->snapshot($run, $task, $attempt, $execution)); + } + + public static function invalidClaims(): iterable + { + yield 'another owner' => ['someone-else', 7]; + yield 'local attempt number instead of workflow epoch' => ['original-worker', 1]; + yield 'new workflow epoch' => ['original-worker', 8]; + yield 'empty owner' => ['', 7]; + yield 'invalid epoch' => ['original-worker', 0]; + } + + public function testProtocol119RefusesControlBeforeAnyMutation(): void + { + [$run, $task, $attempt, $execution] = $this->prepared(); + $before = $this->snapshot($run, $task, $attempt, $execution); + $status = $this->bridge() + ->controlLocalActivity($attempt->id, 'original-worker', 7, true, '1.19'); + $this->assertSame('local_activity_control_requires_protocol_1_20', $status['reason']); + $this->assertFalse($status['renewed']); + $this->assertSame($before, $this->snapshot($run, $task, $attempt, $execution)); + $this->assertSame( + 'activity_attempt_not_found', + PortableLocalActivityControl::poll('unknown', 'original-worker', 7, true, '1.20')['reason'] + ); + } + + #[DataProvider('lostAuthority')] + public function testLostAuthorityCannotBeRenewed(string $kind, string $reason): void + { + [$run, $task, $attempt, $execution] = $this->prepared(); + match ($kind) { + 'takeover' => $task->forceFill([ + 'lease_owner' => 'replacement-worker', + 'attempt_count' => 8, + ])->save(), + 'same owner new epoch' => $task->forceFill([ + 'attempt_count' => 8, + ])->save(), + 'workflow expiry' => $task->forceFill([ + 'lease_expires_at' => now(), + ])->save(), + 'local expiry' => $attempt->forceFill([ + 'lease_expires_at' => now(), + ])->save(), + 'closed run' => $run->forceFill([ + 'status' => RunStatus::Completed, + ])->save(), + 'run deadline' => $run->forceFill([ + 'run_deadline_at' => now(), + ])->save(), + 'execution deadline' => $run->forceFill([ + 'execution_deadline_at' => now(), + ])->save(), + 'per attempt deadline' => $execution->forceFill([ + 'close_deadline_at' => now(), + ])->save(), + 'counter changed' => $execution->forceFill([ + 'attempt_count' => 2, + ])->save(), + 'missing start' => $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted)->delete(), + 'total budget changed' => $execution->forceFill([ + 'schedule_to_close_deadline_at' => now() + ->addMinutes(3), + ])->save(), + }; + $before = $this->snapshot($run, $task, $attempt, $execution); + $status = $this->control($attempt); + $this->assertSame($reason, $status['reason']); + $this->assertTrue($status['stop_required']); + $this->assertFalse($status['renewed']); + $this->assertSame($before, $this->snapshot($run, $task, $attempt, $execution)); + } + + public static function lostAuthority(): iterable + { + yield 'takeover' => ['takeover', 'workflow_claim_mismatch']; + yield 'same owner new epoch' => ['same owner new epoch', 'workflow_claim_mismatch']; + yield 'workflow expiry' => ['workflow expiry', 'workflow_claim_expired']; + yield 'local expiry despite live hosting claim' => ['local expiry', 'local_activity_lease_expired']; + yield 'closed run' => ['closed run', 'run_closed']; + yield 'run deadline' => ['run deadline', 'run_deadline_expired']; + yield 'execution deadline' => ['execution deadline', 'run_deadline_expired']; + yield 'per attempt deadline' => ['per attempt deadline', 'local_activity_deadline_expired']; + yield 'counter changed' => ['counter changed', 'stale_activity_attempt']; + yield 'missing start' => ['missing start', 'local_activity_preparation_mismatch']; + yield 'total budget changed' => ['total budget changed', 'local_activity_preparation_mismatch']; + } + + public function testAStartAfterCancellationCannotAuthorizeAControlFence(): void + { + [$run, $task, $attempt, $execution] = $this->prepared(); + WorkflowStub::loadRun($run->id)->requestCancellation('stop', 30); + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted)->update([ + 'sequence' => 100, + ]); + $before = $this->snapshot($run, $task, $attempt, $execution); + $this->assertSame('local_activity_preparation_mismatch', $this->control($attempt)['reason']); + $this->assertSame($before, $this->snapshot($run, $task, $attempt, $execution)); + } + + public function testMalformedCancellationContextCannotAuthorizeARenewalOrFence(): void + { + [$run, $task, $attempt, $execution] = $this->prepared(); + WorkflowStub::loadRun($run->id)->requestCancellation('stop', 30); + $request = $run->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationRequested)->sole(); + $payload = $request->payload; + $payload['cancellation']['schema'] = 'unsupported'; + $request->forceFill([ + 'payload' => $payload, + ])->save(); + $before = $this->snapshot($run, $task, $attempt, $execution); + $this->assertSame('cancellation_context_not_recorded', $this->control($attempt)['reason']); + $this->assertSame($before, $this->snapshot($run, $task, $attempt, $execution)); + } + + private function bridge(): PreparedLocalActivityTaskBridge + { + return app(PreparedLocalActivityTaskBridge::class); + } + + private function control(ActivityAttempt $attempt, bool $renew = true): array + { + return $this->bridge() + ->controlLocalActivity($attempt->id, 'original-worker', 7, $renew, '1.20'); + } + + private function snapshot( + WorkflowRun $run, + WorkflowTask $task, + ActivityAttempt $attempt, + ActivityExecution $execution + ): array { + return [$run->refresh()->getAttributes(), $task->refresh()->getAttributes(), + $attempt->refresh() + ->getAttributes(), $execution->refresh() + ->getAttributes(), $run->historyEvents() + ->count()]; + } + + private function prepared(array $options = []): array + { + $instance = WorkflowInstance::query()->create([ + 'workflow_class' => TestGreetingWorkflow::class, + 'workflow_type' => 'portable-parent', + ]); + $run = WorkflowRun::query()->create([ + 'workflow_instance_id' => $instance->id, + 'run_number' => 1, + 'workflow_class' => TestGreetingWorkflow::class, + 'workflow_type' => 'portable-parent', + 'status' => RunStatus::Waiting, + 'arguments' => Serializer::serializeWithCodec('avro', ['Taylor']), + 'payload_codec' => 'avro', + 'connection' => 'database', + 'queue' => 'default', + 'compatibility' => 'build-a', + 'started_at' => now() + ->subMinute(), + ]); + $instance->forceFill([ + 'current_run_id' => $run->id, + ])->save(); + $task = WorkflowTask::query()->create([ + 'workflow_run_id' => $run->id, + 'task_type' => TaskType::Workflow, + 'status' => TaskStatus::Leased, + 'attempt_count' => 7, + 'payload' => [], + 'connection' => 'database', + 'queue' => 'default', + 'compatibility' => 'build-a', + 'lease_owner' => 'original-worker', + 'lease_expires_at' => now() + ->addSeconds(5), + ])->refresh(); + $first = $this->bridge() + ->prepareLocalActivity($task->id, 'original-worker', 7, 1, 'sdk-local-attempt', [ + 'type' => 'record_local_activity', + 'activity_type' => 'php-local-opaque', + 'arguments' => Serializer::serializeWithCodec('avro', ['Taylor']), + 'payload_codec' => 'avro', + 'start_to_close_timeout' => 60, + 'schedule_to_close_timeout' => 120, + ...$options, + ], '1.20'); + $this->assertTrue($first['prepared']); + return [$run->refresh(), $task->refresh(), ActivityAttempt::query()->findOrFail($first['activity_attempt_id']), + ActivityExecution::query()->findOrFail($first['activity_execution_id'])]; + } +} diff --git a/tests/Feature/V2/V2PortableLocalActivityOutcomeTest.php b/tests/Feature/V2/V2PortableLocalActivityOutcomeTest.php new file mode 100644 index 000000000..c28cde5b8 --- /dev/null +++ b/tests/Feature/V2/V2PortableLocalActivityOutcomeTest.php @@ -0,0 +1,780 @@ +set('workflows.v2.compatibility.current', 'build-a'); + config() + ->set('workflows.v2.compatibility.supported', ['build-a']); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + public function testSuccessUsesThePreparedAttemptPreservesAvroBytesAndRetainsTheClaimForReplay(): void + { + [$run, $task, $prepared] = $this->preparedClaim(); + $before = $task->getAttributes(); + $report = $this->success(); + $reply = $this->finish($prepared, $report); + $this->assertTrue($reply['recorded']); + $this->assertFalse($reply['duplicate']); + $this->assertFalse($reply['claim_released']); + $this->assertSame([], $reply['created_task_ids']); + $this->assertSame($prepared['activity_attempt_id'], $reply['activity_attempt_id']); + $this->assertSame('sdk-local-attempt', $reply['worker_attempt_id']); + $this->assertSame($before, $task->refresh()->getAttributes()); + $execution = ActivityExecution::query()->findOrFail($prepared['activity_execution_id']); + $this->assertSame(ActivityStatus::Completed, $execution->status); + $this->assertSame($report['result'], $execution->result); + $this->assertSame([ + 'value' => 'Hello, Taylor!', + ], $execution->activityResult()); + $this->assertSame($prepared['activity_attempt_id'], $execution->current_attempt_id); + $this->assertSame(1, $execution->attempts()->count()); + $this->assertSame(ActivityAttemptStatus::Completed, $execution->attempts()->sole()->status); + $this->assertSame(0, $run->tasks()->where('task_type', TaskType::Activity)->count()); + $this->assertSame(2, WorkflowStepHistory::nextDurableCommandSequence($run->fresh())); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $this->assertSame($report['result'], $event->payload['result']); + $this->assertSame($task->id, $event->payload['activity_attempt']['task_id']); + $this->assertSame('sdk-local-attempt', $event->payload['activity_attempt']['worker_attempt_id']); + foreach (HistoryTimeline::forRun($run->fresh()) as $entry) { + $this->assertSame('workflow', $entry['task']['type']); + $this->assertSame(2, $entry['task']['attempt_count']); + $this->assertSame($entry['type'] === 'ActivityScheduled' ? 0 : 1, $entry['activity']['attempt_count']); + } + $count = WorkflowHistoryEvent::query()->count(); + // This opaque alias has no PHP class. Replay must find the recorded + // result before any application/type admission or callback invocation. + $replayed = app(LocalActivityExecutor::class)->execute( + $run->fresh(), + $task->fresh(), + 1, + new LocalActivityCall('python-local-opaque', ['Taylor']) + ); + $this->assertSame('completed', $replayed['status']); + $this->assertSame($reply['event_id'], $replayed['event']->id); + $this->assertSame($count, WorkflowHistoryEvent::query()->count()); + } + + public function testResponseLossReturnsTheOriginalOutcomeAfterClaimTakeoverAndLeaseExpiry(): void + { + [$run, $task, $prepared] = $this->preparedClaim(); + $first = $this->finish($prepared, $this->success()); + $this->assertTrue($first['recorded']); + $task->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 3, + 'lease_expires_at' => now() + ->subSecond(), + ])->save(); + $before = $task->refresh() + ->getAttributes(); + $count = $run->historyEvents() + ->count(); + $second = $this->finish($prepared, $this->success()); + $this->assertSame([ + ...$first, + 'duplicate' => true, + ], $second); + $this->assertSame($count, $run->historyEvents()->count()); + $this->assertSame($before, $task->refresh()->getAttributes()); + $this->assertSame( + 'local_activity_preparation_mismatch', + app(LocalActivityExecutor::class)->recordPortableOutcome( + $prepared['activity_attempt_id'], + 'replacement', + 3, + $this->success(), + '1.20', + )['reason'] + ); + } + + public function testAChangedOutcomeCannotRelabelTheOriginalReceipt(): void + { + [$run, , $prepared] = $this->preparedClaim(); + $this->assertTrue($this->finish($prepared, $this->success())['recorded']); + $count = $run->historyEvents() + ->count(); + $changed = $this->success(); + $changed['result'] = Serializer::serializeWithCodec('avro', 'different'); + $this->assertSame('local_activity_outcome_mismatch', $this->finish($prepared, $changed)['reason']); + $this->assertSame($count, $run->historyEvents()->count()); + } + + public function testTypedNonRetryableFailureUsesTheExistingFailureRecorderAndRetainsTheClaim(): void + { + [$run, $task, $prepared] = $this->preparedClaim([ + 'retry_policy' => [ + 'max_attempts' => 3, + ], + ]); + $before = $task->getAttributes(); + $report = [ + 'outcome' => 'failed', + 'message' => 'cannot continue', + 'exception_type' => 'python.Fatal', + 'non_retryable' => true, + ]; + $reply = $this->finish($prepared, $report); + $this->assertTrue($reply['recorded']); + $this->assertSame('ActivityFailed', $reply['event_type']); + $this->assertFalse($reply['claim_released']); + $this->assertSame($before, $task->refresh()->getAttributes()); + $failure = $run->failures() + ->sole(); + $this->assertSame('python.Fatal', $failure->exception_class); + $this->assertSame('cannot continue', $failure->message); + $this->assertTrue($failure->non_retryable); + $this->assertSame([ + ...$reply, + 'duplicate' => true, + ], $this->finish($prepared, $report)); + $this->assertSame(1, $run->failures()->count()); + } + + public function testRetryRecordsOneDurableWorkflowRetryAndReleasesTheOriginalClaimAtomically(): void + { + [$run, $task, $prepared] = $this->preparedClaim([ + 'retry_policy' => [ + 'max_attempts' => 2, + 'backoff_seconds' => [3], + ], + 'schedule_to_close_timeout' => 30, + ]); + $report = [ + 'outcome' => 'failed', + 'message' => 'temporary', + 'exception_type' => 'rust.Temporary', + ]; + $reply = $this->finish($prepared, $report); + $this->assertTrue($reply['recorded']); + $this->assertTrue($reply['claim_released']); + $this->assertSame('ActivityRetryScheduled', $reply['event_type']); + $this->assertSame(TaskStatus::Completed, $task->refresh()->status); + $this->assertNull($task->lease_expires_at); + $this->assertCount(1, $reply['created_task_ids']); + $retry = WorkflowTask::query()->findOrFail($reply['created_task_ids'][0]); + $this->assertSame(TaskType::Workflow, $retry->task_type); + $this->assertSame(TaskStatus::Ready, $retry->status); + $this->assertSame($prepared['activity_attempt_id'], $retry->payload['retry_after_attempt_id']); + $this->assertSame(3, $retry->payload['retry_backoff_seconds']); + $execution = ActivityExecution::query()->findOrFail($prepared['activity_execution_id']); + $this->assertSame(ActivityStatus::Pending, $execution->status); + $this->assertSame( + $prepared['schedule_to_close_deadline_at'], + $execution->schedule_to_close_deadline_at->toISOString() + ); + $this->assertSame([ + ...$reply, + 'duplicate' => true, + ], $this->finish($prepared, $report)); + $this->assertSame(2, $run->tasks()->count()); + $this->assertSame(0, $run->tasks()->where('task_type', TaskType::Activity)->count()); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityRetryScheduled)->count() + ); + } + + public function testReclaimedOrExpiredClaimsCannotPublishANewResult(): void + { + [$run, $task, $prepared] = $this->preparedClaim(); + $task->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 3, + ])->save(); + $before = $task->refresh() + ->getAttributes(); + $this->assertSame('workflow_claim_mismatch', $this->finish($prepared, $this->success())['reason']); + $this->assertSame($before, $task->refresh()->getAttributes()); + $task->forceFill([ + 'lease_owner' => 'portable-worker', + 'attempt_count' => 2, + 'lease_expires_at' => now() + ->subSecond(), + ])->save(); + $this->assertSame('workflow_claim_expired', $this->finish($prepared, $this->success())['reason']); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCompleted)->count()); + } + + public function testCancellationFencesTheOriginalAttemptAfterTakeoverWithoutPretendingTheCallbackStopped(): void + { + [$run, $task, $prepared] = $this->preparedClaim(); + $this->assertTrue( + WorkflowStub::load($run->workflow_instance_id)->requestCancellation('maintenance', 30)->accepted() + ); + $deadline = $run->refresh() + ->cancellation_deadline_at->toISOString(); + $task->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 3, + ])->save(); + $before = $task->refresh() + ->getAttributes(); + $reply = $this->finish($prepared, $this->success()); + $this->assertFalse($reply['recorded']); + $this->assertSame('cancellation_requested', $reply['reason']); + $this->assertTrue($reply['fenced']); + $this->assertIsString($reply['cancellation_history_event_id']); + $this->assertSame($reply, $this->finish($prepared, $this->success())); + $this->assertSame($before, $task->refresh()->getAttributes()); + $this->assertSame( + ActivityStatus::Cancelled, + ActivityExecution::query()->findOrFail($prepared['activity_execution_id'])->status + ); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCompleted)->count()); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancellationAcknowledged)->count() + ); + $receipt = ActivityCancellationAcknowledgement::recordLocalStopped( + $prepared['activity_attempt_id'], + 'portable-worker', + $run->cancellation_request_command_id, + 2, + ); + $this->assertTrue($receipt['acknowledged']); + $this->assertSame($deadline, $run->refresh()->cancellation_deadline_at->toISOString()); + $this->assertSame($before, $task->refresh()->getAttributes()); + } + + public function testAResponseLostBeforeLaterCancellationStillReturnsThePriorCommittedSuccess(): void + { + [$run, , $prepared] = $this->preparedClaim(); + $first = $this->finish($prepared, $this->success()); + $this->assertTrue($first['recorded']); + $this->assertTrue( + WorkflowStub::load($run->workflow_instance_id)->requestCancellation('maintenance', 30)->accepted() + ); + $count = $run->historyEvents() + ->count(); + $this->assertSame([ + ...$first, + 'duplicate' => true, + ], $this->finish($prepared, $this->success())); + $this->assertSame($count, $run->historyEvents()->count()); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCancelled)->count()); + } + + public function testMissingCanonicalPreparationCannotBeReplacedByMutableRunningRows(): void + { + [$run, $task, $prepared] = $this->preparedClaim(); + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted)->delete(); + $before = $task->getAttributes(); + $this->assertSame('local_activity_preparation_mismatch', $this->finish($prepared, $this->success())['reason']); + $this->assertSame($before, $task->refresh()->getAttributes()); + $this->assertSame( + ActivityStatus::Running, + ActivityExecution::query()->findOrFail($prepared['activity_execution_id'])->status + ); + } + + #[DataProvider('expiredTimeouts')] + public function testAnExpiredActivityDeadlineWinsOverAnIncomingSuccessfulResult(string $field, string $kind): void + { + [$run, $task, $prepared] = $this->preparedClaim([ + $field => 1, + ]); + Carbon::setTestNow(now()->addSeconds(2)); + try { + $before = $task->getAttributes(); + $reply = $this->finish($prepared, $this->success()); + $this->assertTrue($reply['recorded']); + $this->assertSame('ActivityTimedOut', $reply['event_type']); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $this->assertSame($kind, $event->payload['timeout_kind']); + $this->assertSame('completed', $event->payload['local_outcome']['submitted_outcome']); + $this->assertSame($before, $task->refresh()->getAttributes()); + $this->assertSame([ + ...$reply, + 'duplicate' => true, + ], $this->finish($prepared, $this->success())); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCompleted)->count() + ); + } finally { + Carbon::setTestNow(); + } + } + + public static function expiredTimeouts(): iterable + { + yield 'start-to-close' => ['start_to_close_timeout', 'start_to_close']; + yield 'schedule-to-close' => ['schedule_to_close_timeout', 'schedule_to_close']; + yield 'heartbeat' => ['heartbeat_timeout', 'heartbeat']; + } + + public function testAWorkerCannotInventAnEarlyTimeout(): void + { + [$run, , $prepared] = $this->preparedClaim([ + 'start_to_close_timeout' => 10, + ]); + $this->assertSame('local_activity_timeout_not_due', $this->finish($prepared, [ + 'outcome' => 'timed_out', + ])['reason']); + $this->assertSame(2, $run->historyEvents()->count()); + } + + public function testProtocol119DoesNotEnterThePreparedOutcomePath(): void + { + [$run, , $prepared] = $this->preparedClaim(); + $reply = app(LocalActivityExecutor::class)->recordPortableOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 2, + $this->success(), + '1.19', + ); + $this->assertSame('local_activity_outcome_requires_protocol_1_20', $reply['reason']); + $this->assertSame(2, $run->historyEvents()->count()); + } + + #[DataProvider('invalidReports')] + public function testMalformedOrUnpreparedReportsCannotChangeThePreparedRows(array $report): void + { + [$run, $task, $prepared] = $this->preparedClaim(); + $before = $task->getAttributes(); + $reply = $this->finish($prepared, $report); + $this->assertFalse($reply['recorded']); + $this->assertSame('invalid_local_activity_outcome', $reply['reason']); + $this->assertSame($before, $task->refresh()->getAttributes()); + $this->assertSame(2, $run->historyEvents()->count()); + $this->assertSame( + ActivityStatus::Running, + ActivityExecution::query()->findOrFail($prepared['activity_execution_id'])->status + ); + } + + public static function invalidReports(): iterable + { + yield 'missing result' => [[ + 'outcome' => 'completed', + 'payload_codec' => 'avro', + ]]; + yield 'wrong codec' => [[ + 'outcome' => 'completed', + 'result' => 'opaque', + 'payload_codec' => 'json', + ]]; + yield 'attempt reconstruction' => [[ + 'outcome' => 'completed', + 'attempts' => [], + ]]; + yield 'cancellation is not a result report' => [[ + 'outcome' => 'cancelled', + ]]; + yield 'untyped retryability' => [[ + 'outcome' => 'failed', + 'message' => 'failed', + 'non_retryable' => 'true', + ]]; + yield 'invalid failure encoding' => [[ + 'outcome' => 'failed', + 'message' => "invalid-\xFF", + ]]; + yield 'failure cannot carry result bytes' => [[ + 'outcome' => 'failed', + 'message' => 'failed', + 'result' => 'opaque', + ]]; + } + + public function testRetryPreparationRecordsADistinctAttemptBeforeAdmissionAndReplaysItsResult(): void + { + [$run, $task, $first, $retry, $descriptor, $failed] = $this->retryClaim(); + $before = $retry->getAttributes(); + $second = $this->prepareRetry($retry, $descriptor); + $this->assertTrue($second['prepared']); + $this->assertFalse($second['duplicate']); + $this->assertSame(2, $second['attempt_number']); + $this->assertSame(4, $second['workflow_task_attempt']); + $this->assertSame('replacement-local-attempt', $second['worker_attempt_id']); + $this->assertNotSame($first['activity_attempt_id'], $second['activity_attempt_id']); + $this->assertSame($first['activity_execution_id'], $second['activity_execution_id']); + $this->assertSame($first['schedule_to_close_deadline_at'], $second['schedule_to_close_deadline_at']); + $this->assertSame($before, $retry->refresh()->getAttributes()); + $this->assertSame(TaskStatus::Completed, $task->refresh()->status); + $this->assertSame(0, $run->tasks()->where('task_type', TaskType::Activity)->count()); + $this->assertSame(1, $run->historyEvents()->where('event_type', HistoryEventType::ActivityScheduled)->count()); + $this->assertSame(2, $run->historyEvents()->where('event_type', HistoryEventType::ActivityStarted)->count()); + $historyCount = $run->historyEvents() + ->count(); + $duplicate = $this->prepareRetry($retry, $descriptor); + $this->assertTrue($duplicate['duplicate']); + $this->assertSame($second['activity_attempt_id'], $duplicate['activity_attempt_id']); + $this->assertSame($historyCount, $run->historyEvents()->count()); + $oldReceipt = $this->finish($first, $failed); + $this->assertTrue($oldReceipt['duplicate']); + $this->assertSame('local_activity_outcome_mismatch', $this->finish($first, $this->success())['reason']); + $recorded = app(LocalActivityExecutor::class)->recordPortableOutcome( + $second['activity_attempt_id'], + 'replacement-worker', + 4, + $this->success(), + '1.20' + ); + $this->assertTrue($recorded['recorded']); + $this->assertFalse($recorded['claim_released']); + $replayed = app(LocalActivityExecutor::class)->execute( + $run->fresh(), + $retry->fresh(), + 1, + new LocalActivityCall('python-local-opaque', ['Taylor']) + ); + $this->assertSame($recorded['event_id'], $replayed['event']->id); + $execution = ActivityExecution::query()->findOrFail($first['activity_execution_id']); + $this->assertSame( + [ActivityAttemptStatus::Failed, ActivityAttemptStatus::Completed], + $execution->attempts() + ->orderBy('attempt_number') + ->get() + ->pluck('status') + ->all() + ); + $this->assertSame($before, $retry->refresh()->getAttributes()); + } + + public function testMutableRetryAvailabilityCannotSkipTheCanonicalBackoff(): void + { + [$run, , $first, $retry, $descriptor] = $this->retryClaim(elapsed: 0); + $retry->forceFill([ + 'available_at' => now() + ->subMinute(), + ])->save(); + $before = $run->historyEvents() + ->count(); + $this->assertSame('local_activity_retry_not_due', $this->prepareRetry($retry, $descriptor)['reason']); + $this->assertSame($before, $run->historyEvents()->count()); + $this->assertSame(1, ActivityExecution::query()->findOrFail($first['activity_execution_id'])->attempt_count); + Carbon::setTestNow(now()->addSeconds(3)); + $this->assertTrue($this->prepareRetry($retry, $descriptor)['prepared']); + } + + public function testRetryCannotReuseAnEarlierWorkerAttemptIdentity(): void + { + [$run, , , $retry, $descriptor] = $this->retryClaim(); + $before = $run->historyEvents() + ->count(); + $reply = $this->prepareRetry($retry, $descriptor, 'sdk-local-attempt'); + $this->assertSame('local_activity_retry_worker_attempt_reused', $reply['reason']); + $this->assertSame($before, $run->historyEvents()->count()); + } + + public function testAnotherValidWorkflowClaimCannotImpersonateTheDurableRetry(): void + { + [$run, , , $retry, $descriptor] = $this->retryClaim(); + $other = $retry->replicate(); + $other->id = (string) \Illuminate\Support\Str::ulid(); + $other->save(); + $before = $run->historyEvents() + ->count(); + $this->assertSame( + 'local_activity_retry_preparation_mismatch', + $this->prepareRetry($other, $descriptor)['reason'] + ); + $this->assertSame($before, $run->historyEvents()->count()); + } + + public function testAChangedDescriptorCannotReplaceTheOriginalRetryContract(): void + { + [$run, , , $retry, $descriptor] = $this->retryClaim(); + $descriptor['schedule_to_close_timeout'] = 60; + $before = $run->historyEvents() + ->count(); + $this->assertSame( + 'local_activity_retry_preparation_mismatch', + $this->prepareRetry($retry, $descriptor)['reason'] + ); + $this->assertSame($before, $run->historyEvents()->count()); + } + + public function testAnExpiredTotalBudgetRecordsTimeoutWithoutAdmittingAnotherCallback(): void + { + [$run, , $first, $retry, $descriptor, $failed] = $this->retryClaim(totalTimeout: 2); + $before = $retry->getAttributes(); + $reply = $this->prepareRetry($retry, $descriptor); + $this->assertFalse($reply['prepared']); + $this->assertSame('local_activity_deadline_expired', $reply['reason']); + $this->assertSame('ActivityTimedOut', $reply['event_type']); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $this->assertSame('schedule_to_close', $event->payload['timeout_kind']); + $this->assertSame($retry->id, $event->workflow_task_id); + $this->assertSame($first['activity_attempt_id'], $event->payload['activity_attempt_id']); + $execution = ActivityExecution::query()->findOrFail($first['activity_execution_id']); + $this->assertSame(ActivityStatus::Failed, $execution->status); + $this->assertSame(1, $execution->attempt_count); + $this->assertSame(1, $execution->attempts()->count()); + $this->assertSame( + $first['schedule_to_close_deadline_at'], + $execution->schedule_to_close_deadline_at->toISOString() + ); + $this->assertSame($before, $retry->refresh()->getAttributes()); + $count = $run->historyEvents() + ->count(); + $this->assertFalse($this->prepareRetry($retry, $descriptor)['prepared']); + $this->assertSame($count, $run->historyEvents()->count()); + $this->assertSame(1, $run->failures()->count()); + $this->assertTrue($this->finish($first, $failed)['duplicate']); + $replayed = app(LocalActivityExecutor::class)->execute( + $run->fresh(), + $retry->fresh(), + 1, + new LocalActivityCall('python-local-opaque', ['Taylor']) + ); + $this->assertSame($reply['event_id'], $replayed['event']->id); + } + + public function testCancellationBeforeRetryPreparationPreservesTheOriginalRequestBudget(): void + { + [$run, , $first, $retry, $descriptor] = $this->retryClaim(); + $request = WorkflowStub::loadRun($run->id)->requestCancellation('stop before retry', 30); + $deadline = $run->refresh() + ->cancellation_deadline_at->toISOString(); + $before = $retry->refresh() + ->getAttributes(); + $this->assertSame('cancellation_requested', $this->prepareRetry($retry, $descriptor)['reason']); + $this->assertSame($before, $retry->refresh()->getAttributes()); + $this->assertSame($deadline, $run->refresh()->cancellation_deadline_at->toISOString()); + $this->assertSame($request->commandId(), $run->cancellation_request_command_id); + $this->assertSame(1, ActivityExecution::query()->findOrFail($first['activity_execution_id'])->attempt_count); + } + + public function testMissingRetryHistoryCannotInventRetryAuthority(): void + { + [$run, , $first, $retry, $descriptor] = $this->retryClaim(); + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityRetryScheduled)->delete(); + $before = $run->historyEvents() + ->count(); + $this->assertSame( + 'local_activity_retry_preparation_mismatch', + $this->prepareRetry($retry, $descriptor)['reason'] + ); + $this->assertSame($before, $run->historyEvents()->count()); + $this->assertSame( + 1, + ActivityExecution::query()->findOrFail($first['activity_execution_id'])->attempts()->count() + ); + } + + public function testRetryCannotGainMoreTotalTimeFromChangedMutableDeadlineRows(): void + { + [$run, , $first, $retry, $descriptor] = $this->retryClaim(); + ActivityExecution::query()->findOrFail($first['activity_execution_id'])->forceFill([ + 'schedule_to_close_deadline_at' => now() + ->addMinute(), + ])->save(); + $before = $run->historyEvents() + ->count(); + $this->assertSame( + 'local_activity_retry_preparation_mismatch', + $this->prepareRetry($retry, $descriptor)['reason'] + ); + $this->assertSame($before, $run->historyEvents()->count()); + } + + public function testRenewedWorkflowClaimCannotReadmitAnExpiredOriginalLocalAttempt(): void + { + [, , , $retry, $descriptor] = $this->retryClaim(); + $second = $this->prepareRetry($retry, $descriptor); + $this->assertTrue($second['prepared']); + \Workflow\V2\Models\ActivityAttempt::query()->findOrFail($second['activity_attempt_id'])->forceFill([ + 'lease_expires_at' => now() + ->subSecond(), + ])->save(); + $before = $retry->getAttributes(); + $this->assertSame( + 'local_activity_preparation_lease_expired', + $this->prepareRetry($retry, $descriptor)['reason'] + ); + $this->assertSame($before, $retry->refresh()->getAttributes()); + } + + /** + * @return array{WorkflowRun, WorkflowTask, array, WorkflowTask, array, array} + */ + private function retryClaim(int $elapsed = 4, int $totalTimeout = 30): array + { + Carbon::setTestNow('2026-10-02T04:00:00.000000Z'); + $options = [ + 'schedule_to_close_timeout' => $totalTimeout, + 'start_to_close_timeout' => min(10, $totalTimeout), + 'retry_policy' => [ + 'max_attempts' => 3, + 'backoff_seconds' => [3], + ], + ]; + [$run, $task, $first] = $this->preparedClaim($options); + $failed = [ + 'outcome' => 'failed', + 'message' => 'temporary failure', + 'exception_type' => 'python.Temporary', + ]; + $reply = $this->finish($first, $failed); + $this->assertTrue($reply['recorded']); + $this->assertTrue($reply['claim_released']); + $retry = WorkflowTask::query()->findOrFail($reply['created_task_ids'][0]); + Carbon::setTestNow(now()->addSeconds($elapsed)); + $retry->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'replacement-worker', + 'attempt_count' => 4, + 'lease_expires_at' => now() + ->addMinutes(5), + ])->save(); + $descriptor = [ + 'type' => 'record_local_activity', + 'activity_type' => 'python-local-opaque', + 'arguments' => Serializer::serializeWithCodec('avro', ['Taylor']), + 'payload_codec' => 'avro', + ...$options, + ]; + return [$run->refresh(), $task, $first, $retry->refresh(), $descriptor, $failed]; + } + + /** @param array $descriptor + * @return array + */ + private function prepareRetry( + WorkflowTask $task, + array $descriptor, + string $workerAttemptId = 'replacement-local-attempt' + ): array { + return PortableLocalActivityPreparation::prepare( + $task->id, + 'replacement-worker', + 4, + 1, + $workerAttemptId, + $descriptor, + '1.20' + ); + } + + /** + * @return array + */ + private function success(): array + { + return [ + 'outcome' => 'completed', + 'result' => Serializer::serializeWithCodec('avro', [ + 'value' => 'Hello, Taylor!', + ]), + 'payload_codec' => 'avro', + ]; + } + + /** @param array $prepared + * @param array $report + * @return array + */ + private function finish(array $prepared, array $report): array + { + return app(LocalActivityExecutor::class)->recordPortableOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 2, + $report, + '1.20' + ); + } + + /** @param array $options + * @return array{WorkflowRun, WorkflowTask, array} + */ + private function preparedClaim(array $options = []): array + { + $instance = WorkflowInstance::query()->create([ + 'workflow_class' => TestGreetingWorkflow::class, + 'workflow_type' => 'portable-parent', + 'run_count' => 1, + 'started_at' => now() + ->subMinute(), + ]); + $run = WorkflowRun::query()->create([ + 'workflow_instance_id' => $instance->id, + 'run_number' => 1, + 'workflow_class' => TestGreetingWorkflow::class, + 'workflow_type' => 'portable-parent', + 'status' => RunStatus::Waiting, + 'arguments' => Serializer::serializeWithCodec('avro', ['Taylor']), + 'payload_codec' => 'avro', + 'connection' => 'database', + 'queue' => 'default', + 'compatibility' => 'build-a', + 'started_at' => now() + ->subMinute(), + ]); + $instance->forceFill([ + 'current_run_id' => $run->id, + ])->save(); + $task = WorkflowTask::query()->create([ + 'workflow_run_id' => $run->id, + 'task_type' => TaskType::Workflow, + 'status' => TaskStatus::Leased, + 'attempt_count' => 2, + 'payload' => [], + 'connection' => 'database', + 'queue' => 'default', + 'compatibility' => 'build-a', + 'lease_owner' => 'portable-worker', + 'lease_expires_at' => now() + ->addMinutes(5), + ])->refresh(); + $prepared = PortableLocalActivityPreparation::prepare($task->id, 'portable-worker', 2, 1, 'sdk-local-attempt', [ + 'type' => 'record_local_activity', + 'activity_type' => 'python-local-opaque', + 'arguments' => Serializer::serializeWithCodec('avro', ['Taylor']), + 'payload_codec' => 'avro', + ...$options, + ], '1.20'); + $this->assertTrue($prepared['prepared']); + + return [$run->refresh(), $task, $prepared]; + } +} diff --git a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php new file mode 100644 index 000000000..d20d3a1c4 --- /dev/null +++ b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php @@ -0,0 +1,1488 @@ +set('workflows.v2.compatibility.current', 'build-a'); + config() + ->set('workflows.v2.compatibility.supported', ['build-a']); + } + + public function testPreparationPersistsAWorkerAliasAndOriginalClaimBeforeAnyApplicationInvocation(): void + { + [$run, $task] = $this->newClaim(); + $taskBefore = $task->getAttributes(); + $reply = $this->prepare($task); + $this->assertTrue($reply['prepared']); + $this->assertFalse($reply['duplicate']); + $this->assertSame('sdk-local-attempt', $reply['worker_attempt_id']); + $this->assertSame('portable-worker', $reply['lease_owner']); + $this->assertSame(1, $reply['workflow_task_attempt']); + $execution = ActivityExecution::query()->findOrFail($reply['activity_execution_id']); + $this->assertSame('python-local-greeting', $execution->activity_type); + $this->assertSame(['Taylor'], $execution->activityArguments()); + $this->assertSame('avro', $execution->payload_codec); + $this->assertSame(ActivityStatus::Running, $execution->status); + $this->assertSame($reply['activity_attempt_id'], $execution->current_attempt_id); + $attempt = $execution->attempts() + ->sole(); + $this->assertSame(ActivityAttemptStatus::Running, $attempt->status); + $this->assertSame($task->id, $attempt->workflow_task_id); + $this->assertSame($taskBefore, $task->refresh()->getAttributes()); + $events = $run->historyEvents() + ->orderBy('sequence') + ->get(); + $this->assertSame( + [HistoryEventType::ActivityScheduled, HistoryEventType::ActivityStarted], + $events->pluck('event_type') + ->all() + ); + $started = $events->last(); + $this->assertSame($task->id, $started->payload['task']['id']); + $this->assertSame(1, $started->payload['task']['attempt_count']); + $this->assertSame('portable-worker', $started->payload['task']['lease_owner']); + $this->assertSame('sdk-local-attempt', $started->payload['activity_attempt']['worker_attempt_id']); + $this->assertSame(0, $run->tasks()->where('task_type', TaskType::Activity)->count()); + foreach (HistoryTimeline::forRun($run->fresh()) as $entry) { + $this->assertSame('workflow', $entry['task']['type']); + $this->assertSame('leased', $entry['task']['status']); + } + } + + public function testLostPreparationResponseReturnsTheOriginalAttemptWithoutRenewingAnyDeadline(): void + { + [, $task] = $this->newClaim(); + $first = $this->prepare($task, [ + 'start_to_close_timeout' => 10, + 'schedule_to_close_timeout' => 20, + ]); + $this->assertTrue($first['prepared']); + $taskBefore = $task->refresh() + ->getAttributes(); + Carbon::setTestNow(now()->addSecond()); + try { + $second = $this->prepare($task, [ + 'start_to_close_timeout' => 10, + 'schedule_to_close_timeout' => 20, + ]); + $this->assertTrue($second['prepared']); + $this->assertTrue($second['duplicate']); + foreach (['activity_execution_id', 'activity_attempt_id', 'worker_attempt_id', 'workflow_task_id', + 'workflow_task_attempt', 'lease_owner', 'lease_expires_at', 'start_to_close_deadline_at', + 'schedule_to_close_deadline_at'] as $field) { + $this->assertSame($first[$field], $second[$field]); + } + $this->assertSame(1, ActivityExecution::query()->count()); + $this->assertSame(2, WorkflowHistoryEvent::query()->count()); + $this->assertSame($taskBefore, $task->refresh()->getAttributes()); + } finally { + Carbon::setTestNow(); + } + } + + #[DataProvider('preparedLocalCancellationPolicies')] + public function testPreparedLocalPolicyIsPersistedBeforeInvocationAndRetainedOnResponseLoss(?string $policy): void + { + [$run, $task] = $this->newClaim(); + $descriptor = $policy === null ? [] : [ + 'cancellation_policy' => $policy, + ]; + $first = $this->prepare($task, $descriptor); + $this->assertTrue($first['prepared']); + $execution = ActivityExecution::query()->findOrFail($first['activity_execution_id']); + $this->assertSame($policy, $execution->activity_options['cancellation_policy'] ?? null); + foreach ($run->historyEvents()->orderBy('sequence')->get() as $event) { + $this->assertSame($policy, $event->payload['activity']['cancellation_policy'] ?? null); + } + $duplicate = $this->prepare($task, $descriptor); + $this->assertTrue($duplicate['prepared']); + $this->assertTrue($duplicate['duplicate']); + $this->assertSame($first['activity_attempt_id'], $duplicate['activity_attempt_id']); + $this->assertSame(2, $run->historyEvents()->count()); + $this->assertSame(0, $run->tasks()->where('task_type', TaskType::Activity)->count()); + } + + public static function preparedLocalCancellationPolicies(): iterable + { + yield 'historical default' => [null]; + yield 'try cancellation' => ['try_cancel']; + yield 'wait for stop receipt' => ['wait_cancellation_completed']; + } + + public function testAChangedLocalPolicyCannotRelabelAnAdmittedAttempt(): void + { + [$run, $task] = $this->newClaim(); + $this->assertTrue($this->prepare($task, [ + 'cancellation_policy' => 'try_cancel', + ])['prepared']); + $before = $run->historyEvents() + ->orderBy('sequence') + ->get() + ->toArray(); + $changed = $this->prepare($task, [ + 'cancellation_policy' => 'wait_cancellation_completed', + ]); + $this->assertFalse($changed['prepared']); + $this->assertSame('local_activity_preparation_mismatch', $changed['reason']); + $this->assertSame($before, $run->historyEvents()->orderBy('sequence')->get()->toArray()); + } + + #[DataProvider('unsupportedLocalCancellationPolicies')] + public function testUnsupportedLocalPolicyIsRefusedBeforeCallbackAdmission(mixed $policy): void + { + [$run, $task] = $this->newClaim(); + $before = $task->getAttributes(); + $change = [ + 'cancellation_policy' => $policy, + 'schedule_to_close_timeout' => 60, + ]; + try { + PortableLocalActivityPreparation::normalizeDescriptor([...$this->descriptor(), ...$change]); + $this->fail('An unsupported local policy must refuse before admission.'); + } catch (ValidationException $error) { + $this->assertArrayHasKey('local_activity.cancellation_policy', $error->errors()); + } + $reply = $this->prepare($task, $change); + $this->assertFalse($reply['prepared']); + $this->assertSame('invalid_local_activity_preparation', $reply['reason']); + $this->assertSame(0, $run->activityExecutions()->count()); + $this->assertSame(0, $run->historyEvents()->count()); + $this->assertSame($before, $task->refresh()->getAttributes()); + } + + public static function unsupportedLocalCancellationPolicies(): iterable + { + foreach (['abandon', 'unknown', null, false, 1, []] as $policy) { + yield [$policy]; + } + } + + #[DataProvider('changedDescriptors')] + public function testAPreparedAttemptCannotBeRelabelledByARetry(array $change): void + { + [, $task] = $this->newClaim(); + $this->assertTrue($this->prepare($task)['prepared']); + $reply = $this->prepare($task, $change); + $this->assertFalse($reply['prepared']); + $this->assertSame('local_activity_preparation_mismatch', $reply['reason']); + $this->assertSame(1, ActivityExecution::query()->count()); + $this->assertSame(2, WorkflowHistoryEvent::query()->count()); + } + + public static function changedDescriptors(): iterable + { + yield 'different alias' => [[ + 'activity_type' => 'another-local-activity', + ]]; + yield 'different input' => [[ + 'arguments' => Serializer::serializeWithCodec('avro', ['another-input']), + ]]; + yield 'different timeout' => [[ + 'start_to_close_timeout' => 2, + ]]; + yield 'different retry budget' => [[ + 'retry_policy' => [ + 'max_attempts' => 3, + ], + ]]; + } + + public function testChangedOwnerOrClaimAttemptCannotReuseTheOriginalPreparation(): void + { + [, $task] = $this->newClaim(); + $this->assertTrue($this->prepare($task)['prepared']); + $task->forceFill([ + 'lease_owner' => 'replacement-worker', + 'attempt_count' => 2, + ])->save(); + $taskBefore = $task->refresh() + ->getAttributes(); + $this->assertSame('workflow_claim_mismatch', $this->prepare($task)['reason']); + $reply = PortableLocalActivityPreparation::prepare( + $task->id, + 'replacement-worker', + 2, + 1, + 'sdk-local-attempt', + $this->descriptor(), + '1.20', + ); + $this->assertFalse($reply['prepared']); + $this->assertSame('local_activity_preparation_mismatch', $reply['reason']); + $this->assertSame($taskBefore, $task->refresh()->getAttributes()); + $this->assertSame(2, WorkflowHistoryEvent::query()->count()); + } + + #[DataProvider('invalidDescriptors')] + public function testInvalidOrTerminalReportsCannotPretendToPrepareACallback(array $change): void + { + [, $task] = $this->newClaim(); + $reply = $this->prepare($task, $change); + $this->assertFalse($reply['prepared']); + $this->assertSame('invalid_local_activity_preparation', $reply['reason']); + $this->assertSame(0, ActivityExecution::query()->count()); + $this->assertSame(0, WorkflowHistoryEvent::query()->count()); + } + + public static function invalidDescriptors(): iterable + { + yield 'queued routing' => [[ + 'queue' => 'another-queue', + ]]; + yield 'outcome supplied before execution' => [[ + 'outcome' => 'completed', + ]]; + yield 'attempt report supplied before execution' => [[ + 'attempts' => [], + ]]; + yield 'negative timeout' => [[ + 'heartbeat_timeout' => -1, + ]]; + yield 'inconsistent timeouts' => [[ + 'start_to_close_timeout' => 3, + 'schedule_to_close_timeout' => 2, + ]]; + yield 'wrong execution mode' => [[ + 'execution_mode' => 'remote', + ]]; + yield 'invalid identifier encoding' => [[ + 'activity_type' => "invalid-\xFF", + ]]; + } + + public function testProtocol119CannotPreparePortableLocalActivities(): void + { + [, $task] = $this->newClaim(); + $reply = PortableLocalActivityPreparation::prepare( + $task->id, + 'portable-worker', + 1, + 1, + 'sdk-local-attempt', + $this->descriptor(), + '1.19', + ); + $this->assertFalse($reply['prepared']); + $this->assertSame('local_activity_preparation_requires_protocol_1_20', $reply['reason']); + $this->assertSame(0, WorkflowHistoryEvent::query()->count()); + } + + public function testUncommittedEarlierCommandsCannotBeSkipped(): void + { + [$run, $task] = $this->newClaim(); + $reply = PortableLocalActivityPreparation::prepare( + $task->id, + 'portable-worker', + 1, + 2, + 'sdk-local-attempt', + $this->descriptor(), + '1.20', + ); + $this->assertFalse($reply['prepared']); + $this->assertSame('local_activity_command_prefix_not_recorded', $reply['reason']); + $this->assertSame(0, WorkflowHistoryEvent::query()->count()); + WorkflowHistoryEvent::record($run, HistoryEventType::SideEffectRecorded, [ + 'sequence' => 1, + 'result' => Serializer::serializeWithCodec('avro', 'already-recorded'), + ], $task); + $reply = PortableLocalActivityPreparation::prepare( + $task->id, + 'portable-worker', + 1, + 2, + 'sdk-local-attempt', + $this->descriptor(), + '1.20', + ); + $this->assertTrue($reply['prepared']); + $this->assertSame(2, ActivityExecution::query()->sole()->sequence); + $this->assertSame(3, WorkflowHistoryEvent::query()->count()); + } + + public function testCancellationAfterALostResponseCannotAuthorizeApplicationInvocation(): void + { + [$run, $task] = $this->newClaim(); + $first = $this->prepare($task); + $this->assertTrue($first['prepared']); + $this->assertTrue( + WorkflowStub::load($run->workflow_instance_id)->requestCancellation('maintenance', 30)->accepted() + ); + $run->refresh(); + $deadline = $run->cancellation_deadline_at->toISOString(); + $before = $run->historyEvents() + ->count(); + $reply = $this->prepare($task); + $this->assertFalse($reply['prepared']); + $this->assertSame('cancellation_requested', $reply['reason']); + $this->assertSame($before, $run->historyEvents()->count()); + $execution = ActivityExecution::query()->findOrFail($first['activity_execution_id']); + ActivityCancellation::record($run, $execution, command: $run->cancellation_request_command_id); + $task->forceFill([ + 'lease_owner' => 'replacement-worker', + 'attempt_count' => 2, + ])->save(); + $taskBefore = $task->refresh() + ->getAttributes(); + $receipt = ActivityCancellationAcknowledgement::recordLocalStopped( + $first['activity_attempt_id'], + 'portable-worker', + $run->cancellation_request_command_id, + 1, + ); + $this->assertTrue($receipt['acknowledged']); + $this->assertSame($taskBefore, $task->refresh()->getAttributes()); + $this->assertSame($deadline, $run->refresh()->cancellation_deadline_at->toISOString()); + } + + public function testAnExpiredAttemptOrWorkflowClaimCannotBePreparedAgain(): void + { + [, $task] = $this->newClaim(); + $this->assertTrue($this->prepare($task, [ + 'start_to_close_timeout' => 1, + ])['prepared']); + Carbon::setTestNow(now()->addSeconds(2)); + try { + $reply = $this->prepare($task, [ + 'start_to_close_timeout' => 1, + ]); + $this->assertFalse($reply['prepared']); + $this->assertSame('local_activity_deadline_expired', $reply['reason']); + $task->forceFill([ + 'lease_expires_at' => now() + ->subSecond(), + ])->save(); + $this->assertSame('workflow_claim_expired', $this->prepare($task)['reason']); + $this->assertSame(2, WorkflowHistoryEvent::query()->count()); + } finally { + Carbon::setTestNow(); + } + } + + /** + * @return array + */ + public function testCompleteChildLocalGroupIsCommittedBeforeAnyLocalAttempt(): void + { + [$run, $task] = $this->newClaim(); + $lease = $task->lease_expires_at; + $commands = $this->groupCommands(); + $reply = $this->groupCheckpoint($task, $commands); + $this->assertTrue($reply['checkpointed'], $reply['reason'] ?? ''); + $this->assertSame(3, $reply['next_sequence']); + $this->assertCount(1, $reply['local_activities']); + $this->assertSame(2, WorkflowRun::query()->count()); + $this->assertSame(0, ActivityAttempt::query()->count()); + $this->assertSame(0, $run->tasks()->where('task_type', TaskType::Activity)->count()); + $this->assertSame(TaskStatus::Leased, $task->refresh()->status); + $this->assertEquals($lease, $task->lease_expires_at); + $execution = ActivityExecution::query()->sole(); + $this->assertSame(ActivityStatus::Pending, $execution->status); + $this->assertSame(0, $execution->attempt_count); + $this->assertNull($execution->current_attempt_id); + $scheduled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityScheduled)->sole(); + $this->assertGroupPath($commands[1]['parallel_group_path'], $scheduled->payload['parallel_group_path']); + $this->assertSame(1, $scheduled->payload['local_group_admission']['version']); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::ActivityStarted)->count()); + $prepared = $this->prepareGroupMember($task, $commands[1], 2); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $this->assertSame($execution->id, $prepared['activity_execution_id']); + $started = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted)->sole(); + $this->assertGroupPath($commands[1]['parallel_group_path'], $started->payload['parallel_group_path']); + $this->assertSame($task->id, $started->workflow_task_id); + } + + public function testGroupResponseLossCannotCreateAnotherChildOrExtendTheTotalBudget(): void + { + [$run, $task] = $this->newClaim(); + $commands = $this->groupCommands(); + $commands[1]['schedule_to_close_timeout'] = 20; + $first = $this->groupCheckpoint($task, $commands); + $this->assertTrue($first['checkpointed'], $first['reason'] ?? ''); + $deadline = ActivityExecution::query()->sole()->schedule_to_close_deadline_at; + $events = $run->historyEvents() + ->count(); + Carbon::setTestNow(now()->addSeconds(5)); + try { + $duplicate = $this->groupCheckpoint($task, $commands); + $this->assertTrue($duplicate['duplicate']); + $this->assertEquals([ + ...$first, + 'duplicate' => true, + ], $duplicate); + $this->assertSame(2, WorkflowRun::query()->count()); + $this->assertSame($events, $run->historyEvents()->count()); + $prepared = $this->prepareGroupMember($task, $commands[1], 2); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $this->assertEquals($deadline, new Carbon($prepared['schedule_to_close_deadline_at'])); + $this->assertTrue($this->prepareGroupMember($task, $commands[1], 2)['duplicate']); + $this->assertSame(1, ActivityAttempt::query()->count()); + } finally { + Carbon::setTestNow(); + } + } + + public function testReplacementBeforeLocalPreparationCreatesOnlyItsOwnFirstAttempt(): void + { + [$run, $task] = $this->newClaim(); + $commands = $this->groupCommands(); + $this->assertTrue($this->groupCheckpoint($task, $commands)['checkpointed']); + $task->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 2, + ])->save(); + $this->assertSame('workflow_claim_mismatch', $this->prepareGroupMember($task, $commands[1], 2)['reason']); + $prepared = $this->prepareGroupMember($task, $commands[1], 2, 'replacement', 2); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $this->assertSame(2, $prepared['workflow_task_attempt']); + $this->assertSame(1, $prepared['attempt_number']); + $this->assertSame(1, ActivityAttempt::query()->count()); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityRetryScheduled)->count() + ); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancellationAcknowledged)->count() + ); + } + + public function testCancellationBetweenGroupCommitAndPrepareRefusesTheUnstartedCallback(): void + { + [$run, $task] = $this->newClaim(); + $commands = $this->groupCommands(); + $this->assertTrue($this->groupCheckpoint($task, $commands)['checkpointed']); + $run->forceFill([ + 'cancellation_request_command_id' => 'cancellation', + 'cancellation_deadline_at' => now() + ->addSeconds(30), + ])->save(); + $this->assertSame('cancellation_requested', $this->prepareGroupMember($task, $commands[1], 2)['reason']); + $this->assertSame(0, ActivityAttempt::query()->count()); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::ActivityStarted)->count()); + } + + public function testElapsedAdmissionTotalBudgetRecordsTimeoutWithoutFabricatingAnAttempt(): void + { + [$run, $task] = $this->newClaim(); + $commands = $this->groupCommands(); + $commands[1]['schedule_to_close_timeout'] = 2; + $this->assertTrue($this->groupCheckpoint($task, $commands)['checkpointed']); + Carbon::setTestNow(now()->addSeconds(2)); + try { + $reply = $this->prepareGroupMember($task, $commands[1], 2); + $this->assertSame('local_activity_deadline_expired', $reply['reason']); + $this->assertSame('ActivityTimedOut', $reply['event_type']); + $this->assertSame(0, ActivityAttempt::query()->count()); + $this->assertSame(ActivityStatus::Failed, ActivityExecution::query()->sole()->status); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityTimedOut)->count() + ); + } finally { + Carbon::setTestNow(); + } + } + + public function testPreparedGroupOutcomePreservesTheAuthoredPathAndOriginalClaim(): void + { + [$run, $task] = $this->newClaim(); + $commands = $this->groupCommands(); + $this->assertTrue($this->groupCheckpoint($task, $commands)['checkpointed']); + $prepared = $this->prepareGroupMember($task, $commands[1], 2); + $reply = app(DefaultWorkflowTaskBridge::class)->recordLocalActivityOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + [ + 'outcome' => 'completed', + 'result' => Serializer::serializeWithCodec('avro', 'done'), + 'payload_codec' => 'avro', + ], + '1.20' + ); + $this->assertTrue($reply['recorded'], $reply['reason'] ?? ''); + $this->assertFalse($reply['claim_released']); + $event = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCompleted)->sole(); + $this->assertGroupPath($commands[1]['parallel_group_path'], $event->payload['parallel_group_path']); + $this->assertSame($task->id, $event->payload['task']['id']); + $this->assertSame($prepared['activity_attempt_id'], $event->payload['activity_attempt_id']); + } + + public function testCancellationFencesBothPreparedGroupMembersAndRetainsTheirAuthoredPaths(): void + { + [$run, $task] = $this->newClaim(); + $commands = []; + foreach ([0, 1] as $index) { + $commands[] = [ + ...$this->descriptor(), + 'type' => 'prepare_local_activity', + ...ParallelChildGroup::itemMetadata(1, 2, $index, 'activity'), + ]; + } + $this->assertTrue($this->groupCheckpoint($task, $commands)['checkpointed']); + $bridge = app(PreparedLocalActivityTaskBridge::class); + $members = []; + foreach ($commands as $index => $descriptor) { + $members[] = $bridge->prepareLocalActivity( + $task->id, + 'portable-worker', + 1, + $index + 1, + 'group-attempt-' . $index, + [ + ...$descriptor, + 'type' => 'record_local_activity', + ], + '1.20' + ); + $this->assertTrue($members[$index]['prepared']); + } + $context = WorkflowStub::loadRun($run->id)->requestCancellation('stop group', 30)->cancellationContext(); + $taskBefore = $task->refresh() + ->getAttributes(); + foreach ($members as $index => $member) { + $control = $bridge->controlLocalActivity( + $member['activity_attempt_id'], + 'portable-worker', + 1, + true, + '1.20' + ); + $this->assertSame('cancellation_requested', $control['reason']); + $this->assertTrue($control['stop_required']); + $this->assertTrue($control['fenced']); + $this->assertFalse($control['renewed']); + $this->assertSame($context->toArray(), $control['cancellation_request']); + $this->assertSame( + ActivityAttemptStatus::Cancelled, + ActivityAttempt::query()->findOrFail($member['activity_attempt_id'])->status + ); + $event = WorkflowHistoryEvent::query()->findOrFail($control['cancellation_history_event_id']); + $this->assertSame(HistoryEventType::ActivityCancelled, $event->event_type); + $this->assertGroupPath($commands[$index]['parallel_group_path'], $event->payload['parallel_group_path']); + $this->assertSame($member['activity_attempt_id'], $event->payload['activity_attempt_id']); + $this->assertSame($task->id, $event->payload['activity_attempt']['task_id']); + $again = $bridge->controlLocalActivity($member['activity_attempt_id'], 'portable-worker', 1, false, '1.20'); + $this->assertSame($control['cancellation_history_event_id'], $again['cancellation_history_event_id']); + $ack = $bridge->acknowledgeLocalActivityCancellation( + $member['activity_attempt_id'], + 'portable-worker', + $context->requestId, + 1, + '1.20' + ); + $this->assertTrue($ack['acknowledged']); + $receipt = WorkflowHistoryEvent::query()->findOrFail($ack['history_event_id']); + $this->assertSame($context->rootRequestId, $receipt->payload['root_request_id']); + $this->assertSame($context->deadline()->toISOString(), $receipt->payload['cleanup_deadline_at']); + $this->assertFalse($receipt->payload['received_after_deadline']); + } + $this->assertSame($taskBefore, $task->refresh()->getAttributes()); + $this->assertSame(2, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCancelled)->count()); + $this->assertSame( + 2, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancellationAcknowledged)->count() + ); + $this->assertSame( + $context->toArray(), + WorkflowStub::loadRun($run->id)->requestCancellation('duplicate', 600)->cancellationContext()->toArray() + ); + } + + #[DataProvider('groupRetryClaims')] + public function testColdGroupRecoveryTransfersOnlyTheRecordedSiblingRetryChain( + string $change, + ?string $reason + ): void { + Carbon::setTestNow('2026-10-02T10:00:00Z'); + try { + [$run, $original] = $this->newClaim(); + $original->forceFill([ + 'lease_expires_at' => now() + ->addSeconds(10), + ])->save(); + $commands = []; + foreach ([0, 1] as $index) { + $commands[] = [ + ...$this->descriptor(), + 'type' => 'prepare_local_activity', + 'schedule_to_close_timeout' => 120, + 'retry_policy' => [ + 'max_attempts' => 2, + 'backoff_seconds' => [0], + ], + ...ParallelChildGroup::itemMetadata(1, 2, $index, 'activity'), + ]; + } + $this->assertTrue($this->groupCheckpoint($original, $commands)['checkpointed']); + $bridge = app(PreparedLocalActivityTaskBridge::class); + $members = []; + foreach ($commands as $index => $descriptor) { + $members[] = $bridge->prepareLocalActivity( + $original->id, + 'portable-worker', + 1, + $index + 1, + 'original-' . $index, + [ + ...$descriptor, + 'type' => 'record_local_activity', + ], + '1.20' + ); + $this->assertTrue($members[$index]['prepared']); + } + Carbon::setTestNow(now()->addSeconds(11)); + $claim = $original; + $claim->forceFill([ + 'attempt_count' => 2, + 'lease_expires_at' => now() + ->addSeconds(10), + ])->save(); + $recoveries = []; + $claims = []; + foreach ($commands as $index => $descriptor) { + $recovered = $bridge->recoverLocalActivity( + $claim->id, + 'portable-worker', + $claim->attempt_count, + $index + 1, + [ + ...$descriptor, + 'type' => 'record_local_activity', + ], + '1.20' + ); + $this->assertTrue($recovered['recovered'], $recovered['reason'] ?? ''); + $this->assertTrue($recovered['claim_released']); + $this->assertSame('unknown', $recovered['callback_stop_state']); + $recoveries[] = WorkflowHistoryEvent::query()->findOrFail($recovered['event_id']); + $claim = WorkflowTask::query()->findOrFail($recovered['created_task_ids'][0]); + $claim->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'portable-worker', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addSeconds(10), + ])->save(); + $claims[] = $claim; + } + if ($change === 'unrelated claim') { + $claim = $claim->replicate(); + $claim->save(); + } elseif ($change === 'different batch') { + $scheduled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityScheduled) + ->where('payload->sequence', 2) + ->sole(); + $payload = $scheduled->payload; + $payload['local_group_admission']['batch_fingerprint'] = str_repeat('0', 64); + $scheduled->forceFill([ + 'payload' => $payload, + ])->save(); + } elseif ($change === 'missing link') { + $recoveries[1]->delete(); + } elseif ($change === 'unfinished predecessor') { + $claims[0]->forceFill([ + 'status' => TaskStatus::Ready, + ])->save(); + } elseif ($change === 'cycle' || $change === 'wrong source') { + $payload = $recoveries[1]->payload; + $payload[$change === 'cycle' ? 'retry_task_id' : 'retry_of_task_id'] = + $change === 'cycle' ? $claims[0]->id : $original->id; + $recoveries[1]->forceFill([ + 'payload' => $payload, + ])->save(); + } elseif ($change === 'backoff') { + $payload = $recoveries[0]->payload; + $payload['retry_available_at'] = now()->addSecond()->toISOString(); + $recoveries[0]->forceFill([ + 'payload' => $payload, + ])->save(); + $claim->forceFill([ + 'available_at' => now() + ->subMinute(), + ])->save(); + } + $before = $claim->refresh() + ->getAttributes(); + foreach ($commands as $index => $descriptor) { + $prepared = $bridge->prepareLocalActivity( + $claim->id, + 'portable-worker', + 1, + $index + 1, + 'replacement-' . $index, + [ + ...$descriptor, + 'type' => 'record_local_activity', + ], + '1.20' + ); + if ($reason !== null) { + $this->assertFalse($prepared['prepared']); + $this->assertSame($reason, $prepared['reason']); + $this->assertSame(2, ActivityAttempt::query()->count()); + break; + } + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $this->assertSame(2, $prepared['attempt_number']); + $this->assertNotSame($members[$index]['activity_attempt_id'], $prepared['activity_attempt_id']); + $this->assertSame( + $members[$index]['schedule_to_close_deadline_at'], + $prepared['schedule_to_close_deadline_at'] + ); + } + $this->assertSame($before, $claim->refresh()->getAttributes()); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancellationAcknowledged)->count() + ); + } finally { + Carbon::setTestNow(); + } + } + + public static function groupRetryClaims(): iterable + { + yield 'both members on the final claim' => ['unchanged', null]; + foreach ([ + 'unrelated claim', + 'different batch', + 'missing link', + 'unfinished predecessor', + 'cycle', + 'wrong source', + ] as $change) { + yield $change => [$change, 'local_activity_retry_preparation_mismatch']; + } + yield 'mutable availability cannot bypass backoff' => ['backoff', 'local_activity_retry_not_due']; + } + + #[DataProvider('invalidGroupChanges')] + public function testInvalidGroupCreatesNeitherTheChildNorTheLocalExecution(string $change): void + { + [$run, $task] = $this->newClaim(); + $commands = $this->groupCommands(); + match ($change) { + 'partial' => array_pop($commands), + 'missing child' => array_shift($commands), + 'wrong member index' => $commands[1]['parallel_group_path'][0]['parallel_group_index'] = 0, + 'wrong group size' => $commands[1]['parallel_group_size'] = 3, + 'outcome instead of admission' => $commands[1]['type'] = 'record_local_activity', + 'routing' => $commands[1]['queue'] = 'remote', + 'unshielded proof' => $commands[1]['cancellation_cleanup'] = [ + 'request_id' => 'invented', + 'delivery_history_event_id' => 'invented', + ], + }; + $reply = $this->groupCheckpoint($task, $commands); + $this->assertFalse($reply['checkpointed']); + $this->assertSame(0, ActivityExecution::query()->count()); + $this->assertSame(0, ActivityAttempt::query()->count()); + $this->assertSame(1, WorkflowRun::query()->count()); + $this->assertSame(0, $run->historyEvents()->count()); + } + + public static function invalidGroupChanges(): iterable + { + foreach (['partial', 'missing child', 'wrong member index', 'wrong group size', + 'outcome instead of admission', 'routing', 'unshielded proof'] as $change) { + yield $change => [$change]; + } + } + + public function testChangedGroupOrPreparationCannotRelabelTheAdmittedWork(): void + { + [$run, $task] = $this->newClaim(); + $commands = $this->groupCommands(); + $this->assertTrue($this->groupCheckpoint($task, $commands)['checkpointed']); + $before = $run->historyEvents() + ->count(); + $commands[1]['activity_type'] = 'changed'; + $this->assertSame('local_activity_checkpoint_mismatch', $this->groupCheckpoint($task, $commands)['reason']); + $this->assertSame( + 'local_activity_group_admission_mismatch', + $this->prepareGroupMember($task, $commands[1], 2)['reason'] + ); + $this->assertSame($before, $run->historyEvents()->count()); + $this->assertSame(0, ActivityAttempt::query()->count()); + } + + public function testIndividualPreparationCannotCreateAPartialGroup(): void + { + [$run, $task] = $this->newClaim(); + $commands = $this->groupCommands(); + $local = [...$commands[1], ...ParallelChildGroup::itemMetadata(1, 2, 0, 'mixed')]; + $this->assertSame('local_activity_group_not_admitted', $this->prepareGroupMember($task, $local, 1)['reason']); + $this->assertSame(0, ActivityExecution::query()->count()); + $this->assertSame(0, $run->historyEvents()->count()); + } + + public function testDatabaseObjectKeyOrderingPreservesGroupAdmissionButChangedScalarTypesDoNot(): void + { + [$run, $task] = $this->newClaim(); + $commands = $this->groupCommands(); + $this->assertTrue($this->groupCheckpoint($task, $commands)['checkpointed']); + $execution = ActivityExecution::query()->sole(); + $scheduled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityScheduled)->sole(); + $path = $execution->parallel_group_path; + krsort($path[0]); + $payload = $scheduled->payload; + $payload['parallel_group_path'] = $path; + $scheduled->forceFill([ + 'payload' => $payload, + ])->save(); + $path[0]['parallel_group_size'] = '2'; + $execution->forceFill([ + 'parallel_group_path' => $path, + ])->save(); + $this->assertSame( + 'local_activity_group_admission_mismatch', + $this->prepareGroupMember($task, $commands[1], 2)['reason'] + ); + $this->assertSame(0, ActivityAttempt::query()->count()); + $path[0]['parallel_group_size'] = 2; + $execution->forceFill([ + 'parallel_group_path' => $path, + ])->save(); + $reply = $this->prepareGroupMember($task, $commands[1], 2); + $this->assertTrue($reply['prepared'], $reply['reason'] ?? ''); + $this->assertSame(1, ActivityAttempt::query()->count()); + } + + public function testGroupRoleIsOptionalAndProtocol119CannotUseIt(): void + { + [, $task] = $this->newClaim(); + $bridge = app(PreparedLocalActivityGroupTaskBridge::class); + $this->assertSame(app(WorkflowTaskBridge::class), $bridge); + $reply = $bridge->checkpointLocalActivityGroup( + $task->id, + 'portable-worker', + 1, + 'candidate-only', + 1, + $this->groupCommands(), + '1.19' + ); + $this->assertFalse($reply['checkpointed']); + foreach ([WorkflowTaskBridge::class, PreparedLocalActivityTaskBridge::class] as $contract) { + $custom = \Mockery::mock($contract); + $this->app->instance(WorkflowTaskBridge::class, $custom); + $this->assertSame($custom, app(PreparedLocalActivityGroupTaskBridge::class)); + $this->assertNotInstanceOf(PreparedLocalActivityGroupTaskBridge::class, $custom); + } + } + + public function testNestedAllGroupsCommitEveryLeafAndPreserveEachCompletePath(): void + { + [$run, $task] = $this->newClaim(); + $commands = $this->groupCommands(); + $commands[0] = [...$commands[0], ...ParallelChildGroup::itemMetadata(1, 3, 0, 'mixed')]; + for ($offset = 0; $offset < 2; ++$offset) { + $commands[$offset + 1] = [ + ...$this->descriptor(), + 'type' => 'prepare_local_activity', + ...ParallelChildGroup::payloadForPath([ + ParallelChildGroup::groupEntry(1, 3, $offset + 1, 'mixed'), + ParallelChildGroup::groupEntry(2, 2, $offset, 'activity'), + ]), + ]; + } + $reply = $this->groupCheckpoint($task, $commands); + $this->assertTrue($reply['checkpointed'], $reply['reason'] ?? ''); + $this->assertCount(2, $reply['local_activities']); + $this->assertSame(4, $reply['next_sequence']); + $this->assertSame(0, ActivityAttempt::query()->count()); + foreach ([1, 2] as $index) { + $prepared = $this->prepareGroupMember($task, $commands[$index], $index + 1); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $started = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted) + ->where('payload->activity_attempt_id', $prepared['activity_attempt_id'])->sole(); + $this->assertGroupPath($commands[$index]['parallel_group_path'], $started->payload['parallel_group_path']); + } + } + + public function testNestedMixedGroupHeartbeatsPreserveMembershipProgressAndFixedBudgets(): void + { + [$run, $task] = $this->newClaim(); + $commands = []; + foreach ([0, 1] as $index) { + $path = [ParallelChildGroup::groupEntry(1, 3, $index, 'mixed')]; + if ($index === 1) { + $path[] = ParallelChildGroup::groupEntry(2, 2, 0, 'mixed'); + } + $commands[] = [ + ...$this->descriptor(), + 'type' => 'prepare_local_activity', + 'heartbeat_timeout' => 3, + 'start_to_close_timeout' => 8, + 'schedule_to_close_timeout' => 12, + ...ParallelChildGroup::payloadForPath($path), + ]; + } + $commands[] = [ + 'type' => 'start_timer', + 'delay_seconds' => 1, + ...ParallelChildGroup::payloadForPath([ + ParallelChildGroup::groupEntry(1, 3, 2, 'mixed'), + ParallelChildGroup::groupEntry(2, 2, 1, 'mixed'), + ]), + ]; + $checkpoint = $this->groupCheckpoint($task, $commands); + $this->assertTrue($checkpoint['checkpointed'], $checkpoint['reason'] ?? ''); + $prepared = []; + foreach ([0, 1] as $index) { + $prepared[$index] = $this->prepareGroupMember($task, $commands[$index], $index + 1); + $this->assertTrue($prepared[$index]['prepared'], $prepared[$index]['reason'] ?? ''); + } + Carbon::setTestNow(now()->addSecond()); + try { + foreach ($prepared as $index => $admission) { + $progress = [ + 'details' => [ + 'phase' => $index === 0 ? 'first' : 'second', + ], + ]; + $reply = app(PreparedLocalActivityTaskBridge::class)->heartbeatLocalActivity( + $admission['activity_attempt_id'], + 'portable-worker', + 1, + $progress, + '1.20' + ); + $this->assertTrue($reply['active'], $reply['reason'] ?? ''); + $this->assertTrue($reply['heartbeat_recorded']); + $this->assertFalse($reply['renewed']); + foreach ([ + 'lease_expires_at', + 'start_to_close_deadline_at', + 'schedule_to_close_deadline_at', + ] as $field) { + $this->assertSame($admission[$field], $reply[$field]); + } + $this->assertNotSame($admission['heartbeat_deadline_at'], $reply['heartbeat_deadline_at']); + $event = $run->historyEvents() + ->findOrFail($reply['heartbeat_history_event_id']); + $this->assertSame(HistoryEventType::ActivityHeartbeatRecorded, $event->event_type); + $this->assertSame($admission['activity_attempt_id'], $event->payload['activity_attempt_id']); + $this->assertSame($index + 1, $event->payload['sequence']); + $this->assertSame($progress, $event->payload['progress']); + $this->assertGroupPath( + $commands[$index]['parallel_group_path'], + $event->payload['parallel_group_path'] + ); + } + $this->assertSame( + 2, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityHeartbeatRecorded)->count() + ); + } finally { + Carbon::setTestNow(); + } + } + + public function testLocalFirstGroupStillCommitsTheChildBeforeReturningAdmission(): void + { + [$run, $task] = $this->newClaim(); + $original = $this->groupCommands(); + $commands = [[...$original[1], ...ParallelChildGroup::itemMetadata(1, 2, 0, 'mixed')], + [...$original[0], ...ParallelChildGroup::itemMetadata(1, 2, 1, 'mixed')]]; + $reply = $this->groupCheckpoint($task, $commands); + $this->assertTrue($reply['checkpointed'], $reply['reason'] ?? ''); + $this->assertSame(2, WorkflowRun::query()->count()); + $this->assertSame(0, ActivityAttempt::query()->count()); + $this->assertSame(1, ActivityExecution::query()->sole()->sequence); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::ChildWorkflowScheduled)->count() + ); + } + + public function testAdmissionRollsBackTheChildIfALaterLocalStructuralLimitFails(): void + { + [$run, $task] = $this->newClaim(); + config() + ->set('workflows.v2.structural_limits.pending_activity_count', 1); + $commands = $this->groupCommands(); + $commands[0] = [...$commands[0], ...ParallelChildGroup::itemMetadata(1, 3, 0, 'mixed')]; + $commands[1] = [...$commands[1], ...ParallelChildGroup::itemMetadata(1, 3, 1, 'mixed')]; + $commands[2] = [...$commands[1], ...ParallelChildGroup::itemMetadata(1, 3, 2, 'mixed')]; + try { + $this->groupCheckpoint($task, $commands); + $this->fail('The second local member should exceed the structural limit.'); + } catch (\Workflow\V2\Exceptions\StructuralLimitExceededException $exception) { + $this->assertStringContainsString('pending', $exception->getMessage()); + } + $this->assertSame(1, WorkflowRun::query()->count()); + $this->assertSame(0, ActivityExecution::query()->count()); + $this->assertSame(0, $run->historyEvents()->count()); + } + + public function testRecordedLocalScopeSurvivesResponseLossAndCanonicalClaimInspection(): void + { + [$run, $task] = $this->newClaim(); + $scope = CancellationScopeHistory::open($run, $task, 1, '1.20')->payload['scope_id']; + $descriptor = [ + ...$this->descriptor(), + 'cancellation_scope_id' => $scope, + ]; + $before = $task->fresh() + ->getAttributes(); + $first = $this->prepareGroupMember($task, $descriptor, 2); + $duplicate = $this->prepareGroupMember($task, $descriptor, 2); + $this->assertTrue($first['prepared']); + $this->assertTrue($duplicate['duplicate']); + $this->assertSame($first['activity_attempt_id'], $duplicate['activity_attempt_id']); + $this->assertSame($scope, $duplicate['cancellation_scope_id']); + $execution = ActivityExecution::query()->findOrFail($first['activity_execution_id']); + $this->assertSame($scope, $execution->activity_options['cancellation_scope_id']); + foreach ($run->historyEvents()->whereIn('event_type', [HistoryEventType::ActivityScheduled, + HistoryEventType::ActivityStarted])->get() as $event) { + $this->assertSame($scope, $event->payload['activity']['cancellation_scope_id']); + $this->assertSame($scope, ActivitySnapshot::fromEvent($event)['cancellation_scope_id']); + } + $this->assertNotNull(PortableLocalActivityPreparation::originalStart( + $run, + $execution, + $execution->attempts() + ->sole(), + 'portable-worker', + 1, + )); + $this->assertSame($before, $task->fresh()->getAttributes()); + $this->assertSame(0, $run->tasks()->where('task_type', TaskType::Activity)->count()); + $this->assertNull($run->fresh()->cancellation_request_command_id); + } + + #[DataProvider('changedScopeMembership')] + public function testLocalScopeCannotBeChangedOrDroppedBeforeRetryOrColdRecovery(string $change): void + { + [$run, $task] = $this->newClaim(); + $firstScope = CancellationScopeHistory::open($run, $task, 1, '1.20')->payload['scope_id']; + $secondScope = CancellationScopeHistory::open($run, $task, 2, '1.20')->payload['scope_id']; + $descriptor = [ + ...$this->descriptor(), + 'cancellation_scope_id' => $firstScope, + ]; + $first = $this->prepareGroupMember($task, $descriptor, 3); + $this->assertTrue($first['prepared']); + $changed = $this->descriptor(); + if ($change !== 'omitted') { + $changed['cancellation_scope_id'] = $change === 'root' ? CancellationScopeHistory::ROOT_SCOPE_ID : $secondScope; + } + $retry = $this->prepareGroupMember($task, $changed, 3); + $this->assertFalse($retry['prepared']); + $this->assertSame('local_activity_preparation_mismatch', $retry['reason']); + $task->forceFill([ + 'lease_owner' => 'replacement-worker', + 'attempt_count' => 2, + 'lease_expires_at' => now() + ->addMinutes(5), + ])->save(); + $recovery = PortableLocalActivityPreparation::recover($task->id, 'replacement-worker', 2, 3, $changed, '1.20'); + $this->assertFalse($recovery['recovered']); + $this->assertSame('local_activity_preparation_mismatch', $recovery['reason']); + $this->assertFalse($recovery['claim_released']); + $this->assertSame(1, ActivityAttempt::query()->count()); + $this->assertSame($firstScope, ActivityExecution::query()->sole()->activity_options['cancellation_scope_id']); + $this->assertSame(TaskStatus::Leased, $task->fresh()->status); + $this->assertSame('replacement-worker', $task->fresh()->lease_owner); + } + + public static function changedScopeMembership(): iterable + { + yield 'another recorded scope' => ['other']; + yield 'implicit run root' => ['root']; + yield 'membership removed' => ['omitted']; + } + + #[DataProvider('unrecordedScopeMembership')] + public function testLocalScopeMustBelongToTheExactRunBeforeAdmission(string $change): void + { + [$run, $task] = $this->newClaim(); + $scope = 'unknown-scope'; + if ($change === 'foreign') { + [$foreignRun, $foreignTask] = $this->newClaim(); + $scope = CancellationScopeHistory::open($foreignRun, $foreignTask, 1, '1.20')->payload['scope_id']; + } elseif ($change === 'same position') { + $scope = CancellationScopeHistory::open($run, $task, 1, '1.20')->payload['scope_id']; + } + $reply = $this->prepareGroupMember($task, [ + ...$this->descriptor(), + 'cancellation_scope_id' => $scope, + ], 1); + $this->assertFalse($reply['prepared']); + $this->assertSame('local_activity_scope_not_recorded', $reply['reason']); + $this->assertSame(0, ActivityExecution::query()->count()); + $this->assertSame(0, ActivityAttempt::query()->count()); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::ActivityScheduled)->count()); + $this->assertSame(TaskStatus::Leased, $task->fresh()->status); + } + + public static function unrecordedScopeMembership(): iterable + { + yield 'unknown' => ['unknown']; + yield 'scope in another run' => ['foreign']; + yield 'creation is not before operation' => ['same position']; + } + + #[DataProvider('malformedScopeMembership')] + public function testMalformedLocalScopeIsRefusedWithoutCreatingAnAttempt(mixed $scope): void + { + [$run, $task] = $this->newClaim(); + $reply = $this->prepareGroupMember($task, [ + ...$this->descriptor(), + 'cancellation_scope_id' => $scope, + ], 1); + $this->assertFalse($reply['prepared']); + $this->assertSame('invalid_local_activity_preparation', $reply['reason']); + $this->assertSame(0, $run->historyEvents()->count()); + $this->assertSame(0, ActivityAttempt::query()->count()); + } + + public static function malformedScopeMembership(): iterable + { + yield 'null' => [null]; + yield 'empty' => ['']; + yield 'whitespace' => [' ']; + yield 'numeric' => [7]; + yield 'object' => [[ + 'scope_id' => 'root', + ]]; + yield 'too long' => [str_repeat('x', 256)]; + yield 'invalid UTF-8' => ["\xff"]; + } + + public function testUnscopedLocalDescriptorsKeepTheirHistoricalFingerprintAndSnapshots(): void + { + [$run, $task] = $this->newClaim(); + $normalized = PortableLocalActivityPreparation::normalizeDescriptor($this->descriptor()); + $this->assertArrayNotHasKey('cancellation_scope_id', $normalized); + $reply = $this->prepare($task); + $this->assertTrue($reply['prepared']); + $this->assertArrayNotHasKey('cancellation_scope_id', $reply); + foreach ($run->historyEvents()->get() as $event) { + $this->assertArrayNotHasKey('cancellation_scope_id', $event->payload['activity']); + $this->assertArrayNotHasKey('cancellation_scope_id', ActivitySnapshot::fromEvent($event)); + } + $this->assertSame( + hash('sha256', json_encode($normalized, JSON_THROW_ON_ERROR)), + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted)->sole() + ->payload['local_preparation']['descriptor_fingerprint'] + ); + } + + public function testScopedGroupAdmissionRefusesAnUnknownMemberBeforeCreatingAnySibling(): void + { + [$run, $task] = $this->newClaim(); + $scope = CancellationScopeHistory::open($run, $task, 1, '1.20')->payload['scope_id']; + $commands = []; + foreach ([$scope, 'unknown-scope'] as $index => $id) { + $commands[] = [ + ...$this->descriptor(), + 'type' => 'prepare_local_activity', + 'cancellation_scope_id' => $id, + ...ParallelChildGroup::itemMetadata(2, 2, $index, 'mixed'), + ]; + } + $before = $task->fresh() + ->getAttributes(); + $reply = app(PreparedLocalActivityGroupTaskBridge::class)->checkpointLocalActivityGroup( + $task->id, + 'portable-worker', + 1, + 'scoped-group', + 2, + $commands, + '1.20', + ); + $this->assertFalse($reply['checkpointed']); + $this->assertSame('local_activity_scope_not_recorded', $reply['reason']); + $this->assertSame(0, ActivityExecution::query()->count()); + $this->assertSame(0, ActivityAttempt::query()->count()); + $this->assertSame(1, $run->historyEvents()->count()); + $this->assertSame($before, $task->fresh()->getAttributes()); + } + + public function testScopedLocalSiblingsKeepDistinctMembershipOnTheSameHostingClaim(): void + { + [$run, $task] = $this->newClaim(); + $scopes = [CancellationScopeHistory::open($run, $task, 1, '1.20')->payload['scope_id'], + CancellationScopeHistory::open($run, $task, 2, '1.20')->payload['scope_id']]; + $commands = []; + foreach ($scopes as $index => $scope) { + $commands[] = [ + ...$this->descriptor(), + 'type' => 'prepare_local_activity', + 'cancellation_scope_id' => $scope, + ...ParallelChildGroup::itemMetadata(3, 2, $index, 'mixed'), + ]; + } + $reply = app(PreparedLocalActivityGroupTaskBridge::class)->checkpointLocalActivityGroup( + $task->id, + 'portable-worker', + 1, + 'scoped-group', + 3, + $commands, + '1.20', + ); + $this->assertTrue($reply['checkpointed'], $reply['reason'] ?? ''); + foreach ($commands as $index => $command) { + $prepared = $this->prepareGroupMember($task, $command, 3 + $index); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $this->assertSame($scopes[$index], $prepared['cancellation_scope_id']); + $this->assertSame($task->id, $prepared['workflow_task_id']); + } + $this->assertSame(2, ActivityAttempt::query()->count()); + $this->assertSame(TaskStatus::Leased, $task->fresh()->status); + $this->assertSame('portable-worker', $task->fresh()->lease_owner); + $this->assertSame(0, $run->tasks()->where('task_type', TaskType::Activity)->count()); + } + + #[DataProvider('changedCanonicalScopeReceipt')] + public function testChangedCanonicalScopeSnapshotCannotAuthorizeTheOriginalClaim(string $change): void + { + [$run, $task] = $this->newClaim(); + $scope = CancellationScopeHistory::open($run, $task, 1, '1.20')->payload['scope_id']; + $reply = $this->prepareGroupMember($task, [ + ...$this->descriptor(), + 'cancellation_scope_id' => $scope, + ], 2); + $this->assertTrue($reply['prepared']); + $execution = ActivityExecution::query()->findOrFail($reply['activity_execution_id']); + if ($change === 'execution') { + $execution->forceFill([ + 'activity_options' => [ + ...$execution->activity_options, + 'cancellation_scope_id' => CancellationScopeHistory::ROOT_SCOPE_ID, + ], + ])->save(); + } else { + $event = $run->historyEvents() + ->where('event_type', $change === 'scheduled' + ? HistoryEventType::ActivityScheduled : HistoryEventType::ActivityStarted)->sole(); + $payload = $event->payload; + $payload['activity']['cancellation_scope_id'] = CancellationScopeHistory::ROOT_SCOPE_ID; + $event->forceFill([ + 'payload' => $payload, + ])->save(); + } + $this->assertNull(PortableLocalActivityPreparation::originalStart( + $run, + $execution->fresh(), + $execution->attempts() + ->sole(), + 'portable-worker', + 1, + )); + $this->assertSame(TaskStatus::Leased, $task->fresh()->status); + } + + public static function changedCanonicalScopeReceipt(): iterable + { + yield 'execution reparented' => ['execution']; + yield 'scheduled snapshot reparented' => ['scheduled']; + yield 'started snapshot reparented' => ['started']; + } + + /** @param list> $expected + * @param list> $actual + */ + private function assertGroupPath(array $expected, array $actual): void + { + foreach ($expected as &$entry) { + ksort($entry); + } + unset($entry); + foreach ($actual as &$entry) { + ksort($entry); + } + unset($entry); + $this->assertSame($expected, $actual); + } + + /** + * @return list + */ + private function groupCommands(): array + { + return [[ + 'type' => 'start_child_workflow', + 'workflow_type' => 'python-child', + 'arguments' => Serializer::serializeWithCodec('avro', ['child']), + 'payload_codec' => 'avro', + ...ParallelChildGroup::itemMetadata(1, 2, 0, 'mixed'), + ], [ + ...$this->descriptor(), + 'type' => 'prepare_local_activity', + ...ParallelChildGroup::itemMetadata(1, 2, 1, 'mixed'), + ]]; + } + + /** @param list $commands + * @return array + */ + private function groupCheckpoint(WorkflowTask $task, array $commands): array + { + return app(PreparedLocalActivityGroupTaskBridge::class)->checkpointLocalActivityGroup( + $task->id, + 'portable-worker', + 1, + 'group-one', + 1, + $commands, + '1.20' + ); + } + + /** @param array $descriptor + * @return array + */ + private function prepareGroupMember( + WorkflowTask $task, + array $descriptor, + int $sequence, + string $owner = 'portable-worker', + int $epoch = 1 + ): array { + return PortableLocalActivityPreparation::prepare( + $task->id, + $owner, + $epoch, + $sequence, + 'group-local-attempt', + [ + ...$descriptor, + 'type' => 'record_local_activity', + ], + '1.20' + ); + } + + private function descriptor(): array + { + return [ + 'type' => 'record_local_activity', + 'activity_type' => 'python-local-greeting', + 'arguments' => Serializer::serializeWithCodec('avro', ['Taylor']), + 'payload_codec' => 'avro', + ]; + } + + /** @param array $change + * @return array + */ + private function prepare(WorkflowTask $task, array $change = []): array + { + return PortableLocalActivityPreparation::prepare( + $task->id, + 'portable-worker', + 1, + 1, + 'sdk-local-attempt', + [...$this->descriptor(), ...$change], + '1.20', + ); + } + + /** + * @return array{WorkflowRun, WorkflowTask} + */ + private function newClaim(): array + { + $instance = WorkflowInstance::query()->create([ + 'workflow_class' => TestGreetingWorkflow::class, + 'workflow_type' => 'portable-parent', + 'run_count' => 1, + 'started_at' => now() + ->subMinute(), + ]); + $run = WorkflowRun::query()->create([ + 'workflow_instance_id' => $instance->id, + 'run_number' => 1, + 'workflow_class' => TestGreetingWorkflow::class, + 'workflow_type' => 'portable-parent', + 'status' => RunStatus::Waiting, + 'arguments' => Serializer::serialize(['Taylor']), + 'connection' => 'database', + 'queue' => 'default', + 'compatibility' => 'build-a', + 'started_at' => now() + ->subMinute(), + ]); + $instance->forceFill([ + 'current_run_id' => $run->id, + ])->save(); + $task = WorkflowTask::query()->create([ + 'workflow_run_id' => $run->id, + 'task_type' => TaskType::Workflow, + 'status' => TaskStatus::Leased, + 'attempt_count' => 1, + 'payload' => [], + 'connection' => 'database', + 'queue' => 'default', + 'compatibility' => 'build-a', + 'lease_owner' => 'portable-worker', + 'lease_expires_at' => now() + ->addMinutes(5), + ]); + + return [$run, $task->refresh()]; + } +} diff --git a/tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php b/tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php new file mode 100644 index 000000000..9bde846ec --- /dev/null +++ b/tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php @@ -0,0 +1,520 @@ +set('workflows.v2.compatibility.current', 'build-a'); + config() + ->set('workflows.v2.compatibility.supported', ['build-a']); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + public function testColdRecoveryFencesTheOldAttemptAndPreparesARealRetryUnderTheOriginalBudget(): void + { + [$run, $task, $first, $descriptor] = $this->reclaimedCall(); + $reply = $this->recover($task, $descriptor); + $this->assertTrue($reply['recovered']); + $this->assertFalse($reply['duplicate']); + $this->assertTrue($reply['claim_released']); + $this->assertSame('unknown', $reply['callback_stop_state']); + $this->assertSame('ActivityRetryScheduled', $reply['event_type']); + $this->assertSame(TaskStatus::Completed, $task->refresh()->status); + $this->assertNull($task->lease_expires_at); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $this->assertSame('cold_replay', $event->payload['retry_reason']); + $this->assertSame(7, $event->payload['local_recovery']['original_workflow_task_attempt']); + $this->assertSame(8, $event->payload['local_recovery']['workflow_task_attempt']); + $this->assertSame('original-worker', $event->payload['local_recovery']['original_lease_owner']); + $this->assertSame('replacement-worker', $event->payload['local_recovery']['lease_owner']); + $this->assertSame('expired', $event->payload['activity_attempt']['status']); + $this->assertSame(8, $event->payload['task']['attempt_count']); + $this->assertSame(1, $event->payload['activity_attempt']['attempt_number']); + $timeline = collect(HistoryTimeline::forRun($run->fresh()))->firstWhere('id', $event->id); + $this->assertSame('unknown', $timeline['local_recovery']['callback_stop_state']); + $this->assertSame(7, $timeline['local_recovery']['original_workflow_task_attempt']); + $this->assertSame(8, $timeline['local_recovery']['workflow_task_attempt']); + $this->assertStringContainsString('Callback stop is unknown', $timeline['summary']); + $execution = ActivityExecution::query()->findOrFail($first['activity_execution_id']); + $this->assertSame(ActivityStatus::Pending, $execution->status); + $this->assertSame(ActivityAttemptStatus::Expired, $execution->attempts()->sole()->status); + $this->assertSame(1, $execution->attempt_count); + $this->assertSame( + $first['schedule_to_close_deadline_at'], + $execution->schedule_to_close_deadline_at->toISOString() + ); + $this->assertSame('stale_activity_attempt', $this->lateResult($first)['reason']); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancellationAcknowledged)->count() + ); + $this->assertSame(0, $run->tasks()->where('task_type', TaskType::Activity)->count()); + $retry = WorkflowTask::query()->findOrFail($reply['created_task_ids'][0]); + $retry->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'next-worker', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addMinute(), + ])->save(); + $this->assertSame('local_activity_retry_not_due', $this->prepareRetry($retry, $descriptor)['reason']); + Carbon::setTestNow(now()->addSeconds(2)); + $second = $this->prepareRetry($retry, $descriptor); + $this->assertTrue($second['prepared']); + $this->assertSame(2, $second['attempt_number']); + $this->assertSame(1, $second['workflow_task_attempt']); + $this->assertNotSame($first['activity_attempt_id'], $second['activity_attempt_id']); + $this->assertSame($first['schedule_to_close_deadline_at'], $second['schedule_to_close_deadline_at']); + $result = $this->bridge() + ->recordLocalActivityOutcome($second['activity_attempt_id'], 'next-worker', 1, $this->success(), '1.20'); + $this->assertTrue($result['recorded']); + $replayed = app(LocalActivityExecutor::class)->execute( + $run->fresh(), + $retry->fresh(), + 1, + new LocalActivityCall('python-local-opaque', ['Taylor']) + ); + $this->assertSame($result['event_id'], $replayed['event']->id); + $before = $task->refresh() + ->getAttributes(); + $historyCount = $run->historyEvents() + ->count(); + $this->assertSame([ + ...$reply, + 'duplicate' => true, + ], $this->recover($task, $descriptor)); + $this->assertSame($before, $task->refresh()->getAttributes()); + $this->assertSame($historyCount, $run->historyEvents()->count()); + } + + public function testAnIndependentRecoveryClaimRetiresTheExpiredOriginalTask(): void + { + [$run, $original, $first, $descriptor] = $this->reclaimedCall(reclaim: false); + $task = $original->replicate(); + $task->save(); + $task->forceFill([ + 'lease_owner' => 'replacement-worker', + 'attempt_count' => 8, + 'lease_expires_at' => now() + ->addMinute(), + ])->save(); + $reply = $this->recover($task, $descriptor); + $this->assertTrue($reply['recovered']); + $this->assertSame(TaskStatus::Completed, $original->refresh()->status); + $this->assertNull($original->lease_expires_at); + $this->assertSame(TaskStatus::Completed, $task->refresh()->status); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $this->assertSame($task->id, $event->workflow_task_id); + $this->assertSame($original->id, $event->payload['local_recovery']['original_workflow_task_id']); + $this->assertSame($original->id, $event->payload['activity_attempt']['task_id']); + $this->assertSame(3, $run->tasks()->count()); + $this->assertSame('stale_activity_attempt', $this->lateResult($first)['reason']); + } + + public function testResponseLossReturnsItsReceiptAfterTakeoverAndLaterCancellationWithoutRenewal(): void + { + [$run, $task, , $descriptor] = $this->reclaimedCall(); + $first = $this->recover($task, $descriptor); + $task->forceFill([ + 'lease_owner' => 'another-worker', + 'attempt_count' => 9, + 'lease_expires_at' => now() + ->subSecond(), + ])->save(); + $this->assertTrue(WorkflowStub::loadRun($run->id)->requestCancellation('stop', 30)->accepted()); + $deadline = $run->refresh() + ->cancellation_deadline_at->toISOString(); + $before = $task->refresh() + ->getAttributes(); + $historyCount = $run->historyEvents() + ->count(); + $this->assertSame([ + ...$first, + 'duplicate' => true, + ], $this->recover($task, $descriptor)); + $this->assertSame($before, $task->refresh()->getAttributes()); + $this->assertSame($deadline, $run->refresh()->cancellation_deadline_at->toISOString()); + $this->assertSame($historyCount, $run->historyEvents()->count()); + } + + public function testChangedRecoveryContentsCannotRelabelAReceipt(): void + { + [$run, $task, , $descriptor] = $this->reclaimedCall(); + $this->assertTrue($this->recover($task, $descriptor)['recovered']); + $descriptor['schedule_to_close_timeout'] = 60; + $count = $run->historyEvents() + ->count(); + $this->assertSame('local_activity_recovery_mismatch', $this->recover($task, $descriptor)['reason']); + $this->assertSame($count, $run->historyEvents()->count()); + } + + public function testRetryExhaustionRecordsOneReplayableFailureAndRetainsTheReplacementClaim(): void + { + [$run, $task, $first, $descriptor] = $this->reclaimedCall(maxAttempts: 1); + $before = $task->getAttributes(); + $reply = $this->recover($task, $descriptor); + $this->assertTrue($reply['recovered']); + $this->assertSame('ActivityFailed', $reply['event_type']); + $this->assertFalse($reply['claim_released']); + $this->assertSame([], $reply['created_task_ids']); + $this->assertSame($before, $task->refresh()->getAttributes()); + $this->assertSame(1, $run->failures()->count()); + $this->assertSame( + ActivityAttemptStatus::Expired, + ActivityAttempt::query()->findOrFail($first['activity_attempt_id'])->status + ); + $this->assertSame([ + ...$reply, + 'duplicate' => true, + ], $this->recover($task, $descriptor)); + $this->assertSame(1, $run->failures()->count()); + $this->assertSame(1, $run->tasks()->count()); + $replayed = app(LocalActivityExecutor::class)->execute( + $run->fresh(), + $task->fresh(), + 1, + new LocalActivityCall('python-local-opaque', ['Taylor']) + ); + $this->assertSame($reply['event_id'], $replayed['event']->id); + $this->assertSame('failed', $replayed['status']); + } + + public function testTheOriginalTotalDeadlineOverridesColdReplayWithoutAnotherAttempt(): void + { + [$run, $task, $first, $descriptor] = $this->reclaimedCall(totalTimeout: 4); + $before = $task->getAttributes(); + $reply = $this->recover($task, $descriptor); + $this->assertSame('ActivityTimedOut', $reply['event_type']); + $this->assertFalse($reply['claim_released']); + $this->assertSame([], $reply['created_task_ids']); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $this->assertSame('schedule_to_close', $event->payload['timeout_kind']); + $this->assertSame('unknown', $event->payload['local_recovery']['callback_stop_state']); + $this->assertSame('expired', $event->payload['activity_attempt']['status']); + $this->assertSame($before, $task->refresh()->getAttributes()); + $this->assertSame(1, $run->tasks()->count()); + $this->assertSame(1, ActivityExecution::query()->findOrFail($first['activity_execution_id'])->attempt_count); + $this->assertSame([ + ...$reply, + 'duplicate' => true, + ], $this->recover($task, $descriptor)); + } + + public function testAnExpiredPerAttemptDeadlineUsesTheExistingTimeoutRetryPolicy(): void + { + [$run, $task, $first, $descriptor] = $this->reclaimedCall(startTimeout: 4); + $reply = $this->recover($task, $descriptor); + $this->assertSame('ActivityRetryScheduled', $reply['event_type']); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $this->assertSame('timeout', $event->payload['retry_reason']); + $this->assertSame('start_to_close', $event->payload['timeout_kind']); + $retry = WorkflowTask::query()->findOrFail($reply['created_task_ids'][0]); + Carbon::setTestNow(now()->addSeconds(2)); + $retry->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'next-worker', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addMinute(), + ])->save(); + $prepared = $this->prepareRetry($retry, $descriptor); + $this->assertTrue($prepared['prepared']); + $this->assertSame($first['schedule_to_close_deadline_at'], $prepared['schedule_to_close_deadline_at']); + } + + public function testALivePreviousAttemptCannotBeRecoveredEvenAfterTaskOwnershipChanges(): void + { + [$run, $task, , $descriptor] = $this->reclaimedCall(elapsed: 3); + $before = $run->historyEvents() + ->count(); + $this->assertSame('local_activity_previous_attempt_live', $this->recover($task, $descriptor)['reason']); + $this->assertSame($before, $run->historyEvents()->count()); + $this->assertSame(TaskStatus::Leased, $task->refresh()->status); + } + + public function testALiveOriginalHostingClaimCannotBeRecoveredThroughAnotherTask(): void + { + [$run, $original, , $descriptor] = $this->reclaimedCall(reclaim: false); + $task = $original->replicate(); + $task->save(); + $task->forceFill([ + 'lease_owner' => 'replacement-worker', + 'attempt_count' => 8, + 'lease_expires_at' => now() + ->addMinute(), + ])->save(); + $original->forceFill([ + 'lease_expires_at' => now() + ->addMinute(), + ])->save(); + $before = $run->historyEvents() + ->count(); + $this->assertSame('local_activity_original_claim_live', $this->recover($task, $descriptor)['reason']); + $this->assertSame($before, $run->historyEvents()->count()); + } + + public function testTheSameClaimEpochCannotTurnAMissedLocalLeaseIntoColdRecovery(): void + { + [$run, $task, , $descriptor] = $this->reclaimedCall(); + $task->forceFill([ + 'attempt_count' => 7, + ])->save(); + $reply = PortableLocalActivityPreparation::recover($task->id, 'replacement-worker', 7, 1, $descriptor, '1.20'); + $this->assertSame('local_activity_original_claim_not_reclaimed', $reply['reason']); + $this->assertSame(2, $run->historyEvents()->count()); + } + + public function testCancellationFencesBeforeLeaseExpiryAndStillRequiresTheOriginalSupervisorsStopReceipt(): void + { + [$run, $task, $first, $descriptor] = $this->reclaimedCall(elapsed: 3); + $request = WorkflowStub::loadRun($run->id)->requestCancellation('stop', 30); + $this->assertTrue($request->accepted()); + $deadline = $run->refresh() + ->cancellation_deadline_at->toISOString(); + $before = $task->getAttributes(); + $reply = $this->recover($task, $descriptor); + $this->assertFalse($reply['recovered']); + $this->assertSame('cancellation_requested', $reply['reason']); + $this->assertTrue($reply['fenced']); + $this->assertSame($reply, $this->recover($task, $descriptor)); + $this->assertSame($before, $task->refresh()->getAttributes()); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityRetryScheduled)->count() + ); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancellationAcknowledged)->count() + ); + $acknowledgement = $this->bridge() + ->acknowledgeLocalActivityCancellation( + $first['activity_attempt_id'], + 'original-worker', + $request->commandId(), + 7, + '1.20' + ); + $this->assertTrue($acknowledgement['acknowledged']); + $this->assertSame($deadline, $run->refresh()->cancellation_deadline_at->toISOString()); + $this->assertSame($before, $task->refresh()->getAttributes()); + } + + public function testMissingOriginalStartCannotBeReplacedWithMutableRunningRows(): void + { + [$run, $task, , $descriptor] = $this->reclaimedCall(); + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted)->delete(); + $before = $run->historyEvents() + ->count(); + $this->assertSame('local_activity_preparation_mismatch', $this->recover($task, $descriptor)['reason']); + $this->assertSame($before, $run->historyEvents()->count()); + } + + public function testChangedDescriptorCannotRecoverAnotherOperation(): void + { + [$run, $task, , $descriptor] = $this->reclaimedCall(); + $descriptor['arguments'] = Serializer::serializeWithCodec('avro', ['different']); + $before = $run->historyEvents() + ->count(); + $this->assertSame('local_activity_preparation_mismatch', $this->recover($task, $descriptor)['reason']); + $this->assertSame($before, $run->historyEvents()->count()); + } + + public function testProtocol119CannotEnterThePreparedRecoveryPath(): void + { + [$run, $task, , $descriptor] = $this->reclaimedCall(); + $reply = PortableLocalActivityPreparation::recover($task->id, 'replacement-worker', 8, 1, $descriptor, '1.19'); + $this->assertSame('local_activity_recovery_requires_protocol_1_20', $reply['reason']); + $this->assertSame(2, $run->historyEvents()->count()); + } + + public function testPreparedLocalRoleUsesTheExistingWorkflowBindingWithoutExpandingCustomBridges(): void + { + $this->assertSame(app(WorkflowTaskBridge::class), $this->bridge()); + foreach ([WorkflowTaskBridge::class, CooperativeWorkflowTaskBridge::class] as $contract) { + $custom = \Mockery::mock($contract); + $this->app->instance(WorkflowTaskBridge::class, $custom); + $resolved = $this->app->make(PreparedLocalActivityTaskBridge::class); + $this->assertSame($custom, $resolved); + $this->assertNotInstanceOf(PreparedLocalActivityTaskBridge::class, $resolved); + } + } + + public function testLocalStopAdmissionRejectsProtocol119ThroughTheOptionalRole(): void + { + [$run, , $first] = $this->reclaimedCall(); + $before = $run->historyEvents() + ->count(); + $reply = $this->bridge() + ->acknowledgeLocalActivityCancellation( + $first['activity_attempt_id'], + 'original-worker', + 'not-a-recorded-request', + 7, + '1.19' + ); + $this->assertFalse($reply['acknowledged']); + $this->assertSame('local_activity_stop_receipt_requires_protocol_1_20', $reply['reason']); + $this->assertSame($before, $run->historyEvents()->count()); + } + + private function bridge(): PreparedLocalActivityTaskBridge + { + return $this->app->make(PreparedLocalActivityTaskBridge::class); + } + + /** @param array $descriptor + * @return array + */ + private function recover(WorkflowTask $task, array $descriptor): array + { + return $this->bridge() + ->recoverLocalActivity($task->id, 'replacement-worker', 8, 1, $descriptor, '1.20'); + } + + /** @param array $descriptor + * @return array + */ + private function prepareRetry(WorkflowTask $task, array $descriptor): array + { + return $this->bridge() + ->prepareLocalActivity($task->id, 'next-worker', 1, 1, 'new-local-attempt', $descriptor, '1.20'); + } + + /** @param array $first + * @return array + */ + private function lateResult(array $first): array + { + return $this->bridge() + ->recordLocalActivityOutcome($first['activity_attempt_id'], 'original-worker', 7, $this->success(), '1.20'); + } + + /** + * @return array + */ + private function success(): array + { + return [ + 'outcome' => 'completed', + 'payload_codec' => 'avro', + 'result' => Serializer::serializeWithCodec('avro', 'completed'), + ]; + } + + /** + * @return array{WorkflowRun, WorkflowTask, array, array} + */ + private function reclaimedCall( + int $elapsed = 6, + int $maxAttempts = 3, + int $totalTimeout = 30, + int $startTimeout = 20, + bool $reclaim = true, + ): array { + $instance = WorkflowInstance::query()->create([ + 'workflow_class' => TestGreetingWorkflow::class, + 'workflow_type' => 'portable-parent', + 'run_count' => 1, + 'started_at' => now() + ->subMinute(), + ]); + $run = WorkflowRun::query()->create([ + 'workflow_instance_id' => $instance->id, + 'run_number' => 1, + 'workflow_class' => TestGreetingWorkflow::class, + 'workflow_type' => 'portable-parent', + 'status' => RunStatus::Waiting, + 'arguments' => Serializer::serializeWithCodec('avro', ['Taylor']), + 'payload_codec' => 'avro', + 'connection' => 'database', + 'queue' => 'default', + 'compatibility' => 'build-a', + 'started_at' => now() + ->subMinute(), + ]); + $instance->forceFill([ + 'current_run_id' => $run->id, + ])->save(); + $task = WorkflowTask::query()->create([ + 'workflow_run_id' => $run->id, + 'task_type' => TaskType::Workflow, + 'status' => TaskStatus::Leased, + 'attempt_count' => 7, + 'payload' => [], + 'connection' => 'database', + 'queue' => 'default', + 'compatibility' => 'build-a', + 'lease_owner' => 'original-worker', + 'lease_expires_at' => now() + ->addSeconds(5), + ])->refresh(); + $descriptor = [ + 'type' => 'record_local_activity', + 'activity_type' => 'python-local-opaque', + 'arguments' => Serializer::serializeWithCodec('avro', ['Taylor']), + 'payload_codec' => 'avro', + 'start_to_close_timeout' => min($startTimeout, $totalTimeout), + 'schedule_to_close_timeout' => $totalTimeout, + 'retry_policy' => [ + 'max_attempts' => $maxAttempts, + 'backoff_seconds' => [2], + ], + ]; + $first = $this->bridge() + ->prepareLocalActivity($task->id, 'original-worker', 7, 1, 'original-local-attempt', $descriptor, '1.20'); + $this->assertTrue($first['prepared']); + Carbon::setTestNow(now()->addSeconds($elapsed)); + if ($reclaim) { + $task->forceFill([ + 'lease_owner' => 'replacement-worker', + 'attempt_count' => 8, + 'lease_expires_at' => now() + ->addMinute(), + ])->save(); + } + return [$run->refresh(), $task->refresh(), $first, $descriptor]; + } +} diff --git a/tests/Feature/V2/V2ScopedActivityCancellationTest.php b/tests/Feature/V2/V2ScopedActivityCancellationTest.php new file mode 100644 index 000000000..ae5a5c8e5 --- /dev/null +++ b/tests/Feature/V2/V2ScopedActivityCancellationTest.php @@ -0,0 +1,2427 @@ + 'poll', + ]); + Carbon::setTestNow('2026-10-03T00:00:00Z'); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + #[DataProvider('policies')] + public function testRemoteFencePreservesSiblingAndClaimButDoesNotProveCallbackExit(string $policy, bool $wait): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target, $other] = $this->remotePair($run, $task, $scope, $sibling, $policy); + $bridge = app(ActivityTaskBridge::class); + $claim = $bridge->claimStatus($this->activityTask($run, $target)->id, 'activity-owner'); + $this->assertTrue($claim['claimed'], $claim['reason'] ?? ''); + $claimBefore = $task->fresh() + ->getAttributes(); + $siblingBefore = $other->fresh() + ->getAttributes(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $fence = $this->fence($run, $task, $target, $scope); + $this->assertTrue($fence['fenced']); + $this->assertSame($wait, $fence['waiting_for_stop']); + $this->assertSame(ActivityStatus::Cancelled, $target->fresh()->status); + $this->assertSame(TaskStatus::Cancelled, $this->activityTask($run, $target)->status); + $this->assertHostingShortenedWithoutClaimReplacement($claimBefore, $task); + $this->assertSame($siblingBefore, $other->fresh()->getAttributes()); + $this->assertNull($run->fresh()->cancellation_request_command_id); + $this->assertFalse($run->fresh()->status->isTerminal()); + $this->assertFalse($bridge->status($claim['activity_attempt_id'])['can_continue']); + $published = $bridge->complete($claim['activity_attempt_id'], Serializer::serializeWithCodec('avro', 'stale')); + $this->assertFalse($published['recorded']); + $this->assertFalse(ActivityCancellationCompletion::resolved( + $run->fresh(), + $target->id, + CancellationScopeRequests::context($run, $scope) + )); + $wrong = ActivityCancellationAcknowledgement::recordStopped( + $claim['activity_attempt_id'], + 'different-owner', + $request->payload['request_id'] + ); + $this->assertFalse($wrong['acknowledged']); + $ack = ActivityCancellationAcknowledgement::recordStopped( + $claim['activity_attempt_id'], + 'activity-owner', + $request->payload['request_id'] + ); + $this->assertTrue($ack['acknowledged'], $ack['reason'] ?? ''); + $this->assertFalse($this->fence($run->fresh(), $task, $target, $scope)['waiting_for_stop']); + $this->assertSame( + $fence['history_event_id'], + $this->fence($run->fresh(), $task, $target, $scope)['history_event_id'] + ); + $this->assertTrue( + ActivityCancellationAcknowledgement::recordStopped( + $claim['activity_attempt_id'], + 'activity-owner', + $request->payload['request_id'] + )['duplicate'] + ); + $this->assertTrue($bridge->claimStatus($this->activityTask($run, $other)->id, 'sibling-owner')['claimed']); + } + + public function testRemoteActivityTaskLockCannotExtendHostingClaim(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target] = $this->remotePair($run, $task, $scope, $sibling); + $activityTask = $this->activityTask($run, $target); + $claim = app(ActivityTaskBridge::class)->claimStatus($activityTask->id, 'activity-owner'); + $this->assertTrue($claim['claimed'], $claim['reason'] ?? ''); + config() + ->set('workflows.v2.workflow_task_lease_seconds', 5); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + 4, + 'activity', + '1.20' + ); + $crossed = false; + DB::listen(static function (QueryExecuted $query) use ($activityTask, &$crossed): void { + if (! $crossed && str_contains($query->sql, 'workflow_tasks') + && in_array($activityTask->id, $query->bindings, true)) { + Carbon::setTestNow('2026-10-03T00:00:05Z'); + $crossed = true; + } + }); + $before = $run->historyEvents() + ->count(); + try { + ScopedActivityCancellation::fence( + $run->fresh(), + $task, + $target->id, + $scope, + $request->payload['request_id'], + '1.20' + ); + $this->fail('Waiting for the Activity task lock must not renew the hosting claim.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_workflow_claim_mismatch', $error->getMessage()); + } + $this->assertTrue($crossed); + $this->assertSame($before, $run->historyEvents()->count()); + $this->assertSame(ActivityStatus::Running, $target->fresh()->status); + $this->assertSame(TaskStatus::Leased, $activityTask->fresh()->status); + } + + public static function policies(): iterable + { + yield 'try' => [CancellationPolicy::TryCancel->value, false]; + yield 'wait' => [CancellationPolicy::WaitCancellationCompleted->value, true]; + } + + public static function authorityLockDeadlines(): iterable + { + yield 'current ancestor deadline' => [false]; + yield 'original captured deadline' => [true]; + } + + #[DataProvider('authorityLockDeadlines')] + public function testRemoteTaskLockCannotExtendCancellationAuthority(bool $captured): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target] = $this->remotePair($run, $task, $scope, $sibling); + $activityTask = $this->activityTask($run, $target); + $claim = app(ActivityTaskBridge::class)->claimStatus($activityTask->id, 'activity-owner'); + $this->assertTrue($claim['claimed'], $claim['reason'] ?? ''); + if ($captured) { + $run->forceFill([ + 'run_deadline_at' => now() + ->addSeconds(5), + ])->save(); + } + $request = CancellationScopeRequests::request($run->fresh(), $scope, '1.20', 30); + CancellationScopeDelivery::prepare( + $run->fresh(), + $task, + $scope, + $request->payload['request_id'], + 4, + 'activity', + '1.20' + ); + $run->forceFill([ + 'run_deadline_at' => now() + ->addSeconds($captured ? 100 : 5), + ])->save(); + $crossed = false; + DB::listen(static function (QueryExecuted $query) use ($activityTask, &$crossed): void { + if (! $crossed && str_contains($query->sql, 'workflow_tasks') + && in_array($activityTask->id, $query->bindings, true)) { + Carbon::setTestNow('2026-10-03T00:00:05Z'); + $crossed = true; + } + }); + $before = $run->historyEvents() + ->count(); + try { + ScopedActivityCancellation::fence( + $run->fresh(), + $task, + $target->id, + $scope, + $request->payload['request_id'], + '1.20' + ); + $this->fail('Waiting for the Activity task lock must not extend cancellation authority.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_authority_expired', $error->getMessage()); + } + $this->assertTrue($crossed); + $this->assertSame($before, $run->historyEvents()->count()); + $this->assertSame(ActivityStatus::Running, $target->fresh()->status); + $this->assertSame(TaskStatus::Leased, $activityTask->fresh()->status); + } + + #[DataProvider('policies')] + public function testDeliveryRequiresCanonicalFenceAndWaitPolicyRequiresOriginalStopProof( + string $policy, + bool $wait + ): void { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target, $other] = $this->remotePair($run, $task, $scope, $sibling, $policy); + $claim = app(ActivityTaskBridge::class)->claimStatus($this->activityTask($run, $target)->id, 'activity-owner'); + $this->assertTrue($claim['claimed']); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $this->assertDeliveryRefused($run, $task, $scope, 'cancellation_scope_activity_fence_not_established'); + $this->fence($run, $task, $target, $scope); + if ($wait) { + $this->assertDeliveryRefused($run, $task, $scope, 'cancellation_scope_activity_stop_not_acknowledged'); + $this->assertTrue(ActivityCancellationAcknowledgement::recordStopped( + $claim['activity_attempt_id'], + 'activity-owner', + $request->payload['request_id'] + )['acknowledged']); + } + $before = $task->fresh() + ->getAttributes(); + $event = CancellationScopeDelivery::record( + $run->fresh(), + $task, + $scope, + $request->payload['request_id'], + 4, + 'activity', + '1.20' + ); + $this->assertSame($event->id, CancellationScopeDelivery::recorded($run->fresh(), $scope)->id); + $this->assertHostingShortenedWithoutClaimReplacement($before, $task); + $this->assertSame(ActivityStatus::Pending, $other->fresh()->status); + $this->assertSame( + $request->payload['cancellation']['root_context']['cleanup_deadline_at'], + $event->payload['cancellation']['root_context']['cleanup_deadline_at'] + ); + } + + public function testPendingAdmissionFencePermitsDeliveryWithoutInventingAnOwnerStopReport(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target] = $this->remotePair($run, $task, $scope, $sibling); + $request = CancellationScopeRequests::request($run, $scope, '1.20'); + $this->fence($run, $task, $target, $scope); + $event = CancellationScopeDelivery::record( + $run->fresh(), + $task, + $scope, + $request->payload['request_id'], + 4, + 'activity', + '1.20' + ); + $this->assertSame($event->id, CancellationScopeDelivery::recorded($run->fresh(), $scope)->id); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancellationAcknowledged)->count() + ); + } + + #[DataProvider('deliveryBoundaries')] + public function testAllMembersMustHaveStopProofBeforeParallelScopedDelivery(bool $selection): void + { + [, $run, $task, , $scope] = $this->tree(); + [$first, $second] = $this->remotePair($run, $task, $scope, $scope, parallel: true); + $bridge = app(ActivityTaskBridge::class); + $claims = []; + foreach ([$first, $second] as $execution) { + $claims[] = $bridge->claimStatus($this->activityTask($run, $execution)->id, 'activity-owner'); + } + $request = CancellationScopeRequests::request($run, $scope, '1.20'); + CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + $selection ? 6 : 4, + $selection ? 'selection_handle' : 'parallel', + '1.20', + $selection ? 1 : 2, + $selection ? 4 : null, + $selection ? 2 : 1 + ); + foreach ([$first, $second] as $execution) { + $this->fence($run, $task, $execution, $scope); + } + $this->assertTrue( + ActivityCancellationAcknowledgement::recordStopped( + $claims[0]['activity_attempt_id'], + 'activity-owner', + $request->payload['request_id'] + )['acknowledged'] + ); + $before = $run->historyEvents() + ->count(); + $deliver = static fn () => CancellationScopeDelivery::record( + $run->fresh(), + $task, + $scope, + $request->payload['request_id'], + $selection ? 6 : 4, + $selection ? 'selection_handle' : 'parallel', + '1.20', + $selection ? 1 : 2, + $selection ? 4 : null, + $selection ? 2 : 1 + ); + try { + $deliver(); + $this->fail('One remaining callback must prevent parallel delivery.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_activity_stop_not_acknowledged', $error->getMessage()); + } + $this->assertSame($before, $run->historyEvents()->count()); + $this->assertTrue( + ActivityCancellationAcknowledgement::recordStopped( + $claims[1]['activity_attempt_id'], + 'activity-owner', + $request->payload['request_id'] + )['acknowledged'] + ); + $event = $deliver(); + $this->assertSame($event->id, CancellationScopeDelivery::recorded($run->fresh(), $scope)->id); + } + + public static function deliveryBoundaries(): iterable + { + yield 'parallel call' => [false]; + yield 'selection handle' => [true]; + } + + public function testALaterStopReportCannotRetroactivelyValidateAnEarlyHistoricalDelivery(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target] = $this->remotePair($run, $task, $scope, $sibling); + $claim = app(ActivityTaskBridge::class)->claimStatus($this->activityTask($run, $target)->id, 'activity-owner'); + $request = CancellationScopeRequests::request($run, $scope, '1.20'); + $fence = $this->fence($run, $task, $target, $scope); + // Inject a malformed early marker without going through the public primitive. + WorkflowHistoryEvent::record($run, HistoryEventType::CancellationScopeDelivered, [ + 'schema' => CancellationScopeDelivery::SCHEMA, + 'workflow_run_id' => $run->id, + 'scope_id' => $scope, + 'request_id' => $request->payload['request_id'], + 'cancellation' => $request->payload['cancellation'], + 'sequence' => 4, + 'call_kind' => 'activity', + 'sequence_span' => 1, + 'operation_sequence' => null, + 'operation_sequence_span' => 1, + 'authority_deadline_at' => $fence['cleanup_deadline_at'], + ], $task); + $this->assertTrue( + ActivityCancellationAcknowledgement::recordStopped( + $claim['activity_attempt_id'], + 'activity-owner', + $request->payload['request_id'] + )['acknowledged'] + ); + $this->expectExceptionMessage('cancellation_scope_delivery_history_invalid'); + CancellationScopeDelivery::recorded($run->fresh(), $scope); + } + + public function testMutableProjectionCannotReanimateACanonicallyFencedActivityForDelivery(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target] = $this->remotePair($run, $task, $scope, $sibling, CancellationPolicy::TryCancel->value); + CancellationScopeRequests::request($run, $scope, '1.20'); + $this->fence($run, $task, $target, $scope); + $target->refresh() + ->forceFill([ + 'status' => ActivityStatus::Pending, + ])->save(); + $this->assertSame(ActivityStatus::Pending, $target->fresh()->status); + $this->assertDeliveryRefused($run, $task, $scope, 'cancellation_scope_activity_fence_not_established'); + } + + public function testPendingFenceWinsBeforeCallbackAdmissionWithoutInventingStopReceipt(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target] = $this->remotePair($run, $task, $scope, $sibling); + CancellationScopeRequests::request($run, $scope, '1.20', 30); + $fence = $this->fence($run, $task, $target, $scope); + $this->assertTrue($fence['fenced']); + $this->assertFalse($fence['waiting_for_stop']); + $this->assertFalse( + app(ActivityTaskBridge::class)->claimStatus($this->activityTask($run, $target)->id, 'late-owner')['claimed'] + ); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancellationAcknowledged)->count() + ); + } + + public function testInheritedFenceAndLateReceiptKeepOriginalRootAndBudget(): void + { + [, $run, $task, $parent, $scope, $sibling] = $this->tree(); + [$target] = $this->remotePair($run, $task, $scope, $sibling); + $claim = app(ActivityTaskBridge::class)->claimStatus($this->activityTask($run, $target)->id, 'activity-owner'); + $root = CancellationScopeRequests::request($run, $parent, '1.20', 30, 'original reason'); + Carbon::setTestNow('2026-10-03T00:00:07Z'); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 300, parentScopeId: $parent); + $fence = $this->fence($run, $task, $target, $scope); + $this->assertSame($root->payload['request_id'], $fence['root_request_id']); + $this->assertSame('2026-10-03T00:00:30.000000Z', $fence['cleanup_deadline_at']); + $this->advanceWithWorkflowHeartbeats($task, Carbon::parse('2026-10-03T00:00:31Z')); + $ack = ActivityCancellationAcknowledgement::recordStopped( + $claim['activity_attempt_id'], + 'activity-owner', + $request->payload['request_id'] + ); + $this->assertTrue($ack['acknowledged'], $ack['reason'] ?? ''); + $receipt = WorkflowHistoryEvent::query()->findOrFail($ack['history_event_id']); + $this->assertTrue($receipt->payload['received_after_deadline']); + $this->assertSame($root->payload['request_id'], $receipt->payload['root_request_id']); + $this->assertSame( + $request->id, + CancellationScopeRequests::request($run->fresh(), $scope, '1.20', 300, parentScopeId: $parent)->id + ); + $this->expectExceptionMessage('cancellation_scope_authority_expired'); + $this->fence($run->fresh(), $task, $target, $scope); + } + + public function testLocalReceiptUsesOriginalOwnerAfterWorkflowClaimReplacementAndRunRequest(): void + { + [$workflow, $run, $task, , $scope, $sibling] = $this->tree(); + $commands = []; + foreach ([$scope, $sibling] as $index => $address) { + $commands[] = [ + ...$this->localDescriptor($address), + 'type' => 'prepare_local_activity', + ...ParallelChildGroup::itemMetadata(4, 2, $index, 'mixed'), + ]; + } + $group = app(PreparedLocalActivityGroupTaskBridge::class)->checkpointLocalActivityGroup( + $task->id, + 'scope-owner', + 1, + 'locals', + 4, + $commands, + '1.20' + ); + $this->assertTrue($group['checkpointed'], $group['reason'] ?? ''); + $prepared = PortableLocalActivityPreparation::prepare( + $task->id, + 'scope-owner', + 1, + 4, + 'local-attempt', + [ + ...$commands[0], + 'type' => 'record_local_activity', + ], + '1.20' + ); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $target = ActivityExecution::query()->findOrFail($prepared['activity_execution_id']); + $claimBefore = $task->fresh() + ->getAttributes(); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $fence = $this->fence($run, $task, $target, $scope); + $this->assertTrue($fence['waiting_for_stop']); + $this->assertHostingShortenedWithoutClaimReplacement($claimBefore, $task); + $claimAfterFence = $task->fresh() + ->getAttributes(); + $cancelled = WorkflowHistoryEvent::query()->findOrFail($fence['history_event_id']); + $this->assertSame($task->id, $cancelled->payload['workflow_task_id']); + $this->assertSame('scope-owner', $cancelled->payload['task']['lease_owner']); + $this->assertSame(TaskStatus::Leased->value, $cancelled->payload['task']['status']); + $observation = PortableLocalActivityControl::poll( + $prepared['activity_attempt_id'], + 'scope-owner', + 1, + true, + '1.20' + ); + $this->assertFalse($observation['active']); + $this->assertTrue($observation['stop_required']); + $this->assertFalse($observation['renewed']); + $this->assertTrue($observation['fenced']); + $this->assertSame('cancellation_scope_requested', $observation['reason']); + $this->assertSame($fence['history_event_id'], $observation['cancellation_history_event_id']); + $this->assertSame($cancelled->payload['cancellation_scope'], $observation['cancellation_scope']); + $this->assertArrayNotHasKey('cancellation_request', $observation); + $this->assertSame($claimAfterFence, $task->fresh()->getAttributes()); + $other = PortableLocalActivityPreparation::prepare( + $task->id, + 'scope-owner', + 1, + 5, + 'sibling-attempt', + [ + ...$commands[1], + 'type' => 'record_local_activity', + ], + '1.20' + ); + $this->assertTrue($other['prepared'], $other['reason'] ?? ''); + $survivor = PortableLocalActivityControl::poll($other['activity_attempt_id'], 'scope-owner', 1, true, '1.20'); + $this->assertTrue($survivor['active']); + $this->assertTrue($survivor['renewed']); + $this->assertFalse($survivor['stop_required']); + $this->assertFalse($survivor['heartbeat_recorded']); + $this->assertArrayNotHasKey('cancellation_scope', $survivor); + $task->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 2, + ])->save(); + $workflow->requestCancellation('later whole run', 10); + $replaced = $task->fresh() + ->getAttributes(); + $original = PortableLocalActivityControl::poll( + $prepared['activity_attempt_id'], + 'scope-owner', + 1, + true, + '1.20' + ); + $this->assertSame($observation['cancellation_scope'], $original['cancellation_scope']); + $this->assertSame($observation['cancellation_history_event_id'], $original['cancellation_history_event_id']); + $this->assertFalse($original['renewed']); + $this->assertSame($replaced, $task->fresh()->getAttributes()); + $wrongControl = PortableLocalActivityControl::poll( + $prepared['activity_attempt_id'], + 'replacement', + 2, + true, + '1.20' + ); + $this->assertFalse($wrongControl['active']); + $this->assertArrayNotHasKey('cancellation_scope', $wrongControl); + $wrong = ActivityCancellationAcknowledgement::recordLocalStopped( + $prepared['activity_attempt_id'], + 'replacement', + $request->payload['request_id'], + 2 + ); + $this->assertFalse($wrong['acknowledged']); + $ack = ActivityCancellationAcknowledgement::recordLocalStopped( + $prepared['activity_attempt_id'], + 'scope-owner', + $request->payload['request_id'], + 1 + ); + $this->assertTrue($ack['acknowledged'], $ack['reason'] ?? ''); + $receipt = WorkflowHistoryEvent::query()->findOrFail($ack['history_event_id']); + $this->assertSame('scope-owner', $receipt->payload['task']['lease_owner']); + $this->assertSame(1, $receipt->payload['task']['attempt_count']); + $this->assertSame($fence['cleanup_deadline_at'], $receipt->payload['cleanup_deadline_at']); + $this->assertTrue(ActivityCancellationCompletion::resolved($run->fresh(), $target->id)); + $this->assertSame( + $fence['history_event_id'], + $this->fence($run->fresh(), $task->fresh(), $target, $scope)['history_event_id'] + ); + Carbon::setTestNow(now()->addSeconds(31)); + $expired = PortableLocalActivityControl::poll($prepared['activity_attempt_id'], 'scope-owner', 1, true, '1.20'); + $this->assertSame('cancellation_scope_deadline_expired', $expired['reason']); + $this->assertSame($observation['cancellation_scope'], $expired['cancellation_scope']); + $this->assertFalse($expired['renewed']); + $this->assertSame($replaced, $task->fresh()->getAttributes()); + $corrupted = $cancelled->payload; + $corrupted['cancellation_scope']['scope_id'] = $sibling; + $cancelled->forceFill([ + 'payload' => $corrupted, + ])->save(); + $invalid = PortableLocalActivityControl::poll($prepared['activity_attempt_id'], 'scope-owner', 1, true, '1.20'); + $this->assertSame('cancellation_scope_context_not_recorded', $invalid['reason']); + $this->assertFalse($invalid['active']); + $this->assertArrayNotHasKey('cancellation_scope', $invalid); + $this->assertSame($replaced, $task->fresh()->getAttributes()); + } + + public function testLocalCallbackStopProofPrecedesScopedDeliveryWithoutReleasingTheHostingClaim(): void + { + [, $run, $task, , $scope] = $this->tree(); + $commands = []; + foreach ([0, 1] as $index) { + $commands[] = [ + ...$this->localDescriptor($scope), + 'type' => 'prepare_local_activity', + ...ParallelChildGroup::itemMetadata(4, 2, $index, 'mixed'), + ]; + } + $group = app(PreparedLocalActivityGroupTaskBridge::class)->checkpointLocalActivityGroup( + $task->id, + 'scope-owner', + 1, + 'locals', + 4, + $commands, + '1.20' + ); + $this->assertTrue($group['checkpointed'], $group['reason'] ?? ''); + $prepared = PortableLocalActivityPreparation::prepare( + $task->id, + 'scope-owner', + 1, + 4, + 'local-attempt', + [ + ...$commands[0], + 'type' => 'record_local_activity', + ], + '1.20' + ); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $request = CancellationScopeRequests::request($run, $scope, '1.20'); + $bridge = app(DefaultWorkflowTaskBridge::class); + $scopePreparation = $bridge->prepareCancellationScopeDelivery( + $task->id, + 'scope-owner', + 1, + $scope, + $request->payload['request_id'], + 4, + 'parallel', + 2, + protocolVersion: '1.20' + ); + $this->assertTrue($scopePreparation['prepared'], $scopePreparation['reason'] ?? ''); + $before = $task->fresh() + ->getAttributes(); + $deliver = static fn () => $bridge->deliverCancellationScope( + $task->id, + 'scope-owner', + 1, + $scope, + $request->payload['request_id'], + 4, + 'parallel', + 2, + protocolVersion: '1.20' + ); + $waiting = $deliver(); + $this->assertFalse($waiting['delivered']); + $this->assertSame('cancellation_scope_activity_stop_not_acknowledged', $waiting['reason']); + $this->assertCount(2, $waiting['activity_cancellations']); + $this->assertTrue(ActivityCancellationAcknowledgement::recordLocalStopped( + $prepared['activity_attempt_id'], + 'scope-owner', + $request->payload['request_id'], + 1 + )['acknowledged']); + $event = $deliver(); + $this->assertTrue($event['delivered'], $event['reason'] ?? ''); + $this->assertSame($scopePreparation['preparation_history_event_id'], $event['preparation_history_event_id']); + $this->assertHostingShortenedWithoutClaimReplacement($before, $task); + $this->assertSame($event['history_event_id'], CancellationScopeDelivery::recorded($run->fresh(), $scope)->id); + } + + #[DataProvider('badMetadata')] + public function testCorruptScopeSnapshotCannotAcknowledgeOrResolveAWait(string $change): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target] = $this->remotePair($run, $task, $scope, $sibling); + $claim = app(ActivityTaskBridge::class)->claimStatus($this->activityTask($run, $target)->id, 'activity-owner'); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $fence = $this->fence($run, $task, $target, $scope); + $event = WorkflowHistoryEvent::query()->findOrFail($fence['history_event_id']); + $payload = $event->payload; + if ($change === 'scope') { + $payload['cancellation_scope']['scope_id'] = $sibling; + } elseif ($change === 'deadline') { + $payload['cancellation_scope']['authority_deadline_at'] = '2026-10-03T00:01:00.000000Z'; + } elseif ($change === 'identity') { + $payload['cancellation_scope']['cancellation']['request_id'] = 'different'; + } elseif ($change === 'request_event_array') { + $payload['cancellation_scope']['request_history_event_id'] = [$request->id]; + } else { + $payload['cancellation_scope'] = null; + } + $event->forceFill([ + 'payload' => $payload, + ])->save(); + $this->assertNull(ActivityCancellationContext::forEvent($run->fresh(), $event->fresh())); + $this->assertFalse( + ActivityCancellationAcknowledgement::recordStopped( + $claim['activity_attempt_id'], + 'activity-owner', + $request->payload['request_id'] + )['acknowledged'] + ); + $this->assertFalse( + ActivityCancellationCompletion::resolved($run->fresh(), $target->id, CancellationScopeRequests::context( + $run, + $scope + )) + ); + } + + public static function badMetadata(): iterable + { + foreach (['scope', 'deadline', 'identity', 'null', 'request_event_array'] as $change) { + yield $change => [$change]; + } + } + + public function testWrongScopeRefusesWithoutMutatingActivityOrClaim(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target] = $this->remotePair($run, $task, $scope, $sibling); + CancellationScopeRequests::request($run, $sibling, '1.20', 30); + $before = $target->fresh() + ->getAttributes(); + try { + $this->fence($run, $task, $target, $sibling); + $this->fail('Wrong scope must refuse the fence.'); + } catch (LogicException $exception) { + $this->assertSame('cancellation_scope_activity_membership_mismatch', $exception->getMessage()); + } + $this->assertSame($before, $target->fresh()->getAttributes()); + $this->assertSame(TaskStatus::Leased, $task->fresh()->status); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCancelled)->count()); + } + + public function testPendingRetryFencePreservesTheFailedAttemptAndPreventsAnotherAdmission(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target] = $this->remotePair($run, $task, $scope, $sibling); + $bridge = app(ActivityTaskBridge::class); + $claim = $bridge->claimStatus($this->activityTask($run, $target)->id, 'activity-owner'); + $failed = $bridge->fail($claim['activity_attempt_id'], 'retry me'); + $this->assertTrue($failed['recorded'], $failed['reason'] ?? ''); + $this->assertSame(ActivityStatus::Pending, $target->fresh()->status); + $attempt = ActivityAttempt::query()->findOrFail($claim['activity_attempt_id']); + $before = $attempt->getAttributes(); + CancellationScopeRequests::request($run, $scope, '1.20', 30); + $fence = $this->fence($run, $task, $target, $scope); + $this->assertTrue($fence['fenced']); + $this->assertFalse($fence['waiting_for_stop']); + $this->assertSame($before, $attempt->fresh()->getAttributes()); + $this->assertFalse($bridge->claimStatus($failed['next_task_id'], 'retry-owner')['claimed']); + $this->assertSame(1, $target->attempts()->count()); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancellationAcknowledged)->count() + ); + } + + public function testNaturallyCompletedActivityRetainsResultAndHasNoCancellationFence(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target] = $this->remotePair($run, $task, $scope, $sibling); + $bridge = app(ActivityTaskBridge::class); + $claim = $bridge->claimStatus($this->activityTask($run, $target)->id, 'activity-owner'); + $result = $bridge->complete( + $claim['activity_attempt_id'], + Serializer::serializeWithCodec('avro', 'complete'), + 'avro' + ); + $this->assertTrue($result['recorded'], $result['reason'] ?? ''); + $before = $target->fresh() + ->getAttributes(); + CancellationScopeRequests::request($run, $scope, '1.20', 30); + $this->assertFalse($this->fence($run, $task, $target, $scope)['fenced']); + $this->assertSame($before, $target->fresh()->getAttributes()); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCancelled)->count()); + } + + public function testBoundedAbandonLeavesTheOriginalActivityAuthorityAndCompletionIntact(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target] = $this->remotePair($run, $task, $scope, $sibling, 'abandon'); + $bridge = app(ActivityTaskBridge::class); + $claim = $bridge->claimStatus($this->activityTask($run, $target)->id, 'activity-owner'); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $before = $target->fresh() + ->getAttributes(); + $fence = $this->fence($run, $task, $target, $scope); + $this->assertTrue($fence['abandoned']); + $this->assertFalse($fence['fenced']); + $this->assertSame($before, $target->fresh()->getAttributes()); + $event = CancellationScopeDelivery::record( + $run->fresh(), + $task, + $scope, + $request->payload['request_id'], + 4, + 'activity', + '1.20' + ); + $this->assertSame($event->id, CancellationScopeDelivery::recorded($run->fresh(), $scope)->id); + $this->assertTrue($bridge->status($claim['activity_attempt_id'])['can_continue']); + $this->assertTrue( + $bridge->complete( + $claim['activity_attempt_id'], + Serializer::serializeWithCodec('avro', 'continued'), + 'avro' + )['recorded'] + ); + } + + #[DataProvider('invalidDeliveryHistory')] + public function testDeliveryRejectsUnboundedOrContradictoryActivityHistory(string $change, string $reason): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target] = $this->remotePair($run, $task, $scope, $sibling, 'abandon'); + CancellationScopeRequests::request($run, $scope, '1.20'); + $scheduled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityScheduled) + ->where('payload->activity_execution_id', $target->id) + ->sole(); + $payload = $scheduled->payload; + if ($change === 'projection_deadline') { + $target->forceFill([ + 'schedule_to_close_deadline_at' => now() + ->addSeconds(300), + ])->save(); + } elseif ($change === 'duplicate') { + WorkflowHistoryEvent::record($run, HistoryEventType::ActivityScheduled, [ + 'sequence' => $target->sequence, + 'activity_execution_id' => $target->id, + 'activity' => $payload['activity'], + ]); + } else { + $payload['activity'][$change === 'unknown_policy' ? 'cancellation_policy' + : 'schedule_to_close_deadline_at'] = match ($change) { + 'unknown_policy' => 'unknown', + 'unbounded' => null, + 'malformed_deadline' => 'not-a-deadline', + }; + $scheduled->forceFill([ + 'payload' => $payload, + ])->save(); + } + $this->assertDeliveryRefused($run, $task, $scope, $reason); + } + + public static function invalidDeliveryHistory(): iterable + { + foreach (['projection_deadline', 'unbounded', 'malformed_deadline'] as $change) { + yield $change => [$change, 'cancellation_scope_activity_abandon_not_supported']; + } + foreach (['unknown_policy', 'duplicate'] as $change) { + yield $change => [$change, 'cancellation_scope_activity_history_invalid']; + } + } + + #[DataProvider('invalidFences')] + public function testFenceRefusesInvalidAuthorityWithoutChangingRows(string $condition, string $reason): void + { + [, $run, $task, $parent, $scope, $sibling] = $this->tree($condition === 'shield'); + [$target, $other] = $this->remotePair($run, $task, $scope, $sibling); + if ($condition === 'shield') { + CancellationScopeRequests::request($run, $parent, '1.20', 30); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30, parentScopeId: $parent); + } else { + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + } + $originalClaim = clone $task; + if ($condition === 'claim') { + $task->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 2, + ])->save(); + } elseif ($condition === 'terminal') { + $target->forceFill([ + 'status' => ActivityStatus::Completed, + 'closed_at' => now(), + ])->save(); + } elseif ($condition === 'namespace') { + $task->forceFill([ + 'namespace' => 'other-namespace', + ])->save(); + } elseif ($condition === 'attempt') { + $claim = app(ActivityTaskBridge::class)->claimStatus( + $this->activityTask($run, $target) +->id, + 'activity-owner' + ); + $this->assertTrue($claim['claimed']); + ActivityAttempt::query()->findOrFail($claim['activity_attempt_id']) + ->forceFill([ + 'activity_execution_id' => $other->id, + ])->save(); + } + $before = $target->fresh() + ->getAttributes(); + $claimBefore = $task->fresh() + ->getAttributes(); + $historyBefore = $run->historyEvents() + ->count(); + try { + ScopedActivityCancellation::fence( + $run, + $originalClaim, + $target->id, + $scope, + $request->payload['request_id'], + $condition === 'protocol' ? '1.19' : '1.20' + ); + $this->fail('Invalid scope authority must be refused.'); + } catch (LogicException $exception) { + $this->assertSame($reason, $exception->getMessage()); + } + $this->assertSame($before, $target->fresh()->getAttributes()); + $this->assertSame($claimBefore, $task->fresh()->getAttributes()); + $this->assertSame($historyBefore, $run->historyEvents()->count()); + } + + public static function invalidFences(): iterable + { + yield 'stale claim' => ['claim', 'cancellation_scope_workflow_claim_mismatch']; + yield 'old protocol' => ['protocol', 'cancellation_scope_requires_protocol_1_20']; + yield 'inherited shield' => ['shield', 'cancellation_scope_parent_shielded']; + yield 'wrong namespace' => ['namespace', 'cancellation_scope_workflow_claim_mismatch']; + yield 'wrong attempt execution' => ['attempt', 'cancellation_scope_activity_attempt_changed']; + yield 'terminal projection without fact' => [ + 'terminal', + 'cancellation_scope_activity_terminal_history_not_recorded', + ]; + } + + /** + * @return array + */ + public function testPreparationRetainsWholeScopeAndRecoverableClaimWithoutDelivering(): void + { + [, $run, $task, , $scope] = $this->tree(); + [$first, $second] = $this->remotePair($run, $task, $scope, $scope); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $claim = $task->fresh() + ->getAttributes(); + $next = \Workflow\V2\Support\WorkflowStepHistory::nextDurableCommandSequence($run->fresh()); + $prepared = CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + 4, + 'activity', + '1.20' + ); + $this->assertSame( + [$first->id, $second->id], + array_column($prepared->payload['activity_members'], 'activity_execution_id') + ); + $this->assertHostingShortenedWithoutClaimReplacement($claim, $task); + $this->assertSame($next, \Workflow\V2\Support\WorkflowStepHistory::nextDurableCommandSequence($run->fresh())); + $this->assertNull(CancellationScopeDelivery::recorded($run->fresh(), $scope)); + $this->assertSame(ActivityStatus::Pending, $first->fresh()->status); + $this->assertSame(ActivityStatus::Pending, $second->fresh()->status); + $this->assertNull($run->fresh()->cancellation_request_command_id); + $timeline = collect(\Workflow\V2\Support\HistoryTimeline::fromHistory($run->fresh()))->firstWhere( + 'id', + $prepared->id + ); + $this->assertSame('cancellation_scope', $timeline['kind']); + $this->assertCount(2, $timeline['cancellation_scope']['activity_members']); + } + + public function testPreparationCannotRetainCallerLocksAcrossActivityEffects(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + $this->remotePair($run, $task, $scope, $sibling); + $request = CancellationScopeRequests::request($run, $scope, '1.20'); + $history = $run->historyEvents() + ->count(); + try { + $run->getConnection() + ->transaction(static fn () => CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + 4, + 'activity', + '1.20' + )); + $this->fail('Preparation must commit before any attempt/execution locks.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_preparation_requires_own_transaction', $error->getMessage()); + } + $this->assertSame($history, $run->historyEvents()->count()); + } + + public function testReplacementFinishesPartialFencesUsingFirstPreparationAndDeadline(): void + { + [, $run, $task, , $scope] = $this->tree(); + [$first, $second] = $this->remotePair($run, $task, $scope, $scope); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $prepared = CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + 4, + 'activity', + '1.20' + ); + $this->fence($run, $task, $first, $scope); + Carbon::setTestNow('2026-10-03T00:00:11Z'); + $replacement = $task->fresh(); + $replacement->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 2, + 'lease_expires_at' => now() + ->addSeconds(10), + ])->save(); + $duplicate = CancellationScopeDelivery::prepare( + $run->fresh(), + $replacement, + $scope, + $request->payload['request_id'], + 4, + 'activity', + '1.20' + ); + $this->assertSame($prepared->id, $duplicate->id); + $this->assertSame('2026-10-03T00:00:00.000000Z', $duplicate->recorded_at->toISOString()); + $this->assertSame('2026-10-03T00:00:30.000000Z', $duplicate->payload['authority_deadline_at']); + $this->fence($run->fresh(), $replacement, $second, $scope); + $delivered = CancellationScopeDelivery::record( + $run->fresh(), + $replacement, + $scope, + $request->payload['request_id'], + 4, + 'activity', + '1.20' + ); + $this->assertSame($prepared->id, $delivered->payload['preparation_history_event_id']); + $this->assertSame($prepared->payload['authority_deadline_at'], $delivered->payload['authority_deadline_at']); + $this->assertSame($delivered->id, CancellationScopeDelivery::recorded($run->fresh(), $scope)->id); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeDeliveryPrepared)->count() + ); + } + + public function testSelectedHandleStillRequiresStopProofForOtherOriginalScopeMembers(): void + { + [, $run, $task, , $scope] = $this->tree(); + [$first, $second] = $this->remotePair($run, $task, $scope, $scope); + $bridge = app(ActivityTaskBridge::class); + $claims = []; + foreach ([$first, $second] as $execution) { + $claims[] = $bridge->claimStatus($this->activityTask($run, $execution)->id, 'activity-owner'); + } + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + 6, + 'selection_handle', + '1.20', + 1, + 4 + ); + $this->fence($run, $task, $first, $scope); + $this->assertTrue( + ActivityCancellationAcknowledgement::recordStopped( + $claims[0]['activity_attempt_id'], + 'activity-owner', + $request->payload['request_id'] + )['acknowledged'] + ); + $deliver = static fn () => CancellationScopeDelivery::record( + $run->fresh(), + $task, + $scope, + $request->payload['request_id'], + 6, + 'selection_handle', + '1.20', + 1, + 4 + ); + foreach ([ + 'cancellation_scope_activity_fence_not_established', + 'cancellation_scope_activity_stop_not_acknowledged', + ] as $reason) { + try { + $deliver(); + $this->fail('All original scope members need the policy-specific proof.'); + } catch (LogicException $error) { + $this->assertSame($reason, $error->getMessage()); + } + $this->fence($run, $task, $second, $scope); + } + $this->assertTrue( + ActivityCancellationAcknowledgement::recordStopped( + $claims[1]['activity_attempt_id'], + 'activity-owner', + $request->payload['request_id'] + )['acknowledged'] + ); + $delivered = $deliver(); + $this->assertSame($delivered->id, CancellationScopeDelivery::recorded($run->fresh(), $scope)->id); + } + + public function testAnUnpreparedScopeCannotFenceAnUnfinishedActivity(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target] = $this->remotePair($run, $task, $scope, $sibling); + $request = CancellationScopeRequests::request($run, $scope, '1.20'); + $history = $run->historyEvents() + ->count(); + try { + ScopedActivityCancellation::fence( + $run, + $task, + $target->id, + $scope, + $request->payload['request_id'], + '1.20' + ); + $this->fail('Cancellation effects require prior retained preparation.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_delivery_not_prepared', $error->getMessage()); + } + $this->assertSame($history, $run->historyEvents()->count()); + $this->assertSame(ActivityStatus::Pending, $target->fresh()->status); + } + + #[DataProvider('weakenedPolicies')] + public function testAChangedOriginalPolicyCannotWeakenThePreparedWaitContract(string $policy): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target] = $this->remotePair($run, $task, $scope, $sibling); + $request = CancellationScopeRequests::request($run, $scope, '1.20'); + CancellationScopeDelivery::prepare($run, $task, $scope, $request->payload['request_id'], 4, 'activity', '1.20'); + $scheduled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityScheduled)->where( + 'payload->activity_execution_id', + $target->id + )->sole(); + $payload = $scheduled->payload; + $payload['activity']['cancellation_policy'] = $policy; + $scheduled->forceFill([ + 'payload' => $payload, + ])->save(); + $this->expectExceptionMessage('cancellation_scope_delivery_history_invalid'); + ScopedActivityCancellation::fence( + $run->fresh(), + $task, + $target->id, + $scope, + $request->payload['request_id'], + '1.20' + ); + } + + public static function weakenedPolicies(): iterable + { + yield 'try cancel' => ['try_cancel']; + yield 'abandon' => ['abandon']; + } + + #[DataProvider('changedPreparations')] + public function testPartialFencesCannotMoveTheOriginalPreparedCall( + int $sequence, + string $kind, + int $span, + ?int $operation, + int $operationSpan + ): void { + [, $run, $task, , $scope] = $this->tree(); + [$first, $second] = $this->remotePair($run, $task, $scope, $scope); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $prepared = CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + 4, + 'activity', + '1.20' + ); + $this->fence($run, $task, $first, $scope); + $history = $run->historyEvents() + ->count(); + try { + CancellationScopeDelivery::prepare( + $run->fresh(), + $task, + $scope, + $request->payload['request_id'], + $sequence, + $kind, + '1.20', + $span, + $operation, + $operationSpan + ); + $this->fail('Partial effects must preserve the first preparation.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_delivery_mismatch', $error->getMessage()); + } + $this->assertSame($history, $run->historyEvents()->count()); + $this->assertSame(ActivityStatus::Pending, $second->fresh()->status); + $this->assertSame($prepared->id, CancellationScopeDelivery::prepared($run->fresh(), $scope)->id); + } + + public static function changedPreparations(): iterable + { + yield 'move call' => [5, 'activity', 1, null, 1]; + yield 'change kind' => [4, 'timer', 1, null, 1]; + yield 'move selected operation' => [6, 'selection_handle', 1, 5, 1]; + yield 'widen call' => [4, 'parallel', 2, null, 1]; + } + + public function testExpiredPreparationRemainsReadableButCannotRenewEffectAuthority(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target] = $this->remotePair($run, $task, $scope, $sibling); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $prepared = CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + 4, + 'activity', + '1.20' + ); + $this->advanceWithWorkflowHeartbeats($task, Carbon::parse('2026-10-03T00:00:31Z')); + $this->assertSame($prepared->id, CancellationScopeDelivery::prepared($run->fresh(), $scope)->id); + $this->expectExceptionMessage('cancellation_scope_authority_expired'); + ScopedActivityCancellation::fence( + $run->fresh(), + $task, + $target->id, + $scope, + $request->payload['request_id'], + '1.20' + ); + } + + public function testPreparationRejectsNewAdmissionsButAllowsRecordedPrefixReplay(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + $this->remotePair($run, $task, $scope, $sibling); + $request = CancellationScopeRequests::request($run, $scope, '1.20'); + CancellationScopeDelivery::prepare($run, $task, $scope, $request->payload['request_id'], 4, 'activity', '1.20'); + $command = [ + 'type' => 'schedule_activity', + 'activity_type' => TestGreetingActivity::class, + 'arguments' => Serializer::serializeWithCodec('avro', ['Taylor']), + 'payload_codec' => 'avro', + 'cancellation_scope_id' => $scope, + ]; + $history = $run->historyEvents() + ->count(); + $bridge = app(DefaultWorkflowTaskBridge::class); + $this->assertNull($bridge->validateCancellationScopeMembership($run->fresh(), [$command], 4)); + $this->remotePair($run->fresh(), $task, $scope, $sibling); + $this->assertSame($history, $run->historyEvents()->count()); + $result = $bridge->checkpointCancellationScopePrefix( + $task->id, + 'scope-owner', + 1, + 'late-scope-work', + 6, + [$command], + '1.20' + ); + $this->assertFalse($result['checkpointed']); + $this->assertSame('operation_scope_cancellation_prepared', $result['reason']); + $this->assertSame($history, $run->historyEvents()->count()); + $this->assertCount(2, $run->activityExecutions()->get()); + $command['cancellation_scope_id'] = $sibling; + $this->assertNull($bridge->validateCancellationScopeMembership($run->fresh(), [$command], 6)); + } + + #[DataProvider('ancestorShielding')] + public function testPreparedAncestorRejectsNewDescendantWorkUnlessShielded(bool $shield): void + { + [, $run, $task, $parent, $scope, $sibling] = $this->tree($shield); + $this->remotePair($run, $task, $parent, $parent); + $request = CancellationScopeRequests::request($run, $parent, '1.20'); + CancellationScopeDelivery::prepare( + $run, + $task, + $parent, + $request->payload['request_id'], + 4, + 'activity', + '1.20' + ); + $this->assertSame( + $shield ? null : 'operation_scope_cancellation_prepared', + CancellationScopeDelivery::admissionRefusal($run->fresh(), $scope, 6) + ); + $this->assertNull(CancellationScopeDelivery::admissionRefusal($run->fresh(), $sibling, 6)); + // Shielding does not let a new command consume the pending delivery position. + $this->assertSame( + $shield ? 'operation_cancellation_delivery_reserved' : 'operation_scope_cancellation_prepared', + CancellationScopeDelivery::admissionRefusal($run->fresh(), $scope, 4) + ); + } + + public static function ancestorShielding(): iterable + { + yield 'inherited' => [false]; + yield 'shielded' => [true]; + } + + public function testActivityOutsideTheOriginalPreparationCannotBeFenced(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + $this->remotePair($run, $task, $scope, $sibling); + $request = CancellationScopeRequests::request($run, $scope, '1.20'); + CancellationScopeDelivery::prepare($run, $task, $scope, $request->payload['request_id'], 4, 'activity', '1.20'); + $reply = app(DefaultWorkflowTaskBridge::class)->checkpointCancellationScopePrefix( + $task->id, + 'scope-owner', + 1, + 'later-sibling', + 6, + [[ + 'type' => 'schedule_activity', + 'activity_type' => TestGreetingActivity::class, + 'arguments' => Serializer::serializeWithCodec('avro', ['Taylor']), + 'payload_codec' => 'avro', + 'cancellation_scope_id' => $sibling, + ]], + '1.20' + ); + $this->assertTrue($reply['checkpointed'], $reply['reason'] ?? ''); + $execution = $run->activityExecutions() + ->where('sequence', 6) + ->sole(); + $scheduled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityScheduled) + ->where('payload->activity_execution_id', $execution->id) + ->sole(); + // Corrupt a later admission to bypass the bridge's prepared-scope guard. + $payload = $scheduled->payload; + $payload['cancellation_scope_id'] = $scope; + $payload['activity']['cancellation_scope_id'] = $scope; + $scheduled->forceFill([ + 'payload' => $payload, + ])->save(); + $execution->forceFill([ + 'activity_options' => [ + ...$execution->activity_options, + 'cancellation_scope_id' => $scope, + ], + ])->save(); + $history = $run->historyEvents() + ->count(); + try { + ScopedActivityCancellation::fence( + $run->fresh(), + $task, + $execution->id, + $scope, + $request->payload['request_id'], + '1.20' + ); + $this->fail('A later Activity cannot become an original preparation member.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_activity_not_prepared', $error->getMessage()); + } + $this->assertSame($history, $run->historyEvents()->count()); + $this->assertSame(ActivityStatus::Pending, $execution->fresh()->status); + } + + public function testAuthenticatedBridgeRetainsPreparationWhileWaitingForOriginalStopProof(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target, $other] = $this->remotePair($run, $task, $scope, $sibling); + $activityBridge = app(ActivityTaskBridge::class); + $claim = $activityBridge->claimStatus($this->activityTask($run, $target)->id, 'activity-owner'); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $bridge = app(DefaultWorkflowTaskBridge::class); + $this->assertInstanceOf(\Workflow\V2\Contracts\PreparedCancellationScopeTaskBridge::class, $bridge); + $taskBefore = $task->fresh() + ->getAttributes(); + $prepared = $bridge->prepareCancellationScopeDelivery( + $task->id, + 'scope-owner', + 1, + $scope, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + ); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $this->assertFalse($prepared['delivered']); + $this->assertFalse($prepared['claim_released']); + $this->assertCount(1, $prepared['activity_members']); + $this->assertSame($target->id, $prepared['activity_members'][0]['activity_execution_id']); + $this->assertSame($prepared['history_event_id'], $prepared['preparation_history_event_id']); + $this->assertSame(ActivityStatus::Running, $target->fresh()->status); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCancelled)->count()); + $deliver = static fn () => $bridge->deliverCancellationScope( + $task->id, + 'scope-owner', + 1, + $scope, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + ); + $waiting = $deliver(); + $this->assertTrue($waiting['prepared']); + $this->assertFalse($waiting['delivered']); + $this->assertSame('cancellation_scope_activity_stop_not_acknowledged', $waiting['reason']); + $this->assertSame($prepared['preparation_history_event_id'], $waiting['preparation_history_event_id']); + $this->assertSame($prepared['authority_deadline_at'], $waiting['authority_deadline_at']); + $this->assertSame($prepared['cancellation'], $waiting['cancellation']); + $this->assertTrue($waiting['activity_cancellations'][0]['fenced']); + $this->assertTrue($waiting['activity_cancellations'][0]['waiting_for_stop']); + $this->assertSame($target->id, $waiting['activity_cancellations'][0]['activity_execution_id']); + $this->assertFalse($activityBridge->status($claim['activity_attempt_id'])['can_continue']); + $this->assertFalse($activityBridge->complete( + $claim['activity_attempt_id'], + Serializer::serializeWithCodec('avro', 'stale'), + 'avro' + )['recorded']); + $this->assertSame($waiting, $deliver()); + $this->assertTrue(ActivityCancellationAcknowledgement::recordStopped( + $claim['activity_attempt_id'], + 'activity-owner', + $request->payload['request_id'] + )['acknowledged']); + $delivered = $deliver(); + $this->assertTrue($delivered['delivered'], $delivered['reason'] ?? ''); + $this->assertSame($prepared['preparation_history_event_id'], $delivered['preparation_history_event_id']); + $this->assertSame($prepared['activity_members'], $delivered['activity_members']); + $this->assertSame($delivered['history_event_id'], $deliver()['history_event_id']); + $this->assertHostingShortenedWithoutClaimReplacement($taskBefore, $task); + $this->assertSame(ActivityStatus::Pending, $other->fresh()->status); + } + + public function testAuthenticatedDispatchCancelsOriginalMembersOutsideTheSelectedCall(): void + { + [, $run, $task, , $scope] = $this->tree(); + [$first, $second] = $this->remotePair($run, $task, $scope, $scope); + $activityBridge = app(ActivityTaskBridge::class); + $claims = []; + foreach ([$first, $second] as $execution) { + $claims[] = $activityBridge->claimStatus($this->activityTask($run, $execution)->id, 'activity-owner'); + } + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prepared = $bridge->prepareCancellationScopeDelivery( + $task->id, + 'scope-owner', + 1, + $scope, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + ); + $this->assertCount(2, $prepared['activity_members']); + $deliver = static fn () => $bridge->deliverCancellationScope( + $task->id, + 'scope-owner', + 1, + $scope, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + ); + $waiting = $deliver(); + $this->assertFalse($waiting['delivered']); + $this->assertSame('cancellation_scope_activity_stop_not_acknowledged', $waiting['reason']); + $this->assertSame( + [$first->id, $second->id], + array_column($waiting['activity_cancellations'], 'activity_execution_id') + ); + foreach ([$first, $second] as $execution) { + $this->assertSame(ActivityStatus::Cancelled, $execution->fresh()->status); + } + foreach ($claims as $index => $claim) { + $this->assertTrue(ActivityCancellationAcknowledgement::recordStopped( + $claim['activity_attempt_id'], + 'activity-owner', + $request->payload['request_id'] + )['acknowledged']); + $result = $deliver(); + $this->assertSame($index === 1, $result['delivered']); + $this->assertSame($prepared['authority_deadline_at'], $result['authority_deadline_at']); + } + } + + public function testAuthenticatedDispatchRetriesPartialCommittedProgressAfterClaimReplacement(): void + { + [, $run, $task, , $scope] = $this->tree(); + [$first, $second] = $this->remotePair($run, $task, $scope, $scope); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prepared = $bridge->prepareCancellationScopeDelivery( + $task->id, + 'scope-owner', + 1, + $scope, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + ); + $replace = true; + WorkflowHistoryEvent::created(static function (WorkflowHistoryEvent $event) use ( + $task, + $first, + &$replace + ): void { + if (! $replace || $event->event_type !== HistoryEventType::ActivityCancelled + || ($event->payload['activity_execution_id'] ?? null) !== $first->id) { + return; + } + $replace = false; + $event->getConnection() + ->afterCommit(static function () use ($task): void { + Carbon::setTestNow('2026-10-03T00:00:11Z'); + $task->fresh() + ->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 2, + 'lease_expires_at' => now() + ->addSeconds(10), + ])->save(); + }); + }); + $partial = $bridge->deliverCancellationScope( + $task->id, + 'scope-owner', + 1, + $scope, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + ); + $this->assertFalse($partial['delivered']); + $this->assertSame('cancellation_scope_workflow_claim_mismatch', $partial['reason']); + $this->assertCount(1, $partial['activity_cancellations']); + $firstReceipt = $partial['activity_cancellations'][0]; + $this->assertTrue($firstReceipt['fenced']); + $this->assertSame(ActivityStatus::Cancelled, $first->fresh()->status); + $this->assertSame(ActivityStatus::Pending, $second->fresh()->status); + $this->assertNull(CancellationScopeDelivery::recorded($run->fresh(), $scope)); + $resumed = $bridge->deliverCancellationScope( + $task->id, + 'replacement', + 2, + $scope, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + ); + $this->assertTrue($resumed['delivered'], $resumed['reason'] ?? ''); + $this->assertSame($firstReceipt, $resumed['activity_cancellations'][0]); + $this->assertSame($prepared['preparation_history_event_id'], $resumed['preparation_history_event_id']); + $this->assertSame($prepared['authority_deadline_at'], $resumed['authority_deadline_at']); + $this->assertSame('2026-10-03T00:00:30.000000Z', $resumed['authority_deadline_at']); + $this->assertSame(2, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCancelled)->count()); + } + + #[DataProvider('dispatchRefusals')] + public function testAuthenticatedDispatchValidatesTheOriginalBoundaryBeforeEffects( + string $change, + string $reason + ): void { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target, $other] = $this->remotePair($run, $task, $scope, $sibling); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prepared = $bridge->prepareCancellationScopeDelivery( + $task->id, + 'scope-owner', + 1, + $scope, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + ); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + if ($change === 'deadline') { + $this->advanceWithWorkflowHeartbeats($task, Carbon::parse('2026-10-03T00:00:30Z')); + } + $before = $run->historyEvents() + ->count(); + $result = $bridge->deliverCancellationScope( + $task->id, + 'scope-owner', + 1, + $change === 'scope' ? $sibling : $scope, + $change === 'request' ? 'wrong-request' : $request->payload['request_id'], + $change === 'sequence' ? 5 : 4, + $change === 'kind' ? 'timer' : 'activity', + $change === 'span' ? 2 : 1, + $change === 'operation' ? 4 : null, + $change === 'operation-span' ? 2 : 1, + '1.20' + ); + $this->assertFalse($result['delivered']); + $this->assertSame($reason, $result['reason']); + $this->assertSame($before, $run->historyEvents()->count()); + $this->assertSame(ActivityStatus::Pending, $target->fresh()->status); + $this->assertSame(ActivityStatus::Pending, $other->fresh()->status); + } + + public static function dispatchRefusals(): iterable + { + foreach (['sequence', 'kind'] as $change) { + yield $change => [$change, 'cancellation_scope_delivery_mismatch']; + } + yield 'span' => ['span', 'invalid_cancellation_delivery']; + yield 'operation' => ['operation', 'invalid_cancellation_delivery']; + yield 'operation-span' => ['operation-span', 'invalid_cancellation_delivery']; + yield 'request' => ['request', 'cancellation_scope_request_mismatch']; + yield 'scope' => ['scope', 'cancellation_scope_delivery_not_prepared']; + yield 'deadline' => ['deadline', 'cancellation_scope_authority_expired']; + } + + #[DataProvider('dispatchPolicies')] + public function testAuthenticatedDispatchPreservesNaturalCompletionTryAndBoundedAbandon( + string $policy, + bool $completed + ): void { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target, $other] = $this->remotePair($run, $task, $scope, $sibling, $policy); + $activityBridge = app(ActivityTaskBridge::class); + $claim = $activityBridge->claimStatus($this->activityTask($run, $target)->id, 'activity-owner'); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prepared = $bridge->prepareCancellationScopeDelivery( + $task->id, + 'scope-owner', + 1, + $scope, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + ); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + if ($completed) { + $this->assertTrue($activityBridge->complete( + $claim['activity_attempt_id'], + Serializer::serializeWithCodec('avro', 'completed'), + 'avro' + )['recorded']); + } + $before = $target->fresh() + ->getAttributes(); + $delivered = $bridge->deliverCancellationScope( + $task->id, + 'scope-owner', + 1, + $scope, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + ); + $this->assertTrue($delivered['delivered'], $delivered['reason'] ?? ''); + $this->assertSame($prepared['preparation_history_event_id'], $delivered['preparation_history_event_id']); + $receipt = $delivered['activity_cancellations'][0]; + $this->assertFalse($receipt['waiting_for_stop']); + $this->assertSame($policy === 'abandon', $receipt['abandoned']); + $this->assertSame(! $completed && $policy === 'try_cancel', $receipt['fenced']); + if ($completed || $policy === 'abandon') { + $this->assertSame($before, $target->fresh()->getAttributes()); + } else { + $this->assertSame(ActivityStatus::Cancelled, $target->fresh()->status); + } + $this->assertSame(ActivityStatus::Pending, $other->fresh()->status); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancellationAcknowledged)->count() + ); + if ($policy === 'abandon') { + $this->assertTrue($activityBridge->status($claim['activity_attempt_id'])['can_continue']); + $this->assertTrue($activityBridge->complete( + $claim['activity_attempt_id'], + Serializer::serializeWithCodec('avro', 'continued'), + 'avro' + )['recorded']); + } + } + + public static function dispatchPolicies(): iterable + { + yield 'natural completion' => ['wait_cancellation_completed', true]; + yield 'try cancel' => ['try_cancel', false]; + yield 'bounded abandon' => ['abandon', false]; + } + + #[DataProvider('bridgeRefusals')] + public function testAuthenticatedBridgeRefusesInvalidClaimsWithoutEffects( + string $owner, + int $attempt, + string $protocol, + string $requestMode, + int $sequence, + string $kind, + string $reason, + ): void { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target, $other] = $this->remotePair($run, $task, $scope, $sibling); + $request = CancellationScopeRequests::request($run, $scope, '1.20'); + if ($requestMode === 'expired') { + Carbon::setTestNow('2026-10-03T00:05:01Z'); + } elseif ($requestMode === 'namespace') { + $task->forceFill([ + 'namespace' => 'other-namespace', + ])->save(); + } elseif ($requestMode === 'activity-task') { + $task->forceFill([ + 'task_type' => TaskType::Activity, + ])->save(); + } + $history = $run->historyEvents() + ->count(); + $before = $task->fresh() + ->getAttributes(); + $result = app(DefaultWorkflowTaskBridge::class)->prepareCancellationScopeDelivery( + $requestMode === 'missing' ? 'missing-task' : $task->id, + $owner, + $attempt, + $requestMode === 'sibling' ? $sibling : $scope, + $requestMode === 'request' ? 'wrong-request' : $request->payload['request_id'], + $sequence, + $kind, + protocolVersion: $protocol + ); + $this->assertFalse($result['prepared']); + $this->assertFalse($result['delivered']); + $this->assertFalse($result['claim_released']); + $this->assertSame($reason, $result['reason']); + $this->assertSame($history, $run->historyEvents()->count()); + $this->assertSame($before, $task->fresh()->getAttributes()); + $this->assertSame(ActivityStatus::Pending, $target->fresh()->status); + $this->assertSame(ActivityStatus::Pending, $other->fresh()->status); + } + + public static function bridgeRefusals(): iterable + { + yield 'wrong owner' => ['other', 1, '1.20', '', 4, 'activity', 'cancellation_scope_workflow_claim_mismatch']; + yield 'wrong attempt' => [ + 'scope-owner', + 2, + '1.20', + '', + 4, + 'activity', + 'cancellation_scope_workflow_claim_mismatch', + ]; + yield 'invalid attempt' => ['scope-owner', 0, '1.20', '', 4, 'activity', 'invalid_cancellation_scope_delivery']; + yield 'expired claim' => [ + 'scope-owner', + 1, + '1.20', + 'expired', + 4, + 'activity', + 'cancellation_scope_workflow_claim_mismatch', + ]; + yield 'namespace mismatch' => [ + 'scope-owner', + 1, + '1.20', + 'namespace', + 4, + 'activity', + 'cancellation_scope_workflow_claim_mismatch', + ]; + yield 'wrong task type' => [ + 'scope-owner', + 1, + '1.20', + 'activity-task', + 4, + 'activity', + 'cancellation_scope_workflow_claim_mismatch', + ]; + yield 'missing task' => ['scope-owner', 1, '1.20', 'missing', 4, 'activity', 'task_not_found']; + yield 'older protocol' => [ + 'scope-owner', + 1, + '1.19', + '', + 4, + 'activity', + 'cancellation_scope_requires_protocol_1_20', + ]; + yield 'wrong protocol major' => [ + 'scope-owner', + 1, + '2.0', + '', + 4, + 'activity', + 'cancellation_scope_requires_protocol_1_20', + ]; + yield 'wrong request' => [ + 'scope-owner', + 1, + '1.20', + 'request', + 4, + 'activity', + 'cancellation_scope_request_mismatch', + ]; + yield 'wrong scope' => [ + 'scope-owner', + 1, + '1.20', + 'sibling', + 4, + 'activity', + 'cancellation_scope_request_mismatch', + ]; + yield 'later call' => ['scope-owner', 1, '1.20', '', 99, 'activity', 'cancellation_delivery_sequence_mismatch']; + yield 'invalid call' => ['scope-owner', 1, '1.20', '', 4, 'missing', 'invalid_cancellation_delivery']; + } + + public function testAuthenticatedBridgeCannotDeliverWithoutPreparation(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + $this->remotePair($run, $task, $scope, $sibling); + $request = CancellationScopeRequests::request($run, $scope, '1.20'); + $history = $run->historyEvents() + ->count(); + $result = app(DefaultWorkflowTaskBridge::class)->deliverCancellationScope( + $task->id, + 'scope-owner', + 1, + $scope, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + ); + $this->assertFalse($result['prepared']); + $this->assertFalse($result['delivered']); + $this->assertSame('cancellation_scope_delivery_not_prepared', $result['reason']); + $this->assertSame($history, $run->historyEvents()->count()); + } + + public function testAuthenticatedBridgeReplacementReusesFirstPreparationAndBoundary(): void + { + [, $run, $task, , $scope] = $this->tree(); + [$first, $second] = $this->remotePair($run, $task, $scope, $scope); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prepared = $bridge->prepareCancellationScopeDelivery( + $task->id, + 'scope-owner', + 1, + $scope, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + ); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $this->fence($run, $task, $first, $scope); + Carbon::setTestNow('2026-10-03T00:00:11Z'); + $replacement = $task->fresh(); + $replacement->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 2, + 'lease_expires_at' => now() + ->addSeconds(10), + ])->save(); + $history = $run->historyEvents() + ->count(); + $stale = $bridge->prepareCancellationScopeDelivery( + $task->id, + 'scope-owner', + 1, + $scope, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + ); + $this->assertFalse($stale['prepared']); + $this->assertSame('cancellation_scope_workflow_claim_mismatch', $stale['reason']); + $duplicate = $bridge->prepareCancellationScopeDelivery( + $task->id, + 'replacement', + 2, + $scope, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + ); + $this->assertSame($prepared, $duplicate); + $moved = $bridge->prepareCancellationScopeDelivery( + $task->id, + 'replacement', + 2, + $scope, + $request->payload['request_id'], + 5, + 'activity', + protocolVersion: '1.20' + ); + $this->assertFalse($moved['prepared']); + $this->assertSame('cancellation_scope_delivery_mismatch', $moved['reason']); + $this->assertSame($history, $run->historyEvents()->count()); + $delivered = $bridge->deliverCancellationScope( + $task->id, + 'replacement', + 2, + $scope, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + ); + $this->assertTrue($delivered['delivered'], $delivered['reason'] ?? ''); + $this->assertSame($prepared['preparation_history_event_id'], $delivered['preparation_history_event_id']); + $this->assertSame($prepared['cancellation'], $delivered['cancellation']); + $this->assertSame('2026-10-03T00:00:30.000000Z', $delivered['authority_deadline_at']); + $this->assertCount(2, $delivered['activity_cancellations']); + $this->assertSame(ActivityStatus::Cancelled, $second->fresh()->status); + } + + #[DataProvider('bridgeTransactionPhases')] + public function testAuthenticatedBridgeCannotRetainCallerTransactionLocks(bool $preparing): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + $this->remotePair($run, $task, $scope, $sibling); + $request = CancellationScopeRequests::request($run, $scope, '1.20'); + $history = $run->historyEvents() + ->count(); + $method = $preparing ? 'prepareCancellationScopeDelivery' : 'deliverCancellationScope'; + $result = $run->getConnection() + ->transaction(static fn () => app(DefaultWorkflowTaskBridge::class)->{$method}( + $task->id, + 'scope-owner', + 1, + $scope, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + )); + $this->assertFalse($result['prepared']); + $this->assertFalse($result['delivered']); + $this->assertSame('cancellation_scope_preparation_requires_own_transaction', $result['reason']); + $this->assertSame($history, $run->historyEvents()->count()); + } + + public static function bridgeTransactionPhases(): iterable + { + yield 'prepare' => [true]; + yield 'record delivery' => [false]; + } + + #[DataProvider('ancestorShielding')] + public function testAuthenticatedBridgeFencesUnshieldedDescendantsAndPreservesShieldedOnes(bool $shield): void + { + [, $run, $task, $parent, $scope] = $this->tree($shield); + [$first, $descendant] = $this->remotePair($run, $task, $parent, $scope); + $request = CancellationScopeRequests::request($run, $parent, '1.20'); + $bridge = app(DefaultWorkflowTaskBridge::class); + $prepared = $bridge->prepareCancellationScopeDelivery( + $task->id, + 'scope-owner', + 1, + $parent, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + ); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $result = $bridge->deliverCancellationScope( + $task->id, + 'scope-owner', + 1, + $parent, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + ); + $this->assertTrue($result['delivered'], $result['reason'] ?? ''); + $this->assertSame(ActivityStatus::Cancelled, $first->fresh()->status); + $this->assertSame($shield ? ActivityStatus::Pending : ActivityStatus::Cancelled, $descendant->fresh()->status); + $this->assertNull(CancellationScopeDelivery::prepared($run->fresh(), $scope)); + $this->assertSame($prepared['preparation_history_event_id'], $result['preparation_history_event_id']); + } + + public function testAuthenticatedBridgeDispatchesMixedChildrenAndActivitiesWithoutCancellingSiblings(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$first, $other] = $this->remotePair($run, $task, $scope, $sibling); + $bridge = app(DefaultWorkflowTaskBridge::class); + $scheduled = $bridge->checkpointCancellationScopePrefix( + $task->id, + 'scope-owner', + 1, + 'child-prefix', + 6, + [[ + 'type' => 'start_child_workflow', + 'workflow_type' => 'child-scope-fixture', + 'arguments' => Serializer::serializeWithCodec('avro', ['child']), + 'payload_codec' => 'avro', + 'cancellation_policy' => 'try_cancel', + 'cancellation_scope_id' => $scope, + ]], + '1.20' + ); + $this->assertTrue($scheduled['checkpointed'], $scheduled['reason'] ?? ''); + $request = CancellationScopeRequests::request($run, $scope, '1.20'); + $prepared = $bridge->prepareCancellationScopeDelivery( + $task->id, + 'scope-owner', + 1, + $scope, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + ); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $claimBefore = $task->fresh() + ->getAttributes(); + $result = $bridge->deliverCancellationScope( + $task->id, + 'scope-owner', + 1, + $scope, + $request->payload['request_id'], + 4, + 'activity', + protocolVersion: '1.20' + ); + $this->assertTrue($result['prepared']); + $this->assertTrue($result['delivered'], $result['reason'] ?? ''); + $this->assertFalse($result['claim_released']); + $this->assertHostingShortenedWithoutClaimReplacement($claimBefore, $task); + $this->assertSame(ActivityStatus::Cancelled, $first->fresh()->status); + $this->assertSame(ActivityStatus::Pending, $other->fresh()->status); + $this->assertCount(1, $result['activity_cancellations']); + $this->assertCount(1, $result['child_cancellations']); + $receipt = $result['child_cancellations'][0]; + $target = WorkflowRun::query()->findOrFail($receipt['child_workflow_run_id']); + $this->assertFalse($target->status->isTerminal()); + $this->assertSame($receipt['request_id'], $target->cancellation_request_command_id); + $this->assertNotNull(CancellationScopeDelivery::recorded($run->fresh(), $scope)); + } + + public function testConflictingNewScopeCannotFenceWorkBeforeTheOriginalDeliveryBoundaryIsRejected(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target, $other] = $this->remotePair($run, $task, $scope, $sibling, 'try_cancel'); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + CancellationScopeDelivery::prepare($run, $task, $scope, $request->payload['request_id'], 6, 'timer', '1.20'); + // Simulate an already corrupted command history to qualify the actor preflight, + // independently of the admission guard exercised by the following test. + $opened = $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeOpened) + ->where('payload->scope_id', $sibling) + ->sole(); + $openingPayload = $opened->payload; + unset($openingPayload['task']); + WorkflowHistoryEvent::record($run->fresh(), HistoryEventType::CancellationScopeOpened, [ + ...$openingPayload, + 'sequence' => 6, + 'scope_id' => 'conflicting-unrelated-scope', + 'parent_scope_id' => $sibling, + ], $task); + $result = PortableCancellationScopeDelivery::mutate( + false, + $task->id, + 'scope-owner', + 1, + $scope, + $request->payload['request_id'], + 6, + 'timer', + 1, + null, + 1, + '1.20' + ); + $this->assertFalse($result['delivered']); + $this->assertSame('cancellation_delivery_shape_mismatch', $result['reason']); + $this->assertSame(ActivityStatus::Pending, $target->fresh()->status); + $this->assertSame(ActivityStatus::Pending, $other->fresh()->status); + $this->assertSame(TaskStatus::Ready, $this->activityTask($run, $target)->status); + $this->assertNull(CancellationScopeDelivery::recorded($run->fresh(), $scope)); + } + + public function testPendingDeliveryReservesItsFutureCommandWhileExistingSiblingWorkAndReplayContinue(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + [$target, $other] = $this->remotePair($run, $task, $scope, $sibling, 'try_cancel'); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $preparation = CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + $request->payload['request_id'], + 6, + 'timer', + '1.20' + ); + $count = $run->historyEvents() + ->count(); + try { + CancellationScopeHistory::open($run->fresh(), $task, 6, '1.20', $sibling); + $this->fail('A new unrelated scope consumed the prepared future boundary.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_command_sequence_reserved', $error->getMessage()); + } + $this->assertSame($count, $run->historyEvents()->count()); + $this->assertSame(ActivityStatus::Pending, $target->fresh()->status); + $this->assertSame(ActivityStatus::Pending, $other->fresh()->status); + $conflictingPrefix = app(DefaultWorkflowTaskBridge::class)->checkpointCancellationScopePrefix( + $task->id, + 'scope-owner', + 1, + 'conflicting-outside-prefix', + 6, + [[ + 'type' => 'schedule_activity', + 'activity_type' => TestGreetingActivity::class, + 'arguments' => Serializer::serializeWithCodec('avro', ['another']), + 'payload_codec' => 'avro', + 'cancellation_scope_id' => $sibling, + 'cancellation_policy' => 'try_cancel', + 'schedule_to_close_timeout' => 120, + ]], + '1.20' + ); + $this->assertFalse($conflictingPrefix['checkpointed']); + $this->assertSame('operation_cancellation_delivery_reserved', $conflictingPrefix['reason']); + $this->assertSame($count, $run->historyEvents()->count()); + $this->assertSame($preparation->id, CancellationScopeDelivery::prepare( + $run->fresh(), + $task, + $scope, + $request->payload['request_id'], + 6, + 'timer', + '1.20' + )->id); + $this->assertSame( + $sibling, + CancellationScopeHistory::open($run->fresh(), $task, 3, '1.20')->payload['scope_id'] + ); + $result = PortableCancellationScopeDelivery::mutate( + false, + $task->id, + 'scope-owner', + 1, + $scope, + $request->payload['request_id'], + 6, + 'timer', + 1, + null, + 1, + '1.20' + ); + $this->assertTrue($result['delivered'], $result['reason'] ?? ''); + $this->assertSame(ActivityStatus::Cancelled, $target->fresh()->status); + $this->assertSame(ActivityStatus::Pending, $other->fresh()->status); + $later = CancellationScopeHistory::open($run->fresh(), $task, 7, '1.20', $sibling); + $this->assertSame($sibling, $later->payload['parent_scope_id']); + $this->assertSame($preparation->id, CancellationScopeDelivery::prepare( + $run->fresh(), + $task, + $scope, + $request->payload['request_id'], + 6, + 'timer', + '1.20' + )->id); + } + + public function testAnotherAcceptedScopeCannotPrepareTheSameFutureAuthoredPosition(): void + { + [, $run, $task, , $scope, $sibling] = $this->tree(); + $this->remotePair($run, $task, $scope, $sibling, 'try_cancel'); + $first = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $second = CancellationScopeRequests::request($run, $sibling, '1.20', 30); + CancellationScopeDelivery::prepare($run, $task, $scope, $first->payload['request_id'], 6, 'timer', '1.20'); + $count = $run->historyEvents() + ->count(); + try { + CancellationScopeDelivery::prepare( + $run->fresh(), + $task, + $sibling, + $second->payload['request_id'], + 6, + 'timer', + '1.20' + ); + $this->fail('Two scopes prepared the same future authored position.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_command_sequence_reserved', $error->getMessage()); + } + $this->assertSame($count, $run->historyEvents()->count()); + $this->assertNull(CancellationScopeDelivery::prepared($run->fresh(), $sibling)); + } + + /** + * @return array{WorkflowStub, WorkflowRun, WorkflowTask, string, string, string} + */ + private function tree(bool $shield = false): array + { + $workflow = WorkflowStub::make(TestSignalWorkflow::class); + $workflow->start('scoped-activity'); + $run = $workflow->run(); + $task = $run->tasks() + ->where('task_type', TaskType::Workflow)->sole(); + $task->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'scope-owner', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addMinutes(5), + ])->save(); + $parent = CancellationScopeHistory::open($run, $task, 1, '1.20')->payload['scope_id']; + $scope = CancellationScopeHistory::open($run, $task, 2, '1.20', $parent, $shield)->payload['scope_id']; + $sibling = CancellationScopeHistory::open($run, $task, 3, '1.20')->payload['scope_id']; + return [$workflow, $run, $task->refresh(), $parent, $scope, $sibling]; + } + + /** + * @return array{ActivityExecution, ActivityExecution} + */ + private function remotePair( + WorkflowRun $run, + WorkflowTask $task, + string $scope, + string $sibling, + string $policy = 'wait_cancellation_completed', + bool $parallel = false, + ): array { + $commands = []; + foreach ([$scope, $sibling] as $index => $address) { + $commands[] = [ + 'type' => 'schedule_activity', + 'activity_type' => TestGreetingActivity::class, + 'arguments' => Serializer::serializeWithCodec('avro', ['Taylor']), + 'payload_codec' => 'avro', + 'cancellation_scope_id' => $address, + 'cancellation_policy' => $policy, + ...($parallel ? ParallelChildGroup::itemMetadata(4, 2, $index, 'activity') : []), + 'schedule_to_close_timeout' => 120, + 'retry_policy' => [ + 'max_attempts' => 2, + 'backoff_seconds' => [0], + ], + ]; + } + $reply = app(DefaultWorkflowTaskBridge::class)->checkpointCancellationScopePrefix( + $task->id, + 'scope-owner', + 1, + 'remote-pair', + 4, + $commands, + '1.20' + ); + $this->assertTrue($reply['checkpointed'], $reply['reason'] ?? ''); + return [$run->activityExecutions()->where('sequence', 4)->sole(), + $run->activityExecutions() + ->where('sequence', 5) + ->sole()]; + } + + private function activityTask(WorkflowRun $run, ActivityExecution $execution): WorkflowTask + { + return $run->tasks() + ->where('task_type', TaskType::Activity)->where('payload->activity_execution_id', $execution->id)->sole(); + } + + /** + * @return array + */ + private function localDescriptor(string $scope): array + { + return [ + 'type' => 'record_local_activity', + 'activity_type' => 'python-local-greeting', + 'arguments' => Serializer::serializeWithCodec('avro', ['Taylor']), + 'payload_codec' => 'avro', + 'cancellation_scope_id' => $scope, + 'cancellation_policy' => 'wait_cancellation_completed', + ]; + } + + private function fence(WorkflowRun $run, WorkflowTask $task, ActivityExecution $execution, string $scope): array + { + $execution = $execution->fresh(); + if (! in_array($execution->status, [ActivityStatus::Completed, ActivityStatus::Failed], true) + && ($execution->activity_options['cancellation_scope_id'] ?? 'root') === $scope + && CancellationScopeDelivery::prepared($run->fresh(), $scope) === null) { + $scheduled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityScheduled) + ->where('payload->activity_execution_id', $execution->id) + ->sole(); + $size = $scheduled->payload['parallel_group_size'] ?? 1; + $kind = $size > 1 ? 'parallel' + : (\Workflow\V2\Support\CooperativeCancellationDelivery::callKindAt( + $run->fresh(), + $execution->sequence + ) ?? 'activity'); + $base = $scheduled->payload['parallel_group_base_sequence'] ?? $execution->sequence; + $mixed = $size > 1 && $run->activityExecutions() + ->whereBetween('sequence', [$base, $base + $size - 1]) + ->get() + ->contains(static fn (ActivityExecution $member): bool => + ($member->activity_options['cancellation_scope_id'] ?? 'root') !== $scope); + CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + CancellationScopeRequests::context($run, $scope)->requestId, + $mixed ? $base + $size : $base, + $mixed ? 'selection_handle' : $kind, + '1.20', + $mixed ? 1 : $size, + $mixed ? $execution->sequence : null, + 1 + ); + } + return ScopedActivityCancellation::fence( + $run, + $task, + $execution->id, + $scope, + CancellationScopeRequests::context($run, $scope)->requestId, + '1.20' + ); + } + + private function assertDeliveryRefused(WorkflowRun $run, WorkflowTask $task, string $scope, string $reason): void + { + $before = $task->fresh() + ->getAttributes(); + $history = $run->historyEvents() + ->count(); + try { + if (CancellationScopeDelivery::prepared($run->fresh(), $scope) === null) { + CancellationScopeDelivery::prepare( + $run, + $task, + $scope, + CancellationScopeRequests::context($run, $scope)->requestId, + 4, + 'activity', + '1.20' + ); + $history = $run->historyEvents() + ->count(); + $this->assertHostingShortenedWithoutClaimReplacement($before, $task); + $before = $task->fresh() + ->getAttributes(); + } + CancellationScopeDelivery::record( + $run->fresh(), + $task, + $scope, + CancellationScopeRequests::context($run, $scope)->requestId, + 4, + 'activity', + '1.20' + ); + $this->fail('Unproved activity cancellation must prevent delivery.'); + } catch (LogicException $error) { + $this->assertSame($reason, $error->getMessage()); + } + $this->assertSame($before, $task->fresh()->getAttributes()); + $this->assertSame($history, $run->historyEvents()->count()); + } +} diff --git a/tests/Feature/V2/V2ScopedChildCancellationTest.php b/tests/Feature/V2/V2ScopedChildCancellationTest.php new file mode 100644 index 000000000..4623a23fc --- /dev/null +++ b/tests/Feature/V2/V2ScopedChildCancellationTest.php @@ -0,0 +1,925 @@ + 'poll', + ]); + Carbon::setTestNow('2026-10-03T00:00:00Z'); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + public function testPreparationFreezesAllPoliciesWithoutTouchingChildrenSiblingsOrShields(): void + { + [$run, $task, $scope, $sibling, $shield] = $this->tree(); + $target = []; + foreach (['try_cancel', 'wait_cancellation_completed', 'abandon'] as $index => $policy) { + $target[] = $this->child($run, $task, $scope, 4 + $index, $policy); + } + $other = $this->child($run, $task, $sibling, 7); + $shielded = $this->child($run, $task, $shield, 8); + $before = $task->fresh() + ->getAttributes(); + $prepared = $this->prepare($run, $task, $scope); + $this->assertSame(array_column($target, 'child_workflow_run_id'), array_column( + $prepared['child_members'], + 'child_workflow_run_id' + )); + $this->assertSame(['try_cancel', 'wait_cancellation_completed', 'abandon'], array_column( + $prepared['child_members'], + 'cancellation_policy' + )); + foreach ([...$target, $other, $shielded] as $child) { + $this->assertNull( + WorkflowRun::query()->findOrFail($child['child_workflow_run_id'])->cancellation_request_command_id + ); + } + $this->assertHostingShortenedWithoutClaimReplacement($before, $task); + $this->assertNull($run->fresh()->cancellation_request_command_id); + $this->assertFalse($run->fresh()->status->isTerminal()); + $timeline = collect(HistoryTimeline::fromHistory($run->fresh())) + ->firstWhere('id', $prepared['preparation_history_event_id']); + $this->assertSame($prepared['child_members'], ScopedChildCancellation::normalizeMembers( + $timeline['cancellation_scope']['child_members'] + )); + } + + public function testColdPreparationIgnoresMissingProjectionsAndChangedCurrentRun(): void + { + [$run, $task, $scope] = $this->tree(); + $child = $this->child($run, $task, $scope, 4); + $prepared = $this->prepare($run, $task, $scope); + WorkflowChildCall::query()->where('parent_workflow_run_id', $run->id)->delete(); + WorkflowLink::query()->where('parent_workflow_run_id', $run->id)->delete(); + $target = WorkflowRun::query()->findOrFail($child['child_workflow_run_id']); + $target->instance() + ->firstOrFail() + ->forceFill([ + 'current_run_id' => null, + ])->save(); + $cold = CancellationScopeDelivery::prepared($run->fresh(), $scope); + $this->assertSame($prepared['preparation_history_event_id'], $cold->id); + $this->assertSame($prepared['child_members'], ScopedChildCancellation::normalizeMembers( + $cold->payload['child_members'] + )); + $this->assertSame($prepared, $this->prepare($run->fresh(), $task->fresh(), $scope)); + } + + public function testOnlyChildStartsCommittedBeforePreparationSelectItsOriginalRun(): void + { + [$run, $task, $scope] = $this->tree(); + $child = $this->child($run, $task, $scope, 4); + $this->started($run, $task, $child, 'continued-before-preparation'); + $prepared = $this->prepare($run, $task, $scope); + $this->assertSame('continued-before-preparation', $prepared['child_members'][0]['child_workflow_run_id']); + $this->started($run, $task, $child, 'continued-after-preparation'); + $this->assertSame('continued-after-preparation', ScopedChildCancellation::members( + $run->fresh(), + $scope + )[0]['child_workflow_run_id']); + $this->assertSame($prepared, $this->prepare($run->fresh(), $task->fresh(), $scope)); + $this->assertSame($prepared['child_members'], ScopedChildCancellation::normalizeMembers( + CancellationScopeDelivery::prepared($run->fresh(), $scope)->payload['child_members'] + )); + } + + public function testNaturalChildCompletionKeepsTheOriginalPreparedTargetAndPolicy(): void + { + [$run, $task, $scope] = $this->tree(); + $child = $this->child($run, $task, $scope, 4); + $prepared = $this->prepare($run, $task, $scope); + WorkflowHistoryEvent::record($run->fresh(), HistoryEventType::ChildRunCompleted, [ + ...array_intersect_key($child, array_flip([ + 'sequence', 'child_call_id', 'workflow_link_id', 'child_workflow_instance_id', + 'child_workflow_run_id', 'child_workflow_class', 'child_workflow_type', + ])), + 'result' => Serializer::serializeWithCodec('avro', ['done']), + ], $task); + $this->assertSame($prepared, $this->prepare($run->fresh(), $task->fresh(), $scope)); + $this->assertSame(1, $run->historyEvents()->where('event_type', HistoryEventType::ChildRunCompleted)->count()); + } + + #[DataProvider('changedMemberFields')] + public function testColdReplayRejectsAlteredOriginalChildMembership(string $field, mixed $value): void + { + [$run, $task, $scope] = $this->tree(); + $this->child($run, $task, $scope, 4); + $prepared = $this->prepare($run, $task, $scope); + $event = WorkflowHistoryEvent::query()->findOrFail($prepared['preparation_history_event_id']); + $payload = $event->payload; + $payload['child_members'][0][$field] = $value; + $event->forceFill([ + 'payload' => $payload, + ])->save(); + $this->expectExceptionMessage('cancellation_scope_delivery_history_invalid'); + CancellationScopeDelivery::prepared($run->fresh(), $scope); + } + + public static function changedMemberFields(): iterable + { + yield 'sequence' => ['sequence', 99]; + yield 'call' => ['child_call_id', 'other-call']; + yield 'instance' => ['child_workflow_instance_id', 'other-instance']; + yield 'run' => ['child_workflow_run_id', 'other-run']; + yield 'policy' => ['cancellation_policy', 'abandon']; + yield 'hash' => ['descriptor_hash', str_repeat('a', 64)]; + yield 'unknown policy' => ['cancellation_policy', 'invalid']; + yield 'extra field' => ['extra', 'value']; + yield 'empty identity' => ['child_call_id', '']; + yield 'wrong identity type' => ['child_workflow_run_id', 42]; + } + + public function testReplacementClaimReplaysOriginalDeadlineAndDatabaseKeyOrder(): void + { + [$run, $task, $scope] = $this->tree(); + $this->child($run, $task, $scope, 4); + $prepared = $this->prepare($run, $task, $scope); + $event = WorkflowHistoryEvent::query()->findOrFail($prepared['preparation_history_event_id']); + $payload = $event->payload; + $payload['child_members'][0] = array_reverse($payload['child_members'][0], true); + $event->forceFill([ + 'payload' => $payload, + ])->save(); + Carbon::setTestNow('2026-10-03T00:00:20Z'); + $repair = TaskWatchdog::runPass(respectThrottle: false, runIds: [$run->id]); + $this->assertSame([], $repair['existing_task_failures']); + $this->assertSame(1, $repair['repaired_existing_tasks']); + $this->assertTrue( + app(DefaultWorkflowTaskBridge::class)->claimStatus($task->id, 'replacement-child-owner')['claimed'] + ); + $this->assertSame(2, $task->fresh()->attempt_count); + $this->assertSame($prepared, $this->prepare($run->fresh(), $task->fresh(), $scope)); + $this->assertSame('2026-10-03T00:00:30.000000Z', $prepared['authority_deadline_at']); + } + + public function testChildSnapshotCannotBeMistakenForCommittedCancellation(): void + { + [$run, $task, $scope] = $this->tree(); + $child = $this->child($run, $task, $scope, 4); + $prepared = $this->prepare($run, $task, $scope); + $before = $run->historyEvents() + ->count(); + try { + CancellationScopeDelivery::record( + $run->fresh(), + $task->fresh(), + $scope, + $prepared['request_id'], + 4, + 'child', + '1.20' + ); + $this->fail('A child membership snapshot was treated as cancellation proof.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_child_delivery_not_established', $error->getMessage()); + } + $this->assertSame($before, $run->historyEvents()->count()); + $this->assertNull( + WorkflowRun::query()->findOrFail($child['child_workflow_run_id'])->cancellation_request_command_id + ); + $this->assertNull(CancellationScopeDelivery::recorded($run->fresh(), $scope)); + } + + public function testCanonicalChildRequestPreservesThePreparedScopeAndColdDuplicateAfterExpiry(): void + { + [$run, $task, $scope, $sibling] = $this->tree(); + $child = $this->child($run, $task, $scope, 4); + $other = $this->child($run, $task, $sibling, 5); + CancellationScopeRequests::request($run, $scope, '1.20', 30, 'maintenance'); + $prepared = $this->prepare($run, $task, $scope); + WorkflowChildCall::query()->where('parent_workflow_run_id', $run->id)->delete(); + WorkflowLink::query()->where('parent_workflow_run_id', $run->id)->delete(); + Carbon::setTestNow('2026-10-03T00:00:12Z'); + $target = WorkflowStub::loadRun($child['child_workflow_run_id']); + $request = $target->attemptRequestCancellationFromScope( + $run->id, + $scope, + $prepared['preparation_history_event_id'], + $child['child_call_id'] + ); + $this->assertTrue($request->accepted(), $request->rejectionReason() ?? ''); + $context = $request->cancellationContext(); + $this->assertSame($prepared['cancellation'], $context->scopeOrigin->toArray()); + $this->assertSame($prepared['request_id'], $context->parentRequestId); + $this->assertSame('maintenance', $context->reason); + $this->assertSame('2026-10-03T00:00:00.000000Z', $context->requestedAt()->toISOString()); + $this->assertSame('2026-10-03T00:00:30.000000Z', $context->deadline()->toISOString()); + $this->assertFalse($target->run()->fresh()->status->isTerminal()); + $this->assertNull($run->fresh()->cancellation_request_command_id); + $this->assertNull( + WorkflowRun::query()->findOrFail($other['child_workflow_run_id'])->cancellation_request_command_id + ); + $this->assertSame( + $context->toArray(), + CooperativeCancellationDelivery::context($target->run()->fresh())->toArray() + ); + Carbon::setTestNow('2026-10-03T00:01:00Z'); + $duplicate = WorkflowStub::loadRun($child['child_workflow_run_id'])->attemptRequestCancellationFromScope( + $run->id, + $scope, + $prepared['preparation_history_event_id'], + $child['child_call_id'] + ); + $this->assertSame($context->toArray(), $duplicate->cancellationContext()->toArray()); + $this->assertSame( + 1, + $target->run() + ->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationRequested)->count() + ); + } + + #[DataProvider('invalidChildRequests')] + public function testUnpreparedOrExpiredChildRequestCannotCancelTheTarget(string $mutation): void + { + [$run, $task, $scope, $sibling] = $this->tree(); + $child = $this->child($run, $task, $scope, 4, $mutation === 'abandon' ? 'abandon' : 'try_cancel'); + $prepared = $this->prepare($run, $task, $scope); + $preparationId = $prepared['preparation_history_event_id']; + $callId = $child['child_call_id']; + if ($mutation === 'preparation') { + $preparationId = 'wrong-preparation'; + } + if ($mutation === 'call') { + $callId = 'wrong-call'; + } + if ($mutation === 'scope') { + $scope = $sibling; + } + if ($mutation === 'expired') { + Carbon::setTestNow('2026-10-03T00:00:30Z'); + } + if ($mutation === 'shorter run deadline') { + $run->forceFill([ + 'run_deadline_at' => now() + ->addSecond(), + ])->save(); + Carbon::setTestNow('2026-10-03T00:00:01Z'); + } + $request = WorkflowStub::loadRun($child['child_workflow_run_id'])->attemptRequestCancellationFromScope( + $run->id, + $scope, + $preparationId, + $callId + ); + $this->assertTrue($request->rejected()); + $this->assertSame(in_array($mutation, ['expired', 'shorter run deadline'], true) + ? 'cancellation_scope_authority_expired' : 'cancellation_scope_child_target_mismatch', $request->rejectionReason()); + $target = WorkflowRun::query()->findOrFail($child['child_workflow_run_id']); + $this->assertNull($target->cancellation_request_command_id); + $this->assertSame( + 0, + $target->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationRequested)->count() + ); + } + + /** + * @return iterable + */ + public static function invalidChildRequests(): iterable + { + foreach (['preparation', 'call', 'scope', 'abandon', 'expired', 'shorter run deadline'] as $mutation) { + yield $mutation => [$mutation]; + } + } + + #[DataProvider('childPolicies')] + public function testChildActorCommitsPolicyReceiptsWithoutReleasingTheParentClaim(string $policy): void + { + [$run, $task, $scope, $sibling, $shield] = $this->tree(); + $child = $this->child($run, $task, $scope, 4, $policy); + $other = $this->child($run, $task, $sibling, 5); + $shielded = $this->child($run, $task, $shield, 6); + $prepared = $this->prepare($run, $task, $scope); + $claimBefore = $task->fresh() + ->getAttributes(); + $receipt = ScopedChildCancellationDelivery::request( + $run, + $task, + $child['child_call_id'], + $scope, + $prepared['request_id'], + '1.20' + ); + $target = WorkflowStub::loadRun($child['child_workflow_run_id']); + $context = CooperativeCancellationDelivery::context($target->run()->fresh()); + $this->assertSame($policy === 'abandon', $context === null); + $this->assertFalse($target->run()->fresh()->status->isTerminal()); + $this->assertSame($claimBefore, $task->fresh()->getAttributes()); + $this->assertNull($run->fresh()->cancellation_request_command_id); + foreach ([$other, $shielded] as $untouched) { + $this->assertNull( + WorkflowRun::query()->findOrFail($untouched['child_workflow_run_id'])->cancellation_request_command_id + ); + } + if ($policy === 'wait_cancellation_completed') { + $this->assertFalse($receipt['ready']); + try { + CancellationScopeDelivery::record( + $run->fresh(), + $task->fresh(), + $scope, + $prepared['request_id'], + 4, + 'child', + '1.20' + ); + $this->fail('WAIT delivered before canonical child terminal history.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_child_completion_not_established', $error->getMessage()); + } + $target->attemptCancel('terminal fixture'); + $resolved = ScopedChildCancellationDelivery::request( + $run->fresh(), + $task->fresh(), + $child['child_call_id'], + $scope, + $prepared['request_id'], + '1.20' + ); + $this->assertSame($receipt['history_event_id'], $resolved['history_event_id']); + $this->assertTrue($resolved['ready']); + $this->assertNotNull($resolved['resolution_history_event_id']); + } else { + $this->assertTrue($receipt['ready']); + } + $delivery = CancellationScopeDelivery::record( + $run->fresh(), + $task->fresh(), + $scope, + $prepared['request_id'], + 4, + 'child', + '1.20' + ); + Carbon::setTestNow('2026-10-03T00:00:20Z'); + $repair = TaskWatchdog::runPass(respectThrottle: false, runIds: [$run->id]); + $this->assertSame([], $repair['existing_task_failures']); + $this->assertSame(1, $repair['repaired_existing_tasks']); + $this->assertTrue(app(DefaultWorkflowTaskBridge::class)->claimStatus($task->id, 'replacement')['claimed']); + $this->assertSame(2, $task->fresh()->attempt_count); + $cold = ScopedChildCancellationDelivery::request( + $run->fresh(), + $task->fresh(), + $child['child_call_id'], + $scope, + $prepared['request_id'], + '1.20' + ); + $this->assertSame($receipt['history_event_id'], $cold['history_event_id']); + $this->assertSame($delivery->id, CancellationScopeDelivery::recorded($run->fresh(), $scope)->id); + $this->assertSame( + $context?->toArray(), + CooperativeCancellationDelivery::context($target->run()->fresh())?->toArray() + ); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::ChildCancellationRequested)->count() + ); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeDelivered)->count() + ); + } + + /** + * @return iterable + */ + public static function childPolicies(): iterable + { + foreach (['try_cancel', 'wait_cancellation_completed', 'abandon'] as $policy) { + yield $policy => [$policy]; + } + } + + #[DataProvider('childPolicies')] + public function testPortableDeliveryReconcilesOriginalChildPolicyAndReplacementClaim(string $policy): void + { + [$run, $task, $scope, $sibling, $shield] = $this->tree(); + $child = $this->child($run, $task, $scope, 4, $policy); + $other = $this->child($run, $task, $sibling, 5); + $shielded = $this->child($run, $task, $shield, 6); + $prepared = $this->prepare($run, $task, $scope); + $claimBefore = $task->fresh() + ->getAttributes(); + $response = $this->deliver($task, $scope, $prepared['request_id']); + $this->assertTrue($response['prepared']); + $this->assertFalse($response['claim_released']); + $this->assertCount(1, $response['child_cancellations']); + $receipt = $response['child_cancellations'][0]; + $this->assertSame($child['child_call_id'], $receipt['child_call_id']); + $this->assertSame($child['child_workflow_run_id'], $receipt['child_workflow_run_id']); + $this->assertSame($policy, $receipt['policy']); + $target = WorkflowStub::loadRun($child['child_workflow_run_id']); + $context = CooperativeCancellationDelivery::context($target->run()->fresh()); + $this->assertSame($policy === 'abandon', $context === null); + $this->assertFalse($target->run()->fresh()->status->isTerminal()); + if ($context !== null) { + $this->assertSame($prepared['cancellation'], $context->scopeOrigin->toArray()); + $this->assertSame($prepared['request_id'], $context->rootRequestId); + $this->assertSame($prepared['authority_deadline_at'], $context->deadline()->toISOString()); + $this->assertSame($context->requestId, $receipt['request_id']); + } + if ($response['delivered']) { + $this->assertHostingShortenedWithoutClaimReplacement($claimBefore, $task); + } else { + $this->assertSame($claimBefore, $task->fresh()->getAttributes()); + } + foreach ([$other, $shielded] as $untouched) { + $this->assertNull( + WorkflowRun::query()->findOrFail($untouched['child_workflow_run_id'])->cancellation_request_command_id + ); + } + $beforeRetry = $run->historyEvents() + ->count(); + $this->assertSame($response, $this->deliver($task->fresh(), $scope, $prepared['request_id'])); + $this->assertSame($beforeRetry, $run->historyEvents()->count()); + if ($policy === 'wait_cancellation_completed') { + $this->assertFalse($response['delivered']); + $this->assertFalse($receipt['ready']); + $this->assertNull($receipt['resolution_history_event_id']); + $this->assertSame('cancellation_scope_child_completion_not_established', $response['reason']); + $this->assertNull(CancellationScopeDelivery::recorded($run->fresh(), $scope)); + $target->attemptCancel('canonical terminal fixture'); + } else { + $this->assertTrue($response['delivered']); + $this->assertTrue($receipt['ready']); + } + Carbon::setTestNow('2026-10-03T00:00:20Z'); + $repair = TaskWatchdog::runPass(respectThrottle: false, runIds: [$run->id]); + $this->assertSame([], $repair['existing_task_failures']); + $this->assertSame(1, $repair['repaired_existing_tasks']); + $this->assertTrue(app(DefaultWorkflowTaskBridge::class)->claimStatus($task->id, 'replacement')['claimed']); + $this->assertSame(2, $task->fresh()->attempt_count); + $cold = $this->deliver($task->fresh(), $scope, $prepared['request_id']); + $this->assertTrue($cold['delivered'], $cold['reason'] ?? ''); + $this->assertSame($prepared['preparation_history_event_id'], $cold['preparation_history_event_id']); + $this->assertSame($prepared['cancellation'], $cold['cancellation']); + $this->assertSame($prepared['authority_deadline_at'], $cold['authority_deadline_at']); + $this->assertSame($receipt['history_event_id'], $cold['child_cancellations'][0]['history_event_id']); + $this->assertSame($receipt['request_id'], $cold['child_cancellations'][0]['request_id']); + if ($policy === 'wait_cancellation_completed') { + $this->assertTrue($cold['child_cancellations'][0]['ready']); + $this->assertNotNull($cold['child_cancellations'][0]['resolution_history_event_id']); + } else { + $this->assertSame($response['history_event_id'], $cold['history_event_id']); + } + $this->assertSame( + $context?->toArray(), + CooperativeCancellationDelivery::context($target->run()->fresh())?->toArray() + ); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::ChildCancellationRequested)->count() + ); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeDelivered)->count() + ); + $this->assertSame($cold, $this->deliver($task->fresh(), $scope, $prepared['request_id'])); + } + + public function testPortableDeliveryRequestsDirectAndDescendantChildrenUnderOneBoundary(): void + { + [$run, $task, $scope] = $this->tree(); + $descendant = CancellationScopeHistory::open($run, $task, 4, '1.20', $scope)->payload['scope_id']; + $child = $this->child($run, $task, $scope, 5); + $nested = $this->child($run, $task, $descendant, 6); + $prepared = $this->prepare($run, $task, $scope, 5); + $response = $this->deliver($task, $scope, $prepared['request_id'], 5); + $this->assertTrue($response['delivered'], $response['reason'] ?? ''); + foreach ([$child, $nested] as $untouched) { + $this->assertNotNull( + WorkflowRun::query()->findOrFail($untouched['child_workflow_run_id'])->cancellation_request_command_id + ); + } + $this->assertNull(CancellationScopeDelivery::prepared($run->fresh(), $descendant)); + $this->assertSame($response, $this->deliver($task, $scope, $prepared['request_id'], 5)); + } + + public function testPreviouslyClosedChildHasItsOwnCanonicalResolutionReceipt(): void + { + [$run, $task, $scope] = $this->tree(); + $child = $this->child($run, $task, $scope, 4, 'wait_cancellation_completed'); + $prepared = $this->prepare($run, $task, $scope); + WorkflowStub::loadRun($child['child_workflow_run_id'])->attemptCancel('natural terminal fixture'); + $receipt = ScopedChildCancellationDelivery::request( + $run, + $task, + $child['child_call_id'], + $scope, + $prepared['request_id'], + '1.20' + ); + $this->assertTrue($receipt['ready']); + $this->assertNull($receipt['request_id']); + $this->assertSame( + 'already_terminal', + WorkflowHistoryEvent::query()->findOrFail($receipt['history_event_id'])->payload['request_outcome'] + ); + ScopedChildCancellation::assertReady(CancellationScopeDelivery::prepared($run->fresh(), $scope), $run->fresh()); + $this->assertNull( + WorkflowRun::query()->findOrFail($child['child_workflow_run_id'])->cancellation_request_command_id + ); + } + + #[DataProvider('postLockDeadlines')] + public function testChildLockCannotExtendTheRootBudgetOrExpiredHostingLease(int $seconds): void + { + [$run, $task, $scope] = $this->tree(); + $child = $this->child($run, $task, $scope, 4); + config() + ->set('workflows.v2.workflow_task_lease_seconds', 5); + $prepared = $this->prepare($run, $task, $scope); + WorkflowInstance::retrieved(static function (WorkflowInstance $instance) use ($child, $seconds): void { + if ($instance->id === $child['child_workflow_instance_id'] && $instance->getConnection()->transactionLevel() >= 2) { + Carbon::setTestNow('2026-10-03T00:00:' . sprintf('%02d', $seconds) . 'Z'); + } + }); + try { + ScopedChildCancellationDelivery::request( + $run, + $task, + $child['child_call_id'], + $scope, + $prepared['request_id'], + '1.20' + ); + $this->fail('A child request committed after its hosting authority expired.'); + } catch (LogicException $error) { + $this->assertStringContainsString('cancellation_scope_authority_expired', $error->getMessage()); + } + $target = WorkflowRun::query()->findOrFail($child['child_workflow_run_id']); + $this->assertNull($target->cancellation_request_command_id); + $this->assertSame( + 0, + $target->historyEvents() + ->where('event_type', HistoryEventType::CooperativeCancellationRequested)->count() + ); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ChildCancellationRequested)->count() + ); + } + + /** + * @return iterable + */ + public static function postLockDeadlines(): iterable + { + yield 'expired hosting lease' => [5]; + yield 'expired root budget' => [30]; + } + + #[DataProvider('changedChildReceipts')] + public function testAlteredReceiptCannotAuthorizeParentDelivery(string $mutation): void + { + [$run, $task, $scope] = $this->tree(); + $child = $this->child($run, $task, $scope, 4); + $prepared = $this->prepare($run, $task, $scope); + $receipt = ScopedChildCancellationDelivery::request( + $run, + $task, + $child['child_call_id'], + $scope, + $prepared['request_id'], + '1.20' + ); + $event = WorkflowHistoryEvent::query()->findOrFail($receipt['history_event_id']); + $payload = $event->payload; + switch ($mutation) { + case 'parent': $payload['parent_request_id'] = 'wrong'; + break; + case 'target': $payload['child_workflow_run_id'] = 'wrong'; + break; + case 'call': $payload['cancellation_scope']['member']['child_call_id'] = 'wrong'; + break; + case 'preparation': $payload['cancellation_scope']['preparation_history_event_id'] = 'wrong'; + break; + case 'schema': $payload['cancellation_scope']['schema'] = 'wrong'; + break; + case 'deadline': $payload['child_cleanup_deadline_at'] = '2026-10-03T00:00:40.000000Z'; + break; + case 'context': $payload['child_cancellation']['scope_origin'] = []; + break; + case 'outcome': $payload['request_outcome'] = 'abandoned'; + break; + } + $event->forceFill([ + 'payload' => $payload, + ])->save(); + $this->expectException(LogicException::class); + $this->expectExceptionMessage('cancellation_scope_child_delivery_not_established'); + CancellationScopeDelivery::record( + $run->fresh(), + $task->fresh(), + $scope, + $prepared['request_id'], + 4, + 'child', + '1.20' + ); + } + + /** + * @return iterable + */ + public static function changedChildReceipts(): iterable + { + foreach (['parent', 'target', 'call', 'preparation', 'schema', 'deadline', 'context', 'outcome'] as $mutation) { + yield $mutation => [$mutation]; + } + } + + public function testColdReceiptObjectKeyReorderingPreservesTheOriginalProof(): void + { + [$run, $task, $scope] = $this->tree(); + $child = $this->child($run, $task, $scope, 4); + $prepared = $this->prepare($run, $task, $scope); + $receipt = ScopedChildCancellationDelivery::request( + $run, + $task, + $child['child_call_id'], + $scope, + $prepared['request_id'], + '1.20' + ); + $event = WorkflowHistoryEvent::query()->findOrFail($receipt['history_event_id']); + $payload = $event->payload; + ksort($payload['cancellation_scope']); + ksort($payload['cancellation_scope']['member']); + ksort($payload['cancellation_scope']['cancellation']['root_context']['requester']); + ksort($payload['child_cancellation']['requester']); + $event->forceFill([ + 'payload' => $payload, + ])->save(); + $delivery = CancellationScopeDelivery::record( + $run->fresh(), + $task->fresh(), + $scope, + $prepared['request_id'], + 4, + 'child', + '1.20' + ); + $this->assertSame($delivery->id, CancellationScopeDelivery::recorded($run->fresh(), $scope)->id); + } + + public function testAnotherCancellationRootCannotBeReplacedByAScopeReceipt(): void + { + [$run, $task, $scope] = $this->tree(); + $child = $this->child($run, $task, $scope, 4); + $prepared = $this->prepare($run, $task, $scope); + $target = WorkflowStub::loadRun($child['child_workflow_run_id']); + $original = $target->requestCancellation('independent', 20) + ->cancellationContext(); + try { + ScopedChildCancellationDelivery::request( + $run, + $task, + $child['child_call_id'], + $scope, + $prepared['request_id'], + '1.20' + ); + $this->fail('An independent root was replaced by a parent scope.'); + } catch (LogicException $error) { + $this->assertSame( + 'cancellation_scope_child_request_refused:cancellation_root_conflict', + $error->getMessage() + ); + } + $this->assertSame( + $original->toArray(), + CooperativeCancellationDelivery::context($target->run()->fresh())->toArray() + ); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ChildCancellationRequested)->count() + ); + } + + public function testLateResolutionReadCannotCommitAfterTheOriginalBudget(): void + { + [$run, $task, $scope] = $this->tree(); + $child = $this->child($run, $task, $scope, 4, 'wait_cancellation_completed'); + $prepared = $this->prepare($run, $task, $scope); + $receipt = ScopedChildCancellationDelivery::request( + $run, + $task, + $child['child_call_id'], + $scope, + $prepared['request_id'], + '1.20' + ); + WorkflowStub::loadRun($child['child_workflow_run_id'])->attemptCancel('terminal fixture'); + WorkflowCommand::retrieved(static function (WorkflowCommand $command) use ($receipt): void { + if ($command->id === $receipt['request_id'] && $command->getConnection()->transactionLevel() >= 1) { + Carbon::setTestNow('2026-10-03T00:00:30Z'); + } + }); + try { + ScopedChildCancellationDelivery::request( + $run->fresh(), + $task->fresh(), + $child['child_call_id'], + $scope, + $prepared['request_id'], + '1.20' + ); + $this->fail('A late canonical read committed a child resolution receipt.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_authority_expired', $error->getMessage()); + } + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ChildCancellationResolved)->count() + ); + } + + public function testChildInheritsTheEarlierPreparedParentRunDeadline(): void + { + [$run, $task, $scope] = $this->tree(); + $child = $this->child($run, $task, $scope, 4); + $run->forceFill([ + 'run_deadline_at' => now() + ->addSeconds(10), + ])->save(); + $prepared = $this->prepare($run, $task, $scope); + $this->assertSame('2026-10-03T00:00:10.000000Z', $prepared['authority_deadline_at']); + Carbon::setTestNow('2026-10-03T00:00:05Z'); + $receipt = ScopedChildCancellationDelivery::request( + $run, + $task, + $child['child_call_id'], + $scope, + $prepared['request_id'], + '1.20' + ); + $target = WorkflowRun::query()->findOrFail($child['child_workflow_run_id']); + $context = CooperativeCancellationDelivery::context($target); + $this->assertSame('2026-10-03T00:00:10.000000Z', $context->deadline()->toISOString()); + $this->assertSame('2026-10-03T00:00:30.000000Z', $context->scopeOrigin->rootDeadline()->toISOString()); + $this->assertSame($prepared['cancellation'], $context->scopeOrigin->toArray()); + $this->assertSame($context->requestId, $receipt['request_id']); + $this->assertTrue($receipt['ready']); + ScopedChildCancellation::assertReady(CancellationScopeDelivery::prepared($run->fresh(), $scope), $run->fresh()); + } + + /** + * @return array{WorkflowRun, WorkflowTask, string, string, string} + */ + private function tree(): array + { + $workflow = WorkflowStub::make(TestSignalWorkflow::class); + $workflow->start(); + $run = $workflow->run(); + $task = $run->tasks() + ->where('task_type', TaskType::Workflow)->sole(); + $task->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'child-scope-owner', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addMinutes(5), + ])->save(); + $scope = CancellationScopeHistory::open($run, $task, 1, '1.20')->payload['scope_id']; + $sibling = CancellationScopeHistory::open($run, $task, 2, '1.20')->payload['scope_id']; + $shield = CancellationScopeHistory::open($run, $task, 3, '1.20', $scope, true)->payload['scope_id']; + return [$run, $task->refresh(), $scope, $sibling, $shield]; + } + + /** + * @return array + */ + private function child( + WorkflowRun $run, + WorkflowTask $task, + string $scope, + int $sequence, + string $policy = 'try_cancel' + ): array { + $result = app(DefaultWorkflowTaskBridge::class)->checkpointCancellationScopePrefix( + $task->id, + $task->lease_owner, + $task->attempt_count, + 'child-' . $sequence, + $sequence, + [[ + 'type' => 'start_child_workflow', + 'workflow_type' => 'scoped-child-fixture', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'cancellation_scope_id' => $scope, + 'cancellation_policy' => $policy, + ]], + '1.20' + ); + $this->assertTrue($result['checkpointed'], $result['reason'] ?? ''); + return $run->historyEvents() + ->where('event_type', HistoryEventType::ChildWorkflowScheduled) + ->where('payload->sequence', $sequence) + ->sole() +->payload; + } + + /** + * @return array + */ + private function prepare(WorkflowRun $run, WorkflowTask $task, string $scope, int $sequence = 4): array + { + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $result = app(DefaultWorkflowTaskBridge::class)->prepareCancellationScopeDelivery( + $task->id, + $task->lease_owner, + $task->attempt_count, + $scope, + $request->payload['request_id'], + $sequence, + 'child', + protocolVersion: '1.20' + ); + $this->assertTrue($result['prepared'], $result['reason'] ?? ''); + return $result; + } + + /** + * @param array $child + */ + private function started(WorkflowRun $run, WorkflowTask $task, array $child, string $childRunId): void + { + WorkflowHistoryEvent::record($run->fresh(), HistoryEventType::ChildRunStarted, [ + ...array_diff_key($child, [ + 'task' => true, + ]), + 'child_workflow_run_id' => $childRunId, + ], $task); + } + + /** + * @return array + */ + private function deliver(WorkflowTask $task, string $scope, string $request, int $sequence = 4): array + { + return app(DefaultWorkflowTaskBridge::class)->deliverCancellationScope( + $task->id, + $task->lease_owner, + $task->attempt_count, + $scope, + $request, + $sequence, + 'child', + protocolVersion: '1.20' + ); + } +} diff --git a/tests/Feature/V2/V2ScopedLocalControlTest.php b/tests/Feature/V2/V2ScopedLocalControlTest.php new file mode 100644 index 000000000..e62960964 --- /dev/null +++ b/tests/Feature/V2/V2ScopedLocalControlTest.php @@ -0,0 +1,296 @@ + 'poll', + ]); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + #[DataProvider('requests')] + public function testOriginalScalarControlPreparesThenFencesTheAcceptedScope(bool $runRequest, bool $nested): void + { + [$workflow, $run, $task, $parent, $scope, $sequence, $local, $context] = $this->claim($runRequest, $nested); + Carbon::setTestNow(now()->addSecond()); + $before = $task->fresh() + ->getAttributes(); + $control = $this->bridge() + ->controlLocalActivity($local['activity_attempt_id'], 'original', 1, true, '1.20'); + $this->assertSame('cancellation_scope_requested', $control['reason']); + $this->assertFalse($control['active']); + $this->assertFalse($control['renewed']); + $this->assertTrue($control['fenced']); + $this->assertArrayNotHasKey('_scope_control', $control); + $this->assertSameJsonObject($context->toArray(), $control['cancellation_scope']['cancellation']); + $this->assertSame($scope, $control['cancellation_scope']['scope_id']); + $this->assertSame($before, $task->fresh()->getAttributes()); + $preparation = $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeDeliveryPrepared)->sole(); + $fence = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancelled)->sole(); + $this->assertSame($parent, $preparation->payload['scope_id']); + $this->assertSame($sequence, $preparation->payload['sequence']); + $this->assertSame('local_activity', $preparation->payload['call_kind']); + $this->assertLessThan($fence->sequence, $preparation->sequence); + $this->assertSame($context->requestId, $fence->workflow_command_id); + $this->assertSame( + $context->rootContext->rootRequestId, + $fence->payload['cancellation_scope']['cancellation']['root_context']['root_request_id'] + ); + $again = $this->bridge() + ->controlLocalActivity($local['activity_attempt_id'], 'original', 1, true, '1.20'); + $this->assertSame($control['cancellation_scope'], $again['cancellation_scope']); + $this->assertSame(1, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCancelled)->count()); + $this->assertTrue($this->bridge()->acknowledgeLocalActivityCancellation( + $local['activity_attempt_id'], + 'original', + $context->requestId, + 1, + '1.20' + )['acknowledged']); + $parentContext = CancellationScopeRequests::context($run, $parent); + $delivered = $this->bridge() + ->deliverCancellationScope( + $task->id, + 'original', + 1, + $parent, + $parentContext->requestId, + $sequence, + 'local_activity', + protocolVersion: '1.20' + ); + $this->assertTrue($delivered['delivered'], $delivered['reason'] ?? ''); + $this->assertFalse($delivered['claim_released']); + $this->assertSame(TaskStatus::Leased, $task->fresh()->status); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityHeartbeatRecorded)->count() + ); + } + + public static function requests(): iterable + { + foreach ([false, true] as $run) { + foreach ([false, true] as $nested) { + yield ($run ? 'run' : 'direct') . '-' . ($nested ? 'nested' : 'scalar') => [$run, $nested]; + } + } + } + + #[DataProvider('invalidClaims')] + public function testInvalidClaimCannotPrepareOrFenceTheScope(string $fault): void + { + [, $run, $task, , , , $local] = $this->claim(true, false); + $owner = 'original'; + $epoch = 1; + if ($fault === 'owner') { + $owner = 'other'; + } + if ($fault === 'epoch') { + $epoch = 2; + } + if ($fault === 'expired') { + $task->forceFill([ + 'lease_expires_at' => now() + ->subSecond(), + ])->save(); + } + $before = $task->fresh() + ->getAttributes(); + $operation = fn () => $this->bridge() + ->controlLocalActivity($local['activity_attempt_id'], $owner, $epoch, true, '1.20'); + $control = $fault === 'outer transaction' ? DB::transaction($operation) : $operation(); + $this->assertFalse($control['active']); + $this->assertFalse($control['renewed']); + $this->assertArrayNotHasKey('_scope_control', $control); + $this->assertSame( + $fault === 'expired' ? 'workflow_claim_expired' + : ($fault === 'outer transaction' ? 'cancellation_scope_preparation_requires_own_transaction' : 'local_activity_preparation_mismatch'), + $control['reason'] + ); + $this->assertSame($before, $task->fresh()->getAttributes()); + $this->assertSame(0, $run->historyEvents()->whereIn('event_type', [ + HistoryEventType::CancellationScopeDeliveryPrepared, HistoryEventType::ActivityCancelled, + ])->count()); + } + + public static function invalidClaims(): iterable + { + foreach (['owner', 'epoch', 'expired', 'outer transaction'] as $fault) { + yield $fault => [$fault]; + } + } + + public function testTakeoverBetweenPreparationAndFencingCannotBorrowTheReplacementClaim(): void + { + [, $run, $task, , , , $local] = $this->claim(true, false); + $original = WorkflowHistoryEvent::getEventDispatcher(); + $this->assertNotNull($original); + $dispatcher = clone $original; + $dispatcher->listen( + 'eloquent.created: ' . WorkflowHistoryEvent::class, + static function (WorkflowHistoryEvent $event) use ($task): void { + if ($event->event_type === HistoryEventType::CancellationScopeDeliveryPrepared) { + $task->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 2, + 'lease_expires_at' => now() + ->addSeconds(20), + ])->save(); + } + } + ); + WorkflowHistoryEvent::setEventDispatcher($dispatcher); + try { + $control = $this->bridge() + ->controlLocalActivity($local['activity_attempt_id'], 'original', 1, true, '1.20'); + } finally { + WorkflowHistoryEvent::setEventDispatcher($original); + } + $this->assertSame('cancellation_scope_workflow_claim_mismatch', $control['reason']); + $this->assertFalse($control['active']); + $this->assertFalse($control['renewed']); + $this->assertArrayNotHasKey('_scope_control', $control); + $this->assertSame('replacement', $task->fresh()->lease_owner); + $this->assertSame(2, $task->fresh()->attempt_count); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeDeliveryPrepared)->count() + ); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::ActivityCancelled)->count()); + } + + public function testAGroupMemberCannotFreezeAScalarBoundaryOrRevokeItsSibling(): void + { + [, $run, $task, , , , $local] = $this->claim(true, false, true); + $before = $task->fresh() + ->getAttributes(); + $control = $this->bridge() + ->controlLocalActivity($local['activity_attempt_id'], 'original', 1, true, '1.20'); + $this->assertFalse($control['active']); + $this->assertFalse($control['renewed']); + $this->assertSame('cancellation_scope_local_group_control_not_supported', $control['reason']); + $this->assertArrayNotHasKey('_scope_control', $control); + $this->assertSame($before, $task->fresh()->getAttributes()); + $this->assertSame(0, $run->historyEvents()->whereIn('event_type', [ + HistoryEventType::CancellationScopeDeliveryPrepared, HistoryEventType::ActivityCancelled, + ])->count()); + $this->assertSame(2, $run->activityExecutions()->count()); + } + + private function claim(bool $runRequest, bool $nested, bool $group = false): array + { + $workflow = WorkflowStub::make(TestSignalWorkflow::class, 'scope-local-control'); + $workflow->start(); + $run = $workflow->run() + ->fresh(); + $task = $run->tasks() + ->where('task_type', TaskType::Workflow)->sole(); + $task->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'original', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addSeconds(60), + ])->save(); + $parent = CancellationScopeHistory::open($run, $task, 1, '1.20')->payload['scope_id']; + $scope = $nested ? CancellationScopeHistory::open( + $run, + $task, + 2, + '1.20', + $parent + )->payload['scope_id'] : $parent; + $sequence = $nested ? 3 : 2; + $descriptor = [ + 'type' => 'record_local_activity', + 'activity_type' => 'scoped-work', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'cancellation_scope_id' => $scope, + 'cancellation_policy' => 'wait_cancellation_completed', + ]; + if ($group) { + $commands = []; + foreach ([0, 1] as $index) { + $commands[] = [ + ...$descriptor, + 'type' => 'prepare_local_activity', + ...ParallelChildGroup::itemMetadata($sequence, 2, $index, 'mixed'), + ]; + } + $checkpoint = app(PreparedLocalActivityGroupTaskBridge::class)->checkpointLocalActivityGroup( + $task->id, + 'original', + 1, + 'scoped-group', + $sequence, + $commands, + '1.20' + ); + $this->assertTrue($checkpoint['checkpointed'], $checkpoint['reason'] ?? ''); + $descriptor = [ + ...$commands[0], + 'type' => 'record_local_activity', + ]; + } + $local = $this->bridge() + ->prepareLocalActivity($task->id, 'original', 1, $sequence, 'local-original', $descriptor, '1.20'); + $this->assertTrue($local['prepared'], $local['reason'] ?? ''); + if ($runRequest) { + $workflow->requestCancellation('finish the run', 30); + } else { + CancellationScopeRequests::request($run, $parent, '1.20', 30, 'stop only this subtree'); + if ($nested) { + CancellationScopeRequests::request($run, $scope, '1.20', 30, parentScopeId: $parent); + } + } + $run->refresh(); + $context = CancellationScopeRequests::context($run, $scope); + $this->assertNotNull($context); + + return [$workflow, $run, $task, $parent, $scope, $sequence, $local, $context]; + } + + private function bridge(): DefaultWorkflowTaskBridge + { + return app(DefaultWorkflowTaskBridge::class); + } +} diff --git a/tests/Feature/V2/V2ScopedTimerCancellationTest.php b/tests/Feature/V2/V2ScopedTimerCancellationTest.php new file mode 100644 index 000000000..1586e458a --- /dev/null +++ b/tests/Feature/V2/V2ScopedTimerCancellationTest.php @@ -0,0 +1,624 @@ + 'poll', + ]); + Carbon::setTestNow('2026-10-03T00:00:00Z'); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + public function testDispatchFencesEveryOriginalTimerAndPreservesSiblingAndHostingClaim(): void + { + [$run, $task, $scope, $sibling] = $this->tree(); + $target = $this->timer($run, $task, $scope, 3); + $other = $this->timer($run, $task, $sibling, 4); + $outsideCall = $this->timer($run, $task, $scope, 5); + $before = $task->fresh() + ->getAttributes(); + $otherBefore = $other->fresh() + ->getAttributes(); + $prepared = $this->prepare($run, $task, $scope); + $this->assertSame([$target->id, $outsideCall->id], array_column($prepared['timer_members'], 'timer_id')); + $result = $this->dispatch($task, $scope, $prepared['request_id']); + $this->assertTrue($result['delivered'], $result['reason'] ?? ''); + $this->assertCount(2, $result['timer_cancellations']); + foreach ([$target, $outsideCall] as $timer) { + $this->assertSame(TimerStatus::Cancelled, $timer->fresh()->status); + $this->assertSame(TaskStatus::Cancelled, $this->timerTask($run, $timer)->status); + $event = $run->historyEvents() + ->where('event_type', HistoryEventType::TimerCancelled) + ->where('payload->timer_id', $timer->id) + ->sole(); + $this->assertSame( + $prepared['preparation_history_event_id'], + $event->payload['cancellation_scope']['preparation_history_event_id'] + ); + $this->assertSame( + $prepared['authority_deadline_at'], + $event->payload['cancellation_scope']['authority_deadline_at'] + ); + } + $this->assertSame($otherBefore, $other->fresh()->getAttributes()); + $this->assertHostingShortenedWithoutClaimReplacement($before, $task); + $this->assertFalse($run->fresh()->status->isTerminal()); + $this->assertNull($run->fresh()->cancellation_request_command_id); + $this->assertSame($result, $this->dispatch($task, $scope, $prepared['request_id'])); + $this->assertNotNull(CancellationScopeDelivery::recorded($run->fresh(), $scope)); + $timeline = collect(HistoryTimeline::fromHistory($run->fresh())) + ->firstWhere('id', $prepared['preparation_history_event_id']); + $this->assertCount(2, $timeline['cancellation_scope']['timer_members']); + } + + #[DataProvider('projectionDrift')] + public function testDelayedJobCannotFireOrResumeAfterFenceEvenWithProjectionDrift(string $drift): void + { + [$run, $task, $scope] = $this->tree(); + $timer = $this->timer($run, $task, $scope, 3, 1); + $prepared = $this->prepare($run, $task, $scope); + $this->assertTrue($this->dispatch($task, $scope, $prepared['request_id'])['delivered']); + $timerTask = $this->timerTask($run, $timer); + if ($drift === 'missing') { + $timer->delete(); + } elseif ($drift === 'pending') { + $timer->refresh() + ->forceFill([ + 'status' => TimerStatus::Pending, + ])->save(); + } + $timerTask->forceFill([ + 'status' => TaskStatus::Ready, + ])->save(); + Carbon::setTestNow('2026-10-03T00:00:02Z'); + $before = $run->historyEvents() + ->count(); + $this->app->call([new RunTimerTask($timerTask->id), 'handle']); + $this->assertSame($before, $run->historyEvents()->count()); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::TimerFired)->count()); + $this->assertSame(1, $run->tasks()->where('task_type', TaskType::Workflow)->count()); + $this->assertSame(TimerStatus::Cancelled, WorkflowTimer::query()->findOrFail($timer->id)->status); + $this->assertSame(TaskStatus::Cancelled, $timerTask->fresh()->status); + $this->assertNotNull(CancellationScopeDelivery::recorded($run->fresh(), $scope)); + } + + public static function projectionDrift(): iterable + { + yield 'intact' => ['intact']; + yield 'missing row' => ['missing']; + yield 'mutable status reverted' => ['pending']; + } + + #[DataProvider('claimDrift')] + public function testTimerClaimedBeforeFenceCannotFireInItsHandler(bool $drift): void + { + [$run, $task, $scope] = $this->tree(); + $timer = $this->timer($run, $task, $scope, 3, 1); + $prepared = $this->prepare($run, $task, $scope); + $timerTask = $this->timerTask($run, $timer); + $fenced = false; + Event::listen('eloquent.updated: ' . WorkflowTask::class, static function (WorkflowTask $updated) use ( + $run, + $task, + $scope, + $prepared, + $timerTask, + $timer, + $drift, + &$fenced + ): void { + if ($updated->id === $timerTask->id && $updated->status === TaskStatus::Leased) { + DB::afterCommit(static function () use ( + $run, + $task, + $scope, + $prepared, + $timer, + $drift, + &$fenced + ): void { + $fenced = ScopedTimerCancellation::fence( + $run, + $task, + $timer->id, + $scope, + $prepared['request_id'], + '1.20' + )['fenced']; + if ($drift) { + // A claimed job must honor durable cancellation even if + // its mutable projection changes after the fence. + $timer->refresh() + ->forceFill([ + 'status' => TimerStatus::Pending, + ])->save(); + } + }); + } + }); + Carbon::setTestNow('2026-10-03T00:00:02Z'); + $this->app->call([new RunTimerTask($timerTask->id), 'handle']); + $this->assertTrue($fenced); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::TimerFired)->count()); + $this->assertSame(1, $run->tasks()->where('task_type', TaskType::Workflow)->count()); + $this->assertTrue($this->dispatch($task, $scope, $prepared['request_id'])['delivered']); + } + + public static function claimDrift(): iterable + { + yield 'intact projection' => [false]; + yield 'projection changed after fence' => [true]; + } + + public function testFireWinningBeforeFenceKeepsNaturalOutcomeAndCannotFireTwice(): void + { + [$run, $task, $scope] = $this->tree(); + $timer = $this->timer($run, $task, $scope, 3, 1); + $prepared = $this->prepare($run, $task, $scope); + $timerTask = $this->timerTask($run, $timer); + Carbon::setTestNow('2026-10-03T00:00:02Z'); + $this->app->call([new RunTimerTask($timerTask->id), 'handle']); + $before = $run->historyEvents() + ->count(); + $result = $this->dispatch($task, $scope, $prepared['request_id']); + $this->assertTrue($result['delivered'], $result['reason'] ?? ''); + $this->assertFalse($result['timer_cancellations'][0]['fenced']); + $this->assertNull($result['timer_cancellations'][0]['history_event_id']); + $this->assertSame(TimerStatus::Fired, $timer->fresh()->status); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::TimerCancelled)->count()); + $timer->refresh() + ->forceFill([ + 'status' => TimerStatus::Pending, + ])->save(); + $timerTask->forceFill([ + 'status' => TaskStatus::Ready, + ])->save(); + $this->app->call([new RunTimerTask($timerTask->id), 'handle']); + $this->assertSame($before + 1, $run->historyEvents()->count()); + $this->assertSame(1, $run->historyEvents()->where('event_type', HistoryEventType::TimerFired)->count()); + $this->assertSame(2, $run->tasks()->where('task_type', TaskType::Workflow)->count()); + } + + public function testPartialFenceReplacementReusesOriginalMembershipBoundaryAndDeadline(): void + { + [$run, $task, $scope] = $this->tree(); + $first = $this->timer($run, $task, $scope, 3); + $second = $this->timer($run, $task, $scope, 4); + $prepared = $this->prepare($run, $task, $scope); + $receipt = ScopedTimerCancellation::fence($run, $task, $first->id, $scope, $prepared['request_id'], '1.20'); + $this->assertTrue($receipt['fenced']); + $this->assertSame(TimerStatus::Pending, $second->fresh()->status); + Carbon::setTestNow('2026-10-03T00:00:11Z'); + $repair = TaskWatchdog::runPass(respectThrottle: false, runIds: [$run->id]); + $this->assertSame([], $repair['existing_task_failures']); + $this->assertSame(1, $repair['repaired_existing_tasks']); + $this->assertTrue(app(DefaultWorkflowTaskBridge::class)->claimStatus($task->id, 'replacement')['claimed']); + $replacement = $task->fresh(); + $this->assertSame(2, $replacement->attempt_count); + $result = $this->dispatch($replacement, $scope, $prepared['request_id']); + $this->assertTrue($result['delivered'], $result['reason'] ?? ''); + $this->assertSame($receipt, $result['timer_cancellations'][0]); + $this->assertSame($prepared['preparation_history_event_id'], $result['preparation_history_event_id']); + $this->assertSame($prepared['authority_deadline_at'], $result['authority_deadline_at']); + $this->assertSame($prepared['timer_members'], $result['timer_members']); + $this->assertSame($prepared['cancellation'], $result['cancellation']); + $this->assertSame($result, $this->dispatch($replacement, $scope, $prepared['request_id'])); + $this->assertSame( + 'cancellation_scope_workflow_claim_mismatch', + $this->dispatch($task, $scope, $prepared['request_id'])['reason'] + ); + } + + #[DataProvider('refusals')] + public function testActorRefusesStaleOrUnpreparedAuthorityWithoutEffects(string $mutation, string $reason): void + { + [$run, $task, $scope, $sibling] = $this->tree(); + $timer = $this->timer($run, $task, $scope, 3); + $other = $this->timer($run, $task, $sibling, 4); + $prepared = $this->prepare($run, $task, $scope); + $requestId = $prepared['request_id']; + $version = '1.20'; + if ($mutation === 'owner') { + $task->fresh() + ->forceFill([ + 'lease_owner' => 'replacement', + ])->save(); + } elseif ($mutation === 'attempt') { + $task->fresh() + ->forceFill([ + 'attempt_count' => 2, + ])->save(); + } elseif ($mutation === 'lease') { + $task->fresh() + ->forceFill([ + 'lease_expires_at' => now(), + ])->save(); + } elseif ($mutation === 'deadline') { + $this->advanceWithWorkflowHeartbeats($task, Carbon::parse('2026-10-03T00:00:30Z')); + } elseif ($mutation === 'request') { + $requestId = 'another-request'; + } elseif ($mutation === 'sibling') { + $timer = $other; + } elseif ($mutation === 'version') { + $version = '1.19'; + } elseif ($mutation === 'namespace') { + $this->timerTask($run, $timer) + ->forceFill([ + 'namespace' => 'other-namespace', + ])->save(); + } elseif ($mutation === 'projection') { + $timer->forceFill([ + 'status' => TimerStatus::Fired, + ])->save(); + } elseif ($mutation === 'descriptor') { + $event = $run->historyEvents() + ->where('event_type', HistoryEventType::TimerScheduled) + ->where('payload->timer_id', $timer->id) + ->sole(); + $event->forceFill([ + 'payload' => [ + ...$event->payload, + 'delay_seconds' => 61, + ], + ])->save(); + } elseif ($mutation === 'inventory') { + $event = $run->historyEvents() + ->findOrFail($prepared['preparation_history_event_id']); + $event->forceFill([ + 'payload' => [ + ...$event->payload, + 'timer_members' => [], + ], + ])->save(); + } + $before = $run->historyEvents() + ->count(); + try { + ScopedTimerCancellation::fence($run->fresh(), $task, $timer->id, $scope, $requestId, $version); + $this->fail('Invalid authority must not fence a timer.'); + } catch (LogicException $error) { + $this->assertSame($reason, $error->getMessage()); + } + $this->assertSame($before, $run->historyEvents()->count()); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::TimerCancelled)->count()); + $this->assertSame(TimerStatus::Pending, $other->fresh()->status); + } + + public static function refusals(): iterable + { + foreach (['owner', 'attempt', 'lease'] as $case) { + yield $case => [$case, 'cancellation_scope_workflow_claim_mismatch']; + } + yield 'original deadline' => ['deadline', 'cancellation_scope_authority_expired']; + yield 'wrong request' => ['request', 'cancellation_scope_request_mismatch']; + yield 'sibling' => ['sibling', 'cancellation_scope_timer_not_prepared']; + yield 'old protocol' => ['version', 'cancellation_scope_requires_protocol_1_20']; + yield 'foreign timer task' => ['namespace', 'cancellation_scope_timer_task_mismatch']; + yield 'mutable terminal without history' => [ + 'projection', + 'cancellation_scope_timer_terminal_history_not_recorded', + ]; + yield 'changed descriptor' => ['descriptor', 'cancellation_scope_delivery_history_invalid']; + yield 'omitted original member' => ['inventory', 'cancellation_scope_delivery_history_invalid']; + } + + public function testUnpreparedActorAndCallerOuterTransactionAreRefused(): void + { + [$run, $task, $scope] = $this->tree(); + $timer = $this->timer($run, $task, $scope, 3); + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + try { + ScopedTimerCancellation::fence($run, $task, $timer->id, $scope, $request->payload['request_id'], '1.20'); + $this->fail('Preparation is required.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_delivery_not_prepared', $error->getMessage()); + } + DB::transaction(function () use ($run, $task, $scope, $timer, $request): void { + try { + ScopedTimerCancellation::fence( + $run, + $task, + $timer->id, + $scope, + $request->payload['request_id'], + '1.20' + ); + $this->fail('Timer lock ownership requires its own transaction.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_timer_requires_own_transaction', $error->getMessage()); + } + }); + $this->assertSame(TimerStatus::Pending, $timer->fresh()->status); + } + + #[DataProvider('lockExpiry')] + public function testFinalTimerRowLockDoesNotExtendClaimOrCancellationAuthority(int $seconds, string $reason): void + { + [$run, $task, $scope] = $this->tree(); + $timer = $this->timer($run, $task, $scope, 3); + $prepared = $this->prepare($run, $task, $scope); + if ($seconds === 5) { + $task->forceFill([ + 'lease_expires_at' => now() + ->addSeconds(5), + ])->save(); + } else { + $this->advanceWithWorkflowHeartbeats($task, Carbon::parse('2026-10-03T00:00:25Z')); + $this->assertTrue(app(DefaultWorkflowTaskBridge::class)->heartbeat($task->id)['renewed']); + $this->assertSame('2026-10-03T00:00:35.000000Z', $task->fresh()->lease_expires_at->toISOString()); + } + $crossed = false; + DB::listen(static function (QueryExecuted $query) use ($timer, $seconds, &$crossed): void { + if (! $crossed && str_contains($query->sql, 'workflow_run_timers') + && $query->bindings === [$timer->id]) { + // Model the time spent acquiring the actor's final timer row. + Carbon::setTestNow(Carbon::parse('2026-10-03T00:00:00Z')->addSeconds($seconds)); + $crossed = true; + } + }); + $before = $run->historyEvents() + ->count(); + try { + ScopedTimerCancellation::fence( + $run->fresh(), + $task, + $timer->id, + $scope, + $prepared['request_id'], + '1.20' + ); + $this->fail('Time waiting for the final row lock must not renew authority.'); + } catch (LogicException $error) { + $this->assertSame($reason, $error->getMessage()); + } + $this->assertTrue($crossed); + $this->assertSame($before, $run->historyEvents()->count()); + $this->assertSame(TimerStatus::Pending, $timer->fresh()->status); + $this->assertSame(TaskStatus::Ready, $this->timerTask($run, $timer)->status); + } + + public static function lockExpiry(): iterable + { + yield 'original cleanup deadline' => [30, 'cancellation_scope_authority_expired']; + yield 'hosting claim expires first' => [5, 'cancellation_scope_workflow_claim_mismatch']; + } + + public function testBarrierCannotReplaceFenceOrUseReceiptWrittenAfterDeliveryMarker(): void + { + [$run, $task, $scope] = $this->tree(); + $timer = $this->timer($run, $task, $scope, 3); + $prepared = $this->prepare($run, $task, $scope); + try { + CancellationScopeDelivery::record($run, $task, $scope, $prepared['request_id'], 3, 'timer', '1.20'); + $this->fail('A marker cannot substitute for the timer fence.'); + } catch (LogicException $error) { + $this->assertSame('cancellation_scope_timer_fence_not_established', $error->getMessage()); + } + $marker = WorkflowHistoryEvent::record($run, HistoryEventType::CancellationScopeDelivered, [ + 'schema' => CancellationScopeDelivery::SCHEMA, + 'workflow_run_id' => $run->id, + 'scope_id' => $scope, + 'request_id' => $prepared['request_id'], + 'cancellation' => $prepared['cancellation'], + 'sequence' => 3, + 'call_kind' => 'timer', + 'sequence_span' => 1, + 'operation_sequence' => null, + 'operation_sequence_span' => 1, + 'authority_deadline_at' => $prepared['authority_deadline_at'], + 'preparation_history_event_id' => $prepared['preparation_history_event_id'], + ], $task); + $this->assertTrue(ScopedTimerCancellation::fence( + $run->fresh(), + $task, + $timer->id, + $scope, + $prepared['request_id'], + '1.20' + )['fenced']); + $this->assertNotNull($marker); + $this->expectExceptionMessage('cancellation_scope_delivery_history_invalid'); + CancellationScopeDelivery::recorded($run->fresh(), $scope); + } + + #[DataProvider('receiptCorruption')] + public function testColdReplayRejectsChangedCancellationReceipt(string $field): void + { + [$run, $task, $scope] = $this->tree(); + $this->timer($run, $task, $scope, 3); + $prepared = $this->prepare($run, $task, $scope); + $this->assertTrue($this->dispatch($task, $scope, $prepared['request_id'])['delivered']); + $event = $run->historyEvents() + ->where('event_type', HistoryEventType::TimerCancelled)->sole(); + $snapshot = $event->payload['cancellation_scope']; + $snapshot[$field] = 'changed'; + $event->forceFill([ + 'payload' => [ + ...$event->payload, + 'cancellation_scope' => $snapshot, + ], + ])->save(); + $this->expectExceptionMessage('cancellation_scope_delivery_history_invalid'); + CancellationScopeDelivery::recorded($run->fresh(), $scope); + } + + public static function receiptCorruption(): iterable + { + foreach (['request_id', 'scope_id', 'workflow_run_id', 'request_history_event_id', + 'preparation_history_event_id', 'authority_deadline_at', 'schema'] as $field) { + yield $field => [$field]; + } + } + + public function testDescendantChildAndDirectOperationsShareTheOriginalPreparation(): void + { + [$run, $task, $scope] = $this->tree(); + $timer = $this->timer($run, $task, $scope, 3); + $descendant = CancellationScopeHistory::open($run, $task, 4, '1.20', $scope)->payload['scope_id']; + $reply = app(DefaultWorkflowTaskBridge::class)->checkpointCancellationScopePrefix( + $task->id, + $task->lease_owner, + $task->attempt_count, + 'child-preflight', + 5, + [[ + 'type' => 'schedule_activity', + 'activity_type' => TestGreetingActivity::class, + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'cancellation_scope_id' => $scope, + ], [ + 'type' => 'start_child_workflow', + 'workflow_type' => 'child-scope-fixture', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'cancellation_scope_id' => $descendant, + ]], + '1.20' + ); + $this->assertTrue($reply['checkpointed'], $reply['reason'] ?? ''); + $prepared = $this->prepare($run, $task, $scope); + $result = $this->dispatch($task, $scope, $prepared['request_id']); + $this->assertTrue($result['delivered'], $result['reason'] ?? ''); + $this->assertSame(TimerStatus::Cancelled, $timer->fresh()->status); + $this->assertSame(ActivityStatus::Cancelled, $run->activityExecutions()->sole()->status); + $this->assertCount(1, $result['child_cancellations']); + $this->assertNull(CancellationScopeDelivery::prepared($run->fresh(), $descendant)); + $this->assertSame($result, $this->dispatch($task, $scope, $prepared['request_id'])); + } + + /** + * @return array{WorkflowRun, WorkflowTask, string, string} + */ + private function tree(): array + { + $workflow = WorkflowStub::make(TestSignalWorkflow::class); + $workflow->start('scoped-timer'); + $run = $workflow->run(); + $task = $run->tasks() + ->where('task_type', TaskType::Workflow)->sole(); + $task->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'timer-scope-owner', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addMinutes(5), + ])->save(); + $scope = CancellationScopeHistory::open($run, $task, 1, '1.20')->payload['scope_id']; + $sibling = CancellationScopeHistory::open($run, $task, 2, '1.20')->payload['scope_id']; + return [$run, $task->refresh(), $scope, $sibling]; + } + + private function timer( + WorkflowRun $run, + WorkflowTask $task, + string $scope, + int $sequence, + int $delay = 60 + ): WorkflowTimer { + $result = app(DefaultWorkflowTaskBridge::class)->checkpointCancellationScopePrefix( + $task->id, + $task->lease_owner, + $task->attempt_count, + 'timer-' . $sequence, + $sequence, + [[ + 'type' => 'start_timer', + 'delay_seconds' => $delay, + 'cancellation_scope_id' => $scope, + ]], + '1.20' + ); + $this->assertTrue($result['checkpointed'], $result['reason'] ?? ''); + return $run->timers() + ->where('sequence', $sequence) + ->sole(); + } + + /** + * @return array + */ + private function prepare(WorkflowRun $run, WorkflowTask $task, string $scope): array + { + $request = CancellationScopeRequests::request($run, $scope, '1.20', 30); + $result = app(DefaultWorkflowTaskBridge::class)->prepareCancellationScopeDelivery( + $task->id, + $task->lease_owner, + $task->attempt_count, + $scope, + $request->payload['request_id'], + 3, + 'timer', + protocolVersion: '1.20' + ); + $this->assertTrue($result['prepared'], $result['reason'] ?? ''); + return $result; + } + + /** + * @return array + */ + private function dispatch(WorkflowTask $task, string $scope, string $requestId): array + { + return app(DefaultWorkflowTaskBridge::class)->deliverCancellationScope( + $task->id, + $task->lease_owner, + $task->attempt_count, + $scope, + $requestId, + 3, + 'timer', + protocolVersion: '1.20' + ); + } + + private function timerTask(WorkflowRun $run, WorkflowTimer $timer): WorkflowTask + { + return $run->tasks() + ->where('task_type', TaskType::Timer)->where('payload->timer_id', $timer->id)->sole(); + } +} diff --git a/tests/Feature/V2/V2ScopedWaitCancellationTest.php b/tests/Feature/V2/V2ScopedWaitCancellationTest.php new file mode 100644 index 000000000..855936932 --- /dev/null +++ b/tests/Feature/V2/V2ScopedWaitCancellationTest.php @@ -0,0 +1,567 @@ + 'poll', + ]); + Carbon::setTestNow('2026-10-03T00:00:00Z'); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + #[DataProvider('waits')] + public function testDispatchPreservesOriginalWaitIdentityAndClaim(string $kind, ?int $timeout): void + { + [$workflow, $run, $claim, $scope, $prepared] = $this->wait($kind, $timeout); + $before = $claim->fresh() + ->getAttributes(); + $result = $this->dispatch($claim, $scope, $prepared); + $this->assertTrue($result['delivered'], $result['reason'] ?? ''); + $this->assertCount(1, $result['wait_cancellations']); + $this->assertSame($timeout !== 0, $result['wait_cancellations'][0]['cancelled']); + $this->assertSame($prepared['wait_members'], $result['wait_members']); + $this->assertHostingShortenedWithoutClaimReplacement($before, $claim); + $this->assertFalse($run->fresh()->status->isTerminal()); + $this->assertSame($result, $this->dispatch($claim, $scope, $prepared)); + $this->assertNotNull(CancellationScopeDelivery::recorded($run->fresh(), $scope)); + $wait = ($kind === 'signal' ? SignalWaits::forRun($run->fresh()) : ConditionWaits::forRun($run->fresh()))[0]; + if ($timeout !== 0) { + $this->assertSame('cancelled', $wait['status']); + $this->assertSame('scope_cancelled', $wait['source_status']); + $marker = $run->historyEvents() + ->findOrFail($result['wait_cancellations'][0]['history_event_id']); + $this->assertSame( + $prepared['authority_deadline_at'], + $marker->payload['cancellation_scope']['authority_deadline_at'] + ); + $this->assertSame( + $prepared['preparation_history_event_id'], + $marker->payload['cancellation_scope']['preparation_history_event_id'] + ); + } else { + $this->assertSame( + 0, + $run->historyEvents() + ->whereIn( + 'event_type', + [HistoryEventType::SignalWaitCancelled, HistoryEventType::ConditionWaitCancelled] + )->count() + ); + } + } + + public static function waits(): iterable + { + foreach (['signal', 'condition'] as $kind) { + foreach ([null, 0, 5] as $timeout) { + yield $kind . ':' . ($timeout ?? 'untimed') => [$kind, $timeout]; + } + } + } + + #[DataProvider('kinds')] + public function testLostAcknowledgementAndReplacementReuseCommittedWaitReceipt(string $kind): void + { + [$workflow, $run, $claim, $scope, $prepared] = $this->wait($kind, 5); + $member = $prepared['wait_members'][0]; + $receipt = ScopedWaitCancellation::fence( + $run->fresh(), + $claim, + $member['wait_id'], + $scope, + $prepared['request_id'], + '1.20' + ); + $before = $run->historyEvents() + ->count(); + Carbon::setTestNow('2026-10-03T00:00:03Z'); + $claim->forceFill([ + 'lease_owner' => 'replacement', + 'attempt_count' => 2, + ])->save(); + $this->assertSame( + $receipt, + ScopedWaitCancellation::fence( + $run->fresh(), + $claim, + $member['wait_id'], + $scope, + $prepared['request_id'], + '1.20' + ) + ); + $this->assertSame($before, $run->historyEvents()->count()); + $this->assertSame( + $prepared['authority_deadline_at'], + CancellationScopeDelivery::prepared($run->fresh(), $scope)->payload['authority_deadline_at'] + ); + $this->assertTrue($this->dispatch($claim, $scope, $prepared)['delivered']); + Carbon::setTestNow('2026-10-03T00:00:31Z'); + $this->assertNotNull(CancellationScopeDelivery::recorded($run->fresh(), $scope)); + } + + #[DataProvider('kinds')] + public function testWaitAndTimerFenceRollBackTogether(string $kind): void + { + [$workflow, $run, $claim, $scope, $prepared] = $this->wait($kind, 5); + WorkflowHistoryEvent::creating(static function (WorkflowHistoryEvent $event): void { + if ($event->event_type === HistoryEventType::TimerCancelled) { + throw new RuntimeException('injected timer fence write failure'); + } + }); + $before = $run->historyEvents() + ->count(); + try { + ScopedWaitCancellation::fence( + $run->fresh(), + $claim, + $prepared['wait_members'][0]['wait_id'], + $scope, + $prepared['request_id'], + '1.20' + ); + $this->fail('Both records must roll back.'); + } catch (RuntimeException $error) { + $this->assertSame('injected timer fence write failure', $error->getMessage()); + } + $this->assertSame($before, $run->historyEvents()->count()); + $this->assertSame(TimerStatus::Pending, $run->timers()->sole()->status); + $this->assertSame(TaskStatus::Ready, $run->tasks()->where('task_type', TaskType::Timer)->sole()->status); + } + + #[DataProvider('kinds')] + public function testDirectTimerEntryPointCancelsWaitAtomicallyAndLateJobCannotResume(string $kind): void + { + [$workflow, $run, $claim, $scope, $prepared] = $this->wait($kind, 5); + $timer = $run->timers() + ->sole(); + $task = $run->tasks() + ->where('task_type', TaskType::Timer)->sole(); + $this->assertTrue( + ScopedTimerCancellation::fence( + $run->fresh(), + $claim, + $timer->id, + $scope, + $prepared['request_id'], + '1.20' + )['fenced'] + ); + $marker = $run->historyEvents() + ->where( + 'event_type', + $kind === 'signal' ? HistoryEventType::SignalWaitCancelled : HistoryEventType::ConditionWaitCancelled + )->sole(); + $fence = $run->historyEvents() + ->where('event_type', HistoryEventType::TimerCancelled)->sole(); + $this->assertLessThan($fence->sequence, $marker->sequence); + $timer->forceFill([ + 'status' => TimerStatus::Pending, + ])->save(); + $task->forceFill([ + 'status' => TaskStatus::Ready, + ])->save(); + Carbon::setTestNow('2026-10-03T00:00:06Z'); + $before = $run->historyEvents() + ->count(); + $this->app->call([new RunTimerTask($task->id), 'handle']); + $this->assertSame($before, $run->historyEvents()->count()); + $this->assertSame(0, $run->historyEvents()->where('event_type', HistoryEventType::TimerFired)->count()); + $this->assertSame(2, $run->tasks()->where('task_type', TaskType::Workflow)->count()); + $this->assertTrue($this->dispatch($claim, $scope, $prepared)['delivered']); + } + + #[DataProvider('kinds')] + public function testNaturalTimeoutWinningFirstIsRetained(string $kind): void + { + [$workflow, $run, $claim, $scope, $prepared] = $this->wait($kind, 5); + Carbon::setTestNow('2026-10-03T00:00:06Z'); + $this->app->call([new RunTimerTask($run->tasks()->where('task_type', TaskType::Timer)->sole()->id), 'handle']); + $result = $this->dispatch($claim, $scope, $prepared); + $this->assertTrue($result['delivered'], $result['reason'] ?? ''); + $this->assertFalse($result['wait_cancellations'][0]['cancelled']); + $this->assertFalse($result['timer_cancellations'][0]['fenced']); + $this->assertSame(1, $run->historyEvents()->where('event_type', HistoryEventType::TimerFired)->count()); + $this->assertSame( + 0, + $run->historyEvents() + ->whereIn( + 'event_type', + [HistoryEventType::SignalWaitCancelled, HistoryEventType::ConditionWaitCancelled] + )->count() + ); + } + + public function testReceivedSignalRetainsBytesAndNaturalOutcome(): void + { + [$workflow, $run, $claim, $scope, $prepared] = $this->wait('signal', 5); + $workflow->signal('name-provided', 'retained-payload'); + $before = $run->signals() + ->sole() + ->getAttributes(); + $result = $this->dispatch($claim, $scope, $prepared); + $this->assertTrue($result['delivered'], $result['reason'] ?? ''); + $this->assertFalse($result['wait_cancellations'][0]['cancelled']); + $this->assertSame($before, $run->signals()->sole()->getAttributes()); + $this->assertSame('received', SignalWaits::forRun($run->fresh())[0]['source_status']); + } + + public function testSatisfiedConditionKeepsItsNaturalResultAndOriginalTimerCancellation(): void + { + [$workflow, $run, $claim, $scope, $prepared] = $this->wait('condition', 5); + $workflow->signal('name-provided', 'condition-satisfied'); + $signal = $run->signals() + ->sole(); + $claim->forceFill([ + 'payload' => [ + 'resume_source_kind' => 'workflow_signal', + 'workflow_signal_id' => $signal->id, + 'signal_name' => 'name-provided', + ], + ])->save(); + $reply = app(DefaultWorkflowTaskBridge::class)->complete($claim->id, []); + $this->assertTrue($reply['completed'], $reply['reason'] ?? ''); + $satisfied = $run->historyEvents() + ->where('event_type', HistoryEventType::ConditionWaitSatisfied)->sole(); + $timerCancellation = $run->historyEvents() + ->where('event_type', HistoryEventType::TimerCancelled)->sole(); + $this->assertArrayNotHasKey('cancellation_scope', $timerCancellation->payload); + $replacement = $claim->replicate(['id', 'created_at', 'updated_at'])->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'replacement-after-condition', + 'attempt_count' => 1, + 'payload' => null, + ]); + $replacement->save(); + $result = $this->dispatch($replacement, $scope, $prepared); + $this->assertTrue($result['delivered'], $result['reason'] ?? ''); + $this->assertFalse($result['wait_cancellations'][0]['cancelled']); + $this->assertSame($satisfied->id, $result['wait_cancellations'][0]['history_event_id']); + $this->assertTrue($result['timer_cancellations'][0]['fenced']); + $this->assertSame($timerCancellation->id, $result['timer_cancellations'][0]['history_event_id']); + $this->assertSame( + 0, + $run->historyEvents() + ->where('event_type', HistoryEventType::ConditionWaitCancelled)->count() + ); + $this->assertNotNull(CancellationScopeDelivery::recorded($run->fresh(), $scope)); + } + + #[DataProvider('expiredLocks')] + public function testAuthorityIsRecheckedAfterFinalTimerLock(string $kind, int $seconds, string $reason): void + { + [$workflow, $run, $claim, $scope, $prepared] = $this->wait($kind, 5); + if ($seconds === 5) { + $claim->forceFill([ + 'lease_expires_at' => now() + ->addSeconds(5), + ])->save(); + } else { + $this->advanceWithWorkflowHeartbeats($claim, Carbon::parse('2026-10-03T00:00:25Z')); + $this->assertTrue(app(DefaultWorkflowTaskBridge::class)->heartbeat($claim->id)['renewed']); + $this->assertSame('2026-10-03T00:00:35.000000Z', $claim->fresh()->lease_expires_at->toISOString()); + } + $timerId = $run->timers() + ->sole() +->id; + WorkflowTimer::retrieved(static function (WorkflowTimer $timer) use ($timerId, $seconds): void { + if ($timer->id === $timerId) { + Carbon::setTestNow(Carbon::parse('2026-10-03T00:00:00Z')->addSeconds($seconds)); + } + }); + $before = $run->historyEvents() + ->count(); + try { + ScopedWaitCancellation::fence( + $run->fresh(), + $claim, + $prepared['wait_members'][0]['wait_id'], + $scope, + $prepared['request_id'], + '1.20' + ); + $this->fail('A waited lock cannot extend authority.'); + } catch (LogicException $error) { + $this->assertSame($reason, $error->getMessage()); + } + $this->assertSame($before, $run->historyEvents()->count()); + } + + #[DataProvider('kinds')] + public function testLateSignalAndStaleResumeKeepMailboxBytesWithoutRevivingWait(string $kind): void + { + [$workflow, $run, $claim, $scope, $prepared] = $this->wait($kind, 5); + $this->assertTrue($this->dispatch($claim, $scope, $prepared)['delivered']); + $workflow->signal('name-provided', 'buffered-after-cancellation'); + $signal = $run->signals() + ->sole(); + $before = $signal->getAttributes(); + $claim->forceFill([ + 'payload' => [ + 'resume_source_kind' => 'workflow_signal', + 'workflow_signal_id' => $signal->id, + 'signal_name' => 'name-provided', + 'signal_wait_id' => $kind === 'signal' + ? $prepared['wait_members'][0]['wait_id'] : $signal->signal_wait_id, + ], + ])->save(); + $reply = app(DefaultWorkflowTaskBridge::class)->complete($claim->id, []); + $this->assertTrue($reply['completed'], $reply['reason'] ?? ''); + $this->assertSame($before, $run->signals()->sole()->getAttributes()); + $this->assertSame( + 0, + $run->historyEvents() + ->whereIn( + 'event_type', + [HistoryEventType::SignalApplied, HistoryEventType::ConditionWaitSatisfied] + )->count() + ); + $waits = $kind === 'signal' ? SignalWaits::forRun($run->fresh()) : ConditionWaits::forRun($run->fresh()); + $this->assertSame('cancelled', $waits[0]['status']); + $this->assertNotNull(CancellationScopeDelivery::recorded($run->fresh(), $scope)); + } + + public static function expiredLocks(): iterable + { + foreach (['signal', 'condition'] as $kind) { + yield $kind . ':claim' => [$kind, 5, 'cancellation_scope_workflow_claim_mismatch']; + yield $kind . ':deadline' => [$kind, 30, 'cancellation_scope_authority_expired']; + } + } + + #[DataProvider('receiptCorruption')] + public function testColdReplayRejectsChangedReceipt(string $kind, string $field): void + { + [$workflow, $run, $claim, $scope, $prepared] = $this->wait($kind, 5); + $result = $this->dispatch($claim, $scope, $prepared); + $this->assertTrue($result['delivered'], $result['reason'] ?? ''); + $marker = $run->historyEvents() + ->findOrFail($result['wait_cancellations'][0]['history_event_id']); + $snapshot = $marker->payload['cancellation_scope']; + $snapshot[$field] = 'changed'; + $marker->forceFill([ + 'payload' => [ + ...$marker->payload, + 'cancellation_scope' => $snapshot, + ], + ])->save(); + $this->expectExceptionMessage('cancellation_scope_delivery_history_invalid'); + CancellationScopeDelivery::recorded($run->fresh(), $scope); + } + + public static function receiptCorruption(): iterable + { + foreach (['signal', 'condition'] as $kind) { + foreach ([ + 'schema', + 'scope_id', + 'request_id', + 'request_history_event_id', + 'preparation_history_event_id', + 'authority_deadline_at', + ] as $field) { + yield $kind . ':' . $field => [$kind, $field]; + } + } + } + + #[DataProvider('scopeTrees')] + public function testSiblingAndShieldedWaitsSurviveAndInheritedWaitsShareTheOriginalBoundary( + string $kind, + string $mode + ): void { + [$workflow, $run, $claim, $scope, $prepared] = $this->wait($kind, 5, $mode); + $result = $this->dispatch($claim, $scope, $prepared); + $waits = $kind === 'signal' ? SignalWaits::forRun($run->fresh()) : ConditionWaits::forRun($run->fresh()); + $this->assertTrue($result['delivered'], $result['reason'] ?? ''); + $this->assertSame('cancelled', $waits[0]['status']); + $this->assertSame($mode === 'unshielded' ? 'cancelled' : 'open', $waits[1]['status']); + $this->assertSame( + $mode === 'unshielded' ? TimerStatus::Cancelled : TimerStatus::Pending, + $run->timers() + ->where('sequence', 4) + ->sole() +->status + ); + $this->assertSame( + 1, + $run->historyEvents() + ->where('event_type', HistoryEventType::CancellationScopeDeliveryPrepared)->count() + ); + $this->assertSame($result, $this->dispatch($claim, $scope, $prepared)); + } + + public static function scopeTrees(): iterable + { + foreach (['signal', 'condition'] as $kind) { + foreach (['sibling', 'shielded', 'unshielded'] as $mode) { + yield $kind . ':' . $mode => [$kind, $mode]; + } + } + } + + public static function kinds(): iterable + { + yield 'signal' => ['signal']; + yield 'condition' => ['condition']; + } + + /** + * @return array{WorkflowStub, WorkflowRun, WorkflowTask, string, array} + */ + private function wait(string $kind, ?int $timeout, ?string $treeMode = null): array + { + $workflow = WorkflowStub::make(TestSignalWorkflow::class); + $workflow->start(); + $run = $workflow->run(); + $task = $run->tasks() + ->where('task_type', TaskType::Workflow)->sole(); + $task->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'wait-owner', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addMinutes(5), + ])->save(); + $scope = CancellationScopeHistory::open($run, $task, 1, '1.20')->payload['scope_id']; + $sequence = 2; + $otherScope = null; + if ($treeMode !== null) { + $otherScope = CancellationScopeHistory::open( + $run, + $task, + 2, + '1.20', + parentScopeId: $treeMode === 'sibling' ? 'root' : $scope, + shieldParent: $treeMode === 'shielded' + )->payload['scope_id']; + $sequence = 3; + } + $bridge = app(DefaultWorkflowTaskBridge::class); + $command = array_filter( + [ + 'type' => 'open_' . $kind . '_wait', + 'cancellation_scope_id' => $scope, + 'timeout_seconds' => $timeout, + ...($kind === 'signal' ? [ + 'signal_name' => 'name-provided', + ] : [ + 'condition_key' => 'ready', + ]), + ], + static fn (mixed $value): bool => $value !== null + ); + $commands = [$command]; + if ($otherScope !== null) { + $commands[] = [ + ...$command, + 'cancellation_scope_id' => $otherScope, + ]; + } + $this->assertTrue($bridge->complete($task->id, $commands)['completed']); + $claim = $run->tasks() + ->create([ + 'namespace' => $run->namespace, + 'task_type' => TaskType::Workflow, + 'status' => TaskStatus::Leased, + 'lease_owner' => 'wait-hosting', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addMinutes(5), + 'available_at' => now(), + 'connection' => $run->connection, + 'queue' => $run->queue, + 'compatibility' => $run->compatibility, + ]); + $callKind = $kind; + if ($timeout === 0) { + $sequence = 3; + $callKind = 'timer'; + $reply = $bridge->checkpointCancellationScopePrefix( + $claim->id, + $claim->lease_owner, + 1, + 'after-immediate', + 3, + [[ + 'type' => 'start_timer', + 'delay_seconds' => 60, + 'cancellation_scope_id' => $scope, + ]], + '1.20' + ); + $this->assertTrue($reply['checkpointed'], $reply['reason'] ?? ''); + } + $request = CancellationScopeRequests::request($run->fresh(), $scope, '1.20', 30); + $prepared = $bridge->prepareCancellationScopeDelivery( + $claim->id, + $claim->lease_owner, + 1, + $scope, + $request->payload['request_id'], + $sequence, + $callKind, + protocolVersion: '1.20' + ); + $this->assertTrue($prepared['prepared'], $prepared['reason'] ?? ''); + $this->assertCount(1, $prepared['wait_members']); + return [$workflow, $run, $claim, $scope, $prepared]; + } + + /** @param array $prepared + * @return array + */ + private function dispatch(WorkflowTask $claim, string $scope, array $prepared): array + { + return app(DefaultWorkflowTaskBridge::class)->deliverCancellationScope( + $claim->id, + $claim->lease_owner, + $claim->attempt_count, + $scope, + $prepared['request_id'], + $prepared['sequence'], + $prepared['call_kind'], + protocolVersion: '1.20' + ); + } +} diff --git a/tests/Feature/V2/V2WorkflowTaskBridgeTest.php b/tests/Feature/V2/V2WorkflowTaskBridgeTest.php index 9d2fee167..09ad2341a 100644 --- a/tests/Feature/V2/V2WorkflowTaskBridgeTest.php +++ b/tests/Feature/V2/V2WorkflowTaskBridgeTest.php @@ -9435,6 +9435,45 @@ public function testExternalWorkflowCompletionAppliesRequestCancelParentClosePol $this->assertSame($link->child_workflow_run_id, $applied->payload['child_run_id'] ?? null); } + public function testExternalWorkflowCompletionRequestsCooperativeChildCleanup(): void + { + $run = $this->createWaitingRun(); + $task = $this->createLeasedTask($run); + $started = $this->bridge->complete($task->id, [[ + 'type' => 'start_child_workflow', + 'workflow_type' => 'test-greeting-workflow', + 'arguments' => Serializer::serialize(['child-arg']), + 'parent_close_policy' => 'request_cancellation', + ]]); + $this->assertTrue($started['completed']); + $parentCloseTask = $this->createLeasedTask($run->fresh()); + $closed = $this->bridge->complete($parentCloseTask->id, [[ + 'type' => 'complete_workflow', + 'result' => Serializer::serialize([ + 'parent' => 'done', + ]), + ]]); + $this->assertTrue($closed['completed']); + $this->assertSame('completed', $closed['run_status']); + $link = WorkflowLink::query()->where('parent_workflow_run_id', $run->id) + ->where('link_type', 'child_workflow') + ->sole(); + $child = WorkflowRun::query()->findOrFail($link->child_workflow_run_id); + $this->assertFalse($child->status->isTerminal()); + $this->assertNotNull($child->cancellation_request_command_id); + $root = \Workflow\V2\Support\ParentCloseCancellation::context($run->fresh()); + $context = \Workflow\V2\Support\CooperativeCancellationDelivery::context($child); + $this->assertSame($root->rootRequestId, $context->rootRequestId); + $this->assertSame($root->deadline()->toISOString(), $context->deadline()->toISOString()); + $childCall = WorkflowChildCall::query()->where('parent_workflow_run_id', $run->id)->sole(); + $this->assertSame(ChildCallStatus::Started, $childCall->status); + $this->assertNull($run->fresh()->cancellation_request_command_id); + $this->assertSame([], \Workflow\V2\Support\ParentClosePolicyEnforcer::enforce($run->fresh())); + $receipts = $run->historyEvents() + ->where('event_type', HistoryEventType::ParentClosePolicyApplied); + $this->assertSame(1, $receipts->count()); + } + public function testExternalWorkflowFailureAppliesTerminateParentClosePolicy(): void { $run = $this->createWaitingRun(); diff --git a/tests/Fixtures/V2/ReplayRegression/cancellation-context-detached-metadata-cold-replay.json b/tests/Fixtures/V2/ReplayRegression/cancellation-context-detached-metadata-cold-replay.json new file mode 100644 index 000000000..dd40c3633 --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/cancellation-context-detached-metadata-cold-replay.json @@ -0,0 +1,97 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "cancellation-context-detached-metadata-cold-replay", + "protocol_version": "1.0", + "bindings": [ + "php" + ], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestCancellationContextBindingWorkflow", + "arguments": [], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": {}, + "recorded_at": "2026-10-01T00:00:00Z" + }, + { + "sequence": 2, + "event_type": "TimerScheduled", + "payload": { + "sequence": 1 + }, + "recorded_at": "2026-10-01T00:00:01Z" + }, + { + "sequence": 3, + "event_type": "CooperativeCancellationRequested", + "payload": { + "workflow_command_id": "root-1", + "workflow_instance_id": "root-instance", + "workflow_run_id": "root-run", + "command_type": "request_cancellation", + "reason": "maintenance", + "cleanup_deadline_at": "2026-10-01T00:00:32.000000Z", + "cancellation": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "root-1", + "root_request_id": "root-1", + "root_workflow_instance_id": "root-instance", + "root_workflow_run_id": "root-run", + "parent_request_id": null, + "reason": "maintenance", + "requester": { + "type": "operator", + "id": "operator-1" + }, + "source": "control_plane", + "requested_at": "2026-10-01T00:00:02.000000Z", + "cleanup_deadline_at": "2026-10-01T00:00:32.000000Z", + "lineage": [ + { + "request_id": "root-1", + "workflow_instance_id": "root-instance", + "workflow_run_id": "root-run" + } + ] + } + }, + "recorded_at": "2026-10-01T00:00:02Z" + }, + { + "sequence": 4, + "event_type": "TimerCancelled", + "payload": { + "sequence": 1 + }, + "recorded_at": "2026-10-01T00:00:05Z" + }, + { + "sequence": 5, + "event_type": "CooperativeCancellationDelivered", + "payload": { + "workflow_command_id": "root-1", + "workflow_run_id": "root-run", + "sequence": 1, + "call_kind": "timer" + }, + "recorded_at": "2026-10-01T00:00:05Z" + } + ], + "expected": { + "completed": true, + "result": [ + true, + true + ], + "commands": [ + { + "type": "complete_workflow" + } + ] + } +} diff --git a/tests/Fixtures/V2/ReplayRegression/cancellation-context-original-budget-cold-replay.json b/tests/Fixtures/V2/ReplayRegression/cancellation-context-original-budget-cold-replay.json new file mode 100644 index 000000000..dc53501e7 --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/cancellation-context-original-budget-cold-replay.json @@ -0,0 +1,70 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "cancellation-context-original-budget-cold-replay", + "protocol_version": "1.0", + "bindings": ["php"], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestCancellationContextWorkflow", + "arguments": [], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": {}, + "recorded_at": "2026-10-01T00:00:00Z" + }, + { + "sequence": 2, + "event_type": "TimerScheduled", + "payload": {"sequence": 1}, + "recorded_at": "2026-10-01T00:00:01Z" + }, + { + "sequence": 3, + "event_type": "CooperativeCancellationRequested", + "payload": { + "workflow_command_id": "root-1", + "workflow_instance_id": "root-instance", + "workflow_run_id": "root-run", + "command_type": "request_cancellation", + "reason": "maintenance", + "cleanup_deadline_at": "2026-10-01T00:00:32.000000Z", + "cancellation": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "root-1", + "root_request_id": "root-1", + "root_workflow_instance_id": "root-instance", + "root_workflow_run_id": "root-run", + "parent_request_id": null, + "reason": "maintenance", + "requester": {"type": "operator", "id": "operator-1"}, + "source": "control_plane", + "requested_at": "2026-10-01T00:00:02.000000Z", + "cleanup_deadline_at": "2026-10-01T00:00:32.000000Z", + "lineage": [{"request_id": "root-1", "workflow_instance_id": "root-instance", "workflow_run_id": "root-run"}] + } + }, + "recorded_at": "2026-10-01T00:00:02Z" + }, + { + "sequence": 4, + "event_type": "TimerCancelled", + "payload": {"sequence": 1}, + "recorded_at": "2026-10-01T00:00:05Z" + }, + { + "sequence": 5, + "event_type": "CooperativeCancellationDelivered", + "payload": {"workflow_command_id": "root-1", "workflow_run_id": "root-run", "sequence": 1, "call_kind": "timer"}, + "recorded_at": "2026-10-01T00:00:05Z" + } + ], + "expected": { + "completed": false, + "result": null, + "commands": [{"type": "upsert_memo"}, {"type": "start_timer", "delay_seconds": 1}] + } +} diff --git a/tests/Fixtures/V2/ReplayRegression/cancellation-scope-reserved-command-position.json b/tests/Fixtures/V2/ReplayRegression/cancellation-scope-reserved-command-position.json new file mode 100644 index 000000000..0f51d83fe --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/cancellation-scope-reserved-command-position.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "cancellation-scope-reserved-command-position", + "protocol_version": "1.20", + "bindings": ["php"], + "consumers": ["workflow-fiber-runner", "workflow-executor", "query-state-replayer"], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestGreetingWorkflow", + "arguments": ["Ada"], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": {}, + "recorded_at": "2026-10-03T00:00:00Z" + }, + { + "sequence": 2, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "regression-corpus-run-cancellation-scope-reserved-command-position", + "sequence": 1, + "scope_id": "scope-reserved-position-1", + "parent_scope_id": "root", + "shield_parent": false + }, + "recorded_at": "2026-10-03T00:00:01Z" + } + ], + "expected_failure": { + "type": "workflow_history_shape_mismatch", + "exception": "Workflow\\V2\\Exceptions\\HistoryEventShapeMismatchException" + } +} diff --git a/tests/Fixtures/V2/ReplayRegression/cooperative-group-cancelled-history-payload.json b/tests/Fixtures/V2/ReplayRegression/cooperative-group-cancelled-history-payload.json new file mode 100644 index 000000000..aab1d8699 --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/cooperative-group-cancelled-history-payload.json @@ -0,0 +1,47 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "cooperative-group-cancelled-history-payload", + "protocol_version": "1.20", + "bindings": ["php"], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestCooperativeParallelCleanupWorkflow", + "arguments": [false], + "payload_codec": "avro" + }, + "history": [ + {"sequence": 1, "event_type": "WorkflowStarted", "payload": {}, "recorded_at": "2026-10-02T00:00:00+00:00"}, + { + "sequence": 2, + "event_type": "CooperativeCancellationRequested", + "payload": {"workflow_command_id": "cancel-group-1", "command_type": "request_cancellation"}, + "recorded_at": "2026-10-02T00:00:01+00:00" + }, + { + "sequence": 3, + "event_type": "ActivityCancelled", + "payload": { + "activity_execution_id": "activity-one", + "sequence": 1, + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [{"parallel_group_id": "parallel-activities:1:2", "parallel_group_kind": "activity", "parallel_group_base_sequence": 1, "parallel_group_size": 2, "parallel_group_index": 0}] + }, + "recorded_at": "2026-10-02T00:00:02+00:00" + }, + { + "sequence": 4, + "event_type": "CooperativeCancellationDelivered", + "payload": {"workflow_command_id": "cancel-group-1", "sequence": 1, "sequence_span": 2, "call_kind": "parallel"}, + "recorded_at": "2026-10-02T00:00:03+00:00" + } + ], + "expected": { + "completed": false, + "result": null, + "commands": [{"type": "schedule_activity", "activity_type": "Tests\\Fixtures\\V2\\TestGreetingActivity"}] + } +} diff --git a/tests/Fixtures/V2/ReplayRegression/cooperative-group-heartbeat-history-payload.json b/tests/Fixtures/V2/ReplayRegression/cooperative-group-heartbeat-history-payload.json new file mode 100644 index 000000000..1f1fe1164 --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/cooperative-group-heartbeat-history-payload.json @@ -0,0 +1,66 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "cooperative-group-heartbeat-history-payload", + "protocol_version": "1.20", + "bindings": ["php"], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestCooperativeParallelCleanupWorkflow", + "arguments": [false], + "payload_codec": "avro" + }, + "history": [ + {"sequence": 1, "event_type": "WorkflowStarted", "payload": {}, "recorded_at": "2026-10-02T00:00:00+00:00"}, + { + "sequence": 2, + "event_type": "ActivityHeartbeatRecorded", + "payload": { + "activity_execution_id": "activity-one", + "activity_attempt_id": "attempt-one", + "sequence": 1, + "execution_mode": "local", + "local_activity": true, + "progress": {"details": {"phase": "waiting"}}, + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [{"parallel_group_id": "parallel-activities:1:2", "parallel_group_kind": "activity", "parallel_group_base_sequence": 1, "parallel_group_size": 2, "parallel_group_index": 0}] + }, + "recorded_at": "2026-10-02T00:00:00.500000+00:00" + }, + { + "sequence": 3, + "event_type": "CooperativeCancellationRequested", + "payload": {"workflow_command_id": "cancel-group-1", "command_type": "request_cancellation"}, + "recorded_at": "2026-10-02T00:00:01+00:00" + }, + { + "sequence": 4, + "event_type": "ActivityCancelled", + "payload": { + "activity_execution_id": "activity-one", + "sequence": 1, + "parallel_group_id": "parallel-activities:1:2", + "parallel_group_kind": "activity", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [{"parallel_group_id": "parallel-activities:1:2", "parallel_group_kind": "activity", "parallel_group_base_sequence": 1, "parallel_group_size": 2, "parallel_group_index": 0}] + }, + "recorded_at": "2026-10-02T00:00:02+00:00" + }, + { + "sequence": 5, + "event_type": "CooperativeCancellationDelivered", + "payload": {"workflow_command_id": "cancel-group-1", "sequence": 1, "sequence_span": 2, "call_kind": "parallel"}, + "recorded_at": "2026-10-02T00:00:03+00:00" + } + ], + "expected": { + "completed": false, + "result": null, + "commands": [{"type": "schedule_activity", "activity_type": "Tests\\Fixtures\\V2\\TestGreetingActivity"}] + } +} diff --git a/tests/Fixtures/V2/ReplayRegression/root-delivery-scoped-operation-membership.json b/tests/Fixtures/V2/ReplayRegression/root-delivery-scoped-operation-membership.json new file mode 100644 index 000000000..5b239aaa7 --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/root-delivery-scoped-operation-membership.json @@ -0,0 +1,55 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "root-delivery-scoped-operation-membership", + "protocol_version": "1.20", + "bindings": ["php"], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestGreetingWorkflow", + "arguments": ["Ada"], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": {}, + "recorded_at": "2026-10-05T00:00:00Z" + }, + { + "sequence": 2, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "regression-corpus-run-root-delivery-scoped-operation-membership", + "sequence": 1, + "scope_id": "shielded-scope-1", + "parent_scope_id": "root", + "shield_parent": true + }, + "recorded_at": "2026-10-05T00:00:01Z" + }, + { + "sequence": 3, + "event_type": "ActivityScheduled", + "payload": { + "sequence": 2, + "activity_class": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "activity_type": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "activity": {"cancellation_scope_id": "shielded-scope-1"} + }, + "recorded_at": "2026-10-05T00:00:02Z" + }, + { + "sequence": 4, + "event_type": "CooperativeCancellationRequested", + "workflow_command_id": "request-1", + "payload": {"workflow_command_id": "request-1", "command_type": "request_cancellation"}, + "recorded_at": "2026-10-05T00:00:03Z" + } + ], + "expected_failure": { + "type": "workflow_history_shape_mismatch", + "exception": "Workflow\\V2\\Exceptions\\HistoryEventShapeMismatchException" + } +} diff --git a/tests/Fixtures/V2/ReplayRegression/run-inherited-scoped-cleanup-cold-reload.json b/tests/Fixtures/V2/ReplayRegression/run-inherited-scoped-cleanup-cold-reload.json new file mode 100644 index 000000000..75e5511a1 --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/run-inherited-scoped-cleanup-cold-reload.json @@ -0,0 +1,50 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "run-inherited-scoped-cleanup-cold-reload", + "protocol_version": "1.20", + "bindings": ["php"], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestGreetingWorkflow", + "arguments": ["Ada"], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": {}, + "recorded_at": "2026-10-05T00:00:00Z" + }, + { + "sequence": 2, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "regression-corpus-run-run-inherited-scoped-cleanup-cold-reload", + "sequence": 1, + "scope_id": "parent-scope-1", + "parent_scope_id": "root", + "shield_parent": false + }, + "recorded_at": "2026-10-05T00:00:01Z" + }, + { + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "regression-corpus-run-run-inherited-scoped-cleanup-cold-reload", + "sequence": 2, + "scope_id": "child-scope-2", + "parent_scope_id": "parent-scope-1", + "shield_parent": false + }, + "recorded_at": "2026-10-05T00:00:02Z" + } + ], + "expected_failure": { + "type": "workflow_history_shape_mismatch", + "exception": "Workflow\\V2\\Exceptions\\HistoryEventShapeMismatchException" + } +} diff --git a/tests/Fixtures/V2/TestCancellationContextBindingWorkflow.php b/tests/Fixtures/V2/TestCancellationContextBindingWorkflow.php new file mode 100644 index 000000000..7335ea00e --- /dev/null +++ b/tests/Fixtures/V2/TestCancellationContextBindingWorkflow.php @@ -0,0 +1,36 @@ +cancellation ?? throw new LogicException('Canonical cancellation context is missing.'); + $boundBudgetIsCorrect = $context->remaining() === 27.0; + try { + CancellationContext::fromArray($context->toArray())->remaining(); + } catch (LogicException) { + return [$boundBudgetIsCorrect, true]; + } + + return [$boundBudgetIsCorrect, false]; + } + + return [false, false]; + } +} diff --git a/tests/Fixtures/V2/TestCancellationContextWorkflow.php b/tests/Fixtures/V2/TestCancellationContextWorkflow.php new file mode 100644 index 000000000..205fb629b --- /dev/null +++ b/tests/Fixtures/V2/TestCancellationContextWorkflow.php @@ -0,0 +1,29 @@ +cancellation ?? throw new LogicException('Canonical cancellation context is missing.'); + upsertMemo([ + 'cancellation' => $context->toArray(), + 'remaining' => $context->remaining(), + ]); + timer($context->remaining() === 27.0 ? 1 : 2); + throw $cancel; + } + } +} diff --git a/tests/Fixtures/V2/TestChildPolicyCleanupWorkflow.php b/tests/Fixtures/V2/TestChildPolicyCleanupWorkflow.php new file mode 100644 index 000000000..94176e560 --- /dev/null +++ b/tests/Fixtures/V2/TestChildPolicyCleanupWorkflow.php @@ -0,0 +1,27 @@ + 'complete', + ]); + }); + } + } +} diff --git a/tests/Fixtures/V2/TestElapsedLocalCleanupActivity.php b/tests/Fixtures/V2/TestElapsedLocalCleanupActivity.php new file mode 100644 index 000000000..bfde4ffcb --- /dev/null +++ b/tests/Fixtures/V2/TestElapsedLocalCleanupActivity.php @@ -0,0 +1,18 @@ +addSeconds(11)); + + return 'cleaned'; + } +} diff --git a/tests/Fixtures/V2/TestElapsedLocalCleanupWorkflow.php b/tests/Fixtures/V2/TestElapsedLocalCleanupWorkflow.php new file mode 100644 index 000000000..3582aeb1d --- /dev/null +++ b/tests/Fixtures/V2/TestElapsedLocalCleanupWorkflow.php @@ -0,0 +1,26 @@ + localActivity(TestElapsedLocalCleanupActivity::class), + ]); + } + } +} diff --git a/tests/Fixtures/V2/TestParentChildPolicyWorkflow.php b/tests/Fixtures/V2/TestParentChildPolicyWorkflow.php new file mode 100644 index 000000000..602f25d61 --- /dev/null +++ b/tests/Fixtures/V2/TestParentChildPolicyWorkflow.php @@ -0,0 +1,55 @@ + child( + TestChildPolicyCleanupWorkflow::class, + new ChildWorkflowOptions( + parentClosePolicy: ParentClosePolicy::from($parentClosePolicy), + cancellationPolicy: CancellationPolicy::from($policy), + ), + ); + if ($selection) { + $selected = select([ + 'child' => $child, + 'ready' => static fn () => timer(0), + ]); + $selected->handles['child']->await(); + } elseif ($parallel) { + all([ + $child, + static fn () => activity(TestGreetingActivity::class, 'ordinary work'), + ]); + } else { + $child(); + } + } finally { + cancellationShield(static fn () => upsertMemo([ + 'parent_cleanup' => 'complete', + ])); + } + } +} diff --git a/tests/Fixtures/V2/TestParentCloseCooperativeWorkflow.php b/tests/Fixtures/V2/TestParentCloseCooperativeWorkflow.php new file mode 100644 index 000000000..e6cf2cfb6 --- /dev/null +++ b/tests/Fixtures/V2/TestParentCloseCooperativeWorkflow.php @@ -0,0 +1,31 @@ + static fn () => child( + TestChildPolicyCleanupWorkflow::class, + new ChildWorkflowOptions(parentClosePolicy: ParentClosePolicy::RequestCancellation), + ), + 'second' => static fn () => child( + TestChildPolicyCleanupWorkflow::class, + new ChildWorkflowOptions(parentClosePolicy: ParentClosePolicy::RequestCancellation), + ), + 'ready' => static fn () => timer(1), + ]); + } +} diff --git a/tests/TestCase.php b/tests/TestCase.php index da2aed128..0859fda1e 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -4,11 +4,13 @@ namespace Tests; +use Carbon\CarbonInterface; use Dotenv\Dotenv; use Illuminate\Console\OutputStyle; use Illuminate\Database\Schema\Builder as SchemaBuilder; use Illuminate\Filesystem\Filesystem; use Illuminate\Foundation\Testing\DatabaseTruncation; +use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Queue; use Illuminate\Support\Str; @@ -26,6 +28,7 @@ use Workflow\V2\Models\WorkflowRun as V2WorkflowRun; use Workflow\V2\Models\WorkflowTask as V2WorkflowTask; use Workflow\V2\Models\WorkflowUpdate as V2WorkflowUpdate; +use Workflow\V2\Support\DefaultWorkflowTaskBridge; use Workflow\V2\Support\WorkflowDefinition; use Workflow\V2\TaskWatchdog; use Workflow\V2\WorkflowStub as V2WorkflowStub; @@ -168,6 +171,26 @@ protected function tearDown(): void } } + protected function assertHostingShortenedWithoutClaimReplacement(array $before, V2WorkflowTask $task): void + { + $claim = $task->fresh(); + $this->assertSame(now()->addSeconds(10)->toISOString(), $claim->lease_expires_at->toISOString()); + $after = $claim->getAttributes(); + unset($before['lease_expires_at'], $after['lease_expires_at'], $before['updated_at'], $after['updated_at']); + $this->assertSame($before, $after); + } + + protected function advanceWithWorkflowHeartbeats(V2WorkflowTask $task, CarbonInterface $target): void + { + $this->assertTrue($target->gte(now())); + while (($expiry = $task->fresh()->lease_expires_at) !== null && $expiry->lte($target)) { + $this->assertTrue(now()->lt($expiry), 'An expired claim cannot be kept alive by this fixture.'); + Carbon::setTestNow(now()->addMicroseconds((int) (now()->diffInMicroseconds($expiry) / 2))); + $this->assertTrue(app(DefaultWorkflowTaskBridge::class)->heartbeat($task->id)['renewed']); + } + Carbon::setTestNow($target); + } + protected function setUpDatabaseTruncation(): void { // Testbench's PendingCommand binds a mocked OutputStyle while it runs diff --git a/tests/Unit/V2/ActivityCancellationCompletionTest.php b/tests/Unit/V2/ActivityCancellationCompletionTest.php new file mode 100644 index 000000000..e493acc6d --- /dev/null +++ b/tests/Unit/V2/ActivityCancellationCompletionTest.php @@ -0,0 +1,335 @@ +previousContainer = Container::getInstance(); + $app = new Application(dirname(__DIR__, 3)); + $app->instance('config', new Repository()); + } + + protected function tearDown(): void + { + Container::setInstance($this->previousContainer); + parent::tearDown(); + } + + #[DataProvider('callbackModes')] + public function testOnlyTheOriginalOwnersCanonicalStopReceiptResolvesTheWait(bool $local): void + { + $run = $this->fixtureRun($local); + $this->assertFalse(ActivityCancellationCompletion::resolved($run, 'activity')); + $receipt = $this->receipt($local); + $run->historyEvents->push($receipt); + $this->assertTrue(ActivityCancellationCompletion::resolved($run, 'activity')); + $this->assertSame($receipt, ActivityCancellationCompletion::stopReceipt($run, $run->historyEvents[3])); + } + + public static function callbackModes(): iterable + { + yield 'remote' => [false]; + yield 'local' => [true]; + } + + public function testAnOlderScheduleKeepsTryCancelAndAnExplicitCanonicalPolicySurvivesColdReplay(): void + { + $run = $this->fixtureRun(); + $this->assertSame(CancellationPolicy::TryCancel, ActivityCancellationWait::policy($run, 1)); + $scheduled = $run->historyEvents[0]; + $scheduled->payload = [ + ...$scheduled->payload, + 'activity' => [ + 'cancellation_policy' => CancellationPolicy::WaitCancellationCompleted->value, + ], + ]; + $this->assertSame(CancellationPolicy::WaitCancellationCompleted, ActivityCancellationWait::policy($run, 1)); + } + + public function testTheCompletedHostingClaimPreservesBoundaryRootAndDeadlineAcrossReplacement(): void + { + $run = $this->fixtureRun(); + $task = new WorkflowTask([ + 'status' => TaskStatus::Completed, + 'payload' => [ + 'cancellation_activity_wait' => [ + 'request_id' => 'request', + 'root_request_id' => 'request', + 'cleanup_deadline_at' => '2026-10-02T13:00:30.000000Z', + 'execution_ids' => ['activity'], + ], + ], + ]); + ActivityCancellationWait::bindBoundary($task, 1, 'activity', 1, null, 1); + $run->setRelation('tasks', new Collection([$task])); + $this->assertNull(ActivityCancellationWait::validateBoundary($run, 1, 'activity', 1, null, 1)); + $this->assertSame( + 'cancellation_wait_boundary_mismatch', + ActivityCancellationWait::validateBoundary($run, 2, 'timer', 1, null, 1) + ); + $payload = $task->payload; + $payload['cancellation_activity_wait']['cleanup_deadline_at'] = '2026-10-02T13:01:30.000000Z'; + $task->payload = $payload; + $this->assertSame( + 'cancellation_wait_context_mismatch', + ActivityCancellationWait::validateBoundary($run, 1, 'activity', 1, null, 1) + ); + } + + #[DataProvider('invalidReceiptFences')] + public function testAChangedReceiptCannotResolveAnotherAttemptsWait(string $field, mixed $value): void + { + $run = $this->fixtureRun(); + $receipt = $this->receipt(); + $receipt->payload = [ + ...$receipt->payload, + $field => $value, + ]; + $run->historyEvents->push($receipt); + $this->assertFalse(ActivityCancellationCompletion::resolved($run, 'activity')); + } + + public static function invalidReceiptFences(): iterable + { + foreach ([ + 'sequence' => 2, + 'activity_execution_id' => 'other-activity', + 'activity_attempt_id' => 'replacement-attempt', + 'lease_owner' => 'replacement-worker', + 'cancellation_history_event_id' => 'other-fence', + 'request_id' => 'other-request', + 'root_request_id' => 'other-root', + 'cleanup_deadline_at' => '2026-10-02T13:00:31.000000Z', + 'callback_state' => 'unknown', + 'evidence_source' => 'workflow_worker', + ] as $field => $value) { + yield $field => [$field, $value]; + yield $field . ' missing' => [$field, null]; + } + } + + public function testAnExpiredLeaseAndTerminalProjectionDoNotProveCallbackExit(): void + { + $run = $this->fixtureRun(); + $run->status = 'cancelled'; + $run->closed_at = '2026-10-02T13:00:30Z'; + $this->assertFalse(ActivityCancellationCompletion::resolved($run, 'activity')); + } + + public function testCancellationBeforeAdmissionNeedsBothCanonicalScheduleAndNoAttemptFence(): void + { + $run = $this->fixtureRun(); + $run->historyEvents->forget(1); + $cancelled = $run->historyEvents[3]; + $cancelled->payload = [ + ...$cancelled->payload, + 'activity_attempt_id' => null, + 'activity_attempt' => null, + ]; + $this->assertTrue(ActivityCancellationCompletion::resolved($run, 'activity')); + $run->historyEvents->forget(0); + $this->assertFalse(ActivityCancellationCompletion::resolved($run, 'activity')); + } + + public function testAnOmittedAttemptDoesNotImpersonateARecordedNoAttemptFence(): void + { + $run = $this->fixtureRun(); + $run->historyEvents->forget(1); + $payload = $run->historyEvents[3]->payload; + unset($payload['activity_attempt_id']); + $payload['activity_attempt'] = null; + $run->historyEvents[3]->payload = $payload; + $this->assertFalse(ActivityCancellationCompletion::resolved($run, 'activity')); + } + + public function testAnEarlierReceiptOrDifferentCommandCannotResolveTheWait(): void + { + $run = $this->fixtureRun(); + $receipt = $this->receipt(); + $receipt->sequence = 3; + $run->historyEvents->push($receipt); + $this->assertFalse(ActivityCancellationCompletion::resolved($run, 'activity')); + $receipt->sequence = 5; + $receipt->workflow_command_id = 'other-request'; + $this->assertFalse(ActivityCancellationCompletion::resolved($run, 'activity')); + } + + public function testAPostFenceOutcomeCannotImpersonateCallbackStopAcknowledgement(): void + { + $run = $this->fixtureRun(); + $run->historyEvents->push($this->event(HistoryEventType::ActivityCompleted, 5, [ + 'sequence' => 1, + 'activity_execution_id' => 'activity', + 'activity_attempt_id' => 'attempt', + ])); + $this->assertFalse(ActivityCancellationCompletion::resolved($run, 'activity')); + } + + #[DataProvider('outcomes')] + public function testOnlyAnOutcomeOfTheLatestStartedAttemptProvesCallbackCompletion( + HistoryEventType $type, + bool $resolved, + ): void { + $run = $this->fixtureRun(); + $run->historyEvents->forget(3); + $run->historyEvents->push($this->event($type, 4, [ + 'sequence' => 1, + 'activity_execution_id' => 'activity', + 'activity_attempt_id' => 'attempt', + ])); + $this->assertSame($resolved, ActivityCancellationCompletion::resolved($run, 'activity')); + $run->historyEvents->push($this->event(HistoryEventType::ActivityStarted, 5, [ + 'sequence' => 1, + 'activity_execution_id' => 'activity', + 'activity_attempt_id' => 'replacement-attempt', + ])); + $this->assertFalse(ActivityCancellationCompletion::resolved($run, 'activity')); + } + + public static function outcomes(): iterable + { + yield 'completed callback' => [HistoryEventType::ActivityCompleted, true]; + yield 'failed callback' => [HistoryEventType::ActivityFailed, true]; + yield 'timeout fence' => [HistoryEventType::ActivityTimedOut, false]; + yield 'retry scheduling' => [HistoryEventType::ActivityRetryScheduled, false]; + } + + #[DataProvider('retryStopEvidence')] + public function testRetryOnlyProvesExitWithTheOriginalCallbacksRecordedFailure( + string $status, + ?string $timeout, + string $attemptId, + bool $resolved, + ): void { + $run = $this->fixtureRun(); + $run->historyEvents->forget(3); + $run->historyEvents->push($this->event(HistoryEventType::ActivityRetryScheduled, 4, [ + 'sequence' => 1, + 'activity_execution_id' => 'activity', + 'activity_attempt_id' => 'attempt', + 'timeout_kind' => $timeout, + 'activity_attempt' => [ + 'id' => $attemptId, + 'activity_execution_id' => 'activity', + 'status' => $status, + 'closed_at' => '2026-10-02T13:00:01.000000Z', + ], + ])); + $this->assertSame($resolved, ActivityCancellationCompletion::resolved($run, 'activity')); + } + + public static function retryStopEvidence(): iterable + { + yield 'original callback failed' => ['failed', null, 'attempt', true]; + yield 'expired lease' => ['expired', null, 'attempt', false]; + yield 'timeout fence' => ['failed', 'start_to_close', 'attempt', false]; + yield 'different attempt' => ['failed', null, 'replacement', false]; + } + + private function fixtureRun(bool $local = false): WorkflowRun + { + $run = new WorkflowRun([ + 'id' => 'run', + 'cancellation_request_command_id' => 'request', + ]); + $run->setRelation('historyEvents', new Collection([ + $this->event(HistoryEventType::ActivityScheduled, 1, [ + 'sequence' => 1, + 'activity_execution_id' => 'activity', + ]), + $this->event(HistoryEventType::ActivityStarted, 2, [ + 'sequence' => 1, + 'activity_execution_id' => 'activity', + 'activity_attempt_id' => 'attempt', + ]), + $this->event(HistoryEventType::CooperativeCancellationRequested, 3, [ + 'cancellation' => [ + 'schema' => 'durable-workflow.cancellation-context/v1', + 'request_id' => 'request', + 'root_request_id' => 'request', + 'root_workflow_instance_id' => 'instance', + 'root_workflow_run_id' => 'run', + 'parent_request_id' => null, + 'reason' => 'shutdown', + 'requester' => [ + 'type' => 'operator', + ], + 'source' => 'control_plane', + 'requested_at' => '2026-10-02T13:00:00.000000Z', + 'cleanup_deadline_at' => '2026-10-02T13:00:30.000000Z', + 'lineage' => [[ + 'request_id' => 'request', + 'workflow_instance_id' => 'instance', + 'workflow_run_id' => 'run', + ]], + ], + ]), + $this->event(HistoryEventType::ActivityCancelled, 4, [ + 'sequence' => 1, + 'activity_execution_id' => 'activity', + 'activity_attempt_id' => 'attempt', + 'local_activity' => $local, + 'activity_attempt' => [ + 'id' => 'attempt', + 'activity_execution_id' => 'activity', + 'status' => 'cancelled', + 'lease_owner' => 'original-worker', + ], + ]), + ])); + return $run; + } + + private function receipt(bool $local = false): WorkflowHistoryEvent + { + return $this->event(HistoryEventType::ActivityCancellationAcknowledged, 5, [ + 'sequence' => 1, + 'activity_execution_id' => 'activity', + 'activity_attempt_id' => 'attempt', + 'lease_owner' => 'original-worker', + 'cancellation_history_event_id' => 'event-4', + 'request_id' => 'request', + 'root_request_id' => 'request', + 'cleanup_deadline_at' => '2026-10-02T13:00:30.000000Z', + 'callback_state' => 'stopped', + 'evidence_source' => $local ? 'workflow_worker' : 'activity_worker', + ]); + } + + /** + * @param array $payload + */ + private function event(HistoryEventType $type, int $sequence, array $payload): WorkflowHistoryEvent + { + return new WorkflowHistoryEvent([ + 'id' => 'event-' . $sequence, + 'workflow_run_id' => 'run', + 'workflow_command_id' => 'request', + 'event_type' => $type, + 'sequence' => $sequence, + 'payload' => $payload, + ]); + } +} diff --git a/tests/Unit/V2/CancellationContextTest.php b/tests/Unit/V2/CancellationContextTest.php new file mode 100644 index 000000000..fd0e7b353 --- /dev/null +++ b/tests/Unit/V2/CancellationContextTest.php @@ -0,0 +1,308 @@ +snapshot(); + $context = CancellationContext::fromArray($snapshot); + $snapshot['reason'] = 'changed'; + $snapshot['requester']['id'] = 'changed'; + $snapshot['lineage'][0]['request_id'] = 'changed'; + $context->deadline() + ->addHour(); + $context->requestedAt() + ->addDay(); + + $this->assertSame('maintenance', $context->reason); + $this->assertSame('operator-1', $context->requester['id']); + $this->assertSame('root-1', $context->lineage[0]['request_id']); + $this->assertSame($this->snapshot(), $context->toArray()); + $this->assertSame($context->toArray(), CancellationContext::fromArray($context->toArray())->toArray()); + } + + public function testRequesterObjectKeyOrderCannotChangeCanonicalIdentityOrDescendantContext(): void + { + $original = $this->snapshot(); + $original['requester']['label'] = 'Operator'; + $reordered = $original; + $reordered['requester'] = array_reverse($original['requester'], true); + $context = CancellationContext::fromArray($original); + $copy = CancellationContext::fromArray($reordered); + + $this->assertSame($context->toArray(), $copy->toArray()); + $this->assertSame($context->requester, $copy->requester); + $this->assertSame( + $context->forDescendant('child', 'instance-child', 'run-child') + ->toArray(), + $copy->forDescendant('child', 'instance-child', 'run-child') + ->toArray() + ); + $this->assertSame($context->toArray(), CancellationContext::fromArray($copy->toArray())->toArray()); + $this->assertSame($context->deadline()->toISOString(), $copy->deadline()->toISOString()); + } + + public function testRemainingUsesRecordedWorkflowTimeDespiteAChangedHostClock(): void + { + $context = CancellationContext::fromArray($this->snapshot()); + CarbonImmutable::setTestNow('2040-01-01T00:00:00Z'); + try { + $fiber = new Fiber(function () use ($context): void { + WorkflowFiberContext::enter(); + try { + $current = Fiber::getCurrent(); + $this->assertInstanceOf(Fiber::class, $current); + $context->bindToReplayFiber($current); + WorkflowFiberContext::startCancellationTime( + CarbonImmutable::parse('2026-10-01T00:00:03.500000Z'), + Fiber::getCurrent(), + ); + WorkflowFiberContext::setTime(CarbonImmutable::parse('2026-10-01T00:00:03.500000Z')); + $this->assertSame(26.5, $context->remaining()); + WorkflowFiberContext::setTime(CarbonImmutable::parse('2026-10-01T00:00:31Z')); + $this->assertSame(0.0, $context->remaining()); + } finally { + WorkflowFiberContext::leave(); + } + }); + $fiber->start(); + } finally { + CarbonImmutable::setTestNow(); + } + } + + public function testRemainingOutsideAWorkflowRefusesWallClockArithmetic(): void + { + $this->expectException(LogicException::class); + CancellationContext::fromArray($this->snapshot())->remaining(); + } + + public function testConsumedBudgetPreservesMicrosecondsAcrossSynchronousPersistenceAndClockSkew(): void + { + $snapshot = $this->snapshot(); + $snapshot['cleanup_deadline_at'] = '2026-10-01T00:00:30.123456Z'; + $context = CancellationContext::fromArray($snapshot); + $execution = WorkflowExecution::startCallback(static function () use ($context): array { + try { + Fiber::suspend('initial'); + } catch (WorkflowCancellationRequestedException) { + // The real delivery arms the consumed-budget clock. + } + $initial = $context->remaining(); + Fiber::suspend('memo'); + $afterMemo = $context->remaining(); + Fiber::suspend('activity'); + $afterActivity = $context->remaining(); + Fiber::suspend('timer'); + + return [ + $initial, + $afterMemo, + $afterActivity, + $context->remaining(), + WorkflowFiberContext::getRecordedTime()->format('H:i:s.u'), + ]; + }, eventTime: CarbonImmutable::parse('2026-10-01T00:00:08Z')); + + $execution->throw( + new WorkflowCancellationRequestedException(cancellation: $context), + CarbonImmutable::parse('2026-10-01T00:00:08Z'), + ); + $execution->send(null, CarbonImmutable::parse('2026-10-01T00:00:28Z'), advanceCancellationTime: false); + $execution->send(null, CarbonImmutable::parse('2026-10-01T00:00:25Z')); + $execution->send(null, CarbonImmutable::parse('2026-10-01T00:00:20Z')); + + $this->assertSame([22.123456, 22.123456, 5.123456, 5.123456, '00:00:20.000000'], $execution->getReturn()); + } + + public function testMissingBlockingResumeTimestampCannotReuseAnEarlierBudget(): void + { + $context = CancellationContext::fromArray($this->snapshot()); + $execution = WorkflowExecution::startCallback(static function () use ($context): float { + try { + Fiber::suspend('initial'); + } catch (WorkflowCancellationRequestedException) { + // Continue into the durable cleanup wait. + } + Fiber::suspend('activity'); + + return $context->remaining(); + }, eventTime: CarbonImmutable::parse('2026-10-01T00:00:08Z')); + + $execution->throw( + new WorkflowCancellationRequestedException(cancellation: $context), + CarbonImmutable::parse('2026-10-01T00:00:08Z'), + ); + $this->expectException(LogicException::class); + $this->expectExceptionMessage('recorded blocking boundary timestamp'); + $execution->send(null); + } + + public function testRemainingInAnUnseededFiberRefusesWallClockArithmetic(): void + { + $context = CancellationContext::fromArray($this->snapshot()); + $fiber = new Fiber(static function () use ($context): void { + WorkflowFiberContext::enter(); + try { + $context->remaining(); + } finally { + WorkflowFiberContext::leave(); + } + }); + $this->expectException(LogicException::class); + $fiber->start(); + } + + public function testDetachedMetadataCannotBorrowAnActiveDeliveryClock(): void + { + $context = CancellationContext::fromArray($this->snapshot()); + $execution = WorkflowExecution::startCallback(static function (): void { + try { + Fiber::suspend('delivery'); + } catch (WorkflowCancellationRequestedException $cancelled) { + $delivered = $cancelled->cancellation; + self::assertNotNull($delivered); + self::assertSame(27.0, $delivered->remaining()); + $detached = CancellationContext::fromArray($delivered->toArray()); + + $detached->remaining(); + } + }); + $this->expectException(LogicException::class); + $this->expectExceptionMessage('active workflow replay'); + $execution->throw( + new WorkflowCancellationRequestedException(cancellation: $context), + CarbonImmutable::parse('2026-10-01T00:00:03Z'), + ); + } + + public function testDeliveredContextCannotBorrowAnotherFiberAndRetainsMetadataEquality(): void + { + $context = CancellationContext::fromArray($this->snapshot()); + $execution = WorkflowExecution::startCallback(static function (): void { + try { + Fiber::suspend('delivery'); + } catch (WorkflowCancellationRequestedException) { + Fiber::suspend('cleanup'); + } + }); + $execution->throw( + new WorkflowCancellationRequestedException(cancellation: $context), + CarbonImmutable::parse('2026-10-01T00:00:03Z') + ); + $this->assertEquals(CancellationContext::fromArray($context->toArray()), $context); + $other = WorkflowExecution::startCallback(static function () use ($context): void { + try { + Fiber::suspend('delivery'); + } catch (WorkflowCancellationRequestedException $cancelled) { + self::assertNotNull($cancelled->cancellation); + self::assertSame(26.0, $cancelled->cancellation->remaining()); + + $context->remaining(); + } + }); + try { + $this->expectException(LogicException::class); + $this->expectExceptionMessage('active workflow replay'); + $other->throw( + new WorkflowCancellationRequestedException(cancellation: CancellationContext::fromArray( + $this->snapshot() + )), + CarbonImmutable::parse('2026-10-01T00:00:04Z') + ); + } finally { + $execution->send(null); + } + } + + public function testDescendantKeepsTheRootBudgetAndImmediateParentIdentity(): void + { + $snapshot = $this->snapshot(); + $snapshot['request_id'] = 'child-1'; + $snapshot['parent_request_id'] = 'root-1'; + $snapshot['lineage'][] = [ + 'request_id' => 'child-1', + 'workflow_instance_id' => 'child-instance', + 'workflow_run_id' => 'child-run', + ]; + $context = CancellationContext::fromArray($snapshot); + $this->assertSame('root-1', $context->rootRequestId); + $this->assertSame('root-1', $context->parentRequestId); + $this->assertSame($snapshot, $context->toArray()); + } + + public function testRequesterCannotCarryUnrelatedRequestOrAuthenticationMetadata(): void + { + $snapshot = $this->snapshot(); + $snapshot['requester']['headers'] = 'unexpected'; + $this->expectException(InvalidArgumentException::class); + CancellationContext::fromArray($snapshot); + } + + public function testMismatchedLineageIsRefused(): void + { + $snapshot = $this->snapshot(); + $snapshot['root_request_id'] = 'different-root'; + $this->expectException(InvalidArgumentException::class); + CancellationContext::fromArray($snapshot); + } + + public function testCyclicLineageIsRefused(): void + { + $snapshot = $this->snapshot(); + $snapshot['parent_request_id'] = 'root-1'; + $snapshot['lineage'][] = $snapshot['lineage'][0]; + $this->expectException(InvalidArgumentException::class); + CancellationContext::fromArray($snapshot); + } + + public function testMalformedCalendarTimestampIsRefused(): void + { + $snapshot = $this->snapshot(); + $snapshot['requested_at'] = '2026-02-30T00:00:00Z'; + $this->expectException(InvalidArgumentException::class); + CancellationContext::fromArray($snapshot); + } + + /** + * @return array + */ + private function snapshot(): array + { + return [ + 'schema' => 'durable-workflow.cancellation-context/v1', + 'request_id' => 'root-1', + 'root_request_id' => 'root-1', + 'root_workflow_instance_id' => 'root-instance', + 'root_workflow_run_id' => 'root-run', + 'parent_request_id' => null, + 'reason' => 'maintenance', + 'requester' => [ + 'type' => 'operator', + 'id' => 'operator-1', + ], + 'source' => 'control_plane', + 'requested_at' => '2026-10-01T00:00:00.000000Z', + 'cleanup_deadline_at' => '2026-10-01T00:00:30.000000Z', + 'lineage' => [[ + 'request_id' => 'root-1', + 'workflow_instance_id' => 'root-instance', + 'workflow_run_id' => 'root-run', + ]], + ]; + } +} diff --git a/tests/Unit/V2/CancellationHistoryTimelineTest.php b/tests/Unit/V2/CancellationHistoryTimelineTest.php new file mode 100644 index 000000000..5d5eb2e8f --- /dev/null +++ b/tests/Unit/V2/CancellationHistoryTimelineTest.php @@ -0,0 +1,255 @@ + 'durable-workflow.cancellation-scope/v1', + 'scope_id' => 'scope-child', + 'parent_scope_id' => 'scope-parent', + 'shield_parent' => true, + ]; + $entry = $this->entry(HistoryEventType::CancellationScopeOpened, $scope); + $this->assertSame($scope, $entry['cancellation_scope']); + $this->assertSame('cancellation_scope', $entry['kind']); + $this->assertSame( + 'Cancellation scope scope-child opened under scope-parent with parent shielding.', + $entry['summary'] + ); + $this->assertArrayNotHasKey('cancellation_request', $entry); + } + + #[DataProvider('cleanupOutcomes')] + public function testCleanupOutcomeRetainsTheOriginalBudgetWhenInspectedLater(string $outcome, string $summary): void + { + $cleanup = [ + 'request_id' => 'request-1', + 'outcome' => $outcome, + 'cleanup_deadline_at' => '2026-10-02T00:00:30.000000Z', + 'delivery_history_event_id' => in_array($outcome, ['not_delivered', 'unavailable'], true) + ? null : 'delivery-1', + 'delivery_sequence' => in_array($outcome, ['not_delivered', 'unavailable'], true) ? null : 4, + 'finished_at' => $outcome === 'deadline_expired' + ? '2026-10-02T00:00:30.000000Z' : '2026-10-02T00:00:20.000000Z', + ]; + Carbon::setTestNow('2026-12-01T00:00:00Z'); + try { + $entry = $this->entry(HistoryEventType::WorkflowCancelled, [ + 'cancellation_cleanup' => $cleanup, + ]); + + $this->assertSame($cleanup, $entry['cancellation_cleanup']); + $this->assertSame($summary, $entry['summary']); + } finally { + Carbon::setTestNow(); + } + } + + /** + * @return iterable + */ + public static function cleanupOutcomes(): iterable + { + yield 'completed' => ['completed', 'Workflow cancelled after cleanup completed.']; + yield 'expired' => ['deadline_expired', 'Workflow cancelled at the cleanup deadline.']; + yield 'not delivered' => [ + 'not_delivered', + 'Workflow cancelled without cancellation delivery to workflow code.', + ]; + yield 'unavailable' => ['unavailable', 'Workflow cancelled. Cleanup delivery evidence is unavailable.']; + } + + #[DataProvider('historyEventTypes')] + public function testEverySupportedHistoryEventCanBeInspectedWithSparseMetadata(HistoryEventType $type): void + { + $entry = $this->entry($type, []); + + $this->assertSame($type->value, $entry['type']); + $this->assertNotSame('', $entry['summary']); + } + + /** + * @return iterable + */ + public static function historyEventTypes(): iterable + { + foreach (HistoryEventType::cases() as $type) { + yield $type->value => [$type]; + } + } + + #[DataProvider('receiptTiming')] + public function testStopReceiptExplainsTheOriginalAttemptAndBudgetWithoutGrantingAuthority(bool $late): void + { + $payload = [ + 'sequence' => 4, + 'activity_execution_id' => 'activity-1', + 'activity_attempt_id' => 'original-attempt', + 'cancellation_history_event_id' => 'cancel-event', + 'request_id' => 'child-request', + 'root_request_id' => 'root-request', + 'cleanup_deadline_at' => '2026-10-02T00:00:30.000000Z', + 'callback_state' => 'stopped', + 'evidence_source' => 'activity_worker', + 'acknowledged_at' => $late ? '2026-10-02T00:00:31.000000Z' : '2026-10-02T00:00:10.000000Z', + 'received_after_deadline' => $late, + ]; + $entry = $this->entry(HistoryEventType::ActivityCancellationAcknowledged, $payload); + + $this->assertSame('activity', $entry['kind']); + $this->assertSame('activity_execution', $entry['source_kind']); + $this->assertSame('activity-1', $entry['source_id']); + $this->assertSame('original-attempt', $entry['activity']['attempt_id']); + $this->assertSame('cancelled', $entry['activity_status']); + $this->assertSame('activity', $entry['task']['type']); + $this->assertSame('cancelled', $entry['task']['status']); + $this->assertSame(array_diff_key($payload, [ + 'sequence' => true, + 'activity_execution_id' => true, + ]), $entry['cancellation_acknowledgement']); + $this->assertSame( + 'Worker reported stopped callback for activity' . ($late ? ' after the cleanup deadline' : '') . '.', + $entry['summary'], + ); + } + + /** + * @return iterable + */ + public static function receiptTiming(): iterable + { + yield 'before original deadline' => [false]; + yield 'after original deadline' => [true]; + } + + public function testLocalStopReceiptDoesNotDescribeItsWorkflowClaimAsACancelledActivityTask(): void + { + $entry = $this->entry(HistoryEventType::ActivityCancellationAcknowledged, [ + 'activity_execution_id' => 'local-activity-1', + 'activity_attempt_id' => 'original-local-attempt', + 'local_activity' => true, + 'execution_mode' => 'local', + 'evidence_source' => 'workflow_worker', + 'task' => [ + 'id' => 'task-1', + 'type' => 'workflow', + 'status' => 'leased', + 'attempt_count' => 1, + ], + ]); + + $this->assertSame('cancelled', $entry['activity_status']); + $this->assertSame('workflow', $entry['task']['type']); + $this->assertSame('leased', $entry['task']['status']); + $this->assertSame(1, $entry['task']['attempt_count']); + } + + #[DataProvider('localActivityEvents')] + public function testLocalActivityHistoryCannotInventAClosedOrdinaryActivityTask(HistoryEventType $type): void + { + $entry = $this->entry($type, [ + 'activity_execution_id' => 'local-activity-1', + 'execution_mode' => 'local', + 'task' => [ + 'id' => 'task-1', + 'type' => 'workflow', + 'status' => 'leased', + 'attempt_count' => 1, + ], + ]); + + $this->assertSame('workflow', $entry['task']['type']); + $this->assertSame('leased', $entry['task']['status']); + } + + /** + * @return iterable + */ + public static function localActivityEvents(): iterable + { + foreach ([HistoryEventType::ActivityScheduled, HistoryEventType::ActivityStarted, + HistoryEventType::ActivityHeartbeatRecorded, HistoryEventType::ActivityRetryScheduled, + HistoryEventType::ActivityCompleted, HistoryEventType::ActivityFailed, + HistoryEventType::ActivityCancelled, HistoryEventType::ActivityTimedOut, + HistoryEventType::ActivityCancellationAcknowledged] as $type) { + yield $type->value => [$type]; + } + } + + #[DataProvider('localActivityEvents')] + public function testARepairedWorkflowClaimCannotChangeTheLocalActivityAttemptCount(HistoryEventType $type): void + { + $expected = $type === HistoryEventType::ActivityScheduled ? 0 : 2; + $entry = $this->entry($type, [ + 'activity_execution_id' => 'local-activity-1', + 'execution_mode' => 'local', + 'activity' => [ + 'id' => 'local-activity-1', + 'attempt_count' => $expected, + ], + 'task' => [ + 'id' => 'task-1', + 'type' => 'workflow', + 'status' => 'leased', + 'attempt_count' => 5, + ], + ]); + + $this->assertSame($expected, $entry['activity']['attempt_count']); + $this->assertSame(5, $entry['task']['attempt_count']); + } + + public function testSparseLocalHistoryUsesTheActivityAttemptNumberInsteadOfTheWorkflowClaimCount(): void + { + $entry = $this->entry(HistoryEventType::ActivityStarted, [ + 'activity_execution_id' => 'local-activity-1', + 'local_activity' => true, + 'attempt_number' => 3, + 'task' => [ + 'id' => 'task-1', + 'type' => 'workflow', + 'status' => 'leased', + 'attempt_count' => 7, + ], + ]); + + $this->assertSame(3, $entry['activity']['attempt_count']); + $this->assertSame(7, $entry['task']['attempt_count']); + } + + /** @param array $payload + * @return array + */ + private function entry(HistoryEventType $type, array $payload): array + { + $run = new WorkflowRun(); + foreach (['commands', 'tasks', 'activityExecutions', 'timers', 'failures'] as $relation) { + $run->setRelation($relation, new Collection()); + } + $event = new WorkflowHistoryEvent(); + $event->forceFill([ + 'id' => 'event-1', + 'sequence' => 1, + 'workflow_task_id' => 'task-1', + 'event_type' => $type, + 'payload' => $payload, + ]); + $run->setRelation('historyEvents', new Collection([$event])); + + return HistoryTimeline::fromHistory($run)[0]; + } +} diff --git a/tests/Unit/V2/CancellationPropagationContextTest.php b/tests/Unit/V2/CancellationPropagationContextTest.php new file mode 100644 index 000000000..5163d3834 --- /dev/null +++ b/tests/Unit/V2/CancellationPropagationContextTest.php @@ -0,0 +1,269 @@ + 'database', + ]); + Queue::fake(); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + parent::tearDown(); + } + + public function testChildAndGrandchildInheritTheOriginalRootAndDeadlineWithoutClosingImmediately(): void + { + Carbon::setTestNow('2026-10-01T00:00:00Z'); + $parent = $this->workflow(); + $child = $this->workflow(); + $grandchild = $this->workflow(); + $this->link($parent, $child); + $this->link($child, $grandchild); + $root = $parent->requestCancellation('maintenance', 30) + ->cancellationContext(); + + Carbon::setTestNow('2026-10-01T00:00:05Z'); + $childRequest = $child->attemptRequestCancellationFromParent($parent->runId()); + $this->assertTrue($childRequest->accepted()); + $childContext = $childRequest->cancellationContext(); + $this->assertNotSame($root->requestId, $childContext->requestId); + $this->assertSame($root->requestId, $childContext->rootRequestId); + $this->assertSame($root->requestId, $childContext->parentRequestId); + $this->assertSame($root->requestedAt()->toISOString(), $childContext->requestedAt()->toISOString()); + $this->assertSame($root->deadline()->toISOString(), $childContext->deadline()->toISOString()); + $this->assertSame($root->requester, $childContext->requester); + $this->assertSame($root->source, $childContext->source); + $this->assertSame('maintenance', $childContext->reason); + $this->assertFalse($child->run()->fresh()->status->isTerminal()); + $this->assertSame( + $childContext->toArray(), + CooperativeCancellationDelivery::context($child->run()->fresh())->toArray() + ); + + Carbon::setTestNow('2026-10-01T00:00:10Z'); + $grandchildRequest = $grandchild->attemptRequestCancellationFromParent($child->runId()); + $grandchildContext = $grandchildRequest->cancellationContext(); + $this->assertSame($root->requestId, $grandchildContext->rootRequestId); + $this->assertSame($childContext->requestId, $grandchildContext->parentRequestId); + $this->assertSame($root->deadline()->toISOString(), $grandchildContext->deadline()->toISOString()); + $this->assertCount(3, $grandchildContext->lineage); + $this->assertFalse($grandchild->run()->fresh()->status->isTerminal()); + + $duplicate = $child->attemptRequestCancellationFromParent($parent->runId()); + $this->assertSame($childRequest->commandId(), $duplicate->commandId()); + $this->assertSame($childContext->toArray(), $duplicate->cancellationContext()->toArray()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('workflow_run_id', $child->runId()) + ->where('event_type', HistoryEventType::CooperativeCancellationRequested->value)->count()); + } + + public function testLatePropagationDoesNotGrantAFreshBudgetAndRepairClosesIt(): void + { + Carbon::setTestNow('2026-10-01T00:00:00Z'); + $parent = $this->workflow(); + $child = $this->workflow(); + $this->link($parent, $child); + $root = $parent->requestCancellation('maintenance', 30) + ->cancellationContext(); + Carbon::setTestNow('2026-10-01T00:00:35Z'); + $propagated = $child->attemptRequestCancellationFromParent($parent->runId()); + $this->assertTrue($propagated->accepted()); + $this->assertSame( + $root->deadline() + ->toISOString(), + $child->run() + ->fresh() + ->cancellation_deadline_at->toISOString() + ); + $report = TaskWatchdog::runPass(); + $this->assertSame(2, $report['cancellation_deadlines_enforced']); + $this->assertSame(RunStatus::Cancelled, $child->run()->fresh()->status); + $duplicate = $child->attemptRequestCancellationFromParent($parent->runId()); + $this->assertSame($propagated->commandId(), $duplicate->commandId()); + $this->assertSame( + $root->deadline() + ->toISOString(), + $duplicate->cancellationContext() + ->deadline() + ->toISOString() + ); + } + + public function testAnIndependentChildRequestWinsAndACompetingRootIsExplicitlyRejected(): void + { + $parent = $this->workflow(); + $child = $this->workflow(); + $this->link($parent, $child); + $independent = $child->requestCancellation('child operator request', 60); + $root = $parent->requestCancellation('ancestor request', 30) + ->cancellationContext(); + $conflict = $child->attemptRequestCancellationFromParent($parent->runId()); + $this->assertTrue($conflict->rejected()); + $this->assertSame('cancellation_root_conflict', $conflict->rejectionReason()); + $this->assertSame([ + 'existing_request_id' => $independent->commandId(), + 'existing_root_request_id' => $independent->commandId(), + 'existing_cleanup_deadline_at' => $independent->cancellationContext() + ->deadline() + ->toISOString(), + 'incoming_root_request_id' => $root->requestId, + 'incoming_cleanup_deadline_at' => $root->deadline() + ->toISOString(), + ], $conflict->payloadValues([ + 'existing_request_id', 'existing_root_request_id', 'existing_cleanup_deadline_at', + 'incoming_root_request_id', 'incoming_cleanup_deadline_at', + ])); + $this->assertSame($independent->commandId(), $child->run()->fresh()->cancellation_request_command_id); + $this->assertSame( + $independent->cancellationContext() + ->toArray(), + $child->requestCancellation('duplicate', 3600) + ->cancellationContext() + ->toArray() + ); + } + + public function testAnInheritedRequestWinsOverALaterIndependentDuplicate(): void + { + $parent = $this->workflow(); + $child = $this->workflow(); + $this->link($parent, $child); + $parent->requestCancellation('ancestor request', 30); + $inherited = $child->attemptRequestCancellationFromParent($parent->runId()); + $duplicate = $child->requestCancellation('independent later request', 3600); + $this->assertSame($inherited->commandId(), $duplicate->commandId()); + $this->assertSame($inherited->cancellationContext()->toArray(), $duplicate->cancellationContext()->toArray()); + } + + public function testAnotherParentWithTheSameRootKeepsTheFirstRecordedDeliveryRoute(): void + { + $root = $this->workflow(); + $firstParent = $this->workflow(); + $otherParent = $this->workflow(); + $child = $this->workflow(); + $this->link($root, $firstParent); + $this->link($root, $otherParent); + $this->link($firstParent, $child); + $this->link($otherParent, $child); + $root->requestCancellation('cascade', 30); + $firstParent->attemptRequestCancellationFromParent($root->runId()); + $otherParent->attemptRequestCancellationFromParent($root->runId()); + $first = $child->attemptRequestCancellationFromParent($firstParent->runId()); + $second = $child->attemptRequestCancellationFromParent($otherParent->runId()); + $this->assertSame($first->commandId(), $second->commandId()); + $this->assertSame($first->cancellationContext()->toArray(), $second->cancellationContext()->toArray()); + } + + public function testUnlinkedAndUnrequestedParentsCannotSupplyCancellationAuthority(): void + { + $parent = $this->workflow(); + $child = $this->workflow(); + $parent->requestCancellation('request', 30); + $unlinked = $child->attemptRequestCancellationFromParent($parent->runId()); + $this->assertSame('cancellation_parent_not_linked', $unlinked->rejectionReason()); + $unrequested = $this->workflow(); + $this->link($unrequested, $child); + $missing = $child->attemptRequestCancellationFromParent($unrequested->runId()); + $this->assertSame('cancellation_parent_context_unavailable', $missing->rejectionReason()); + $this->assertNull($child->run()->fresh()->cancellation_request_command_id); + } + + public function testLegacyParentRequestsRequireAnExplicitContextCapability(): void + { + $parent = $this->workflow(); + $child = $this->workflow(); + $this->link($parent, $child); + $request = $parent->requestCancellation('legacy', 30); + $command = WorkflowCommand::query()->findOrFail($request->commandId()); + $command->forceFill([ + 'payload' => \Workflow\Serializers\Serializer::serializeWithCodec($command->payload_codec, [ + 'reason' => 'legacy', + 'cleanup_deadline_at' => $parent->run() + ->fresh() + ->cancellation_deadline_at->toISOString(), + ]), + ])->save(); + $result = $child->attemptRequestCancellationFromParent($parent->runId()); + $this->assertSame('cancellation_parent_context_unavailable', $result->rejectionReason()); + $this->assertNull($child->run()->fresh()->cancellation_request_command_id); + } + + public function testAStoredChildLinkCannotCreateAnAncestorCycle(): void + { + $parent = $this->workflow(); + $child = $this->workflow(); + $this->link($parent, $child); + $this->link($child, $parent); + $root = $parent->requestCancellation('cascade', 30); + $child->attemptRequestCancellationFromParent($parent->runId()); + $cycle = $parent->attemptRequestCancellationFromParent($child->runId()); + $this->assertSame('cancellation_lineage_cycle', $cycle->rejectionReason()); + $this->assertSame($root->commandId(), $parent->run()->fresh()->cancellation_request_command_id); + } + + public function testParentContextMustMatchTheAcceptedCommandAndItsStoredParentRun(): void + { + $parent = $this->workflow(); + $child = $this->workflow(); + $unrelated = $this->workflow(); + $this->link($parent, $child); + $original = $parent->requestCancellation('parent', 30); + $other = $unrelated->requestCancellation('other', 60); + $command = WorkflowCommand::query()->findOrFail($original->commandId()); + $command->forceFill([ + 'payload' => \Workflow\Serializers\Serializer::serializeWithCodec($command->payload_codec, [ + 'reason' => 'other', + 'cleanup_deadline_at' => $other->cancellationContext() + ->deadline() + ->toISOString(), + 'cancellation' => $other->cancellationContext() + ->toArray(), + ]), + ])->save(); + $result = $child->attemptRequestCancellationFromParent($parent->runId()); + $this->assertSame('cancellation_parent_context_mismatch', $result->rejectionReason()); + $this->assertNull($child->run()->fresh()->cancellation_request_command_id); + } + + private function workflow(): WorkflowStub + { + $workflow = WorkflowStub::make(TestCancellationContextWorkflow::class); + $workflow->start(); + + return $workflow; + } + + private function link(WorkflowStub $parent, WorkflowStub $child): void + { + WorkflowLink::query()->create([ + 'link_type' => 'child_workflow', + 'parent_workflow_instance_id' => $parent->id(), + 'parent_workflow_run_id' => $parent->runId(), + 'child_workflow_instance_id' => $child->id(), + 'child_workflow_run_id' => $child->runId(), + 'is_primary_parent' => true, + ]); + } +} diff --git a/tests/Unit/V2/CancellationRequestContextTest.php b/tests/Unit/V2/CancellationRequestContextTest.php new file mode 100644 index 000000000..b0fe5d4e3 --- /dev/null +++ b/tests/Unit/V2/CancellationRequestContextTest.php @@ -0,0 +1,74 @@ + 'database', + ]); + Queue::fake(); + $workflow = WorkflowStub::make(TestCancellationContextWorkflow::class) + ->withCommandContext(CommandContext::controlPlane()->withPrincipal('operator', 'operator-1', 'Maintainer') + ->with([ + 'principal' => [ + 'unrelated' => 'omit', + ], + 'request' => [ + 'unrelated' => 'omit', + ], + ])); + $workflow->start(); + $first = $workflow->requestCancellation('maintenance', 30); + $context = $first->cancellationContext(); + $this->assertNotNull($context); + $this->assertSame($first->commandId(), $context->requestId); + $this->assertSame($context->requestId, $context->rootRequestId); + $this->assertSame($workflow->id(), $context->rootWorkflowInstanceId); + $this->assertSame($workflow->runId(), $context->rootWorkflowRunId); + $this->assertNull($context->parentRequestId); + $this->assertSame('maintenance', $context->reason); + $this->assertSame('control_plane', $context->source); + $this->assertSame([ + 'type' => 'operator', + 'id' => 'operator-1', + 'label' => 'Maintainer', + ], $context->requester); + $this->assertSame(30.0, (float) $context->requestedAt()->diffInSeconds($context->deadline())); + $this->assertCount(1, $context->lineage); + + $run = $workflow->run() + ->fresh('historyEvents'); + $requested = $run->historyEvents->firstWhere('event_type', HistoryEventType::CooperativeCancellationRequested); + $this->assertSame($context->toArray(), $requested->payload['cancellation']); + $this->assertSame($context->toArray(), CooperativeCancellationDelivery::context($run)->toArray()); + + Carbon::setTestNow($context->deadline()->addMinute()); + try { + $duplicate = $workflow->withCommandContext( + CommandContext::phpApi()->withPrincipal('operator', 'operator-2') + ) + ->requestCancellation('different reason', 3600); + $this->assertSame($first->commandId(), $duplicate->commandId()); + $this->assertSame($context->toArray(), $duplicate->cancellationContext()->toArray()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('workflow_run_id', $run->id) + ->where('event_type', HistoryEventType::CooperativeCancellationRequested->value)->count()); + } finally { + Carbon::setTestNow(); + } + } +} diff --git a/tests/Unit/V2/ChildCallServiceTest.php b/tests/Unit/V2/ChildCallServiceTest.php index e4151d2a7..7c995959f 100644 --- a/tests/Unit/V2/ChildCallServiceTest.php +++ b/tests/Unit/V2/ChildCallServiceTest.php @@ -5,6 +5,7 @@ namespace Tests\Unit\V2; use Tests\TestCase; +use Workflow\V2\Enums\CancellationPolicy; use Workflow\V2\Enums\ChildCallStatus; use Workflow\V2\Enums\ParentClosePolicy; use Workflow\V2\Models\WorkflowChildCall; @@ -78,6 +79,20 @@ public function testItSchedulesChildWithRoutingOverrides(): void $this->assertEquals('high-priority', $childCall->queue); } + public function testCancellationPolicyIsRecordedSeparatelyFromParentClosePolicy(): void + { + foreach (CancellationPolicy::cases() as $policy) { + $childCall = $this->service->scheduleChild( + $this->createRun(), + new ChildWorkflowCall('TestChildWorkflow', [], new ChildWorkflowOptions(cancellationPolicy: $policy)), + 10, + ); + $this->assertSame(ParentClosePolicy::Abandon, $childCall->parent_close_policy); + $this->assertSame($policy !== CancellationPolicy::Abandon, $childCall->cancellation_propagation); + $this->assertSame($policy->value, $childCall->metadata['cancellation_policy']); + } + } + public function testItInheritsRoutingFromParentWhenNotOverridden(): void { $parentRun = $this->createRun(); diff --git a/tests/Unit/V2/CooperativeCancellationDeliveryTest.php b/tests/Unit/V2/CooperativeCancellationDeliveryTest.php index 8b29e45d3..396e80a8b 100644 --- a/tests/Unit/V2/CooperativeCancellationDeliveryTest.php +++ b/tests/Unit/V2/CooperativeCancellationDeliveryTest.php @@ -40,6 +40,150 @@ public function testPersistedRequestRequiresItsMatchingCanonicalHistory(): void $this->assertSame('cancellation_request_history_missing', $this->validate($run)); } + #[DataProvider('operationScopeMembership')] + public function testRootDeliveryRequiresCanonicalRootOperationMembership(array $payload, ?string $expected): void + { + $run = $this->runWithHistory([ + $this->event(HistoryEventType::ActivityScheduled, [ + 'sequence' => 1, + ...$payload, + ]), + $this->request(), + ]); + $this->assertSame($expected, $this->validate($run)); + } + + public static function operationScopeMembership(): iterable + { + yield 'historical omission' => [[], null]; + yield 'explicit root' => [[ + 'cancellation_scope_id' => 'root', + ], null]; + yield 'nested root' => [[ + 'activity' => [ + 'cancellation_scope_id' => 'root', + ], + ], null]; + yield 'both root' => [[ + 'cancellation_scope_id' => 'root', + 'activity' => [ + 'cancellation_scope_id' => 'root', + ], + ], null]; + yield 'application value' => [[ + 'arguments' => [ + 'cancellation_scope_id' => 'app-value', + ], + ], null]; + yield 'flat scope' => [[ + 'cancellation_scope_id' => 'scope-1', + ], 'cancellation_delivery_requires_scope']; + yield 'nested scope' => [[ + 'activity' => [ + 'cancellation_scope_id' => 'scope-1', + ], + ], 'cancellation_delivery_requires_scope']; + yield 'conflicting snapshots' => [[ + 'cancellation_scope_id' => 'root', + 'activity' => [ + 'cancellation_scope_id' => 'scope-1', + ], + ], 'cancellation_delivery_scope_membership_invalid']; + yield 'null descriptor' => [[ + 'activity' => null, + ], 'cancellation_delivery_scope_membership_invalid']; + yield 'scalar descriptor' => [[ + 'activity' => 'root', + ], 'cancellation_delivery_scope_membership_invalid']; + foreach ([null, false, 1, '', ['root']] as $index => $invalid) { + yield 'invalid flat ' . $index => [[ + 'cancellation_scope_id' => $invalid, + ], 'cancellation_delivery_scope_membership_invalid']; + yield 'invalid nested ' . $index => [[ + 'activity' => [ + 'cancellation_scope_id' => $invalid, + ], + ], 'cancellation_delivery_scope_membership_invalid']; + } + } + + public function testScopeMembershipIsCheckedAgainWhenReplayingAnExistingRootReceipt(): void + { + $run = $this->runWithHistory([ + $this->event(HistoryEventType::ActivityScheduled, [ + 'sequence' => 1, + ]), + $this->request(), + $this->event(HistoryEventType::CooperativeCancellationDelivered, [ + 'workflow_command_id' => 'request-1', + 'sequence' => 1, + 'call_kind' => 'activity', + ]), + ]); + $run->cancellation_delivery_sequence = 1; + $this->assertNull($this->validate($run)); + $run->historyEvents[0]->payload = [ + 'sequence' => 1, + 'activity' => [ + 'cancellation_scope_id' => 'scope-1', + ], + ]; + $this->assertSame('cancellation_delivery_requires_scope', $this->validate($run)); + } + + #[DataProvider('scopedCallKinds')] + public function testEveryRecordedScopedWaitRequiresScopeDelivery( + HistoryEventType $type, + string $kind, + array $membership + ): void { + $run = $this->runWithHistory([ + $this->event($type, [ + 'sequence' => 1, + ...$membership, + ]), + $this->request(), + ]); + $this->assertSame('cancellation_delivery_requires_scope', $this->validate($run, 1, $kind)); + } + + public static function scopedCallKinds(): iterable + { + yield 'remote activity' => [ + HistoryEventType::ActivityScheduled, 'activity', [ + 'activity' => [ + 'cancellation_scope_id' => 'scope-1', + ], + ]]; + yield 'local activity' => [ + HistoryEventType::ActivityScheduled, 'local_activity', [ + 'local_activity' => true, + 'activity' => [ + 'cancellation_scope_id' => 'scope-1', + ], + ]]; + yield 'timer' => [ + HistoryEventType::TimerScheduled, 'timer', [ + 'timer' => [ + 'cancellation_scope_id' => 'scope-1', + ], + ]]; + yield 'child' => [ + HistoryEventType::ChildWorkflowScheduled, 'child', [ + 'child_workflow' => [ + 'cancellation_scope_id' => 'scope-1', + ], + ]]; + yield 'condition' => [ + HistoryEventType::ConditionWaitOpened, 'condition', [ + 'cancellation_scope_id' => 'scope-1', + ]]; + yield 'signal' => [ + HistoryEventType::SignalWaitOpened, 'signal', [ + 'cancellation_scope_id' => 'scope-1', + ]]; + } + public function testASequenceHoleCannotBeUsedToMoveDeliveryEarlier(): void { $run = $this->runWithHistory([ diff --git a/tests/Unit/V2/PlatformConformanceSuiteTest.php b/tests/Unit/V2/PlatformConformanceSuiteTest.php index 7ea57d71b..96d4e46a4 100644 --- a/tests/Unit/V2/PlatformConformanceSuiteTest.php +++ b/tests/Unit/V2/PlatformConformanceSuiteTest.php @@ -1031,7 +1031,10 @@ public function testWorkflowSourceReleaseAndQualifiedSdkTupleRemainExplicit(): v $contracts = $suiteManifest['fixture_catalog']['signal_query_runtime_contract']['required_scenario_contracts']; $this->assertIsString($workflowSourceRelease); - $this->assertMatchesRegularExpression('/^2\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)$/D', $workflowSourceRelease); + $this->assertMatchesRegularExpression( + '/^2\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-(?:alpha|beta|rc)\.(?:0|[1-9]\d*))?$/D', + $workflowSourceRelease + ); $this->assertSame($workflowSourceRelease, PlatformConformanceSuite::workflowSourceRelease()); foreach ($sdkCompatibility as $sdk) { diff --git a/tests/Unit/V2/PlatformProtocolSpecsTest.php b/tests/Unit/V2/PlatformProtocolSpecsTest.php index 65f89fffc..613031211 100644 --- a/tests/Unit/V2/PlatformProtocolSpecsTest.php +++ b/tests/Unit/V2/PlatformProtocolSpecsTest.php @@ -436,7 +436,7 @@ public function testCurrentWorkerProtocolAddsDurableSelectionWithoutChangingHist ); } - public function testRuntimeAndCurrentWorkerSpecsCannotDriftOnMessageStreamContract(): void + public function testRuntimePreservesTheFrozenProtocol119MessageStreamContract(): void { $root = dirname(__DIR__, 3) . '/resources/conformance/suite-v47/platform-protocol-specs'; $openApi = Yaml::parseFile($root . '/worker-protocol-api.openapi.yaml'); @@ -451,7 +451,7 @@ public function testRuntimeAndCurrentWorkerSpecsCannotDriftOnMessageStreamContra foreach ([$openApi, $asyncApi] as $spec) { $this->assertSame( - WorkerProtocolVersion::VERSION, + '1.19', $spec['x-durable-workflow-worker-protocol-negotiation']['default_advertised_version'], ); $this->assertSame( @@ -555,13 +555,12 @@ public function testRuntimeAndCurrentWorkerSpecsCannotDriftOnMessageStreamContra } $this->assertSame(WorkerProtocolVersion::VERSION, $negotiation['default_advertised_version']); - $this->assertSame($expectedVersions, $negotiation['accepted_request_versions_by_default']); - $this->assertSame( - WorkerProtocolVersion::VERSION, - $openApi['components']['schemas']['AdvertisedWorkerProtocolVersion']['const'], - ); + foreach ($expectedVersions as $version) { + $this->assertContains($version, $negotiation['accepted_request_versions_by_default']); + } + $this->assertSame('1.19', $openApi['components']['schemas']['AdvertisedWorkerProtocolVersion']['const']); $this->assertSame( - WorkerProtocolVersion::VERSION, + '1.19', $asyncApi['components']['schemas']['ProtocolEnvelope']['properties']['protocol_version']['const'], ); diff --git a/tests/Unit/V2/ReplayRegressionCorpusTest.php b/tests/Unit/V2/ReplayRegressionCorpusTest.php index 5e850037c..59245cb7a 100644 --- a/tests/Unit/V2/ReplayRegressionCorpusTest.php +++ b/tests/Unit/V2/ReplayRegressionCorpusTest.php @@ -15,6 +15,7 @@ use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowRun; +use Workflow\V2\Support\CooperativeCancellationDelivery; use Workflow\V2\Support\HistoryEventPayloadContract; use Workflow\V2\Support\WorkflowFiberRunner; use Workflow\V2\Support\WorkflowStep; @@ -64,6 +65,21 @@ public function testFixtureExecutesThroughColdReplayRunner(array $fixture): void { $this->assertHistoryPayloadContract($fixture); + if ($fixture['id'] === 'root-delivery-scoped-operation-membership') { + $run = new WorkflowRun([ + 'cancellation_request_command_id' => 'request-1', + ]); + $run->setRelation('historyEvents', collect($fixture['history'])->map( + static fn (array $event): WorkflowHistoryEvent => new WorkflowHistoryEvent($event), + )); + $this->assertSame('cancellation_delivery_requires_scope', CooperativeCancellationDelivery::validate( + $run, + 'request-1', + 2, + 'activity', + )); + } + $workflow = $fixture['workflow']; $workflowClass = $workflow['type']; diff --git a/tests/Unit/V2/ScopedCancellationContextTest.php b/tests/Unit/V2/ScopedCancellationContextTest.php new file mode 100644 index 000000000..a8248a626 --- /dev/null +++ b/tests/Unit/V2/ScopedCancellationContextTest.php @@ -0,0 +1,258 @@ +root(); + $context = ScopedCancellationContext::fromRunContext($root) + ->forDescendant('scope-request-a', 'instance-root', 'run-root', 'scope-a') + ->forDescendant('scope-request-b', 'instance-root', 'run-root', 'scope-b') + ->forDescendant('child-request', 'instance-child', 'run-child', 'root'); + $snapshot = $context->toArray(); + CarbonImmutable::setTestNow('2040-01-01T00:00:00Z'); + try { + $reloaded = ScopedCancellationContext::fromArray(json_decode( + json_encode($snapshot, JSON_THROW_ON_ERROR), + true, + flags: JSON_THROW_ON_ERROR, + )); + $this->assertSame($snapshot, $reloaded->toArray()); + $this->assertSame($root->toArray(), $reloaded->rootContext->toArray()); + $this->assertSame('child-request', $reloaded->requestId); + $this->assertSame('scope-request-b', $reloaded->parentRequestId); + $this->assertSame('instance-child', $reloaded->workflowInstanceId); + $this->assertSame('run-child', $reloaded->workflowRunId); + $this->assertSame('root', $reloaded->scopeId); + $this->assertSame('root', $reloaded->rootScopeId); + $this->assertCount(4, $reloaded->lineage); + $this->assertSame('2026-10-03T00:00:00.000000Z', $reloaded->requestedAt()->toISOString()); + $this->assertSame('2026-10-03T00:00:30.123456Z', $reloaded->deadline()->toISOString()); + $this->assertSame($reloaded->rootDeadline(), $reloaded->rootContext->deadline()); + } finally { + CarbonImmutable::setTestNow(); + } + } + + public function testStricterAcceptedScopeBudgetDoesNotRewriteOriginalMetadataOrGrantDescendantsMoreTime(): void + { + $original = ScopedCancellationContext::fromRunContext($this->root()); + $before = $original->toArray(); + $scope = $original->forDescendant( + 'scope-request-a', + 'instance-root', + 'run-root', + 'scope-a', + CarbonImmutable::parse('2026-10-03T00:00:20.000001Z'), + ); + $child = $scope->forDescendant('child-request', 'instance-child', 'run-child', 'root'); + $this->assertSame($before, $original->toArray()); + $this->assertSame($before['root_context'], $child->toArray()['root_context']); + $this->assertSame('2026-10-03T00:00:30.123456Z', $child->rootDeadline()->toISOString()); + $this->assertSame('2026-10-03T00:00:20.000001Z', $child->deadline()->toISOString()); + $this->assertSame('maintenance', $child->rootContext->reason); + $this->assertSame([ + 'type' => 'operator', + 'id' => 'operator-1', + ], $child->rootContext->requester); + $this->assertSame('api', $child->rootContext->source); + $copy = $child->toArray(); + $copy['lineage'][1]['scope_id'] = 'changed'; + $child->deadline() + ->addHour(); + $child->rootDeadline() + ->addHour(); + $this->assertSame('scope-a', $child->lineage[1]['scope_id']); + $this->assertSame('2026-10-03T00:00:20.000001Z', $child->deadline()->toISOString()); + $this->expectException(InvalidArgumentException::class); + $scope->forDescendant( + 'longer-child', + 'instance-child', + 'run-child', + 'root', + CarbonImmutable::parse('2026-10-03T00:00:21Z'), + ); + } + + public function testLegacyRunLineageIsAdaptedWithoutChangingItsV1Snapshot(): void + { + $root = $this->root(); + $legacy = $root->forDescendant('child-request', 'instance-child', 'run-child'); + $before = $legacy->toArray(); + $context = ScopedCancellationContext::fromRunContext($legacy); + $this->assertSame($root->toArray(), $context->rootContext->toArray()); + $this->assertSame($before, $legacy->toArray()); + $this->assertSame(['root', 'root'], array_column($context->lineage, 'scope_id')); + $this->assertSame(['root-request', 'child-request'], array_column($context->lineage, 'request_id')); + $this->assertSame('child-request', $context->requestId); + $this->assertSame('root-request', $context->parentRequestId); + $this->assertSame($legacy->deadline()->toISOString(), $context->deadline()->toISOString()); + } + + public function testDirectScopeRootHasItsOwnOriginalAddressWithoutChangingV1RootMetadata(): void + { + $snapshot = ScopedCancellationContext::fromRunContext($this->root())->toArray(); + $snapshot['lineage'][0]['scope_id'] = 'direct-root-scope'; + $context = ScopedCancellationContext::fromArray($snapshot); + $this->assertSame('direct-root-scope', $context->rootScopeId); + $this->assertSame('direct-root-scope', $context->scopeId); + $this->assertNull($context->parentRequestId); + $this->assertSame($this->root()->toArray(), $context->rootContext->toArray()); + } + + public function testLegacyParserStillRejectsRepeatedRunsAndDoesNotAcceptTheNewSchema(): void + { + try { + $this->root() + ->forDescendant('scope-request', 'instance-root', 'run-root'); + $this->fail('Legacy v1 parser accepted multiple addresses inside one run.'); + } catch (InvalidArgumentException $error) { + $this->assertStringContainsString('cycle', $error->getMessage()); + } + $this->expectExceptionMessage('Unsupported cancellation context schema.'); + CancellationContext::fromArray(ScopedCancellationContext::fromRunContext($this->root())->toArray()); + } + + public function testAddressComparisonCannotConfuseDelimiterCharactersWithRunScopeBoundaries(): void + { + $context = ScopedCancellationContext::fromRunContext($this->root()) + ->forDescendant('first', 'instance-first', 'run:first', 'scope') + ->forDescendant('second', 'instance-second', 'run', 'first:scope'); + $this->assertCount(3, $context->lineage); + $this->assertSame('second', $context->requestId); + } + + public function testDatabaseObjectKeyReorderingDoesNotChangeCanonicalScopeAddresses(): void + { + $context = ScopedCancellationContext::fromRunContext($this->root()) + ->forDescendant('scope-request', 'instance-root', 'run-root', 'scope-a'); + $snapshot = $context->toArray(); + foreach ($snapshot['lineage'] as &$entry) { + ksort($entry); + } + unset($entry); + $this->assertSame($context->toArray(), ScopedCancellationContext::fromArray($snapshot)->toArray()); + } + + #[DataProvider('invalidSnapshots')] + public function testMalformedOrExpandedMetadataCannotChangeAnAcceptedContext(string $mutation): void + { + $original = ScopedCancellationContext::fromRunContext($this->root()) + ->forDescendant( + 'scope-request-a', + 'instance-root', + 'run-root', + 'scope-a', + CarbonImmutable::parse('2026-10-03T00:00:20Z') + ) + ->forDescendant('scope-request-b', 'instance-root', 'run-root', 'scope-b'); + $snapshot = $original->toArray(); + $before = $snapshot; + switch ($mutation) { + case 'schema': $snapshot['schema'] = 'durable-workflow.scoped-cancellation-context/v2'; + break; + case 'extra top field': $snapshot['authority'] = true; + break; + case 'missing root': unset($snapshot['root_context']); + break; + case 'empty lineage': $snapshot['lineage'] = []; + break; + case 'non-list lineage': $snapshot['lineage'] = [ + 'root' => $snapshot['lineage'][0], + ]; + break; + case 'non-array entry': $snapshot['lineage'][1] = 'scope-a'; + break; + case 'extra address field': $snapshot['lineage'][1]['shield_parent'] = false; + break; + case 'missing address field': unset($snapshot['lineage'][1]['scope_id']); + break; + case 'wrong root request': $snapshot['lineage'][0]['request_id'] = 'other-root'; + break; + case 'wrong root run': $snapshot['lineage'][0]['workflow_run_id'] = 'other-run'; + break; + case 'duplicate request': $snapshot['lineage'][2]['request_id'] = 'scope-request-a'; + break; + case 'repeated address': $snapshot['lineage'][2]['scope_id'] = 'scope-a'; + break; + case 'contradictory instance': $snapshot['lineage'][2]['workflow_instance_id'] = 'other-instance'; + break; + case 'longer descendant': $snapshot['lineage'][2]['cleanup_deadline_at'] = '2026-10-03T00:00:21Z'; + break; + case 'changed root deadline': $snapshot['lineage'][0]['cleanup_deadline_at'] = '2026-10-03T00:00:29Z'; + break; + case 'malformed deadline': $snapshot['lineage'][2]['cleanup_deadline_at'] = 'tomorrow'; + break; + case 'expired initial budget': $snapshot['lineage'][2]['cleanup_deadline_at'] = '2026-10-03T00:00:00Z'; + break; + case 'empty scope': $snapshot['lineage'][2]['scope_id'] = ' '; + break; + case 'numeric request': $snapshot['lineage'][2]['request_id'] = 17; + break; + case 'extra root metadata': $snapshot['root_context']['scope_id'] = 'forged'; + break; + case 'descendant used as root': $snapshot['root_context'] = $this->root() + ->forDescendant('child-request', 'instance-child', 'run-child') + ->toArray(); + break; + } + try { + ScopedCancellationContext::fromArray($snapshot); + $this->fail('Invalid scoped metadata was accepted: ' . $mutation); + } catch (InvalidArgumentException) { + $this->assertSame($before, $original->toArray()); + } + } + + /** + * @return iterable + */ + public static function invalidSnapshots(): iterable + { + foreach ([ + 'schema', 'extra top field', 'missing root', 'empty lineage', 'non-list lineage', + 'non-array entry', 'extra address field', 'missing address field', 'wrong root request', + 'wrong root run', 'duplicate request', 'repeated address', 'contradictory instance', + 'longer descendant', 'changed root deadline', 'malformed deadline', 'expired initial budget', + 'empty scope', 'numeric request', 'extra root metadata', 'descendant used as root', + ] as $mutation) { + yield $mutation => [$mutation]; + } + } + + private function root(): CancellationContext + { + return CancellationContext::fromArray([ + 'schema' => 'durable-workflow.cancellation-context/v1', + 'request_id' => 'root-request', + 'root_request_id' => 'root-request', + 'root_workflow_instance_id' => 'instance-root', + 'root_workflow_run_id' => 'run-root', + 'parent_request_id' => null, + 'reason' => 'maintenance', + 'requester' => [ + 'type' => 'operator', + 'id' => 'operator-1', + ], + 'source' => 'api', + 'requested_at' => '2026-10-03T00:00:00Z', + 'cleanup_deadline_at' => '2026-10-03T00:00:30.123456Z', + 'lineage' => [[ + 'request_id' => 'root-request', + 'workflow_instance_id' => 'instance-root', + 'workflow_run_id' => 'run-root', + ]], + ]); + } +} diff --git a/tests/Unit/V2/ScopedChildCancellationTest.php b/tests/Unit/V2/ScopedChildCancellationTest.php new file mode 100644 index 000000000..5b9b369ad --- /dev/null +++ b/tests/Unit/V2/ScopedChildCancellationTest.php @@ -0,0 +1,200 @@ +historyRun(); + $first = $this->event('scheduled-a', 3); + $second = $this->event('scheduled-b', 4, [ + 'sequence' => 5, + 'child_call_id' => 'call-b', + ]); + $run->setRelation('historyEvents', new Collection([$second, $first])); + $original = ScopedChildCancellation::members($run, 'scope-a'); + $this->assertSame([4, 5], array_column($original, 'sequence')); + $first->payload = [ + ...$first->payload, + 'arguments' => 'secret application bytes', + ]; + $this->assertSame($original, ScopedChildCancellation::members($run, 'scope-a')); + $this->assertSame($original, ScopedChildCancellation::normalizeMembers(array_map( + static fn (array $member): array => array_reverse($member, true), + $original + ))); + $this->assertSame([], ScopedChildCancellation::members($run, 'scope-sibling')); + } + + public function testHistoricalMissingPolicyRemainsAbandonAndTerminalHistoryDoesNotRetarget(): void + { + $scheduled = $this->event('scheduled', 3); + $payload = $scheduled->payload; + unset($payload['cancellation_policy']); + $scheduled->payload = $payload; + $terminal = $this->event('terminal', 6, [ + 'child_workflow_run_id' => 'different-terminal-run', + ]); + $terminal->event_type = HistoryEventType::ChildRunCompleted; + $run = $this->historyRun(); + $run->setRelation('historyEvents', new Collection([$scheduled, $terminal])); + $member = ScopedChildCancellation::members($run, 'scope-a')[0]; + $this->assertSame('abandon', $member['cancellation_policy']); + $this->assertSame('child-run', $member['child_workflow_run_id']); + } + + #[DataProvider('invalidHistory')] + public function testMalformedChildAuthorityCannotEnterPreparation(array $changes, bool $started = false): void + { + $scheduled = $this->event('scheduled', 3, $started ? [] : $changes); + $history = [$scheduled]; + if ($started) { + $row = $this->event('started', 4, $changes); + $row->event_type = HistoryEventType::ChildRunStarted; + $history[] = $row; + } + $run = $this->historyRun(); + $run->setRelation('historyEvents', new Collection($history)); + $this->expectException(LogicException::class); + ScopedChildCancellation::members($run, 'scope-a'); + } + + public static function invalidHistory(): iterable + { + foreach ([ + 'sequence' => 0, + 'child_call_id' => '', + 'child_workflow_instance_id' => false, + 'child_workflow_run_id' => 'parent-run', + 'cancellation_policy' => 'terminate', + 'child_workflow' => false, + ] as $key => $value) { + yield 'scheduled:' . $key => [[ + $key => $value, + ]]; + } + yield 'conflicting descriptor address' => [[ + 'child_workflow' => [ + 'cancellation_scope_id' => 'scope-b', + ], + ]]; + foreach ([ + 'child_call_id' => 'different-call', + 'child_workflow_instance_id' => 'different-instance', + 'child_workflow_run_id' => null, + 'cancellation_scope_id' => 'scope-b', + 'cancellation_policy' => 'abandon', + ] as $key => $value) { + yield 'started:' . $key => [[ + $key => $value, + ], true]; + } + } + + public function testRepeatedChildCallIdentityCannotCreateAnotherCancellationTarget(): void + { + $run = $this->historyRun(); + $run->setRelation('historyEvents', new Collection([ + $this->event('first', 3), $this->event('duplicate', 4, [ + 'sequence' => 5, + ]), + ])); + $this->expectExceptionMessage('cancellation_scope_child_history_invalid'); + ScopedChildCancellation::members($run, 'scope-a'); + } + + #[DataProvider('invalidMembership')] + public function testColdNormalizationRejectsMalformedSnapshots(mixed $snapshot): void + { + $this->expectExceptionMessage('cancellation_scope_preparation_history_invalid'); + ScopedChildCancellation::normalizeMembers($snapshot); + } + + public static function invalidMembership(): iterable + { + yield 'not an array' => [null]; + yield 'not a list' => [[ + 'member' => [], + ]]; + yield 'not a member' => [[false]]; + $member = [ + 'sequence' => 4, + 'child_call_id' => 'call-a', + 'child_workflow_instance_id' => 'child-instance', + 'child_workflow_run_id' => 'child-run', + 'cancellation_policy' => 'try_cancel', + 'descriptor_hash' => str_repeat('a', 64), + ]; + foreach ([ + 'sequence' => '4', + 'child_call_id' => null, + 'child_workflow_instance_id' => '', + 'child_workflow_run_id' => false, + 'cancellation_policy' => 'terminate', + 'descriptor_hash' => 'invalid', + ] as $key => $value) { + yield $key => [[[ + $key => $value, + ] + $member]]; + } + } + + public function testEmptyChildSetCanCrossTheBarrierButMembershipCannotPretendToBeAReceipt(): void + { + $preparation = new WorkflowHistoryEvent([ + 'payload' => [ + 'child_members' => [], + ], + ]); + ScopedChildCancellation::assertReady($preparation); + $run = $this->historyRun(); + $run->setRelation('historyEvents', new Collection([$this->event('scheduled', 3)])); + $preparation->payload = [ + 'child_members' => ScopedChildCancellation::members($run, 'scope-a'), + ]; + $this->expectExceptionMessage('cancellation_scope_child_delivery_not_established'); + ScopedChildCancellation::assertReady($preparation); + } + + private function historyRun(): WorkflowRun + { + $run = $this->createPartialMock(WorkflowRun::class, ['loadMissing']); + $run->expects($this->atLeastOnce()) + ->method('loadMissing') + ->willReturnSelf(); + $run->id = 'parent-run'; + return $run; + } + + /** + * @param array $changes + */ + private function event(string $id, int $historySequence, array $changes = []): WorkflowHistoryEvent + { + return new WorkflowHistoryEvent([ + 'id' => $id, + 'sequence' => $historySequence, + 'event_type' => HistoryEventType::ChildWorkflowScheduled, + 'payload' => [...[ + 'sequence' => 4, + 'child_call_id' => 'call-a', + 'child_workflow_instance_id' => 'child-instance', + 'child_workflow_run_id' => 'child-run', + 'cancellation_policy' => 'try_cancel', + 'cancellation_scope_id' => 'scope-a', + ], ...$changes], + ]); + } +} diff --git a/tests/Unit/V2/ScopedRunCancellationContextTest.php b/tests/Unit/V2/ScopedRunCancellationContextTest.php new file mode 100644 index 000000000..e2ae00607 --- /dev/null +++ b/tests/Unit/V2/ScopedRunCancellationContextTest.php @@ -0,0 +1,216 @@ +origin(); + $child = CancellationContext::fromScopeContext($origin, 'child-request', 'child-instance', 'child-run'); + $this->assertSame('root-request', $child->rootRequestId); + $this->assertSame('inner-request', $child->parentRequestId); + $this->assertSame($origin->toArray(), $child->scopeOrigin->toArray()); + $this->assertSame('2026-10-04T00:00:00.000000Z', $child->requestedAt()->toISOString()); + $this->assertSame('2026-10-04T00:00:20.000000Z', $child->deadline()->toISOString()); + $this->assertSame('2026-10-04T00:00:30.000000Z', $child->scopeOrigin->rootDeadline()->toISOString()); + $this->assertSame(['root-request', 'child-request'], array_column($child->lineage, 'request_id')); + $this->assertSame(['outer', 'inner'], array_column($child->scopeOrigin->lineage, 'scope_id')); + $this->assertSame('maintenance', $child->reason); + $this->assertSame([ + 'type' => 'operator', + 'id' => 'operator-1', + ], $child->requester); + $this->assertSame('api', $child->source); + } + + public function testColdAvroRoundTripAndRunScopeRunPropagationPreserveTheCompleteTree(): void + { + $origin = $this->origin(); + $child = CancellationContext::fromScopeContext($origin, 'child-request', 'child-instance', 'child-run'); + $decoded = Serializer::unserializeWithCodec('avro', Serializer::serializeWithCodec('avro', $child->toArray())); + $cold = CancellationContext::fromArray($decoded); + $this->assertSame($child->toArray(), $cold->toArray()); + $childScope = ScopedCancellationContext::fromRunContext($cold) + ->forDescendant('child-scope-request', 'child-instance', 'child-run', 'child-scope'); + $grandchild = CancellationContext::fromScopeContext( + $childScope, + 'grandchild-request', + 'grandchild-instance', + 'grandchild-run' + ); + $this->assertSame('child-scope-request', $grandchild->parentRequestId); + $this->assertSame($origin->lineage, array_slice($grandchild->scopeOrigin->lineage, 0, 2)); + $this->assertSame(['outer', 'inner', 'root', 'child-scope'], array_column( + $grandchild->scopeOrigin->lineage, + 'scope_id' + )); + $this->assertSame($origin->deadline()->toISOString(), $grandchild->deadline()->toISOString()); + $this->assertSame(['root-request', 'child-scope-request', 'grandchild-request'], array_column( + $grandchild->lineage, + 'request_id' + )); + $directGrandchild = $cold->forDescendant('other-child', 'other-instance', 'other-run'); + $this->assertSame($cold->requestId, $directGrandchild->parentRequestId); + $this->assertSame(['outer', 'inner', 'root'], array_column( + $directGrandchild->scopeOrigin->lineage, + 'scope_id' + )); + $this->assertSame($origin->rootContext->toArray(), $directGrandchild->scopeOrigin->rootContext->toArray()); + } + + public function testObjectKeyOrderingCannotAlterTheRecordedOrigin(): void + { + $context = CancellationContext::fromScopeContext($this->origin(), 'child', 'instance-child', 'run-child'); + $snapshot = $context->toArray(); + foreach ($snapshot['lineage'] as &$entry) { + ksort($entry); + } + unset($entry); + ksort($snapshot['requester']); + $cold = CancellationContext::fromArray($snapshot); + $this->assertSame($context->lineage, $cold->lineage); + $this->assertSame($context->scopeOrigin->toArray(), $cold->scopeOrigin->toArray()); + } + + public function testEarlierParentAuthorityNarrowsChildBudgetWithoutChangingTheRootOrOrigin(): void + { + $origin = $this->origin(); + $child = CancellationContext::fromScopeContext( + $origin, + 'child', + 'child-instance', + 'child-run', + CarbonImmutable::parse('2026-10-04T00:00:10Z') + ); + $this->assertSame($origin->toArray(), $child->scopeOrigin->toArray()); + $this->assertSame('2026-10-04T00:00:10.000000Z', $child->deadline()->toISOString()); + $this->assertSame($child->toArray(), CancellationContext::fromArray($child->toArray())->toArray()); + $grandchild = $child->forDescendant('grandchild', 'grandchild-instance', 'grandchild-run'); + $this->assertSame($child->deadline()->toISOString(), $grandchild->deadline()->toISOString()); + $this->assertSame($origin->rootContext->toArray(), $grandchild->scopeOrigin->rootContext->toArray()); + $this->expectException(InvalidArgumentException::class); + CancellationContext::fromScopeContext( + $origin, + 'wider', + 'other-instance', + 'other-run', + CarbonImmutable::parse('2026-10-04T00:00:21Z') + ); + } + + #[DataProvider('invalidMutations')] + public function testInvalidOriginCannotBeSilentlyDroppedOrGrantedANewBudget(string $mutation): void + { + $snapshot = CancellationContext::fromScopeContext($this->origin(), 'child', 'instance-child', 'run-child') + ->toArray(); + switch ($mutation) { + case 'legacy schema': $snapshot['schema'] = 'durable-workflow.cancellation-context/v1'; + break; + case 'missing origin': unset($snapshot['scope_origin']); + break; + case 'invalid origin': $snapshot['scope_origin'] = []; + break; + case 'root': $snapshot['root_request_id'] = 'different'; + break; + case 'parent': $snapshot['parent_request_id'] = 'root-request'; + break; + case 'reason': $snapshot['reason'] = 'different'; + break; + case 'source': $snapshot['source'] = 'different'; + break; + case 'requester': $snapshot['requester']['id'] = 'different'; + break; + case 'requested at': $snapshot['requested_at'] = '2026-10-04T00:00:01Z'; + break; + case 'wider deadline': $snapshot['cleanup_deadline_at'] = '2026-10-04T00:00:30Z'; + break; + case 'different deadline': $snapshot['cleanup_deadline_at'] = '2026-10-04T00:00:10Z'; + break; + case 'discarded address': array_shift($snapshot['scope_origin']['lineage']); + break; + case 'reused request': $snapshot['request_id'] = 'inner-request'; + $snapshot['lineage'][1]['request_id'] = 'inner-request'; + break; + case 'reused run': $snapshot['lineage'][1]['workflow_run_id'] = 'root-run'; + break; + case 'altered lineage': $snapshot['lineage'][0]['workflow_instance_id'] = 'different'; + break; + } + $this->expectException(InvalidArgumentException::class); + CancellationContext::fromArray($snapshot); + } + + /** + * @return iterable + */ + public static function invalidMutations(): iterable + { + foreach (['legacy schema', 'missing origin', 'invalid origin', 'root', 'parent', 'reason', 'source', + 'requester', 'requested at', 'wider deadline', 'different deadline', 'discarded address', + 'reused request', 'reused run', 'altered lineage'] as $mutation) { + yield $mutation => [$mutation]; + } + } + + public function testLineageCannotReenterAnEarlierRunThroughANewScopeAddress(): void + { + $origin = $this->origin() + ->forDescendant('child', 'child-instance', 'child-run', 'root'); + $this->expectException(InvalidArgumentException::class); + $origin->forDescendant('cycle', 'root-instance', 'root-run', 'unused-scope'); + } + + private function origin(): ScopedCancellationContext + { + $root = CancellationContext::fromArray([ + 'schema' => 'durable-workflow.cancellation-context/v1', + 'request_id' => 'root-request', + 'root_request_id' => 'root-request', + 'root_workflow_instance_id' => 'root-instance', + 'root_workflow_run_id' => 'root-run', + 'parent_request_id' => null, + 'reason' => 'maintenance', + 'requester' => [ + 'type' => 'operator', + 'id' => 'operator-1', + ], + 'source' => 'api', + 'requested_at' => '2026-10-04T00:00:00Z', + 'cleanup_deadline_at' => '2026-10-04T00:00:30Z', + 'lineage' => [[ + 'request_id' => 'root-request', + 'workflow_instance_id' => 'root-instance', + 'workflow_run_id' => 'root-run', + ]], + ]); + return ScopedCancellationContext::fromArray([ + 'schema' => ScopedCancellationContext::SCHEMA, + 'root_context' => $root->toArray(), + 'lineage' => [[ + 'request_id' => $root->requestId, + 'workflow_instance_id' => 'root-instance', + 'workflow_run_id' => 'root-run', + 'scope_id' => 'outer', + 'cleanup_deadline_at' => $root->deadline() + ->toISOString(), + ]], + ])->forDescendant( + 'inner-request', + 'root-instance', + 'root-run', + 'inner', + CarbonImmutable::parse('2026-10-04T00:00:20Z') + ); + } +} diff --git a/tests/Unit/V2/SurfaceStabilityContractTest.php b/tests/Unit/V2/SurfaceStabilityContractTest.php index b7dd41cb0..5f922cc05 100644 --- a/tests/Unit/V2/SurfaceStabilityContractTest.php +++ b/tests/Unit/V2/SurfaceStabilityContractTest.php @@ -29,7 +29,7 @@ public function testManifestAdvertisesAuthorityIdentity(): void $manifest = SurfaceStabilityContract::manifest(); $this->assertSame('durable-workflow.v2.surface-stability.contract', $manifest['schema']); - $this->assertSame(4, $manifest['version']); + $this->assertSame(5, $manifest['version']); $this->assertSame('https://durable-workflow.github.io/docs/2.0/compatibility', $manifest['authority_url']); } @@ -145,7 +145,7 @@ public function testWorkerProtocolNegotiationIncludesRustProtocolAndFailsClosed( $negotiation = $manifest['surface_families']['worker_protocol']['negotiation']; $this->assertSame('worker_protocol.version', $negotiation['advertised_version_path']); - $this->assertSame('1.19', $negotiation['default_advertised_version']); + $this->assertSame('1.20', $negotiation['default_advertised_version']); $this->assertSame( 'same_major_and_minor_less_than_or_equal_to_advertised', $negotiation['request_header_rule'], @@ -153,6 +153,7 @@ public function testWorkerProtocolNegotiationIncludesRustProtocolAndFailsClosed( $this->assertSame('1.0', $negotiation['accepted_request_versions_by_default'][0]); $this->assertContains('1.2', $negotiation['accepted_request_versions_by_default']); $this->assertSame('1.19', $negotiation['accepted_request_versions_by_default'][19]); + $this->assertSame('1.20', $negotiation['accepted_request_versions_by_default'][20]); $this->assertSame('advertised_version', $negotiation['response_version']); $this->assertSame( ['missing_header', 'malformed_version', 'different_major', 'minor_greater_than_advertised'], diff --git a/tests/Unit/V2/WorkerProtocolVersionTest.php b/tests/Unit/V2/WorkerProtocolVersionTest.php index 759905a9a..f6a72abb7 100644 --- a/tests/Unit/V2/WorkerProtocolVersionTest.php +++ b/tests/Unit/V2/WorkerProtocolVersionTest.php @@ -19,7 +19,7 @@ public function testVersionIsNonEmptyString(): void public function testVersionTracksServiceOperationCommandShape(): void { - $this->assertSame('1.19', WorkerProtocolVersion::VERSION); + $this->assertSame('1.20', WorkerProtocolVersion::VERSION); $this->assertContains('start_service_operation', WorkerProtocolVersion::nonTerminalCommandTypes()); $this->assertSame(0, WorkerProtocolVersion::longPollSemantics()['min_timeout_seconds']); } diff --git a/tests/Unit/V2/WorkflowCommandNormalizerTest.php b/tests/Unit/V2/WorkflowCommandNormalizerTest.php index c1f9bf3bf..703ced887 100644 --- a/tests/Unit/V2/WorkflowCommandNormalizerTest.php +++ b/tests/Unit/V2/WorkflowCommandNormalizerTest.php @@ -15,6 +15,175 @@ final class WorkflowCommandNormalizerTest extends NonDatabaseTestCase { + public function testScopedCleanupTimerProofRequiresTheCandidateProtocolAndCannotBeDroppedOrExtended(): void + { + $command = [ + 'type' => 'start_timer', + 'delay_seconds' => 1, + 'cancellation_scope_id' => 'scope', + 'cancellation_cleanup' => [ + 'scope_id' => 'scope', + 'request_id' => 'request', + 'delivery_history_event_id' => 'delivery', + ], + ]; + $this->assertSame($command, WorkflowCommandNormalizer::normalize([$command], '1.20')[0]); + foreach (['1.19', '2.0'] as $protocol) { + try { + WorkflowCommandNormalizer::normalize([$command], $protocol); + $this->fail('Unqualified protocol accepted a cleanup authority proof.'); + } catch (ValidationException $error) { + $this->assertArrayHasKey('commands.0.cancellation_cleanup', $error->errors()); + } + } + foreach ([ + null, [], [ + 'scope_id' => 'scope', + ], [ + ...$command['cancellation_cleanup'], + 'cleanup_deadline_at' => '2027-01-01T00:00:00.000000Z', + ]] as $proof) { + $command['cancellation_cleanup'] = $proof; + try { + WorkflowCommandNormalizer::normalize([$command], '1.20'); + $this->fail('Malformed cleanup authority was silently dropped or accepted.'); + } catch (ValidationException $error) { + $this->assertArrayHasKey('commands.0.cancellation_cleanup', $error->errors()); + } + } + } + + public function testOperationScopeMembershipRequiresTheCandidateProtocolAndPreservesExactAddresses(): void + { + foreach ($this->scopedOperationCommands() as $command) { + $normalized = WorkflowCommandNormalizer::normalize([$command], '1.20')[0]; + $this->assertSame('recorded-scope', $normalized['cancellation_scope_id']); + foreach (['1.19', '2.0', '1.20.extra'] as $protocol) { + try { + WorkflowCommandNormalizer::normalize([$command], $protocol); + $this->fail('Unqualified protocol accepted operation scope membership.'); + } catch (ValidationException $error) { + $this->assertArrayHasKey('commands.0.cancellation_scope_id', $error->errors()); + } + } + } + $this->assertTrue(WorkerProtocolVersion::supportsCancellationScopeMembership('1.20')); + $this->assertFalse(WorkerProtocolVersion::supportsCancellationScopeMembership('1.19')); + } + + public function testMalformedScopeAddressesCannotBeDroppedDuringOperationNormalization(): void + { + foreach ($this->scopedOperationCommands() as $command) { + foreach ([null, '', ' ', 7, [], str_repeat('x', 256), "\xff"] as $id) { + $command['cancellation_scope_id'] = $id; + try { + WorkflowCommandNormalizer::normalize([$command], '1.20'); + $this->fail('Invalid operation scope was silently omitted.'); + } catch (ValidationException $error) { + $this->assertArrayHasKey('commands.0.cancellation_scope_id', $error->errors()); + } + } + } + } + + public function testScopeMembershipCannotBeAttachedToTerminalOrUnrelatedCommands(): void + { + foreach ([[ + 'type' => 'complete_workflow', + ], [ + 'type' => 'fail_workflow', + 'message' => 'failed', + ], + [ + 'type' => 'record_side_effect', + 'result' => Serializer::serializeWithCodec('avro', 1), + ]] as $command) { + try { + WorkflowCommandNormalizer::normalize([[ + ...$command, + 'cancellation_scope_id' => 'recorded-scope', + ]], '1.20'); + $this->fail('Unrelated command accepted operation scope membership.'); + } catch (ValidationException $error) { + $this->assertArrayHasKey('commands.0.cancellation_scope_id', $error->errors()); + } + } + } + + public function testUnscopedOperationsKeepTheirPublishedCommandShape(): void + { + foreach ($this->scopedOperationCommands() as $command) { + unset($command['cancellation_scope_id']); + $published = WorkflowCommandNormalizer::normalize([$command], '1.19'); + $candidate = WorkflowCommandNormalizer::normalize([$command], '1.20'); + $this->assertSame($published, $candidate); + $this->assertArrayNotHasKey('cancellation_scope_id', $published[0]); + } + } + + public function testExplicitRemoteActivityPoliciesPreserveTheirAuthoredPolicyOnlyOnTheSourceProtocol(): void + { + foreach (['try_cancel', 'wait_cancellation_completed', 'abandon'] as $policy) { + $command = [ + 'type' => 'schedule_activity', + 'activity_type' => 'remote', + 'cancellation_policy' => $policy, + 'schedule_to_close_timeout' => 30, + ]; + $normalized = WorkflowCommandNormalizer::normalize([$command], '1.20')[0]; + $this->assertSame($policy, $normalized['cancellation_policy']); + $this->assertSame(30, $normalized['schedule_to_close_timeout']); + foreach (['1.19', '2.0'] as $protocol) { + try { + WorkflowCommandNormalizer::normalize([$command], $protocol); + $this->fail('Explicit remote policies require the cooperative Source protocol.'); + } catch (ValidationException $exception) { + $this->assertArrayHasKey('commands.0.cancellation_policy', $exception->errors()); + } + } + } + $historical = WorkflowCommandNormalizer::normalize([ + [ + 'type' => 'schedule_activity', + 'activity_type' => 'remote', + ], + ], '1.19')[0]; + $this->assertArrayNotHasKey('cancellation_policy', $historical); + } + + public function testRemoteAbandonRefusesAnAbsentInvalidOrUnlimitedTotalLifetime(): void + { + foreach ([null, 0, -1, '30', [], 1.5] as $timeout) { + try { + WorkflowCommandNormalizer::normalize([[ + 'type' => 'schedule_activity', + 'activity_type' => 'remote', + 'cancellation_policy' => 'abandon', + 'schedule_to_close_timeout' => $timeout, + ]], '1.20'); + $this->fail('Remote Abandon must have a finite positive integer total timeout.'); + } catch (ValidationException $exception) { + $this->assertArrayHasKey('commands.0.schedule_to_close_timeout', $exception->errors()); + } + } + } + + public function testRemoteActivityPolicyRefusesUnknownAndNonStringValues(): void + { + foreach (['unknown', [], 1, true] as $policy) { + try { + WorkflowCommandNormalizer::normalize([[ + 'type' => 'schedule_activity', + 'activity_type' => 'remote', + 'cancellation_policy' => $policy, + ]], '1.20'); + $this->fail('Remote activity policy must be a known policy string.'); + } catch (ValidationException $exception) { + $this->assertArrayHasKey('commands.0.cancellation_policy', $exception->errors()); + } + } + } + public function testPayloadEnvelopeFieldContractNamesCodecBearingCommandPayloads(): void { $this->assertSame([ @@ -1095,6 +1264,71 @@ public function testStartChildWorkflowAcceptsKnownPolicy(): void ]], $out); } + public function testCooperativeChildPoliciesSurviveNormalizationAtTheirProtocolFloor(): void + { + foreach (['try_cancel', 'wait_cancellation_completed', 'abandon'] as $policy) { + $command = [ + 'type' => 'start_child_workflow', + 'workflow_type' => 'Child', + 'parent_close_policy' => 'request_cancellation', + 'cancellation_policy' => $policy, + ]; + $this->assertSame([$command], WorkflowCommandNormalizer::normalize([$command], '1.20')); + } + } + + public function testCooperativeChildPoliciesRejectOldAndWrongMajorProtocols(): void + { + foreach (['1.19', '2.20', 'invalid'] as $version) { + try { + WorkflowCommandNormalizer::normalize([[ + 'type' => 'start_child_workflow', + 'workflow_type' => 'Child', + 'parent_close_policy' => 'request_cancellation', + 'cancellation_policy' => 'wait_cancellation_completed', + ]], $version); + $this->fail('Unsupported protocol accepted cooperative child policies.'); + } catch (ValidationException $exception) { + $this->assertArrayHasKey('commands.0.parent_close_policy', $exception->errors()); + $this->assertArrayHasKey('commands.0.cancellation_policy', $exception->errors()); + } + } + } + + public function testExplicitAbandonKeepsLegacyNormalization(): void + { + $command = [ + 'type' => 'start_child_workflow', + 'workflow_type' => 'Child', + 'parent_close_policy' => 'request_cancel', + 'cancellation_policy' => 'abandon', + ]; + $this->assertSame([$command], WorkflowCommandNormalizer::normalize([$command], '1.19')); + } + + public function testInvalidAndMisplacedChildCancellationPoliciesAreRejected(): void + { + foreach ([ + [ + 'type' => 'start_child_workflow', + 'workflow_type' => 'Child', + 'cancellation_policy' => 'unknown', + ], + [ + 'type' => 'start_child_workflow', + 'workflow_type' => 'Child', + 'cancellation_policy' => [], + ], + [ + 'type' => 'start_timer', + 'delay_seconds' => 1, + 'cancellation_policy' => 'try_cancel', + ], + ] as $command) { + $this->assertArrayHasKey('commands.0.cancellation_policy', $this->normalizeAndCaptureErrors([$command])); + } + } + public function testStartChildWorkflowPreservesRetryPolicyAndTimeouts(): void { $out = WorkflowCommandNormalizer::normalize([ @@ -2361,6 +2595,40 @@ public function testParallelMetadataRejectsPartialOrIncompatibleIdentity(): void $this->assertArrayHasKey('commands.0.parallel_group_path', $errors); } + /** + * @return list + */ + private function scopedOperationCommands(): array + { + return [ + [ + 'type' => 'schedule_activity', + 'activity_type' => 'remote', + 'cancellation_scope_id' => 'recorded-scope', + ], + [ + 'type' => 'start_timer', + 'delay_seconds' => 5, + 'cancellation_scope_id' => 'recorded-scope', + ], + [ + 'type' => 'start_child_workflow', + 'workflow_type' => 'child', + 'cancellation_scope_id' => 'recorded-scope', + ], + [ + 'type' => 'open_signal_wait', + 'signal_name' => 'ready', + 'cancellation_scope_id' => 'recorded-scope', + ], + [ + 'type' => 'open_condition_wait', + 'condition_key' => 'ready', + 'cancellation_scope_id' => 'recorded-scope', + ], + ]; + } + /** * @param list> $commands * @return array> diff --git a/tests/Unit/V2/WorkflowFiberRunnerTest.php b/tests/Unit/V2/WorkflowFiberRunnerTest.php index fca2bb706..652ef8add 100644 --- a/tests/Unit/V2/WorkflowFiberRunnerTest.php +++ b/tests/Unit/V2/WorkflowFiberRunnerTest.php @@ -107,6 +107,44 @@ public function testCooperativeCancellationHistoryResumesAnExistingRunner(): voi $this->assertSame('Tests\\Fixtures\\V2\\TestRetryActivity', $resumed->commands[0]['activity_type']); } + public function testSynchronousMemoPersistencePreservesCancellationBudgetDuringColdReplay(): void + { + $fixture = json_decode( + (string) file_get_contents( + __DIR__ . '/../../Fixtures/V2/ReplayRegression/cancellation-context-original-budget-cold-replay.json' + ), + true, + flags: JSON_THROW_ON_ERROR, + ); + $workflow = $fixture['workflow']; + $runner = static fn (array $history): WorkflowFiberRunner => WorkflowFiberRunner::forClass( + $workflow['type'], + 'root-instance', + 'root-run', + [], + 'avro', + )->withHistoryEvents($history); + + $first = $runner($fixture['history'])->step(); + $this->assertSame('upsert_memo', $first->commands[0]['type']); + $this->assertSame(1, $first->commands[1]['delay_seconds']); + $fixture['history'][] = [ + 'sequence' => 6, + 'event_type' => 'MemoUpserted', + 'payload' => [ + 'sequence' => 2, + 'entries' => $first->commands[0]['entries'], + ], + 'recorded_at' => '2026-10-01T00:00:12Z', + ]; + + $cold = $runner($fixture['history'])->step(); + $this->assertSame([[ + 'type' => 'start_timer', + 'delay_seconds' => 1, + ]], $cold->commands); + } + public function testRunnerAuthorsNestedSelectionAsOneExactFlattenedCommandBatchAndThenWaits(): void { $scheduled = $this->runnerFor(WorkerProtocolRunnerNestedSelectionWorkflow::class)->step(); diff --git a/tests/Unit/V2/WorkflowStepHistoryTest.php b/tests/Unit/V2/WorkflowStepHistoryTest.php index 5fadb8367..ac815054b 100644 --- a/tests/Unit/V2/WorkflowStepHistoryTest.php +++ b/tests/Unit/V2/WorkflowStepHistoryTest.php @@ -16,6 +16,19 @@ final class WorkflowStepHistoryTest extends TestCase { + public function testScopeOpeningReservesItsOwnDurableCommandPosition(): void + { + $run = $this->runWithHistoryEvents([ + $this->historyEvent(HistoryEventType::CancellationScopeOpened, [ + 'sequence' => 1, + ]), + ]); + $this->assertSame(2, WorkflowStepHistory::nextDurableCommandSequence($run)); + WorkflowStepHistory::assertCompatible($run, 1, WorkflowStepHistory::CANCELLATION_SCOPE); + $this->expectException(HistoryEventShapeMismatchException::class); + WorkflowStepHistory::assertCompatible($run, 1, WorkflowStepHistory::ACTIVITY); + } + public function testCancellationDeliveryReservesTheInterruptedDurableCallSequence(): void { $run = $this->runWithHistoryEvents([ diff --git a/tests/Unit/V2/WorkflowTaskLeaseTest.php b/tests/Unit/V2/WorkflowTaskLeaseTest.php index ea5f79de5..31be48a1f 100644 --- a/tests/Unit/V2/WorkflowTaskLeaseTest.php +++ b/tests/Unit/V2/WorkflowTaskLeaseTest.php @@ -6,6 +6,8 @@ use Illuminate\Support\Carbon; use Tests\TestCase; +use Workflow\V2\Models\WorkflowRun; +use Workflow\V2\Support\CancellationCleanupLease; use Workflow\V2\Support\WorkflowTaskLease; final class WorkflowTaskLeaseTest extends TestCase @@ -36,4 +38,24 @@ public function testInvalidRuntimeConfigurationFallsBackToEmbeddedDefault(): voi ->set(WorkflowTaskLease::CONFIG_KEY, 'invalid'); $this->assertSame(WorkflowTaskLease::DEFAULT_SECONDS, WorkflowTaskLease::seconds()); } + + public function testAcceptedCancellationBoundsOwnershipBeforeRootDelivery(): void + { + Carbon::setTestNow('2026-10-05T00:00:24Z'); + $this->beforeApplicationDestroyed(static function (): void { + Carbon::setTestNow(); + }); + config() + ->set(WorkflowTaskLease::CONFIG_KEY, 60); + $run = new WorkflowRun([ + 'cancellation_request_command_id' => 'original-request', + 'cancellation_deadline_at' => '2026-10-05T00:00:30Z', + 'cancellation_delivered_at' => null, + ]); + $this->assertSame('2026-10-05T00:00:30.000000Z', CancellationCleanupLease::expiresAt($run)->toISOString()); + $this->assertSame('2026-10-05T00:00:30.000000Z', CancellationCleanupLease::forScopes($run)->toISOString()); + Carbon::setTestNow('2026-10-05T00:00:00Z'); + $this->assertSame('2026-10-05T00:00:10.000000Z', CancellationCleanupLease::expiresAt($run)->toISOString()); + $this->assertSame('2026-10-05T00:00:30.000000Z', CancellationCleanupLease::forScopes($run)->toISOString()); + } }