From 0be5da4238e21bf80ea5ce998d87e8afe42c012a Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 30 Sep 2026 22:58:59 +0000 Subject: [PATCH 01/57] feat: add cooperative cancellation request and claim-bound delivery draft --- .../Api/CooperativeCancellationController.php | 201 ++++++++++++++++ app/Http/Controllers/Api/WorkerController.php | 11 + app/Support/CooperativeCancellationPolicy.php | 78 +++++++ app/Support/PollRequestLeaseBinding.php | 5 + app/Support/WorkflowTaskPoller.php | 35 ++- routes/api.php | 4 + .../CooperativeCancellationProtocolTest.php | 215 ++++++++++++++++++ .../WorkerProtocolVersionCoverageTest.php | 1 + 8 files changed, 546 insertions(+), 4 deletions(-) create mode 100644 app/Http/Controllers/Api/CooperativeCancellationController.php create mode 100644 app/Support/CooperativeCancellationPolicy.php create mode 100644 tests/Feature/CooperativeCancellationProtocolTest.php diff --git a/app/Http/Controllers/Api/CooperativeCancellationController.php b/app/Http/Controllers/Api/CooperativeCancellationController.php new file mode 100644 index 00000000..f31a80b8 --- /dev/null +++ b/app/Http/Controllers/Api/CooperativeCancellationController.php @@ -0,0 +1,201 @@ + 'cooperative_cancellation_not_supported', + 'minimum_protocol_version' => CooperativeCancellationPolicy::MINIMUM_PROTOCOL_VERSION, + ], 409); + } + + $namespace = (string) $request->attributes->get('namespace'); + if (! NamespaceWorkflowScope::workflowBound($namespace, $workflowId)) { + return ControlPlaneProtocol::jsonForRequest($request, ['reason' => 'instance_not_found'], 404); + } + + $validated = $request->validate([ + 'reason' => ['nullable', 'string', 'max:1000'], + 'cleanup_timeout_seconds' => ['nullable', 'integer', 'min:1', 'max:3600'], + ]); + + $result = $this->controlMutations->run(fn (): array => DB::transaction(function () use ( + $request, $namespace, $workflowId, $runId, $validated, + ): array { + $instance = WorkflowInstance::query()->where('namespace', $namespace) + ->lockForUpdate()->find($workflowId); + if (! $instance instanceof WorkflowInstance) { + return ['reason' => 'instance_not_found', 'http_status' => 404]; + } + + $run = NamespaceWorkflowScope::runQuery($namespace, $workflowId) + ->lockForUpdate()->find($runId ?? $instance->current_run_id); + if (! $run instanceof WorkflowRun) { + return ['reason' => 'run_not_found', 'http_status' => 404]; + } + if ($run->id !== $instance->current_run_id) { + return ['reason' => 'selected_run_not_current', 'http_status' => 409]; + } + if (LegacyV1Projection::isProjectedRun($run)) { + return ['reason' => 'legacy_v1_operation_not_supported', 'http_status' => 409]; + } + + // Duplicates keep the original identity and deadline even after + // cleanup ended or the worker registration changed. + if (($pending = CooperativeCancellationPolicy::pending($run)) !== null) { + return [ + 'workflow_id' => $workflowId, + 'run_id' => $run->id, + 'accepted' => true, + 'duplicate' => true, + 'run_status' => $run->status->value, + 'cancellation_request' => $pending, + 'http_status' => 200, + ]; + } + if ($run->status->isTerminal()) { + return ['reason' => 'run_not_active', 'run_status' => $run->status->value, 'http_status' => 409]; + } + + $claims = NamespaceWorkflowScope::taskQuery($namespace) + ->where('workflow_tasks.workflow_run_id', $run->id) + ->where('workflow_tasks.task_type', TaskType::Workflow->value) + ->where('workflow_tasks.status', TaskStatus::Leased->value) + ->where('workflow_tasks.lease_expires_at', '>', now()) + ->lockForUpdate()->get(); + foreach ($claims as $claim) { + if (! CooperativeCancellationPolicy::claimSupportsCancellation($claim)) { + return [ + 'reason' => 'active_claim_cancellation_not_supported', + 'task_id' => $claim->id, + 'workflow_task_attempt' => $claim->attempt_count, + 'http_status' => 409, + ]; + } + } + + $command = WorkflowStub::loadSelection($workflowId, $runId, $namespace) + ->withCommandContext($this->contexts->make($request, $workflowId, 'request_cancellation')) + ->attemptRequestCancellation( + $validated['reason'] ?? null, + $validated['cleanup_timeout_seconds'] ?? 600, + ); + $run->refresh(); + + return [ + 'workflow_id' => $workflowId, + 'run_id' => $run->id, + 'accepted' => $command->accepted(), + 'duplicate' => false, + 'run_status' => $run->status->value, + 'cancellation_request' => CooperativeCancellationPolicy::pending($run), + 'reason' => $command->rejectionReason(), + 'http_status' => $command->accepted() ? 202 : 409, + ]; + })); + + if (($result['accepted'] ?? false) === true) { + app(LongPollSignalStore::class)->signalWorkflowTaskQueuesForWorkflow($workflowId, $namespace); + } + $status = $result['http_status']; + unset($result['http_status']); + + return ControlPlaneProtocol::jsonForRequest($request, $result, $status); + } + + public function deliver(Request $request, string $taskId): JsonResponse + { + if ($response = WorkerProtocol::rejectUnsupported($request)) { + return $response; + } + if (! WorkerProtocol::versionMeetsMinimum( + WorkerProtocol::requestVersion($request), CooperativeCancellationPolicy::MINIMUM_PROTOCOL_VERSION, + )) { + return WorkerProtocol::json(['reason' => 'cooperative_cancellation_not_supported'], 409); + } + + $validated = $request->validate([ + 'lease_owner' => ['required', 'string'], + 'workflow_task_attempt' => ['required', 'integer', 'min:1'], + 'request_id' => ['required', 'string'], + 'sequence' => ['required', 'integer', 'min:1'], + 'call_kind' => ['required', 'string', 'in:activity,local_activity,timer,condition,signal,child,parallel,selection_handle'], + 'sequence_span' => ['nullable', 'integer', 'min:1'], + 'operation_sequence' => ['nullable', 'integer', 'min:1'], + 'operation_sequence_span' => ['nullable', 'integer', 'min:1'], + ]); + $namespace = (string) $request->attributes->get('namespace'); + $bridge = app(WorkflowTaskBridge::class); + if (! $bridge instanceof CooperativeWorkflowTaskBridge) { + return WorkerProtocol::json(['reason' => 'cooperative_cancellation_not_supported'], 409); + } + + $result = $this->workerMutations->run(fn (): array => DB::transaction(function () use ( + $namespace, $taskId, $validated, $bridge, + ): array { + // Keep the ownership check and engine delivery under the same + // task lock. A reclaim cannot replace the checked attempt. + $task = NamespaceWorkflowScope::taskQuery($namespace)->lockForUpdate()->find($taskId); + if (! $task instanceof WorkflowTask) { + return ['delivered' => false, 'reason' => 'task_not_found']; + } + if ($task->lease_owner !== $validated['lease_owner']) { + return ['delivered' => false, 'reason' => 'lease_owner_mismatch']; + } + if ($task->attempt_count !== (int) $validated['workflow_task_attempt']) { + return ['delivered' => false, 'reason' => 'workflow_task_attempt_mismatch']; + } + if (! CooperativeCancellationPolicy::claimSupportsCancellation($task)) { + return ['delivered' => false, 'reason' => 'active_claim_cancellation_not_supported']; + } + + return $bridge->deliverCancellation( + $taskId, $validated['request_id'], (int) $validated['sequence'], $validated['call_kind'], + (int) ($validated['sequence_span'] ?? 1), + isset($validated['operation_sequence']) ? (int) $validated['operation_sequence'] : null, + (int) ($validated['operation_sequence_span'] ?? 1), + ); + })); + + return WorkerProtocol::json($result, ($result['delivered'] ?? false) ? 200 + : (($result['reason'] ?? null) === 'task_not_found' ? 404 : 409)); + } +} diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index db0d7501..af97ebc3 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -9,6 +9,7 @@ use App\Support\AvroPayloadEnvelopeResolver; use App\Support\BackendLockPressure; use App\Support\CachedPollTaskKindConflict; +use App\Support\CooperativeCancellationPolicy; use App\Support\ExternalPayloadStorageUnavailable; use App\Support\HistoryRetentionEnforcer; use App\Support\LongPollCapacityExhaustedException; @@ -3226,6 +3227,15 @@ public function heartbeatWorkflowTask(Request $request, string $taskId): JsonRes ); } + $observation = []; + if (($status['renewed'] ?? false) === true) { + $task = NamespaceWorkflowScope::task($namespace, $taskId); + if ($task?->run instanceof WorkflowRun + && ($pending = CooperativeCancellationPolicy::pending($task->run)) !== null) { + $observation['cancellation_request'] = $pending; + } + } + return WorkerProtocol::json([ 'task_id' => $taskId, 'workflow_task_attempt' => (int) $validated['workflow_task_attempt'], @@ -3235,6 +3245,7 @@ public function heartbeatWorkflowTask(Request $request, string $taskId): JsonRes 'run_status' => $status['run_status'], 'task_status' => $status['task_status'], 'reason' => $status['reason'], + ...$observation, ], $this->workflowOutcomeStatus($status['reason'])); } diff --git a/app/Support/CooperativeCancellationPolicy.php b/app/Support/CooperativeCancellationPolicy.php new file mode 100644 index 00000000..4ecc7f12 --- /dev/null +++ b/app/Support/CooperativeCancellationPolicy.php @@ -0,0 +1,78 @@ + */ + public static function workerSnapshot(WorkerRegistration $worker, ?string $protocolVersion): array + { + return [ + ...WorkerPollFence::snapshot($worker), + 'registration_id' => $worker->getKey(), + 'protocol_version' => $protocolVersion, + 'capabilities' => is_array($worker->capabilities) ? $worker->capabilities : [], + ]; + } + + /** @param array $snapshot */ + public static function bindClaim(WorkflowTask $task, array $snapshot): void + { + $payload = is_array($task->payload) ? $task->payload : []; + $payload[self::CLAIM_KEY] = [ + ...$snapshot, + 'task_id' => $task->id, + 'run_id' => $task->workflow_run_id, + 'lease_owner' => $task->lease_owner, + 'attempt' => $task->attempt_count, + ]; + $task->forceFill(['payload' => $payload])->save(); + } + + public static function claimSupportsCancellation(WorkflowTask $task): bool + { + $claim = $task->payload[self::CLAIM_KEY] ?? null; + + return is_array($claim) + && ($claim['task_id'] ?? null) === $task->id + && ($claim['run_id'] ?? null) === $task->workflow_run_id + && ($claim['namespace'] ?? null) === $task->namespace + && ($claim['worker_id'] ?? null) === $task->lease_owner + && ($claim['lease_owner'] ?? null) === $task->lease_owner + && ($claim['attempt'] ?? null) === $task->attempt_count + && self::supports($claim['capabilities'] ?? [], $claim['protocol_version'] ?? null); + } + + /** @param list $capabilities */ + public static function supports(array $capabilities, ?string $protocolVersion): bool + { + return in_array(self::CAPABILITY, $capabilities, true) + && WorkerProtocol::versionMeetsMinimum($protocolVersion, self::MINIMUM_PROTOCOL_VERSION); + } + + /** @return array|null */ + public static function pending(WorkflowRun $run): ?array + { + if (! is_string($run->cancellation_request_command_id)) { + return null; + } + + return [ + 'request_id' => $run->cancellation_request_command_id, + 'requested_at' => $run->cancellation_requested_at?->toISOString(), + 'cleanup_deadline_at' => $run->cancellation_deadline_at?->toISOString(), + 'delivery_sequence' => $run->cancellation_delivery_sequence, + 'delivered_at' => $run->cancellation_delivered_at?->toISOString(), + ]; + } +} diff --git a/app/Support/PollRequestLeaseBinding.php b/app/Support/PollRequestLeaseBinding.php index c901dced..b1307537 100644 --- a/app/Support/PollRequestLeaseBinding.php +++ b/app/Support/PollRequestLeaseBinding.php @@ -76,6 +76,7 @@ public function bindClaimedTask( string $taskId, string $leaseOwner, ?string $pollRequestId, + array $workerClaim = [], ): void { /** @var WorkflowTask|null $task */ $task = NamespaceWorkflowScope::taskQuery($namespace) @@ -100,5 +101,9 @@ public function bindClaimedTask( $task->forceFill([ 'payload' => $payload === [] ? null : $payload, ])->save(); + + if ($workerClaim !== []) { + CooperativeCancellationPolicy::bindClaim($task, $workerClaim); + } } } diff --git a/app/Support/WorkflowTaskPoller.php b/app/Support/WorkflowTaskPoller.php index 3152bcb5..9b222cb7 100644 --- a/app/Support/WorkflowTaskPoller.php +++ b/app/Support/WorkflowTaskPoller.php @@ -458,10 +458,10 @@ private function performPoll( 'poll_status' => 'empty', 'next_probe_at' => null, ]; - $workerPollFence = [ - ...WorkerPollFence::snapshot($worker), - 'protocol_version' => WorkerProtocol::requestVersion($request), - ]; + $workerPollFence = CooperativeCancellationPolicy::workerSnapshot( + $worker, + WorkerProtocol::requestVersion($request), + ); $supportsQueryTasks = in_array('workflow', $taskKinds, true) && $this->cache->available() && $this->queryTasks->workerSupportsQueryTasks($namespace, $worker); @@ -1070,6 +1070,16 @@ private function claimReadyTask( return ['claimed' => false, 'reason' => 'stale_worker_registration']; } + // Request admission takes the same run lock before inspecting + // leases. Either it observes this claim's immutable proof, or + // this claim observes the newly accepted request. + $run = WorkflowRun::query()->where('namespace', $namespace) + ->lockForUpdate()->find($runId); + + if (! $run instanceof WorkflowRun) { + return ['claimed' => false, 'reason' => 'run_not_found']; + } + if (! $this->workerCanReplayRun( $namespace, $leaseOwner, @@ -1096,6 +1106,7 @@ private function claimReadyTask( $taskId, $leaseOwner, $pollRequestId, + $workerPollFence, ); } @@ -1912,6 +1923,11 @@ private function refreshCachedTaskPayload(string $namespace, ?array $task): ?arr $payload['lease_expires_at'] = $workflowTask->lease_expires_at?->toJSON() ?? ($payload['lease_expires_at'] ?? null); + if ($workflowTask->run instanceof WorkflowRun + && ($pending = CooperativeCancellationPolicy::pending($workflowTask->run)) !== null) { + $payload['cancellation_request'] = $pending; + } + return $payload; } @@ -2062,6 +2078,11 @@ private function taskPayload( $payload = array_merge($payload, $this->workflowTaskResumeContext($namespace, (string) $claim['task_id'])); + $run = WorkflowRun::query()->where('namespace', $namespace)->find($claim['workflow_run_id']); + if ($run instanceof WorkflowRun && ($pending = CooperativeCancellationPolicy::pending($run)) !== null) { + $payload['cancellation_request'] = $pending; + } + // Include pagination metadata when history was fetched via // historyPayloadPaginated() so the controller can build page tokens. if (array_key_exists('has_more', $history)) { @@ -2312,6 +2333,12 @@ private function workerCanReplayRun( )) : []; + $run = WorkflowRun::query()->where('namespace', $namespace)->find($runId); + if ($run instanceof WorkflowRun && is_string($run->cancellation_request_command_id) + && ! CooperativeCancellationPolicy::supports($capabilities, $protocolVersion)) { + return false; + } + return WorkflowMetadataCapabilityPolicy::canReplayRun( $runId, $capabilities, diff --git a/routes/api.php b/routes/api.php index 77e400ca..1803003c 100644 --- a/routes/api.php +++ b/routes/api.php @@ -3,6 +3,7 @@ use App\Http\Controllers\Api\ActivityController; use App\Http\Controllers\Api\ActivityTaskController; use App\Http\Controllers\Api\BridgeAdapterController; +use App\Http\Controllers\Api\CooperativeCancellationController; use App\Http\Controllers\Api\DeploymentController; use App\Http\Controllers\Api\EmbeddedV2ImportController; use App\Http\Controllers\Api\HealthController; @@ -140,6 +141,7 @@ Route::post('/{workflowId}/query/{queryName}', [WorkflowController::class, 'query']); Route::post('/{workflowId}/update/{updateName}', [WorkflowController::class, 'update']); Route::post('/{workflowId}/cancel', [WorkflowController::class, 'cancel']); + Route::post('/{workflowId}/request-cancellation', [CooperativeCancellationController::class, 'request']); Route::post('/{workflowId}/terminate', [WorkflowController::class, 'terminate']); Route::post('/{workflowId}/repair', [WorkflowController::class, 'repair']); Route::post('/{workflowId}/archive', [WorkflowController::class, 'archive']); @@ -149,6 +151,7 @@ Route::post('/{workflowId}/runs/{runId}/query/{queryName}', [WorkflowController::class, 'queryRun']); Route::post('/{workflowId}/runs/{runId}/update/{updateName}', [WorkflowController::class, 'updateRun']); Route::post('/{workflowId}/runs/{runId}/cancel', [WorkflowController::class, 'cancelRun']); + Route::post('/{workflowId}/runs/{runId}/request-cancellation', [CooperativeCancellationController::class, 'request']); Route::post('/{workflowId}/runs/{runId}/terminate', [WorkflowController::class, 'terminateRun']); Route::post('/{workflowId}/runs/{runId}/repair', [WorkflowController::class, 'repairRun']); Route::post('/{workflowId}/runs/{runId}/redrive', [WorkflowController::class, 'redriveRun']); @@ -230,6 +233,7 @@ Route::post('/workflow-tasks/poll', [WorkerController::class, 'pollWorkflowTasks']); Route::post('/workflow-tasks/{taskId}/history', [WorkerController::class, 'workflowTaskHistory']); Route::post('/workflow-tasks/{taskId}/heartbeat', [WorkerController::class, 'heartbeatWorkflowTask']); + Route::post('/workflow-tasks/{taskId}/deliver-cancellation', [CooperativeCancellationController::class, 'deliver']); Route::post('/workflow-tasks/{taskId}/complete', [WorkerController::class, 'completeWorkflowTask']); Route::post('/workflow-tasks/{taskId}/fail', [WorkerController::class, 'failWorkflowTask']); diff --git a/tests/Feature/CooperativeCancellationProtocolTest.php b/tests/Feature/CooperativeCancellationProtocolTest.php new file mode 100644 index 00000000..f8b33e36 --- /dev/null +++ b/tests/Feature/CooperativeCancellationProtocolTest.php @@ -0,0 +1,215 @@ + '1.20', + 'workflows.v2.types.workflows' => ['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class], + ]); + WorkflowNamespace::query()->create([ + 'name' => 'default', 'description' => 'Test', 'retention_days' => 30, 'status' => 'active', + ]); + } + + public function test_request_before_claim_routes_only_to_capable_workers_and_keeps_original_deadline(): void + { + [$workflowId, $runId] = $this->start(); + $this->register('old', false); + $this->register('new', true); + $accepted = $this->requestCancellation($workflowId, ['cleanup_timeout_seconds' => 120]); + $accepted->assertStatus(202)->assertJsonPath('accepted', true)->assertJsonPath('run_id', $runId); + $original = $accepted->json('cancellation_request'); + $this->assertFalse(WorkflowRun::query()->findOrFail($runId)->status->isTerminal()); + $this->poll('old', '1.19')->assertOk()->assertJsonPath('task', null); + $this->poll('new')->assertOk()->assertJsonPath('task.cancellation_request', $original); + $this->travel(10)->seconds(); + $this->requestCancellation($workflowId, ['cleanup_timeout_seconds' => 3600]) + ->assertOk()->assertJsonPath('duplicate', true)->assertJsonPath('cancellation_request', $original); + $this->assertSame(1, $this->eventCount($runId, HistoryEventType::CooperativeCancellationRequested)); + } + + public function test_active_claim_observes_request_on_heartbeat_and_cached_poll_without_losing_lease(): void + { + [$workflowId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new', pollRequestId: 'repeat')->json('task'); + $accepted = $this->requestCancellation($workflowId)->assertStatus(202); + $pending = $accepted->json('cancellation_request'); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/heartbeat", [ + 'lease_owner' => 'new', 'workflow_task_attempt' => $task['workflow_task_attempt'], + ])->assertOk()->assertJsonPath('renewed', true)->assertJsonPath('cancellation_request', $pending); + $this->poll('new', pollRequestId: 'repeat')->assertOk() + ->assertJsonPath('task.task_id', $task['task_id'])->assertJsonPath('task.cancellation_request', $pending); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + } + + public function test_request_refuses_incompatible_claim_without_partial_command_or_history(): void + { + [$workflowId, $runId] = $this->start(); + $this->register('old', false); + $this->poll('old', '1.19')->assertOk(); + $before = WorkflowHistoryEvent::query()->where('workflow_run_id', $runId)->count(); + $this->requestCancellation($workflowId)->assertStatus(409) + ->assertJsonPath('reason', 'active_claim_cancellation_not_supported'); + $this->assertNull(WorkflowRun::query()->findOrFail($runId)->cancellation_request_command_id); + $this->assertSame($before, WorkflowHistoryEvent::query()->where('workflow_run_id', $runId)->count()); + } + + public function test_reregistration_cannot_upgrade_an_old_claim(): void + { + [$workflowId] = $this->start(); + $this->register('same-id', false); + $this->poll('same-id', '1.19')->assertOk(); + $this->register('same-id', true); + $this->requestCancellation($workflowId)->assertStatus(409) + ->assertJsonPath('reason', 'active_claim_cancellation_not_supported'); + } + + public function test_reregistration_does_not_erase_a_capable_claims_original_proof(): void + { + [$workflowId] = $this->start(); + $this->register('same-id', true); + $task = $this->poll('same-id')->json('task'); + $this->register('same-id', false); + $this->requestCancellation($workflowId)->assertStatus(202); + $this->assertTrue(CooperativeCancellationPolicy::claimSupportsCancellation( + WorkflowTask::query()->findOrFail($task['task_id']), + )); + } + + public function test_delivery_retry_records_one_marker_and_keeps_cleanup_authority(): void + { + [$workflowId, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $requestId = $this->requestCancellation($workflowId)->json('cancellation_request.request_id'); + $this->deliver($task, $requestId)->assertOk()->assertJsonPath('delivered', true)->assertJsonPath('sequence', 1); + $this->deliver($task, $requestId)->assertOk()->assertJsonPath('delivered', true); + $this->assertSame(1, $this->eventCount($runId, HistoryEventType::CooperativeCancellationDelivered)); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + $this->deliver($task, $requestId, ['call_kind' => 'timer'])->assertStatus(409) + ->assertJsonPath('reason', 'cancellation_delivery_mismatch'); + } + + public function test_delivery_fences_the_actual_claim_attempt_owner_and_request(): void + { + [$workflowId, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $requestId = $this->requestCancellation($workflowId)->json('cancellation_request.request_id'); + $this->deliver($task, $requestId, ['lease_owner' => 'other'])->assertStatus(409) + ->assertJsonPath('reason', 'lease_owner_mismatch'); + $this->deliver($task, $requestId, ['workflow_task_attempt' => 2])->assertStatus(409) + ->assertJsonPath('reason', 'workflow_task_attempt_mismatch'); + $this->deliver($task, 'different')->assertStatus(409)->assertJsonPath('reason', 'cancellation_request_mismatch'); + $this->deliver($task, $requestId, ['sequence' => 9])->assertStatus(409) + ->assertJsonPath('reason', 'cancellation_delivery_sequence_mismatch'); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::CooperativeCancellationDelivered)); + } + + public function test_cleanup_deadline_revokes_delivery_authority(): void + { + [$workflowId, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $requestId = $this->requestCancellation($workflowId, ['cleanup_timeout_seconds' => 1]) + ->json('cancellation_request.request_id'); + $this->travel(2)->seconds(); + $this->deliver($task, $requestId)->assertStatus(409)->assertJsonPath('reason', 'run_cancelled'); + $this->assertSame('cancelled', WorkflowRun::query()->findOrFail($runId)->status->value); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::CooperativeCancellationDelivered)); + } + + public function test_terminal_cancel_keeps_its_existing_semantics_and_request_is_separate(): void + { + [$workflowId, $runId] = $this->start(); + $this->withHeaders($this->controlHeaders())->postJson("/api/workflows/{$workflowId}/cancel", [])->assertOk(); + $this->requestCancellation($workflowId)->assertStatus(409)->assertJsonPath('reason', 'run_not_active'); + $this->assertSame('cancelled', WorkflowRun::query()->findOrFail($runId)->status->value); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::CooperativeCancellationRequested)); + } + + public function test_request_is_unavailable_until_the_server_protocol_supports_it(): void + { + [$workflowId] = $this->start(); + config(['server.worker_protocol.version' => '1.19']); + $this->requestCancellation($workflowId)->assertStatus(409) + ->assertJsonPath('reason', 'cooperative_cancellation_not_supported'); + } + + private function start(): array + { + $response = $this->withHeaders($this->controlHeaders())->postJson('/api/workflows', [ + 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'cooperative', 'input' => ['Ada'], + ])->assertCreated(); + + return [$response->json('workflow_id'), $response->json('run_id')]; + } + + private function register(string $workerId, bool $capable): void + { + WorkerRegistration::query()->updateOrCreate(['namespace' => 'default', 'worker_id' => $workerId], [ + 'task_queue' => 'cooperative', 'runtime' => 'php', 'supported_workflow_types' => ['tests.external-greeting-workflow'], + 'capabilities' => $capable ? [CooperativeCancellationPolicy::CAPABILITY] : [], + 'max_concurrent_workflow_tasks' => 1, 'last_heartbeat_at' => now(), 'status' => 'active', + ]); + } + + private function poll(string $workerId, string $version = '1.20', ?string $pollRequestId = null): TestResponse + { + return $this->withHeaders($this->headers($version))->postJson('/api/worker/workflow-tasks/poll', array_filter([ + 'worker_id' => $workerId, 'task_queue' => 'cooperative', 'poll_request_id' => $pollRequestId, + ], static fn ($value) => $value !== null)); + } + + private function requestCancellation(string $workflowId, array $body = []): TestResponse + { + return $this->withHeaders($this->controlHeaders())->postJson("/api/workflows/{$workflowId}/request-cancellation", $body); + } + + private function deliver(array $task, string $requestId, array $overrides = []): TestResponse + { + return $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/deliver-cancellation", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'request_id' => $requestId, 'sequence' => 1, 'call_kind' => 'activity', ...$overrides, + ]); + } + + private function eventCount(string $runId, HistoryEventType $type): int + { + return WorkflowHistoryEvent::query()->where('workflow_run_id', $runId)->where('event_type', $type->value)->count(); + } + + private function headers(string $version = '1.20'): array + { + return ['X-Namespace' => 'default', WorkerProtocol::HEADER => $version]; + } + + private function controlHeaders(): array + { + return ['X-Namespace' => 'default', 'X-Durable-Workflow-Control-Plane-Version' => '2']; + } +} diff --git a/tests/Feature/WorkerProtocolVersionCoverageTest.php b/tests/Feature/WorkerProtocolVersionCoverageTest.php index 467eb4be..d311a0e0 100644 --- a/tests/Feature/WorkerProtocolVersionCoverageTest.php +++ b/tests/Feature/WorkerProtocolVersionCoverageTest.php @@ -42,6 +42,7 @@ public static function workerEndpointProvider(): array 'workflow-tasks.poll' => ['method' => 'post', 'path' => '/api/worker/workflow-tasks/poll'], 'workflow-tasks.history' => ['method' => 'post', 'path' => '/api/worker/workflow-tasks/task-1/history'], 'workflow-tasks.heartbeat' => ['method' => 'post', 'path' => '/api/worker/workflow-tasks/task-1/heartbeat'], + 'workflow-tasks.deliver-cancellation' => ['method' => 'post', 'path' => '/api/worker/workflow-tasks/task-1/deliver-cancellation'], 'workflow-tasks.complete' => ['method' => 'post', 'path' => '/api/worker/workflow-tasks/task-1/complete'], 'workflow-tasks.fail' => ['method' => 'post', 'path' => '/api/worker/workflow-tasks/task-1/fail'], 'query-tasks.poll' => ['method' => 'post', 'path' => '/api/worker/query-tasks/poll'], From ec352c57e34a204f3ecac202a1ec23262f58b3d5 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 30 Sep 2026 23:12:26 +0000 Subject: [PATCH 02/57] test: qualify cooperative request races and cold cleanup recovery --- .../Api/CooperativeCancellationController.php | 23 ++- app/Http/Controllers/Api/WorkerController.php | 10 + app/Support/PollRequestLeaseBinding.php | 1 + .../control-plane-api.openapi.yaml | 70 ++++++- .../worker-protocol-api.openapi.yaml | 116 ++++++++++- .../worker-protocol-stream.asyncapi.yaml | 33 +++- .../CooperativeCancellationProtocolTest.php | 186 +++++++++++++++++- .../CooperativeCancellationRaceTest.php | 154 +++++++++++++++ tests/Feature/WorkflowWorkerProtocolTest.php | 20 ++ .../CooperativeCancellationRaceRequest.php | 67 +++++++ 10 files changed, 665 insertions(+), 15 deletions(-) create mode 100644 tests/Feature/CooperativeCancellationRaceTest.php create mode 100644 tests/Fixtures/CooperativeCancellationRaceRequest.php diff --git a/app/Http/Controllers/Api/CooperativeCancellationController.php b/app/Http/Controllers/Api/CooperativeCancellationController.php index f31a80b8..03e24af0 100644 --- a/app/Http/Controllers/Api/CooperativeCancellationController.php +++ b/app/Http/Controllers/Api/CooperativeCancellationController.php @@ -2,12 +2,14 @@ namespace App\Http\Controllers\Api; +use App\Models\WorkerRegistration; use App\Support\ControlPlaneMutationRetrier; use App\Support\ControlPlaneProtocol; use App\Support\CooperativeCancellationPolicy; use App\Support\LegacyV1Projection; use App\Support\LongPollSignalStore; use App\Support\NamespaceWorkflowScope; +use App\Support\WorkerPollFence; use App\Support\WorkerProtocol; use App\Support\WorkerProtocolMutationRetrier; use App\Support\WorkflowCommandContextFactory; @@ -42,6 +44,7 @@ public function request(Request $request, string $workflowId, ?string $runId = n CooperativeCancellationPolicy::MINIMUM_PROTOCOL_VERSION, )) { return ControlPlaneProtocol::jsonForRequest($request, [ + 'message' => 'This Server does not support cooperative cancellation requests.', 'reason' => 'cooperative_cancellation_not_supported', 'minimum_protocol_version' => CooperativeCancellationPolicy::MINIMUM_PROTOCOL_VERSION, ], 409); @@ -49,7 +52,7 @@ public function request(Request $request, string $workflowId, ?string $runId = n $namespace = (string) $request->attributes->get('namespace'); if (! NamespaceWorkflowScope::workflowBound($namespace, $workflowId)) { - return ControlPlaneProtocol::jsonForRequest($request, ['reason' => 'instance_not_found'], 404); + return ControlPlaneProtocol::jsonForRequest($request, ['message' => 'Workflow not found.', 'reason' => 'instance_not_found'], 404); } $validated = $request->validate([ @@ -130,13 +133,23 @@ public function request(Request $request, string $workflowId, ?string $runId = n 'reason' => $command->rejectionReason(), 'http_status' => $command->accepted() ? 202 : 409, ]; - })); + }), allBackends: true); if (($result['accepted'] ?? false) === true) { app(LongPollSignalStore::class)->signalWorkflowTaskQueuesForWorkflow($workflowId, $namespace); } $status = $result['http_status']; unset($result['http_status']); + if (($result['accepted'] ?? false) !== true) { + $result['message'] = match ($result['reason'] ?? null) { + 'active_claim_cancellation_not_supported' => 'The active workflow-task claim cannot perform cooperative cancellation. Retry after a capable worker claims the task.', + 'selected_run_not_current' => 'The selected run is no longer the current run.', + 'run_not_active' => 'The workflow run is already closed.', + 'legacy_v1_operation_not_supported' => 'Imported legacy workflows do not support cooperative cancellation.', + 'run_not_found' => 'Workflow run not found.', + default => 'Workflow not found.', + }; + } return ControlPlaneProtocol::jsonForRequest($request, $result, $status); } @@ -187,6 +200,12 @@ public function deliver(Request $request, string $taskId): JsonResponse return ['delivered' => false, 'reason' => 'active_claim_cancellation_not_supported']; } + $worker = WorkerRegistration::query()->where('namespace', $namespace) + ->where('worker_id', $validated['lease_owner'])->first(); + if (! $worker instanceof WorkerRegistration || ! WorkerPollFence::isFresh($worker)) { + return ['delivered' => false, 'reason' => 'stale_worker_registration']; + } + return $bridge->deliverCancellation( $taskId, $validated['request_id'], (int) $validated['sequence'], $validated['call_kind'], (int) ($validated['sequence_span'] ?? 1), diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index af97ebc3..82d9e82b 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -224,6 +224,16 @@ public function register(Request $request): JsonResponse ]); $workerCapabilities = $this->nonEmptyStringArray($validated['capabilities'] ?? []); + if (in_array(CooperativeCancellationPolicy::CAPABILITY, $workerCapabilities, true) + && ! CooperativeCancellationPolicy::supports($workerCapabilities, WorkerProtocol::requestVersion($request))) { + return WorkerProtocol::json([ + 'registered' => false, + 'reason' => 'cooperative_cancellation_protocol_mismatch', + 'minimum_protocol_version' => CooperativeCancellationPolicy::MINIMUM_PROTOCOL_VERSION, + 'requested_version' => WorkerProtocol::requestVersion($request), + ], 409); + } + $capabilityManifest = $this->portableWorkerCapabilityManifest( is_array($validated['capability_manifest'] ?? null) ? $validated['capability_manifest'] : [], ); diff --git a/app/Support/PollRequestLeaseBinding.php b/app/Support/PollRequestLeaseBinding.php index b1307537..7cfcd5f1 100644 --- a/app/Support/PollRequestLeaseBinding.php +++ b/app/Support/PollRequestLeaseBinding.php @@ -71,6 +71,7 @@ public function ensureUnbound( } } + /** @param array $workerClaim */ public function bindClaimedTask( string $namespace, string $taskId, diff --git a/resources/platform-protocol-specs/control-plane-api.openapi.yaml b/resources/platform-protocol-specs/control-plane-api.openapi.yaml index ff6aee5b..8b5ae1ca 100644 --- a/resources/platform-protocol-specs/control-plane-api.openapi.yaml +++ b/resources/platform-protocol-specs/control-plane-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.control-plane-api - version: "10" + version: "11" summary: Durable Workflow control-plane HTTP+JSON API description: > Normative OpenAPI specification for the standalone Durable Workflow @@ -318,6 +318,47 @@ paths: requestBody: { $ref: "#/components/requestBodies/ReasonObject" } responses: "202": { $ref: "#/components/responses/ControlPlaneEnvelope" } + /workflows/{workflowId}/request-cancellation: + post: + tags: [workflows] + operationId: requestWorkflowCancellation + summary: Request cooperative cancellation with bounded workflow cleanup. + x-durable-workflow-minimum-worker-protocol-version: "1.20" + parameters: + - $ref: "#/components/parameters/ControlPlaneVersionHeader" + - $ref: "#/components/parameters/WorkflowIdPath" + requestBody: + required: false + content: + application/json: + schema: { $ref: "#/components/schemas/CooperativeCancellationAdmissionRequest" } + responses: + "200": { $ref: "#/components/responses/CooperativeCancellationAdmission" } + "202": { $ref: "#/components/responses/CooperativeCancellationAdmission" } + "404": { $ref: "#/components/responses/ControlPlaneError" } + "409": { $ref: "#/components/responses/ControlPlaneError" } + "422": { $ref: "#/components/responses/ControlPlaneError" } + /workflows/{workflowId}/runs/{runId}/request-cancellation: + post: + tags: [workflows] + operationId: requestWorkflowRunCancellation + summary: Request cooperative cancellation of the selected current run. + x-durable-workflow-minimum-worker-protocol-version: "1.20" + parameters: + - $ref: "#/components/parameters/ControlPlaneVersionHeader" + - $ref: "#/components/parameters/WorkflowIdPath" + - $ref: "#/components/parameters/RunIdPath" + requestBody: + required: false + content: + application/json: + schema: { $ref: "#/components/schemas/CooperativeCancellationAdmissionRequest" } + responses: + "200": { $ref: "#/components/responses/CooperativeCancellationAdmission" } + "202": { $ref: "#/components/responses/CooperativeCancellationAdmission" } + "404": { $ref: "#/components/responses/ControlPlaneError" } + "409": { $ref: "#/components/responses/ControlPlaneError" } + "422": { $ref: "#/components/responses/ControlPlaneError" } /workflows/{workflowId}/terminate: post: tags: [workflows] @@ -900,6 +941,14 @@ components: content: application/json: schema: { $ref: "#/components/schemas/JsonObject" } + CooperativeCancellationAdmission: + description: The original request and immutable cleanup deadline, including duplicate acceptance. + headers: + X-Durable-Workflow-Control-Plane-Version: + schema: { type: string, const: "2" } + content: + application/json: + schema: { $ref: "#/components/schemas/CooperativeCancellationAdmission" } ControlPlaneEnvelope: description: Successful control-plane response. headers: @@ -933,6 +982,25 @@ components: application/json: schema: { $ref: "#/components/schemas/ScheduleListError" } schemas: + CooperativeCancellationAdmissionRequest: + type: object + additionalProperties: true + properties: + reason: { type: [string, "null"], maxLength: 1000 } + cleanup_timeout_seconds: { type: [integer, "null"], minimum: 1, maximum: 3600, default: 600 } + CooperativeCancellationAdmission: + allOf: + - $ref: "#/components/schemas/ControlPlaneEnvelope" + - type: object + required: [workflow_id, run_id, accepted, duplicate, run_status, cancellation_request] + properties: + workflow_id: { type: string, minLength: 1 } + run_id: { type: string, minLength: 1 } + accepted: { type: boolean, const: true } + duplicate: { type: boolean } + run_status: { type: string } + cancellation_request: + $ref: "./worker-protocol-api.openapi.yaml#/components/schemas/CooperativeCancellationRequest" JsonObject: type: object additionalProperties: true diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 33f9e658..aa060dde 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "23" + version: "24" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -23,6 +23,22 @@ x-durable-workflow-catalog-entry: worker_protocol_api x-durable-workflow-catalog-schema: durable-workflow.v2.platform-protocol-specs.catalog x-durable-workflow-catalog-version: 16 x-durable-workflow-evolution-rule: additive_minor_breaking_major +x-durable-workflow-cooperative-cancellation-contract: + minimum_protocol_version: "1.20" + worker_capability: cooperative_cancellation + request_operation: POST /workflows/{workflowId}/request-cancellation + observation_field: cancellation_request + observation_surfaces: [workflow_task_claim, successful_workflow_task_heartbeat] + delivery_operation: POST /worker/workflow-tasks/{taskId}/deliver-cancellation + canonical_history_event: CooperativeCancellationDelivered + delivery_identity: [request_id, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span] + claim_proof: immutable_registration_snapshot_bound_to_task_run_owner_and_attempt + request_against_incompatible_active_claim: { status: 409, reason: active_claim_cancellation_not_supported, history_appended: false } + request_claim_race: serialized_on_workflow_run_lock + prior_completed_calls: replay_normally_before_canonical_delivery_boundary + cleanup_deadline: immutable_original_request_deadline + cleanup_completion: cancelled_with_original_request_id + terminal_cancel_and_terminate: unchanged x-durable-workflow-message-streams-contract: discovery_path: /cluster/info#/message_streams_contract minimum_protocol_version: "1.15" @@ -342,12 +358,39 @@ paths: required: false content: application/json: - schema: { $ref: "#/components/schemas/TaskHeartbeatRequest" } + schema: { $ref: "#/components/schemas/WorkflowTaskHeartbeatRequest" } responses: - "200": { $ref: "#/components/responses/WorkerEnvelope" } + "200": + description: Lease renewal with optional durable cancellation observation. + content: + application/json: + schema: { $ref: "#/components/schemas/WorkflowTaskHeartbeatResponse" } "404": { $ref: "#/components/responses/WorkerError" } "409": { $ref: "#/components/responses/WorkerError" } "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/deliver-cancellation: + post: + operationId: deliverWorkflowCancellation + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: cooperative_cancellation + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/CooperativeCancellationDeliveryRequest" } + responses: + "200": + description: Canonical delivery committed or an identical acknowledgment retried. + content: + application/json: + schema: { $ref: "#/components/schemas/CooperativeCancellationDeliveryResponse" } + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } /worker/workflow-tasks/{taskId}/complete: post: operationId: completeWorkflowTask @@ -1124,6 +1167,7 @@ components: local_activities: "1.18" worker_sessions: "1.18" sticky_execution: "1.18" + cooperative_cancellation: "1.20" capability_manifest: $ref: "#/components/schemas/PortableWorkerCapabilityManifest" WorkerHeartbeatRequest: @@ -1255,6 +1299,48 @@ components: lease_owner: { type: string, minLength: 1 } lease_expires_at: { type: string, format: date-time } history_events: { type: array, items: { $ref: "#/components/schemas/WorkerHistoryEvent" } } + cancellation_request: + $ref: "#/components/schemas/CooperativeCancellationRequest" + x-durable-workflow-minimum-protocol-version: "1.20" + CooperativeCancellationRequest: + type: object + additionalProperties: false + required: [request_id, requested_at, cleanup_deadline_at, delivery_sequence, delivered_at] + properties: + request_id: { type: string, minLength: 1 } + requested_at: { type: string, format: date-time } + cleanup_deadline_at: { type: string, format: date-time } + delivery_sequence: { type: [integer, "null"], minimum: 1 } + delivered_at: { type: [string, "null"], format: date-time } + CooperativeCancellationDeliveryRequest: + type: object + additionalProperties: true + required: [lease_owner, workflow_task_attempt, request_id, sequence, call_kind] + properties: + lease_owner: { type: string, minLength: 1 } + workflow_task_attempt: { type: integer, minimum: 1 } + request_id: { type: string, minLength: 1 } + sequence: { type: integer, minimum: 1 } + call_kind: { type: string, enum: [activity, local_activity, timer, condition, signal, child, parallel, selection_handle] } + sequence_span: { type: [integer, "null"], minimum: 1, default: 1 } + operation_sequence: { type: [integer, "null"], minimum: 1 } + operation_sequence_span: { type: [integer, "null"], minimum: 1, default: 1 } + CooperativeCancellationDeliveryResponse: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - type: object + required: [delivered, task_id, workflow_run_id, request_id, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span, reason] + properties: + delivered: { type: boolean, const: true } + task_id: { type: string, minLength: 1 } + workflow_run_id: { type: string, minLength: 1 } + request_id: { type: string, minLength: 1 } + sequence: { type: integer, minimum: 1 } + call_kind: { type: string, enum: [activity, local_activity, timer, condition, signal, child, parallel, selection_handle] } + sequence_span: { type: integer, minimum: 1 } + operation_sequence: { type: [integer, "null"], minimum: 1 } + operation_sequence_span: { type: integer, minimum: 1 } + reason: { type: "null" } WorkerHistoryEvent: type: object additionalProperties: true @@ -1518,6 +1604,30 @@ components: properties: worker_id: { type: string } progress: true + WorkflowTaskHeartbeatRequest: + type: object + additionalProperties: true + required: [lease_owner, workflow_task_attempt] + properties: + lease_owner: { type: string, minLength: 1 } + workflow_task_attempt: { type: integer, minimum: 1 } + WorkflowTaskHeartbeatResponse: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - type: object + required: [task_id, workflow_task_attempt, lease_owner, renewed, lease_expires_at, run_status, task_status, reason] + properties: + task_id: { type: string, minLength: 1 } + workflow_task_attempt: { type: integer, minimum: 1 } + lease_owner: { type: string, minLength: 1 } + renewed: { type: boolean } + lease_expires_at: { type: [string, "null"], format: date-time } + run_status: { type: [string, "null"] } + task_status: { type: [string, "null"] } + reason: { type: [string, "null"] } + cancellation_request: + $ref: "#/components/schemas/CooperativeCancellationRequest" + x-durable-workflow-minimum-protocol-version: "1.20" WorkflowTaskCompleteRequest: type: object additionalProperties: true diff --git a/resources/platform-protocol-specs/worker-protocol-stream.asyncapi.yaml b/resources/platform-protocol-specs/worker-protocol-stream.asyncapi.yaml index c933ad78..429cead1 100644 --- a/resources/platform-protocol-specs/worker-protocol-stream.asyncapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-stream.asyncapi.yaml @@ -2,7 +2,7 @@ asyncapi: 2.6.0 id: durable-workflow.v2.worker-protocol-stream info: title: Durable Workflow worker-protocol long-poll stream semantics - version: "12" + version: "13" description: > Normative AsyncAPI description of the public worker-protocol stream semantics. Durable uses HTTP long-poll request/response rather than a @@ -21,6 +21,37 @@ x-durable-workflow-catalog-entry: worker_protocol_stream x-durable-workflow-catalog-schema: durable-workflow.v2.platform-protocol-specs.catalog x-durable-workflow-catalog-version: 16 x-durable-workflow-evolution-rule: additive_minor_breaking_major +x-durable-workflow-cooperative-cancellation-contract: + minimum_protocol_version: "1.20" + worker_capability: cooperative_cancellation + request: + path: /workflows/{workflowId}/request-cancellation + run_remains_active: true + duplicates: return_original_request_and_deadline + incompatible_active_claim: reject_without_history_or_command + request_claim_race: serialized_on_workflow_run_lock + observation: + field: cancellation_request + surfaces: [workflow_task_claim, successful_workflow_task_heartbeat] + identity: [request_id, requested_at, cleanup_deadline_at] + observation_alone: does_not_throw_before_authored_delivery_boundary + delivery: + path: /worker/workflow-tasks/{taskId}/deliver-cancellation + lease_fences: [namespace, task_id, lease_owner, workflow_task_attempt, unexpired_lease] + capability_proof: immutable_registration_snapshot_at_claim + history_event: CooperativeCancellationDelivered + retry_identity: [request_id, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span] + prior_completed_calls: replay_normally + marker_retry: exactly_one_canonical_event + workflow_lease: retained_for_bounded_cleanup + affected_open_wait: cancelled_on_committed_delivery + cleanup: + deadline: original_immutable_cleanup_deadline_at + cold_restart: replay_canonical_delivery_and_completed_cleanup_work + completion_outcome: cancelled_with_original_request_id + expiry_or_termination: revoke_remaining_authority + external_effects: at_least_once_with_application_idempotency_or_reconciliation + terminal_cancel_and_terminate: unchanged x-durable-workflow-message-streams-contract: discovery_path: /cluster/info#/message_streams_contract minimum_protocol_version: "1.15" diff --git a/tests/Feature/CooperativeCancellationProtocolTest.php b/tests/Feature/CooperativeCancellationProtocolTest.php index f8b33e36..4b7098e1 100644 --- a/tests/Feature/CooperativeCancellationProtocolTest.php +++ b/tests/Feature/CooperativeCancellationProtocolTest.php @@ -10,12 +10,15 @@ use Illuminate\Support\Facades\Queue; use Illuminate\Testing\TestResponse; use Tests\Fixtures\ExternalGreetingWorkflow; +use Tests\Support\OpenApiSchema; use Tests\TestCase; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\TaskStatus; +use Workflow\V2\Jobs\RunTimerTask; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; +use Workflow\V2\Models\WorkflowTimer; class CooperativeCancellationProtocolTest extends TestCase { @@ -78,22 +81,25 @@ public function test_request_refuses_incompatible_claim_without_partial_command_ $this->assertSame($before, WorkflowHistoryEvent::query()->where('workflow_run_id', $runId)->count()); } - public function test_reregistration_cannot_upgrade_an_old_claim(): void + public function test_mutating_registration_cannot_upgrade_an_old_claim(): void { [$workflowId] = $this->start(); $this->register('same-id', false); $this->poll('same-id', '1.19')->assertOk(); - $this->register('same-id', true); + WorkerRegistration::query()->where('worker_id', 'same-id')->firstOrFail()->forceFill([ + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY], + ])->save(); $this->requestCancellation($workflowId)->assertStatus(409) ->assertJsonPath('reason', 'active_claim_cancellation_not_supported'); } - public function test_reregistration_does_not_erase_a_capable_claims_original_proof(): void + public function test_mutating_registration_does_not_erase_a_capable_claims_original_proof(): void { [$workflowId] = $this->start(); $this->register('same-id', true); $task = $this->poll('same-id')->json('task'); - $this->register('same-id', false); + WorkerRegistration::query()->where('worker_id', 'same-id')->firstOrFail() + ->forceFill(['capabilities' => []])->save(); $this->requestCancellation($workflowId)->assertStatus(202); $this->assertTrue(CooperativeCancellationPolicy::claimSupportsCancellation( WorkflowTask::query()->findOrFail($task['task_id']), @@ -160,6 +166,167 @@ public function test_request_is_unavailable_until_the_server_protocol_supports_i ->assertJsonPath('reason', 'cooperative_cancellation_not_supported'); } + public function test_waiting_timer_is_interrupted_only_when_delivery_is_committed(): void + { + [$workflowId, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $this->complete($task, [['type' => 'start_timer', 'delay_seconds' => 300]]) + ->assertOk()->assertJsonPath('run_status', 'waiting'); + $requestId = $this->requestCancellation($workflowId)->assertStatus(202)->json('cancellation_request.request_id'); + $this->assertSame('pending', WorkflowTimer::query()->where('workflow_run_id', $runId)->firstOrFail()->status->value); + $resumed = $this->poll('new')->assertOk()->json('task'); + $this->deliver($resumed, $requestId, ['call_kind' => 'timer'])->assertOk(); + $this->assertSame('cancelled', WorkflowTimer::query()->where('workflow_run_id', $runId)->firstOrFail()->status->value); + $this->assertSame(1, $this->eventCount($runId, HistoryEventType::TimerCancelled)); + $this->complete($resumed, [['type' => 'complete_workflow']])->assertOk()->assertJsonPath('run_status', 'cancelled'); + $event = WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::WorkflowCancelled->value)->firstOrFail(); + $this->assertSame($requestId, $event->workflow_command_id); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::WorkflowCompleted)); + } + + public function test_prior_committed_result_cannot_become_the_cancellation_boundary(): void + { + [$workflowId, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $run = WorkflowRun::query()->findOrFail($runId); + // A completed timer represents a durable call resolved before the request. + WorkflowHistoryEvent::record($run, HistoryEventType::TimerScheduled, [ + 'sequence' => 1, 'timer_id' => 'completed-timer', 'delay_seconds' => 1, + ]); + WorkflowHistoryEvent::record($run, HistoryEventType::TimerFired, [ + 'sequence' => 1, 'timer_id' => 'completed-timer', + ]); + $requestId = $this->requestCancellation($workflowId)->json('cancellation_request.request_id'); + $this->deliver($task, $requestId, ['call_kind' => 'timer'])->assertStatus(409) + ->assertJsonPath('reason', 'cancellation_delivery_not_eligible'); + $this->deliver($task, $requestId, ['sequence' => 2])->assertOk()->assertJsonPath('sequence', 2); + $this->assertSame(1, $this->eventCount($runId, HistoryEventType::CooperativeCancellationDelivered)); + } + + public function test_termination_during_cleanup_revokes_late_delivery_and_completion(): void + { + [$workflowId, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $requestId = $this->requestCancellation($workflowId)->json('cancellation_request.request_id'); + $this->deliver($task, $requestId)->assertOk(); + $this->withHeaders($this->controlHeaders())->postJson("/api/workflows/{$workflowId}/terminate", [])->assertOk(); + $this->deliver($task, $requestId)->assertStatus(409)->assertJsonPath('reason', 'task_not_leased'); + $this->complete($task, [['type' => 'complete_workflow']])->assertStatus(409)->assertJsonPath('can_continue', false); + $this->assertSame('terminated', WorkflowRun::query()->findOrFail($runId)->status->value); + $this->assertSame(1, $this->eventCount($runId, HistoryEventType::WorkflowTerminated)); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::WorkflowCompleted)); + } + + public function test_claim_proof_cannot_be_reused_for_another_attempt(): void + { + [$workflowId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + WorkflowTask::query()->findOrFail($task['task_id'])->increment('attempt_count'); + $this->requestCancellation($workflowId)->assertStatus(409) + ->assertJsonPath('reason', 'active_claim_cancellation_not_supported'); + } + + public function test_real_reregistration_requires_a_new_claim_and_fences_the_old_attempt(): void + { + [$workflowId] = $this->start(); + $this->register('same-id', false); + $oldTask = $this->poll('same-id', '1.19')->json('task'); + $this->register('same-id', true, '2026-09-30T00:01:00Z'); + $requestId = $this->requestCancellation($workflowId)->assertStatus(202)->json('cancellation_request.request_id'); + $newTask = $this->poll('same-id')->assertOk()->json('task'); + $this->assertSame($oldTask['task_id'], $newTask['task_id']); + $this->assertSame(2, $newTask['workflow_task_attempt']); + $this->deliver($oldTask, $requestId)->assertStatus(409) + ->assertJsonPath('reason', 'workflow_task_attempt_mismatch'); + $this->deliver($newTask, $requestId)->assertOk(); + } + + public function test_cold_reclaim_preserves_delivery_and_can_commit_bounded_cleanup(): void + { + [$workflowId, $runId] = $this->start(); + $this->register('same-id', true); + $task = $this->poll('same-id')->json('task'); + $requestId = $this->requestCancellation($workflowId)->json('cancellation_request.request_id'); + $this->deliver($task, $requestId)->assertOk(); + // The process dies after the marker commits, before the acknowledgment + // or any cleanup submission. Re-registration is the real recovery path. + $this->register('same-id', true, '2026-09-30T00:01:00Z'); + $cold = $this->poll('same-id')->assertOk()->json('task'); + $this->assertSame(2, $cold['workflow_task_attempt']); + $markers = array_values(array_filter($cold['history_events'], static fn ($event) => $event['event_type'] === HistoryEventType::CooperativeCancellationDelivered->value)); + $this->assertCount(1, $markers); + $this->assertSame(1, $markers[0]['payload']['sequence']); + $this->deliver($cold, $requestId)->assertOk(); + $this->complete($cold, [['type' => 'start_timer', 'delay_seconds' => 1]]) + ->assertOk()->assertJsonPath('run_status', 'waiting'); + $timerTask = WorkflowTask::query()->where('workflow_run_id', $runId)->where('task_type', 'timer') + ->where('status', TaskStatus::Ready->value)->firstOrFail(); + $this->travel(2)->seconds(); + (new RunTimerTask($timerTask->id))->handle(); + $resumed = $this->poll('same-id')->assertOk()->json('task'); + $this->complete($resumed, [['type' => 'complete_workflow']])->assertOk()->assertJsonPath('run_status', 'cancelled'); + $this->assertSame(1, $this->eventCount($runId, HistoryEventType::CooperativeCancellationDelivered)); + $this->assertSame(1, $this->eventCount($runId, HistoryEventType::TimerScheduled)); + $this->assertSame(1, $this->eventCount($runId, HistoryEventType::WorkflowCancelled)); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::WorkflowCompleted)); + } + + public function test_registration_cannot_advertise_cooperation_below_its_protocol_floor(): void + { + $this->withHeaders($this->headers('1.19'))->postJson('/api/worker/register', [ + 'worker_id' => 'invalid', 'task_queue' => 'cooperative', 'runtime' => 'php', + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY], + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + ])->assertStatus(409)->assertJsonPath('reason', 'cooperative_cancellation_protocol_mismatch'); + $this->assertFalse(WorkerRegistration::query()->where('worker_id', 'invalid')->exists()); + } + + public function test_new_admission_observation_and_delivery_fields_match_the_versioned_schemas(): void + { + [$workflowId] = $this->start(); + $this->register('new', true); + $accepted = $this->requestCancellation($workflowId)->assertStatus(202); + $specDirectory = dirname(__DIR__, 2).'/resources/platform-protocol-specs/'; + OpenApiSchema::fromFile($specDirectory.'control-plane-api.openapi.yaml') + ->assertReferenceMatches('#/components/schemas/CooperativeCancellationAdmission', json_decode($accepted->getContent(), flags: JSON_THROW_ON_ERROR)); + $workerSpec = OpenApiSchema::fromFile($specDirectory.'worker-protocol-api.openapi.yaml'); + $poll = $this->poll('new')->assertOk(); + $task = $poll->json('task'); + $workerSpec->assertReferenceMatches('#/components/schemas/WorkflowTask', json_decode($poll->getContent(), flags: JSON_THROW_ON_ERROR)->task); + $heartbeat = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/heartbeat", [ + 'lease_owner' => 'new', 'workflow_task_attempt' => $task['workflow_task_attempt'], + ])->assertOk(); + $workerSpec->assertReferenceMatches('#/components/schemas/WorkflowTaskHeartbeatResponse/allOf/1', json_decode($heartbeat->getContent(), flags: JSON_THROW_ON_ERROR)); + $delivery = $this->deliver($task, $accepted->json('cancellation_request.request_id'))->assertOk(); + $workerSpec->assertReferenceMatches('#/components/schemas/CooperativeCancellationDeliveryResponse/allOf/1', json_decode($delivery->getContent(), flags: JSON_THROW_ON_ERROR)); + } + + public function test_run_target_and_namespace_are_checked_before_request_admission(): void + { + [$workflowId, $runId] = $this->start(); + $this->withHeaders($this->controlHeaders())->postJson( + "/api/workflows/{$workflowId}/runs/missing/request-cancellation", [], + )->assertNotFound()->assertJsonPath('reason', 'run_not_found'); + $accepted = $this->withHeaders($this->controlHeaders())->postJson( + "/api/workflows/{$workflowId}/runs/{$runId}/request-cancellation", [], + )->assertStatus(202); + $this->assertSame($runId, $accepted->json('run_id')); + $this->requestCancellation('missing')->assertNotFound()->assertJsonPath('reason', 'instance_not_found'); + } + + private function complete(array $task, array $commands): TestResponse + { + return $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => $commands, + ]); + } + private function start(): array { $response = $this->withHeaders($this->controlHeaders())->postJson('/api/workflows', [ @@ -169,13 +336,16 @@ private function start(): array return [$response->json('workflow_id'), $response->json('run_id')]; } - private function register(string $workerId, bool $capable): void + private function register(string $workerId, bool $capable, string $processStart = '2026-09-30T00:00:00Z'): void { - WorkerRegistration::query()->updateOrCreate(['namespace' => 'default', 'worker_id' => $workerId], [ + $this->withHeaders($this->headers($capable ? '1.20' : '1.19'))->postJson('/api/worker/register', [ + 'worker_id' => $workerId, 'task_queue' => 'cooperative', 'runtime' => 'php', 'supported_workflow_types' => ['tests.external-greeting-workflow'], 'capabilities' => $capable ? [CooperativeCancellationPolicy::CAPABILITY] : [], - 'max_concurrent_workflow_tasks' => 1, 'last_heartbeat_at' => now(), 'status' => 'active', - ]); + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + 'max_concurrent_workflow_tasks' => 1, + 'process_metrics' => ['host' => 'test-worker', 'process_started_at' => $processStart, 'process_id' => 10], + ])->assertCreated(); } private function poll(string $workerId, string $version = '1.20', ?string $pollRequestId = null): TestResponse diff --git a/tests/Feature/CooperativeCancellationRaceTest.php b/tests/Feature/CooperativeCancellationRaceTest.php new file mode 100644 index 00000000..8691ef81 --- /dev/null +++ b/tests/Feature/CooperativeCancellationRaceTest.php @@ -0,0 +1,154 @@ +assertIsString($path); + $this->databasePath = $path; + config([ + 'database.default' => 'sqlite', + 'database.connections.sqlite.database' => $path, + 'database.connections.sqlite.busy_timeout' => 100, + 'database.connections.sqlite.journal_mode' => 'WAL', + 'database.connections.sqlite.transaction_mode' => 'IMMEDIATE', + 'cache.default' => 'array', + 'server.worker_protocol.version' => '1.20', + 'workflows.v2.types.workflows' => ['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class], + ]); + DB::purge('sqlite'); + $this->artisan('migrate:fresh', ['--force' => true])->assertExitCode(0); + Queue::fake(); + WorkflowNamespace::query()->create([ + 'name' => 'default', 'description' => 'Test', 'retention_days' => 30, 'status' => 'active', + ]); + } + + protected function tearDown(): void + { + DB::disconnect('sqlite'); + foreach ([$this->databasePath, $this->databasePath.'-wal', $this->databasePath.'-shm'] as $path) { + if (is_file($path)) { + unlink($path); + } + } + parent::tearDown(); + } + + public static function workerCapabilities(): array + { + return ['old worker' => [false], 'cooperative worker' => [true]]; + } + + #[DataProvider('workerCapabilities')] + public function test_concurrent_request_and_claim_cannot_accept_an_incompatible_lease(bool $capable): void + { + // Repeat real process races. Deterministic before/after cases live in + // CooperativeCancellationProtocolTest; this covers overlapping writes. + for ($iteration = 1; $iteration <= 3; $iteration++) { + $workerId = 'race-worker-'.$iteration; + $queue = 'race-'.$iteration; + $start = $this->withHeaders($this->controlHeaders())->postJson('/api/workflows', [ + 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => $queue, 'input' => ['Ada'], + ])->assertCreated(); + $workflowId = $start->json('workflow_id'); + $runId = $start->json('run_id'); + WorkerRegistration::query()->create([ + 'namespace' => 'default', 'worker_id' => $workerId, 'task_queue' => $queue, + 'runtime' => 'php', 'supported_workflow_types' => ['tests.external-greeting-workflow'], + 'capabilities' => $capable ? [CooperativeCancellationPolicy::CAPABILITY] : [], + 'max_concurrent_workflow_tasks' => 1, 'last_heartbeat_at' => now(), 'status' => 'active', + ]); + + [$request, $poll] = $this->race($workflowId, $workerId, $queue, $capable ? '1.20' : '1.19'); + $this->assertSame(200, $poll['status'], json_encode($poll)); + $this->assertContains($request['status'], [202, 409], json_encode($request)); + $run = WorkflowRun::query()->findOrFail($runId); + $task = $poll['body']['task']; + + if ($request['status'] === 202) { + $this->assertNotNull($run->cancellation_request_command_id); + if ($capable) { + $this->assertIsArray($task); + $this->assertTrue(CooperativeCancellationPolicy::claimSupportsCancellation( + WorkflowTask::query()->findOrFail($task['task_id']), + )); + } else { + $this->assertNull($task); + } + } else { + $this->assertFalse($capable); + $this->assertSame('active_claim_cancellation_not_supported', $request['body']['reason']); + $this->assertNull($run->cancellation_request_command_id); + $this->assertIsArray($task); + } + } + } + + /** @return list> */ + private function race(string $workflowId, string $workerId, string $queue, string $protocol): array + { + $children = []; + try { + foreach (['request', 'poll'] as $operation) { + $process = proc_open([ + PHP_BINARY, dirname(__DIR__).'/Fixtures/CooperativeCancellationRaceRequest.php', + $this->databasePath, $operation, $workflowId, $workerId, $queue, $protocol, + ], [0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + $this->assertIsResource($process); + stream_set_timeout($pipes[1], 10); + $children[] = ['process' => $process, 'pipes' => $pipes]; + } + foreach ($children as $child) { + $this->assertSame("ready\n", fgets($child['pipes'][1])); + } + foreach ($children as $child) { + fwrite($child['pipes'][0], "go\n"); + fclose($child['pipes'][0]); + } + + $results = []; + foreach ($children as $child) { + $output = stream_get_contents($child['pipes'][1]); + $this->assertFalse(stream_get_meta_data($child['pipes'][1])['timed_out'], 'Concurrent request timed out.'); + $results[] = json_decode($output, true, flags: JSON_THROW_ON_ERROR); + } + + return $results; + } finally { + foreach ($children as $child) { + if (proc_get_status($child['process'])['running']) { + proc_terminate($child['process']); + } + foreach ($child['pipes'] as $pipe) { + if (is_resource($pipe)) { + fclose($pipe); + } + } + proc_close($child['process']); + } + } + } + + private function controlHeaders(): array + { + return ['X-Namespace' => 'default', 'X-Durable-Workflow-Control-Plane-Version' => '2']; + } +} diff --git a/tests/Feature/WorkflowWorkerProtocolTest.php b/tests/Feature/WorkflowWorkerProtocolTest.php index af566687..101f7286 100644 --- a/tests/Feature/WorkflowWorkerProtocolTest.php +++ b/tests/Feature/WorkflowWorkerProtocolTest.php @@ -2473,6 +2473,16 @@ public function test_it_drops_claimed_workflow_tasks_when_the_bridge_cannot_buil 'run_count' => 0, ]); + WorkflowRun::query()->create([ + 'id' => 'run-bridge-missing-task', + 'workflow_instance_id' => 'wf-bridge-missing-task', + 'workflow_class' => ExternalGreetingWorkflow::class, + 'workflow_type' => 'tests.external-greeting-workflow', + 'namespace' => 'default', + 'run_number' => 1, + 'status' => 'pending', + ]); + $recordedAt = now()->toJSON(); $this->mock(WorkflowTaskBridge::class, function (MockInterface $mock) use ($recordedAt): void { @@ -5872,6 +5882,16 @@ public function test_poll_response_compresses_history_above_threshold_via_mock() 'run_count' => 0, ]); + WorkflowRun::query()->create([ + 'id' => 'run-compress', + 'workflow_instance_id' => 'wf-compress-mock', + 'workflow_class' => ExternalGreetingWorkflow::class, + 'workflow_type' => 'tests.external-greeting-workflow', + 'namespace' => 'default', + 'run_number' => 1, + 'status' => 'pending', + ]); + $recordedAt = now()->toJSON(); $events = []; diff --git a/tests/Fixtures/CooperativeCancellationRaceRequest.php b/tests/Fixtures/CooperativeCancellationRaceRequest.php new file mode 100644 index 00000000..1d69d5ce --- /dev/null +++ b/tests/Fixtures/CooperativeCancellationRaceRequest.php @@ -0,0 +1,67 @@ +make(Kernel::class); +$app->make(Illuminate\Contracts\Console\Kernel::class)->bootstrap(); + +config([ + 'app.env' => 'testing', + 'app.key' => 'base64:dGVzdGluZy10ZXN0aW5nLXRlc3RpbmctdGVzdGluZzEyMzQ1Ng==', + 'database.default' => 'sqlite', + 'database.connections.sqlite.database' => $argv[1], + 'database.connections.sqlite.busy_timeout' => 100, + 'database.connections.sqlite.journal_mode' => 'WAL', + 'database.connections.sqlite.transaction_mode' => 'IMMEDIATE', + 'cache.default' => 'array', + 'server.auth.driver' => 'none', + 'server.worker_protocol.version' => '1.20', + 'server.polling.timeout' => 0, + 'server.polling.interval_ms' => 1, + 'server.polling.cache_path' => sys_get_temp_dir().'/dw-cooperative-race-'.getmypid(), + 'workflows.v2.types.workflows' => [ + 'tests.external-greeting-workflow' => ExternalGreetingWorkflow::class, + ], +]); +DB::purge('sqlite'); +Queue::fake(); +register_shutdown_function(static function (): void { + (new Filesystem)->deleteDirectory( + sys_get_temp_dir().'/dw-cooperative-race-'.getmypid(), + ); +}); + +fwrite(STDOUT, "ready\n"); +if (trim((string) fgets(STDIN)) !== 'go') { + exit(2); +} + +$operation = $argv[2]; +$path = $operation === 'request' + ? '/api/workflows/'.$argv[3].'/request-cancellation' + : '/api/worker/workflow-tasks/poll'; +$body = $operation === 'request' ? [] : [ + 'worker_id' => $argv[4], 'task_queue' => $argv[5], 'poll_request_id' => 'race-poll', +]; +$request = Request::create($path, 'POST', server: [ + 'CONTENT_TYPE' => 'application/json', + 'HTTP_ACCEPT' => 'application/json', + 'HTTP_X_NAMESPACE' => 'default', + 'HTTP_X_DURABLE_WORKFLOW_PROTOCOL_VERSION' => $argv[6], + 'HTTP_X_DURABLE_WORKFLOW_CONTROL_PLANE_VERSION' => '2', +], content: json_encode($body, JSON_THROW_ON_ERROR)); +$response = $kernel->handle($request); +fwrite(STDOUT, json_encode([ + 'status' => $response->getStatusCode(), + 'body' => json_decode($response->getContent(), true, flags: JSON_THROW_ON_ERROR), +], JSON_THROW_ON_ERROR)); +$kernel->terminate($request, $response); From a23e64435a3d86676d26619bd475c757cc9a3b6c Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 30 Sep 2026 23:22:05 +0000 Subject: [PATCH 03/57] Reload canonical cancellation history and recover race pressure --- app/Http/Controllers/Api/WorkerController.php | 15 ++------ app/Support/CooperativeCancellationPolicy.php | 1 + app/Support/WorkflowHistoryPageToken.php | 24 +++++++++++++ .../worker-protocol-api.openapi.yaml | 14 ++++++-- .../worker-protocol-stream.asyncapi.yaml | 1 + .../CooperativeCancellationProtocolTest.php | 21 +++++++++++ .../CooperativeCancellationRaceTest.php | 11 ++++++ .../CooperativeCancellationRaceRequest.php | 35 +++++++++++++------ 8 files changed, 97 insertions(+), 25 deletions(-) create mode 100644 app/Support/WorkflowHistoryPageToken.php diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index 82d9e82b..76358f5c 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -37,6 +37,7 @@ use App\Support\WorkerProtocol; use App\Support\WorkerProtocolMutationRetrier; use App\Support\WorkerTerminalEventAttribution; +use App\Support\WorkflowHistoryPageToken; use App\Support\WorkflowMetadataCapabilityPolicy; use App\Support\WorkflowQueryTaskBroker; use App\Support\WorkflowStreamCommandProcessor; @@ -4126,22 +4127,12 @@ private function formatTaskHistoryPagination(?array $task): ?array private static function encodeHistoryPageToken(int $sequence): string { - return base64_encode((string) $sequence); + return WorkflowHistoryPageToken::encode($sequence); } private static function decodeHistoryPageToken(?string $token): ?int { - if (! is_string($token) || trim($token) === '') { - return null; - } - - $decoded = base64_decode($token, true); - - if (! is_string($decoded) || ! ctype_digit($decoded)) { - return null; - } - - return (int) $decoded; + return WorkflowHistoryPageToken::decode($token); } /** diff --git a/app/Support/CooperativeCancellationPolicy.php b/app/Support/CooperativeCancellationPolicy.php index 4ecc7f12..677ec87f 100644 --- a/app/Support/CooperativeCancellationPolicy.php +++ b/app/Support/CooperativeCancellationPolicy.php @@ -73,6 +73,7 @@ public static function pending(WorkflowRun $run): ?array 'cleanup_deadline_at' => $run->cancellation_deadline_at?->toISOString(), 'delivery_sequence' => $run->cancellation_delivery_sequence, 'delivered_at' => $run->cancellation_delivered_at?->toISOString(), + 'history_refresh_page_token' => WorkflowHistoryPageToken::encode(0), ]; } } diff --git a/app/Support/WorkflowHistoryPageToken.php b/app/Support/WorkflowHistoryPageToken.php new file mode 100644 index 00000000..dc2b0a06 --- /dev/null +++ b/app/Support/WorkflowHistoryPageToken.php @@ -0,0 +1,24 @@ +assertSame(0, $this->eventCount($runId, HistoryEventType::CooperativeCancellationDelivered)); } + public function test_worker_reloads_canonical_delivery_after_acknowledgment_loss_with_the_observed_token(): void + { + [$workflowId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $pending = $this->requestCancellation($workflowId)->assertStatus(202)->json('cancellation_request'); + $this->deliver($task, $pending['request_id'])->assertOk(); + // The old task payload predates the request and marker. Read their + // real canonical history rather than constructing an event locally. + $history = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/history", [ + 'lease_owner' => 'new', + 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'next_history_page_token' => $pending['history_refresh_page_token'], + ])->assertOk(); + $markers = array_values(array_filter($history->json('history_events'), static fn ($event) => $event['event_type'] === HistoryEventType::CooperativeCancellationDelivered->value)); + $this->assertCount(1, $markers); + $this->assertSame($pending['request_id'], $markers[0]['workflow_command_id']); + $this->assertSame(1, $markers[0]['payload']['sequence']); + $this->assertNotContains(HistoryEventType::CooperativeCancellationDelivered->value, array_column($task['history_events'], 'event_type')); + } + public function test_cleanup_deadline_revokes_delivery_authority(): void { [$workflowId, $runId] = $this->start(); diff --git a/tests/Feature/CooperativeCancellationRaceTest.php b/tests/Feature/CooperativeCancellationRaceTest.php index 8691ef81..3e7320ab 100644 --- a/tests/Feature/CooperativeCancellationRaceTest.php +++ b/tests/Feature/CooperativeCancellationRaceTest.php @@ -10,6 +10,8 @@ use PHPUnit\Framework\Attributes\DataProvider; use Tests\Fixtures\ExternalGreetingWorkflow; use Tests\TestCase; +use Workflow\V2\Enums\HistoryEventType; +use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; @@ -78,10 +80,19 @@ public function test_concurrent_request_and_claim_cannot_accept_an_incompatible_ ]); [$request, $poll] = $this->race($workflowId, $workerId, $queue, $capable ? '1.20' : '1.19'); + foreach ([$request, $poll] as $result) { + foreach (array_slice($result['attempts'], 0, -1) as $retry) { + $this->assertSame(['status' => 503, 'reason' => 'backend_lock_pressure'], $retry); + } + } $this->assertSame(200, $poll['status'], json_encode($poll)); $this->assertContains($request['status'], [202, 409], json_encode($request)); $run = WorkflowRun::query()->findOrFail($runId); $task = $poll['body']['task']; + $this->assertSame($request['status'] === 202 ? 1 : 0, WorkflowHistoryEvent::query() + ->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::CooperativeCancellationRequested->value) + ->count()); if ($request['status'] === 202) { $this->assertNotNull($run->cancellation_request_command_id); diff --git a/tests/Fixtures/CooperativeCancellationRaceRequest.php b/tests/Fixtures/CooperativeCancellationRaceRequest.php index 1d69d5ce..7d8652ba 100644 --- a/tests/Fixtures/CooperativeCancellationRaceRequest.php +++ b/tests/Fixtures/CooperativeCancellationRaceRequest.php @@ -33,6 +33,9 @@ ], ]); DB::purge('sqlite'); +// Initialize the connection before releasing the request barrier. The race +// exercises the durable mutations rather than connection setup PRAGMAs. +DB::connection()->getPdo(); Queue::fake(); register_shutdown_function(static function (): void { (new Filesystem)->deleteDirectory( @@ -52,16 +55,28 @@ $body = $operation === 'request' ? [] : [ 'worker_id' => $argv[4], 'task_queue' => $argv[5], 'poll_request_id' => 'race-poll', ]; -$request = Request::create($path, 'POST', server: [ - 'CONTENT_TYPE' => 'application/json', - 'HTTP_ACCEPT' => 'application/json', - 'HTTP_X_NAMESPACE' => 'default', - 'HTTP_X_DURABLE_WORKFLOW_PROTOCOL_VERSION' => $argv[6], - 'HTTP_X_DURABLE_WORKFLOW_CONTROL_PLANE_VERSION' => '2', -], content: json_encode($body, JSON_THROW_ON_ERROR)); -$response = $kernel->handle($request); +$attempts = []; +for ($attempt = 1; $attempt <= 3; $attempt++) { + $request = Request::create($path, 'POST', server: [ + 'CONTENT_TYPE' => 'application/json', + 'HTTP_ACCEPT' => 'application/json', + 'HTTP_X_NAMESPACE' => 'default', + 'HTTP_X_DURABLE_WORKFLOW_PROTOCOL_VERSION' => $argv[6], + 'HTTP_X_DURABLE_WORKFLOW_CONTROL_PLANE_VERSION' => '2', + ], content: json_encode($body, JSON_THROW_ON_ERROR)); + $response = $kernel->handle($request); + $responseBody = json_decode($response->getContent(), true, flags: JSON_THROW_ON_ERROR); + $attempts[] = ['status' => $response->getStatusCode(), 'reason' => $responseBody['reason'] ?? null]; + $kernel->terminate($request, $response); + if ($response->getStatusCode() !== 503 || ($responseBody['reason'] ?? null) !== 'backend_lock_pressure' || $attempt === 3) { + break; + } + // Follow the published pressure response without changing request or poll + // identity. Every other failure remains visible to the parent assertion. + usleep(max(1, (int) ($responseBody['retry_after_seconds'] ?? 1)) * 1000000); +} fwrite(STDOUT, json_encode([ 'status' => $response->getStatusCode(), - 'body' => json_decode($response->getContent(), true, flags: JSON_THROW_ON_ERROR), + 'body' => $responseBody, + 'attempts' => $attempts, ], JSON_THROW_ON_ERROR)); -$kernel->terminate($request, $response); From 2e2d6b31df981e98c1a054a8953e9efe95d17e6e Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 02:20:43 +0000 Subject: [PATCH 04/57] Discover cooperative cancellation only on the configured candidate protocol --- .../Api/CooperativeCancellationController.php | 5 +---- app/Support/CooperativeCancellationPolicy.php | 8 ++++++++ app/Support/WorkerProtocol.php | 2 ++ .../worker-protocol-api.openapi.yaml | 3 +++ .../CooperativeCancellationProtocolTest.php | 14 ++++++++++++++ 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/app/Http/Controllers/Api/CooperativeCancellationController.php b/app/Http/Controllers/Api/CooperativeCancellationController.php index 03e24af0..1706feee 100644 --- a/app/Http/Controllers/Api/CooperativeCancellationController.php +++ b/app/Http/Controllers/Api/CooperativeCancellationController.php @@ -39,10 +39,7 @@ public function request(Request $request, string $workflowId, ?string $runId = n return $response; } - if (! WorkerProtocol::versionMeetsMinimum( - config('server.worker_protocol.version', WorkerProtocol::VERSION), - CooperativeCancellationPolicy::MINIMUM_PROTOCOL_VERSION, - )) { + if (! CooperativeCancellationPolicy::serverSupported()) { return ControlPlaneProtocol::jsonForRequest($request, [ 'message' => 'This Server does not support cooperative cancellation requests.', 'reason' => 'cooperative_cancellation_not_supported', diff --git a/app/Support/CooperativeCancellationPolicy.php b/app/Support/CooperativeCancellationPolicy.php index 677ec87f..777b8c76 100644 --- a/app/Support/CooperativeCancellationPolicy.php +++ b/app/Support/CooperativeCancellationPolicy.php @@ -14,6 +14,14 @@ final class CooperativeCancellationPolicy private const CLAIM_KEY = '_server_workflow_claim'; + public static function serverSupported(): bool + { + return WorkerProtocol::versionMeetsMinimum( + (string) config('server.worker_protocol.version', WorkerProtocol::VERSION), + self::MINIMUM_PROTOCOL_VERSION, + ); + } + /** @return array */ public static function workerSnapshot(WorkerRegistration $worker, ?string $protocolVersion): array { diff --git a/app/Support/WorkerProtocol.php b/app/Support/WorkerProtocol.php index 9f67d032..a9035119 100644 --- a/app/Support/WorkerProtocol.php +++ b/app/Support/WorkerProtocol.php @@ -413,6 +413,7 @@ public static function workflowMemoUpdateSemantics(): array * query_task_timeouts: array{control_plane_timeout_seconds: int, lease_timeout_seconds: int, lease_grace_seconds: int}, * activity_retry_policy: bool, * activity_timeouts: bool, + * cooperative_cancellation: bool, * local_activities: array, * worker_session_verbs: list, * worker_sessions: array, @@ -544,6 +545,7 @@ public static function serverCapabilities(): array ], 'activity_retry_policy' => true, 'activity_timeouts' => true, + 'cooperative_cancellation' => CooperativeCancellationPolicy::serverSupported(), 'local_activities' => [ ...WorkerProtocolVersion::localActivitySemantics(), 'supported' => $portableWorkerAffinitySupported, diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index e5845d77..1c7eebdc 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -993,6 +993,9 @@ components: additionalProperties: true required: [supported_workflow_task_commands, workflow_task_poll_request_idempotency, workflow_memo_updates, message_streams, typed_search_attributes, condition_wait_occurrence_identity, update_validation_tasks, synchronous_update_validation, local_activities, sticky_execution] properties: + cooperative_cancellation: + type: boolean + description: Explicit support for claim-bound cooperative request delivery, requiring worker protocol 1.20 or newer in the same major version. False at the default protocol 1.19. supported_workflow_task_commands: type: array uniqueItems: true diff --git a/tests/Feature/CooperativeCancellationProtocolTest.php b/tests/Feature/CooperativeCancellationProtocolTest.php index 9e4ea795..7e13b11a 100644 --- a/tests/Feature/CooperativeCancellationProtocolTest.php +++ b/tests/Feature/CooperativeCancellationProtocolTest.php @@ -187,6 +187,20 @@ public function test_request_is_unavailable_until_the_server_protocol_supports_i ->assertJsonPath('reason', 'cooperative_cancellation_not_supported'); } + public function test_discovery_matches_request_support_and_preserves_the_default_protocol(): void + { + foreach (['1.19' => false, '1.20' => true, '2.0' => false, 'malformed' => false] as $version => $supported) { + config(['server.worker_protocol.version' => $version]); + $this->withHeaders($this->controlHeaders())->getJson('/api/cluster/info') + ->assertOk() + ->assertJsonPath('worker_protocol.version', $version) + ->assertJsonPath('worker_protocol.server_capabilities.cooperative_cancellation', $supported); + $this->assertSame($supported, CooperativeCancellationPolicy::serverSupported()); + } + + $this->assertSame('1.19', WorkerProtocol::VERSION); + } + public function test_waiting_timer_is_interrupted_only_when_delivery_is_committed(): void { [$workflowId, $runId] = $this->start(); From 073a516bbd4063f57d4ad65ad732ec423131c8ff Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 05:53:49 +0000 Subject: [PATCH 05/57] Observe remote activity ownership without renewing user progress --- .../Api/ActivityTaskController.php | 91 +++++ .../Middleware/EnforceStorageAdmission.php | 1 + docs/worker-activity-observation.md | 36 ++ .../worker-protocol-api.openapi.yaml | 77 +++- routes/api.php | 1 + tests/Feature/ActivityAttemptStatusTest.php | 338 ++++++++++++++++++ 6 files changed, 543 insertions(+), 1 deletion(-) create mode 100644 docs/worker-activity-observation.md create mode 100644 tests/Feature/ActivityAttemptStatusTest.php diff --git a/app/Http/Controllers/Api/ActivityTaskController.php b/app/Http/Controllers/Api/ActivityTaskController.php index ae4ca4f0..bcff482e 100644 --- a/app/Http/Controllers/Api/ActivityTaskController.php +++ b/app/Http/Controllers/Api/ActivityTaskController.php @@ -8,6 +8,7 @@ use App\Support\ActivityTaskPoller; use App\Support\AvroPayloadEnvelopeResolver; use App\Support\BackendLockPressure; +use App\Support\CooperativeCancellationPolicy; use App\Support\ExternalExecutorConfigContract; use App\Support\ExternalPayloadEnvelopeService; use App\Support\ExternalPayloadStorageUnavailable; @@ -22,6 +23,7 @@ use App\Support\WorkerProtocol; use App\Support\WorkerProtocolMutationRetrier; use App\Support\WorkerSessionRegistry; +use Carbon\CarbonImmutable; use Carbon\CarbonInterface; use Illuminate\Http\JsonResponse; use Illuminate\Http\Request; @@ -414,6 +416,94 @@ public function fail(Request $request, string $taskId): JsonResponse ], $stopStatus), $this->outcomeStatus($outcome['reason'])); } + /** Observe an owned attempt without renewing its lease or recording progress. */ + public function status(Request $request, string $taskId): JsonResponse + { + if ($response = WorkerProtocol::rejectUnsupported($request)) { + return $response; + } + + if (! CooperativeCancellationPolicy::serverSupported() + || ! WorkerProtocol::versionMeetsMinimum(WorkerProtocol::requestVersion($request), '1.20')) { + return WorkerProtocol::json([ + 'error' => 'Activity attempt observation requires worker protocol 1.20.', + 'reason' => 'activity_attempt_status_unavailable', + 'minimum_protocol_version' => '1.20', + ], 422); + } + + $validated = $request->validate([ + 'activity_attempt_id' => ['required', 'string'], + 'lease_owner' => ['required', 'string'], + ]); + $status = null; + try { + if ($response = $this->guardAttemptOwnership( + $request->attributes->get('namespace'), + $taskId, + $validated['activity_attempt_id'], + $validated['lease_owner'], + $status, + )) { + return $response; + } + $deadlines = $this->executionDeadlines($status['activity_execution_id'] ?? null) ?? []; + $workerSession = $this->workerSessions->workerSessionForExecution($status['activity_execution_id'] ?? null); + } catch (\Throwable $exception) { + if (! BackendLockPressure::is($exception)) { + throw $exception; + } + + return BackendLockPressure::workerOperationResponse($request, false); + } + + if ($status['can_continue'] === true) { + $expiresAt = $status['lease_expires_at'] ?? null; + if (! is_string($expiresAt) || ! CarbonImmutable::parse($expiresAt)->isAfter(now())) { + $status['can_continue'] = false; + $status['reason'] = 'lease_expired'; + } + foreach (['heartbeat', 'start_to_close', 'schedule_to_close'] as $kind) { + if ($status['can_continue'] === true && isset($deadlines[$kind]) + && ! CarbonImmutable::parse($deadlines[$kind])->isAfter(now())) { + $status['can_continue'] = false; + $status['reason'] = $kind.'_timeout'; + } + } + if ($status['can_continue'] === true && $workerSession !== null) { + $sessionLease = $workerSession['lease_expires_at'] ?? null; + $sessionTtl = $workerSession['ttl_expires_at'] ?? null; + if (($workerSession['status'] ?? null) !== 'active' + || ($workerSession['lease_owner'] ?? null) !== $validated['lease_owner'] + || ! is_string($sessionLease) || ! CarbonImmutable::parse($sessionLease)->isAfter(now()) + || ! is_string($sessionTtl) || ! CarbonImmutable::parse($sessionTtl)->isAfter(now())) { + $status['can_continue'] = false; + $status['reason'] = 'worker_session_unavailable'; + } + } + } + + return WorkerProtocol::json([ + 'task_id' => $taskId, + 'activity_attempt_id' => $validated['activity_attempt_id'], + 'lease_owner' => $status['lease_owner'], + 'can_continue' => $status['can_continue'], + 'cancel_requested' => $status['cancel_requested'], + 'reason' => $status['reason'], + 'heartbeat_recorded' => false, + 'lease_expires_at' => $status['lease_expires_at'], + 'last_heartbeat_at' => $status['last_heartbeat_at'], + 'run_status' => $status['run_status'], + 'run_closed_reason' => $status['run_closed_reason'] ?? null, + 'run_closed_at' => $status['run_closed_at'] ?? null, + 'activity_status' => $status['activity_status'], + 'attempt_status' => $status['attempt_status'], + 'task_status' => $status['task_status'], + 'deadlines' => $deadlines === [] ? null : $deadlines, + 'worker_session' => $workerSession, + ]); + } + /** * Heartbeat an in-progress activity task. * @@ -686,6 +776,7 @@ private function guardAttemptOwnership( string $taskId, string $attemptId, string $leaseOwner, + ?array &$status = null, ): ?JsonResponse { $task = NamespaceWorkflowScope::task($namespace, $taskId); diff --git a/app/Http/Middleware/EnforceStorageAdmission.php b/app/Http/Middleware/EnforceStorageAdmission.php index 4e9baf79..cbabe08b 100644 --- a/app/Http/Middleware/EnforceStorageAdmission.php +++ b/app/Http/Middleware/EnforceStorageAdmission.php @@ -58,6 +58,7 @@ public function handle(Request $request, Closure $next): Response } if ($snapshot['state'] === 'normal' || $request->isMethodSafe() || $action === 'WorkerController@workflowTaskHistory' + || $action === 'ActivityTaskController@status' || ($snapshot['state'] === 'draining' && in_array($action, self::DRAIN_ACTIONS, true))) { return $next($request); } diff --git a/docs/worker-activity-observation.md b/docs/worker-activity-observation.md new file mode 100644 index 00000000..f29c3ba7 --- /dev/null +++ b/docs/worker-activity-observation.md @@ -0,0 +1,36 @@ +# Candidate remote activity observation + +Worker protocol1.20 adds `POST /api/worker/activity-tasks/{taskId}/status`. +This remains a source candidate. The default released protocol is1.19. + +Send the namespace and worker credential used to claim the task, the1.20 +protocol header, and the exact `activity_attempt_id` and `lease_owner` in JSON. +The response reports whether the observed attempt can continue, its stop reason, +lease expiry, user heartbeat time, execution deadlines and any required worker +session. Namespace, task, attempt and owner mismatches refuse the observation. + +Observation does not renew the activity lease, worker registration or required +session. It does not reset the application's heartbeat deadline or write +history. The activity retains the existing five-minute lease. Authored user +heartbeats retain their existing progress and renewal behavior. Required +sessions also retain their own lease and TTL. A worker must keep the separate +registration heartbeat current while it executes work. + +An expired lease, heartbeat or execution deadline refuses continuation before +the timeout scanner runs. A required session must be active, owned by the same +worker and within its lease and TTL. The read remains available under draining +or fenced storage admission. Backend lock pressure returns retryable503 with +the attempted fence and does not grant permission to continue. + +An accepted cooperative request remains pending until the workflow records its +canonical delivery. Pending acceptance alone does not cancel a remote activity. +Once delivery cancels the activity, observation returns `cancel_requested=true` +and `can_continue=false`. Terminal cancel and terminate retain their existing +behavior. Workers must stop unsafe callback execution and check the attempt +again before encoding or publication. A prior successful observation is not an +ownership reservation, and completion/failure still validate the actual fence. + +Stopping an activity process cannot undo external effects. Safe retries still +require idempotency or reconciliation. This endpoint supplies the observation +contract for connected SDK qualification and does not by itself prove callback +supervision, graceful shutdown or cold replacement in any SDK. diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 1c7eebdc..ecd14341 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "24" + version: "25" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -554,6 +554,41 @@ paths: - $ref: "#/components/schemas/DrainingPoll" "429": { $ref: "#/components/responses/WorkerPollBackpressure" } "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/activity-tasks/{taskId}/status: + post: + operationId: observeActivityTask + tags: [activity-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + description: > + Observe an activity attempt using its namespace, task, attempt and owner + fence. This call never renews an attempt lease, worker session or user + heartbeat and never writes history. An expired lease or execution/heartbeat + deadline refuses continuation even before timeout repair. A required + worker session must remain active, owned and within its lease and TTL. + Observation + remains available under draining or fenced storage admission. User + heartbeats retain the existing ownership + renewal contract. A successful observation is not a reservation for later + publication, which must independently validate the same attempt fence. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/ActivityTaskStatusRequest" } + responses: + "200": + description: Observed continuation or stop state, with no ownership or progress renewal. + content: + application/json: + schema: { $ref: "#/components/schemas/ActivityTaskStatusResponse" } + "400": { $ref: "#/components/responses/WorkerError" } + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } /worker/activity-tasks/{taskId}/heartbeat: post: operationId: heartbeatActivityTask @@ -1609,6 +1644,46 @@ components: next_history_page_token: { type: string, minLength: 1 } history_page_size: { type: [integer, "null"], minimum: 1 } accept_history_encoding: { type: [string, "null"] } + ActivityTaskStatusRequest: + type: object + additionalProperties: true + required: [activity_attempt_id, lease_owner] + properties: + activity_attempt_id: { type: string, minLength: 1 } + lease_owner: { type: string, minLength: 1 } + ActivityTaskStatusResponse: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - type: object + required: [task_id, activity_attempt_id, lease_owner, can_continue, cancel_requested, reason, heartbeat_recorded, lease_expires_at, last_heartbeat_at, run_status, activity_status, attempt_status, task_status] + properties: + task_id: { type: string, minLength: 1 } + activity_attempt_id: { type: string, minLength: 1 } + lease_owner: { type: string, minLength: 1 } + can_continue: { type: boolean } + cancel_requested: { type: boolean } + reason: { type: [string, "null"] } + heartbeat_recorded: { type: boolean, const: false } + lease_expires_at: { type: [string, "null"], format: date-time } + last_heartbeat_at: { type: [string, "null"], format: date-time } + run_status: { type: [string, "null"] } + run_closed_reason: { type: [string, "null"] } + run_closed_at: { type: [string, "null"], format: date-time } + activity_status: { type: [string, "null"] } + attempt_status: { type: [string, "null"] } + task_status: { type: [string, "null"] } + deadlines: + type: [object, "null"] + description: Existing execution deadlines. Null when no deadline is configured. + properties: + schedule_to_start: { type: string, format: date-time } + start_to_close: { type: string, format: date-time } + schedule_to_close: { type: string, format: date-time } + heartbeat: { type: string, format: date-time } + worker_session: + type: [object, "null"] + additionalProperties: true + description: Existing required-session observation, never a renewal. TaskHeartbeatRequest: type: object additionalProperties: true diff --git a/routes/api.php b/routes/api.php index 1803003c..f0e0a51e 100644 --- a/routes/api.php +++ b/routes/api.php @@ -249,6 +249,7 @@ // Activity tasks (long-poll) Route::post('/activity-tasks/poll', [ActivityTaskController::class, 'poll']); + Route::post('/activity-tasks/{taskId}/status', [ActivityTaskController::class, 'status']); Route::post('/activity-tasks/{taskId}/complete', [ActivityTaskController::class, 'complete']); Route::post('/activity-tasks/{taskId}/fail', [ActivityTaskController::class, 'fail']); Route::post('/activity-tasks/{taskId}/heartbeat', [ActivityTaskController::class, 'heartbeat']); diff --git a/tests/Feature/ActivityAttemptStatusTest.php b/tests/Feature/ActivityAttemptStatusTest.php new file mode 100644 index 00000000..253793f3 --- /dev/null +++ b/tests/Feature/ActivityAttemptStatusTest.php @@ -0,0 +1,338 @@ + '1.20']); + WorkflowNamespace::query()->create([ + 'name' => 'default', 'description' => 'Test', 'retention_days' => 30, 'status' => 'active', + ]); + } + + protected function tearDown(): void + { + $this->removeStoragePressure(); + parent::tearDown(); + } + + public function test_observation_preserves_user_progress_lease_and_registration(): void + { + $task = $this->lease(); + $execution = ActivityExecution::query()->findOrFail($task['activity_execution_id']); + $execution->forceFill(['retry_policy' => ['heartbeat_timeout' => 10]])->save(); + $this->withHeaders($this->headers())->postJson($this->path($task, 'heartbeat'), $this->fence($task) + [ + 'message' => 'authored progress', 'current' => 1, 'total' => 3, + ])->assertOk()->assertJsonPath('heartbeat_recorded', true); + $before = $this->snapshot($task); + $this->travel(2)->seconds(); + $response = $this->observe($task)->assertOk() + ->assertJsonPath('can_continue', true) + ->assertJsonPath('cancel_requested', false) + ->assertJsonPath('heartbeat_recorded', false) + ->assertJsonPath('task_id', $task['task_id']) + ->assertJsonPath('activity_attempt_id', $task['activity_attempt_id']) + ->assertJsonPath('lease_owner', $task['lease_owner']); + $this->assertSame($before, $this->snapshot($task)); + OpenApiSchema::fromFile(base_path('resources/platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/ActivityTaskStatusResponse/allOf/1', json_decode($response->getContent(), flags: JSON_THROW_ON_ERROR)); + } + + public function test_pending_request_does_not_cancel_before_canonical_delivery(): void + { + $task = $this->lease(); + $this->withHeaders($this->headers())->postJson("/api/workflows/{$task['workflow_id']}/request-cancellation", []) + ->assertStatus(202)->assertJsonPath('accepted', true); + $before = $this->snapshot($task); + $this->observe($task)->assertOk()->assertJsonPath('can_continue', true) + ->assertJsonPath('cancel_requested', false)->assertJsonPath('heartbeat_recorded', false); + $this->assertSame($before, $this->snapshot($task)); + } + + public function test_required_session_is_observed_without_renewal(): void + { + $task = $this->lease(); + $session = $this->requireSession($task); + $before = $session->fresh()->getRawOriginal(); + $this->travel(2)->seconds(); + $this->observe($task)->assertOk()->assertJsonPath('can_continue', true) + ->assertJsonPath('worker_session.session_id', 'required-session') + ->assertJsonPath('worker_session.lease_owner', $task['lease_owner']); + $this->assertSame($before, $session->fresh()->getRawOriginal()); + } + + public static function sessionProvider(): array + { + return [['lease_expires_at'], ['ttl_expires_at'], ['lease_owner'], ['status'], ['absent']]; + } + + #[DataProvider('sessionProvider')] + public function test_required_session_loss_refuses_continuation_without_repair(string $field): void + { + $task = $this->lease(); + $session = $this->requireSession($task); + if ($field === 'absent') { + $session->delete(); + } else { + $value = match ($field) { + 'lease_owner' => 'replacement-worker', + 'status' => 'closed', + default => now(), + }; + $session->forceFill([$field => $value])->save(); + } + $before = WorkerSessionLease::query()->get()->map->getRawOriginal()->all(); + $this->observe($task)->assertOk()->assertJsonPath('can_continue', false) + ->assertJsonPath('reason', 'worker_session_unavailable'); + $this->assertSame($before, WorkerSessionLease::query()->get()->map->getRawOriginal()->all()); + } + + public function test_operator_credentials_cannot_observe_worker_attempts(): void + { + $task = $this->lease(); + config(['server.auth.driver' => 'token', 'server.auth.role_tokens' => [ + 'operator' => 'fixture-operator', 'worker' => 'fixture-worker', + ]]); + $this->withHeaders($this->headers() + ['Authorization' => 'Bearer fixture-operator']) + ->postJson($this->path($task), $this->fence($task))->assertForbidden()->assertJsonPath('reason', 'forbidden'); + $this->withHeaders($this->headers() + ['Authorization' => 'Bearer fixture-worker']) + ->postJson($this->path($task), $this->fence($task))->assertOk()->assertJsonPath('can_continue', true); + } + + public function test_expired_lease_cannot_continue_before_repair_runs(): void + { + $task = $this->lease(); + $before = $this->snapshot($task); + $this->travel(5)->minutes(); + $this->observe($task)->assertOk()->assertJsonPath('can_continue', false) + ->assertJsonPath('reason', 'lease_expired')->assertJsonPath('heartbeat_recorded', false); + $this->assertSame($before, $this->snapshot($task)); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + } + + public static function deadlineProvider(): array + { + return [ + ['heartbeat_deadline_at', 'heartbeat_timeout'], + ['close_deadline_at', 'start_to_close_timeout'], + ['schedule_to_close_deadline_at', 'schedule_to_close_timeout'], + ]; + } + + #[DataProvider('deadlineProvider')] + public function test_expired_application_deadline_is_not_hidden_by_observation(string $field, string $reason): void + { + $task = $this->lease(); + ActivityExecution::query()->findOrFail($task['activity_execution_id']) + ->forceFill([$field => now()])->save(); + $before = $this->snapshot($task); + $this->observe($task)->assertOk()->assertJsonPath('can_continue', false) + ->assertJsonPath('reason', $reason)->assertJsonPath('heartbeat_recorded', false); + $this->assertSame($before, $this->snapshot($task)); + $this->assertSame(ActivityStatus::Running, ActivityExecution::query()->findOrFail($task['activity_execution_id'])->status); + } + + public function test_canonical_activity_cancellation_stops_without_another_history_event(): void + { + $task = $this->lease(); + $execution = ActivityExecution::query()->findOrFail($task['activity_execution_id']); + ActivityCancellation::record( + WorkflowRun::query()->findOrFail($task['run_id']), $execution, + WorkflowTask::query()->findOrFail($task['task_id']), + ); + $before = $this->snapshot($task); + $this->observe($task)->assertOk()->assertJsonPath('can_continue', false) + ->assertJsonPath('cancel_requested', true)->assertJsonPath('heartbeat_recorded', false); + $this->assertSame($before, $this->snapshot($task)); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id']) + ->where('event_type', HistoryEventType::ActivityCancelled->value)->count()); + } + + public static function closureProvider(): array + { + return [['cancel', 'run_cancelled'], ['terminate', 'run_terminated']]; + } + + #[DataProvider('closureProvider')] + public function test_terminal_close_is_observed_without_progress(string $action, string $reason): void + { + $task = $this->lease(); + $this->withHeaders($this->headers())->postJson("/api/workflows/{$task['workflow_id']}/{$action}", []) + ->assertSuccessful(); + $before = $this->snapshot($task); + $this->observe($task)->assertOk()->assertJsonPath('can_continue', false) + ->assertJsonPath('cancel_requested', true)->assertJsonPath('reason', $reason); + $this->assertSame($before, $this->snapshot($task)); + } + + public function test_namespace_task_attempt_and_owner_fences_are_required(): void + { + $task = $this->lease(); + $other = $this->lease('other'); + $this->observe($task, ['lease_owner' => 'different'])->assertStatus(409) + ->assertJsonPath('reason', 'lease_owner_mismatch'); + $this->observe($task, ['activity_attempt_id' => $other['activity_attempt_id']])->assertStatus(409) + ->assertJsonPath('reason', 'task_mismatch'); + $this->observe($task, ['activity_attempt_id' => 'absent'])->assertNotFound() + ->assertJsonPath('reason', 'attempt_not_found'); + WorkflowNamespace::query()->create([ + 'name' => 'isolated', 'description' => 'Other', 'retention_days' => 30, 'status' => 'active', + ]); + $this->observe($task, namespace: 'isolated')->assertNotFound()->assertJsonPath('reason', 'task_not_found'); + $this->withHeaders($this->headers())->postJson($this->path($task), [])->assertStatus(422) + ->assertJsonValidationErrors(['activity_attempt_id', 'lease_owner']); + } + + public function test_closed_old_attempt_cannot_continue_after_reclaim(): void + { + $task = $this->lease(); + $model = WorkflowTask::query()->findOrFail($task['task_id']); + $model->forceFill(['status' => TaskStatus::Ready, 'lease_owner' => null, 'lease_expires_at' => null])->save(); + $replacement = app(ActivityTaskBridge::class)->claim($model->id, 'replacement-worker'); + $this->assertNotNull($replacement); + $this->assertNotSame($task['activity_attempt_id'], $replacement['activity_attempt_id']); + $before = $this->snapshot($task); + $this->observe($task)->assertOk()->assertJsonPath('can_continue', false); + $this->assertSame($before, $this->snapshot($task)); + } + + public function test_lock_pressure_does_not_claim_permission_to_continue(): void + { + $task = $this->lease(); + $this->mock(ActivityTaskBridge::class, static function (MockInterface $mock): void { + $mock->shouldReceive('status')->andThrow(new BackendLockPressureException('database is locked')); + $mock->shouldNotReceive('heartbeat'); + }); + $this->observe($task)->assertStatus(503)->assertHeader('Retry-After', '1') + ->assertJsonPath('reason', 'backend_lock_pressure')->assertJsonPath('recorded', false) + ->assertJsonPath('activity_attempt_id', $task['activity_attempt_id']) + ->assertJsonMissingPath('can_continue'); + } + + public function test_read_is_available_when_storage_is_draining_or_fenced(): void + { + $task = $this->lease(); + $this->configureStoragePressure(); + $before = $this->snapshot($task); + foreach (['draining', 'fenced'] as $state) { + $this->observeStoragePressure($state); + $this->observe($task)->assertOk()->assertJsonPath('can_continue', true) + ->assertJsonPath('heartbeat_recorded', false); + } + $this->assertSame($before, $this->snapshot($task)); + } + + public function test_default_and_old_protocols_do_not_enable_observation(): void + { + $task = $this->lease(); + config(['server.worker_protocol.version' => '1.19']); + $this->observe($task)->assertStatus(400)->assertJsonPath('reason', 'unsupported_protocol_version'); + $old = $this->headers(); + $old[WorkerProtocol::HEADER] = '1.19'; + $this->withHeaders($old)->postJson($this->path($task), $this->fence($task)) + ->assertStatus(422)->assertJsonPath('reason', 'activity_attempt_status_unavailable'); + config(['server.worker_protocol.version' => '1.20']); + $this->withHeaders($old)->postJson($this->path($task), $this->fence($task)) + ->assertStatus(422)->assertJsonPath('reason', 'activity_attempt_status_unavailable'); + } + + private function lease(string $suffix = 'one'): array + { + $workflow = WorkflowStub::make(ExternalGreetingWorkflow::class, 'status-'.$suffix); + $start = $workflow->start('Ada'); + NamespaceWorkflowScope::bind('default', $workflow->id(), ExternalGreetingWorkflow::class); + $id = WorkflowTask::query()->where('workflow_run_id', $start->runId()) + ->where('task_type', 'workflow')->where('status', 'ready')->value('id'); + (new RunWorkflowTask($id))->handle(app(WorkflowExecutor::class)); + WorkerRegistration::query()->updateOrCreate(['worker_id' => 'status-worker-'.$suffix, 'namespace' => 'default'], [ + 'task_queue' => 'external-activities', 'runtime' => 'php', + 'supported_activity_types' => ['tests.external-greeting-activity'], + 'last_heartbeat_at' => now(), 'status' => 'active', + ]); + + return $this->withHeaders($this->headers())->postJson('/api/worker/activity-tasks/poll', [ + 'worker_id' => 'status-worker-'.$suffix, 'task_queue' => 'external-activities', + ])->assertOk()->json('task'); + } + + private function requireSession(array $task): WorkerSessionLease + { + ActivityExecution::query()->findOrFail($task['activity_execution_id']) + ->forceFill(['activity_options' => ['worker_session' => [ + 'session_id' => 'required-session', 'lease_seconds' => 120, + ]]])->save(); + + return WorkerSessionLease::query()->create([ + 'namespace' => 'default', 'session_id' => 'required-session', 'queue' => 'external-activities', + 'status' => 'active', 'lease_owner' => $task['lease_owner'], 'lease_expires_at' => now()->addSeconds(120), + 'ttl_expires_at' => now()->addMinutes(30), 'last_heartbeat_at' => now(), + ]); + } + + private function headers(string $namespace = 'default'): array + { + return ['X-Namespace' => $namespace, 'X-Durable-Workflow-Control-Plane-Version' => '2', WorkerProtocol::HEADER => '1.20']; + } + + private function path(array $task, string $action = 'status'): string + { + return "/api/worker/activity-tasks/{$task['task_id']}/{$action}"; + } + + private function fence(array $task): array + { + return ['activity_attempt_id' => $task['activity_attempt_id'], 'lease_owner' => $task['lease_owner']]; + } + + private function observe(array $task, array $overrides = [], string $namespace = 'default'): TestResponse + { + return $this->withHeaders($this->headers($namespace))->postJson($this->path($task), array_replace($this->fence($task), $overrides)); + } + + private function snapshot(array $task): array + { + return [ + ActivityExecution::query()->findOrFail($task['activity_execution_id'])->getRawOriginal(), + ActivityAttempt::query()->findOrFail($task['activity_attempt_id'])->getRawOriginal(), + WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(), + WorkerRegistration::query()->where('worker_id', $task['lease_owner'])->firstOrFail()->getRawOriginal(), + WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id'])->count(), + ]; + } +} From 6be6cd3936a1163b2a7136f179c90f031870a739 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 10:14:51 +0000 Subject: [PATCH 06/57] fix: fence cooperative heartbeat renewal against lease reclaim --- app/Http/Controllers/Api/WorkerController.php | 47 +++++---- .../CooperativeCancellationRaceTest.php | 96 +++++++++++++++++++ .../CooperativeCancellationRaceRequest.php | 47 +++++++-- 3 files changed, 165 insertions(+), 25 deletions(-) diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index 76358f5c..4487f04d 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -3209,22 +3209,36 @@ public function heartbeatWorkflowTask(Request $request, string $taskId): JsonRes 'workflow_task_attempt' => ['required', 'integer', 'min:1'], ]); - if ($response = $this->guardWorkflowTaskOwnership( - $request, - $namespace, - $taskId, - (int) $validated['workflow_task_attempt'], - $validated['lease_owner'], - )) { - return $response; - } - /** @var WorkflowTaskBridge $bridge */ $bridge = app(WorkflowTaskBridge::class); try { $status = $this->storageMutations->run( - static fn (): array => $bridge->heartbeat($taskId), + fn (): array|JsonResponse => DB::transaction(function () use ($request, $namespace, $taskId, $validated, $bridge): array|JsonResponse { + // Ownership must remain valid until renewal commits. A + // check before this lock can acknowledge a reclaimed lease. + NamespaceWorkflowScope::taskQuery($namespace)->lockForUpdate()->find($taskId); + if ($response = $this->guardWorkflowTaskOwnership( + $request, + $namespace, + $taskId, + (int) $validated['workflow_task_attempt'], + $validated['lease_owner'], + )) { + return $response; + } + + $status = $bridge->heartbeat($taskId); + if (($status['renewed'] ?? false) === true) { + $task = NamespaceWorkflowScope::task($namespace, $taskId); + if ($task?->run instanceof WorkflowRun + && ($pending = CooperativeCancellationPolicy::pending($task->run)) !== null) { + $status['cancellation_request'] = $pending; + } + } + + return $status; + }), ); } catch (\Throwable $exception) { if (! BackendLockPressure::is($exception)) { @@ -3238,13 +3252,8 @@ public function heartbeatWorkflowTask(Request $request, string $taskId): JsonRes ); } - $observation = []; - if (($status['renewed'] ?? false) === true) { - $task = NamespaceWorkflowScope::task($namespace, $taskId); - if ($task?->run instanceof WorkflowRun - && ($pending = CooperativeCancellationPolicy::pending($task->run)) !== null) { - $observation['cancellation_request'] = $pending; - } + if ($status instanceof JsonResponse) { + return $status; } return WorkerProtocol::json([ @@ -3256,7 +3265,7 @@ public function heartbeatWorkflowTask(Request $request, string $taskId): JsonRes 'run_status' => $status['run_status'], 'task_status' => $status['task_status'], 'reason' => $status['reason'], - ...$observation, + ...(isset($status['cancellation_request']) ? ['cancellation_request' => $status['cancellation_request']] : []), ], $this->workflowOutcomeStatus($status['reason'])); } diff --git a/tests/Feature/CooperativeCancellationRaceTest.php b/tests/Feature/CooperativeCancellationRaceTest.php index 3e7320ab..fae8c9a8 100644 --- a/tests/Feature/CooperativeCancellationRaceTest.php +++ b/tests/Feature/CooperativeCancellationRaceTest.php @@ -59,6 +59,102 @@ public static function workerCapabilities(): array return ['old worker' => [false], 'cooperative worker' => [true]]; } + public function test_heartbeat_cannot_acknowledge_an_old_claim_after_concurrent_reclaim(): void + { + $clock = now()->startOfSecond(); + $this->travelTo($clock); + config(['workflows.v2.workflow_task_lease_seconds' => 1]); + $start = $this->withHeaders($this->controlHeaders())->postJson('/api/workflows', [ + 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'heartbeat-race', 'input' => ['Ada'], + ])->assertCreated(); + $runId = $start->json('run_id'); + foreach (['original', 'replacement'] as $workerId) { + WorkerRegistration::query()->create([ + 'namespace' => 'default', 'worker_id' => $workerId, 'task_queue' => 'heartbeat-race', + 'runtime' => 'php', 'supported_workflow_types' => ['tests.external-greeting-workflow'], + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY], + 'max_concurrent_workflow_tasks' => 1, 'last_heartbeat_at' => now(), 'status' => 'active', + ]); + } + $pending = $this->withHeaders($this->controlHeaders()) + ->postJson('/api/workflows/'.$start->json('workflow_id').'/request-cancellation', []) + ->assertStatus(202)->json('cancellation_request'); + $claim = $this->withHeaders([ + 'X-Namespace' => 'default', 'X-Durable-Workflow-Protocol-Version' => '1.20', + ])->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'original', 'task_queue' => 'heartbeat-race', 'poll_request_id' => 'original-poll', + ])->assertOk()->json('task'); + $this->assertIsArray($claim); + + $children = []; + try { + foreach (['heartbeat' => 'original', 'poll' => 'replacement'] as $operation => $workerId) { + $process = proc_open([ + PHP_BINARY, dirname(__DIR__).'/Fixtures/CooperativeCancellationRaceRequest.php', + $this->databasePath, $operation, $claim['task_id'], $workerId, 'heartbeat-race', '1.20', + $clock->toIso8601String(), (string) $claim['workflow_task_attempt'], + ], [0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + $this->assertIsResource($process); + stream_set_timeout($pipes[1], 10); + $children[$operation] = ['process' => $process, 'pipes' => $pipes]; + $this->assertSame("ready\n", fgets($pipes[1])); + } + fwrite($children['heartbeat']['pipes'][0], "go\n"); + $this->assertSame("checked\n", fgets($children['heartbeat']['pipes'][1])); + // The original guard passed before expiry. Advance both actual + // HTTP kernels beyond expiry and let Native try a replacement + // claim while renewal is paused, without editing any lease row. + fwrite($children['poll']['pipes'][0], "go\n"); + $read = [$children['poll']['pipes'][1]]; + $write = $except = null; + stream_select($read, $write, $except, 1); + fwrite($children['heartbeat']['pipes'][0], "go\n"); + + $results = []; + foreach ($children as $operation => $child) { + fclose($child['pipes'][0]); + $output = stream_get_contents($child['pipes'][1]); + $this->assertFalse(stream_get_meta_data($child['pipes'][1])['timed_out'], 'Heartbeat race timed out.'); + $results[$operation] = json_decode($output, true, flags: JSON_THROW_ON_ERROR); + } + $heartbeat = $results['heartbeat']; + $this->assertSame(200, $heartbeat['status'], json_encode($heartbeat)); + $this->assertTrue($heartbeat['body']['renewed']); + $task = WorkflowTask::query()->findOrFail($claim['task_id']); + $this->assertSame($task->lease_owner, $heartbeat['body']['lease_owner'], json_encode([ + 'heartbeat' => array_intersect_key($heartbeat['body'], array_flip([ + 'task_id', 'lease_owner', 'workflow_task_attempt', 'renewed', + ])), + 'replacement_claim' => is_array($results['poll']['body']['task'] ?? null) + ? array_intersect_key($results['poll']['body']['task'], array_flip([ + 'task_id', 'lease_owner', 'workflow_task_attempt', + ])) : null, + ])); + $this->assertSame($task->attempt_count, $heartbeat['body']['workflow_task_attempt']); + $this->assertSame('original', $task->lease_owner); + $this->assertSame($claim['workflow_task_attempt'], $task->attempt_count); + $this->assertSame(200, $results['poll']['status'], json_encode($results['poll'])); + $this->assertNull($results['poll']['body']['task']); + $this->assertSame($pending, $heartbeat['body']['cancellation_request']); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::CooperativeCancellationRequested->value)->count()); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered->value)->count()); + } finally { + foreach ($children as $child) { + if (proc_get_status($child['process'])['running']) { + proc_terminate($child['process']); + } + foreach ($child['pipes'] as $pipe) { + if (is_resource($pipe)) { + fclose($pipe); + } + } + proc_close($child['process']); + } + } + } + #[DataProvider('workerCapabilities')] public function test_concurrent_request_and_claim_cannot_accept_an_incompatible_lease(bool $capable): void { diff --git a/tests/Fixtures/CooperativeCancellationRaceRequest.php b/tests/Fixtures/CooperativeCancellationRaceRequest.php index 7d8652ba..4acb7f69 100644 --- a/tests/Fixtures/CooperativeCancellationRaceRequest.php +++ b/tests/Fixtures/CooperativeCancellationRaceRequest.php @@ -3,9 +3,12 @@ use Illuminate\Contracts\Http\Kernel; use Illuminate\Filesystem\Filesystem; use Illuminate\Http\Request; +use Illuminate\Support\Carbon; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Queue; use Tests\Fixtures\ExternalGreetingWorkflow; +use Workflow\V2\Contracts\WorkflowTaskBridge; +use Workflow\V2\Support\WorkflowTaskOwnership; // Separate HTTP kernels and database connections exercise the request/claim // race. The parent supplies only a disposable database and fixture identities. @@ -37,6 +40,34 @@ // exercises the durable mutations rather than connection setup PRAGMAs. DB::connection()->getPdo(); Queue::fake(); +if (isset($argv[7])) { + Carbon::setTestNow(Carbon::parse($argv[7])); + config(['workflows.v2.workflow_task_lease_seconds' => 1]); + if ($argv[2] === 'poll') { + Carbon::setTestNow(Carbon::parse($argv[7])->addSeconds(2)); + } elseif ($argv[2] === 'heartbeat') { + $native = $app->make(WorkflowTaskBridge::class); + // Only add an IPC barrier. Both ownership status and renewal still + // execute the published Native implementation against the real DB. + $bridge = Mockery::mock(WorkflowTaskBridge::class); + $bridge->shouldReceive('status')->andReturnUsing($native->status(...)); + $paused = false; + $bridge->shouldReceive('heartbeat')->andReturnUsing(static function (string $taskId) use ($native, &$paused, $argv): array { + if (! $paused) { + $paused = true; + Carbon::setTestNow(Carbon::parse($argv[7])->addSeconds(2)); + fwrite(STDOUT, "checked\n"); + if (trim((string) fgets(STDIN)) !== 'go') { + exit(2); + } + } + + return $native->heartbeat($taskId); + }); + $app->instance(WorkflowTaskBridge::class, $bridge); + $app->instance(WorkflowTaskOwnership::class, new WorkflowTaskOwnership($bridge)); + } +} register_shutdown_function(static function (): void { (new Filesystem)->deleteDirectory( sys_get_temp_dir().'/dw-cooperative-race-'.getmypid(), @@ -49,12 +80,16 @@ } $operation = $argv[2]; -$path = $operation === 'request' - ? '/api/workflows/'.$argv[3].'/request-cancellation' - : '/api/worker/workflow-tasks/poll'; -$body = $operation === 'request' ? [] : [ - 'worker_id' => $argv[4], 'task_queue' => $argv[5], 'poll_request_id' => 'race-poll', -]; +$path = match ($operation) { + 'request' => '/api/workflows/'.$argv[3].'/request-cancellation', + 'heartbeat' => '/api/worker/workflow-tasks/'.$argv[3].'/heartbeat', + default => '/api/worker/workflow-tasks/poll', +}; +$body = match ($operation) { + 'request' => [], + 'heartbeat' => ['lease_owner' => $argv[4], 'workflow_task_attempt' => (int) $argv[8]], + default => ['worker_id' => $argv[4], 'task_queue' => $argv[5], 'poll_request_id' => 'race-poll'], +}; $attempts = []; for ($attempt = 1; $attempt <= 3; $attempt++) { $request = Request::create($path, 'POST', server: [ From 0f6b1dd9679ef46418e8af9d3740bba3b5f5ce8b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 13:05:21 +0000 Subject: [PATCH 07/57] test: preserve cancellation fencing when reclaim wins --- .../CooperativeCancellationRaceTest.php | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/tests/Feature/CooperativeCancellationRaceTest.php b/tests/Feature/CooperativeCancellationRaceTest.php index fae8c9a8..4de41bd8 100644 --- a/tests/Feature/CooperativeCancellationRaceTest.php +++ b/tests/Feature/CooperativeCancellationRaceTest.php @@ -118,24 +118,37 @@ public function test_heartbeat_cannot_acknowledge_an_old_claim_after_concurrent_ $results[$operation] = json_decode($output, true, flags: JSON_THROW_ON_ERROR); } $heartbeat = $results['heartbeat']; - $this->assertSame(200, $heartbeat['status'], json_encode($heartbeat)); - $this->assertTrue($heartbeat['body']['renewed']); $task = WorkflowTask::query()->findOrFail($claim['task_id']); - $this->assertSame($task->lease_owner, $heartbeat['body']['lease_owner'], json_encode([ + $trace = json_encode([ + 'heartbeat_status' => $heartbeat['status'], 'heartbeat' => array_intersect_key($heartbeat['body'], array_flip([ - 'task_id', 'lease_owner', 'workflow_task_attempt', 'renewed', + 'task_id', 'lease_owner', 'workflow_task_attempt', 'renewed', 'reason', ])), 'replacement_claim' => is_array($results['poll']['body']['task'] ?? null) ? array_intersect_key($results['poll']['body']['task'], array_flip([ 'task_id', 'lease_owner', 'workflow_task_attempt', ])) : null, - ])); - $this->assertSame($task->attempt_count, $heartbeat['body']['workflow_task_attempt']); - $this->assertSame('original', $task->lease_owner); - $this->assertSame($claim['workflow_task_attempt'], $task->attempt_count); + ]); + $this->assertContains($heartbeat['status'], [200, 409], $trace); $this->assertSame(200, $results['poll']['status'], json_encode($results['poll'])); - $this->assertNull($results['poll']['body']['task']); - $this->assertSame($pending, $heartbeat['body']['cancellation_request']); + if ($heartbeat['status'] === 200) { + $this->assertTrue($heartbeat['body']['renewed']); + $this->assertSame($task->lease_owner, $heartbeat['body']['lease_owner'], $trace); + $this->assertSame($task->attempt_count, $heartbeat['body']['workflow_task_attempt']); + $this->assertSame('original', $task->lease_owner); + $this->assertSame($claim['workflow_task_attempt'], $task->attempt_count); + $this->assertNull($results['poll']['body']['task']); + $this->assertSame($pending, $heartbeat['body']['cancellation_request']); + } else { + $this->assertSame('lease_owner_mismatch', $heartbeat['body']['reason'], $trace); + $this->assertFalse($heartbeat['body']['renewed'] ?? false); + $this->assertSame('replacement', $task->lease_owner); + $this->assertSame($claim['workflow_task_attempt'] + 1, $task->attempt_count); + $this->assertSame($claim['task_id'], $results['poll']['body']['task']['task_id']); + $this->assertSame($task->lease_owner, $results['poll']['body']['task']['lease_owner']); + $this->assertSame($task->attempt_count, $results['poll']['body']['task']['workflow_task_attempt']); + $this->assertSame($pending, $results['poll']['body']['task']['cancellation_request']); + } $this->assertSame(1, WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) ->where('event_type', HistoryEventType::CooperativeCancellationRequested->value)->count()); $this->assertSame(0, WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) From 555cf7ab4ed4e9a79b52475be76a6dfcf886dd70 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 14:51:38 +0000 Subject: [PATCH 08/57] Keep pending cancellation runnable after workflow prefix completion --- app/Http/Controllers/Api/WorkerController.php | 8 ++- app/Support/CooperativeCancellationPolicy.php | 48 +++++++++++++++++ .../CooperativeCancellationProtocolTest.php | 51 +++++++++++++++++++ 3 files changed, 106 insertions(+), 1 deletion(-) diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index 4487f04d..75a5a28a 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -1792,7 +1792,7 @@ function () use ( ->where('worker_id', $validated['lease_owner']) ->lockForUpdate() ->first(); - NamespaceWorkflowScope::taskQuery((string) $namespace) + $claimedTask = NamespaceWorkflowScope::taskQuery((string) $namespace) ->whereKey($taskId) ->lockForUpdate() ->first(); @@ -1843,6 +1843,12 @@ function () use ( WorkerProtocol::requestVersion($request), ); $outcome = $bridge->complete($taskId, $commands); + if (($outcome['completed'] ?? false) === true && $claimedTask instanceof WorkflowTask) { + $successor = CooperativeCancellationPolicy::resumeUndeliveredRequest($claimedTask->refresh()); + if ($successor instanceof WorkflowTask) { + $outcome['created_task_ids'][] = $successor->id; + } + } $this->applyStickyCacheClaim( $taskId, $validated['lease_owner'], diff --git a/app/Support/CooperativeCancellationPolicy.php b/app/Support/CooperativeCancellationPolicy.php index 777b8c76..0ebfbb6f 100644 --- a/app/Support/CooperativeCancellationPolicy.php +++ b/app/Support/CooperativeCancellationPolicy.php @@ -3,6 +3,8 @@ namespace App\Support; use App\Models\WorkerRegistration; +use Workflow\V2\Enums\TaskStatus; +use Workflow\V2\Enums\TaskType; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; @@ -68,6 +70,52 @@ public static function supports(array $capabilities, ?string $protocolVersion): && WorkerProtocol::versionMeetsMinimum($protocolVersion, self::MINIMUM_PROTOCOL_VERSION); } + /** + * Keep an undelivered request runnable when a worker commits the commands + * preceding its cancellation boundary. Call inside the fenced completion + * transaction so the prefix and its successor commit together. + */ + public static function resumeUndeliveredRequest(WorkflowTask $completedTask): ?WorkflowTask + { + if ($completedTask->status !== TaskStatus::Completed + || ! self::claimSupportsCancellation($completedTask)) { + return null; + } + + $run = WorkflowRun::query()->where('namespace', $completedTask->namespace) + ->lockForUpdate()->find($completedTask->workflow_run_id); + if (! $run instanceof WorkflowRun + || $run->status->isTerminal() + || ! is_string($run->cancellation_request_command_id) + || $run->cancellation_delivery_sequence !== null + || $run->cancellation_deadline_at === null + || ! $run->cancellation_deadline_at->isFuture()) { + return null; + } + + if (WorkflowTask::query()->where('workflow_run_id', $run->id) + ->where('task_type', TaskType::Workflow->value) + ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value])->exists()) { + return null; + } + + return WorkflowTask::query()->create([ + 'workflow_run_id' => $run->id, + 'namespace' => $run->namespace, + 'task_type' => TaskType::Workflow->value, + 'status' => TaskStatus::Ready->value, + 'available_at' => now(), + 'payload' => [ + 'resume_source_kind' => 'cancellation_request', + 'resume_source_id' => $run->cancellation_request_command_id, + 'workflow_command_id' => $run->cancellation_request_command_id, + ], + 'connection' => $run->connection, + 'queue' => $run->queue, + 'compatibility' => $run->compatibility, + ]); + } + /** @return array|null */ public static function pending(WorkflowRun $run): ?array { diff --git a/tests/Feature/CooperativeCancellationProtocolTest.php b/tests/Feature/CooperativeCancellationProtocolTest.php index 7e13b11a..8284e06e 100644 --- a/tests/Feature/CooperativeCancellationProtocolTest.php +++ b/tests/Feature/CooperativeCancellationProtocolTest.php @@ -9,9 +9,11 @@ use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\Queue; use Illuminate\Testing\TestResponse; +use PHPUnit\Framework\Attributes\DataProvider; use Tests\Fixtures\ExternalGreetingWorkflow; use Tests\Support\OpenApiSchema; use Tests\TestCase; +use Workflow\Serializers\Serializer; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Jobs\RunTimerTask; @@ -120,6 +122,55 @@ public function test_delivery_retry_records_one_marker_and_keeps_cleanup_authori ->assertJsonPath('reason', 'cancellation_delivery_mismatch'); } + public static function cancellationRequestTiming(): array + { + return ['before claim' => [true], 'after claim' => [false]]; + } + + #[DataProvider('cancellationRequestTiming')] + public function test_prefix_completion_keeps_pending_cancellation_deliverable(bool $beforeClaim): void + { + [$workflowId, $runId] = $this->start(); + $this->register('original', true); + $requestId = $beforeClaim + ? $this->requestCancellation($workflowId)->assertStatus(202)->json('cancellation_request.request_id') + : null; + $task = $this->poll('original')->assertOk()->json('task'); + $requestId ??= $this->requestCancellation($workflowId)->assertStatus(202)->json('cancellation_request.request_id'); + + $completed = $this->complete($task, [ + ['type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 7)], + ['type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 8)], + ])->assertOk()->assertJsonPath('recorded', true)->assertJsonPath('run_status', 'waiting'); + $this->assertCount(1, $completed->json('created_task_ids')); + $this->assertSame(2, $this->eventCount($runId, HistoryEventType::SideEffectRecorded)); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::CooperativeCancellationDelivered)); + $this->assertSame(1, WorkflowTask::query()->where('workflow_run_id', $runId) + ->where('task_type', 'workflow')->whereIn('status', ['ready', 'leased'])->count()); + $successor = WorkflowTask::query()->findOrFail($completed->json('created_task_ids.0')); + $this->assertSame($requestId, $successor->payload['resume_source_id']); + $this->assertSame('cancellation_request', $successor->payload['resume_source_kind']); + + // A repeated completion cannot publish a second successor or prefix. + $this->complete($task, [ + ['type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 7)], + ])->assertStatus(409); + $this->assertSame(2, WorkflowTask::query()->where('workflow_run_id', $runId) + ->where('task_type', 'workflow')->count()); + $this->assertSame(2, $this->eventCount($runId, HistoryEventType::SideEffectRecorded)); + + $this->register('successor', true); + $resumed = $this->poll('successor')->assertOk()->json('task'); + $this->assertSame($successor->id, $resumed['task_id']); + $this->assertSame($requestId, $resumed['cancellation_request']['request_id']); + $this->deliver($resumed, $requestId, ['sequence' => 3])->assertOk()->assertJsonPath('delivered', true); + $this->complete($resumed, [['type' => 'complete_workflow']])->assertOk() + ->assertJsonPath('run_status', 'cancelled')->assertJsonPath('created_task_ids', []); + $this->assertSame(1, $this->eventCount($runId, HistoryEventType::CooperativeCancellationDelivered)); + $this->assertSame(0, WorkflowTask::query()->where('workflow_run_id', $runId) + ->where('task_type', 'workflow')->whereIn('status', ['ready', 'leased'])->count()); + } + public function test_delivery_fences_the_actual_claim_attempt_owner_and_request(): void { [$workflowId, $runId] = $this->start(); From 3c15bfb0f98e038febb657412f1ca97bbdb23ad8 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 15:07:41 +0000 Subject: [PATCH 09/57] Avoid cancellation successor reads for ordinary worker claims --- app/Http/Controllers/Api/WorkerController.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index 75a5a28a..d5e7174a 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -1843,7 +1843,9 @@ function () use ( WorkerProtocol::requestVersion($request), ); $outcome = $bridge->complete($taskId, $commands); - if (($outcome['completed'] ?? false) === true && $claimedTask instanceof WorkflowTask) { + if (($outcome['completed'] ?? false) === true + && $claimedTask instanceof WorkflowTask + && CooperativeCancellationPolicy::claimSupportsCancellation($claimedTask)) { $successor = CooperativeCancellationPolicy::resumeUndeliveredRequest($claimedTask->refresh()); if ($successor instanceof WorkflowTask) { $outcome['created_task_ids'][] = $successor->id; From 7675ab976c7c24b7aadfc940fd8930ef2e907a7c Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 20:15:59 +0000 Subject: [PATCH 10/57] fix: use published workflow 2.3.3 deadline recovery --- composer.json | 2 +- composer.lock | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/composer.json b/composer.json index 99f9652c..e8ab0550 100644 --- a/composer.json +++ b/composer.json @@ -6,7 +6,7 @@ "require": { "php": "^8.2", "apache/avro": "^1.12", - "durable-workflow/workflow": "2.3.2", + "durable-workflow/workflow": "2.3.3", "laravel/framework": "^13.30", "laravel/tinker": "^3.0", "league/flysystem-aws-s3-v3": "^3.35.3" diff --git a/composer.lock b/composer.lock index 5ce093af..15df71db 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f2948ea5779b2cc004db4330c85d7a32", + "content-hash": "d32f88e742d9ca2ce35fe99df1a2a0d9", "packages": [ { "name": "apache/avro", @@ -655,16 +655,16 @@ }, { "name": "durable-workflow/workflow", - "version": "2.3.2", + "version": "2.3.3", "source": { "type": "git", "url": "https://github.com/durable-workflow/workflow.git", - "reference": "d8594a67661f63f17d76cd5a1a1349c2d28f4a6d" + "reference": "70d4fe48efd7dd796c35c1078b3d5ac43f738f4f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/durable-workflow/workflow/zipball/d8594a67661f63f17d76cd5a1a1349c2d28f4a6d", - "reference": "d8594a67661f63f17d76cd5a1a1349c2d28f4a6d", + "url": "https://api.github.com/repos/durable-workflow/workflow/zipball/70d4fe48efd7dd796c35c1078b3d5ac43f738f4f", + "reference": "70d4fe48efd7dd796c35c1078b3d5ac43f738f4f", "shasum": "" }, "require": { @@ -697,7 +697,7 @@ "dev-main": "2.0.x-dev" }, "durable-workflow": { - "product-train": "2.3.2", + "product-train": "2.3.3", "laravel-embedded-upgrade-contract": "resources/laravel-embedded-upgrade-contract.json", "laravel-dependency-security-policy": "resources/laravel-dependency-security-policy.json" } @@ -724,9 +724,9 @@ "description": "Embedded durable workflow runtime and orchestration engine for Laravel applications.", "support": { "issues": "https://github.com/durable-workflow/workflow/issues", - "source": "https://github.com/durable-workflow/workflow/tree/2.3.2" + "source": "https://github.com/durable-workflow/workflow/tree/2.3.3" }, - "time": "2026-10-01T17:45:47+00:00" + "time": "2026-10-01T19:54:10+00:00" }, { "name": "egulias/email-validator", From 42e8a8318446693566d1dc17578bb38e6218fde9 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 21:25:47 +0000 Subject: [PATCH 11/57] feat: return pending child cancellation without releasing the claim --- .../Api/CooperativeCancellationController.php | 3 +- .../worker-protocol-api.openapi.yaml | 43 +++++++++++++------ .../CooperativeCancellationProtocolTest.php | 29 +++++++++++++ 3 files changed, 60 insertions(+), 15 deletions(-) diff --git a/app/Http/Controllers/Api/CooperativeCancellationController.php b/app/Http/Controllers/Api/CooperativeCancellationController.php index 1706feee..9a9aec91 100644 --- a/app/Http/Controllers/Api/CooperativeCancellationController.php +++ b/app/Http/Controllers/Api/CooperativeCancellationController.php @@ -211,7 +211,8 @@ public function deliver(Request $request, string $taskId): JsonResponse ); })); - return WorkerProtocol::json($result, ($result['delivered'] ?? false) ? 200 + return WorkerProtocol::json($result, (($result['delivered'] ?? false) + || ($result['reason'] ?? null) === 'cancellation_waiting_for_child') ? 200 : (($result['reason'] ?? null) === 'task_not_found' ? 404 : 409)); } } diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index ecd14341..9159286f 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -32,6 +32,7 @@ x-durable-workflow-cooperative-cancellation-contract: delivery_operation: POST /worker/workflow-tasks/{taskId}/deliver-cancellation canonical_history_event: CooperativeCancellationDelivered delivery_identity: [request_id, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span] + pending_child_delivery: { status: 200, delivered: false, reason: cancellation_waiting_for_child, lease_released: false, canonical_delivery_recorded: false } claim_proof: immutable_registration_snapshot_bound_to_task_run_owner_and_attempt request_against_incompatible_active_claim: { status: 409, reason: active_claim_cancellation_not_supported, history_appended: false } request_claim_race: serialized_on_workflow_run_lock @@ -384,7 +385,7 @@ paths: schema: { $ref: "#/components/schemas/CooperativeCancellationDeliveryRequest" } responses: "200": - description: Canonical delivery committed or an identical acknowledgment retried. + description: Canonical delivery committed, an identical acknowledgment retried, or child cancellation completion still pending. content: application/json: schema: { $ref: "#/components/schemas/CooperativeCancellationDeliveryResponse" } @@ -1370,19 +1371,33 @@ components: CooperativeCancellationDeliveryResponse: allOf: - $ref: "#/components/schemas/WorkerEnvelope" - - type: object - required: [delivered, task_id, workflow_run_id, request_id, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span, reason] - properties: - delivered: { type: boolean, const: true } - task_id: { type: string, minLength: 1 } - workflow_run_id: { type: string, minLength: 1 } - request_id: { type: string, minLength: 1 } - sequence: { type: integer, minimum: 1 } - call_kind: { type: string, enum: [activity, local_activity, timer, condition, signal, child, parallel, selection_handle] } - sequence_span: { type: integer, minimum: 1 } - operation_sequence: { type: [integer, "null"], minimum: 1 } - operation_sequence_span: { type: integer, minimum: 1 } - reason: { type: "null" } + - oneOf: + - type: object + required: [delivered, task_id, workflow_run_id, request_id, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span, reason] + properties: + delivered: { type: boolean, const: true } + task_id: { type: string, minLength: 1 } + workflow_run_id: { type: string, minLength: 1 } + request_id: { type: string, minLength: 1 } + sequence: { type: integer, minimum: 1 } + call_kind: { type: string, enum: [activity, local_activity, timer, condition, signal, child, parallel, selection_handle] } + sequence_span: { type: integer, minimum: 1 } + operation_sequence: { type: [integer, "null"], minimum: 1 } + operation_sequence_span: { type: integer, minimum: 1 } + reason: { type: "null" } + - type: object + required: [delivered, task_id, workflow_run_id, request_id, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span, reason] + properties: + delivered: { type: boolean, const: false } + task_id: { type: string, minLength: 1 } + workflow_run_id: { type: string, minLength: 1 } + request_id: { type: "null" } + sequence: { type: "null" } + call_kind: { type: "null" } + sequence_span: { type: "null" } + operation_sequence: { type: "null" } + operation_sequence_span: { type: "null" } + reason: { type: string, const: cancellation_waiting_for_child } WorkerHistoryEvent: type: object additionalProperties: true diff --git a/tests/Feature/CooperativeCancellationProtocolTest.php b/tests/Feature/CooperativeCancellationProtocolTest.php index 8284e06e..09899dde 100644 --- a/tests/Feature/CooperativeCancellationProtocolTest.php +++ b/tests/Feature/CooperativeCancellationProtocolTest.php @@ -14,6 +14,8 @@ use Tests\Support\OpenApiSchema; use Tests\TestCase; use Workflow\Serializers\Serializer; +use Workflow\V2\Contracts\CooperativeWorkflowTaskBridge; +use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Jobs\RunTimerTask; @@ -122,6 +124,33 @@ public function test_delivery_retry_records_one_marker_and_keeps_cleanup_authori ->assertJsonPath('reason', 'cancellation_delivery_mismatch'); } + public function test_child_acknowledgement_pending_is_a_valid_reply_and_keeps_the_exact_claim(): void + { + [$workflowId, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $requestId = $this->requestCancellation($workflowId)->json('cancellation_request.request_id'); + $pending = ['delivered' => false, 'task_id' => $task['task_id'], 'workflow_run_id' => $runId, + 'request_id' => null, 'sequence' => null, 'call_kind' => null, 'sequence_span' => null, + 'operation_sequence' => null, 'operation_sequence_span' => null, + 'reason' => 'cancellation_waiting_for_child']; + $bridge = \Mockery::mock(CooperativeWorkflowTaskBridge::class); + $bridge->shouldReceive('deliverCancellation')->once()->with( + $task['task_id'], $requestId, 1, 'child', 1, null, 1, + )->andReturn($pending); + $this->app->instance(WorkflowTaskBridge::class, $bridge); + + $response = $this->deliver($task, $requestId, ['call_kind' => 'child'])->assertOk() + ->assertJsonPath('delivered', false)->assertJsonPath('reason', 'cancellation_waiting_for_child'); + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/CooperativeCancellationDeliveryResponse/allOf/1', json_decode($response->getContent(), flags: JSON_THROW_ON_ERROR)); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::CooperativeCancellationDelivered)); + $claim = WorkflowTask::query()->findOrFail($task['task_id']); + $this->assertSame(TaskStatus::Leased, $claim->status); + $this->assertSame($task['lease_owner'], $claim->lease_owner); + $this->assertSame($task['workflow_task_attempt'], $claim->attempt_count); + } + public static function cancellationRequestTiming(): array { return ['before claim' => [true], 'after claim' => [false]]; From 4d9a4248d740579b99668ed97e94c893959fba0a Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 21:37:49 +0000 Subject: [PATCH 12/57] fix: acknowledge parked child cancellation claims explicitly --- .../worker-protocol-api.openapi.yaml | 5 +++-- tests/Feature/CooperativeCancellationProtocolTest.php | 11 ++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 9159286f..c394697c 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -32,7 +32,7 @@ x-durable-workflow-cooperative-cancellation-contract: delivery_operation: POST /worker/workflow-tasks/{taskId}/deliver-cancellation canonical_history_event: CooperativeCancellationDelivered delivery_identity: [request_id, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span] - pending_child_delivery: { status: 200, delivered: false, reason: cancellation_waiting_for_child, lease_released: false, canonical_delivery_recorded: false } + pending_child_delivery: { status: 200, delivered: false, reason: cancellation_waiting_for_child, claim_released: true, canonical_delivery_recorded: false, resume_source: child_terminal_history } claim_proof: immutable_registration_snapshot_bound_to_task_run_owner_and_attempt request_against_incompatible_active_claim: { status: 409, reason: active_claim_cancellation_not_supported, history_appended: false } request_claim_race: serialized_on_workflow_run_lock @@ -1386,9 +1386,10 @@ components: operation_sequence_span: { type: integer, minimum: 1 } reason: { type: "null" } - type: object - required: [delivered, task_id, workflow_run_id, request_id, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span, reason] + required: [delivered, task_id, workflow_run_id, request_id, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span, reason, claim_released] properties: delivered: { type: boolean, const: false } + claim_released: { type: boolean, const: true } task_id: { type: string, minLength: 1 } workflow_run_id: { type: string, minLength: 1 } request_id: { type: "null" } diff --git a/tests/Feature/CooperativeCancellationProtocolTest.php b/tests/Feature/CooperativeCancellationProtocolTest.php index 09899dde..95e6edb1 100644 --- a/tests/Feature/CooperativeCancellationProtocolTest.php +++ b/tests/Feature/CooperativeCancellationProtocolTest.php @@ -124,7 +124,7 @@ public function test_delivery_retry_records_one_marker_and_keeps_cleanup_authori ->assertJsonPath('reason', 'cancellation_delivery_mismatch'); } - public function test_child_acknowledgement_pending_is_a_valid_reply_and_keeps_the_exact_claim(): void + public function test_child_acknowledgement_pending_is_a_valid_reply_with_explicit_claim_release(): void { [$workflowId, $runId] = $this->start(); $this->register('new', true); @@ -133,7 +133,7 @@ public function test_child_acknowledgement_pending_is_a_valid_reply_and_keeps_th $pending = ['delivered' => false, 'task_id' => $task['task_id'], 'workflow_run_id' => $runId, 'request_id' => null, 'sequence' => null, 'call_kind' => null, 'sequence_span' => null, 'operation_sequence' => null, 'operation_sequence_span' => null, - 'reason' => 'cancellation_waiting_for_child']; + 'reason' => 'cancellation_waiting_for_child', 'claim_released' => true]; $bridge = \Mockery::mock(CooperativeWorkflowTaskBridge::class); $bridge->shouldReceive('deliverCancellation')->once()->with( $task['task_id'], $requestId, 1, 'child', 1, null, 1, @@ -141,14 +141,11 @@ public function test_child_acknowledgement_pending_is_a_valid_reply_and_keeps_th $this->app->instance(WorkflowTaskBridge::class, $bridge); $response = $this->deliver($task, $requestId, ['call_kind' => 'child'])->assertOk() - ->assertJsonPath('delivered', false)->assertJsonPath('reason', 'cancellation_waiting_for_child'); + ->assertJsonPath('delivered', false)->assertJsonPath('reason', 'cancellation_waiting_for_child') + ->assertJsonPath('claim_released', true); OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) ->assertReferenceMatches('#/components/schemas/CooperativeCancellationDeliveryResponse/allOf/1', json_decode($response->getContent(), flags: JSON_THROW_ON_ERROR)); $this->assertSame(0, $this->eventCount($runId, HistoryEventType::CooperativeCancellationDelivered)); - $claim = WorkflowTask::query()->findOrFail($task['task_id']); - $this->assertSame(TaskStatus::Leased, $claim->status); - $this->assertSame($task['lease_owner'], $claim->lease_owner); - $this->assertSame($task['workflow_task_attempt'], $claim->attempt_count); } public static function cancellationRequestTiming(): array From eafa77721b6840234e0de11e18a4a2aa10deb299 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 22:18:26 +0000 Subject: [PATCH 13/57] Preserve and gate cooperative child cancellation commands --- app/Http/Controllers/Api/WorkerController.php | 76 ++++++++++++- app/Support/CooperativeCancellationPolicy.php | 32 ++++++ .../worker-protocol-api.openapi.yaml | 15 ++- .../CooperativeCancellationProtocolTest.php | 101 ++++++++++++++++++ 4 files changed, 220 insertions(+), 4 deletions(-) diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index d5e7174a..c4734bdf 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -1535,6 +1535,7 @@ public function completeWorkflowTask(Request $request, string $taskId): JsonResp 'commands.*.entries' => ['nullable', 'array'], 'commands.*.non_retryable' => ['nullable', 'boolean'], 'commands.*.parent_close_policy' => ['nullable', 'string'], + 'commands.*.cancellation_policy' => ['nullable', 'string', 'in:try_cancel,wait_cancellation_completed,abandon'], 'commands.*.condition_key' => ['nullable', 'string'], 'commands.*.condition_definition_fingerprint' => ['nullable', 'string'], 'commands.*.condition_wait_occurrence_id' => ['nullable', 'string'], @@ -1659,6 +1660,16 @@ public function completeWorkflowTask(Request $request, string $taskId): JsonResp return $response; } + if ($response = $this->guardChildCancellationPoliciesAvailable( + $request, + (string) $namespace, + $taskId, + (int) $validated['workflow_task_attempt'], + $commands, + )) { + return $response; + } + $commands = $this->authorizeServiceOperationCommands($request, (string) $namespace, $commands); if ($response = $this->guardWorkerSessionCommandsAvailable( @@ -1818,6 +1829,17 @@ function () use ( return $response; } + if ($response = $this->guardChildCancellationPoliciesAvailable( + $request, + (string) $namespace, + $taskId, + (int) $validated['workflow_task_attempt'], + $commands, + $claimedTask, + )) { + return $response; + } + if ($response = $this->guardPortableWorkerAffinityCompletion( $request, (string) $namespace, @@ -1980,9 +2002,52 @@ function () use ( ], $this->workflowOutcomeStatus($outcome['reason'])); } - /** - * @param list> $commands - */ + /** @param array> $commands */ + private function guardChildCancellationPoliciesAvailable( + Request $request, + string $namespace, + string $taskId, + int $workflowTaskAttempt, + array $commands, + ?WorkflowTask $claimedTask = null, + ): ?JsonResponse { + $usesPolicy = false; + foreach ($commands as $command) { + if (($command['type'] ?? null) === 'start_child_workflow' + && (($command['parent_close_policy'] ?? null) === 'request_cancellation' + || in_array($command['cancellation_policy'] ?? null, ['try_cancel', 'wait_cancellation_completed'], true))) { + $usesPolicy = true; + break; + } + } + if (! $usesPolicy) { + return null; + } + $task = $claimedTask ?? NamespaceWorkflowScope::taskQuery($namespace)->whereKey($taskId)->first(); + $unavailable = CooperativeCancellationPolicy::childPolicyUnavailableReasons( + $task, + WorkerProtocol::requestVersion($request), + ); + if ($unavailable === []) { + return null; + } + + return WorkerProtocol::json([ + 'task_id' => $taskId, + 'workflow_task_attempt' => $workflowTaskAttempt, + 'worker_id' => $task?->lease_owner, + 'outcome' => 'rejected', + 'recorded' => false, + 'reason' => 'child_cancellation_policy_not_supported', + 'required_capability' => CooperativeCancellationPolicy::CAPABILITY, + 'minimum_protocol_version' => CooperativeCancellationPolicy::MINIMUM_PROTOCOL_VERSION, + 'requested_version' => WorkerProtocol::requestVersion($request), + 'unavailable' => $unavailable, + 'remediation' => 'Use a runtime with child cancellation policies and a claim from a cooperative worker on protocol 1.20 or newer.', + ], 409); + } + + /** @param list> $commands */ private function guardWorkerSessionCommandsAvailable( Request $request, string $taskId, @@ -2685,6 +2750,11 @@ private function validateWorkflowTaskCommandScopes(array $commands): void } } + if ($this->hasCommandValue($command, 'cancellation_policy') && $type !== 'start_child_workflow') { + $errors["commands.{$index}.cancellation_policy"][] = + 'cancellation_policy is only supported for start_child_workflow commands.'; + } + if ($this->hasCommandValue($command, 'non_retryable') && ! in_array($type, ['fail_workflow', 'fail_update', 'record_local_activity'], true) ) { diff --git a/app/Support/CooperativeCancellationPolicy.php b/app/Support/CooperativeCancellationPolicy.php index 0ebfbb6f..cf4d9232 100644 --- a/app/Support/CooperativeCancellationPolicy.php +++ b/app/Support/CooperativeCancellationPolicy.php @@ -3,10 +3,14 @@ namespace App\Support; use App\Models\WorkerRegistration; +use Workflow\V2\Enums\CancellationPolicy; +use Workflow\V2\Enums\ParentClosePolicy; use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Enums\TaskType; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; +use Workflow\V2\Support\ChildCancellation; +use Workflow\V2\WorkflowStub; final class CooperativeCancellationPolicy { @@ -24,6 +28,34 @@ public static function serverSupported(): bool ); } + public static function childPolicyBackendSupported(): bool + { + return enum_exists(CancellationPolicy::class) + && defined(ParentClosePolicy::class.'::RequestCancellation') + && class_exists(ChildCancellation::class) + && method_exists(WorkflowStub::class, 'attemptRequestCancellationFromParent'); + } + + /** @return list */ + public static function childPolicyUnavailableReasons(?WorkflowTask $task, ?string $requestVersion): array + { + $reasons = []; + if (! self::serverSupported()) { + $reasons[] = 'server_protocol'; + } + if (! WorkerProtocol::versionMeetsMinimum($requestVersion, self::MINIMUM_PROTOCOL_VERSION)) { + $reasons[] = 'request_protocol'; + } + if ($task === null || ! self::claimSupportsCancellation($task)) { + $reasons[] = 'worker_claim_capability'; + } + if (! self::childPolicyBackendSupported()) { + $reasons[] = 'installed_runtime_child_policy'; + } + + return $reasons; + } + /** @return array */ public static function workerSnapshot(WorkerRegistration $worker, ?string $protocolVersion): array { diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index c394697c..d4b5324d 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "25" + version: "26" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -33,6 +33,11 @@ x-durable-workflow-cooperative-cancellation-contract: canonical_history_event: CooperativeCancellationDelivered delivery_identity: [request_id, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span] pending_child_delivery: { status: 200, delivered: false, reason: cancellation_waiting_for_child, claim_released: true, canonical_delivery_recorded: false, resume_source: child_terminal_history } + child_operation_policies: [try_cancel, wait_cancellation_completed, abandon] + cooperative_parent_close_policy: request_cancellation + legacy_terminal_parent_close_policy: request_cancel + child_policy_admission: [server_protocol, request_protocol, immutable_worker_claim_capability, installed_runtime_child_policy] + unsupported_child_policy: { status: 409, reason: child_cancellation_policy_not_supported, recorded: false } claim_proof: immutable_registration_snapshot_bound_to_task_run_owner_and_attempt request_against_incompatible_active_claim: { status: 409, reason: active_claim_cancellation_not_supported, history_appended: false } request_claim_race: serialized_on_workflow_run_lock @@ -1795,6 +1800,14 @@ components: required: [type] properties: type: { type: string } + parent_close_policy: + type: [string, "null"] + description: Child parent-close policy. request_cancel retains legacy terminal cancellation. request_cancellation requests cooperative cleanup under protocol 1.20 and the original shared budget. + cancellation_policy: + type: [string, "null"] + enum: [try_cancel, wait_cancellation_completed, abandon, null] + x-durable-workflow-minimum-protocol-version: "1.20" + description: Per-operation child cancellation policy. Absent or null retains abandon. Cooperative policies require a capable immutable worker claim and an installed runtime implementation. allOf: - if: properties: diff --git a/tests/Feature/CooperativeCancellationProtocolTest.php b/tests/Feature/CooperativeCancellationProtocolTest.php index 95e6edb1..7704e31b 100644 --- a/tests/Feature/CooperativeCancellationProtocolTest.php +++ b/tests/Feature/CooperativeCancellationProtocolTest.php @@ -20,6 +20,7 @@ use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Jobs\RunTimerTask; use Workflow\V2\Models\WorkflowHistoryEvent; +use Workflow\V2\Models\WorkflowLink; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; use Workflow\V2\Models\WorkflowTimer; @@ -431,6 +432,106 @@ public function test_run_target_and_namespace_are_checked_before_request_admissi $this->requestCancellation('missing')->assertNotFound()->assertJsonPath('reason', 'instance_not_found'); } + public function test_child_policy_is_preserved_or_reports_the_missing_installed_runtime(): void + { + [, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $response = $this->complete($task, [[ + 'type' => 'start_child_workflow', + 'workflow_type' => 'tests.external-greeting-workflow', + 'arguments' => Serializer::serialize(['child']), + 'cancellation_policy' => 'wait_cancellation_completed', + 'parent_close_policy' => 'request_cancellation', + ]]); + if (! CooperativeCancellationPolicy::childPolicyBackendSupported()) { + $response->assertStatus(409)->assertJsonPath('reason', 'child_cancellation_policy_not_supported') + ->assertJsonPath('unavailable', ['installed_runtime_child_policy'])->assertJsonPath('recorded', false); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::ChildWorkflowScheduled)); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + + return; + } + $response->assertOk()->assertJsonPath('outcome', 'completed'); + $event = WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::ChildWorkflowScheduled)->sole(); + $this->assertSame('wait_cancellation_completed', $event->payload['cancellation_policy']); + $this->assertSame('request_cancellation', $event->payload['parent_close_policy']); + $this->assertSame('request_cancellation', WorkflowLink::query() + ->where('parent_workflow_run_id', $runId)->where('link_type', 'child_workflow')->sole()->parent_close_policy); + } + + public function test_old_claim_cannot_acquire_child_policy_support_from_a_changed_registration(): void + { + [, $runId] = $this->start(); + $this->register('old', false); + $task = $this->poll('old', '1.19')->json('task'); + WorkerRegistration::query()->where('worker_id', 'old')->sole()->forceFill([ + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY], + ])->save(); + $this->complete($task, [[ + 'type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', + 'cancellation_policy' => 'try_cancel', + ]])->assertStatus(409)->assertJsonPath('reason', 'child_cancellation_policy_not_supported') + ->assertJsonPath('worker_id', 'old')->assertJsonPath('recorded', false) + ->assertJsonFragment(['worker_claim_capability']); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::ChildWorkflowScheduled)); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + } + + public function test_capable_claim_still_requires_a_cooperative_completion_protocol(): void + { + [, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $this->withHeaders($this->headers('1.19')) + ->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [[ + 'type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', + 'parent_close_policy' => 'request_cancellation', + ]], + ])->assertStatus(409)->assertJsonPath('reason', 'child_cancellation_policy_not_supported') + ->assertJsonFragment(['request_protocol']); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::ChildWorkflowScheduled)); + } + + public function test_invalid_or_misplaced_child_cancellation_policy_is_rejected_before_completion(): void + { + [, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + foreach ([ + ['type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', 'cancellation_policy' => 'unknown'], + ['type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', 'cancellation_policy' => []], + ['type' => 'start_timer', 'delay_seconds' => 1, 'cancellation_policy' => 'try_cancel'], + ] as $command) { + $this->complete($task, [$command])->assertStatus(422) + ->assertJsonValidationErrors('commands.0.cancellation_policy'); + } + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::ChildWorkflowScheduled)); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::TimerScheduled)); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + } + + public function test_legacy_terminal_parent_close_policy_keeps_its_old_protocol_path(): void + { + [, $runId] = $this->start(); + $this->register('old', false); + $task = $this->poll('old', '1.19')->json('task'); + $this->withHeaders($this->headers('1.19')) + ->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [[ + 'type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', + 'parent_close_policy' => 'request_cancel', + ]], + ])->assertOk()->assertJsonPath('outcome', 'completed'); + $event = WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::ChildWorkflowScheduled)->sole(); + $this->assertSame('request_cancel', $event->payload['parent_close_policy']); + } + private function complete(array $task, array $commands): TestResponse { return $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ From a63b06a5dfa3e6f9b7c3bff2c0647ea07539d43a Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 23:59:08 +0000 Subject: [PATCH 14/57] Admit fenced remote callback-stop receipts --- .../Api/ActivityTaskController.php | 145 +++++++++++++++ .../Middleware/EnforceStorageAdmission.php | 1 + app/Support/ActivityTaskPoller.php | 11 +- app/Support/CooperativeCancellationPolicy.php | 7 + app/Support/WorkerProtocol.php | 3 + docs/contracts/external-execution-surface.md | 28 +++ .../worker-protocol-api.openapi.yaml | 84 ++++++++- routes/api.php | 1 + tests/Feature/ActivityAttemptStatusTest.php | 169 +++++++++++++++++- .../WorkerProtocolVersionCoverageTest.php | 2 + 10 files changed, 447 insertions(+), 4 deletions(-) diff --git a/app/Http/Controllers/Api/ActivityTaskController.php b/app/Http/Controllers/Api/ActivityTaskController.php index bcff482e..6bea3576 100644 --- a/app/Http/Controllers/Api/ActivityTaskController.php +++ b/app/Http/Controllers/Api/ActivityTaskController.php @@ -27,12 +27,19 @@ use Carbon\CarbonInterface; use Illuminate\Http\JsonResponse; use Illuminate\Http\Request; +use Illuminate\Support\Facades\DB; use Illuminate\Validation\Rule; use Illuminate\Validation\ValidationException; use InvalidArgumentException; use Workflow\V2\Contracts\ActivityTaskBridge as ActivityTaskBridgeContract; +use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Exceptions\ExternalPayloadIntegrityException; use Workflow\V2\Models\ActivityExecution; +use Workflow\V2\Models\WorkflowHistoryEvent; +use Workflow\V2\Models\WorkflowRun; +use Workflow\V2\Support\ActivityCancellationAcknowledgement; +use Workflow\V2\Support\ActivityRowLockOrder; +use Workflow\V2\Support\CooperativeCancellationDelivery; use Workflow\V2\Support\WorkerProtocolVersion; class ActivityTaskController @@ -121,6 +128,7 @@ public function poll(Request $request): JsonResponse $request, ), timeoutSeconds: $timeoutSeconds, + protocolVersion: WorkerProtocol::requestVersion($request), ); } catch (\Throwable $exception) { if ($exception instanceof InvalidArgumentException @@ -449,6 +457,15 @@ public function status(Request $request, string $taskId): JsonResponse } $deadlines = $this->executionDeadlines($status['activity_execution_id'] ?? null) ?? []; $workerSession = $this->workerSessions->workerSessionForExecution($status['activity_execution_id'] ?? null); + $cancellation = $status['cancel_requested'] === true + ? $this->activityCancellationReceipt( + (string) $request->attributes->get('namespace'), + $taskId, + $validated['activity_attempt_id'], + $validated['lease_owner'], + $status['activity_execution_id'] ?? null, + ) + : null; } catch (\Throwable $exception) { if (! BackendLockPressure::is($exception)) { throw $exception; @@ -501,9 +518,137 @@ public function status(Request $request, string $taskId): JsonResponse 'task_status' => $status['task_status'], 'deadlines' => $deadlines === [] ? null : $deadlines, 'worker_session' => $workerSession, + ...($cancellation === null ? [] : ['cancellation_acknowledgement' => $cancellation]), ]); } + /** Record an original owner's report after its remote callback has stopped and joined. */ + public function acknowledgeCancellation(Request $request, string $taskId): JsonResponse + { + if ($response = WorkerProtocol::rejectUnsupported($request)) { + return $response; + } + if (! CooperativeCancellationPolicy::serverSupported() + || ! WorkerProtocol::versionMeetsMinimum(WorkerProtocol::requestVersion($request), '1.20')) { + return WorkerProtocol::json([ + 'acknowledged' => false, + 'reason' => 'activity_cancellation_acknowledgement_not_supported', + 'minimum_protocol_version' => '1.20', + 'unavailable' => ['worker_protocol'], + ], 422); + } + $validated = $request->validate([ + 'activity_attempt_id' => ['required', 'string', 'max:255'], + 'lease_owner' => ['required', 'string', 'max:255'], + 'request_id' => ['required', 'string', 'max:255'], + ]); + $namespace = (string) $request->attributes->get('namespace'); + try { + if ($response = $this->guardAttemptOwnership( + $namespace, $taskId, $validated['activity_attempt_id'], $validated['lease_owner'], + )) { + return $response; + } + if (! CooperativeCancellationPolicy::activityAcknowledgementBackendSupported()) { + return $this->activityAcknowledgementUnavailable($validated['lease_owner'], ['installed_runtime_activity_acknowledgement']); + } + return $this->storageMutations->run(fn (): JsonResponse => DB::transaction(function () use ($namespace, $taskId, $validated): JsonResponse { + // Acquire the package's activity lock order before checking the + // immutable claim and calling the nested package transaction. + $rows = ActivityRowLockOrder::lockForAttempt($validated['activity_attempt_id']); + $attempt = $rows['attempt']; + if ($attempt === null || $attempt->workflow_task_id !== $taskId) { + return WorkerProtocol::json(['acknowledged' => false, 'reason' => 'activity_attempt_not_found'], 404); + } + $run = WorkflowRun::query()->where('namespace', $namespace)->lockForUpdate()->find($attempt->workflow_run_id); + $task = NamespaceWorkflowScope::taskQuery($namespace)->lockForUpdate()->find($taskId); + if ($run === null || $task === null || $task->workflow_run_id !== $run->id) { + return WorkerProtocol::json(['acknowledged' => false, 'reason' => 'task_not_found'], 404); + } + if (! CooperativeCancellationPolicy::claimSupportsCancellation($task)) { + return $this->activityAcknowledgementUnavailable($validated['lease_owner'], ['worker_claim_capability']); + } + $outcome = ActivityCancellationAcknowledgement::recordStopped( + $validated['activity_attempt_id'], $validated['lease_owner'], $validated['request_id'], + ); + return WorkerProtocol::json([ + 'task_id' => $taskId, + 'activity_attempt_id' => $validated['activity_attempt_id'], + 'lease_owner' => $validated['lease_owner'], + 'request_id' => $validated['request_id'], + ...$outcome, + 'heartbeat_recorded' => false, + ], $outcome['acknowledged'] ? 200 : ($outcome['reason'] === 'activity_attempt_not_found' ? 404 : 409)); + })); + } catch (\Throwable $exception) { + if (! BackendLockPressure::is($exception)) { + throw $exception; + } + return BackendLockPressure::workerOperationResponse($request, false); + } + } + + /** @param list $unavailable */ + private function activityAcknowledgementUnavailable(string $workerId, array $unavailable): JsonResponse + { + return WorkerProtocol::json([ + 'acknowledged' => false, + 'reason' => 'activity_cancellation_acknowledgement_not_supported', + 'required_capability' => CooperativeCancellationPolicy::CAPABILITY, + 'minimum_protocol_version' => '1.20', + 'worker_id' => $workerId, + 'unavailable' => $unavailable, + 'remediation' => 'Use a runtime with activity stop acknowledgements and an original claim from a cooperative worker on protocol 1.20 or newer.', + ], 409); + } + + /** @return array|null */ + private function activityCancellationReceipt( + string $namespace, string $taskId, string $attemptId, string $leaseOwner, ?string $executionId, + ): ?array { + if (! CooperativeCancellationPolicy::activityAcknowledgementBackendSupported() || $executionId === null) { + return null; + } + $execution = ActivityExecution::query()->find($executionId); + if ($execution === null || $execution->current_attempt_id !== $attemptId) { + return null; + } + $run = WorkflowRun::query()->where('namespace', $namespace)->find($execution->workflow_run_id); + if ($run === null || ! is_string($run->cancellation_request_command_id)) { + return null; + } + $context = CooperativeCancellationDelivery::context($run); + if ($context === null || $context->requestId !== $run->cancellation_request_command_id) { + return null; + } + $events = WorkflowHistoryEvent::query()->where('workflow_run_id', $run->id) + ->where('workflow_command_id', $context->requestId) + ->where('payload->activity_execution_id', $executionId) + ->whereIn('event_type', [HistoryEventType::ActivityCancelled->value, HistoryEventType::ActivityCancellationAcknowledged->value]) + ->get(); + $cancelled = $events->first(fn (WorkflowHistoryEvent $event): bool => $event->event_type === HistoryEventType::ActivityCancelled + && ($event->payload['activity_attempt_id'] ?? null) === $attemptId); + $snapshot = $cancelled?->payload['activity_attempt'] ?? null; + if (! is_array($snapshot) || ($snapshot['id'] ?? null) !== $attemptId + || ($snapshot['task_id'] ?? null) !== $taskId || ($snapshot['lease_owner'] ?? null) !== $leaseOwner + || ($snapshot['activity_execution_id'] ?? null) !== $executionId || ($snapshot['status'] ?? null) !== 'cancelled') { + return null; + } + $ack = $events->first(fn (WorkflowHistoryEvent $event): bool => $event->event_type === HistoryEventType::ActivityCancellationAcknowledged + && ($event->payload['cancellation_history_event_id'] ?? null) === $cancelled->id + && ($event->payload['activity_attempt_id'] ?? null) === $attemptId); + return [ + 'request_id' => $context->requestId, + 'root_request_id' => $context->rootRequestId, + 'cleanup_deadline_at' => $context->deadline()->toISOString(), + 'cancellation_history_event_id' => $cancelled->id, + 'callback_state' => $ack === null ? 'unknown' : 'stopped', + 'history_event_id' => $ack?->id, + 'acknowledged_at' => $ack?->payload['acknowledged_at'] ?? null, + 'received_after_deadline' => $ack?->payload['received_after_deadline'] ?? null, + ]; + } + /** * Heartbeat an in-progress activity task. * diff --git a/app/Http/Middleware/EnforceStorageAdmission.php b/app/Http/Middleware/EnforceStorageAdmission.php index cbabe08b..c8f579c9 100644 --- a/app/Http/Middleware/EnforceStorageAdmission.php +++ b/app/Http/Middleware/EnforceStorageAdmission.php @@ -25,6 +25,7 @@ final class EnforceStorageAdmission 'ActivityTaskController@complete', 'ActivityTaskController@fail', 'ActivityTaskController@heartbeat', + 'ActivityTaskController@acknowledgeCancellation', 'WorkerSessionController@heartbeat', ]; diff --git a/app/Support/ActivityTaskPoller.php b/app/Support/ActivityTaskPoller.php index d2b7e77e..277ea853 100644 --- a/app/Support/ActivityTaskPoller.php +++ b/app/Support/ActivityTaskPoller.php @@ -46,6 +46,7 @@ public function poll( array $supportedActivityTypes = [], bool $workerSessionsAvailable = true, ?int $timeoutSeconds = null, + ?string $protocolVersion = null, ): array { $pollRequestId = $this->nonEmptyString($pollRequestId); @@ -87,6 +88,7 @@ public function poll( supportedActivityTypes: $supportedActivityTypes, workerSessionsAvailable: $workerSessionsAvailable, timeoutSeconds: $timeoutSeconds, + protocolVersion: $protocolVersion, ); } @@ -100,6 +102,7 @@ public function poll( supportedActivityTypes: $supportedActivityTypes, workerSessionsAvailable: $workerSessionsAvailable, timeoutSeconds: $timeoutSeconds, + protocolVersion: $protocolVersion, ); } @@ -117,6 +120,7 @@ private function coordinatedPoll( array $supportedActivityTypes = [], bool $workerSessionsAvailable = true, ?int $timeoutSeconds = null, + ?string $protocolVersion = null, ): array { $workerPollFence = WorkerPollFence::snapshot($worker); @@ -161,6 +165,7 @@ private function coordinatedPoll( supportedActivityTypes: $supportedActivityTypes, workerSessionsAvailable: $workerSessionsAvailable, timeoutSeconds: $timeoutSeconds, + protocolVersion: $protocolVersion, ); } @@ -380,6 +385,7 @@ private function runCoordinatedPollLeader( array $supportedActivityTypes = [], bool $workerSessionsAvailable = true, ?int $timeoutSeconds = null, + ?string $protocolVersion = null, ): array { try { $task = $this->performPoll( @@ -392,6 +398,7 @@ private function runCoordinatedPollLeader( supportedActivityTypes: $supportedActivityTypes, workerSessionsAvailable: $workerSessionsAvailable, timeoutSeconds: $timeoutSeconds, + protocolVersion: $protocolVersion, ); } catch (\Throwable $exception) { $this->pollRequests->forgetPending( @@ -432,6 +439,7 @@ private function performPoll( array $supportedActivityTypes = [], bool $workerSessionsAvailable = true, ?int $timeoutSeconds = null, + ?string $protocolVersion = null, ): array { $limit = max(10, max(1, (int) config('server.polling.max_tasks_per_poll', 1)) * 10); $nextProbeAt = null; @@ -440,7 +448,7 @@ private function performPoll( 'poll_status' => 'empty', 'next_probe_at' => null, ]; - $workerPollFence = WorkerPollFence::snapshot($worker); + $workerPollFence = CooperativeCancellationPolicy::workerSnapshot($worker, $protocolVersion); $crossKindWake = WorkerProtocol::supportsCrossKindPollWake($worker); @@ -932,6 +940,7 @@ private function claimReadyTask( $taskId, $leaseOwner, $pollRequestId, + $workerPollFence, ); return $claim; diff --git a/app/Support/CooperativeCancellationPolicy.php b/app/Support/CooperativeCancellationPolicy.php index cf4d9232..3cf8a856 100644 --- a/app/Support/CooperativeCancellationPolicy.php +++ b/app/Support/CooperativeCancellationPolicy.php @@ -9,6 +9,7 @@ use Workflow\V2\Enums\TaskType; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; +use Workflow\V2\Support\ActivityCancellationAcknowledgement; use Workflow\V2\Support\ChildCancellation; use Workflow\V2\WorkflowStub; @@ -36,6 +37,12 @@ public static function childPolicyBackendSupported(): bool && method_exists(WorkflowStub::class, 'attemptRequestCancellationFromParent'); } + public static function activityAcknowledgementBackendSupported(): bool + { + return class_exists(ActivityCancellationAcknowledgement::class) + && method_exists(ActivityCancellationAcknowledgement::class, 'recordStopped'); + } + /** @return list */ public static function childPolicyUnavailableReasons(?WorkflowTask $task, ?string $requestVersion): array { diff --git a/app/Support/WorkerProtocol.php b/app/Support/WorkerProtocol.php index a9035119..d98e55dc 100644 --- a/app/Support/WorkerProtocol.php +++ b/app/Support/WorkerProtocol.php @@ -414,6 +414,7 @@ public static function workflowMemoUpdateSemantics(): array * activity_retry_policy: bool, * activity_timeouts: bool, * cooperative_cancellation: bool, + * activity_cancellation_acknowledgement: bool, * local_activities: array, * worker_session_verbs: list, * worker_sessions: array, @@ -546,6 +547,8 @@ public static function serverCapabilities(): array 'activity_retry_policy' => true, 'activity_timeouts' => true, 'cooperative_cancellation' => CooperativeCancellationPolicy::serverSupported(), + 'activity_cancellation_acknowledgement' => CooperativeCancellationPolicy::serverSupported() + && CooperativeCancellationPolicy::activityAcknowledgementBackendSupported(), 'local_activities' => [ ...WorkerProtocolVersion::localActivitySemantics(), 'supported' => $portableWorkerAffinitySupported, diff --git a/docs/contracts/external-execution-surface.md b/docs/contracts/external-execution-surface.md index a12b72d5..341989df 100644 --- a/docs/contracts/external-execution-surface.md +++ b/docs/contracts/external-execution-surface.md @@ -77,3 +77,31 @@ Stable adjacent contract docs live in: - `docs/contracts/bridge-adapters.md` - `docs/contracts/external-task-input.md` - `docs/contracts/external-task-result.md` + +## Candidate cooperative activity stop receipts + +The cooperative cancellation draft adds remote callback-stop receipts at worker +protocol 1.20. The default remains 1.19 and the candidate contract is not frozen. +The installed Native runtime must provide the acknowledgment primitive. Discovery +advertises `server_capabilities.activity_cancellation_acknowledgement` only when +both conditions hold. + +`POST /api/worker/activity-tasks/{taskId}/status` remains read-only. After canonical +cooperative cancellation it reports the original request, root identity, immutable +cleanup deadline and cancellation history event. `callback_state: unknown` means +that the worker has not reported an actual stop. A cancelled durable row or +expired lease does not make that state `stopped`. + +The original cooperative worker may post `activity_attempt_id`, `lease_owner` +and `request_id` to `/acknowledge-cancellation` after stopping and joining the +remote callback. The Server checks the namespace, task, original claim capability, +attempt and canonical cancellation snapshot in a fenced transaction. A later +registration or request protocol cannot upgrade an old claim. Duplicates return +the original receipt event and late receipt remains explicitly late. + +This report grants no renewed lease, application heartbeat, result authority or +cleanup budget. It describes the worker's stopped callback, not reversal of +external effects. Local activities require separate workflow claim authority. +Receipt writes are admitted during storage draining and refused when storage is +fenced. SDK stop/join emission, local receipts and activity waiting policies still +need connected qualification before this candidate is published. diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index d4b5324d..e9c01744 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "26" + version: "27" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -595,6 +595,43 @@ paths: "409": { $ref: "#/components/responses/WorkerError" } "422": { $ref: "#/components/responses/WorkerError" } "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/activity-tasks/{taskId}/acknowledge-cancellation: + post: + operationId: acknowledgeActivityCancellation + tags: [activity-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + description: > + Report that the original remote callback has stopped and been joined. + Requires the immutable cooperative worker claim, original task, attempt, + owner and local cancellation request, plus matching canonical cancellation + history. Cancelled rows, lease expiry and a disconnected worker are not + stop reports. Duplicates return the original receipt event. This report + never renews leases, records application heartbeats, restores publication + authority or extends the cleanup deadline. Late receipt remains late. + A worker report does not undo external effects. Local activities require + separate workflow claim authority and cannot use this endpoint. Installed + runtimes without this primitive return explicit capability diagnostics. + Available during draining storage admission. Fenced storage refuses writes. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/ActivityCancellationAcknowledgementRequest" } + responses: + "200": + description: Original callback-stop receipt, including an idempotent duplicate. + content: + application/json: + schema: { $ref: "#/components/schemas/ActivityCancellationAcknowledgementResponse" } + "400": { $ref: "#/components/responses/WorkerError" } + "403": { $ref: "#/components/responses/WorkerError" } + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } /worker/activity-tasks/{taskId}/heartbeat: post: operationId: heartbeatActivityTask @@ -1037,6 +1074,9 @@ components: cooperative_cancellation: type: boolean description: Explicit support for claim-bound cooperative request delivery, requiring worker protocol 1.20 or newer in the same major version. False at the default protocol 1.19. + activity_cancellation_acknowledgement: + type: boolean + description: Original remote callback-stop receipts require an installed runtime primitive and worker protocol 1.20. False for the default protocol and for runtimes without that primitive. supported_workflow_task_commands: type: array uniqueItems: true @@ -1672,6 +1712,45 @@ components: properties: activity_attempt_id: { type: string, minLength: 1 } lease_owner: { type: string, minLength: 1 } + ActivityCancellationAcknowledgementRequest: + type: object + additionalProperties: true + required: [activity_attempt_id, lease_owner, request_id] + properties: + activity_attempt_id: { type: string, minLength: 1, maxLength: 255 } + lease_owner: { type: string, minLength: 1, maxLength: 255 } + request_id: { type: string, minLength: 1, maxLength: 255 } + ActivityCancellationAcknowledgementResponse: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - type: object + required: [task_id, activity_attempt_id, lease_owner, request_id, acknowledged, duplicate, reason, history_event_id, heartbeat_recorded] + properties: + task_id: { type: string, minLength: 1 } + activity_attempt_id: { type: string, minLength: 1 } + lease_owner: { type: string, minLength: 1 } + request_id: { type: string, minLength: 1 } + acknowledged: { type: boolean, const: true } + duplicate: { type: boolean } + reason: { type: "null" } + history_event_id: { type: string, minLength: 1 } + heartbeat_recorded: { type: boolean, const: false } + ActivityCancellationReceipt: + type: object + description: > + Canonical cooperative request and callback-stop receipt. Unknown means no + original worker stop report has been recorded, even when publication is + already fenced. The deadline is the original immutable cleanup budget. + required: [request_id, root_request_id, cleanup_deadline_at, cancellation_history_event_id, callback_state, history_event_id, acknowledged_at, received_after_deadline] + properties: + request_id: { type: string, minLength: 1 } + root_request_id: { type: string, minLength: 1 } + cleanup_deadline_at: { type: string, format: date-time } + cancellation_history_event_id: { type: string, minLength: 1 } + callback_state: { type: string, enum: [unknown, stopped] } + history_event_id: { type: [string, "null"] } + acknowledged_at: { type: [string, "null"], format: date-time } + received_after_deadline: { type: [boolean, "null"] } ActivityTaskStatusResponse: allOf: - $ref: "#/components/schemas/WorkerEnvelope" @@ -1693,6 +1772,9 @@ components: activity_status: { type: [string, "null"] } attempt_status: { type: [string, "null"] } task_status: { type: [string, "null"] } + cancellation_acknowledgement: + $ref: "#/components/schemas/ActivityCancellationReceipt" + x-durable-workflow-minimum-protocol-version: "1.20" deadlines: type: [object, "null"] description: Existing execution deadlines. Null when no deadline is configured. diff --git a/routes/api.php b/routes/api.php index f0e0a51e..57bdf806 100644 --- a/routes/api.php +++ b/routes/api.php @@ -250,6 +250,7 @@ // Activity tasks (long-poll) Route::post('/activity-tasks/poll', [ActivityTaskController::class, 'poll']); Route::post('/activity-tasks/{taskId}/status', [ActivityTaskController::class, 'status']); + Route::post('/activity-tasks/{taskId}/acknowledge-cancellation', [ActivityTaskController::class, 'acknowledgeCancellation']); Route::post('/activity-tasks/{taskId}/complete', [ActivityTaskController::class, 'complete']); Route::post('/activity-tasks/{taskId}/fail', [ActivityTaskController::class, 'fail']); Route::post('/activity-tasks/{taskId}/heartbeat', [ActivityTaskController::class, 'heartbeat']); diff --git a/tests/Feature/ActivityAttemptStatusTest.php b/tests/Feature/ActivityAttemptStatusTest.php index 253793f3..7bd87d8a 100644 --- a/tests/Feature/ActivityAttemptStatusTest.php +++ b/tests/Feature/ActivityAttemptStatusTest.php @@ -6,6 +6,7 @@ use App\Models\WorkerSessionLease; use App\Models\WorkflowNamespace; use App\Support\BackendLockPressureException; +use App\Support\CooperativeCancellationPolicy; use App\Support\NamespaceWorkflowScope; use App\Support\WorkerProtocol; use Illuminate\Foundation\Testing\RefreshDatabase; @@ -18,6 +19,7 @@ use Tests\Support\OpenApiSchema; use Tests\TestCase; use Workflow\V2\Contracts\ActivityTaskBridge; +use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Enums\ActivityStatus; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\TaskStatus; @@ -272,7 +274,169 @@ public function test_default_and_old_protocols_do_not_enable_observation(): void ->assertStatus(422)->assertJsonPath('reason', 'activity_attempt_status_unavailable'); } - private function lease(string $suffix = 'one'): array + public function test_stop_acknowledgement_requires_a_capable_installed_runtime(): void + { + $this->assertSame(CooperativeCancellationPolicy::activityAcknowledgementBackendSupported(), WorkerProtocol::serverCapabilities()['activity_cancellation_acknowledgement']); + config(['server.worker_protocol.version' => '1.19']); + $this->assertFalse(WorkerProtocol::serverCapabilities()['activity_cancellation_acknowledgement']); + config(['server.worker_protocol.version' => '1.20']); + $task = $this->lease(capable: true); + $response = $this->acknowledge($task, 'not-yet-requested'); + if (! CooperativeCancellationPolicy::activityAcknowledgementBackendSupported()) { + $response->assertStatus(409)->assertJsonPath('acknowledged', false) + ->assertJsonPath('unavailable', ['installed_runtime_activity_acknowledgement']); + } else { + $response->assertStatus(409)->assertJsonPath('reason', 'cancellation_request_mismatch'); + } + } + + public function test_stop_receipt_is_distinct_from_fencing_and_duplicate_keeps_original_history(): void + { + $this->requireAcknowledgementBackend(); + $task = $this->lease(capable: true); + $request = $this->cancelCooperatively($task); + $before = $this->snapshot($task); + $observed = $this->observe($task)->assertOk()->assertJsonPath('can_continue', false) + ->assertJsonPath('cancellation_acknowledgement.callback_state', 'unknown') + ->assertJsonPath('cancellation_acknowledgement.request_id', $request['request_id']) + ->assertJsonPath('cancellation_acknowledgement.root_request_id', $request['request_id']) + ->assertJsonPath('cancellation_acknowledgement.cleanup_deadline_at', $request['cleanup_deadline_at']); + $this->assertSame($before, $this->snapshot($task)); + $response = $this->acknowledge($task, $request['request_id'])->assertOk() + ->assertJsonPath('acknowledged', true)->assertJsonPath('duplicate', false) + ->assertJsonPath('heartbeat_recorded', false); + $eventId = $response->json('history_event_id'); + $schema = OpenApiSchema::fromFile(base_path('resources/platform-protocol-specs/worker-protocol-api.openapi.yaml')); + $schema->assertReferenceMatches('#/components/schemas/ActivityCancellationAcknowledgementResponse/allOf/1', json_decode($response->getContent(), flags: JSON_THROW_ON_ERROR)); + $schema->assertReferenceMatches('#/components/schemas/ActivityCancellationReceipt', json_decode($observed->getContent(), flags: JSON_THROW_ON_ERROR)->cancellation_acknowledgement); + $this->travel(40)->seconds(); + $this->acknowledge($task, $request['request_id'])->assertOk() + ->assertJsonPath('duplicate', true)->assertJsonPath('history_event_id', $eventId); + $this->observe($task)->assertOk()->assertJsonPath('cancellation_acknowledgement.callback_state', 'stopped') + ->assertJsonPath('cancellation_acknowledgement.history_event_id', $eventId) + ->assertJsonPath('cancellation_acknowledgement.received_after_deadline', false) + ->assertJsonPath('cancellation_acknowledgement.cleanup_deadline_at', $request['cleanup_deadline_at']); + $after = $this->snapshot($task); + $this->assertSame($before[4] + 1, $after[4]); + unset($before[4], $after[4]); + $this->assertSame($before, $after); + $this->withHeaders($this->headers())->postJson($this->path($task, 'complete'), $this->fence($task) + ['result' => null]) + ->assertJsonPath('recorded', false); + } + + public function test_late_stop_receipt_cannot_renew_original_deadline(): void + { + $this->requireAcknowledgementBackend(); + $task = $this->lease(capable: true); + $request = $this->cancelCooperatively($task); + $this->travel(31)->seconds(); + $this->acknowledge($task, $request['request_id'])->assertOk()->assertJsonPath('acknowledged', true); + $this->observe($task)->assertOk()->assertJsonPath('cancellation_acknowledgement.received_after_deadline', true) + ->assertJsonPath('cancellation_acknowledgement.cleanup_deadline_at', $request['cleanup_deadline_at']); + $this->assertSame($request['cleanup_deadline_at'], WorkflowRun::query()->findOrFail($task['run_id'])->cancellation_deadline_at->toISOString()); + } + + public function test_changed_registration_cannot_upgrade_or_erase_original_activity_claim(): void + { + $this->requireAcknowledgementBackend(); + foreach ([false, true] as $originallyCapable) { + $task = $this->lease($originallyCapable ? 'capable' : 'legacy', capable: $originallyCapable); + $request = $this->cancelCooperatively($task); + WorkerRegistration::query()->where('worker_id', $task['lease_owner'])->firstOrFail()->forceFill([ + 'capabilities' => $originallyCapable ? [] : [CooperativeCancellationPolicy::CAPABILITY], + ])->save(); + $response = $this->acknowledge($task, $request['request_id']); + if ($originallyCapable) { + $response->assertOk()->assertJsonPath('acknowledged', true); + } else { + $response->assertStatus(409)->assertJsonPath('unavailable', ['worker_claim_capability']); + } + } + } + + public function test_acknowledgement_requires_original_request_owner_task_attempt_and_namespace(): void + { + $this->requireAcknowledgementBackend(); + $task = $this->lease(capable: true); + $other = $this->lease('other', capable: true); + $request = $this->cancelCooperatively($task); + $before = $this->snapshot($task); + $this->acknowledge($task, 'different')->assertStatus(409)->assertJsonPath('reason', 'cancellation_request_mismatch'); + $this->acknowledge($task, $request['request_id'], ['lease_owner' => 'different'])->assertStatus(409)->assertJsonPath('reason', 'lease_owner_mismatch'); + $this->acknowledge($task, $request['request_id'], ['activity_attempt_id' => $other['activity_attempt_id']]) + ->assertStatus(409)->assertJsonPath('reason', 'task_mismatch'); + WorkflowNamespace::query()->create(['name' => 'isolated', 'description' => 'Other', 'retention_days' => 30, 'status' => 'active']); + $this->withHeaders($this->headers('isolated'))->postJson($this->path($task, 'acknowledge-cancellation'), + $this->fence($task) + ['request_id' => $request['request_id']])->assertNotFound(); + $this->assertSame($before, $this->snapshot($task)); + } + + public function test_new_request_protocol_cannot_upgrade_a_legacy_activity_claim(): void + { + $this->requireAcknowledgementBackend(); + $task = $this->lease(capable: true, protocolVersion: '1.19'); + $request = $this->cancelCooperatively($task); + $before = $this->snapshot($task); + $this->acknowledge($task, $request['request_id'])->assertStatus(409) + ->assertJsonPath('unavailable', ['worker_claim_capability']); + $this->assertSame($before, $this->snapshot($task)); + } + + public function test_acknowledgement_accepts_drain_but_fenced_storage_preserves_unknown_stop_state(): void + { + $this->requireAcknowledgementBackend(); + $task = $this->lease(capable: true); + $request = $this->cancelCooperatively($task); + $this->configureStoragePressure(); + $this->observeStoragePressure('fenced'); + $before = $this->snapshot($task); + $this->acknowledge($task, $request['request_id'])->assertStatus(503)->assertJsonPath('request_admitted', false); + $this->observe($task)->assertOk()->assertJsonPath('cancellation_acknowledgement.callback_state', 'unknown'); + $this->assertSame($before, $this->snapshot($task)); + $this->observeStoragePressure('draining'); + $this->acknowledge($task, $request['request_id'])->assertOk()->assertJsonPath('acknowledged', true); + } + + public function test_old_protocol_and_operator_role_cannot_acknowledge_callbacks(): void + { + $task = $this->lease(capable: true); + $old = $this->headers(); + $old[WorkerProtocol::HEADER] = '1.19'; + $this->withHeaders($old)->postJson($this->path($task, 'acknowledge-cancellation'), $this->fence($task) + ['request_id' => 'request']) + ->assertStatus(422)->assertJsonPath('reason', 'activity_cancellation_acknowledgement_not_supported'); + config(['server.auth.driver' => 'token', 'server.auth.role_tokens' => ['operator' => 'fixture-operator', 'worker' => 'fixture-worker']]); + $this->withHeaders($this->headers() + ['Authorization' => 'Bearer fixture-operator']) + ->postJson($this->path($task, 'acknowledge-cancellation'), $this->fence($task) + ['request_id' => 'request']) + ->assertForbidden()->assertJsonPath('reason', 'forbidden'); + } + + private function requireAcknowledgementBackend(): void + { + if (! CooperativeCancellationPolicy::activityAcknowledgementBackendSupported()) { + $this->markTestSkipped('Requires the candidate Native acknowledgement primitive for source qualification.'); + } + } + + private function cancelCooperatively(array $task): array + { + $result = WorkflowStub::load($task['workflow_id'])->requestCancellation('cleanup', 30); + $this->assertTrue($result->accepted()); + $run = WorkflowRun::query()->findOrFail($task['run_id']); + $ready = $run->tasks()->where('task_type', 'workflow')->where('status', 'ready')->sole(); + $bridge = app(WorkflowTaskBridge::class); + $claim = $bridge->claimStatus($ready->id, 'cleanup-owner'); + $this->assertTrue($claim['claimed']); + $this->assertTrue($bridge->deliverCancellation($ready->id, $run->cancellation_request_command_id, 1, 'activity')['delivered']); + return ['request_id' => $run->cancellation_request_command_id, 'cleanup_deadline_at' => $run->cancellation_deadline_at->toISOString()]; + } + + private function acknowledge(array $task, string $requestId, array $overrides = []): TestResponse + { + return $this->withHeaders($this->headers())->postJson($this->path($task, 'acknowledge-cancellation'), + array_replace($this->fence($task) + ['request_id' => $requestId], $overrides)); + } + + private function lease(string $suffix = 'one', bool $capable = false, string $protocolVersion = '1.20'): array { $workflow = WorkflowStub::make(ExternalGreetingWorkflow::class, 'status-'.$suffix); $start = $workflow->start('Ada'); @@ -283,10 +447,11 @@ private function lease(string $suffix = 'one'): array WorkerRegistration::query()->updateOrCreate(['worker_id' => 'status-worker-'.$suffix, 'namespace' => 'default'], [ 'task_queue' => 'external-activities', 'runtime' => 'php', 'supported_activity_types' => ['tests.external-greeting-activity'], + 'capabilities' => $capable ? [CooperativeCancellationPolicy::CAPABILITY] : [], 'last_heartbeat_at' => now(), 'status' => 'active', ]); - return $this->withHeaders($this->headers())->postJson('/api/worker/activity-tasks/poll', [ + return $this->withHeaders(array_replace($this->headers(), [WorkerProtocol::HEADER => $protocolVersion]))->postJson('/api/worker/activity-tasks/poll', [ 'worker_id' => 'status-worker-'.$suffix, 'task_queue' => 'external-activities', ])->assertOk()->json('task'); } diff --git a/tests/Feature/WorkerProtocolVersionCoverageTest.php b/tests/Feature/WorkerProtocolVersionCoverageTest.php index d311a0e0..2fbbc5ee 100644 --- a/tests/Feature/WorkerProtocolVersionCoverageTest.php +++ b/tests/Feature/WorkerProtocolVersionCoverageTest.php @@ -49,6 +49,8 @@ public static function workerEndpointProvider(): array 'query-tasks.complete' => ['method' => 'post', 'path' => '/api/worker/query-tasks/task-1/complete'], 'query-tasks.fail' => ['method' => 'post', 'path' => '/api/worker/query-tasks/task-1/fail'], 'activity-tasks.poll' => ['method' => 'post', 'path' => '/api/worker/activity-tasks/poll'], + 'activity-tasks.status' => ['method' => 'post', 'path' => '/api/worker/activity-tasks/task-1/status'], + 'activity-tasks.acknowledge-cancellation' => ['method' => 'post', 'path' => '/api/worker/activity-tasks/task-1/acknowledge-cancellation'], 'activity-tasks.complete' => ['method' => 'post', 'path' => '/api/worker/activity-tasks/task-1/complete'], 'activity-tasks.fail' => ['method' => 'post', 'path' => '/api/worker/activity-tasks/task-1/fail'], 'activity-tasks.heartbeat' => ['method' => 'post', 'path' => '/api/worker/activity-tasks/task-1/heartbeat'], From 546b1ca456b7d31bee1e8cd4a563da61fc584ce8 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 00:02:44 +0000 Subject: [PATCH 15/57] Format callback stop receipt paths --- app/Http/Controllers/Api/ActivityTaskController.php | 4 ++++ tests/Feature/ActivityAttemptStatusTest.php | 1 + 2 files changed, 5 insertions(+) diff --git a/app/Http/Controllers/Api/ActivityTaskController.php b/app/Http/Controllers/Api/ActivityTaskController.php index 6bea3576..3452edb3 100644 --- a/app/Http/Controllers/Api/ActivityTaskController.php +++ b/app/Http/Controllers/Api/ActivityTaskController.php @@ -552,6 +552,7 @@ public function acknowledgeCancellation(Request $request, string $taskId): JsonR if (! CooperativeCancellationPolicy::activityAcknowledgementBackendSupported()) { return $this->activityAcknowledgementUnavailable($validated['lease_owner'], ['installed_runtime_activity_acknowledgement']); } + return $this->storageMutations->run(fn (): JsonResponse => DB::transaction(function () use ($namespace, $taskId, $validated): JsonResponse { // Acquire the package's activity lock order before checking the // immutable claim and calling the nested package transaction. @@ -571,6 +572,7 @@ public function acknowledgeCancellation(Request $request, string $taskId): JsonR $outcome = ActivityCancellationAcknowledgement::recordStopped( $validated['activity_attempt_id'], $validated['lease_owner'], $validated['request_id'], ); + return WorkerProtocol::json([ 'task_id' => $taskId, 'activity_attempt_id' => $validated['activity_attempt_id'], @@ -584,6 +586,7 @@ public function acknowledgeCancellation(Request $request, string $taskId): JsonR if (! BackendLockPressure::is($exception)) { throw $exception; } + return BackendLockPressure::workerOperationResponse($request, false); } } @@ -637,6 +640,7 @@ private function activityCancellationReceipt( $ack = $events->first(fn (WorkflowHistoryEvent $event): bool => $event->event_type === HistoryEventType::ActivityCancellationAcknowledged && ($event->payload['cancellation_history_event_id'] ?? null) === $cancelled->id && ($event->payload['activity_attempt_id'] ?? null) === $attemptId); + return [ 'request_id' => $context->requestId, 'root_request_id' => $context->rootRequestId, diff --git a/tests/Feature/ActivityAttemptStatusTest.php b/tests/Feature/ActivityAttemptStatusTest.php index 7bd87d8a..deadee0a 100644 --- a/tests/Feature/ActivityAttemptStatusTest.php +++ b/tests/Feature/ActivityAttemptStatusTest.php @@ -427,6 +427,7 @@ private function cancelCooperatively(array $task): array $claim = $bridge->claimStatus($ready->id, 'cleanup-owner'); $this->assertTrue($claim['claimed']); $this->assertTrue($bridge->deliverCancellation($ready->id, $run->cancellation_request_command_id, 1, 'activity')['delivered']); + return ['request_id' => $run->cancellation_request_command_id, 'cleanup_deadline_at' => $run->cancellation_deadline_at->toISOString()]; } From dae9bb04d1828b0d45fa8d65ebe0a94d2d555937 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 00:59:09 +0000 Subject: [PATCH 16/57] Verify run inspection and stale publication after callback-stop receipts --- tests/Feature/ActivityAttemptStatusTest.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/Feature/ActivityAttemptStatusTest.php b/tests/Feature/ActivityAttemptStatusTest.php index deadee0a..5e72887e 100644 --- a/tests/Feature/ActivityAttemptStatusTest.php +++ b/tests/Feature/ActivityAttemptStatusTest.php @@ -309,6 +309,13 @@ public function test_stop_receipt_is_distinct_from_fencing_and_duplicate_keeps_o $schema = OpenApiSchema::fromFile(base_path('resources/platform-protocol-specs/worker-protocol-api.openapi.yaml')); $schema->assertReferenceMatches('#/components/schemas/ActivityCancellationAcknowledgementResponse/allOf/1', json_decode($response->getContent(), flags: JSON_THROW_ON_ERROR)); $schema->assertReferenceMatches('#/components/schemas/ActivityCancellationReceipt', json_decode($observed->getContent(), flags: JSON_THROW_ON_ERROR)->cancellation_acknowledgement); + $this->withHeaders($this->headers())->getJson("/api/workflows/{$task['workflow_id']}/runs/{$task['run_id']}")->assertOk(); + $this->withHeaders($this->headers())->getJson("/api/workflows/{$task['workflow_id']}/runs/{$task['run_id']}/debug")->assertOk(); + $this->withHeaders($this->headers())->postJson($this->path($task, 'complete'), $this->fence($task) + ['result' => null]) + ->assertStatus(409)->assertJsonPath('recorded', false); + $this->withHeaders($this->headers())->postJson($this->path($task, 'fail'), $this->fence($task) + ['failure' => ['message' => 'stale callback failure', 'non_retryable' => true]]) + ->assertStatus(409)->assertJsonPath('recorded', false); + $this->assertSame($before[4] + 1, $this->snapshot($task)[4]); $this->travel(40)->seconds(); $this->acknowledge($task, $request['request_id'])->assertOk() ->assertJsonPath('duplicate', true)->assertJsonPath('history_event_id', $eventId); From b272f4fa18eccaac0e1741e6fa1797d9be0bcfb2 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 05:35:24 +0000 Subject: [PATCH 17/57] Add claim-bound prepared local activity worker endpoints --- .github/workflows/phpunit-feature.yml | 101 ++++ .../Api/PreparedLocalActivityController.php | 317 +++++++++++ app/Http/Controllers/Api/WorkerController.php | 90 ++- .../RuntimeExternalPayloadTransport.php | 13 + .../PreparedLocalActivityAdmissionRefused.php | 6 + app/Support/PreparedLocalActivityPolicy.php | 133 +++++ app/Support/WorkerProtocol.php | 1 + .../worker-protocol-api.openapi.yaml | 409 +++++++++++++- routes/api.php | 7 + .../PreparedLocalActivityProtocolTest.php | 533 ++++++++++++++++++ 10 files changed, 1603 insertions(+), 7 deletions(-) create mode 100644 app/Http/Controllers/Api/PreparedLocalActivityController.php create mode 100644 app/Support/PreparedLocalActivityAdmissionRefused.php create mode 100644 app/Support/PreparedLocalActivityPolicy.php create mode 100644 tests/Feature/PreparedLocalActivityProtocolTest.php diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index ad15f156..993dcd1d 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -11,6 +11,10 @@ on: description: Full Git commit SHA to classify regression-corpus changes against required: true type: string + prepared_local_workflow_commit: + description: Optional full Native commit for candidate prepared-local source qualification. Does not replace locked-package CI. + required: false + type: string permissions: contents: read @@ -253,3 +257,100 @@ jobs: test "$PREFLIGHT_RESULT" = success test "$FEATURE_RESULT" = success echo "Server repository contracts and feature tests succeeded." + + prepared-local-source: + name: Candidate prepared-local source qualification + needs: [action-policy, preflight] + if: ${{ github.event_name == 'workflow_dispatch' && inputs.prepared_local_workflow_commit != '' }} + runs-on: ubuntu-latest + timeout-minutes: 20 + services: + mysql: + image: mysql@sha256:679e7e924f38a3cbb62a3d7df32924b83f7321a602d3f9f967c01b3df18495d6 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + MYSQL_DATABASE: prepared_local_test + MYSQL_ROOT_HOST: "%" + ports: ["3317:3306"] + options: >- + --health-cmd="mysqladmin --silent --host=127.0.0.1 ping" + --health-interval=2s --health-timeout=5s --health-retries=30 + postgres: + image: postgres@sha256:f1c3376c26f2609ab9f29f71f824103fe2fcd8ee0346485cb6122a4f93df6f94 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: prepared_local_test + ports: ["5439:5432"] + options: >- + --health-cmd="pg_isready --host=127.0.0.1 --username=postgres" + --health-interval=2s --health-timeout=5s --health-retries=30 + env: + DW_PREPARED_WORKFLOW_COMMIT: ${{ inputs.prepared_local_workflow_commit }} + steps: + - name: Require immutable candidate source + run: | + set -euo pipefail + [[ "$DW_PREPARED_WORKFLOW_COMMIT" =~ ^[0-9a-f]{40}$ ]] + [[ ! "$DW_PREPARED_WORKFLOW_COMMIT" =~ ^0+$ ]] + - name: Check out Server candidate + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Check out exact Native candidate + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: durable-workflow/workflow + ref: ${{ inputs.prepared_local_workflow_commit }} + path: prepared-workflow-source + persist-credentials: false + - name: Record source binding and qualify all supported databases + run: | + set -euo pipefail + test "$(git -C prepared-workflow-source rev-parse HEAD)" = "$DW_PREPARED_WORKFLOW_COMMIT" + mkdir -p prepared-local-evidence + { + echo "Candidate source qualification only. No image, release or SDK conformance claim." + echo "Server commit: $(git rev-parse HEAD)" + echo "Native source overlay: $DW_PREPARED_WORKFLOW_COMMIT" + echo "Locked package baseline retained by ordinary feature CI." + } > prepared-local-evidence/provenance.txt + docker build --target base -t dw-server-prepared-local-base . + tar --exclude=vendor --exclude='*/vendor' --exclude=build --exclude='*/build' \ + --exclude=prepared-local-evidence -cf - . > "$RUNNER_TEMP/prepared-local-source.tar" + for database in sqlite mysql pgsql; do + database_name=prepared_local_test + database_user=root + database_port=3317 + if [ "$database" = sqlite ]; then database_name=:memory:; fi + if [ "$database" = pgsql ]; then database_user=postgres; database_port=5439; fi + docker run --rm --init --network host \ + --user "$(id -u):$(id -g)" \ + --tmpfs "/app:rw,uid=$(id -u),gid=$(id -g),mode=0755" \ + --volume "$RUNNER_TEMP/prepared-local-source.tar:/source.tar:ro" \ + --volume "$PWD/prepared-local-evidence:/evidence" \ + --env COMPOSER_HOME=/tmp/dw-prepared-composer \ + --env DB_CONNECTION="$database" --env DB_DATABASE="$database_name" \ + --env DB_HOST=127.0.0.1 --env DB_PORT="$database_port" \ + --env DB_USERNAME="$database_user" --env DB_PASSWORD= \ + --env DW_PREPARED_DATABASE="$database" \ + --workdir /app --entrypoint sh dw-server-prepared-local-base -ec ' + tar --no-same-owner -xf /source.tar -C /app + composer install --no-interaction --no-progress --prefer-dist + mv vendor/durable-workflow/workflow /tmp/locked-workflow-package + cp -a prepared-workflow-source vendor/durable-workflow/workflow + php -r '\''require "vendor/autoload.php"; if (!interface_exists(Workflow\V2\Contracts\PreparedLocalActivityTaskBridge::class)) { exit(1); }'\'' + vendor/bin/phpunit tests/Feature/PreparedLocalActivityProtocolTest.php --log-junit "/evidence/$DW_PREPARED_DATABASE.xml" --colors=never + php -r '\''$xml = simplexml_load_file("/evidence/".getenv("DW_PREPARED_DATABASE").".xml"); if ($xml === false || count($xml->xpath("//testcase/skipped")) > 0) { exit(1); }'\'' + if [ "$DW_PREPARED_DATABASE" = sqlite ]; then + vendor/bin/phpunit tests/Feature/CooperativeCancellationProtocolTest.php tests/Feature/WorkflowWorkerProtocolTest.php tests/Feature/ActivityWorkerProtocolTest.php tests/Feature/SearchAttributeValueValidationTest.php tests/Feature/NamespaceDurableStateQuotaTest.php tests/Feature/RuntimeExternalPayloadTransportTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php --log-junit /evidence/affected-regression.xml --colors=never + fi + ' > "prepared-local-evidence/$database.log" 2>&1 + done + - name: Retain exact source qualification evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: candidate-prepared-local-source + path: prepared-local-evidence + if-no-files-found: error + retention-days: 90 diff --git a/app/Http/Controllers/Api/PreparedLocalActivityController.php b/app/Http/Controllers/Api/PreparedLocalActivityController.php new file mode 100644 index 00000000..6e449c79 --- /dev/null +++ b/app/Http/Controllers/Api/PreparedLocalActivityController.php @@ -0,0 +1,317 @@ +admit($request, $taskId, false); + } + + public function recover(Request $request, string $taskId): JsonResponse + { + return $this->admit($request, $taskId, true); + } + + private function admit(Request $request, string $taskId, bool $recover): JsonResponse + { + if ($response = self::unavailable($request)) { + return $response; + } + $validated = $request->validate([ + ...self::claimRules(), + 'sequence' => ['required', 'integer', 'min:1'], + 'worker_attempt_id' => [$recover ? 'nullable' : 'required', 'string', 'max:255'], + 'descriptor' => ['required', 'array'], + ]); + $namespace = (string) $request->attributes->get('namespace'); + $task = NamespaceWorkflowScope::task($namespace, $taskId); + if ($task === null) { + return WorkerProtocol::json(['reason' => 'task_not_found'], 404); + } + $claim = PreparedLocalActivityPolicy::currentClaim($task); + if ($claim === null) { + return self::refused($validated['lease_owner'], ['worker_claim_capability']); + } + try { + $validated['descriptor'] = $this->resolvePayload($validated['descriptor'], 'arguments', 'descriptor', $namespace); + $reply = $this->mutations->run(fn (): array|JsonResponse => DB::transaction(function () use ($request, $namespace, $taskId, $validated, $claim, $recover): array|JsonResponse { + $quotaSnapshot = $this->quota->snapshotForMutation($namespace, self::quotaResources()); + // Registration precedes Native's attempt/execution/run/task + // locks. Do not lock the hosting task before entering Native. + if (! WorkerPollFence::isCurrentForUpdate($claim)) { + return WorkerProtocol::json(['reason' => 'stale_worker_registration'], 409); + } + /** @var PreparedLocalActivityTaskBridge $bridge */ + $bridge = app(WorkflowTaskBridge::class); + $reply = $recover + ? $bridge->recoverLocalActivity($taskId, $validated['lease_owner'], (int) $validated['workflow_task_attempt'], (int) $validated['sequence'], $validated['descriptor'], WorkerProtocol::requestVersion($request)) + : $bridge->prepareLocalActivity($taskId, $validated['lease_owner'], (int) $validated['workflow_task_attempt'], (int) $validated['sequence'], $validated['worker_attempt_id'], $validated['descriptor'], WorkerProtocol::requestVersion($request)); + if (! $recover && ($reply['prepared'] ?? false)) { + $task = NamespaceWorkflowScope::taskQuery($namespace)->lockForUpdate()->find($taskId); + $attemptId = $reply['activity_attempt_id'] ?? null; + $attempt = is_string($attemptId) ? ActivityAttempt::query()->find($attemptId) : null; + if (! $task instanceof WorkflowTask || ! $attempt instanceof ActivityAttempt + || ! PreparedLocalActivityPolicy::remember($task, $attempt, $claim)) { + throw new PreparedLocalActivityAdmissionRefused; + } + } + + $this->quota->assertNoIncreasePastLimit($quotaSnapshot); + + return $reply; + })); + } catch (PreparedLocalActivityAdmissionRefused) { + return self::refused($validated['lease_owner'], ['original_claim_persistence']); + } catch (\Throwable $exception) { + if ($response = $this->payloadFailure($exception)) { + return $response; + } + if (! BackendLockPressure::is($exception)) { + throw $exception; + } + + return BackendLockPressure::workerOperationResponse($request, false); + } + + return $this->reply($request, $reply); + } + + public function control(Request $request, string $taskId, string $attemptId): JsonResponse + { + return $this->attemptOperation($request, $taskId, $attemptId, 'control'); + } + + public function outcome(Request $request, string $taskId, string $attemptId): JsonResponse + { + return $this->attemptOperation($request, $taskId, $attemptId, 'outcome'); + } + + public function acknowledgeCancellation(Request $request, string $taskId, string $attemptId): JsonResponse + { + return $this->attemptOperation($request, $taskId, $attemptId, 'acknowledge'); + } + + private function attemptOperation(Request $request, string $taskId, string $attemptId, string $operation): JsonResponse + { + if ($response = self::unavailable($request)) { + return $response; + } + $validated = $request->validate([ + ...self::claimRules(), + 'renew_lease' => ['nullable', 'boolean'], + 'report' => [$operation === 'outcome' ? 'required' : 'nullable', 'array'], + 'request_id' => [$operation === 'acknowledge' ? 'required' : 'nullable', 'string', 'max:255'], + ]); + $namespace = (string) $request->attributes->get('namespace'); + $task = NamespaceWorkflowScope::task($namespace, $taskId); + $attempt = ActivityAttempt::query()->where('workflow_task_id', $taskId)->find($attemptId); + if ($task === null || $attempt === null || $attempt->workflow_run_id !== $task->workflow_run_id) { + return WorkerProtocol::json(['reason' => 'activity_attempt_not_found'], 404); + } + $claim = PreparedLocalActivityPolicy::originalClaim($task, $attempt, $validated['lease_owner'], (int) $validated['workflow_task_attempt']); + if ($claim === null) { + return self::refused($validated['lease_owner'], ['original_prepared_local_claim']); + } + try { + if ($operation === 'outcome' && ($validated['report']['outcome'] ?? null) === 'completed') { + $validated['report'] = $this->resolvePayload($validated['report'], 'result', 'report', $namespace); + } + $reply = $this->mutations->run(fn (): array => DB::transaction(function () use ($request, $validated, $claim, $attemptId, $operation): array { + $quotaSnapshot = $this->quota->snapshotForMutation($claim['namespace'], self::quotaResources()); + /** @var PreparedLocalActivityTaskBridge $bridge */ + $bridge = app(WorkflowTaskBridge::class); + $version = WorkerProtocol::requestVersion($request); + $epoch = (int) $validated['workflow_task_attempt']; + $owner = $validated['lease_owner']; + if ($operation === 'acknowledge') { + // A dead, drained or replaced registration may still have + // a surviving supervisor reporting its actual joined stop. + $reply = $bridge->acknowledgeLocalActivityCancellation($attemptId, $owner, $validated['request_id'], $epoch, $version); + $this->quota->assertNoIncreasePastLimit($quotaSnapshot); + + return $reply; + } + $current = WorkerPollFence::isCurrentForUpdate($claim); + if ($operation === 'outcome') { + // Lost responses may still read their immutable receipt. + // Cancellation may fence, but a stale registration cannot + // publish a fresh result under an unreclaimed task epoch. + if (! $current && ! WorkflowHistoryEvent::query() + ->where('workflow_run_id', $claim['run_id']) + ->where('payload->activity_attempt_id', $attemptId) + ->whereNotNull('payload->local_outcome')->exists() + && WorkflowRun::query()->find($claim['run_id'])?->cancellation_request_command_id === null) { + return ['recorded' => false, 'reason' => 'stale_worker_registration']; + } + $reply = $bridge->recordLocalActivityOutcome($attemptId, $owner, $epoch, $validated['report'], $version); + $this->quota->assertNoIncreasePastLimit($quotaSnapshot); + + return $reply; + } + $reply = $bridge->controlLocalActivity($attemptId, $owner, $epoch, $current && ($validated['renew_lease'] ?? false), $version); + if (! $current && ($reply['active'] ?? false)) { + $reply = [...$reply, 'active' => false, 'renewed' => false, 'stop_required' => true, 'reason' => 'stale_worker_registration']; + } + $this->quota->assertNoIncreasePastLimit($quotaSnapshot); + + return $reply; + })); + } catch (\Throwable $exception) { + if ($response = $this->payloadFailure($exception)) { + return $response; + } + if (! BackendLockPressure::is($exception)) { + throw $exception; + } + + return BackendLockPressure::workerOperationResponse($request, false); + } + + return $this->reply($request, $reply); + } + + /** @return array> */ + private static function claimRules(): array + { + return ['lease_owner' => ['required', 'string', 'max:255'], 'workflow_task_attempt' => ['required', 'integer', 'min:1']]; + } + + /** @return list */ + private static function quotaResources(): array + { + return [NamespaceDurableStateQuota::WORKFLOW_HISTORY_EVENTS, NamespaceDurableStateQuota::WORKFLOW_TASKS, + NamespaceDurableStateQuota::PENDING_WORKFLOW_TASKS]; + } + + /** @param array $payload + * @return array + */ + private function resolvePayload(array $payload, string $key, string $prefix, string $namespace): array + { + if (array_key_exists('payload_codec', $payload)) { + try { + $payload['payload_codec'] = PayloadCodecContract::canonicalize($payload['payload_codec']); + } catch (\InvalidArgumentException $exception) { + throw ValidationException::withMessages([$prefix.'.payload_codec' => [$exception->getMessage()]]); + } + } + $value = $payload[$key] ?? null; + if (is_string($value)) { + AvroPayloadEnvelopeResolver::assertSerializedPayload($value, $prefix.'.'.$key); + } elseif (is_array($value)) { + $resolved = AvroPayloadEnvelopeResolver::resolveCommandPayloadWithCodec( + $value, $prefix.'.'.$key, app(NamespaceExternalPayloadStorage::class)->driverFor($namespace), + ); + $payload[$key] = $resolved['payload']; + $payload['payload_codec'] ??= $resolved['codec']; + } + + return $payload; + } + + private function payloadFailure(\Throwable $exception): ?JsonResponse + { + if ($exception instanceof StructuralLimitExceededException) { + return WorkerProtocol::json(['recorded' => false, 'reason' => 'structural_limit_exceeded', + 'limit_kind' => $exception->limitKind->value, 'current_value' => $exception->currentValue, + 'configured_limit' => $exception->configuredLimit], 422); + } + if ($exception instanceof ExternalPayloadIntegrityException || $exception instanceof ExternalPayloadStorageUnavailable) { + return WorkerProtocol::json(['recorded' => false, 'reason' => $exception instanceof ExternalPayloadIntegrityException + ? 'external_payload_integrity_failed' : 'external_payload_storage_unavailable'], + $exception instanceof ExternalPayloadIntegrityException ? 422 : 503); + } + + return null; + } + + public static function unavailable(Request $request): ?JsonResponse + { + if ($response = WorkerProtocol::rejectUnsupported($request)) { + return $response; + } + $reasons = PreparedLocalActivityPolicy::unavailableReasons(WorkerProtocol::requestVersion($request)); + + return $reasons === [] ? null : self::refused((string) $request->input('lease_owner', ''), $reasons); + } + + /** @param list $reasons */ + public static function refused(string $workerId, array $reasons): JsonResponse + { + return WorkerProtocol::json([ + 'reason' => 'prepared_local_activity_not_supported', + 'required_capability' => PreparedLocalActivityPolicy::CAPABILITY, + 'minimum_protocol_version' => PreparedLocalActivityPolicy::MINIMUM_PROTOCOL_VERSION, + 'worker_id' => $workerId, + 'unavailable' => $reasons, + 'remediation' => 'Use a prepared-local backend and a claim issued to a capable protocol 1.20 worker.', + ], 409); + } + + /** @param array|JsonResponse $reply */ + private function reply(Request $request, array|JsonResponse $reply): JsonResponse + { + if ($reply instanceof JsonResponse) { + return $reply; + } + // This is a cursor, not authority. History paging still requires the + // current namespace, lease owner and workflow claim epoch. + $reply['history_refresh_page_token'] = WorkflowHistoryPageToken::encode(0); + try { + $this->mutations->run(function () use ($reply): void { + app(ServiceModeTimerDispatcher::class)->dispatchCreatedTaskIds($reply['created_task_ids'] ?? []); + foreach ($reply['created_task_ids'] ?? [] as $taskId) { + $task = WorkflowTask::query()->find($taskId); + if ($task instanceof WorkflowTask) { + app(LongPollSignalStore::class)->signalTask($task); + } + } + }); + } catch (\Throwable $exception) { + if (! BackendLockPressure::is($exception)) { + throw $exception; + } + + return BackendLockPressure::workerOperationResponse($request, true); + } + $reason = $reply['reason'] ?? null; + $ok = $reason === null || isset($reply['cancellation_request']); + + return WorkerProtocol::json($reply, $ok ? 200 : (in_array($reason, ['task_not_found', 'activity_attempt_not_found'], true) ? 404 : 409)); + } +} diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index c4734bdf..0bbad670 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -20,6 +20,8 @@ use App\Support\NamespaceWorkflowScope; use App\Support\PayloadCodecContract; use App\Support\PollRequestTaskKindsConflict; +use App\Support\PreparedLocalActivityAdmissionRefused; +use App\Support\PreparedLocalActivityPolicy; use App\Support\QueryTaskQueueUnavailableException; use App\Support\RouteAuthorizationResource; use App\Support\RuntimeExternalPayloadAudit; @@ -53,6 +55,7 @@ use Illuminate\Support\Str; use Illuminate\Validation\ValidationException; use Workflow\V2\Contracts\HistoryProjectionRole; +use Workflow\V2\Contracts\PreparedLocalActivityTaskBridge; use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Enums\ActivityAttemptStatus; use Workflow\V2\Enums\HistoryEventType; @@ -225,6 +228,16 @@ public function register(Request $request): JsonResponse ]); $workerCapabilities = $this->nonEmptyStringArray($validated['capabilities'] ?? []); + if (in_array(PreparedLocalActivityPolicy::CAPABILITY, $workerCapabilities, true) + && ! CooperativeCancellationPolicy::supports($workerCapabilities, WorkerProtocol::requestVersion($request))) { + return WorkerProtocol::json([ + 'registered' => false, + 'reason' => 'prepared_local_activity_capability_mismatch', + 'required_capability' => CooperativeCancellationPolicy::CAPABILITY, + 'minimum_protocol_version' => PreparedLocalActivityPolicy::MINIMUM_PROTOCOL_VERSION, + 'requested_version' => WorkerProtocol::requestVersion($request), + ], 409); + } if (in_array(CooperativeCancellationPolicy::CAPABILITY, $workerCapabilities, true) && ! CooperativeCancellationPolicy::supports($workerCapabilities, WorkerProtocol::requestVersion($request))) { return WorkerProtocol::json([ @@ -1446,6 +1459,27 @@ public function workflowTaskHistory(Request $request, string $taskId): JsonRespo * Complete a claimed workflow task with commands emitted by an external worker. */ public function completeWorkflowTask(Request $request, string $taskId): JsonResponse + { + return $this->mutateWorkflowTaskCommands($request, $taskId); + } + + public function checkpointLocalActivityPrefix(Request $request, string $taskId): JsonResponse + { + if ($response = PreparedLocalActivityController::unavailable($request)) { + return $response; + } + $task = NamespaceWorkflowScope::task((string) $request->attributes->get('namespace'), $taskId); + if ($task === null) { + return WorkerProtocol::json(['reason' => 'task_not_found'], 404); + } + if (PreparedLocalActivityPolicy::currentClaim($task) === null) { + return PreparedLocalActivityController::refused((string) $request->input('lease_owner', ''), ['worker_claim_capability']); + } + + return $this->mutateWorkflowTaskCommands($request, $taskId, true); + } + + private function mutateWorkflowTaskCommands(Request $request, string $taskId, bool $checkpoint = false): JsonResponse { if ($response = WorkerProtocol::rejectUnsupported($request)) { return $response; @@ -1453,6 +1487,14 @@ public function completeWorkflowTask(Request $request, string $taskId): JsonResp $namespace = $request->attributes->get('namespace'); + $checkpointInput = $checkpoint ? $request->validate([ + 'checkpoint_id' => ['required', 'string', 'max:255'], + 'start_sequence' => ['required', 'integer', 'min:1'], + 'message_stream_cursors' => ['prohibited'], + 'message_stream_waits' => ['prohibited'], + 'sticky_cache' => ['prohibited'], + ]) : []; + $messageStreamCompletion = $request->validate([ 'message_stream_cursors' => ['nullable', 'array', 'max:100'], 'message_stream_cursors.*.stream_name' => ['required', 'string', 'max:128'], @@ -1779,6 +1821,8 @@ function () use ( $validated, $messageStreamCursors, $messageStreamWaits, + $checkpoint, + $checkpointInput, ): array|JsonResponse { return DB::transaction(function () use ( $bridge, @@ -1789,6 +1833,8 @@ function () use ( $validated, $messageStreamCursors, $messageStreamWaits, + $checkpoint, + $checkpointInput, ): array|JsonResponse { $quotaSnapshot = $this->durableStateQuota->snapshotForMutation( (string) $namespace, @@ -1864,7 +1910,22 @@ function () use ( $commands, WorkerProtocol::requestVersion($request), ); - $outcome = $bridge->complete($taskId, $commands); + if ($checkpoint) { + if (PreparedLocalActivityPolicy::currentClaim($claimedTask) === null + || ! $bridge instanceof PreparedLocalActivityTaskBridge) { + throw new PreparedLocalActivityAdmissionRefused; + } + $outcome = $bridge->checkpointLocalActivityPrefix( + $taskId, $validated['lease_owner'], (int) $validated['workflow_task_attempt'], + $checkpointInput['checkpoint_id'], (int) $checkpointInput['start_sequence'], $commands, + WorkerProtocol::requestVersion($request), + ); + // Shared post-processing must not release this claim + // or attribute a terminal event to a prefix receipt. + $outcome['completed'] = false; + } else { + $outcome = $bridge->complete($taskId, $commands); + } if (($outcome['completed'] ?? false) === true && $claimedTask instanceof WorkflowTask && CooperativeCancellationPolicy::claimSupportsCancellation($claimedTask)) { @@ -1917,6 +1978,8 @@ function () use ( 'current_value' => $e->currentValue, 'configured_limit' => $e->configuredLimit, ], 422); + } catch (PreparedLocalActivityAdmissionRefused) { + return PreparedLocalActivityController::refused($validated['lease_owner'], ['worker_claim_capability']); } catch (ExternalPayloadStorageUnavailable $exception) { return $this->externalPayloadFailure($taskId, (int) $validated['workflow_task_attempt'], $exception, 503); } catch (StreamFullException $exception) { @@ -1990,6 +2053,13 @@ function () use ( return BackendLockPressure::workerOperationResponse($request, true); } + if ($checkpoint) { + unset($outcome['completed']); + $outcome['history_refresh_page_token'] = WorkflowHistoryPageToken::encode(0); + + return WorkerProtocol::json($outcome, $this->workflowOutcomeStatus($outcome['reason'])); + } + return WorkerProtocol::json([ 'task_id' => $taskId, 'workflow_task_attempt' => (int) $validated['workflow_task_attempt'], @@ -2481,7 +2551,7 @@ private function persistTypedSearchAttributeHistoryIdentity( array $commands, array $outcome, ): void { - if (($outcome['completed'] ?? false) !== true) { + if (($outcome['completed'] ?? false) !== true && ($outcome['checkpointed'] ?? false) !== true) { return; } @@ -2494,11 +2564,19 @@ private function persistTypedSearchAttributeHistoryIdentity( return; } - $events = WorkflowHistoryEvent::query() + $eventQuery = WorkflowHistoryEvent::query() ->where('workflow_task_id', $taskId) - ->where('event_type', HistoryEventType::SearchAttributesUpserted->value) - ->orderBy('sequence') - ->get(); + ->where('event_type', HistoryEventType::SearchAttributesUpserted->value); + if (($outcome['checkpointed'] ?? false) === true) { + $eventQuery->where('payload->sequence', '>=', $outcome['start_sequence']) + ->where('payload->sequence', '<', $outcome['next_sequence']); + } else { + $prefixEnd = WorkflowTask::query()->find($taskId)?->payload['portable_local_checkpoint']['next_sequence'] ?? null; + if (is_int($prefixEnd) && $prefixEnd > 0) { + $eventQuery->where('payload->sequence', '>=', $prefixEnd); + } + } + $events = $eventQuery->orderBy('sequence')->get(); if ($events->count() !== count($upserts)) { throw new \RuntimeException('Typed search-attribute commands did not produce a one-to-one history event set.'); diff --git a/app/Http/Middleware/RuntimeExternalPayloadTransport.php b/app/Http/Middleware/RuntimeExternalPayloadTransport.php index 1c7d017d..c05e6d90 100644 --- a/app/Http/Middleware/RuntimeExternalPayloadTransport.php +++ b/app/Http/Middleware/RuntimeExternalPayloadTransport.php @@ -27,6 +27,16 @@ class RuntimeExternalPayloadTransport 'ScheduleController@update' => [['action', 'input']], 'ServiceCatalogController@executeOperation' => [['arguments']], 'WorkerController@completeQueryTask' => [['result_envelope']], + 'PreparedLocalActivityController@prepare' => [['descriptor', 'arguments']], + 'PreparedLocalActivityController@recover' => [['descriptor', 'arguments']], + 'PreparedLocalActivityController@outcome' => [['report', 'result']], + 'WorkerController@checkpointLocalActivityPrefix' => [ + ['commands', '*', 'arguments'], + ['commands', '*', 'entries'], + ['commands', '*', 'exception', 'details'], + ['commands', '*', 'request_payload'], + ['commands', '*', 'result'], + ], 'WorkerController@completeWorkflowTask' => [ ['commands', '*', 'arguments'], ['commands', '*', 'entries'], @@ -45,6 +55,9 @@ class RuntimeExternalPayloadTransport /** @var array>> */ private const INCOMING_REFERENCE_PATHS = [ + 'WorkerController@checkpointLocalActivityPrefix' => [ + ['commands', '*', 'workflow_stream', 'items', '*', 'payload_reference'], + ], 'WorkerController@completeWorkflowTask' => [ ['commands', '*', 'workflow_stream', 'items', '*', 'payload_reference'], ], diff --git a/app/Support/PreparedLocalActivityAdmissionRefused.php b/app/Support/PreparedLocalActivityAdmissionRefused.php new file mode 100644 index 00000000..a08d75e2 --- /dev/null +++ b/app/Support/PreparedLocalActivityAdmissionRefused.php @@ -0,0 +1,6 @@ + */ + public static function unavailableReasons(?string $requestVersion): array + { + $reasons = []; + if (! CooperativeCancellationPolicy::serverSupported()) { + $reasons[] = 'server_protocol'; + } + if (! WorkerProtocol::versionMeetsMinimum($requestVersion, self::MINIMUM_PROTOCOL_VERSION)) { + $reasons[] = 'request_protocol'; + } + if (! self::backendSupported()) { + $reasons[] = 'installed_runtime_prepared_local_activity'; + } + + return $reasons; + } + + /** @return array|null */ + public static function currentClaim(WorkflowTask $task): ?array + { + $claim = $task->payload[self::CLAIM_KEY] ?? null; + if (! is_array($claim) || ! CooperativeCancellationPolicy::claimSupportsCancellation($task) + || ! in_array(self::CAPABILITY, $claim['capabilities'] ?? [], true)) { + return null; + } + + return $claim; + } + + /** + * Persist only after Native preparation succeeds, in the same transaction. + * Native's attempt/execution/run/task locks must already be held. This + * receipt is issued authority, not a second lease or attempt tracker. + * + * @param array $claim + */ + public static function remember(WorkflowTask $task, ActivityAttempt $attempt, array $claim): bool + { + if (self::currentClaim($task) !== $claim || $attempt->workflow_task_id !== $task->id + || $attempt->workflow_run_id !== $task->workflow_run_id + || $attempt->lease_owner !== ($claim['lease_owner'] ?? null) + || ! is_string($attempt->worker_attempt_id) || $attempt->worker_attempt_id === '') { + return false; + } + $payload = is_array($task->payload) ? $task->payload : []; + $receipts = $payload[self::LOCAL_CLAIMS_KEY] ?? []; + if (! is_array($receipts)) { + return false; + } + $receipt = [ + 'schema' => 'durable-workflow.server.prepared-local-authority/v1', + 'claim' => $claim, + 'activity_execution_id' => $attempt->activity_execution_id, + 'activity_attempt_id' => $attempt->id, + 'worker_attempt_id' => $attempt->worker_attempt_id, + ]; + if (array_key_exists($attempt->id, $receipts)) { + // JSON objects may be reordered by MySQL. The validated original + // claim and current claim are both read from the same task payload. + return self::originalClaim($task, $attempt, $claim['lease_owner'], $claim['attempt']) === $claim; + } + $receipts[$attempt->id] = $receipt; + $payload[self::LOCAL_CLAIMS_KEY] = $receipts; + $task->forceFill(['payload' => $payload])->save(); + + return true; + } + + /** + * Takeover may replace the current claim. Stop observation and receipts + * still use the immutable claim that admitted this particular callback. + * Native separately validates its canonical Started event and authority. + * + * @return array|null + */ + public static function originalClaim( + WorkflowTask $task, + ActivityAttempt $attempt, + string $leaseOwner, + int $workflowTaskAttempt, + ): ?array { + $receipt = $task->payload[self::LOCAL_CLAIMS_KEY][$attempt->id] ?? null; + $claim = is_array($receipt) ? ($receipt['claim'] ?? null) : null; + if (! is_array($claim) + || ($receipt['schema'] ?? null) !== 'durable-workflow.server.prepared-local-authority/v1' + || ($receipt['activity_execution_id'] ?? null) !== $attempt->activity_execution_id + || ($receipt['activity_attempt_id'] ?? null) !== $attempt->id + || ($receipt['worker_attempt_id'] ?? null) !== $attempt->worker_attempt_id + || $attempt->workflow_task_id !== $task->id || $attempt->workflow_run_id !== $task->workflow_run_id + || $attempt->lease_owner !== $leaseOwner || $workflowTaskAttempt < 1 + || ($claim['task_id'] ?? null) !== $task->id || ($claim['run_id'] ?? null) !== $task->workflow_run_id + || ($claim['namespace'] ?? null) !== $task->namespace + || ($claim['worker_id'] ?? null) !== $leaseOwner || ($claim['lease_owner'] ?? null) !== $leaseOwner + || ($claim['attempt'] ?? null) !== $workflowTaskAttempt + || ! CooperativeCancellationPolicy::supports($claim['capabilities'] ?? [], $claim['protocol_version'] ?? null) + || ! in_array(self::CAPABILITY, $claim['capabilities'] ?? [], true)) { + return null; + } + + return $claim; + } +} diff --git a/app/Support/WorkerProtocol.php b/app/Support/WorkerProtocol.php index d98e55dc..f8501c61 100644 --- a/app/Support/WorkerProtocol.php +++ b/app/Support/WorkerProtocol.php @@ -547,6 +547,7 @@ public static function serverCapabilities(): array 'activity_retry_policy' => true, 'activity_timeouts' => true, 'cooperative_cancellation' => CooperativeCancellationPolicy::serverSupported(), + 'prepared_local_activities' => PreparedLocalActivityPolicy::serverSupported(), 'activity_cancellation_acknowledgement' => CooperativeCancellationPolicy::serverSupported() && CooperativeCancellationPolicy::activityAcknowledgementBackendSupported(), 'local_activities' => [ diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index e9c01744..38b1fd58 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "27" + version: "28" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -45,6 +45,23 @@ x-durable-workflow-cooperative-cancellation-contract: cleanup_deadline: immutable_original_request_deadline cleanup_completion: cancelled_with_original_request_id terminal_cancel_and_terminate: unchanged +x-durable-workflow-prepared-local-activity-contract: + status: candidate + minimum_protocol_version: "1.20" + worker_capabilities: [cooperative_cancellation, prepared_local_activities] + backend_role: Workflow\V2\Contracts\PreparedLocalActivityTaskBridge + advertised_support: configured_protocol_and_actual_bound_optional_role + preparation: canonical_scheduled_and_started_commit_before_callback_admission + prefix: ordinary_command_validation_authorization_and_quota_with_retained_claim + supervisor: original_issued_claim_bound_to_namespace_task_run_owner_epoch_and_attempt + renewal: atomic_workflow_and_local_attempt_leases_without_application_heartbeat + cancellation: original_root_identity_and_deadline_without_releasing_cleanup_claim + stop_acknowledgement: original_supervisor_reports_only_after_callback_is_stopped_and_joined + expired_callback_stop: unknown_until_original_supervisor_reports_join + retry: durable_workflow_task_backoff_and_original_total_deadline + outcome: immutable_duplicate_receipt_and_stale_publication_refusal + history_refresh_page_token: opaque_cursor_from_first_event_with_current_claim_authority_still_required + published_protocol_1_19: unchanged x-durable-workflow-message-streams-contract: discovery_path: /cluster/info#/message_streams_contract minimum_protocol_version: "1.15" @@ -397,6 +414,189 @@ paths: "404": { $ref: "#/components/responses/WorkerError" } "409": { $ref: "#/components/responses/WorkerError" } "422": { $ref: "#/components/responses/WorkerError" } + /worker/workflow-tasks/{taskId}/local-activities/prepare: + post: + operationId: prepareLocalActivity + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: prepared_local_activities + description: >- + Commit callback admission before application code starts. Retry an identical preparation with the same workflow claim and SDK attempt nonce. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/PreparedLocalPrepareRequest" } + responses: + "200": + description: The durable prepare result. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - $ref: "#/components/schemas/PreparedLocalPreparationResult" + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "429": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/local-activities/checkpoint: + post: + operationId: checkpointLocalActivityPrefix + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: prepared_local_activities + description: >- + Commit the preceding command prefix using ordinary completion validation, authorization and quota. Retain the hosting claim. Checkpoint commit grants no callback admission. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/PreparedLocalCheckpointRequest" } + responses: + "200": + description: The durable checkpoint result. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - $ref: "#/components/schemas/PreparedLocalCheckpointResult" + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "429": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/local-activities/recover: + post: + operationId: recoverLocalActivity + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: prepared_local_activities + description: >- + A replacement workflow claim recovers an expired original local attempt. Durable recovery records callback stop as unknown and preserves the original total timeout budget. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/PreparedLocalRecoverRequest" } + responses: + "200": + description: The durable recover result. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - $ref: "#/components/schemas/PreparedLocalRecoveryResult" + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "429": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/local-activities/{attemptId}/control: + post: + operationId: controlLocalActivity + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: prepared_local_activities + description: >- + Observe the original local attempt and optionally renew both workflow and attempt leases atomically. No application heartbeat is recorded. Accepted cancellation returns its original identity and deadline even after workflow claim takeover. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + - $ref: "#/components/parameters/PreparedLocalAttemptIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/PreparedLocalControlRequest" } + responses: + "200": + description: The durable control result. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - $ref: "#/components/schemas/PreparedLocalControlResult" + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "429": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/local-activities/{attemptId}/outcome: + post: + operationId: recordLocalActivityOutcome + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: prepared_local_activities + description: >- + Commit an original callback outcome under its issued authority. An identical committed receipt remains readable after takeover. A stale worker cannot publish a fresh result. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + - $ref: "#/components/parameters/PreparedLocalAttemptIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/PreparedLocalOutcomeRequest" } + responses: + "200": + description: The durable outcome result. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - $ref: "#/components/schemas/PreparedLocalOutcomeResult" + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "429": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/local-activities/{attemptId}/acknowledge-cancellation: + post: + operationId: acknowledgeLocalActivityCancellation + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: prepared_local_activities + description: >- + Report only after the original supervisor has stopped and joined the callback. A surviving supervisor may report after registration drain or claim takeover without gaining result or lease authority. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + - $ref: "#/components/parameters/PreparedLocalAttemptIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/PreparedLocalStopRequest" } + responses: + "200": + description: The durable acknowledge-cancellation result. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - $ref: "#/components/schemas/PreparedLocalStopResult" + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "429": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } /worker/workflow-tasks/{taskId}/complete: post: operationId: completeWorkflowTask @@ -704,6 +904,12 @@ components: in: path required: true schema: { type: string, minLength: 1 } + PreparedLocalAttemptIdPath: + name: attemptId + in: path + required: true + description: Canonical durable attempt ID returned by preparation, distinct from worker_attempt_id. + schema: { type: string, minLength: 1 } SessionIdPath: name: sessionId in: path @@ -1077,6 +1283,9 @@ components: activity_cancellation_acknowledgement: type: boolean description: Original remote callback-stop receipts require an installed runtime primitive and worker protocol 1.20. False for the default protocol and for runtimes without that primitive. + prepared_local_activities: + type: boolean + description: Candidate preparation, prefix checkpoint, supervisor control, outcome, recovery and joined-stop receipts. Requires protocol 1.20 and the actual bound PreparedLocalActivityTaskBridge role. False at the default protocol 1.19. supported_workflow_task_commands: type: array uniqueItems: true @@ -1444,6 +1653,204 @@ components: operation_sequence: { type: "null" } operation_sequence_span: { type: "null" } reason: { type: string, const: cancellation_waiting_for_child } + PreparedLocalClaimRequest: + type: object + required: [lease_owner, workflow_task_attempt] + properties: + lease_owner: { type: string, minLength: 1, maxLength: 255 } + workflow_task_attempt: { type: integer, minimum: 1 } + PreparedLocalDescriptor: + type: object + additionalProperties: false + required: [type, activity_type, arguments] + properties: + type: { type: string, const: record_local_activity } + activity_type: { type: string, minLength: 1 } + arguments: + description: Encoded Avro or a standard Avro envelope resolved by this namespace's configured runtime storage. + oneOf: + - { type: string, minLength: 1 } + - { type: object, required: [codec], properties: { codec: { const: avro } } } + payload_codec: { type: string, const: avro } + execution_mode: { type: string, const: local } + start_to_close_timeout: { type: [integer, "null"], minimum: 1 } + schedule_to_close_timeout: { type: [integer, "null"], minimum: 1 } + heartbeat_timeout: { type: [integer, "null"], minimum: 1 } + retry_policy: + type: [object, "null"] + properties: + max_attempts: { type: [integer, "null"], minimum: 1 } + backoff_seconds: { type: array, items: { type: integer, minimum: 0 } } + non_retryable_exceptions: { type: array, items: { type: string } } + PreparedLocalRecoverRequest: + allOf: + - $ref: "#/components/schemas/PreparedLocalClaimRequest" + - type: object + required: [sequence, descriptor] + properties: + sequence: { type: integer, minimum: 1 } + descriptor: { $ref: "#/components/schemas/PreparedLocalDescriptor" } + PreparedLocalPrepareRequest: + allOf: + - $ref: "#/components/schemas/PreparedLocalRecoverRequest" + - type: object + required: [worker_attempt_id] + properties: + worker_attempt_id: + type: string + minLength: 1 + maxLength: 255 + description: Stable SDK nonce for this callback admission. It is distinct from the durable activity_attempt_id. + PreparedLocalCheckpointRequest: + allOf: + - $ref: "#/components/schemas/PreparedLocalClaimRequest" + - type: object + required: [checkpoint_id, start_sequence, commands] + properties: + checkpoint_id: { type: string, minLength: 1, maxLength: 255 } + start_sequence: { type: integer, minimum: 1 } + commands: + type: array + maxItems: 100 + description: Nonterminal prefix commands. Turn-closing waits, local outcome reports and selection cancellation are excluded. Nonempty completion-only message cursor, wait and sticky cache metadata are rejected. + items: { $ref: "#/components/schemas/WorkflowCommand" } + PreparedLocalControlRequest: + allOf: + - $ref: "#/components/schemas/PreparedLocalClaimRequest" + - type: object + properties: + renew_lease: { type: boolean, default: false } + PreparedLocalOutcomeRequest: + allOf: + - $ref: "#/components/schemas/PreparedLocalClaimRequest" + - type: object + required: [report] + properties: + report: + oneOf: + - type: object + additionalProperties: false + required: [outcome, result] + properties: + outcome: { const: completed } + result: + oneOf: + - { type: string, minLength: 1 } + - { type: object, required: [codec], properties: { codec: { const: avro } } } + payload_codec: { const: avro } + - type: object + additionalProperties: false + required: [outcome, message] + properties: + outcome: { const: failed } + message: { type: string } + exception_type: { type: string, minLength: 1, maxLength: 255 } + non_retryable: { type: boolean, default: false } + - type: object + additionalProperties: false + required: [outcome] + properties: + outcome: { const: timed_out } + PreparedLocalStopRequest: + allOf: + - $ref: "#/components/schemas/PreparedLocalClaimRequest" + - type: object + required: [request_id] + properties: + request_id: { type: string, minLength: 1, maxLength: 255 } + PreparedLocalPreparationResult: + type: object + required: [prepared, duplicate, reason, activity_execution_id, activity_attempt_id, worker_attempt_id, attempt_number, workflow_task_id, workflow_task_attempt, lease_owner, lease_expires_at, server_time] + properties: + prepared: { const: true } + duplicate: { type: boolean } + reason: { type: "null" } + activity_execution_id: { type: string, minLength: 1 } + activity_attempt_id: { type: string, minLength: 1 } + worker_attempt_id: { type: string, minLength: 1 } + attempt_number: { type: integer, minimum: 1 } + workflow_task_id: { type: string, minLength: 1 } + workflow_task_attempt: { type: integer, minimum: 1 } + lease_owner: { type: string, minLength: 1 } + lease_expires_at: { type: string, format: date-time } + server_time: { type: string, format: date-time } + start_to_close_deadline_at: { type: [string, "null"], format: date-time } + schedule_to_close_deadline_at: { type: [string, "null"], format: date-time } + heartbeat_deadline_at: { type: [string, "null"], format: date-time } + PreparedLocalCheckpointResult: + type: object + required: [checkpointed, duplicate, reason, checkpoint_id, task_id, workflow_run_id, workflow_task_attempt, start_sequence, next_sequence, fingerprint, created_task_ids, recorded_at] + properties: + checkpointed: { const: true } + duplicate: { type: boolean } + reason: { type: "null" } + checkpoint_id: { type: string, minLength: 1 } + task_id: { type: string, minLength: 1 } + workflow_run_id: { type: string, minLength: 1 } + workflow_task_attempt: { type: integer, minimum: 1 } + start_sequence: { type: integer, minimum: 1 } + next_sequence: { type: integer, minimum: 1 } + fingerprint: { type: string, pattern: "^[0-9a-f]{64}$" } + created_task_ids: { type: array, items: { type: string } } + recorded_at: { type: string, format: date-time } + PreparedLocalOutcomeResult: + type: object + required: [recorded, duplicate, reason, event_id, event_type, workflow_run_id, workflow_task_id, workflow_task_attempt, activity_execution_id, activity_attempt_id, worker_attempt_id, recorded_at, claim_released, created_task_ids] + properties: + recorded: { const: true } + duplicate: { type: boolean } + reason: { type: "null" } + event_id: { type: string, minLength: 1 } + event_type: { type: string, enum: [ActivityCompleted, ActivityFailed, ActivityTimedOut, ActivityRetryScheduled] } + workflow_run_id: { type: string, minLength: 1 } + workflow_task_id: { type: string, minLength: 1 } + workflow_task_attempt: { type: integer, minimum: 1 } + activity_execution_id: { type: string, minLength: 1 } + activity_attempt_id: { type: string, minLength: 1 } + worker_attempt_id: { type: string, minLength: 1 } + recorded_at: { type: string, format: date-time } + claim_released: { type: boolean } + created_task_ids: { type: array, items: { type: string } } + PreparedLocalRecoveryResult: + type: object + required: [recovered, duplicate, reason, event_id, event_type, callback_stop_state, claim_released, created_task_ids] + properties: + recovered: { const: true } + duplicate: { type: boolean } + reason: { type: "null" } + event_id: { type: string, minLength: 1 } + event_type: { type: string, enum: [ActivityFailed, ActivityTimedOut, ActivityRetryScheduled] } + callback_stop_state: { const: unknown } + claim_released: { type: boolean } + created_task_ids: { type: array, items: { type: string } } + PreparedLocalControlResult: + type: object + required: [active, renewed, stop_required, reason, activity_execution_id, activity_attempt_id, workflow_task_id, workflow_task_attempt, server_time] + properties: + active: { type: boolean } + renewed: { type: boolean } + stop_required: { type: boolean } + reason: { type: [string, "null"] } + activity_execution_id: { type: string, minLength: 1 } + activity_attempt_id: { type: string, minLength: 1 } + workflow_task_id: { type: string, minLength: 1 } + workflow_task_attempt: { type: integer, minimum: 1 } + server_time: { type: string, format: date-time } + cancellation_request: + type: object + required: [request_id, root_request_id, cleanup_deadline_at] + properties: + request_id: { type: string, minLength: 1 } + root_request_id: { type: string, minLength: 1 } + cleanup_deadline_at: { type: string, format: date-time } + PreparedLocalStopResult: + type: object + required: [acknowledged, duplicate, reason, history_event_id] + properties: + acknowledged: { const: true } + duplicate: { type: boolean } + reason: { type: "null" } + history_event_id: { type: string, minLength: 1 } WorkerHistoryEvent: type: object additionalProperties: true diff --git a/routes/api.php b/routes/api.php index 57bdf806..1e652c48 100644 --- a/routes/api.php +++ b/routes/api.php @@ -11,6 +11,7 @@ use App\Http\Controllers\Api\LegacyV1ProjectionController; use App\Http\Controllers\Api\MessageStreamController; use App\Http\Controllers\Api\NamespaceController; +use App\Http\Controllers\Api\PreparedLocalActivityController; use App\Http\Controllers\Api\RuntimeCredentialController; use App\Http\Controllers\Api\RuntimeExternalPayloadController; use App\Http\Controllers\Api\ScheduleController; @@ -234,6 +235,12 @@ Route::post('/workflow-tasks/{taskId}/history', [WorkerController::class, 'workflowTaskHistory']); Route::post('/workflow-tasks/{taskId}/heartbeat', [WorkerController::class, 'heartbeatWorkflowTask']); Route::post('/workflow-tasks/{taskId}/deliver-cancellation', [CooperativeCancellationController::class, 'deliver']); + Route::post('/workflow-tasks/{taskId}/local-activities/prepare', [PreparedLocalActivityController::class, 'prepare']); + Route::post('/workflow-tasks/{taskId}/local-activities/checkpoint', [WorkerController::class, 'checkpointLocalActivityPrefix']); + Route::post('/workflow-tasks/{taskId}/local-activities/recover', [PreparedLocalActivityController::class, 'recover']); + Route::post('/workflow-tasks/{taskId}/local-activities/{attemptId}/control', [PreparedLocalActivityController::class, 'control']); + Route::post('/workflow-tasks/{taskId}/local-activities/{attemptId}/outcome', [PreparedLocalActivityController::class, 'outcome']); + Route::post('/workflow-tasks/{taskId}/local-activities/{attemptId}/acknowledge-cancellation', [PreparedLocalActivityController::class, 'acknowledgeCancellation']); Route::post('/workflow-tasks/{taskId}/complete', [WorkerController::class, 'completeWorkflowTask']); Route::post('/workflow-tasks/{taskId}/fail', [WorkerController::class, 'failWorkflowTask']); diff --git a/tests/Feature/PreparedLocalActivityProtocolTest.php b/tests/Feature/PreparedLocalActivityProtocolTest.php new file mode 100644 index 00000000..ec490dae --- /dev/null +++ b/tests/Feature/PreparedLocalActivityProtocolTest.php @@ -0,0 +1,533 @@ + '1.20', + 'workflows.v2.workflow_task_lease_seconds' => 10, + 'workflows.v2.types.workflows' => ['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class], + ]); + WorkflowNamespace::query()->create([ + 'name' => 'default', 'description' => 'Test', 'retention_days' => 30, 'status' => 'active', + ]); + } + + protected function tearDown(): void + { + if ($this->payloadDirectory !== null) { + File::deleteDirectory($this->payloadDirectory); + } + parent::tearDown(); + } + + public function test_default_protocol_does_not_advertise_or_admit_prepared_callbacks(): void + { + config(['server.worker_protocol.version' => '1.19']); + $this->withHeaders($this->headers('1.19'))->postJson('/api/worker/workflow-tasks/missing/local-activities/prepare', []) + ->assertStatus(409)->assertJsonPath('reason', 'prepared_local_activity_not_supported') + ->assertJsonPath('server_capabilities.prepared_local_activities', false) + ->assertJsonPath('unavailable', fn ($reasons) => in_array('server_protocol', $reasons, true) && in_array('request_protocol', $reasons, true)); + } + + public function test_actual_bound_bridge_must_supply_the_optional_role(): void + { + $this->app->instance(WorkflowTaskBridge::class, \Mockery::mock(WorkflowTaskBridge::class)); + $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/missing/local-activities/prepare', []) + ->assertStatus(409)->assertJsonPath('reason', 'prepared_local_activity_not_supported') + ->assertJsonPath('server_capabilities.prepared_local_activities', false) + ->assertJsonFragment(['installed_runtime_prepared_local_activity']); + } + + public function test_prepared_capability_requires_cooperation_and_protocol_1_20(): void + { + foreach (['1.19', '1.20'] as $version) { + $this->withHeaders($this->headers($version))->postJson('/api/worker/register', [ + 'worker_id' => 'incorrect', 'task_queue' => 'prepared', 'runtime' => 'php', + 'capabilities' => [PreparedLocalActivityPolicy::CAPABILITY], + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + ])->assertStatus(409)->assertJsonPath('reason', 'prepared_local_activity_capability_mismatch'); + } + $this->assertSame(0, WorkerRegistration::query()->count()); + } + + public function test_preparation_records_admission_before_callback_and_duplicate_keeps_authority(): void + { + $task = $this->claim(); + $prepared = $this->prepare($task)->assertOk()->assertJsonPath('prepared', true)->json(); + $attempt = ActivityAttempt::query()->findOrFail($prepared['activity_attempt_id']); + $this->assertNotSame('sdk-local-1', $attempt->id); + $this->assertSame('sdk-local-1', $attempt->worker_attempt_id); + $this->assertSame(1, $attempt->attempt_number); + $this->assertSame($task['task_id'], $attempt->workflow_task_id); + $hosting = WorkflowTask::query()->findOrFail($task['task_id']); + $this->assertSame(TaskStatus::Leased, $hosting->status); + $this->assertNotNull(PreparedLocalActivityPolicy::originalClaim($hosting, $attempt, $task['lease_owner'], $task['workflow_task_attempt'])); + $before = $attempt->getAttributes(); + $this->travel(2)->seconds(); + $this->prepare($task)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('activity_attempt_id', $attempt->id) + ->assertJsonPath('lease_expires_at', $prepared['lease_expires_at']); + $this->assertSame($before, $attempt->refresh()->getAttributes()); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityScheduled)); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityStarted)); + $this->assertSame(0, WorkflowTask::query()->where('task_type', 'activity')->count()); + } + + public function test_claim_cannot_be_upgraded_by_later_registration_edit(): void + { + $task = $this->claim(prepared: false); + WorkerRegistration::query()->where('worker_id', 'original')->sole()->forceFill([ + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY, PreparedLocalActivityPolicy::CAPABILITY], + ])->save(); + $this->prepare($task)->assertStatus(409)->assertJsonFragment(['worker_claim_capability']); + $this->assertSame(0, ActivityAttempt::query()->count()); + } + + public function test_malformed_and_non_avro_arguments_are_refused_before_admission(): void + { + $task = $this->claim(); + $fixture = json_decode(file_get_contents(base_path('tests/Fixtures/CodecRegression/worker-malformed-serialized-payload.json')), true, flags: JSON_THROW_ON_ERROR); + $badBytes = base64_decode($fixture['framing']['wire_base64'], true); + $this->prepare($task, ['descriptor' => $this->descriptor(['arguments' => $badBytes])]) + ->assertStatus(422)->assertJsonValidationErrors('descriptor.arguments'); + $this->prepare($task, ['descriptor' => $this->descriptor(['payload_codec' => 'json'])]) + ->assertStatus(422)->assertJsonValidationErrors('descriptor.payload_codec'); + $this->assertSame(0, ActivityAttempt::query()->count()); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityScheduled)); + } + + public function test_inline_envelopes_preserve_exact_arguments_and_result_bytes(): void + { + $task = $this->claim(); + $input = Serializer::serializeWithCodec('avro', ['Ada', ['nested' => true]]); + $prepared = $this->prepare($task, ['descriptor' => $this->descriptor(['arguments' => ['codec' => 'avro', 'blob' => $input]])]) + ->assertOk()->json(); + $this->assertSame($input, ActivityExecution::query()->findOrFail($prepared['activity_execution_id'])->arguments); + $result = Serializer::serializeWithCodec('avro', ['binary' => AvroBinaryValue::fromBytes("\0\xff")]); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => [ + 'outcome' => 'completed', 'result' => ['codec' => 'avro', 'blob' => $result], + ]])->assertOk()->assertJsonPath('recorded', true); + $event = WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id']) + ->where('event_type', HistoryEventType::ActivityCompleted)->sole(); + $this->assertSame($result, $event->payload['result']); + } + + public function test_external_arguments_and_result_references_are_namespace_bound(): void + { + $task = $this->claim(); + $this->payloadDirectory = sys_get_temp_dir().'/dw-prepared-payload-'.getmypid(); + foreach (['default', 'other'] as $namespace) { + WorkflowNamespace::query()->updateOrCreate(['name' => $namespace], [ + 'description' => 'Fixture', 'retention_days' => 30, 'status' => 'active', + 'external_payload_storage' => ['driver' => 'local', 'enabled' => true, 'threshold_bytes' => 4096, + 'config' => ['uri' => 'file://'.$this->payloadDirectory.'/'.$namespace]], + ]); + } + $arguments = Serializer::serializeWithCodec('avro', ['external arguments']); + $foreign = $this->uploadPayload($arguments, 'other'); + $this->prepare($task, ['descriptor' => $this->descriptor(['arguments' => ['codec' => 'avro', 'external_payload' => $foreign]])]) + ->assertStatus(404)->assertJsonPath('reason', 'external_payload_not_found'); + $this->assertSame(0, ActivityAttempt::query()->count()); + $reference = $this->uploadPayload($arguments, 'default'); + $prepared = $this->prepare($task, ['descriptor' => $this->descriptor(['arguments' => ['codec' => 'avro', 'external_payload' => $reference]])]) + ->assertOk()->json(); + $this->assertSame($arguments, ActivityExecution::query()->findOrFail($prepared['activity_execution_id'])->arguments); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => [ + 'outcome' => 'completed', 'result' => ['codec' => 'avro', 'external_payload' => $foreign], + ]])->assertStatus(404)->assertJsonPath('reason', 'external_payload_not_found'); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + $result = Serializer::serializeWithCodec('avro', ['external result']); + $resultReference = $this->uploadPayload($result, 'default'); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => [ + 'outcome' => 'completed', 'result' => ['codec' => 'avro', 'external_payload' => $resultReference], + ]])->assertOk()->assertJsonPath('recorded', true); + $this->assertSame($result, ActivityExecution::query()->findOrFail($prepared['activity_execution_id'])->result); + } + + public function test_checkpoint_commits_prefix_keeps_claim_and_reuses_completion_validation(): void + { + $task = $this->claim(); + $claim = WorkflowTask::query()->findOrFail($task['task_id']); + $expiry = $claim->lease_expires_at->toISOString(); + $body = ['checkpoint_id' => 'prefix-1', 'start_sequence' => 1, 'commands' => [ + ['type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 7)], + ]]; + $this->localRequest($task, 'checkpoint', $body)->assertOk()->assertJsonPath('checkpointed', true) + ->assertJsonPath('next_sequence', 2); + $this->localRequest($task, 'checkpoint', $body)->assertOk()->assertJsonPath('duplicate', true); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::SideEffectRecorded)); + $this->assertSame(TaskStatus::Leased, $claim->refresh()->status); + $this->assertSame($expiry, $claim->lease_expires_at->toISOString()); + $this->localRequest($task, 'checkpoint', [...$body, 'commands' => [['type' => 'start_timer', 'delay_seconds' => 2]]]) + ->assertStatus(409)->assertJsonPath('reason', 'local_activity_checkpoint_mismatch'); + $this->localRequest($task, 'checkpoint', [...$body, 'commands' => [[ + 'type' => 'start_timer', 'delay_seconds' => 1, 'cancellation_policy' => 'try_cancel', + ]]])->assertStatus(422)->assertJsonValidationErrors('commands.0.cancellation_policy'); + $this->localRequest($task, 'checkpoint', [...$body, 'commands' => [['type' => 'complete_workflow']]]) + ->assertStatus(409)->assertJsonPath('reason', 'invalid_local_activity_checkpoint_commands'); + $this->prepare($task, ['sequence' => 2])->assertOk()->assertJsonPath('prepared', true); + } + + public function test_checkpoint_rejects_completion_only_metadata(): void + { + $task = $this->claim(); + $this->localRequest($task, 'checkpoint', ['checkpoint_id' => 'one', 'start_sequence' => 1, 'commands' => [], + 'message_stream_cursors' => [['stream_name' => 'input', 'cursor' => 1]], + ])->assertStatus(422)->assertJsonValidationErrors('message_stream_cursors'); + $this->assertArrayNotHasKey('portable_local_checkpoint', WorkflowTask::query()->findOrFail($task['task_id'])->payload); + } + + public function test_sequential_checkpoints_keep_typed_search_history_identity(): void + { + SearchAttributeDefinition::query()->create(['namespace' => 'default', 'name' => 'Tier', 'type' => 'keyword']); + $task = $this->claim(extraCapabilities: ['typed_search_attributes']); + foreach ([1 => 'basic', 2 => 'pro'] as $sequence => $value) { + $body = ['checkpoint_id' => "search-{$sequence}", 'start_sequence' => $sequence, 'commands' => [[ + 'type' => 'upsert_search_attributes', 'attributes' => ['Tier' => $value], 'attribute_types' => ['Tier' => 'keyword'], + ]]]; + $this->localRequest($task, 'checkpoint', $body)->assertOk()->assertJsonPath('checkpointed', true); + $this->localRequest($task, 'checkpoint', $body)->assertOk()->assertJsonPath('duplicate', true); + } + $events = WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id']) + ->where('event_type', HistoryEventType::SearchAttributesUpserted)->orderBy('sequence')->get(); + $this->assertCount(2, $events); + foreach ($events as $event) { + $this->assertSame(['Tier' => 'keyword'], $event->payload['attribute_types']); + } + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [ + ['type' => 'upsert_search_attributes', 'attributes' => ['Tier' => 'enterprise'], 'attribute_types' => ['Tier' => 'keyword']], + ['type' => 'complete_workflow'], + ], + ])->assertOk()->assertJsonPath('recorded', true)->assertJsonPath('run_status', 'completed'); + $final = WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id']) + ->where('event_type', HistoryEventType::SearchAttributesUpserted)->orderByDesc('sequence')->firstOrFail(); + $this->assertSame(3, $final->payload['sequence']); + $this->assertSame(['Tier' => 'keyword'], $final->payload['attribute_types']); + } + + public function test_supervisor_renews_both_leases_without_application_heartbeats(): void + { + $task = $this->claim(); + $prepared = $this->prepare($task)->assertOk()->json(); + $attempt = ActivityAttempt::query()->findOrFail($prepared['activity_attempt_id']); + $heartbeat = $attempt->last_heartbeat_at->toISOString(); + $execution = ActivityExecution::query()->findOrFail($prepared['activity_execution_id'])->getAttributes(); + $this->travel(4)->seconds(); + $this->localRequest($task, "{$attempt->id}/control", ['renew_lease' => true])->assertOk() + ->assertJsonPath('active', true)->assertJsonPath('renewed', true); + $this->assertSame($heartbeat, $attempt->refresh()->last_heartbeat_at->toISOString()); + $this->assertSame($execution, ActivityExecution::query()->findOrFail($prepared['activity_execution_id'])->getAttributes()); + $this->assertSame($attempt->lease_expires_at->toISOString(), WorkflowTask::query()->findOrFail($task['task_id'])->lease_expires_at->toISOString()); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityStarted)); + } + + public function test_same_claim_history_refresh_and_completion_do_not_repeat_a_prepared_callback(): void + { + $task = $this->claim(); + $this->localRequest($task, 'checkpoint', ['checkpoint_id' => 'first', 'start_sequence' => 1, 'commands' => [[ + 'type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 7), + ]]])->assertOk(); + $prepared = $this->prepare($task, ['sequence' => 2])->assertOk()->json(); + $receipt = $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => $this->success()])->assertOk()->json(); + $history = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/history", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'next_history_page_token' => $receipt['history_refresh_page_token'], + ])->assertOk()->json('history_events'); + $this->assertContains('SideEffectRecorded', array_column($history, 'event_type')); + $this->assertContains('ActivityCompleted', array_column($history, 'event_type')); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [['type' => 'complete_workflow', 'result' => $this->success()['result'], 'payload_codec' => 'avro']], + ])->assertOk()->assertJsonPath('recorded', true)->assertJsonPath('run_status', 'completed'); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityScheduled)); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityStarted)); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::SideEffectRecorded)); + $this->assertSame(TaskStatus::Completed, WorkflowTask::query()->findOrFail($task['task_id'])->status); + } + + public function test_changed_registration_cannot_renew_or_publish_a_fresh_result(): void + { + $task = $this->claim(); + $attempt = $this->prepare($task)->assertOk()->json('activity_attempt_id'); + WorkerRegistration::query()->where('worker_id', 'original')->sole()->forceFill(['status' => 'draining'])->save(); + $this->localRequest($task, "{$attempt}/control", ['renew_lease' => true])->assertStatus(409) + ->assertJsonPath('active', false)->assertJsonPath('renewed', false)->assertJsonPath('stop_required', true) + ->assertJsonPath('reason', 'stale_worker_registration'); + $this->localRequest($task, "{$attempt}/outcome", ['report' => $this->success()])->assertStatus(409) + ->assertJsonPath('recorded', false)->assertJsonPath('reason', 'stale_worker_registration'); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + } + + public function test_success_receipt_is_idempotent_after_takeover_and_changed_result_is_refused(): void + { + $task = $this->claim(); + $attempt = $this->prepare($task)->assertOk()->json('activity_attempt_id'); + $first = $this->localRequest($task, "{$attempt}/outcome", ['report' => $this->success()])->assertOk() + ->assertJsonPath('recorded', true)->assertJsonPath('claim_released', false)->json(); + $this->replaceClaim($task); + $this->localRequest($task, "{$attempt}/outcome", ['report' => $this->success()])->assertOk() + ->assertJsonPath('duplicate', true)->assertJsonPath('event_id', $first['event_id']); + $this->localRequest($task, "{$attempt}/outcome", ['report' => $this->success('changed')])->assertStatus(409) + ->assertJsonPath('reason', 'local_activity_outcome_mismatch'); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + } + + public function test_failure_schedules_one_durable_retry_and_retains_the_total_deadline(): void + { + $task = $this->claim(); + $descriptor = $this->descriptor(['schedule_to_close_timeout' => 60, 'retry_policy' => [ + 'max_attempts' => 2, 'backoff_seconds' => [2], + ]]); + $first = $this->prepare($task, ['descriptor' => $descriptor])->assertOk()->json(); + $failure = ['outcome' => 'failed', 'message' => 'retry me', 'exception_type' => \RuntimeException::class]; + $result = $this->localRequest($task, "{$first['activity_attempt_id']}/outcome", ['report' => $failure]) + ->assertOk()->assertJsonPath('recorded', true)->assertJsonPath('claim_released', true) + ->assertJsonPath('event_type', 'ActivityRetryScheduled')->json(); + $this->localRequest($task, "{$first['activity_attempt_id']}/outcome", ['report' => $failure]) + ->assertOk()->assertJsonPath('duplicate', true)->assertJsonPath('event_id', $result['event_id']); + $this->assertCount(1, $result['created_task_ids']); + $this->assertSame(TaskStatus::Completed, WorkflowTask::query()->findOrFail($task['task_id'])->status); + $this->assertSame(0, WorkflowTask::query()->where('task_type', 'activity')->count()); + $this->poll('original')->assertOk()->assertJsonPath('task', null); + $this->travel(2)->seconds(); + $retry = $this->poll('original')->assertOk()->json('task'); + $this->assertSame($result['created_task_ids'][0], $retry['task_id']); + $second = $this->prepare($retry, ['worker_attempt_id' => 'sdk-local-2', 'descriptor' => $descriptor]) + ->assertOk()->assertJsonPath('attempt_number', 2)->json(); + $this->assertNotSame($first['activity_attempt_id'], $second['activity_attempt_id']); + $this->assertSame($first['schedule_to_close_deadline_at'], $second['schedule_to_close_deadline_at']); + $this->localRequest($retry, "{$second['activity_attempt_id']}/outcome", ['report' => $this->success()]) + ->assertOk()->assertJsonPath('recorded', true); + } + + public function test_cold_recovery_records_unknown_stop_then_retries_under_original_budget(): void + { + $task = $this->claim(); + $descriptor = $this->descriptor(['schedule_to_close_timeout' => 60, 'retry_policy' => [ + 'max_attempts' => 2, 'backoff_seconds' => [2], + ]]); + $first = $this->prepare($task, ['descriptor' => $descriptor])->assertOk()->json(); + ActivityAttempt::query()->findOrFail($first['activity_attempt_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $replacement = $this->replaceClaim($task); + $body = ['sequence' => 1, 'descriptor' => $descriptor]; + $recovered = $this->localRequest($replacement, 'recover', $body)->assertOk() + ->assertJsonPath('recovered', true)->assertJsonPath('claim_released', true) + ->assertJsonPath('callback_stop_state', 'unknown')->json(); + $this->localRequest($replacement, 'recover', $body)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('event_id', $recovered['event_id']); + $this->localRequest($task, "{$first['activity_attempt_id']}/outcome", ['report' => $this->success()]) + ->assertStatus(409)->assertJsonPath('reason', 'stale_activity_attempt'); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityCancellationAcknowledged)); + $this->travel(2)->seconds(); + $retry = $this->poll('replacement')->assertOk()->json('task'); + $this->assertSame($recovered['created_task_ids'][0], $retry['task_id']); + $second = $this->prepare($retry, ['worker_attempt_id' => 'sdk-recovered', 'descriptor' => $descriptor]) + ->assertOk()->assertJsonPath('attempt_number', 2)->json(); + $this->assertSame($first['schedule_to_close_deadline_at'], $second['schedule_to_close_deadline_at']); + $this->assertSame(0, WorkflowTask::query()->where('task_type', 'activity')->count()); + } + + public function test_accepted_cancellation_refuses_publication_without_releasing_cleanup_claim(): void + { + $task = $this->claim(); + $attempt = $this->prepare($task)->assertOk()->json('activity_attempt_id'); + $this->withHeaders($this->controlHeaders())->postJson("/api/workflows/{$task['workflow_id']}/request-cancellation", []) + ->assertStatus(202); + $this->localRequest($task, "{$attempt}/outcome", ['report' => $this->success()])->assertStatus(409) + ->assertJsonPath('recorded', false)->assertJsonPath('fenced', true)->assertJsonPath('reason', 'cancellation_requested'); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityCancelled)); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityCancellationAcknowledged)); + } + + public function test_original_supervisor_observes_cancellation_and_records_join_after_takeover(): void + { + $task = $this->claim(); + $attempt = $this->prepare($task)->assertOk()->json('activity_attempt_id'); + $replacement = $this->replaceClaim($task); + $accepted = $this->withHeaders($this->controlHeaders())->postJson("/api/workflows/{$task['workflow_id']}/request-cancellation", [ + 'cleanup_timeout_seconds' => 30, + ])->assertStatus(202)->json('cancellation_request'); + $this->localRequest($task, "{$attempt}/control", ['renew_lease' => true])->assertOk() + ->assertJsonPath('active', false)->assertJsonPath('renewed', false) + ->assertJsonPath('stop_required', true)->assertJsonPath('cancellation_request.request_id', $accepted['request_id']) + ->assertJsonPath('cancellation_request.cleanup_deadline_at', $accepted['cleanup_deadline_at']); + $this->localRequest($task, "{$attempt}/acknowledge-cancellation", ['request_id' => $accepted['request_id']]) + ->assertOk()->assertJsonPath('acknowledged', true); + $this->localRequest($task, "{$attempt}/acknowledge-cancellation", ['request_id' => $accepted['request_id']]) + ->assertOk()->assertJsonPath('duplicate', true); + $current = WorkflowTask::query()->findOrFail($task['task_id']); + $this->assertSame(TaskStatus::Leased, $current->status); + $this->assertSame($replacement['lease_owner'], $current->lease_owner); + $this->assertSame($replacement['workflow_task_attempt'], $current->attempt_count); + } + + public function test_wrong_owner_epoch_namespace_and_unknown_attempt_cannot_act(): void + { + $task = $this->claim(); + $attempt = $this->prepare($task)->assertOk()->json('activity_attempt_id'); + foreach ([['lease_owner' => 'different'], ['workflow_task_attempt' => 99]] as $wrong) { + $this->localRequest($task, "{$attempt}/control", $wrong)->assertStatus(409) + ->assertJsonFragment(['original_prepared_local_claim']); + } + WorkflowNamespace::query()->create(['name' => 'other', 'description' => 'Other', 'retention_days' => 30, 'status' => 'active']); + $this->withHeaders([...$this->headers(), 'X-Namespace' => 'other']) + ->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/{$attempt}/control", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + ])->assertStatus(404)->assertJsonPath('reason', 'activity_attempt_not_found'); + $this->localRequest($task, 'missing/control')->assertStatus(404); + $this->withHeaders($this->headers('1.19'))->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/{$attempt}/control", []) + ->assertStatus(409)->assertJsonFragment(['request_protocol']); + } + + public function test_quota_refusal_rolls_back_admission_and_original_authority_receipt(): void + { + $task = $this->claim(); + $before = WorkflowHistoryEvent::query()->count(); + config(['server.namespace_durable_state.limits' => ['max_workflow_history_events' => $before + 1]]); + $this->prepare($task)->assertStatus(429)->assertJsonPath('reason', 'namespace_workflow_history_events_exhausted'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertSame(0, ActivityAttempt::query()->count()); + $this->assertArrayNotHasKey('_server_prepared_local_activity_claims', WorkflowTask::query()->findOrFail($task['task_id'])->payload); + } + + private function claim(bool $prepared = true, array $extraCapabilities = []): array + { + if (! PreparedLocalActivityPolicy::backendSupported()) { + $this->markTestSkipped('Prepared-local source binding required. Published protocol 1.19 refusal is covered separately.'); + } + $this->withHeaders($this->controlHeaders())->postJson('/api/workflows', [ + 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'prepared', 'input' => ['Ada'], + ])->assertCreated(); + $this->register('original', $prepared, $extraCapabilities); + + return $this->poll('original')->assertOk()->json('task'); + } + + private function register(string $id, bool $prepared = true, array $extraCapabilities = []): void + { + $this->withHeaders($this->headers())->postJson('/api/worker/register', [ + 'worker_id' => $id, 'task_queue' => 'prepared', 'runtime' => 'php', + 'supported_workflow_types' => ['tests.external-greeting-workflow'], + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY, ...($prepared ? [PreparedLocalActivityPolicy::CAPABILITY] : []), ...$extraCapabilities], + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + 'max_concurrent_workflow_tasks' => 1, + 'process_metrics' => ['host' => 'test-worker', 'process_started_at' => now()->toISOString(), 'process_id' => $id === 'original' ? 10 : 20], + ])->assertCreated(); + } + + private function poll(string $id): TestResponse + { + return $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', ['worker_id' => $id, 'task_queue' => 'prepared']); + } + + private function replaceClaim(array $task): array + { + // Exercise the real poll takeover using an expired workflow claim. + // Keep the prepared callback lease live to test original stop authority. + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $this->register('replacement'); + $replacement = $this->poll('replacement')->assertOk()->json('task'); + $this->assertSame($task['task_id'], $replacement['task_id']); + $this->assertSame($task['workflow_task_attempt'] + 1, $replacement['workflow_task_attempt']); + + return $replacement; + } + + private function prepare(array $task, array $overrides = []): TestResponse + { + return $this->localRequest($task, 'prepare', ['sequence' => 1, 'worker_attempt_id' => 'sdk-local-1', 'descriptor' => $this->descriptor(), ...$overrides]); + } + + private function descriptor(array $overrides = []): array + { + return ['type' => 'record_local_activity', 'activity_type' => 'opaque-local', + 'arguments' => Serializer::serializeWithCodec('avro', ['Ada']), 'payload_codec' => 'avro', ...$overrides]; + } + + private function success(string $value = 'done'): array + { + return ['outcome' => 'completed', 'result' => Serializer::serializeWithCodec('avro', $value), 'payload_codec' => 'avro']; + } + + private function localRequest(array $task, string $path, array $body = []): TestResponse + { + $response = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/{$path}", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], ...$body, + ]); + if ($response->status() === 200) { + $operation = substr($path, strrpos('/'.$path, '/')); + $schema = match ($operation) { + 'prepare' => 'PreparedLocalPreparationResult', 'recover' => 'PreparedLocalRecoveryResult', + 'checkpoint' => 'PreparedLocalCheckpointResult', 'control' => 'PreparedLocalControlResult', + 'outcome' => 'PreparedLocalOutcomeResult', 'acknowledge-cancellation' => 'PreparedLocalStopResult', + }; + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/'.$schema, json_decode($response->getContent(), flags: JSON_THROW_ON_ERROR)); + } + + return $response; + } + + private function uploadPayload(string $payload, string $namespace): array + { + return $this->call('POST', '/api/external-payloads/v1', [], [], [], [ + 'CONTENT_TYPE' => 'application/octet-stream', 'HTTP_X_NAMESPACE' => $namespace, + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_CODEC' => 'avro', + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_SIZE' => (string) strlen($payload), + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_SHA256' => hash('sha256', $payload), + ], $payload)->assertCreated()->json('reference'); + } + + private function eventCount(array $task, HistoryEventType $type): int + { + return WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id'])->where('event_type', $type)->count(); + } + + private function headers(string $version = '1.20'): array + { + return ['X-Namespace' => 'default', WorkerProtocol::HEADER => $version]; + } + + private function controlHeaders(): array + { + return ['X-Namespace' => 'default', 'X-Durable-Workflow-Control-Plane-Version' => '2']; + } +} From 04c9023ba55c6a0a0eeca062a296936a940280fc Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 05:39:46 +0000 Subject: [PATCH 18/57] Preserve prepared local completions under storage draining --- .github/workflows/phpunit-feature.yml | 15 +++---- .../Middleware/EnforceStorageAdmission.php | 6 +++ .../worker-protocol-api.openapi.yaml | 2 + .../PreparedLocalActivityProtocolTest.php | 45 +++++++++++++++++++ 4 files changed, 60 insertions(+), 8 deletions(-) diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index 993dcd1d..779a1838 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -308,6 +308,7 @@ jobs: set -euo pipefail test "$(git -C prepared-workflow-source rev-parse HEAD)" = "$DW_PREPARED_WORKFLOW_COMMIT" mkdir -p prepared-local-evidence + chmod 0777 prepared-local-evidence { echo "Candidate source qualification only. No image, release or SDK conformance claim." echo "Server commit: $(git rev-parse HEAD)" @@ -316,7 +317,7 @@ jobs: } > prepared-local-evidence/provenance.txt docker build --target base -t dw-server-prepared-local-base . tar --exclude=vendor --exclude='*/vendor' --exclude=build --exclude='*/build' \ - --exclude=prepared-local-evidence -cf - . > "$RUNNER_TEMP/prepared-local-source.tar" + --exclude=prepared-local-evidence -cf ../prepared-local-source.tar . for database in sqlite mysql pgsql; do database_name=prepared_local_test database_user=root @@ -324,9 +325,9 @@ jobs: if [ "$database" = sqlite ]; then database_name=:memory:; fi if [ "$database" = pgsql ]; then database_user=postgres; database_port=5439; fi docker run --rm --init --network host \ - --user "$(id -u):$(id -g)" \ - --tmpfs "/app:rw,uid=$(id -u),gid=$(id -g),mode=0755" \ - --volume "$RUNNER_TEMP/prepared-local-source.tar:/source.tar:ro" \ + --user 1000:1000 \ + --tmpfs /app:rw,uid=1000,gid=1000,mode=0755 \ + --volume "$PWD/../prepared-local-source.tar:/source.tar:ro" \ --volume "$PWD/prepared-local-evidence:/evidence" \ --env COMPOSER_HOME=/tmp/dw-prepared-composer \ --env DB_CONNECTION="$database" --env DB_DATABASE="$database_name" \ @@ -338,11 +339,9 @@ jobs: composer install --no-interaction --no-progress --prefer-dist mv vendor/durable-workflow/workflow /tmp/locked-workflow-package cp -a prepared-workflow-source vendor/durable-workflow/workflow - php -r '\''require "vendor/autoload.php"; if (!interface_exists(Workflow\V2\Contracts\PreparedLocalActivityTaskBridge::class)) { exit(1); }'\'' - vendor/bin/phpunit tests/Feature/PreparedLocalActivityProtocolTest.php --log-junit "/evidence/$DW_PREPARED_DATABASE.xml" --colors=never - php -r '\''$xml = simplexml_load_file("/evidence/".getenv("DW_PREPARED_DATABASE").".xml"); if ($xml === false || count($xml->xpath("//testcase/skipped")) > 0) { exit(1); }'\'' + php vendor/bin/phpunit tests/Feature/PreparedLocalActivityProtocolTest.php --fail-on-skipped --log-junit "/evidence/$DW_PREPARED_DATABASE.xml" --colors=never if [ "$DW_PREPARED_DATABASE" = sqlite ]; then - vendor/bin/phpunit tests/Feature/CooperativeCancellationProtocolTest.php tests/Feature/WorkflowWorkerProtocolTest.php tests/Feature/ActivityWorkerProtocolTest.php tests/Feature/SearchAttributeValueValidationTest.php tests/Feature/NamespaceDurableStateQuotaTest.php tests/Feature/RuntimeExternalPayloadTransportTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php --log-junit /evidence/affected-regression.xml --colors=never + php vendor/bin/phpunit tests/Feature/CooperativeCancellationProtocolTest.php tests/Feature/WorkflowWorkerProtocolTest.php tests/Feature/ActivityWorkerProtocolTest.php tests/Feature/SearchAttributeValueValidationTest.php tests/Feature/NamespaceDurableStateQuotaTest.php tests/Feature/RuntimeExternalPayloadTransportTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php --log-junit /evidence/affected-regression.xml --colors=never fi ' > "prepared-local-evidence/$database.log" 2>&1 done diff --git a/app/Http/Middleware/EnforceStorageAdmission.php b/app/Http/Middleware/EnforceStorageAdmission.php index c8f579c9..8cf107fe 100644 --- a/app/Http/Middleware/EnforceStorageAdmission.php +++ b/app/Http/Middleware/EnforceStorageAdmission.php @@ -17,6 +17,12 @@ final class EnforceStorageAdmission 'WorkerController@heartbeat', 'WorkerController@heartbeatWorkflowTask', 'WorkerController@completeWorkflowTask', + 'WorkerController@checkpointLocalActivityPrefix', + 'PreparedLocalActivityController@prepare', + 'PreparedLocalActivityController@recover', + 'PreparedLocalActivityController@control', + 'PreparedLocalActivityController@outcome', + 'PreparedLocalActivityController@acknowledgeCancellation', 'WorkerController@failWorkflowTask', 'WorkerController@completeQueryTask', 'WorkerController@failQueryTask', diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 38b1fd58..00662b59 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -61,6 +61,8 @@ x-durable-workflow-prepared-local-activity-contract: retry: durable_workflow_task_backoff_and_original_total_deadline outcome: immutable_duplicate_receipt_and_stale_publication_refusal history_refresh_page_token: opaque_cursor_from_first_event_with_current_claim_authority_still_required + storage_draining: existing_issued_claim_may_finish_and_report_join + storage_fenced: all_six_local_mutation_endpoints_refused_before_handler published_protocol_1_19: unchanged x-durable-workflow-message-streams-contract: discovery_path: /cluster/info#/message_streams_contract diff --git a/tests/Feature/PreparedLocalActivityProtocolTest.php b/tests/Feature/PreparedLocalActivityProtocolTest.php index ec490dae..7e0ed23f 100644 --- a/tests/Feature/PreparedLocalActivityProtocolTest.php +++ b/tests/Feature/PreparedLocalActivityProtocolTest.php @@ -12,6 +12,7 @@ use Illuminate\Support\Facades\File; use Illuminate\Support\Facades\Queue; use Illuminate\Testing\TestResponse; +use Tests\Feature\Concerns\StoragePressureFixture; use Tests\Fixtures\ExternalGreetingWorkflow; use Tests\Support\OpenApiSchema; use Tests\TestCase; @@ -28,6 +29,7 @@ final class PreparedLocalActivityProtocolTest extends TestCase { use RefreshDatabase; + use StoragePressureFixture; private ?string $payloadDirectory = null; @@ -47,6 +49,7 @@ protected function setUp(): void protected function tearDown(): void { + $this->removeStoragePressure(); if ($this->payloadDirectory !== null) { File::deleteDirectory($this->payloadDirectory); } @@ -428,6 +431,48 @@ public function test_quota_refusal_rolls_back_admission_and_original_authority_r $this->assertArrayNotHasKey('_server_prepared_local_activity_claims', WorkflowTask::query()->findOrFail($task['task_id'])->payload); } + public function test_draining_allows_an_existing_claim_to_commit_prefix_prepare_and_finish(): void + { + $task = $this->claim(); + $this->configureStoragePressure('draining'); + $this->localRequest($task, 'checkpoint', ['checkpoint_id' => 'drain', 'start_sequence' => 1, 'commands' => [[ + 'type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 7), + ]]])->assertOk()->assertJsonPath('checkpointed', true); + $prepared = $this->prepare($task, ['sequence' => 2])->assertOk()->json(); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/control", ['renew_lease' => true]) + ->assertOk()->assertJsonPath('renewed', true); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => $this->success()]) + ->assertOk()->assertJsonPath('recorded', true); + } + + public function test_draining_preserves_original_cancellation_observation_and_join_receipt(): void + { + $task = $this->claim(); + $attempt = $this->prepare($task)->assertOk()->json('activity_attempt_id'); + $pending = $this->withHeaders($this->controlHeaders())->postJson("/api/workflows/{$task['workflow_id']}/request-cancellation", []) + ->assertStatus(202)->json('cancellation_request'); + $this->configureStoragePressure('draining'); + $this->localRequest($task, "{$attempt}/control")->assertOk()->assertJsonPath('stop_required', true) + ->assertJsonPath('cancellation_request.request_id', $pending['request_id']); + $this->localRequest($task, "{$attempt}/acknowledge-cancellation", ['request_id' => $pending['request_id']]) + ->assertOk()->assertJsonPath('acknowledged', true); + } + + public function test_storage_fence_refuses_all_local_mutations_without_changing_leases_or_history(): void + { + $task = $this->claim(); + $attemptId = $this->prepare($task)->assertOk()->json('activity_attempt_id'); + $attempt = ActivityAttempt::query()->findOrFail($attemptId); + $before = $attempt->getAttributes(); + $history = WorkflowHistoryEvent::query()->count(); + $this->configureStoragePressure('fenced'); + foreach (['prepare', 'recover', 'checkpoint', "{$attemptId}/control", "{$attemptId}/outcome", "{$attemptId}/acknowledge-cancellation"] as $path) { + $this->localRequest($task, $path)->assertStatus(503)->assertJsonPath('request_admitted', false); + } + $this->assertSame($before, $attempt->refresh()->getAttributes()); + $this->assertSame($history, WorkflowHistoryEvent::query()->count()); + } + private function claim(bool $prepared = true, array $extraCapabilities = []): array { if (! PreparedLocalActivityPolicy::backendSupported()) { From d3dcc3b9ba5a4af4ac0ddfd18c06cc0a554977c0 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 05:49:18 +0000 Subject: [PATCH 19/57] Preserve strict worker fencing across JSON key order --- .github/workflows/phpunit-feature.yml | 10 ++++++--- app/Support/WorkerPollFence.php | 5 +++++ .../PreparedLocalActivityProtocolTest.php | 4 +++- tests/Unit/WorkerPollFenceTest.php | 21 +++++++++++++++++++ 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index 779a1838..6bffcb70 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -318,13 +318,14 @@ jobs: docker build --target base -t dw-server-prepared-local-base . tar --exclude=vendor --exclude='*/vendor' --exclude=build --exclude='*/build' \ --exclude=prepared-local-evidence -cf ../prepared-local-source.tar . + qualification_status=0 for database in sqlite mysql pgsql; do database_name=prepared_local_test database_user=root database_port=3317 if [ "$database" = sqlite ]; then database_name=:memory:; fi if [ "$database" = pgsql ]; then database_user=postgres; database_port=5439; fi - docker run --rm --init --network host \ + if ! docker run --rm --init --network host \ --user 1000:1000 \ --tmpfs /app:rw,uid=1000,gid=1000,mode=0755 \ --volume "$PWD/../prepared-local-source.tar:/source.tar:ro" \ @@ -339,12 +340,15 @@ jobs: composer install --no-interaction --no-progress --prefer-dist mv vendor/durable-workflow/workflow /tmp/locked-workflow-package cp -a prepared-workflow-source vendor/durable-workflow/workflow - php vendor/bin/phpunit tests/Feature/PreparedLocalActivityProtocolTest.php --fail-on-skipped --log-junit "/evidence/$DW_PREPARED_DATABASE.xml" --colors=never + php vendor/bin/phpunit tests/Feature/PreparedLocalActivityProtocolTest.php tests/Unit/WorkerPollFenceTest.php --fail-on-skipped --log-junit "/evidence/$DW_PREPARED_DATABASE.xml" --colors=never if [ "$DW_PREPARED_DATABASE" = sqlite ]; then php vendor/bin/phpunit tests/Feature/CooperativeCancellationProtocolTest.php tests/Feature/WorkflowWorkerProtocolTest.php tests/Feature/ActivityWorkerProtocolTest.php tests/Feature/SearchAttributeValueValidationTest.php tests/Feature/NamespaceDurableStateQuotaTest.php tests/Feature/RuntimeExternalPayloadTransportTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php --log-junit /evidence/affected-regression.xml --colors=never fi - ' > "prepared-local-evidence/$database.log" 2>&1 + ' > "prepared-local-evidence/$database.log" 2>&1; then + qualification_status=1 + fi done + test "$qualification_status" = 0 - name: Retain exact source qualification evidence if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 diff --git a/app/Support/WorkerPollFence.php b/app/Support/WorkerPollFence.php index 6e5a6b7d..4d3c3daf 100644 --- a/app/Support/WorkerPollFence.php +++ b/app/Support/WorkerPollFence.php @@ -92,6 +92,11 @@ private static function matchesCurrent(array $snapshot, bool $lockForUpdate): bo : []; if ($processIdentity !== [] || $currentProcessIdentity !== []) { + // MySQL may reorder keys when a claim is persisted as JSON. + // Preserve strict field/value checks without depending on order. + ksort($processIdentity); + ksort($currentProcessIdentity); + return $processIdentity === $currentProcessIdentity; } diff --git a/tests/Feature/PreparedLocalActivityProtocolTest.php b/tests/Feature/PreparedLocalActivityProtocolTest.php index 7e0ed23f..5ee44e8f 100644 --- a/tests/Feature/PreparedLocalActivityProtocolTest.php +++ b/tests/Feature/PreparedLocalActivityProtocolTest.php @@ -89,7 +89,9 @@ public function test_prepared_capability_requires_cooperation_and_protocol_1_20( public function test_preparation_records_admission_before_callback_and_duplicate_keeps_authority(): void { $task = $this->claim(); - $prepared = $this->prepare($task)->assertOk()->assertJsonPath('prepared', true)->json(); + $response = $this->prepare($task); + $this->assertSame(200, $response->status(), json_encode($response->json('reason'), JSON_THROW_ON_ERROR)); + $prepared = $response->assertJsonPath('prepared', true)->json(); $attempt = ActivityAttempt::query()->findOrFail($prepared['activity_attempt_id']); $this->assertNotSame('sdk-local-1', $attempt->id); $this->assertSame('sdk-local-1', $attempt->worker_attempt_id); diff --git a/tests/Unit/WorkerPollFenceTest.php b/tests/Unit/WorkerPollFenceTest.php index d80ec470..bc6ec00e 100644 --- a/tests/Unit/WorkerPollFenceTest.php +++ b/tests/Unit/WorkerPollFenceTest.php @@ -73,6 +73,27 @@ public function test_worker_fence_rejects_superseded_registration_even_with_fres $this->assertFalse(WorkerPollFence::isCurrent($snapshot)); } + public function test_persisted_process_identity_ignores_key_order_and_preserves_strict_fencing(): void + { + $worker = $this->createWorker('php-worker-json-fence'); + $worker->forceFill(['process_metrics' => [ + 'host' => 'worker-host', 'process_started_at' => now()->toISOString(), 'process_id' => 123, + ]])->save(); + $snapshot = WorkerPollFence::snapshot($worker); + $snapshot['process_identity'] = array_reverse($snapshot['process_identity'], true); + + $this->assertTrue(WorkerPollFence::isCurrent($snapshot)); + $this->assertTrue(WorkerPollFence::isCurrentForUpdate($snapshot)); + foreach ([ + [...$snapshot['process_identity'], 'process_id' => 124], + [...$snapshot['process_identity'], 'process_id' => '123'], + [...$snapshot['process_identity'], 'extra' => 'unissued'], + array_diff_key($snapshot['process_identity'], ['host' => true]), + ] as $changedIdentity) { + $this->assertFalse(WorkerPollFence::isCurrent([...$snapshot, 'process_identity' => $changedIdentity])); + } + } + private function createWorker(string $workerId): WorkerRegistration { return WorkerRegistration::query()->create([ From 929b457ce89ffad1f0d11d517f81a9f7aa4854f0 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 06:37:17 +0000 Subject: [PATCH 20/57] Admit bounded prepared cleanup and distinct application heartbeats --- .../Api/PreparedLocalActivityController.php | 24 +++- .../Middleware/EnforceStorageAdmission.php | 1 + app/Support/PreparedLocalActivityPolicy.php | 3 +- .../worker-protocol-api.openapi.yaml | 76 ++++++++++++- routes/api.php | 1 + .../PreparedLocalActivityProtocolTest.php | 105 +++++++++++++++++- 6 files changed, 198 insertions(+), 12 deletions(-) diff --git a/app/Http/Controllers/Api/PreparedLocalActivityController.php b/app/Http/Controllers/Api/PreparedLocalActivityController.php index 6e449c79..c249145f 100644 --- a/app/Http/Controllers/Api/PreparedLocalActivityController.php +++ b/app/Http/Controllers/Api/PreparedLocalActivityController.php @@ -27,7 +27,6 @@ use Workflow\V2\Exceptions\StructuralLimitExceededException; use Workflow\V2\Models\ActivityAttempt; use Workflow\V2\Models\WorkflowHistoryEvent; -use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; final class PreparedLocalActivityController @@ -121,6 +120,11 @@ public function outcome(Request $request, string $taskId, string $attemptId): Js return $this->attemptOperation($request, $taskId, $attemptId, 'outcome'); } + public function heartbeat(Request $request, string $taskId, string $attemptId): JsonResponse + { + return $this->attemptOperation($request, $taskId, $attemptId, 'heartbeat'); + } + public function acknowledgeCancellation(Request $request, string $taskId, string $attemptId): JsonResponse { return $this->attemptOperation($request, $taskId, $attemptId, 'acknowledge'); @@ -136,6 +140,7 @@ private function attemptOperation(Request $request, string $taskId, string $atte 'renew_lease' => ['nullable', 'boolean'], 'report' => [$operation === 'outcome' ? 'required' : 'nullable', 'array'], 'request_id' => [$operation === 'acknowledge' ? 'required' : 'nullable', 'string', 'max:255'], + 'progress' => ['nullable', 'array'], ]); $namespace = (string) $request->attributes->get('namespace'); $task = NamespaceWorkflowScope::task($namespace, $taskId); @@ -175,17 +180,26 @@ private function attemptOperation(Request $request, string $taskId, string $atte ->where('workflow_run_id', $claim['run_id']) ->where('payload->activity_attempt_id', $attemptId) ->whereNotNull('payload->local_outcome')->exists() - && WorkflowRun::query()->find($claim['run_id'])?->cancellation_request_command_id === null) { - return ['recorded' => false, 'reason' => 'stale_worker_registration']; + ) { + // A stale registration can observe and fence its old + // cancelled callback. It cannot publish shielded + // cleanup merely because cancellation was accepted. + $control = $bridge->controlLocalActivity($attemptId, $owner, $epoch, false, $version); + if (! in_array($control['reason'] ?? null, ['cancellation_requested', 'cancellation_deadline_expired'], true)) { + return ['recorded' => false, 'reason' => 'stale_worker_registration']; + } } $reply = $bridge->recordLocalActivityOutcome($attemptId, $owner, $epoch, $validated['report'], $version); $this->quota->assertNoIncreasePastLimit($quotaSnapshot); return $reply; } - $reply = $bridge->controlLocalActivity($attemptId, $owner, $epoch, $current && ($validated['renew_lease'] ?? false), $version); + $reply = $operation === 'heartbeat' && $current + ? $bridge->heartbeatLocalActivity($attemptId, $owner, $epoch, $validated['progress'] ?? [], $version) + : $bridge->controlLocalActivity($attemptId, $owner, $epoch, $current && ($validated['renew_lease'] ?? false), $version); if (! $current && ($reply['active'] ?? false)) { - $reply = [...$reply, 'active' => false, 'renewed' => false, 'stop_required' => true, 'reason' => 'stale_worker_registration']; + $reply = [...$reply, 'active' => false, 'renewed' => false, 'heartbeat_recorded' => false, + 'stop_required' => true, 'reason' => 'stale_worker_registration']; } $this->quota->assertNoIncreasePastLimit($quotaSnapshot); diff --git a/app/Http/Middleware/EnforceStorageAdmission.php b/app/Http/Middleware/EnforceStorageAdmission.php index 8cf107fe..b22682b9 100644 --- a/app/Http/Middleware/EnforceStorageAdmission.php +++ b/app/Http/Middleware/EnforceStorageAdmission.php @@ -21,6 +21,7 @@ final class EnforceStorageAdmission 'PreparedLocalActivityController@prepare', 'PreparedLocalActivityController@recover', 'PreparedLocalActivityController@control', + 'PreparedLocalActivityController@heartbeat', 'PreparedLocalActivityController@outcome', 'PreparedLocalActivityController@acknowledgeCancellation', 'WorkerController@failWorkflowTask', diff --git a/app/Support/PreparedLocalActivityPolicy.php b/app/Support/PreparedLocalActivityPolicy.php index f63f4a53..53665cb1 100644 --- a/app/Support/PreparedLocalActivityPolicy.php +++ b/app/Support/PreparedLocalActivityPolicy.php @@ -21,7 +21,8 @@ final class PreparedLocalActivityPolicy public static function backendSupported(): bool { return interface_exists(PreparedLocalActivityTaskBridge::class) - && app(WorkflowTaskBridge::class) instanceof PreparedLocalActivityTaskBridge; + && app(WorkflowTaskBridge::class) instanceof PreparedLocalActivityTaskBridge + && method_exists(app(WorkflowTaskBridge::class), 'heartbeatLocalActivity'); } public static function serverSupported(): bool diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 00662b59..9167bc7b 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "28" + version: "29" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -55,6 +55,8 @@ x-durable-workflow-prepared-local-activity-contract: prefix: ordinary_command_validation_authorization_and_quota_with_retained_claim supervisor: original_issued_claim_bound_to_namespace_task_run_owner_epoch_and_attempt renewal: atomic_workflow_and_local_attempt_leases_without_application_heartbeat + application_heartbeat: canonical_progress_and_heartbeat_timeout_without_lease_renewal + shielded_cleanup: canonical_request_and_delivery_identity_with_runtime_owned_original_deadline cancellation: original_root_identity_and_deadline_without_releasing_cleanup_claim stop_acknowledgement: original_supervisor_reports_only_after_callback_is_stopped_and_joined expired_callback_stop: unknown_until_original_supervisor_reports_join @@ -62,7 +64,7 @@ x-durable-workflow-prepared-local-activity-contract: outcome: immutable_duplicate_receipt_and_stale_publication_refusal history_refresh_page_token: opaque_cursor_from_first_event_with_current_claim_authority_still_required storage_draining: existing_issued_claim_may_finish_and_report_join - storage_fenced: all_six_local_mutation_endpoints_refused_before_handler + storage_fenced: all_seven_local_mutation_endpoints_refused_before_handler published_protocol_1_19: unchanged x-durable-workflow-message-streams-contract: discovery_path: /cluster/info#/message_streams_contract @@ -537,6 +539,37 @@ paths: "422": { $ref: "#/components/responses/WorkerError" } "429": { $ref: "#/components/responses/WorkerError" } "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/local-activities/{attemptId}/heartbeat: + post: + operationId: heartbeatLocalActivity + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: prepared_local_activities + description: >- + Record an application heartbeat and bounded progress under the original prepared claim. Update the heartbeat timeout without renewing either lease or moving fixed execution or root cleanup deadlines. Stale workers can observe cancellation but cannot record a heartbeat. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + - $ref: "#/components/parameters/PreparedLocalAttemptIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/PreparedLocalHeartbeatRequest" } + responses: + "200": + description: The durable heartbeat or stop result. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - $ref: "#/components/schemas/PreparedLocalControlResult" + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "429": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } /worker/workflow-tasks/{taskId}/local-activities/{attemptId}/outcome: post: operationId: recordLocalActivityOutcome @@ -1287,7 +1320,7 @@ components: description: Original remote callback-stop receipts require an installed runtime primitive and worker protocol 1.20. False for the default protocol and for runtimes without that primitive. prepared_local_activities: type: boolean - description: Candidate preparation, prefix checkpoint, supervisor control, outcome, recovery and joined-stop receipts. Requires protocol 1.20 and the actual bound PreparedLocalActivityTaskBridge role. False at the default protocol 1.19. + description: Candidate preparation, prefix checkpoint, supervisor control, application heartbeat, outcome, recovery and joined-stop receipts. Requires protocol 1.20 and the actual bound PreparedLocalActivityTaskBridge role including heartbeat support. False at the default protocol 1.19. supported_workflow_task_commands: type: array uniqueItems: true @@ -1678,12 +1711,20 @@ components: start_to_close_timeout: { type: [integer, "null"], minimum: 1 } schedule_to_close_timeout: { type: [integer, "null"], minimum: 1 } heartbeat_timeout: { type: [integer, "null"], minimum: 1 } + cancellation_cleanup: + type: object + additionalProperties: false + required: [request_id, delivery_history_event_id] + description: Explicit shielded cleanup after canonical delivery. The runtime supplies the original root identity and deadline. + properties: + request_id: { type: string, minLength: 1 } + delivery_history_event_id: { type: string, minLength: 1 } retry_policy: type: [object, "null"] properties: max_attempts: { type: [integer, "null"], minimum: 1 } backoff_seconds: { type: array, items: { type: integer, minimum: 0 } } - non_retryable_exceptions: { type: array, items: { type: string } } + non_retryable_error_types: { type: array, items: { type: string } } PreparedLocalRecoverRequest: allOf: - $ref: "#/components/schemas/PreparedLocalClaimRequest" @@ -1722,6 +1763,29 @@ components: - type: object properties: renew_lease: { type: boolean, default: false } + PreparedLocalHeartbeatRequest: + allOf: + - $ref: "#/components/schemas/PreparedLocalClaimRequest" + - type: object + properties: + progress: + type: [object, "null"] + additionalProperties: false + properties: + message: { type: string, minLength: 1, maxLength: 280 } + current: { type: [number, string], description: Nonnegative finite number or numeric string. } + total: { type: [number, string], description: Nonnegative finite number or numeric string. } + unit: { type: string, minLength: 1, maxLength: 64 } + details: { type: object, maxProperties: 20 } + PreparedLocalCleanupSnapshot: + type: [object, "null"] + additionalProperties: false + required: [request_id, root_request_id, delivery_history_event_id, cleanup_deadline_at] + properties: + request_id: { type: string, minLength: 1 } + root_request_id: { type: string, minLength: 1 } + delivery_history_event_id: { type: string, minLength: 1 } + cleanup_deadline_at: { type: string, format: date-time } PreparedLocalOutcomeRequest: allOf: - $ref: "#/components/schemas/PreparedLocalClaimRequest" @@ -1779,6 +1843,7 @@ components: start_to_close_deadline_at: { type: [string, "null"], format: date-time } schedule_to_close_deadline_at: { type: [string, "null"], format: date-time } heartbeat_deadline_at: { type: [string, "null"], format: date-time } + cancellation_cleanup: { $ref: "#/components/schemas/PreparedLocalCleanupSnapshot" } PreparedLocalCheckpointResult: type: object required: [checkpointed, duplicate, reason, checkpoint_id, task_id, workflow_run_id, workflow_task_attempt, start_sequence, next_sequence, fingerprint, created_task_ids, recorded_at] @@ -1838,6 +1903,9 @@ components: workflow_task_id: { type: string, minLength: 1 } workflow_task_attempt: { type: integer, minimum: 1 } server_time: { type: string, format: date-time } + heartbeat_recorded: { type: boolean } + heartbeat_history_event_id: { type: [string, "null"] } + cancellation_cleanup: { $ref: "#/components/schemas/PreparedLocalCleanupSnapshot" } cancellation_request: type: object required: [request_id, root_request_id, cleanup_deadline_at] diff --git a/routes/api.php b/routes/api.php index 1e652c48..e49fc28c 100644 --- a/routes/api.php +++ b/routes/api.php @@ -239,6 +239,7 @@ Route::post('/workflow-tasks/{taskId}/local-activities/checkpoint', [WorkerController::class, 'checkpointLocalActivityPrefix']); Route::post('/workflow-tasks/{taskId}/local-activities/recover', [PreparedLocalActivityController::class, 'recover']); Route::post('/workflow-tasks/{taskId}/local-activities/{attemptId}/control', [PreparedLocalActivityController::class, 'control']); + Route::post('/workflow-tasks/{taskId}/local-activities/{attemptId}/heartbeat', [PreparedLocalActivityController::class, 'heartbeat']); Route::post('/workflow-tasks/{taskId}/local-activities/{attemptId}/outcome', [PreparedLocalActivityController::class, 'outcome']); Route::post('/workflow-tasks/{taskId}/local-activities/{attemptId}/acknowledge-cancellation', [PreparedLocalActivityController::class, 'acknowledgeCancellation']); Route::post('/workflow-tasks/{taskId}/complete', [WorkerController::class, 'completeWorkflowTask']); diff --git a/tests/Feature/PreparedLocalActivityProtocolTest.php b/tests/Feature/PreparedLocalActivityProtocolTest.php index 5ee44e8f..0261b2b4 100644 --- a/tests/Feature/PreparedLocalActivityProtocolTest.php +++ b/tests/Feature/PreparedLocalActivityProtocolTest.php @@ -286,6 +286,87 @@ public function test_same_claim_history_refresh_and_completion_do_not_repeat_a_p $this->assertSame(TaskStatus::Completed, WorkflowTask::query()->findOrFail($task['task_id'])->status); } + public function test_application_heartbeat_records_progress_without_renewing_either_lease(): void + { + $task = $this->claim(); + $prepared = $this->prepare($task, ['descriptor' => $this->descriptor([ + 'heartbeat_timeout' => 3, 'start_to_close_timeout' => 60, 'schedule_to_close_timeout' => 120, + ])])->assertOk()->json(); + $lease = $prepared['lease_expires_at']; + $this->travel(2)->seconds(); + $heartbeat = $this->localRequest($task, "{$prepared['activity_attempt_id']}/heartbeat", ['progress' => ['message' => 'working']]) + ->assertOk()->assertJsonPath('active', true)->assertJsonPath('heartbeat_recorded', true) + ->assertJsonPath('renewed', false)->json(); + $this->assertSame($lease, $heartbeat['lease_expires_at']); + $this->assertSame($lease, $heartbeat['workflow_lease_expires_at']); + $this->assertSame($prepared['start_to_close_deadline_at'], $heartbeat['start_to_close_deadline_at']); + $this->assertSame($prepared['schedule_to_close_deadline_at'], $heartbeat['schedule_to_close_deadline_at']); + $this->assertNotSame($prepared['heartbeat_deadline_at'], $heartbeat['heartbeat_deadline_at']); + $event = WorkflowHistoryEvent::query()->whereKey($heartbeat['heartbeat_history_event_id'])->firstOrFail(); + $this->assertSame(['message' => 'working'], $event->payload['progress']); + $this->assertSame($task['run_id'], $event->workflow_run_id); + $this->travel(2)->seconds(); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/control", ['renew_lease' => true])->assertOk()->assertJsonPath('active', true); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityHeartbeatRecorded)); + } + + public function test_expired_or_stale_claims_cannot_record_application_heartbeats(): void + { + $task = $this->claim(); + $prepared = $this->prepare($task, ['descriptor' => $this->descriptor(['heartbeat_timeout' => 3])])->assertOk()->json(); + WorkerRegistration::query()->where('worker_id', 'original')->sole()->forceFill(['status' => 'draining'])->save(); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/heartbeat", ['progress' => ['message' => 'stale']]) + ->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration')->assertJsonPath('heartbeat_recorded', false); + WorkerRegistration::query()->where('worker_id', 'original')->sole()->forceFill(['status' => 'active'])->save(); + $this->travel(3)->seconds(); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/heartbeat")->assertStatus(409) + ->assertJsonPath('reason', 'local_activity_deadline_expired')->assertJsonPath('heartbeat_recorded', false); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityHeartbeatRecorded)); + } + + public function test_shielded_cleanup_is_bound_to_canonical_delivery_and_original_budget(): void + { + [$task, $descriptor, $accepted] = $this->cleanupClaim(); + $this->prepare($task, ['sequence' => 2])->assertStatus(409)->assertJsonPath('reason', 'cancellation_requested'); + $this->localRequest($task, 'checkpoint', ['checkpoint_id' => 'cleanup-prefix', 'start_sequence' => 2, 'commands' => [[ + 'type' => 'record_side_effect', 'marker_id' => 'cleanup', 'result' => $this->success()['result'], 'payload_codec' => 'avro', + ]]])->assertOk()->assertJsonPath('next_sequence', 3); + $response = $this->prepare($task, ['sequence' => 3, 'descriptor' => $descriptor]); + $this->assertSame(200, $response->status(), json_encode($response->json('reason'), JSON_THROW_ON_ERROR)); + $prepared = $response->assertOk() + ->assertJsonPath('cancellation_cleanup.request_id', $accepted['request_id']) + ->assertJsonPath('cancellation_cleanup.root_request_id', $accepted['request_id']) + ->assertJsonPath('cancellation_cleanup.cleanup_deadline_at', $accepted['cleanup_deadline_at'])->json(); + $this->assertSame($accepted['cleanup_deadline_at'], $prepared['schedule_to_close_deadline_at']); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/heartbeat", ['progress' => ['message' => 'cleaning']]) + ->assertOk()->assertJsonPath('active', true)->assertJsonPath('heartbeat_recorded', true); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/control", ['renew_lease' => true])->assertOk()->assertJsonPath('active', true); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => $this->success()])->assertOk()->assertJsonPath('recorded', true); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [['type' => 'complete_workflow']], + ])->assertOk()->assertJsonPath('run_status', 'cancelled'); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::CooperativeCancellationDelivered)); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + } + + public function test_stale_registration_cannot_publish_cleanup_despite_accepted_cancellation(): void + { + [$task, $descriptor] = $this->cleanupClaim(); + $response = $this->prepare($task, ['sequence' => 2, 'descriptor' => $descriptor]); + $this->assertSame(200, $response->status(), json_encode($response->json('reason'), JSON_THROW_ON_ERROR)); + $prepared = $response->assertOk()->json(); + $attempt = $prepared['activity_attempt_id']; + $before = ActivityAttempt::query()->findOrFail($attempt)->getAttributes(); + WorkerRegistration::query()->where('worker_id', 'original')->sole()->forceFill(['status' => 'draining'])->save(); + $this->localRequest($task, "{$attempt}/outcome", ['report' => $this->success()])->assertStatus(409) + ->assertJsonPath('reason', 'stale_worker_registration')->assertJsonPath('recorded', false); + $this->localRequest($task, "{$attempt}/heartbeat")->assertStatus(409)->assertJsonPath('heartbeat_recorded', false); + $this->assertSame($before, ActivityAttempt::query()->findOrFail($attempt)->getAttributes()); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityHeartbeatRecorded)); + } + public function test_changed_registration_cannot_renew_or_publish_a_fresh_result(): void { $task = $this->claim(); @@ -443,6 +524,8 @@ public function test_draining_allows_an_existing_claim_to_commit_prefix_prepare_ $prepared = $this->prepare($task, ['sequence' => 2])->assertOk()->json(); $this->localRequest($task, "{$prepared['activity_attempt_id']}/control", ['renew_lease' => true]) ->assertOk()->assertJsonPath('renewed', true); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/heartbeat", ['progress' => ['message' => 'draining']]) + ->assertOk()->assertJsonPath('heartbeat_recorded', true)->assertJsonPath('renewed', false); $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => $this->success()]) ->assertOk()->assertJsonPath('recorded', true); } @@ -468,13 +551,31 @@ public function test_storage_fence_refuses_all_local_mutations_without_changing_ $before = $attempt->getAttributes(); $history = WorkflowHistoryEvent::query()->count(); $this->configureStoragePressure('fenced'); - foreach (['prepare', 'recover', 'checkpoint', "{$attemptId}/control", "{$attemptId}/outcome", "{$attemptId}/acknowledge-cancellation"] as $path) { + foreach (['prepare', 'recover', 'checkpoint', "{$attemptId}/control", "{$attemptId}/heartbeat", "{$attemptId}/outcome", "{$attemptId}/acknowledge-cancellation"] as $path) { $this->localRequest($task, $path)->assertStatus(503)->assertJsonPath('request_admitted', false); } $this->assertSame($before, $attempt->refresh()->getAttributes()); $this->assertSame($history, WorkflowHistoryEvent::query()->count()); } + private function cleanupClaim(): array + { + $task = $this->claim(); + $accepted = $this->withHeaders($this->controlHeaders())->postJson("/api/workflows/{$task['workflow_id']}/request-cancellation", [ + 'cleanup_timeout_seconds' => 30, + ])->assertStatus(202)->json('cancellation_request'); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/deliver-cancellation", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'request_id' => $accepted['request_id'], 'sequence' => 1, 'call_kind' => 'timer', + ])->assertOk()->assertJsonPath('delivered', true); + $delivery = WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id']) + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered)->sole(); + + return [$task, $this->descriptor(['cancellation_cleanup' => [ + 'request_id' => $accepted['request_id'], 'delivery_history_event_id' => $delivery->id, + ]]), $accepted]; + } + private function claim(bool $prepared = true, array $extraCapabilities = []): array { if (! PreparedLocalActivityPolicy::backendSupported()) { @@ -543,7 +644,7 @@ private function localRequest(array $task, string $path, array $body = []): Test $operation = substr($path, strrpos('/'.$path, '/')); $schema = match ($operation) { 'prepare' => 'PreparedLocalPreparationResult', 'recover' => 'PreparedLocalRecoveryResult', - 'checkpoint' => 'PreparedLocalCheckpointResult', 'control' => 'PreparedLocalControlResult', + 'checkpoint' => 'PreparedLocalCheckpointResult', 'control', 'heartbeat' => 'PreparedLocalControlResult', 'outcome' => 'PreparedLocalOutcomeResult', 'acknowledge-cancellation' => 'PreparedLocalStopResult', }; OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) From 4d904df5851428fc645db35d19c5c33a98e179f3 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 07:22:53 +0000 Subject: [PATCH 21/57] Bound prepared local completion uploads to original claim during draining --- .github/workflows/phpunit-feature.yml | 2 +- .../RuntimeExternalPayloadReference.php | 2 + .../RuntimePayloadCompletionContext.php | 47 +++- app/Support/RuntimePayloadCompletionLease.php | 66 ++++++ docs/contracts/external-payload-storage.md | 35 +++ .../worker-protocol-api.openapi.yaml | 19 +- .../PreparedLocalActivityProtocolTest.php | 213 ++++++++++++++++++ .../RuntimePayloadCompletionContextTest.php | 45 ++++ 8 files changed, 421 insertions(+), 8 deletions(-) diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index 6bffcb70..51528ece 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -342,7 +342,7 @@ jobs: cp -a prepared-workflow-source vendor/durable-workflow/workflow php vendor/bin/phpunit tests/Feature/PreparedLocalActivityProtocolTest.php tests/Unit/WorkerPollFenceTest.php --fail-on-skipped --log-junit "/evidence/$DW_PREPARED_DATABASE.xml" --colors=never if [ "$DW_PREPARED_DATABASE" = sqlite ]; then - php vendor/bin/phpunit tests/Feature/CooperativeCancellationProtocolTest.php tests/Feature/WorkflowWorkerProtocolTest.php tests/Feature/ActivityWorkerProtocolTest.php tests/Feature/SearchAttributeValueValidationTest.php tests/Feature/NamespaceDurableStateQuotaTest.php tests/Feature/RuntimeExternalPayloadTransportTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php --log-junit /evidence/affected-regression.xml --colors=never + php vendor/bin/phpunit tests/Feature/CooperativeCancellationProtocolTest.php tests/Feature/WorkflowWorkerProtocolTest.php tests/Feature/ActivityWorkerProtocolTest.php tests/Feature/SearchAttributeValueValidationTest.php tests/Feature/NamespaceDurableStateQuotaTest.php tests/Feature/RuntimeExternalPayloadTransportTest.php tests/Feature/RuntimePayloadCompletionUploadsTest.php tests/Unit/RuntimePayloadCompletionContextTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php --log-junit /evidence/affected-regression.xml --colors=never fi ' > "prepared-local-evidence/$database.log" 2>&1; then qualification_status=1 diff --git a/app/Support/RuntimeExternalPayloadReference.php b/app/Support/RuntimeExternalPayloadReference.php index 91931776..d9575455 100644 --- a/app/Support/RuntimeExternalPayloadReference.php +++ b/app/Support/RuntimeExternalPayloadReference.php @@ -74,6 +74,8 @@ public static function transportManifest(): array 'idempotency' => 'content_addressed_per_namespace', 'completion_context' => [ 'schema' => RuntimePayloadCompletionContext::SCHEMA, + 'prepared_schema' => PreparedLocalActivityPolicy::serverSupported() + ? RuntimePayloadCompletionContext::PREPARED_SCHEMA : null, 'header' => RuntimePayloadCompletionContext::HEADER, 'use' => 'retry_draining_refusal_for_current_worker_completion_only', 'max_bytes_per_lease' => RuntimePayloadCompletionUploads::maxBytes(), diff --git a/app/Support/RuntimePayloadCompletionContext.php b/app/Support/RuntimePayloadCompletionContext.php index a38a0de8..6451df1a 100644 --- a/app/Support/RuntimePayloadCompletionContext.php +++ b/app/Support/RuntimePayloadCompletionContext.php @@ -10,6 +10,8 @@ public const SCHEMA = 'durable-workflow.v2.payload-completion-context.v1'; + public const PREPARED_SCHEMA = 'durable-workflow.v2.payload-completion-context.v2'; + /** @param list $slot */ private function __construct( public string $kind, @@ -18,6 +20,10 @@ private function __construct( public string $leaseOwner, public string $operation, public array $slot, + public ?int $sequence = null, + public ?string $activityAttemptId = null, + public string $schema = self::SCHEMA, + public ?string $checkpointId = null, ) {} public static function parse(string $header): self @@ -31,21 +37,41 @@ public static function parse(string $header): self throw self::invalid(); } $keys = ['schema', 'kind', 'task_id', 'attempt', 'lease_owner', 'operation', 'slot']; + $prepared = is_array($value) && ($value['schema'] ?? null) === self::PREPARED_SCHEMA; + if ($prepared) { + $keys[] = match ($value['operation'] ?? null) { + 'local_activity_outcome' => 'activity_attempt_id', + 'local_activity_checkpoint' => 'checkpoint_id', + default => 'sequence', + }; + } if (! is_array($value) || count($value) !== count($keys) || array_diff($keys, array_keys($value)) !== [] - || ($value['schema'] ?? null) !== self::SCHEMA + || ! in_array($value['schema'] ?? null, [self::SCHEMA, self::PREPARED_SCHEMA], true) || ! in_array($value['kind'] ?? null, ['activity', 'workflow', 'query'], true) || ! is_string($value['operation']) || ! self::identifier($value['task_id']) || ! self::identifier($value['lease_owner']) || ! is_array($value['slot']) || ! array_is_list($value['slot'])) { throw self::invalid(); } + if ($prepared && ($value['kind'] !== 'workflow' || ! match ($value['operation'] ?? null) { + 'local_activity_outcome' => self::identifier($value['activity_attempt_id']), + 'local_activity_checkpoint' => self::identifier($value['checkpoint_id']), + default => is_int($value['sequence']) && $value['sequence'] > 0, + })) { + throw self::invalid(); + } if ($value['kind'] === 'activity' ? ! self::identifier($value['attempt']) : (! is_int($value['attempt']) || $value['attempt'] < 1)) { throw self::invalid(); } - $validSlot = match ($value['kind'].'.'.($value['operation'] ?? '')) { + $validSlot = $prepared ? match ($value['operation']) { + 'local_activity_checkpoint' => self::workflowSlot($value['slot']), + 'local_activity_prepare', 'local_activity_recover' => $value['slot'] === ['descriptor', 'arguments'], + 'local_activity_outcome' => $value['slot'] === ['report', 'result'], + default => false, + } : match ($value['kind'].'.'.($value['operation'] ?? '')) { 'activity.complete' => $value['slot'] === ['result'], 'activity.fail' => $value['slot'] === ['failure', 'details'], 'query.complete' => $value['slot'] === ['result_envelope'], @@ -57,7 +83,8 @@ public static function parse(string $header): self } return new self($value['kind'], $value['task_id'], $value['attempt'], - $value['lease_owner'], $value['operation'], $value['slot']); + $value['lease_owner'], $value['operation'], $value['slot'], + $value['sequence'] ?? null, $value['activity_attempt_id'] ?? null, $value['schema'], $value['checkpoint_id'] ?? null); } public function scope(string $namespace): string @@ -69,14 +96,22 @@ public function scope(string $namespace): string public function slotIdentity(): string { - return hash('sha256', json_encode([$this->operation, $this->slot], JSON_THROW_ON_ERROR)); + $identity = [$this->operation, $this->slot]; + if ($this->schema === self::PREPARED_SCHEMA) { + $identity[] = $this->checkpointId ?? $this->sequence ?? $this->activityAttemptId; + } + + return hash('sha256', json_encode($identity, JSON_THROW_ON_ERROR)); } public function toArray(): array { - return ['schema' => self::SCHEMA, 'kind' => $this->kind, 'task_id' => $this->taskId, + return ['schema' => $this->schema, 'kind' => $this->kind, 'task_id' => $this->taskId, 'attempt' => $this->attempt, 'lease_owner' => $this->leaseOwner, - 'operation' => $this->operation, 'slot' => $this->slot]; + 'operation' => $this->operation, 'slot' => $this->slot, + ...($this->sequence === null ? [] : ['sequence' => $this->sequence]), + ...($this->activityAttemptId === null ? [] : ['activity_attempt_id' => $this->activityAttemptId]), + ...($this->checkpointId === null ? [] : ['checkpoint_id' => $this->checkpointId])]; } private static function identifier(mixed $value): bool diff --git a/app/Support/RuntimePayloadCompletionLease.php b/app/Support/RuntimePayloadCompletionLease.php index 27022196..9a2ea86b 100644 --- a/app/Support/RuntimePayloadCompletionLease.php +++ b/app/Support/RuntimePayloadCompletionLease.php @@ -5,7 +5,11 @@ use App\Models\WorkerRegistration; use Carbon\CarbonImmutable; use Workflow\V2\Contracts\ActivityTaskBridge; +use Workflow\V2\Contracts\PreparedLocalActivityTaskBridge; +use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Enums\TaskType; +use Workflow\V2\Models\ActivityAttempt; +use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Support\WorkflowTaskOwnership; final class RuntimePayloadCompletionLease @@ -102,6 +106,68 @@ private function workflow(string $namespace, RuntimePayloadCompletionContext $co throw self::rejected(); } + if ($context->schema === RuntimePayloadCompletionContext::PREPARED_SCHEMA) { + $task = $guard['task']; + $claim = PreparedLocalActivityPolicy::currentClaim($task); + if (! PreparedLocalActivityPolicy::serverSupported() || $claim === null || ! WorkerPollFence::isCurrent($claim)) { + throw self::rejected(); + } + $expires = [$guard['status']['lease_expires_at'] ?? null]; + $run = WorkflowRun::query()->find($task->workflow_run_id); + if (! $run instanceof WorkflowRun) { + throw self::rejected(); + } + if ($run->cancellation_request_command_id !== null) { + if ($run->cancellation_deadline_at === null) { + throw self::rejected(); + } + $expires[] = $run->cancellation_deadline_at->toISOString(); + } + if ($context->operation === 'local_activity_outcome') { + $attempt = ActivityAttempt::query()->where('workflow_task_id', $task->id)->find($context->activityAttemptId); + if (! $attempt instanceof ActivityAttempt + || PreparedLocalActivityPolicy::originalClaim($task, $attempt, $context->leaseOwner, $context->attempt) !== $claim) { + throw self::rejected(); + } + /** @var PreparedLocalActivityTaskBridge $bridge */ + $bridge = app(WorkflowTaskBridge::class); + $control = $bridge->controlLocalActivity($attempt->id, $context->leaseOwner, $context->attempt, false, + PreparedLocalActivityPolicy::MINIMUM_PROTOCOL_VERSION); + if (($control['active'] ?? null) !== true + || ($control['activity_attempt_id'] ?? null) !== $attempt->id + || ($control['workflow_task_id'] ?? null) !== $task->id + || ($control['workflow_task_attempt'] ?? null) !== $context->attempt + || ($control['lease_owner'] ?? null) !== $context->leaseOwner + || ($control['renewed'] ?? null) !== false) { + throw self::rejected(); + } + foreach (['lease_expires_at', 'workflow_lease_expires_at', 'start_to_close_deadline_at', + 'schedule_to_close_deadline_at', 'heartbeat_deadline_at'] as $field) { + if (! array_key_exists($field, $control) + || (str_contains($field, 'lease_expires_at') && (! is_string($control[$field]) || $control[$field] === ''))) { + throw self::rejected(); + } + if (($control[$field] ?? null) !== null) { + $expires[] = $control[$field]; + } + } + } + $earliest = null; + foreach ($expires as $expiry) { + if (! is_string($expiry) || $expiry === '') { + throw self::rejected(); + } + try { + $deadline = CarbonImmutable::parse($expiry); + } catch (\Exception) { + throw self::rejected(); + } + $earliest = $earliest === null || $deadline->lt($earliest) ? $deadline : $earliest; + } + + return $earliest?->toISOString(); + } + return $guard['status']['lease_expires_at'] ?? null; } diff --git a/docs/contracts/external-payload-storage.md b/docs/contracts/external-payload-storage.md index c3cdba3b..1b93286c 100644 --- a/docs/contracts/external-payload-storage.md +++ b/docs/contracts/external-payload-storage.md @@ -220,6 +220,41 @@ must not replace the last verified recovery point. A restored database may contain the historical hold; release it after validating the restored copy or allow its original deadline to expire. +## Payload completion while storage drains + +An existing worker claim can retry a draining upload refusal once using the +completion schema and header advertised in namespace discovery. Every slot of +the same claim shares one byte allowance and a maximum of 128 slots. New client +work remains subject to ordinary storage admission. + +The prepared local activity candidate on explicit protocol 1.20 additionally +advertises `completion_context.prepared_schema` as +`durable-workflow.v2.payload-completion-context.v2`. Its exact header object +contains `schema`, `kind: workflow`, `task_id`, the original workflow `attempt`, +`lease_owner`, `operation` and `slot`, plus one operation identity: + +| Operation | Identity | Payload slot | +| --- | --- | --- | +| `local_activity_checkpoint` | `checkpoint_id` | Ordinary command payload slot | +| `local_activity_prepare` | `sequence` | `["descriptor", "arguments"]` | +| `local_activity_recover` | `sequence` | `["descriptor", "arguments"]` | +| `local_activity_outcome` | `activity_attempt_id` | `["report", "result"]` | + +All prepared operations and legacy completion uploads share the original +workflow claim's allowance. A new sequence, checkpoint or activity attempt +does not reset it. Prepared uploads require the original issued registration, +current workflow epoch and live authority. Results additionally require the +matching admitted activity attempt and its live execution deadlines. Accepted +cancellation fences results from ordinary callbacks. Shielded cleanup keeps +the original root deadline. + +An upload does not renew workflow or activity leases, application heartbeats, +or cancellation deadlines. An exact stored reference can be read after the +claim closes without rewriting its object or extending retention. Changed +bytes still require live authority. Fenced or stale storage admission refuses +both ordinary and completion uploads. Default published protocol 1.19 keeps +its existing completion schema. + ## Typed outcomes and retryability The transport returns a stable diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 9167bc7b..7a4d03c3 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "29" + version: "30" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -64,6 +64,23 @@ x-durable-workflow-prepared-local-activity-contract: outcome: immutable_duplicate_receipt_and_stale_publication_refusal history_refresh_page_token: opaque_cursor_from_first_event_with_current_claim_authority_still_required storage_draining: existing_issued_claim_may_finish_and_report_join + external_payload_completion: + discovery: namespace.external_payload_storage.transport.upload.completion_context.prepared_schema + header: X-Durable-Workflow-Payload-Completion + schema: durable-workflow.v2.payload-completion-context.v2 + required: [schema, kind, task_id, attempt, lease_owner, operation, slot] + kind: workflow + attempt: original_workflow_task_attempt + operations: + local_activity_checkpoint: { identity: checkpoint_id, slots: ordinary_workflow_command_payload_slots } + local_activity_prepare: { identity: sequence, slots: [[descriptor, arguments]] } + local_activity_recover: { identity: sequence, slots: [[descriptor, arguments]] } + local_activity_outcome: { identity: activity_attempt_id, slots: [[report, result]] } + exact_shape: operation_identity_is_required_and_other_extra_fields_are_refused + admission: current_original_issued_prepared_claim_and_unexpired_execution_and_root_budget + allowance: shared_with_legacy_v1_completion_across_every_operation_and_slot_of_one_workflow_claim + duplicate: exact_stored_reference_may_be_read_after_claim_closes_without_object_or_retention_mutation + lifetime: no_workflow_or_activity_lease_or_application_heartbeat_or_cleanup_deadline_renewal storage_fenced: all_seven_local_mutation_endpoints_refused_before_handler published_protocol_1_19: unchanged x-durable-workflow-message-streams-contract: diff --git a/tests/Feature/PreparedLocalActivityProtocolTest.php b/tests/Feature/PreparedLocalActivityProtocolTest.php index 0261b2b4..f4968d60 100644 --- a/tests/Feature/PreparedLocalActivityProtocolTest.php +++ b/tests/Feature/PreparedLocalActivityProtocolTest.php @@ -2,11 +2,15 @@ namespace Tests\Feature; +use App\Models\RuntimeExternalPayload; +use App\Models\RuntimePayloadCompletionBudget; use App\Models\SearchAttributeDefinition; use App\Models\WorkerRegistration; use App\Models\WorkflowNamespace; use App\Support\CooperativeCancellationPolicy; use App\Support\PreparedLocalActivityPolicy; +use App\Support\RuntimeExternalPayloadReference; +use App\Support\RuntimePayloadCompletionContext; use App\Support\WorkerProtocol; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\File; @@ -25,6 +29,7 @@ use Workflow\V2\Models\ActivityExecution; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowTask; +use Workflow\V2\Support\ExternalPayloads; final class PreparedLocalActivityProtocolTest extends TestCase { @@ -59,6 +64,7 @@ protected function tearDown(): void public function test_default_protocol_does_not_advertise_or_admit_prepared_callbacks(): void { config(['server.worker_protocol.version' => '1.19']); + $this->assertNull(RuntimeExternalPayloadReference::transportManifest()['upload']['completion_context']['prepared_schema']); $this->withHeaders($this->headers('1.19'))->postJson('/api/worker/workflow-tasks/missing/local-activities/prepare', []) ->assertStatus(409)->assertJsonPath('reason', 'prepared_local_activity_not_supported') ->assertJsonPath('server_capabilities.prepared_local_activities', false) @@ -68,6 +74,7 @@ public function test_default_protocol_does_not_advertise_or_admit_prepared_callb public function test_actual_bound_bridge_must_supply_the_optional_role(): void { $this->app->instance(WorkflowTaskBridge::class, \Mockery::mock(WorkflowTaskBridge::class)); + $this->assertNull(RuntimeExternalPayloadReference::transportManifest()['upload']['completion_context']['prepared_schema']); $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/missing/local-activities/prepare', []) ->assertStatus(409)->assertJsonPath('reason', 'prepared_local_activity_not_supported') ->assertJsonPath('server_capabilities.prepared_local_activities', false) @@ -543,6 +550,176 @@ public function test_draining_preserves_original_cancellation_observation_and_jo ->assertOk()->assertJsonPath('acknowledged', true); } + public function test_draining_external_prefix_arguments_recovery_and_result_complete_without_extending_authority(): void + { + $task = $this->claim(); + $this->enableCompletionPayloads(); + $this->withHeaders($this->controlHeaders())->getJson('/api/cluster/info')->assertOk() + ->assertJsonPath('namespace.external_payload_storage.transport.upload.completion_context.prepared_schema', RuntimePayloadCompletionContext::PREPARED_SCHEMA); + $before = WorkflowTask::query()->findOrFail($task['task_id'])->lease_expires_at->toISOString(); + $this->configureStoragePressure('draining'); + $prefix = Serializer::serializeWithCodec('avro', str_repeat('prefix', 30)); + $prefixReference = $this->completionUpload($prefix, $this->completionContext($task, 'checkpoint', 'prefix-1')) + ->assertCreated()->json('reference'); + $this->localRequest($task, 'checkpoint', ['checkpoint_id' => 'prefix-1', 'start_sequence' => 1, 'commands' => [[ + 'type' => 'record_side_effect', 'result' => ['codec' => 'avro', 'external_payload' => $prefixReference], + ]]])->assertOk(); + $arguments = Serializer::serializeWithCodec('avro', [str_repeat('arguments', 30)]); + $argumentReference = $this->completionUpload($arguments, $this->completionContext($task, 'prepare', 2)) + ->assertCreated()->json('reference'); + $prepared = $this->prepare($task, ['sequence' => 2, 'descriptor' => $this->descriptor([ + 'arguments' => ['codec' => 'avro', 'external_payload' => $argumentReference], + ])])->assertOk()->json(); + $this->completionUpload($arguments, $this->completionContext($task, 'recover', 2))->assertCreated(); + $result = Serializer::serializeWithCodec('avro', str_repeat('result', 30)); + $context = $this->completionContext($task, 'outcome', $prepared['activity_attempt_id']); + $reference = $this->completionUpload($result, $context)->assertCreated()->json('reference'); + $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->lease_expires_at->toISOString()); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => [ + 'outcome' => 'completed', 'result' => ['codec' => 'avro', 'external_payload' => $reference], + ]])->assertOk()->assertJsonPath('recorded', true); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [['type' => 'complete_workflow']], + ])->assertOk()->assertJsonPath('run_status', 'completed'); + $this->assertSame($result, ExternalPayloads::resolveStoredPayload(ActivityExecution::query()->findOrFail($prepared['activity_execution_id'])->result, 'avro', 'default')); + $this->assertDatabaseCount('runtime_payload_completion_budgets', 1); + $budget = RuntimePayloadCompletionBudget::query()->sole(); + $this->assertCount(4, $budget->slots); + $this->assertCount(3, $budget->objects); + $rows = RuntimeExternalPayload::query()->orderBy('id')->get()->map->getAttributes()->all(); + $budgetBefore = $budget->getAttributes(); + $this->completionUpload($result, $context)->assertCreated()->assertJsonPath('reference', $reference); + $this->assertSame($rows, RuntimeExternalPayload::query()->orderBy('id')->get()->map->getAttributes()->all()); + $this->assertSame($budgetBefore, $budget->refresh()->getAttributes()); + $this->completionUpload('new bytes', $context)->assertStatus(409)->assertJsonPath('reason', 'external_payload_completion_lease_rejected'); + } + + public function test_prepared_operations_cannot_multiply_the_legacy_workflow_byte_allowance(): void + { + $task = $this->claim(); + $this->enableCompletionPayloads(); + config(['server.external_payload_transport.completion_max_bytes' => 8]); + $this->configureStoragePressure('draining'); + $legacy = ['schema' => RuntimePayloadCompletionContext::SCHEMA, 'kind' => 'workflow', + 'task_id' => $task['task_id'], 'attempt' => $task['workflow_task_attempt'], 'lease_owner' => $task['lease_owner'], + 'operation' => 'complete', 'slot' => ['commands', 0, 'result']]; + $this->completionUpload('aaaa', $legacy)->assertCreated(); + $this->completionUpload('bbbb', $this->completionContext($task, 'prepare', 1))->assertCreated(); + foreach ([$this->completionContext($task, 'prepare', 2), $this->completionContext($task, 'checkpoint', 'prefix-2')] as $context) { + $this->completionUpload('cccc', $context)->assertStatus(503)->assertJsonPath('request_admitted', false); + } + $this->assertDatabaseCount('runtime_external_payloads', 2); + $this->assertDatabaseCount('runtime_payload_completion_budgets', 1); + $budget = RuntimePayloadCompletionBudget::query()->sole(); + $this->assertCount(2, $budget->slots); + $this->assertSame(8, array_sum($budget->objects)); + } + + public function test_prepared_upload_refuses_unissued_claim_protocol_namespace_and_attempt(): void + { + $task = $this->claim(); + $this->enableCompletionPayloads(); + $prepared = $this->prepare($task)->assertOk()->json(); + $this->configureStoragePressure('draining'); + $context = $this->completionContext($task, 'prepare', 2); + foreach ([['attempt' => 99], ['lease_owner' => 'different']] as $changes) { + $this->completionUpload('bytes', array_replace($context, $changes))->assertStatus(409); + } + $this->completionUpload('bytes', $context, 'other')->assertStatus(409); + $this->completionUpload('bytes', $this->completionContext($task, 'outcome', 'unknown'))->assertStatus(409); + config(['server.worker_protocol.version' => '1.19']); + $this->completionUpload('bytes', $context)->assertStatus(409); + config(['server.worker_protocol.version' => '1.20']); + $this->observeStoragePressure('normal'); + $this->register('original'); + $this->observeStoragePressure('draining'); + foreach ([$context, $this->completionContext($task, 'outcome', $prepared['activity_attempt_id'])] as $value) { + $this->completionUpload('bytes', $value)->assertStatus(409); + } + $this->assertDatabaseCount('runtime_external_payloads', 0); + $this->assertDatabaseCount('runtime_payload_completion_budgets', 0); + } + + public function test_cooperative_only_worker_cannot_claim_prepared_upload_authority(): void + { + $task = $this->claim(false); + $this->enableCompletionPayloads(); + $this->configureStoragePressure('draining'); + $this->completionUpload('bytes', $this->completionContext($task, 'prepare', 1)) + ->assertStatus(409)->assertJsonPath('reason', 'external_payload_completion_lease_rejected'); + $this->assertDatabaseCount('runtime_external_payloads', 0); + $this->assertDatabaseCount('runtime_payload_completion_budgets', 0); + } + + public function test_accepted_cancellation_fences_old_callback_result_upload(): void + { + $task = $this->claim(); + $this->enableCompletionPayloads(); + $prepared = $this->prepare($task)->assertOk()->json(); + $this->withHeaders($this->controlHeaders())->postJson("/api/workflows/{$task['workflow_id']}/request-cancellation", []) + ->assertStatus(202); + $this->configureStoragePressure('draining'); + $this->completionUpload('bytes', $this->completionContext($task, 'outcome', $prepared['activity_attempt_id'])) + ->assertStatus(409)->assertJsonPath('reason', 'external_payload_completion_lease_rejected'); + $this->assertDatabaseCount('runtime_external_payloads', 0); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + } + + public function test_cleanup_upload_does_not_extend_original_root_budget_or_heartbeat(): void + { + [$task, $descriptor, $accepted] = $this->cleanupClaim(); + $this->enableCompletionPayloads(); + $prepared = $this->prepare($task, ['sequence' => 2, 'descriptor' => $descriptor])->assertOk()->json(); + $this->configureStoragePressure('draining'); + $attempt = ActivityAttempt::query()->findOrFail($prepared['activity_attempt_id']); + $before = $attempt->getAttributes(); + $context = $this->completionContext($task, 'outcome', $attempt->id); + $result = Serializer::serializeWithCodec('avro', str_repeat('cleanup', 30)); + $reference = $this->completionUpload($result, $context)->assertCreated()->json('reference'); + $this->assertSame($before, $attempt->refresh()->getAttributes()); + $this->localRequest($task, "{$attempt->id}/outcome", ['report' => [ + 'outcome' => 'completed', 'result' => ['codec' => 'avro', 'external_payload' => $reference], + ]])->assertOk()->assertJsonPath('recorded', true); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [['type' => 'complete_workflow']], + ])->assertOk()->assertJsonPath('run_status', 'cancelled'); + $this->assertSame($accepted['cleanup_deadline_at'], $prepared['cancellation_cleanup']['cleanup_deadline_at']); + } + + public function test_result_upload_is_fenced_by_fixed_activity_deadline_even_with_live_workflow_lease(): void + { + $task = $this->claim(); + $this->enableCompletionPayloads(); + $prepared = $this->prepare($task, ['descriptor' => $this->descriptor(['start_to_close_timeout' => 1])])->assertOk()->json(); + $this->configureStoragePressure('draining'); + $this->travel(2)->seconds(); + $this->observeStoragePressure('draining'); + $this->assertTrue(WorkflowTask::query()->findOrFail($task['task_id'])->lease_expires_at->isFuture()); + $this->completionUpload('bytes', $this->completionContext($task, 'outcome', $prepared['activity_attempt_id'])) + ->assertStatus(409)->assertJsonPath('reason', 'external_payload_completion_lease_rejected'); + $this->assertDatabaseCount('runtime_external_payloads', 0); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + } + + public function test_expired_root_budget_refuses_new_preparation_upload_with_live_workflow_lease(): void + { + $task = $this->claim(); + $this->enableCompletionPayloads(); + $this->withHeaders($this->controlHeaders())->postJson("/api/workflows/{$task['workflow_id']}/request-cancellation", [ + 'cleanup_timeout_seconds' => 1, + ])->assertStatus(202); + $this->configureStoragePressure('draining'); + $this->travel(2)->seconds(); + $this->observeStoragePressure('draining'); + $this->assertTrue(WorkflowTask::query()->findOrFail($task['task_id'])->lease_expires_at->isFuture()); + $this->completionUpload('bytes', $this->completionContext($task, 'prepare', 2)) + ->assertStatus(409)->assertJsonPath('reason', 'external_payload_completion_lease_rejected'); + $this->assertDatabaseCount('runtime_external_payloads', 0); + $this->assertDatabaseCount('runtime_payload_completion_budgets', 0); + } + public function test_storage_fence_refuses_all_local_mutations_without_changing_leases_or_history(): void { $task = $this->claim(); @@ -664,6 +841,42 @@ private function uploadPayload(string $payload, string $namespace): array ], $payload)->assertCreated()->json('reference'); } + private function enableCompletionPayloads(): void + { + $this->payloadDirectory = sys_get_temp_dir().'/dw-prepared-completion-'.getmypid(); + foreach (['default', 'other'] as $namespace) { + WorkflowNamespace::query()->updateOrCreate(['name' => $namespace], [ + 'description' => 'Fixture', 'retention_days' => 30, 'status' => 'active', + 'external_payload_storage' => ['driver' => 'local', 'enabled' => true, 'threshold_bytes' => 32, + 'config' => ['uri' => 'file://'.$this->payloadDirectory.'/'.$namespace]], + ]); + } + } + + private function completionContext(array $task, string $operation, int|string $identity): array + { + return ['schema' => RuntimePayloadCompletionContext::PREPARED_SCHEMA, 'kind' => 'workflow', + 'task_id' => $task['task_id'], 'attempt' => $task['workflow_task_attempt'], 'lease_owner' => $task['lease_owner'], + 'operation' => 'local_activity_'.$operation, + 'slot' => match ($operation) { + 'checkpoint' => ['commands', 0, 'result'], 'outcome' => ['report', 'result'], default => ['descriptor', 'arguments'], + }, + match ($operation) { + 'checkpoint' => 'checkpoint_id', 'outcome' => 'activity_attempt_id', default => 'sequence', + } => $identity]; + } + + private function completionUpload(string $bytes, array $context, string $namespace = 'default'): TestResponse + { + return $this->call('POST', '/api/external-payloads/v1', [], [], [], [ + 'CONTENT_TYPE' => 'application/octet-stream', 'HTTP_X_NAMESPACE' => $namespace, + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_CODEC' => 'avro', + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_SIZE' => (string) strlen($bytes), + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_SHA256' => hash('sha256', $bytes), + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_COMPLETION' => json_encode($context, JSON_THROW_ON_ERROR), + ], $bytes); + } + private function eventCount(array $task, HistoryEventType $type): int { return WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id'])->where('event_type', $type)->count(); diff --git a/tests/Unit/RuntimePayloadCompletionContextTest.php b/tests/Unit/RuntimePayloadCompletionContextTest.php index 868eec12..d079e3cd 100644 --- a/tests/Unit/RuntimePayloadCompletionContextTest.php +++ b/tests/Unit/RuntimePayloadCompletionContextTest.php @@ -79,6 +79,51 @@ public function test_oversized_header_is_rejected_before_json_decode(): void RuntimePayloadCompletionContext::parse(str_repeat(' ', 4097)); } + public function test_prepared_operations_and_legacy_completion_share_one_workflow_allowance(): void + { + $legacy = RuntimePayloadCompletionContext::parse(json_encode($this->value('workflow', 'complete', ['commands', 0, 'result']), JSON_THROW_ON_ERROR)); + $identities = []; + foreach ([ + ['local_activity_checkpoint', ['commands', 0, 'result'], ['checkpoint_id' => 'prefix-1']], + ['local_activity_checkpoint', ['commands', 0, 'result'], ['checkpoint_id' => 'prefix-2']], + ['local_activity_prepare', ['descriptor', 'arguments'], ['sequence' => 2]], + ['local_activity_prepare', ['descriptor', 'arguments'], ['sequence' => 3]], + ['local_activity_recover', ['descriptor', 'arguments'], ['sequence' => 2]], + ['local_activity_outcome', ['report', 'result'], ['activity_attempt_id' => 'attempt-1']], + ['local_activity_outcome', ['report', 'result'], ['activity_attempt_id' => 'attempt-2']], + ] as [$operation, $slot, $identity]) { + $value = array_replace($this->value('workflow', $operation, $slot), ['schema' => RuntimePayloadCompletionContext::PREPARED_SCHEMA], $identity); + $context = RuntimePayloadCompletionContext::parse(json_encode($value, JSON_THROW_ON_ERROR)); + $this->assertSame($value, $context->toArray()); + $this->assertSame($legacy->scope('default'), $context->scope('default')); + $this->assertNotSame($context->scope('default'), $context->scope('other')); + $this->assertSame($context->slotIdentity(), RuntimePayloadCompletionContext::parse(json_encode(array_reverse($value, true), JSON_THROW_ON_ERROR))->slotIdentity()); + $identities[] = $context->slotIdentity(); + } + $this->assertCount(count($identities), array_unique($identities)); + } + + #[DataProvider('invalidPreparedContexts')] + public function test_prepared_context_requires_exact_claim_identity_and_operation_slot(array $changes): void + { + $this->expectException(RuntimeExternalPayloadException::class); + $value = array_replace($this->value('workflow', 'local_activity_prepare', ['descriptor', 'arguments']), + ['schema' => RuntimePayloadCompletionContext::PREPARED_SCHEMA, 'sequence' => 2], $changes); + RuntimePayloadCompletionContext::parse(json_encode($value, JSON_THROW_ON_ERROR)); + } + + public static function invalidPreparedContexts(): array + { + return array_map(static fn (array $changes): array => [$changes], [ + ['sequence' => 0], ['sequence' => '2'], ['sequence' => null], ['sequence' => []], + ['kind' => 'activity'], ['kind' => 'query'], ['operation' => 'complete'], + ['slot' => ['report', 'result']], ['attempt' => '1'], ['attempt' => 0], + ['checkpoint_id' => 'extra'], ['activity_attempt_id' => 'extra'], + ['operation' => 'local_activity_checkpoint'], ['operation' => 'local_activity_outcome'], + ['schema' => RuntimePayloadCompletionContext::SCHEMA], + ]); + } + private function value(string $kind = 'activity', string $operation = 'complete', array $slot = ['result']): array { return ['schema' => RuntimePayloadCompletionContext::SCHEMA, 'kind' => $kind, From e288b8f0f0e46cae91d706e99bb1f4fb56030da2 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 07:28:02 +0000 Subject: [PATCH 22/57] Compare immutable payload reference objects independently of JSON key order --- tests/Feature/PreparedLocalActivityProtocolTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Feature/PreparedLocalActivityProtocolTest.php b/tests/Feature/PreparedLocalActivityProtocolTest.php index f4968d60..e8acdb6a 100644 --- a/tests/Feature/PreparedLocalActivityProtocolTest.php +++ b/tests/Feature/PreparedLocalActivityProtocolTest.php @@ -589,7 +589,7 @@ public function test_draining_external_prefix_arguments_recovery_and_result_comp $this->assertCount(3, $budget->objects); $rows = RuntimeExternalPayload::query()->orderBy('id')->get()->map->getAttributes()->all(); $budgetBefore = $budget->getAttributes(); - $this->completionUpload($result, $context)->assertCreated()->assertJsonPath('reference', $reference); + $this->assertSame($reference, RuntimeExternalPayloadReference::validate($this->completionUpload($result, $context)->assertCreated()->json('reference'))); $this->assertSame($rows, RuntimeExternalPayload::query()->orderBy('id')->get()->map->getAttributes()->all()); $this->assertSame($budgetBefore, $budget->refresh()->getAttributes()); $this->completionUpload('new bytes', $context)->assertStatus(409)->assertJsonPath('reason', 'external_payload_completion_lease_rejected'); From c1bda7f81323e9939ef433c7ebca66f62365cdfd Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 09:32:08 +0000 Subject: [PATCH 23/57] Expose atomic prepared local activity group admission --- .../Api/PreparedLocalActivityController.php | 7 +- app/Http/Controllers/Api/WorkerController.php | 127 ++++++++- .../Middleware/EnforceStorageAdmission.php | 1 + .../RuntimeExternalPayloadTransport.php | 10 + app/Support/PreparedLocalActivityPolicy.php | 29 ++ .../RuntimePayloadCompletionContext.php | 6 +- app/Support/RuntimePayloadCompletionLease.php | 4 + app/Support/WorkerProtocol.php | 1 + .../worker-protocol-api.openapi.yaml | 74 ++++- routes/api.php | 1 + .../PreparedLocalActivityProtocolTest.php | 257 +++++++++++++++++- .../RuntimePayloadCompletionContextTest.php | 3 + 12 files changed, 494 insertions(+), 26 deletions(-) diff --git a/app/Http/Controllers/Api/PreparedLocalActivityController.php b/app/Http/Controllers/Api/PreparedLocalActivityController.php index c249145f..b7a57682 100644 --- a/app/Http/Controllers/Api/PreparedLocalActivityController.php +++ b/app/Http/Controllers/Api/PreparedLocalActivityController.php @@ -285,11 +285,12 @@ public static function unavailable(Request $request): ?JsonResponse } /** @param list $reasons */ - public static function refused(string $workerId, array $reasons): JsonResponse + public static function refused(string $workerId, array $reasons, string $capability = PreparedLocalActivityPolicy::CAPABILITY): JsonResponse { return WorkerProtocol::json([ - 'reason' => 'prepared_local_activity_not_supported', - 'required_capability' => PreparedLocalActivityPolicy::CAPABILITY, + 'reason' => $capability === PreparedLocalActivityPolicy::GROUP_CAPABILITY + ? 'prepared_local_activity_groups_not_supported' : 'prepared_local_activity_not_supported', + 'required_capability' => $capability, 'minimum_protocol_version' => PreparedLocalActivityPolicy::MINIMUM_PROTOCOL_VERSION, 'worker_id' => $workerId, 'unavailable' => $reasons, diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index 0bbad670..35d2975d 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -55,6 +55,7 @@ use Illuminate\Support\Str; use Illuminate\Validation\ValidationException; use Workflow\V2\Contracts\HistoryProjectionRole; +use Workflow\V2\Contracts\PreparedLocalActivityGroupTaskBridge; use Workflow\V2\Contracts\PreparedLocalActivityTaskBridge; use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Enums\ActivityAttemptStatus; @@ -71,6 +72,7 @@ use Workflow\V2\Models\WorkflowServiceCall; use Workflow\V2\Models\WorkflowTask; use Workflow\V2\Support\StickyExecution; +use Workflow\V2\Support\PortableLocalActivityPreparation; use Workflow\V2\Support\WorkerProtocolVersion; use Workflow\V2\Support\WorkflowCommandNormalizer; use Workflow\V2\Support\WorkflowTaskOwnership; @@ -228,6 +230,17 @@ public function register(Request $request): JsonResponse ]); $workerCapabilities = $this->nonEmptyStringArray($validated['capabilities'] ?? []); + if (in_array(PreparedLocalActivityPolicy::GROUP_CAPABILITY, $workerCapabilities, true) + && (! in_array(PreparedLocalActivityPolicy::CAPABILITY, $workerCapabilities, true) + || ! CooperativeCancellationPolicy::supports($workerCapabilities, WorkerProtocol::requestVersion($request)))) { + return WorkerProtocol::json([ + 'registered' => false, + 'reason' => 'prepared_local_activity_group_capability_mismatch', + 'required_capabilities' => [CooperativeCancellationPolicy::CAPABILITY, PreparedLocalActivityPolicy::CAPABILITY], + 'minimum_protocol_version' => PreparedLocalActivityPolicy::MINIMUM_PROTOCOL_VERSION, + 'requested_version' => WorkerProtocol::requestVersion($request), + ], 409); + } if (in_array(PreparedLocalActivityPolicy::CAPABILITY, $workerCapabilities, true) && ! CooperativeCancellationPolicy::supports($workerCapabilities, WorkerProtocol::requestVersion($request))) { return WorkerProtocol::json([ @@ -1479,7 +1492,27 @@ public function checkpointLocalActivityPrefix(Request $request, string $taskId): return $this->mutateWorkflowTaskCommands($request, $taskId, true); } - private function mutateWorkflowTaskCommands(Request $request, string $taskId, bool $checkpoint = false): JsonResponse + public function checkpointLocalActivityGroup(Request $request, string $taskId): JsonResponse + { + if ($response = WorkerProtocol::rejectUnsupported($request)) { + return $response; + } + $reasons = PreparedLocalActivityPolicy::groupUnavailableReasons(WorkerProtocol::requestVersion($request)); + if ($reasons !== []) { + return PreparedLocalActivityController::refused((string) $request->input('lease_owner', ''), $reasons, PreparedLocalActivityPolicy::GROUP_CAPABILITY); + } + $task = NamespaceWorkflowScope::task((string) $request->attributes->get('namespace'), $taskId); + if ($task === null) { + return WorkerProtocol::json(['reason' => 'task_not_found'], 404); + } + if (PreparedLocalActivityPolicy::currentGroupClaim($task) === null) { + return PreparedLocalActivityController::refused((string) $request->input('lease_owner', ''), ['worker_claim_capability'], PreparedLocalActivityPolicy::GROUP_CAPABILITY); + } + + return $this->mutateWorkflowTaskCommands($request, $taskId, true, true); + } + + private function mutateWorkflowTaskCommands(Request $request, string $taskId, bool $checkpoint = false, bool $group = false): JsonResponse { if ($response = WorkerProtocol::rejectUnsupported($request)) { return $response; @@ -1639,6 +1672,12 @@ private function mutateWorkflowTaskCommands(Request $request, string $taskId, bo ]; $topLevelRules = []; + if ($group) { + $completionRules['commands'] = ['required', 'array', 'min:1', 'max:100']; + $completionRules['commands.*.cancellation_cleanup'] = ['nullable', 'array:request_id,delivery_history_event_id']; + $completionRules['commands.*.cancellation_cleanup.request_id'] = ['required_with:commands.*.cancellation_cleanup', 'string', 'max:255']; + $completionRules['commands.*.cancellation_cleanup.delivery_history_event_id'] = ['required_with:commands.*.cancellation_cleanup', 'string', 'max:255']; + } $commandRules = ['commands' => ['required', 'array', 'min:1']]; foreach ($completionRules as $field => $rules) { if (str_starts_with($field, 'commands.*.')) { @@ -1666,7 +1705,11 @@ private function mutateWorkflowTaskCommands(Request $request, string $taskId, bo } foreach ($chunkValidator->validated()['commands'] as $index => $command) { - $validatedCommands[$index] = $command; + // Prepared descriptors have a closed Native grammar. Keep + // original keys so transport validation cannot silently + // discard routing, fabricated reports or unknown fields. + $validatedCommands[$index] = $group && ($command['type'] ?? null) === 'prepare_local_activity' + ? $chunk[$index] : $command; } } } @@ -1678,7 +1721,8 @@ private function mutateWorkflowTaskCommands(Request $request, string $taskId, bo $validated['commands'] = $validatedCommands; $commands = $this->normalizeWorkflowTaskCommandIntegerFields($validated['commands']); - $commands = WorkflowCommandNormalizer::preflightParallelMetadata($commands); + $commands = $group ? $this->preflightPreparedLocalGroupMetadata($commands) + : WorkflowCommandNormalizer::preflightParallelMetadata($commands); $commands = $this->applyWorkerSessionRoutingDefaults($commands); $this->validateWorkflowTaskCommandScopes($commands); @@ -1823,6 +1867,7 @@ function () use ( $messageStreamWaits, $checkpoint, $checkpointInput, + $group, ): array|JsonResponse { return DB::transaction(function () use ( $bridge, @@ -1835,6 +1880,7 @@ function () use ( $messageStreamWaits, $checkpoint, $checkpointInput, + $group, ): array|JsonResponse { $quotaSnapshot = $this->durableStateQuota->snapshotForMutation( (string) $namespace, @@ -1906,16 +1952,17 @@ function () use ( (string) $namespace, $commands, ); - $commands = WorkflowCommandNormalizer::normalize( - $commands, - WorkerProtocol::requestVersion($request), - ); + $commands = $group ? $this->normalizePreparedLocalGroupCommands($commands, WorkerProtocol::requestVersion($request)) + : WorkflowCommandNormalizer::normalize($commands, WorkerProtocol::requestVersion($request)); if ($checkpoint) { if (PreparedLocalActivityPolicy::currentClaim($claimedTask) === null - || ! $bridge instanceof PreparedLocalActivityTaskBridge) { + || ! $bridge instanceof PreparedLocalActivityTaskBridge + || ($group && (PreparedLocalActivityPolicy::currentGroupClaim($claimedTask) === null + || ! $bridge instanceof PreparedLocalActivityGroupTaskBridge))) { throw new PreparedLocalActivityAdmissionRefused; } - $outcome = $bridge->checkpointLocalActivityPrefix( + $method = $group ? 'checkpointLocalActivityGroup' : 'checkpointLocalActivityPrefix'; + $outcome = $bridge->$method( $taskId, $validated['lease_owner'], (int) $validated['workflow_task_attempt'], $checkpointInput['checkpoint_id'], (int) $checkpointInput['start_sequence'], $commands, WorkerProtocol::requestVersion($request), @@ -1979,7 +2026,8 @@ function () use ( 'configured_limit' => $e->configuredLimit, ], 422); } catch (PreparedLocalActivityAdmissionRefused) { - return PreparedLocalActivityController::refused($validated['lease_owner'], ['worker_claim_capability']); + return PreparedLocalActivityController::refused($validated['lease_owner'], ['worker_claim_capability'], + $group ? PreparedLocalActivityPolicy::GROUP_CAPABILITY : PreparedLocalActivityPolicy::CAPABILITY); } catch (ExternalPayloadStorageUnavailable $exception) { return $this->externalPayloadFailure($taskId, (int) $validated['workflow_task_attempt'], $exception, 503); } catch (StreamFullException $exception) { @@ -2072,6 +2120,45 @@ function () use ( ], $this->workflowOutcomeStatus($outcome['reason'])); } + /** @param array> $commands */ + private function preflightPreparedLocalGroupMetadata(array $commands): array + { + $locals = []; + foreach ($commands as $index => $command) { + if (($command['type'] ?? null) === 'prepare_local_activity') { + $locals[] = $index; + $commands[$index]['type'] = 'schedule_activity'; + } + } + $commands = WorkflowCommandNormalizer::preflightParallelMetadata($commands); + foreach ($locals as $index) { + $commands[$index]['type'] = 'prepare_local_activity'; + } + + return $commands; + } + + /** @param array> $commands */ + private function normalizePreparedLocalGroupCommands(array $commands, string $protocolVersion): array + { + $locals = []; + foreach ($commands as $index => $command) { + if (($command['type'] ?? null) !== 'prepare_local_activity') { + continue; + } + $descriptor = PortableLocalActivityPreparation::normalizeDescriptor([...$command, 'type' => 'record_local_activity']); + $locals[$index] = [...$descriptor, 'type' => 'prepare_local_activity']; + unset($descriptor['execution_mode'], $descriptor['cancellation_cleanup']); + $commands[$index] = [...$descriptor, 'type' => 'schedule_activity']; + } + $commands = WorkflowCommandNormalizer::normalize($commands, $protocolVersion); + foreach ($locals as $index => $descriptor) { + $commands[$index] = $descriptor; + } + + return $commands; + } + /** @param array> $commands */ private function guardChildCancellationPoliciesAvailable( Request $request, @@ -2571,7 +2658,9 @@ private function persistTypedSearchAttributeHistoryIdentity( $eventQuery->where('payload->sequence', '>=', $outcome['start_sequence']) ->where('payload->sequence', '<', $outcome['next_sequence']); } else { - $prefixEnd = WorkflowTask::query()->find($taskId)?->payload['portable_local_checkpoint']['next_sequence'] ?? null; + $payload = WorkflowTask::query()->find($taskId)?->payload ?? []; + $prefixEnd = max($payload['portable_local_checkpoint']['next_sequence'] ?? 0, + $payload['portable_local_group_checkpoint']['next_sequence'] ?? 0); if (is_int($prefixEnd) && $prefixEnd > 0) { $eventQuery->where('payload->sequence', '>=', $prefixEnd); } @@ -2801,7 +2890,7 @@ private function validateWorkflowTaskCommandScopes(array $commands): void } if ($this->hasCommandValue($command, 'retry_policy') - && ! in_array($type, ['schedule_activity', 'record_local_activity', 'start_child_workflow'], true) + && ! in_array($type, ['schedule_activity', 'record_local_activity', 'prepare_local_activity', 'start_child_workflow'], true) ) { $errors["commands.{$index}.retry_policy"][] = 'retry_policy is only supported for schedule_activity and start_child_workflow commands.'; @@ -2809,7 +2898,7 @@ private function validateWorkflowTaskCommandScopes(array $commands): void foreach (['start_to_close_timeout', 'schedule_to_start_timeout', 'schedule_to_close_timeout', 'heartbeat_timeout'] as $field) { if ($this->hasCommandValue($command, $field) - && ! in_array($type, ['schedule_activity', 'record_local_activity'], true) + && ! in_array($type, ['schedule_activity', 'record_local_activity', 'prepare_local_activity'], true) ) { $errors["commands.{$index}.{$field}"][] = "{$field} is only supported for schedule_activity commands."; @@ -2864,7 +2953,7 @@ private function validateWorkflowTaskCommandScopes(array $commands): void $this->validateActivityTimeoutEnvelope($command, $index, $errors); } - if ($type === 'record_local_activity') { + if (in_array($type, ['record_local_activity', 'prepare_local_activity'], true)) { $this->validateActivityTimeoutEnvelope($command, $index, $errors); } @@ -3146,7 +3235,8 @@ private function resolveWorkflowTaskCommandPayloadReferences(array $commands, st } } - $payloadFields = WorkflowCommandNormalizer::payloadEnvelopeFields()[$commandType] ?? []; + $payloadFields = $commandType === 'prepare_local_activity' ? ['arguments'] + : (WorkflowCommandNormalizer::payloadEnvelopeFields()[$commandType] ?? []); foreach ($payloadFields as $field) { if (is_string($command[$field] ?? null)) { AvroPayloadEnvelopeResolver::assertSerializedPayload($command[$field], "commands.{$index}.{$field}"); @@ -3164,6 +3254,13 @@ private function resolveWorkflowTaskCommandPayloadReferences(array $commands, st retainExternal: in_array($commandType, ['complete_workflow', 'schedule_activity'], true), ); + if ($commandType === 'prepare_local_activity') { + $commands[$index][$field] = $resolved['payload']; + $commands[$index]['payload_codec'] ??= $resolved['codec']; + + continue; + } + if ($resolved['codec'] === null) { $commands[$index][$field] = $resolved['payload']; diff --git a/app/Http/Middleware/EnforceStorageAdmission.php b/app/Http/Middleware/EnforceStorageAdmission.php index b22682b9..16359388 100644 --- a/app/Http/Middleware/EnforceStorageAdmission.php +++ b/app/Http/Middleware/EnforceStorageAdmission.php @@ -18,6 +18,7 @@ final class EnforceStorageAdmission 'WorkerController@heartbeatWorkflowTask', 'WorkerController@completeWorkflowTask', 'WorkerController@checkpointLocalActivityPrefix', + 'WorkerController@checkpointLocalActivityGroup', 'PreparedLocalActivityController@prepare', 'PreparedLocalActivityController@recover', 'PreparedLocalActivityController@control', diff --git a/app/Http/Middleware/RuntimeExternalPayloadTransport.php b/app/Http/Middleware/RuntimeExternalPayloadTransport.php index c05e6d90..cb686f15 100644 --- a/app/Http/Middleware/RuntimeExternalPayloadTransport.php +++ b/app/Http/Middleware/RuntimeExternalPayloadTransport.php @@ -37,6 +37,13 @@ class RuntimeExternalPayloadTransport ['commands', '*', 'request_payload'], ['commands', '*', 'result'], ], + 'WorkerController@checkpointLocalActivityGroup' => [ + ['commands', '*', 'arguments'], + ['commands', '*', 'entries'], + ['commands', '*', 'exception', 'details'], + ['commands', '*', 'request_payload'], + ['commands', '*', 'result'], + ], 'WorkerController@completeWorkflowTask' => [ ['commands', '*', 'arguments'], ['commands', '*', 'entries'], @@ -58,6 +65,9 @@ class RuntimeExternalPayloadTransport 'WorkerController@checkpointLocalActivityPrefix' => [ ['commands', '*', 'workflow_stream', 'items', '*', 'payload_reference'], ], + 'WorkerController@checkpointLocalActivityGroup' => [ + ['commands', '*', 'workflow_stream', 'items', '*', 'payload_reference'], + ], 'WorkerController@completeWorkflowTask' => [ ['commands', '*', 'workflow_stream', 'items', '*', 'payload_reference'], ], diff --git a/app/Support/PreparedLocalActivityPolicy.php b/app/Support/PreparedLocalActivityPolicy.php index 53665cb1..b7c23ddf 100644 --- a/app/Support/PreparedLocalActivityPolicy.php +++ b/app/Support/PreparedLocalActivityPolicy.php @@ -2,6 +2,7 @@ namespace App\Support; +use Workflow\V2\Contracts\PreparedLocalActivityGroupTaskBridge; use Workflow\V2\Contracts\PreparedLocalActivityTaskBridge; use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Models\ActivityAttempt; @@ -12,6 +13,8 @@ final class PreparedLocalActivityPolicy { public const CAPABILITY = 'prepared_local_activities'; + public const GROUP_CAPABILITY = 'prepared_local_activity_groups'; + public const MINIMUM_PROTOCOL_VERSION = '1.20'; private const CLAIM_KEY = '_server_workflow_claim'; @@ -30,6 +33,24 @@ public static function serverSupported(): bool return CooperativeCancellationPolicy::serverSupported() && self::backendSupported(); } + public static function groupsSupported(): bool + { + return self::serverSupported() && interface_exists(PreparedLocalActivityGroupTaskBridge::class) + && app(WorkflowTaskBridge::class) instanceof PreparedLocalActivityGroupTaskBridge; + } + + /** @return list */ + public static function groupUnavailableReasons(?string $requestVersion): array + { + $reasons = self::unavailableReasons($requestVersion); + if (! interface_exists(PreparedLocalActivityGroupTaskBridge::class) + || ! app(WorkflowTaskBridge::class) instanceof PreparedLocalActivityGroupTaskBridge) { + $reasons[] = 'installed_runtime_prepared_local_activity_groups'; + } + + return $reasons; + } + /** @return list */ public static function unavailableReasons(?string $requestVersion): array { @@ -59,6 +80,14 @@ public static function currentClaim(WorkflowTask $task): ?array return $claim; } + /** @return array|null */ + public static function currentGroupClaim(WorkflowTask $task): ?array + { + $claim = self::currentClaim($task); + + return $claim !== null && in_array(self::GROUP_CAPABILITY, $claim['capabilities'] ?? [], true) ? $claim : null; + } + /** * Persist only after Native preparation succeeds, in the same transaction. * Native's attempt/execution/run/task locks must already be held. This diff --git a/app/Support/RuntimePayloadCompletionContext.php b/app/Support/RuntimePayloadCompletionContext.php index 6451df1a..b87efcb7 100644 --- a/app/Support/RuntimePayloadCompletionContext.php +++ b/app/Support/RuntimePayloadCompletionContext.php @@ -41,7 +41,7 @@ public static function parse(string $header): self if ($prepared) { $keys[] = match ($value['operation'] ?? null) { 'local_activity_outcome' => 'activity_attempt_id', - 'local_activity_checkpoint' => 'checkpoint_id', + 'local_activity_checkpoint', 'local_activity_group_checkpoint' => 'checkpoint_id', default => 'sequence', }; } @@ -56,7 +56,7 @@ public static function parse(string $header): self } if ($prepared && ($value['kind'] !== 'workflow' || ! match ($value['operation'] ?? null) { 'local_activity_outcome' => self::identifier($value['activity_attempt_id']), - 'local_activity_checkpoint' => self::identifier($value['checkpoint_id']), + 'local_activity_checkpoint', 'local_activity_group_checkpoint' => self::identifier($value['checkpoint_id']), default => is_int($value['sequence']) && $value['sequence'] > 0, })) { throw self::invalid(); @@ -67,7 +67,7 @@ public static function parse(string $header): self throw self::invalid(); } $validSlot = $prepared ? match ($value['operation']) { - 'local_activity_checkpoint' => self::workflowSlot($value['slot']), + 'local_activity_checkpoint', 'local_activity_group_checkpoint' => self::workflowSlot($value['slot']), 'local_activity_prepare', 'local_activity_recover' => $value['slot'] === ['descriptor', 'arguments'], 'local_activity_outcome' => $value['slot'] === ['report', 'result'], default => false, diff --git a/app/Support/RuntimePayloadCompletionLease.php b/app/Support/RuntimePayloadCompletionLease.php index 9a2ea86b..971a21fd 100644 --- a/app/Support/RuntimePayloadCompletionLease.php +++ b/app/Support/RuntimePayloadCompletionLease.php @@ -112,6 +112,10 @@ private function workflow(string $namespace, RuntimePayloadCompletionContext $co if (! PreparedLocalActivityPolicy::serverSupported() || $claim === null || ! WorkerPollFence::isCurrent($claim)) { throw self::rejected(); } + if ($context->operation === 'local_activity_group_checkpoint' + && (! PreparedLocalActivityPolicy::groupsSupported() || PreparedLocalActivityPolicy::currentGroupClaim($task) === null)) { + throw self::rejected(); + } $expires = [$guard['status']['lease_expires_at'] ?? null]; $run = WorkflowRun::query()->find($task->workflow_run_id); if (! $run instanceof WorkflowRun) { diff --git a/app/Support/WorkerProtocol.php b/app/Support/WorkerProtocol.php index f8501c61..08fad7f9 100644 --- a/app/Support/WorkerProtocol.php +++ b/app/Support/WorkerProtocol.php @@ -548,6 +548,7 @@ public static function serverCapabilities(): array 'activity_timeouts' => true, 'cooperative_cancellation' => CooperativeCancellationPolicy::serverSupported(), 'prepared_local_activities' => PreparedLocalActivityPolicy::serverSupported(), + 'prepared_local_activity_groups' => PreparedLocalActivityPolicy::groupsSupported(), 'activity_cancellation_acknowledgement' => CooperativeCancellationPolicy::serverSupported() && CooperativeCancellationPolicy::activityAcknowledgementBackendSupported(), 'local_activities' => [ diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 7a4d03c3..7a17edb2 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -73,6 +73,7 @@ x-durable-workflow-prepared-local-activity-contract: attempt: original_workflow_task_attempt operations: local_activity_checkpoint: { identity: checkpoint_id, slots: ordinary_workflow_command_payload_slots } + local_activity_group_checkpoint: { identity: checkpoint_id, slots: ordinary_workflow_command_payload_slots } local_activity_prepare: { identity: sequence, slots: [[descriptor, arguments]] } local_activity_recover: { identity: sequence, slots: [[descriptor, arguments]] } local_activity_outcome: { identity: activity_attempt_id, slots: [[report, result]] } @@ -81,7 +82,7 @@ x-durable-workflow-prepared-local-activity-contract: allowance: shared_with_legacy_v1_completion_across_every_operation_and_slot_of_one_workflow_claim duplicate: exact_stored_reference_may_be_read_after_claim_closes_without_object_or_retention_mutation lifetime: no_workflow_or_activity_lease_or_application_heartbeat_or_cleanup_deadline_renewal - storage_fenced: all_seven_local_mutation_endpoints_refused_before_handler + storage_fenced: every_local_mutation_endpoint_refused_before_handler published_protocol_1_19: unchanged x-durable-workflow-message-streams-contract: discovery_path: /cluster/info#/message_streams_contract @@ -495,6 +496,36 @@ paths: "422": { $ref: "#/components/responses/WorkerError" } "429": { $ref: "#/components/responses/WorkerError" } "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/local-activities/checkpoint-group: + post: + operationId: checkpointLocalActivityGroup + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: prepared_local_activity_groups + description: >- + Atomically commit every member of a complete all group, including prepare_local_activity descriptors without outcomes. Local executions are pending without attempts until separate preparation. Retain the original workflow claim, total deadlines and immutable group receipt. Partial and selection groups are refused. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/PreparedLocalGroupCheckpointRequest" } + responses: + "200": + description: The durable complete-group checkpoint result. Callback execution is not admitted. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - $ref: "#/components/schemas/PreparedLocalGroupCheckpointResult" + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "429": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } /worker/workflow-tasks/{taskId}/local-activities/recover: post: operationId: recoverLocalActivity @@ -1338,6 +1369,9 @@ components: prepared_local_activities: type: boolean description: Candidate preparation, prefix checkpoint, supervisor control, application heartbeat, outcome, recovery and joined-stop receipts. Requires protocol 1.20 and the actual bound PreparedLocalActivityTaskBridge role including heartbeat support. False at the default protocol 1.19. + prepared_local_activity_groups: + type: boolean + description: Candidate atomic complete-all-group admission. Requires prepared local activities, protocol 1.20 and the actual bound PreparedLocalActivityGroupTaskBridge. Existing custom prepared bridges do not acquire this role. supported_workflow_task_commands: type: array uniqueItems: true @@ -1736,6 +1770,16 @@ components: properties: request_id: { type: string, minLength: 1 } delivery_history_event_id: { type: string, minLength: 1 } + parallel_group_id: { type: string, minLength: 1 } + parallel_group_kind: { type: string, enum: [activity, mixed] } + parallel_group_mode: { type: string, const: all } + parallel_group_base_sequence: { type: integer, minimum: 1 } + parallel_group_size: { type: integer, minimum: 1 } + parallel_group_index: { type: integer, minimum: 0 } + parallel_group_path: + type: array + minItems: 1 + items: { type: object } retry_policy: type: [object, "null"] properties: @@ -1774,6 +1818,17 @@ components: maxItems: 100 description: Nonterminal prefix commands. Turn-closing waits, local outcome reports and selection cancellation are excluded. Nonempty completion-only message cursor, wait and sticky cache metadata are rejected. items: { $ref: "#/components/schemas/WorkflowCommand" } + PreparedLocalGroupCheckpointRequest: + allOf: + - $ref: "#/components/schemas/PreparedLocalCheckpointRequest" + - type: object + properties: + commands: + type: array + minItems: 1 + maxItems: 100 + description: Complete all groups with at least one prepare_local_activity. Local descriptors use the PreparedLocalDescriptor fields with type prepare_local_activity and an authored parallel group path. They contain no outcome, result, routing or attempt reports. Cleanup proof is validated before any sibling is created. + items: { $ref: "#/components/schemas/WorkflowCommand" } PreparedLocalControlRequest: allOf: - $ref: "#/components/schemas/PreparedLocalClaimRequest" @@ -1877,6 +1932,23 @@ components: fingerprint: { type: string, pattern: "^[0-9a-f]{64}$" } created_task_ids: { type: array, items: { type: string } } recorded_at: { type: string, format: date-time } + PreparedLocalGroupCheckpointResult: + allOf: + - $ref: "#/components/schemas/PreparedLocalCheckpointResult" + - type: object + required: [local_activities] + properties: + local_activities: + type: array + minItems: 1 + maxItems: 100 + items: + type: object + additionalProperties: false + required: [sequence, activity_execution_id] + properties: + sequence: { type: integer, minimum: 1 } + activity_execution_id: { type: string, minLength: 1 } PreparedLocalOutcomeResult: type: object required: [recorded, duplicate, reason, event_id, event_type, workflow_run_id, workflow_task_id, workflow_task_attempt, activity_execution_id, activity_attempt_id, worker_attempt_id, recorded_at, claim_released, created_task_ids] diff --git a/routes/api.php b/routes/api.php index e49fc28c..f029ef7d 100644 --- a/routes/api.php +++ b/routes/api.php @@ -237,6 +237,7 @@ Route::post('/workflow-tasks/{taskId}/deliver-cancellation', [CooperativeCancellationController::class, 'deliver']); Route::post('/workflow-tasks/{taskId}/local-activities/prepare', [PreparedLocalActivityController::class, 'prepare']); Route::post('/workflow-tasks/{taskId}/local-activities/checkpoint', [WorkerController::class, 'checkpointLocalActivityPrefix']); + Route::post('/workflow-tasks/{taskId}/local-activities/checkpoint-group', [WorkerController::class, 'checkpointLocalActivityGroup']); Route::post('/workflow-tasks/{taskId}/local-activities/recover', [PreparedLocalActivityController::class, 'recover']); Route::post('/workflow-tasks/{taskId}/local-activities/{attemptId}/control', [PreparedLocalActivityController::class, 'control']); Route::post('/workflow-tasks/{taskId}/local-activities/{attemptId}/heartbeat', [PreparedLocalActivityController::class, 'heartbeat']); diff --git a/tests/Feature/PreparedLocalActivityProtocolTest.php b/tests/Feature/PreparedLocalActivityProtocolTest.php index e8acdb6a..a0731149 100644 --- a/tests/Feature/PreparedLocalActivityProtocolTest.php +++ b/tests/Feature/PreparedLocalActivityProtocolTest.php @@ -23,6 +23,7 @@ use Workflow\Serializers\AvroBinaryValue; use Workflow\Serializers\Serializer; use Workflow\V2\Contracts\WorkflowTaskBridge; +use Workflow\V2\Contracts\PreparedLocalActivityTaskBridge; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Models\ActivityAttempt; @@ -30,6 +31,7 @@ use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowTask; use Workflow\V2\Support\ExternalPayloads; +use Workflow\V2\Support\ParallelChildGroup; final class PreparedLocalActivityProtocolTest extends TestCase { @@ -728,16 +730,16 @@ public function test_storage_fence_refuses_all_local_mutations_without_changing_ $before = $attempt->getAttributes(); $history = WorkflowHistoryEvent::query()->count(); $this->configureStoragePressure('fenced'); - foreach (['prepare', 'recover', 'checkpoint', "{$attemptId}/control", "{$attemptId}/heartbeat", "{$attemptId}/outcome", "{$attemptId}/acknowledge-cancellation"] as $path) { + foreach (['prepare', 'recover', 'checkpoint', 'checkpoint-group', "{$attemptId}/control", "{$attemptId}/heartbeat", "{$attemptId}/outcome", "{$attemptId}/acknowledge-cancellation"] as $path) { $this->localRequest($task, $path)->assertStatus(503)->assertJsonPath('request_admitted', false); } $this->assertSame($before, $attempt->refresh()->getAttributes()); $this->assertSame($history, WorkflowHistoryEvent::query()->count()); } - private function cleanupClaim(): array + private function cleanupClaim(bool $group = false): array { - $task = $this->claim(); + $task = $group ? $this->groupClaim() : $this->claim(); $accepted = $this->withHeaders($this->controlHeaders())->postJson("/api/workflows/{$task['workflow_id']}/request-cancellation", [ 'cleanup_timeout_seconds' => 30, ])->assertStatus(202)->json('cancellation_request'); @@ -753,6 +755,252 @@ private function cleanupClaim(): array ]]), $accepted]; } + public function test_group_capability_requires_preparation_cooperation_and_protocol(): void + { + foreach ([['1.19', [CooperativeCancellationPolicy::CAPABILITY, PreparedLocalActivityPolicy::CAPABILITY]], + ['1.20', [CooperativeCancellationPolicy::CAPABILITY]], ['1.20', [PreparedLocalActivityPolicy::CAPABILITY]]] as [$version, $capabilities]) { + $this->withHeaders($this->headers($version))->postJson('/api/worker/register', [ + 'worker_id' => 'incorrect-group', 'task_queue' => 'prepared', 'runtime' => 'php', + 'capabilities' => [...$capabilities, PreparedLocalActivityPolicy::GROUP_CAPABILITY], + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + ])->assertStatus(409)->assertJsonPath('reason', 'prepared_local_activity_group_capability_mismatch'); + } + $this->assertSame(0, WorkerRegistration::query()->count()); + } + + public function test_default_protocol_refuses_group_admission_with_capability_diagnostics(): void + { + config(['server.worker_protocol.version' => '1.19']); + $this->withHeaders($this->headers('1.19'))->postJson('/api/worker/workflow-tasks/missing/local-activities/checkpoint-group', []) + ->assertStatus(409)->assertJsonPath('reason', 'prepared_local_activity_groups_not_supported') + ->assertJsonPath('required_capability', PreparedLocalActivityPolicy::GROUP_CAPABILITY) + ->assertJsonPath('server_capabilities.prepared_local_activity_groups', false); + } + + public function test_existing_custom_prepared_bridge_does_not_acquire_group_capability(): void + { + $task = $this->groupClaim(); + $this->app->instance(WorkflowTaskBridge::class, \Mockery::mock(PreparedLocalActivityTaskBridge::class)); + $this->localRequest($task, 'checkpoint-group', $this->groupBody())->assertStatus(409) + ->assertJsonFragment(['installed_runtime_prepared_local_activity_groups']) + ->assertJsonPath('server_capabilities.prepared_local_activity_groups', false); + $this->assertDatabaseCount('activity_executions', 0); + } + + public function test_child_and_local_group_commit_before_callback_preparation_and_duplicate_is_immutable(): void + { + $task = $this->groupClaim(); + $hosting = WorkflowTask::query()->findOrFail($task['task_id']); + $expiry = $hosting->lease_expires_at->toISOString(); + $body = $this->groupBody(); + $this->localRequest($task, 'checkpoint-group', $body)->assertOk()->assertJsonPath('checkpointed', true) + ->assertJsonPath('next_sequence', 3)->assertJsonCount(1, 'local_activities'); + $this->assertDatabaseCount('workflow_runs', 2); + $this->assertDatabaseCount('activity_attempts', 0); + $execution = ActivityExecution::query()->sole(); + $this->assertSame('pending', $execution->status->value); + $this->assertSame(0, $execution->attempt_count); + $before = $execution->getAttributes(); + $events = WorkflowHistoryEvent::query()->count(); + $this->travel(2)->seconds(); + $this->localRequest($task, 'checkpoint-group', $body)->assertOk()->assertJsonPath('duplicate', true); + $this->assertSame($before, $execution->refresh()->getAttributes()); + $this->assertSame($events, WorkflowHistoryEvent::query()->count()); + $this->assertSame($expiry, $hosting->refresh()->lease_expires_at->toISOString()); + $prepared = $this->prepare($task, ['sequence' => 2, 'descriptor' => [...$body['commands'][1], 'type' => 'record_local_activity']]) + ->assertOk()->assertJsonPath('activity_execution_id', $execution->id)->json(); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => $this->success()]) + ->assertOk()->assertJsonPath('recorded', true)->assertJsonPath('claim_released', false); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + } + + public function test_group_claim_cannot_be_upgraded_after_issue_and_is_namespace_bound(): void + { + $task = $this->claim(); + WorkerRegistration::query()->where('worker_id', 'original')->sole()->forceFill([ + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY, PreparedLocalActivityPolicy::CAPABILITY, PreparedLocalActivityPolicy::GROUP_CAPABILITY], + ])->save(); + $this->localRequest($task, 'checkpoint-group', $this->groupBody())->assertStatus(409)->assertJsonFragment(['worker_claim_capability']); + WorkflowNamespace::query()->create(['name' => 'other', 'description' => 'Other', 'retention_days' => 30, 'status' => 'active']); + $this->withHeaders([...$this->headers(), 'X-Namespace' => 'other']) + ->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/checkpoint-group", $this->groupBody()) + ->assertNotFound()->assertJsonPath('reason', 'task_not_found'); + $this->assertDatabaseCount('workflow_runs', 1); + $this->assertDatabaseCount('activity_executions', 0); + } + + public function test_partial_group_and_terminal_report_do_not_create_any_sibling(): void + { + $task = $this->groupClaim(); + $body = $this->groupBody(); + $before = WorkflowHistoryEvent::query()->count(); + $this->localRequest($task, 'checkpoint-group', [...$body, 'commands' => [$body['commands'][1]]]) + ->assertStatus(409)->assertJsonPath('reason', 'invalid_local_activity_checkpoint_commands'); + $this->localRequest($task, 'checkpoint-group', [...$body, 'commands' => [...$body['commands'], ['type' => 'complete_workflow']]]) + ->assertStatus(409)->assertJsonPath('reason', 'invalid_local_activity_checkpoint_commands'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseCount('workflow_runs', 1); + $this->assertDatabaseCount('activity_executions', 0); + } + + public function test_group_quota_refusal_rolls_back_child_local_and_receipt(): void + { + $task = $this->groupClaim(); + $before = WorkflowHistoryEvent::query()->count(); + config(['server.namespace_durable_state.limits' => ['max_workflow_history_events' => $before + 1]]); + $this->localRequest($task, 'checkpoint-group', $this->groupBody())->assertStatus(429); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseCount('workflow_runs', 1); + $this->assertDatabaseCount('activity_executions', 0); + $this->assertArrayNotHasKey('portable_local_group_checkpoint', WorkflowTask::query()->findOrFail($task['task_id'])->payload); + } + + public function test_local_group_descriptor_cannot_hide_unknown_fields_routing_or_a_fabricated_outcome(): void + { + $task = $this->groupClaim(); + $before = WorkflowHistoryEvent::query()->count(); + foreach ([['unknown_field' => true], ['queue' => 'remote'], ['outcome' => 'completed']] as $changes) { + $body = $this->groupBody(); + $body['commands'][1] = [...$body['commands'][1], ...$changes]; + $this->localRequest($task, 'checkpoint-group', $body)->assertStatus(422); + } + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseCount('workflow_runs', 1); + $this->assertDatabaseCount('activity_executions', 0); + } + + public function test_draining_group_uploads_and_admission_share_one_original_claim_budget(): void + { + $task = $this->groupClaim(); + $this->enableCompletionPayloads(); + $this->configureStoragePressure('draining'); + $expiry = WorkflowTask::query()->findOrFail($task['task_id'])->lease_expires_at->toISOString(); + $body = $this->groupBody(); + $contexts = []; + foreach ([0, 1] as $index) { + $contexts[$index] = ['schema' => RuntimePayloadCompletionContext::PREPARED_SCHEMA, 'kind' => 'workflow', + 'task_id' => $task['task_id'], 'attempt' => $task['workflow_task_attempt'], 'lease_owner' => $task['lease_owner'], + 'operation' => 'local_activity_group_checkpoint', 'checkpoint_id' => $body['checkpoint_id'], + 'slot' => ['commands', $index, 'arguments']]; + $bytes = $body['commands'][$index]['arguments']; + $reference = $this->completionUpload($bytes, $contexts[$index])->assertCreated()->json('reference'); + $body['commands'][$index]['arguments'] = ['codec' => 'avro', 'external_payload' => $reference]; + } + $this->localRequest($task, 'checkpoint-group', $body)->assertOk()->assertJsonPath('checkpointed', true); + $this->assertSame($expiry, WorkflowTask::query()->findOrFail($task['task_id'])->lease_expires_at->toISOString()); + $this->assertDatabaseCount('runtime_payload_completion_budgets', 1); + $this->assertCount(2, RuntimePayloadCompletionBudget::query()->sole()->slots); + $prepared = $this->prepare($task, ['sequence' => 2, 'descriptor' => [...$body['commands'][1], 'type' => 'record_local_activity']]) + ->assertOk()->json(); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => $this->success()])->assertOk(); + } + + public function test_prepared_claim_without_group_capability_cannot_upload_group_payloads(): void + { + $task = $this->claim(); + $this->enableCompletionPayloads(); + $this->configureStoragePressure('draining'); + $context = ['schema' => RuntimePayloadCompletionContext::PREPARED_SCHEMA, 'kind' => 'workflow', + 'task_id' => $task['task_id'], 'attempt' => $task['workflow_task_attempt'], 'lease_owner' => $task['lease_owner'], + 'operation' => 'local_activity_group_checkpoint', 'checkpoint_id' => 'group-1', 'slot' => ['commands', 1, 'arguments']]; + $this->completionUpload('bytes', $context)->assertStatus(409)->assertJsonPath('reason', 'external_payload_completion_lease_rejected'); + $this->assertDatabaseCount('runtime_external_payloads', 0); + } + + public function test_replacement_can_prepare_the_committed_group_without_inventing_an_old_attempt(): void + { + $task = $this->groupClaim(); + $body = $this->groupBody(); + $body['commands'][0]['queue'] = 'children'; + $this->localRequest($task, 'checkpoint-group', $body)->assertOk(); + $execution = ActivityExecution::query()->sole(); + $deadline = $execution->schedule_to_close_deadline_at->toISOString(); + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $this->register('replacement', extraCapabilities: [PreparedLocalActivityPolicy::GROUP_CAPABILITY]); + $replacement = $this->poll('replacement')->assertOk()->json('task'); + $this->assertSame($task['task_id'], $replacement['task_id']); + $this->assertSame($task['workflow_task_attempt'] + 1, $replacement['workflow_task_attempt']); + $this->prepare($task, ['sequence' => 2, 'descriptor' => [...$body['commands'][1], 'type' => 'record_local_activity']])->assertStatus(409); + $prepared = $this->prepare($replacement, ['sequence' => 2, 'descriptor' => [...$body['commands'][1], 'type' => 'record_local_activity']]) + ->assertOk()->assertJsonPath('activity_execution_id', $execution->id)->assertJsonPath('attempt_number', 1) + ->assertJsonPath('workflow_task_attempt', $replacement['workflow_task_attempt'])->json(); + $this->assertSame($deadline, $prepared['schedule_to_close_deadline_at']); + $this->assertDatabaseCount('activity_attempts', 1); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityCancellationAcknowledged)); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityRetryScheduled)); + } + + public function test_cleanup_group_members_preserve_the_one_original_root_budget(): void + { + [$task, $descriptor, $accepted] = $this->cleanupClaim(group: true); + $commands = []; + foreach ([0, 1] as $index) { + $commands[] = [...$descriptor, 'type' => 'prepare_local_activity', + ...ParallelChildGroup::itemMetadata(2, 2, $index, 'activity')]; + } + $this->localRequest($task, 'checkpoint-group', ['checkpoint_id' => 'cleanup-group', 'start_sequence' => 2, 'commands' => $commands]) + ->assertOk()->assertJsonCount(2, 'local_activities'); + $this->assertDatabaseCount('activity_attempts', 0); + $this->travel(3)->seconds(); + foreach ($commands as $index => $command) { + $prepared = $this->prepare($task, ['sequence' => $index + 2, 'worker_attempt_id' => 'cleanup-'.$index, + 'descriptor' => [...$command, 'type' => 'record_local_activity']])->assertOk() + ->assertJsonPath('cancellation_cleanup.root_request_id', $accepted['request_id']) + ->assertJsonPath('cancellation_cleanup.cleanup_deadline_at', $accepted['cleanup_deadline_at'])->json(); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => $this->success()])->assertOk(); + } + $this->assertSame(1, $this->eventCount($task, HistoryEventType::CooperativeCancellationDelivered)); + $this->assertSame(2, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + } + + public function test_typed_search_history_identity_survives_a_group_checkpoint_then_completion(): void + { + if (! PreparedLocalActivityPolicy::groupsSupported()) { + $this->markTestSkipped('Prepared-group source binding required.'); + } + SearchAttributeDefinition::query()->create(['namespace' => 'default', 'name' => 'Tier', 'type' => 'keyword']); + $task = $this->claim(extraCapabilities: [PreparedLocalActivityPolicy::GROUP_CAPABILITY, 'typed_search_attributes']); + $body = $this->groupBody(); + foreach ([0, 1] as $index) { + $body['commands'][$index] = [...$body['commands'][$index], ...ParallelChildGroup::itemMetadata(2, 2, $index, 'mixed')]; + } + array_unshift($body['commands'], ['type' => 'upsert_search_attributes', 'attributes' => ['Tier' => 'basic'], 'attribute_types' => ['Tier' => 'keyword']]); + $this->localRequest($task, 'checkpoint-group', $body)->assertOk()->assertJsonPath('next_sequence', 4); + $prepared = $this->prepare($task, ['sequence' => 3, 'descriptor' => [...$body['commands'][2], 'type' => 'record_local_activity']])->assertOk()->json(); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => $this->success()])->assertOk(); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [['type' => 'upsert_search_attributes', 'attributes' => ['Tier' => 'pro'], 'attribute_types' => ['Tier' => 'keyword']], + ['type' => 'complete_workflow']], + ])->assertOk()->assertJsonPath('run_status', 'completed'); + $events = WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id']) + ->where('event_type', HistoryEventType::SearchAttributesUpserted)->orderBy('sequence')->get(); + $this->assertCount(2, $events); + $this->assertSame([1, 4], $events->map(fn ($event) => $event->payload['sequence'])->all()); + foreach ($events as $event) { + $this->assertSame(['Tier' => 'keyword'], $event->payload['attribute_types']); + } + } + + private function groupClaim(): array + { + if (! PreparedLocalActivityPolicy::groupsSupported()) { + $this->markTestSkipped('Prepared-group source binding required.'); + } + + return $this->claim(extraCapabilities: [PreparedLocalActivityPolicy::GROUP_CAPABILITY]); + } + + private function groupBody(): array + { + return ['checkpoint_id' => 'group-1', 'start_sequence' => 1, 'commands' => [[ + 'type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', + 'arguments' => Serializer::serializeWithCodec('avro', ['child']), 'payload_codec' => 'avro', + ...ParallelChildGroup::itemMetadata(1, 2, 0, 'mixed'), + ], [...$this->descriptor(['schedule_to_close_timeout' => 30]), 'type' => 'prepare_local_activity', + ...ParallelChildGroup::itemMetadata(1, 2, 1, 'mixed')]]]; + } + private function claim(bool $prepared = true, array $extraCapabilities = []): array { if (! PreparedLocalActivityPolicy::backendSupported()) { @@ -821,7 +1069,8 @@ private function localRequest(array $task, string $path, array $body = []): Test $operation = substr($path, strrpos('/'.$path, '/')); $schema = match ($operation) { 'prepare' => 'PreparedLocalPreparationResult', 'recover' => 'PreparedLocalRecoveryResult', - 'checkpoint' => 'PreparedLocalCheckpointResult', 'control', 'heartbeat' => 'PreparedLocalControlResult', + 'checkpoint' => 'PreparedLocalCheckpointResult', 'checkpoint-group' => 'PreparedLocalGroupCheckpointResult', + 'control', 'heartbeat' => 'PreparedLocalControlResult', 'outcome' => 'PreparedLocalOutcomeResult', 'acknowledge-cancellation' => 'PreparedLocalStopResult', }; OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) diff --git a/tests/Unit/RuntimePayloadCompletionContextTest.php b/tests/Unit/RuntimePayloadCompletionContextTest.php index d079e3cd..8d50e380 100644 --- a/tests/Unit/RuntimePayloadCompletionContextTest.php +++ b/tests/Unit/RuntimePayloadCompletionContextTest.php @@ -86,6 +86,9 @@ public function test_prepared_operations_and_legacy_completion_share_one_workflo foreach ([ ['local_activity_checkpoint', ['commands', 0, 'result'], ['checkpoint_id' => 'prefix-1']], ['local_activity_checkpoint', ['commands', 0, 'result'], ['checkpoint_id' => 'prefix-2']], + ['local_activity_group_checkpoint', ['commands', 0, 'arguments'], ['checkpoint_id' => 'group-1']], + ['local_activity_group_checkpoint', ['commands', 1, 'arguments'], ['checkpoint_id' => 'group-1']], + ['local_activity_group_checkpoint', ['commands', 0, 'arguments'], ['checkpoint_id' => 'group-2']], ['local_activity_prepare', ['descriptor', 'arguments'], ['sequence' => 2]], ['local_activity_prepare', ['descriptor', 'arguments'], ['sequence' => 3]], ['local_activity_recover', ['descriptor', 'arguments'], ['sequence' => 2]], From 0f02ecf96c9c0144bac946d1e90a8513794b56a8 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 14:54:01 +0000 Subject: [PATCH 24/57] Hold detached activity history and payloads until canonical closure --- .github/workflows/phpunit-feature.yml | 2 +- app/Support/DetachedActivityRetentionHold.php | 39 ++++++++++ .../ExternalPayloadRetentionCleanup.php | 4 + app/Support/HistoryRetentionEnforcer.php | 5 ++ tests/Feature/HistoryRetentionTest.php | 74 +++++++++++++++++++ 5 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 app/Support/DetachedActivityRetentionHold.php diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index 51528ece..5a33d516 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -340,7 +340,7 @@ jobs: composer install --no-interaction --no-progress --prefer-dist mv vendor/durable-workflow/workflow /tmp/locked-workflow-package cp -a prepared-workflow-source vendor/durable-workflow/workflow - php vendor/bin/phpunit tests/Feature/PreparedLocalActivityProtocolTest.php tests/Unit/WorkerPollFenceTest.php --fail-on-skipped --log-junit "/evidence/$DW_PREPARED_DATABASE.xml" --colors=never + php vendor/bin/phpunit tests/Feature/PreparedLocalActivityProtocolTest.php tests/Feature/HistoryRetentionTest.php tests/Unit/WorkerPollFenceTest.php --fail-on-skipped --log-junit "/evidence/$DW_PREPARED_DATABASE.xml" --colors=never if [ "$DW_PREPARED_DATABASE" = sqlite ]; then php vendor/bin/phpunit tests/Feature/CooperativeCancellationProtocolTest.php tests/Feature/WorkflowWorkerProtocolTest.php tests/Feature/ActivityWorkerProtocolTest.php tests/Feature/SearchAttributeValueValidationTest.php tests/Feature/NamespaceDurableStateQuotaTest.php tests/Feature/RuntimeExternalPayloadTransportTest.php tests/Feature/RuntimePayloadCompletionUploadsTest.php tests/Unit/RuntimePayloadCompletionContextTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php --log-junit /evidence/affected-regression.xml --colors=never fi diff --git a/app/Support/DetachedActivityRetentionHold.php b/app/Support/DetachedActivityRetentionHold.php new file mode 100644 index 00000000..c86e92ce --- /dev/null +++ b/app/Support/DetachedActivityRetentionHold.php @@ -0,0 +1,39 @@ + $runIds */ + public static function forRuns(string $namespace, array $runIds): ?string + { + $runs = WorkflowRun::query()->where('namespace', $namespace)->whereIn('id', $runIds)->get(); + foreach ($runs as $run) { + if (method_exists(WorkflowRunRetentionCleanup::class, 'retentionHoldReason')) { + $reason = WorkflowRunRetentionCleanup::retentionHoldReason($run); + if ($reason !== null) { + return $reason; + } + + continue; + } + // A downgraded backend must preserve newer detached work and its + // payloads even when it cannot execute that policy itself. + $schedules = WorkflowHistoryEvent::query()->where('workflow_run_id', $run->id) + ->where('event_type', HistoryEventType::ActivityScheduled->value) + ->where('payload->activity->cancellation_policy', 'abandon')->get(); + if ($schedules->isNotEmpty()) { + // Restore a compatible backend to evaluate its canonical + // outcome contract. Do not parse newer history with old enums. + return 'activity_policy_backend_unsupported'; + } + } + + return null; + } +} diff --git a/app/Support/ExternalPayloadRetentionCleanup.php b/app/Support/ExternalPayloadRetentionCleanup.php index f6f3ca7e..426673bd 100644 --- a/app/Support/ExternalPayloadRetentionCleanup.php +++ b/app/Support/ExternalPayloadRetentionCleanup.php @@ -59,6 +59,10 @@ public function deleteForRuns( array_map(static fn (mixed $itemId): int => (int) $itemId, $releasedInboundStreamItemIds), static fn (int $itemId): bool => $itemId > 0, ))); + $reason = DetachedActivityRetentionHold::forRuns($namespace, $runIds); + if ($reason !== null) { + return ['found' => 0, 'deleted' => 0, 'blocked' => true, 'reason' => $reason]; + } $references = $this->referencesForRuns($namespace, $runIds); $this->collectPayloadColumn( WorkflowInboundStreamItem::query()->whereIn('id', $releasedInboundStreamItemIds), diff --git a/app/Support/HistoryRetentionEnforcer.php b/app/Support/HistoryRetentionEnforcer.php index c2497cc1..50ca7bf2 100644 --- a/app/Support/HistoryRetentionEnforcer.php +++ b/app/Support/HistoryRetentionEnforcer.php @@ -200,6 +200,11 @@ public static function pruneRun(string $namespace, string $runId): array return self::skippedRetentionResult('run_archived'); } + $reason = DetachedActivityRetentionHold::forRuns($namespace, [$runId]); + if ($reason !== null) { + return self::skippedRetentionResult($reason); + } + $messageStreams = app(MessageStreamRetentionCleanup::class); $streamPlan = $messageStreams->planForRun( $namespace, diff --git a/tests/Feature/HistoryRetentionTest.php b/tests/Feature/HistoryRetentionTest.php index b30616b5..4401ac39 100644 --- a/tests/Feature/HistoryRetentionTest.php +++ b/tests/Feature/HistoryRetentionTest.php @@ -8,6 +8,7 @@ use App\Models\WorkflowInboundStream; use App\Models\WorkflowInboundStreamItem; use App\Models\WorkflowNamespace; +use App\Support\ExternalPayloadRetentionCleanup; use App\Support\MessageStreamService; use App\Support\NamespaceWorkflowScope; use App\Support\RuntimeExternalPayloadRegistry; @@ -35,6 +36,7 @@ use Workflow\V2\Models\WorkflowSearchAttribute; use Workflow\V2\Models\WorkflowTask; use Workflow\V2\Support\ExternalPayloads; +use Workflow\V2\Support\WorkflowRunRetentionCleanup; use Workflow\V2\WorkflowStub; class HistoryRetentionTest extends TestCase @@ -42,6 +44,78 @@ class HistoryRetentionTest extends TestCase use RefreshDatabase; use ServerTestHelpers; + public function test_retention_holds_detached_activity_history_and_payload_bytes_until_it_closes(): void + { + Queue::fake(); + config(['filesystems.disks.detached-retention-payloads' => [ + 'driver' => 'local', 'root' => storage_path('framework/testing/detached-retention-payloads'), + ]]); + Storage::fake('detached-retention-payloads'); + $this->createNamespace('default'); + WorkflowNamespace::where('name', 'default')->update(['external_payload_storage' => [ + 'driver' => 's3', 'enabled' => true, 'config' => [ + 'disk' => 'detached-retention-payloads', 'bucket' => 'dw-payloads', 'prefix' => 'retention/', + ], + ]]); + $expectedReason = method_exists(WorkflowRunRetentionCleanup::class, 'retentionHoldReason') + ? 'detached_activity_still_open' : 'activity_policy_backend_unsupported'; + foreach ([ActivityStatus::Pending, ActivityStatus::Running] as $status) { + $runId = $this->createExpiredClosedRun('default', 'wf-detached-retention-'.$status->value); + $run = WorkflowRun::findOrFail($runId); + $payload = "independent detached bytes\0".$status->value; + $key = 'retention/avro/'.substr(hash('sha256', $payload), 0, 2).'/'.hash('sha256', $payload); + Storage::disk('detached-retention-payloads')->put($key, $payload); + $execution = ActivityExecution::query()->create([ + 'workflow_run_id' => $runId, 'sequence' => 999, + 'activity_class' => 'detached-fixture-activity', 'activity_type' => 'detached-fixture-activity', + 'status' => $status, 'arguments' => $this->storedExternalPayloadReference('s3://dw-payloads/'.$key, $payload), + 'activity_options' => ['cancellation_policy' => 'abandon'], + 'schedule_to_close_deadline_at' => now()->addMinutes(3), + ]); + WorkflowHistoryEvent::query()->create([ + 'workflow_run_id' => $runId, 'sequence' => ($run->historyEvents()->max('sequence') ?? 0) + 1, + 'event_type' => HistoryEventType::ActivityScheduled, + 'payload' => ['sequence' => 999, 'activity_execution_id' => $execution->id, + 'activity' => ['id' => $execution->id, 'cancellation_policy' => 'abandon']], + 'recorded_at' => now(), + ]); + $historyCount = $run->historyEvents()->count(); + $this->withHeaders($this->apiHeaders())->postJson('/api/system/retention/pass', ['run_ids' => [$runId]]) + ->assertOk()->assertJsonPath('pruned', 0)->assertJsonPath('skipped', 1) + ->assertJsonPath('results.0.reason', $expectedReason); + $direct = app(ExternalPayloadRetentionCleanup::class)->deleteForRun('default', $runId); + $this->assertTrue($direct['blocked']); + $this->assertSame($expectedReason, $direct['reason']); + $this->assertSame(0, $direct['deleted']); + $this->assertSame($payload, Storage::disk('detached-retention-payloads')->get($key)); + $this->assertSame($historyCount, $run->historyEvents()->count()); + $this->assertNull($run->refresh()->details_pruned_at); + $execution->forceFill(['status' => ActivityStatus::Completed, 'closed_at' => now()])->save(); + $this->withHeaders($this->apiHeaders())->postJson('/api/system/retention/pass', ['run_ids' => [$runId]]) + ->assertOk()->assertJsonPath('pruned', 0)->assertJsonPath('results.0.reason', $expectedReason); + $this->assertSame($payload, Storage::disk('detached-retention-payloads')->get($key)); + WorkflowHistoryEvent::query()->create([ + 'workflow_run_id' => $runId, 'sequence' => $run->historyEvents()->max('sequence') + 1, + 'event_type' => HistoryEventType::ActivityCancelled, + 'payload' => ['sequence' => 999, 'activity_execution_id' => $execution->id, 'activity_attempt_id' => null], + 'recorded_at' => now(), + ]); + $execution->forceFill(['status' => ActivityStatus::Cancelled])->save(); + if (! method_exists(WorkflowRunRetentionCleanup::class, 'retentionHoldReason')) { + $this->withHeaders($this->apiHeaders())->postJson('/api/system/retention/pass', ['run_ids' => [$runId]]) + ->assertOk()->assertJsonPath('pruned', 0)->assertJsonPath('results.0.reason', $expectedReason); + $this->assertSame($payload, Storage::disk('detached-retention-payloads')->get($key)); + + continue; + } + $this->withHeaders($this->apiHeaders())->postJson('/api/system/retention/pass', ['run_ids' => [$runId]]) + ->assertOk()->assertJsonPath('pruned', 1)->assertJsonPath('results.0.external_payloads_deleted', 1); + Storage::disk('detached-retention-payloads')->assertMissing($key); + $this->assertSame(0, ActivityExecution::query()->whereKey($execution->id)->count()); + $this->assertNull(WorkflowRunSummary::find($runId)); + } + } + // ── Retention Status Endpoint ────────────────────────────────── public function test_retention_status_returns_empty_when_no_expired_runs(): void From 2fc521b5f132c5a6d0909ec6239f2fbbc435bae6 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 15:15:21 +0000 Subject: [PATCH 25/57] Accept released cancellation claims while activity stop remains pending --- .../Api/CooperativeCancellationController.php | 3 ++- .../worker-protocol-api.openapi.yaml | 5 +++-- .../CooperativeCancellationProtocolTest.php | 18 +++++++++++++----- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/app/Http/Controllers/Api/CooperativeCancellationController.php b/app/Http/Controllers/Api/CooperativeCancellationController.php index 9a9aec91..4e85cb8c 100644 --- a/app/Http/Controllers/Api/CooperativeCancellationController.php +++ b/app/Http/Controllers/Api/CooperativeCancellationController.php @@ -212,7 +212,8 @@ public function deliver(Request $request, string $taskId): JsonResponse })); return WorkerProtocol::json($result, (($result['delivered'] ?? false) - || ($result['reason'] ?? null) === 'cancellation_waiting_for_child') ? 200 + || (($result['claim_released'] ?? null) === true && in_array($result['reason'] ?? null, + ['cancellation_waiting_for_child', 'cancellation_waiting_for_activity'], true))) ? 200 : (($result['reason'] ?? null) === 'task_not_found' ? 404 : 409)); } } diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 7a17edb2..348a8dd2 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "30" + version: "31" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -33,6 +33,7 @@ x-durable-workflow-cooperative-cancellation-contract: canonical_history_event: CooperativeCancellationDelivered delivery_identity: [request_id, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span] pending_child_delivery: { status: 200, delivered: false, reason: cancellation_waiting_for_child, claim_released: true, canonical_delivery_recorded: false, resume_source: child_terminal_history } + pending_activity_delivery: { status: 200, delivered: false, reason: cancellation_waiting_for_activity, claim_released: true, canonical_delivery_recorded: false, resume_source: canonical_callback_outcome_or_original_stop_receipt_or_fenced_never_started_operation } child_operation_policies: [try_cancel, wait_cancellation_completed, abandon] cooperative_parent_close_policy: request_cancellation legacy_terminal_parent_close_policy: request_cancel @@ -1738,7 +1739,7 @@ components: sequence_span: { type: "null" } operation_sequence: { type: "null" } operation_sequence_span: { type: "null" } - reason: { type: string, const: cancellation_waiting_for_child } + reason: { type: string, enum: [cancellation_waiting_for_child, cancellation_waiting_for_activity] } PreparedLocalClaimRequest: type: object required: [lease_owner, workflow_task_attempt] diff --git a/tests/Feature/CooperativeCancellationProtocolTest.php b/tests/Feature/CooperativeCancellationProtocolTest.php index 7704e31b..b38f5974 100644 --- a/tests/Feature/CooperativeCancellationProtocolTest.php +++ b/tests/Feature/CooperativeCancellationProtocolTest.php @@ -125,7 +125,8 @@ public function test_delivery_retry_records_one_marker_and_keeps_cleanup_authori ->assertJsonPath('reason', 'cancellation_delivery_mismatch'); } - public function test_child_acknowledgement_pending_is_a_valid_reply_with_explicit_claim_release(): void + #[DataProvider('pendingCancellationKinds')] + public function test_acknowledgement_pending_is_a_valid_reply_with_explicit_claim_release(string $kind, string $reason): void { [$workflowId, $runId] = $this->start(); $this->register('new', true); @@ -134,21 +135,28 @@ public function test_child_acknowledgement_pending_is_a_valid_reply_with_explici $pending = ['delivered' => false, 'task_id' => $task['task_id'], 'workflow_run_id' => $runId, 'request_id' => null, 'sequence' => null, 'call_kind' => null, 'sequence_span' => null, 'operation_sequence' => null, 'operation_sequence_span' => null, - 'reason' => 'cancellation_waiting_for_child', 'claim_released' => true]; + 'reason' => $reason, 'claim_released' => true]; $bridge = \Mockery::mock(CooperativeWorkflowTaskBridge::class); $bridge->shouldReceive('deliverCancellation')->once()->with( - $task['task_id'], $requestId, 1, 'child', 1, null, 1, + $task['task_id'], $requestId, 1, $kind, 1, null, 1, )->andReturn($pending); $this->app->instance(WorkflowTaskBridge::class, $bridge); - $response = $this->deliver($task, $requestId, ['call_kind' => 'child'])->assertOk() - ->assertJsonPath('delivered', false)->assertJsonPath('reason', 'cancellation_waiting_for_child') + $response = $this->deliver($task, $requestId, ['call_kind' => $kind])->assertOk() + ->assertJsonPath('delivered', false)->assertJsonPath('reason', $reason) ->assertJsonPath('claim_released', true); OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) ->assertReferenceMatches('#/components/schemas/CooperativeCancellationDeliveryResponse/allOf/1', json_decode($response->getContent(), flags: JSON_THROW_ON_ERROR)); $this->assertSame(0, $this->eventCount($runId, HistoryEventType::CooperativeCancellationDelivered)); } + public static function pendingCancellationKinds(): array + { + return ['child' => ['child', 'cancellation_waiting_for_child'], + 'remote activity' => ['activity', 'cancellation_waiting_for_activity'], + 'local activity' => ['local_activity', 'cancellation_waiting_for_activity']]; + } + public static function cancellationRequestTiming(): array { return ['before claim' => [true], 'after claim' => [false]]; From 8940fb8a866e24d7874947e850f7fc864fbdfd3c Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 16:10:26 +0000 Subject: [PATCH 26/57] Admit remote activity cancellation policies on capable source claims --- app/Http/Controllers/Api/WorkerController.php | 72 ++++++++++++++++- app/Support/CooperativeCancellationPolicy.php | 30 ++++++++ .../worker-protocol-api.openapi.yaml | 20 ++++- .../CooperativeCancellationProtocolTest.php | 77 +++++++++++++++++++ 4 files changed, 194 insertions(+), 5 deletions(-) diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index 35d2975d..c9ca937e 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -71,8 +71,8 @@ use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowServiceCall; use Workflow\V2\Models\WorkflowTask; -use Workflow\V2\Support\StickyExecution; use Workflow\V2\Support\PortableLocalActivityPreparation; +use Workflow\V2\Support\StickyExecution; use Workflow\V2\Support\WorkerProtocolVersion; use Workflow\V2\Support\WorkflowCommandNormalizer; use Workflow\V2\Support\WorkflowTaskOwnership; @@ -1746,6 +1746,16 @@ private function mutateWorkflowTaskCommands(Request $request, string $taskId, bo return $response; } + if ($response = $this->guardRemoteActivityCancellationPoliciesAvailable( + $request, + (string) $namespace, + $taskId, + (int) $validated['workflow_task_attempt'], + $commands, + )) { + return $response; + } + if ($response = $this->guardChildCancellationPoliciesAvailable( $request, (string) $namespace, @@ -1921,6 +1931,17 @@ function () use ( return $response; } + if ($response = $this->guardRemoteActivityCancellationPoliciesAvailable( + $request, + (string) $namespace, + $taskId, + (int) $validated['workflow_task_attempt'], + $commands, + $claimedTask, + )) { + return $response; + } + if ($response = $this->guardChildCancellationPoliciesAvailable( $request, (string) $namespace, @@ -2159,6 +2180,50 @@ private function normalizePreparedLocalGroupCommands(array $commands, string $pr return $commands; } + /** @param array> $commands */ + private function guardRemoteActivityCancellationPoliciesAvailable( + Request $request, + string $namespace, + string $taskId, + int $workflowTaskAttempt, + array $commands, + ?WorkflowTask $claimedTask = null, + ): ?JsonResponse { + $usesPolicy = false; + foreach ($commands as $command) { + if (($command['type'] ?? null) === 'schedule_activity' + && $this->hasCommandValue($command, 'cancellation_policy')) { + $usesPolicy = true; + break; + } + } + if (! $usesPolicy) { + return null; + } + $task = $claimedTask ?? NamespaceWorkflowScope::taskQuery($namespace)->whereKey($taskId)->first(); + $unavailable = CooperativeCancellationPolicy::remoteActivityPolicyUnavailableReasons( + $task, + WorkerProtocol::requestVersion($request), + ); + if ($unavailable === []) { + return null; + } + + return WorkerProtocol::json([ + 'task_id' => $taskId, + 'workflow_task_attempt' => $workflowTaskAttempt, + 'worker_id' => $task?->lease_owner, + 'outcome' => 'rejected', + 'recorded' => false, + 'reason' => 'activity_cancellation_policy_not_supported', + 'required_capability' => CooperativeCancellationPolicy::CAPABILITY, + 'minimum_protocol_version' => CooperativeCancellationPolicy::MINIMUM_PROTOCOL_VERSION, + 'requested_version' => WorkerProtocol::requestVersion($request), + 'unavailable' => $unavailable, + 'remediation' => 'Use a runtime with remote activity cancellation policies and a cooperative worker claim on protocol 1.20 or newer.', + ], 409); + } + /** @param array> $commands */ private function guardChildCancellationPoliciesAvailable( Request $request, @@ -2917,9 +2982,10 @@ private function validateWorkflowTaskCommandScopes(array $commands): void } } - if ($this->hasCommandValue($command, 'cancellation_policy') && $type !== 'start_child_workflow') { + if ($this->hasCommandValue($command, 'cancellation_policy') + && ! in_array($type, ['start_child_workflow', 'schedule_activity'], true)) { $errors["commands.{$index}.cancellation_policy"][] = - 'cancellation_policy is only supported for start_child_workflow commands.'; + 'cancellation_policy is only supported for start_child_workflow or schedule_activity commands.'; } if ($this->hasCommandValue($command, 'non_retryable') diff --git a/app/Support/CooperativeCancellationPolicy.php b/app/Support/CooperativeCancellationPolicy.php index 3cf8a856..86bea5db 100644 --- a/app/Support/CooperativeCancellationPolicy.php +++ b/app/Support/CooperativeCancellationPolicy.php @@ -3,6 +3,7 @@ namespace App\Support; use App\Models\WorkerRegistration; +use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Enums\CancellationPolicy; use Workflow\V2\Enums\ParentClosePolicy; use Workflow\V2\Enums\TaskStatus; @@ -43,6 +44,35 @@ public static function activityAcknowledgementBackendSupported(): bool && method_exists(ActivityCancellationAcknowledgement::class, 'recordStopped'); } + public static function remoteActivityPolicyBackendSupported(): bool + { + $bridge = app(WorkflowTaskBridge::class); + + return method_exists($bridge, 'supportsRemoteActivityCancellationPolicies') + && is_callable([$bridge, 'supportsRemoteActivityCancellationPolicies']) + && $bridge->supportsRemoteActivityCancellationPolicies() === true; + } + + /** @return list */ + public static function remoteActivityPolicyUnavailableReasons(?WorkflowTask $task, ?string $requestVersion): array + { + $reasons = []; + if (! self::serverSupported()) { + $reasons[] = 'server_protocol'; + } + if (! WorkerProtocol::versionMeetsMinimum($requestVersion, self::MINIMUM_PROTOCOL_VERSION)) { + $reasons[] = 'request_protocol'; + } + if ($task === null || ! self::claimSupportsCancellation($task)) { + $reasons[] = 'worker_claim_capability'; + } + if (! self::remoteActivityPolicyBackendSupported()) { + $reasons[] = 'installed_runtime_activity_policy'; + } + + return $reasons; + } + /** @return list */ public static function childPolicyUnavailableReasons(?WorkflowTask $task, ?string $requestVersion): array { diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 348a8dd2..475474c5 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "31" + version: "32" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -39,6 +39,13 @@ x-durable-workflow-cooperative-cancellation-contract: legacy_terminal_parent_close_policy: request_cancel child_policy_admission: [server_protocol, request_protocol, immutable_worker_claim_capability, installed_runtime_child_policy] unsupported_child_policy: { status: 409, reason: child_cancellation_policy_not_supported, recorded: false } + remote_activity_operation_policies: [try_cancel, wait_cancellation_completed, abandon] + remote_activity_policy_admission: [server_protocol, request_protocol, immutable_worker_claim_capability, installed_runtime_activity_policy] + remote_activity_policy_default: try_cancel + remote_activity_abandon_lifetime: original_finite_schedule_to_close_timeout + unsupported_remote_activity_policy: { status: 409, reason: activity_cancellation_policy_not_supported, recorded: false } + local_activity_policy_authoring: refused_pending_portable_admission + local_activity_abandon: refused_pending_independent_callback_lifetime claim_proof: immutable_registration_snapshot_bound_to_task_run_owner_and_attempt request_against_incompatible_active_claim: { status: 409, reason: active_claim_cancellation_not_supported, history_appended: false } request_claim_race: serialized_on_workflow_run_lock @@ -2456,7 +2463,7 @@ components: type: [string, "null"] enum: [try_cancel, wait_cancellation_completed, abandon, null] x-durable-workflow-minimum-protocol-version: "1.20" - description: Per-operation child cancellation policy. Absent or null retains abandon. Cooperative policies require a capable immutable worker claim and an installed runtime implementation. + description: Per-operation child or remote Activity cancellation policy. Absent or null retains child abandon and Activity try_cancel. Explicit remote policies require protocol 1.20, a capable immutable worker claim and an installed runtime implementation. Remote abandon requires a finite schedule_to_close_timeout. Explicit local Activity policies are not admitted. allOf: - if: properties: @@ -2541,6 +2548,15 @@ components: outcome: { const: timed_out } then: required: [timeout_kind] + - if: + properties: + type: { const: schedule_activity } + cancellation_policy: { const: abandon } + required: [type, cancellation_policy] + then: + required: [schedule_to_close_timeout] + properties: + schedule_to_close_timeout: { type: integer, minimum: 1 } LocalActivityAttemptReport: type: object additionalProperties: false diff --git a/tests/Feature/CooperativeCancellationProtocolTest.php b/tests/Feature/CooperativeCancellationProtocolTest.php index b38f5974..6cdbaadc 100644 --- a/tests/Feature/CooperativeCancellationProtocolTest.php +++ b/tests/Feature/CooperativeCancellationProtocolTest.php @@ -469,6 +469,83 @@ public function test_child_policy_is_preserved_or_reports_the_missing_installed_ ->where('parent_workflow_run_id', $runId)->where('link_type', 'child_workflow')->sole()->parent_close_policy); } + #[DataProvider('remoteActivityPolicies')] + public function test_remote_activity_policy_is_canonical_or_reports_the_missing_installed_runtime(string $policy): void + { + [, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $response = $this->complete($task, [['type' => 'schedule_activity', + 'activity_type' => 'tests.remote', 'arguments' => Serializer::serialize(['work']), + 'cancellation_policy' => $policy, 'schedule_to_close_timeout' => 180]]); + if (! CooperativeCancellationPolicy::remoteActivityPolicyBackendSupported()) { + $response->assertStatus(409)->assertJsonPath('reason', 'activity_cancellation_policy_not_supported') + ->assertJsonPath('unavailable', ['installed_runtime_activity_policy'])->assertJsonPath('recorded', false); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::ActivityScheduled)); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + + return; + } + $response->assertOk()->assertJsonPath('outcome', 'completed'); + $event = WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::ActivityScheduled)->sole(); + $this->assertSame($policy, $event->payload['activity']['cancellation_policy']); + if ($policy === 'abandon') { + $this->assertNotNull($event->payload['activity']['schedule_to_close_deadline_at']); + } + } + + public static function remoteActivityPolicies(): array + { + return [['try_cancel'], ['wait_cancellation_completed'], ['abandon']]; + } + + public function test_remote_activity_policy_cannot_upgrade_an_old_claim_from_a_changed_registration(): void + { + [, $runId] = $this->start(); + $this->register('old', false); + $task = $this->poll('old', '1.19')->json('task'); + WorkerRegistration::query()->where('worker_id', 'old')->sole()->forceFill([ + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY], + ])->save(); + $this->complete($task, [['type' => 'schedule_activity', 'activity_type' => 'tests.remote', + 'cancellation_policy' => 'wait_cancellation_completed']])->assertStatus(409) + ->assertJsonPath('reason', 'activity_cancellation_policy_not_supported')->assertJsonPath('worker_id', 'old') + ->assertJsonPath('recorded', false)->assertJsonFragment(['worker_claim_capability']); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::ActivityScheduled)); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + } + + public function test_remote_abandon_requires_a_total_lifetime_before_scheduling(): void + { + [, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $response = $this->complete($task, [['type' => 'schedule_activity', 'activity_type' => 'tests.remote', + 'cancellation_policy' => 'abandon']]); + if (CooperativeCancellationPolicy::remoteActivityPolicyBackendSupported()) { + $response->assertStatus(422)->assertJsonValidationErrors('commands.0.schedule_to_close_timeout'); + } else { + $response->assertStatus(409)->assertJsonPath('reason', 'activity_cancellation_policy_not_supported'); + } + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::ActivityScheduled)); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + } + + public function test_remote_activity_policy_refuses_completion_on_the_legacy_protocol(): void + { + [, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $this->withHeaders($this->headers('1.19'))->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => 'new', 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [['type' => 'schedule_activity', 'activity_type' => 'tests.remote', + 'cancellation_policy' => 'try_cancel']], + ])->assertStatus(409)->assertJsonPath('reason', 'activity_cancellation_policy_not_supported') + ->assertJsonFragment(['request_protocol'])->assertJsonPath('recorded', false); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::ActivityScheduled)); + } + public function test_old_claim_cannot_acquire_child_policy_support_from_a_changed_registration(): void { [, $runId] = $this->start(); From 70807856b416b8792099108712cdd031e2c66c89 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 19:37:37 +0000 Subject: [PATCH 27/57] Expose the canonical cancellation cascade in run diagnostics --- .github/workflows/phpunit-feature.yml | 4 +- app/Support/WorkflowRunDiagnostics.php | 5 + docs/server-reference.md | 25 ++ tests/Feature/WorkflowDebugTest.php | 3 + .../CancellationCascadeDiagnosticsTest.php | 245 ++++++++++++++++++ 5 files changed, 280 insertions(+), 2 deletions(-) create mode 100644 tests/Source/CancellationCascadeDiagnosticsTest.php diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index 5a33d516..64012abf 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -259,7 +259,7 @@ jobs: echo "Server repository contracts and feature tests succeeded." prepared-local-source: - name: Candidate prepared-local source qualification + name: Candidate cancellation and prepared-local source qualification needs: [action-policy, preflight] if: ${{ github.event_name == 'workflow_dispatch' && inputs.prepared_local_workflow_commit != '' }} runs-on: ubuntu-latest @@ -340,7 +340,7 @@ jobs: composer install --no-interaction --no-progress --prefer-dist mv vendor/durable-workflow/workflow /tmp/locked-workflow-package cp -a prepared-workflow-source vendor/durable-workflow/workflow - php vendor/bin/phpunit tests/Feature/PreparedLocalActivityProtocolTest.php tests/Feature/HistoryRetentionTest.php tests/Unit/WorkerPollFenceTest.php --fail-on-skipped --log-junit "/evidence/$DW_PREPARED_DATABASE.xml" --colors=never + php vendor/bin/phpunit tests/Feature/PreparedLocalActivityProtocolTest.php tests/Feature/HistoryRetentionTest.php tests/Unit/WorkerPollFenceTest.php tests/Source/CancellationCascadeDiagnosticsTest.php tests/Feature/WorkflowDebugTest.php --fail-on-skipped --log-junit "/evidence/$DW_PREPARED_DATABASE.xml" --colors=never if [ "$DW_PREPARED_DATABASE" = sqlite ]; then php vendor/bin/phpunit tests/Feature/CooperativeCancellationProtocolTest.php tests/Feature/WorkflowWorkerProtocolTest.php tests/Feature/ActivityWorkerProtocolTest.php tests/Feature/SearchAttributeValueValidationTest.php tests/Feature/NamespaceDurableStateQuotaTest.php tests/Feature/RuntimeExternalPayloadTransportTest.php tests/Feature/RuntimePayloadCompletionUploadsTest.php tests/Unit/RuntimePayloadCompletionContextTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php --log-junit /evidence/affected-regression.xml --colors=never fi diff --git a/app/Support/WorkflowRunDiagnostics.php b/app/Support/WorkflowRunDiagnostics.php index 98ee4135..cdb5ba7b 100644 --- a/app/Support/WorkflowRunDiagnostics.php +++ b/app/Support/WorkflowRunDiagnostics.php @@ -16,6 +16,7 @@ use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowRunSummary; use Workflow\V2\Models\WorkflowTask; +use Workflow\V2\Support\CancellationCascadeView; use Workflow\V2\Support\StandaloneWorkerVisibility; use Workflow\V2\Support\TaskCompatibility; use Workflow\V2\Support\TaskRepairPolicy; @@ -72,6 +73,10 @@ public function forRun(string $namespace, WorkflowRun $run, bool $includeLastEve 'recent_failures' => $recentFailures, 'latest_workflow_task_failure' => $latestWorkflowTaskFailure, 'compatibility' => $this->compatibility($namespace, $run, $summary, $taskQueue), + 'cancellation_cascade_supported' => class_exists(CancellationCascadeView::class), + 'cancellation_cascade' => class_exists(CancellationCascadeView::class) && $run->namespace === $namespace + ? CancellationCascadeView::forRun($run) + : null, ]; $payload['findings'] = $this->findings($payload); diff --git a/docs/server-reference.md b/docs/server-reference.md index 06a7e61c..704d1caf 100644 --- a/docs/server-reference.md +++ b/docs/server-reference.md @@ -638,6 +638,31 @@ metadata by default; add `include_last_event_payload=true` to include at most a 4 KiB JSON preview. Use the history endpoints when a full replay/debug archive is needed. +The candidate cooperative-cancellation runtime adds `cancellation_cascade_supported` +and `cancellation_cascade` to both debug responses. A supported runtime returns +`null` when the selected run has no cooperative request. An older installed +runtime reports support as `false`. This read-only view does not require worker +protocol 1.20 to inspect already recorded evidence. + +The `durable-workflow.cancellation-cascade/v1` view joins the original root +request, its immutable cleanup deadline, each visible run's local request and +lifecycle, child propagation, activity fencing and matching stop receipts, +cleanup lease recovery, and the recorded terminal cleanup outcome. Selecting a +historical run keeps that run selected. Every related run is checked against +the namespace before its identity or history is returned. Activity fencing +alone does not prove that its callback stopped. Recovery records identify the +old and replacement grants without guessing why a worker lost its lease. + +The view inspects at most 20 runs, 100 relationship edges, 128 recent relevant +history events per run and 512 recent history events in total. Original request +events are loaded separately so clipping recent history does not change the +root budget. Each request text field is capped at 8192 bytes. Missing, +inaccessible, inconsistent or clipped evidence is explained in `findings` and +sets `inspection_complete` to `false`. `inspection_complete` describes the +evidence inventory, not whether cleanup succeeded. Use the history endpoints +for evidence outside these bounds. This candidate contract is not a claim +about the currently published Server image. + ### History - `GET /api/workflows/{id}/runs/{runId}/history` — Get event history - `GET /api/workflows/{id}/runs/{runId}/history/export` — Export replay bundle diff --git a/tests/Feature/WorkflowDebugTest.php b/tests/Feature/WorkflowDebugTest.php index b1931beb..95ab8f1d 100644 --- a/tests/Feature/WorkflowDebugTest.php +++ b/tests/Feature/WorkflowDebugTest.php @@ -21,6 +21,7 @@ use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; +use Workflow\V2\Support\CancellationCascadeView; use Workflow\V2\Support\WorkerCompatibilityFleet; class WorkflowDebugTest extends TestCase @@ -86,6 +87,8 @@ public function test_it_aggregates_a_one_shot_workflow_debug_diagnostic(): void ->assertJsonPath('workflow_id', 'wf-debug') ->assertJsonPath('run_id', $runId) ->assertJsonPath('namespace', 'default') + ->assertJsonPath('cancellation_cascade_supported', class_exists(CancellationCascadeView::class)) + ->assertJsonPath('cancellation_cascade', null) ->assertJsonPath('diagnostic_status', 'pending_work') ->assertJsonPath('execution.status', 'pending') ->assertJsonPath('execution.task_queue', 'debug-queue') diff --git a/tests/Source/CancellationCascadeDiagnosticsTest.php b/tests/Source/CancellationCascadeDiagnosticsTest.php new file mode 100644 index 00000000..bb2de119 --- /dev/null +++ b/tests/Source/CancellationCascadeDiagnosticsTest.php @@ -0,0 +1,245 @@ +assertTrue(class_exists(CancellationCascadeView::class), 'This qualification requires the exact Native cancellation inspection source.'); + Queue::fake(); + config([ + 'server.worker_protocol.version' => '1.20', + 'workflows.v2.types.workflows' => ['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class], + ]); + WorkflowNamespace::query()->create([ + 'name' => 'default', 'description' => 'Test', 'retention_days' => 30, 'status' => 'active', + ]); + $this->travelTo(now()->startOfSecond()); + } + + public function test_requested_delivered_and_completed_cleanup_share_the_original_budget(): void + { + [$workflowId, $runId] = $this->start(); + $task = $this->poll()->json('task'); + $original = $this->requestCancellation($workflowId)->assertStatus(202)->json('cancellation_request'); + $before = WorkflowHistoryEvent::query()->count(); + $requested = $this->debug($workflowId)->assertOk() + ->assertJsonPath('cancellation_cascade_supported', true) + ->assertJsonPath('cancellation_cascade.schema', 'durable-workflow.cancellation-cascade/v1') + ->assertJsonPath('cancellation_cascade.selected_run_id', $runId) + ->assertJsonPath('cancellation_cascade.root.root_request_id', $original['request_id']) + ->assertJsonPath('cancellation_cascade.root.cleanup_deadline_at', $original['cleanup_deadline_at']) + ->assertJsonPath('cancellation_cascade.runs.0.lifecycle', 'requested') + ->assertJsonPath('cancellation_cascade.inspection_complete', true) + ->assertJsonPath('cancellation_cascade.truncated', false); + $this->assertSame($before, WorkflowHistoryEvent::query()->count(), 'Diagnostics must not mutate history.'); + $this->assertSame([], $requested->json('cancellation_cascade.runs.0.activity_stops')); + + $this->deliver($task, $original['request_id'])->assertOk()->assertJsonPath('delivered', true); + $this->debug($workflowId)->assertOk() + ->assertJsonPath('cancellation_cascade.runs.0.lifecycle', 'delivered') + ->assertJsonPath('cancellation_cascade.runs.0.delivery.sequence', 1); + $this->complete($task, [['type' => 'complete_workflow']])->assertOk()->assertJsonPath('run_status', 'cancelled'); + $finished = $this->debug($workflowId, $runId)->assertOk() + ->assertJsonPath('cancellation_cascade.runs.0.lifecycle', 'cancelled') + ->assertJsonPath('cancellation_cascade.runs.0.cleanup.outcome', 'completed') + ->assertJsonPath('cancellation_cascade.runs.0.cleanup.cleanup_deadline_at', $original['cleanup_deadline_at']) + ->assertJsonPath('cancellation_cascade.inspection_complete', true) + ->json('cancellation_cascade'); + $this->travel(10)->seconds(); + $this->requestCancellation($workflowId, 300)->assertOk() + ->assertJsonPath('duplicate', true)->assertJsonPath('cancellation_request.request_id', $original['request_id']) + ->assertJsonPath('cancellation_request.cleanup_deadline_at', $original['cleanup_deadline_at']); + $this->assertSame($finished, $this->debug($workflowId, $runId)->assertOk()->json('cancellation_cascade')); + } + + public function test_child_propagation_is_visible_from_either_run_without_inventing_a_second_root(): void + { + [$workflowId, $runId] = $this->start(); + $task = $this->poll()->json('task'); + $this->complete($task, [[ + 'type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', + 'cancellation_policy' => 'wait_cancellation_completed', + ]])->assertOk(); + $link = WorkflowLink::query()->where('parent_workflow_run_id', $runId)->where('link_type', 'child_workflow')->sole(); + $original = $this->requestCancellation($workflowId)->assertStatus(202)->json('cancellation_request'); + // The child was scheduled first. Hold its claim so the following poll obtains the parent cancellation task. + $childTask = $this->poll('child')->assertOk()->json('task'); + $this->assertSame($link->child_workflow_run_id, $childTask['run_id']); + $parentTask = $this->poll('parent')->assertOk()->json('task'); + $this->assertSame($runId, $parentTask['run_id']); + $this->deliver($parentTask, $original['request_id'], 'child')->assertOk() + ->assertJsonPath('delivered', false)->assertJsonPath('reason', 'cancellation_waiting_for_child'); + + foreach ([[$workflowId, $runId], [$link->child_workflow_instance_id, $link->child_workflow_run_id]] as [$selectedWorkflow, $selectedRun]) { + $view = $this->debug($selectedWorkflow, $selectedRun)->assertOk() + ->assertJsonPath('cancellation_cascade.selected_run_id', $selectedRun) + ->assertJsonPath('cancellation_cascade.root.root_request_id', $original['request_id']) + ->assertJsonPath('cancellation_cascade.root.cleanup_deadline_at', $original['cleanup_deadline_at']) + ->assertJsonPath('cancellation_cascade.inspection_complete', true)->json('cancellation_cascade'); + $this->assertCount(2, $view['runs']); + $this->assertCount(1, $view['edges']); + $this->assertSame($runId, $view['edges'][0]['parent_run_id']); + $this->assertSame($link->child_workflow_run_id, $view['edges'][0]['child_run_id']); + $this->assertSame('requested', $view['runs'][1]['lifecycle']); + $this->assertTrue($view['runs'][1]['same_root_budget']); + $this->assertNotSame($original['request_id'], $view['runs'][1]['request']['request_id']); + $this->assertSame($original['request_id'], $view['runs'][1]['request']['parent_request_id']); + $this->assertSame('wait_cancellation_completed', $view['runs'][0]['child_propagation'][0]['policy']); + } + } + + public function test_a_deadline_expiry_is_a_terminal_outcome_separate_from_completed_cleanup(): void + { + [$workflowId] = $this->start(); + $task = $this->poll()->json('task'); + $original = $this->requestCancellation($workflowId, 1)->assertStatus(202)->json('cancellation_request'); + $this->travel(2)->seconds(); + $this->deliver($task, $original['request_id'])->assertStatus(409)->assertJsonPath('reason', 'run_cancelled'); + $this->debug($workflowId)->assertOk() + ->assertJsonPath('cancellation_cascade.runs.0.projected_status', 'cancelled') + ->assertJsonPath('cancellation_cascade.runs.0.lifecycle', 'deadline_expired') + ->assertJsonPath('cancellation_cascade.runs.0.cleanup.outcome', 'deadline_expired') + ->assertJsonPath('cancellation_cascade.runs.0.delivery', null) + ->assertJsonPath('cancellation_cascade.inspection_complete', true); + } + + public function test_missing_canonical_history_cannot_be_presented_as_a_successful_cascade(): void + { + [$workflowId, $runId] = $this->start(); + $this->requestCancellation($workflowId)->assertStatus(202); + WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::CooperativeCancellationRequested->value)->delete(); + $view = $this->debug($workflowId)->assertOk() + ->assertJsonPath('cancellation_cascade.root', null) + ->assertJsonPath('cancellation_cascade.inspection_complete', false)->json('cancellation_cascade'); + $this->assertContains('request_history_unavailable', array_column($view['findings'], 'code')); + } + + public function test_related_foreign_runs_and_their_cancellation_metadata_are_not_exposed(): void + { + [$workflowId, $runId] = $this->start(); + $this->requestCancellation($workflowId)->assertStatus(202); + [$foreignWorkflow, $foreignRunId] = $this->start(); + $this->requestCancellation($foreignWorkflow)->assertStatus(202); + $foreign = WorkflowRun::query()->findOrFail($foreignRunId); + $foreign->instance->forceFill(['namespace' => 'foreign'])->save(); + $foreign->forceFill(['namespace' => 'foreign'])->save(); + WorkflowLink::query()->create([ + 'parent_workflow_instance_id' => $workflowId, 'parent_workflow_run_id' => $runId, + 'child_workflow_instance_id' => $foreignWorkflow, 'child_workflow_run_id' => $foreignRunId, + 'link_type' => 'child_workflow', 'sequence' => 99, 'is_primary_parent' => false, + ]); + $view = $this->debug($workflowId)->assertOk() + ->assertJsonPath('cancellation_cascade.inspection_complete', false) + ->assertJsonPath('cancellation_cascade.edges.0.child_run_id', null)->json('cancellation_cascade'); + $encoded = json_encode($view, JSON_THROW_ON_ERROR); + $this->assertStringNotContainsString($foreignWorkflow, $encoded); + $this->assertStringNotContainsString($foreignRunId, $encoded); + $this->debug($foreignWorkflow)->assertNotFound(); + $this->debug($workflowId, $foreignRunId)->assertNotFound(); + } + + public function test_run_scoped_debug_keeps_the_historical_cancellation_selection(): void + { + [$workflowId, $runId] = $this->start(); + $original = $this->requestCancellation($workflowId)->assertStatus(202)->json('cancellation_request'); + $selected = WorkflowRun::query()->findOrFail($runId); + $replacement = $selected->replicate(['id']); + $replacement->run_number = $selected->run_number + 1; + $replacement->cancellation_request_command_id = null; + $replacement->cancellation_requested_at = null; + $replacement->cancellation_deadline_at = null; + $replacement->save(); + $selected->instance->forceFill(['current_run_id' => $replacement->id])->save(); + $this->debug($workflowId)->assertOk()->assertJsonPath('run_id', $replacement->id) + ->assertJsonPath('cancellation_cascade', null); + $historical = $this->debug($workflowId, $runId)->assertOk() + ->assertJsonPath('run_id', $runId)->assertJsonPath('cancellation_cascade.selected_run_id', $runId) + ->assertJsonPath('cancellation_cascade.root.root_request_id', $original['request_id']); + $this->assertStringNotContainsString($replacement->id, json_encode($historical->json('cancellation_cascade'), JSON_THROW_ON_ERROR)); + } + + private function start(): array + { + $started = $this->postJson('/api/workflows', [ + 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'cooperative', 'input' => ['Ada'], + ], $this->controlHeaders())->assertCreated(); + + return [$started->json('workflow_id'), $started->json('run_id')]; + } + + private function poll(string $workerId = 'inspector'): TestResponse + { + $this->postJson('/api/worker/register', [ + 'worker_id' => $workerId, 'task_queue' => 'cooperative', 'runtime' => 'php', + 'supported_workflow_types' => ['tests.external-greeting-workflow'], + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY], + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + 'max_concurrent_workflow_tasks' => 2, + ], $this->workerHeaders())->assertCreated(); + + return $this->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => $workerId, 'task_queue' => 'cooperative', + ], $this->workerHeaders())->assertOk(); + } + + private function requestCancellation(string $workflowId, int $timeout = 30): TestResponse + { + return $this->postJson("/api/workflows/{$workflowId}/request-cancellation", [ + 'reason' => 'Maintenance', 'cleanup_timeout_seconds' => $timeout, + ], $this->controlHeaders()); + } + + private function deliver(array $task, string $requestId, string $kind = 'activity'): TestResponse + { + return $this->postJson("/api/worker/workflow-tasks/{$task['task_id']}/deliver-cancellation", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'request_id' => $requestId, 'sequence' => 1, 'call_kind' => $kind, + ], $this->workerHeaders()); + } + + private function complete(array $task, array $commands): TestResponse + { + return $this->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => $commands, + ], $this->workerHeaders()); + } + + private function debug(string $workflowId, ?string $runId = null): TestResponse + { + $target = $runId === null ? "/api/workflows/{$workflowId}" : "/api/workflows/{$workflowId}/runs/{$runId}"; + + return $this->getJson($target.'/debug', $this->controlHeaders()); + } + + private function controlHeaders(): array + { + return ['X-Namespace' => 'default', 'X-Durable-Workflow-Control-Plane-Version' => '2']; + } + + private function workerHeaders(): array + { + return ['X-Namespace' => 'default', WorkerProtocol::HEADER => '1.20']; + } +} From 172dedb57a71629048fc145175d21eb626bf386d Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 22:47:20 +0000 Subject: [PATCH 28/57] Distinguish recovered task history in debug findings --- app/Support/WorkflowRunDiagnostics.php | 9 ++- tests/Feature/WorkflowDebugTest.php | 81 ++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/app/Support/WorkflowRunDiagnostics.php b/app/Support/WorkflowRunDiagnostics.php index cdb5ba7b..705286fb 100644 --- a/app/Support/WorkflowRunDiagnostics.php +++ b/app/Support/WorkflowRunDiagnostics.php @@ -889,10 +889,13 @@ private function findings(array $payload): array } if ((bool) data_get($payload, 'execution.task_problem', false)) { + $historical = data_get($payload, 'execution.task_problem_badge.code') === 'history'; $findings[] = [ - 'severity' => 'warning', - 'code' => 'task_problem', - 'message' => 'The run summary has a task problem flag.', + 'severity' => $historical ? 'info' : 'warning', + 'code' => $historical ? 'task_recovery_history' : 'task_problem', + 'message' => $historical + ? 'The run previously needed workflow-task repair or replay recovery.' + : 'The run summary has a task problem flag.', ]; } diff --git a/tests/Feature/WorkflowDebugTest.php b/tests/Feature/WorkflowDebugTest.php index 95ab8f1d..4826cde8 100644 --- a/tests/Feature/WorkflowDebugTest.php +++ b/tests/Feature/WorkflowDebugTest.php @@ -5,6 +5,7 @@ use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Queue; +use PHPUnit\Framework\Attributes\DataProvider; use Tests\Feature\Concerns\ServerTestHelpers; use Tests\Fixtures\AwaitApprovalWorkflow; use Tests\TestCase; @@ -22,6 +23,7 @@ use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; use Workflow\V2\Support\CancellationCascadeView; +use Workflow\V2\Support\RunSummaryProjector; use Workflow\V2\Support\WorkerCompatibilityFleet; class WorkflowDebugTest extends TestCase @@ -41,6 +43,85 @@ protected function setUp(): void ]); } + #[DataProvider('recoveredTerminalOutcomes')] + public function test_recovered_tasks_are_informational_history_after_a_run_closes(string $outcome): void + { + $workflowId = 'debug-recovered-'.$outcome; + [$runId, $task, $workerId, $attempt] = $this->diagnosticTask($workflowId); + $task->forceFill(['repair_count' => 1])->save(); + + if ($outcome === 'cancelled') { + $this->postJson("/api/workflows/{$workflowId}/cancel", [], $this->controlPlaneHeadersWithWorkerProtocol()) + ->assertOk(); + } else { + $this->postJson("/api/worker/workflow-tasks/{$task->id}/complete", [ + 'lease_owner' => $workerId, + 'workflow_task_attempt' => $attempt, + 'commands' => [['type' => 'complete_workflow']], + ], $this->workerHeaders())->assertOk()->assertJsonPath('run_status', 'completed'); + } + + $this->getJson("/api/workflows/{$workflowId}/debug", $this->controlPlaneHeadersWithWorkerProtocol()) + ->assertOk() + ->assertJsonPath('run_id', $runId) + ->assertJsonPath('diagnostic_status', 'terminal') + ->assertJsonPath('execution.status', $outcome) + ->assertJsonPath('execution.task_problem', true) + ->assertJsonPath('execution.task_problem_badge.code', 'history') + ->assertJsonPath('findings.0.code', 'task_recovery_history') + ->assertJsonPath('findings.0.severity', 'info') + ->assertJsonPath('findings.0.message', 'The run previously needed workflow-task repair or replay recovery.') + ->assertJsonMissing(['code' => 'task_problem']); + } + + public static function recoveredTerminalOutcomes(): array + { + return ['completed' => ['completed'], 'cancelled' => ['cancelled']]; + } + + public function test_active_replay_failure_keeps_its_warning_and_error(): void + { + $workflowId = 'debug-active-replay'; + [$runId, $task] = $this->diagnosticTask($workflowId); + $task->forceFill([ + 'status' => TaskStatus::Failed, + 'payload' => [...($task->payload ?? []), 'replay_blocked' => true], + 'last_error' => 'Recorded workflow commands changed.', + ])->save(); + RunSummaryProjector::project(WorkflowRun::query()->findOrFail($runId)); + + $debug = $this->getJson("/api/workflows/{$workflowId}/debug", $this->controlPlaneHeadersWithWorkerProtocol()) + ->assertOk() + ->assertJsonPath('diagnostic_status', 'needs_attention') + ->assertJsonPath('execution.task_problem_badge.code', 'replay_blocked'); + $findings = collect($debug->json('findings'))->keyBy('code'); + $this->assertSame('warning', $findings['task_problem']['severity']); + $this->assertSame('error', $findings['workflow_replay_blocked']['severity']); + $this->assertFalse($findings->has('task_recovery_history')); + } + + private function diagnosticTask(string $workflowId): array + { + $workerId = $workflowId.'-worker'; + $queue = $workflowId.'-queue'; + $this->registerWorker($workerId, $queue, supportedWorkflowTypes: ['tests.await-approval-workflow']); + $started = $this->postJson('/api/workflows', [ + 'workflow_id' => $workflowId, + 'workflow_type' => 'tests.await-approval-workflow', + 'task_queue' => $queue, + ], $this->controlPlaneHeadersWithWorkerProtocol())->assertCreated(); + $polled = $this->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => $workerId, 'task_queue' => $queue, + ], $this->workerHeaders())->assertOk(); + + return [ + $started->json('run_id'), + WorkflowTask::query()->findOrFail($polled->json('task.task_id')), + $workerId, + (int) $polled->json('task.workflow_task_attempt'), + ]; + } + public function test_it_aggregates_a_one_shot_workflow_debug_diagnostic(): void { $this->registerWorker( From dd8996fdd6d488b48d575b1b934821bd8bedc8ed Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 23:45:36 +0000 Subject: [PATCH 29/57] Bind prepared local cancellation policies to issued worker claims --- .github/workflows/phpunit-feature.yml | 2 +- .../Api/PreparedLocalActivityController.php | 29 ++ app/Http/Controllers/Api/WorkerController.php | 38 ++- app/Support/PreparedLocalActivityPolicy.php | 70 +++++ app/Support/WorkerProtocol.php | 1 + app/Support/WorkflowTaskPoller.php | 11 +- docs/contracts/external-execution-surface.md | 29 ++ .../worker-protocol-api.openapi.yaml | 22 +- .../PreparedLocalCancellationPolicyTest.php | 282 ++++++++++++++++++ 9 files changed, 473 insertions(+), 11 deletions(-) create mode 100644 tests/Source/PreparedLocalCancellationPolicyTest.php diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index 64012abf..ce5ccb71 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -340,7 +340,7 @@ jobs: composer install --no-interaction --no-progress --prefer-dist mv vendor/durable-workflow/workflow /tmp/locked-workflow-package cp -a prepared-workflow-source vendor/durable-workflow/workflow - php vendor/bin/phpunit tests/Feature/PreparedLocalActivityProtocolTest.php tests/Feature/HistoryRetentionTest.php tests/Unit/WorkerPollFenceTest.php tests/Source/CancellationCascadeDiagnosticsTest.php tests/Feature/WorkflowDebugTest.php --fail-on-skipped --log-junit "/evidence/$DW_PREPARED_DATABASE.xml" --colors=never + php vendor/bin/phpunit tests/Feature/PreparedLocalActivityProtocolTest.php tests/Feature/HistoryRetentionTest.php tests/Unit/WorkerPollFenceTest.php tests/Source/CancellationCascadeDiagnosticsTest.php tests/Source/PreparedLocalCancellationPolicyTest.php tests/Feature/WorkflowDebugTest.php --fail-on-skipped --log-junit "/evidence/$DW_PREPARED_DATABASE.xml" --colors=never if [ "$DW_PREPARED_DATABASE" = sqlite ]; then php vendor/bin/phpunit tests/Feature/CooperativeCancellationProtocolTest.php tests/Feature/WorkflowWorkerProtocolTest.php tests/Feature/ActivityWorkerProtocolTest.php tests/Feature/SearchAttributeValueValidationTest.php tests/Feature/NamespaceDurableStateQuotaTest.php tests/Feature/RuntimeExternalPayloadTransportTest.php tests/Feature/RuntimePayloadCompletionUploadsTest.php tests/Unit/RuntimePayloadCompletionContextTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php --log-junit /evidence/affected-regression.xml --colors=never fi diff --git a/app/Http/Controllers/Api/PreparedLocalActivityController.php b/app/Http/Controllers/Api/PreparedLocalActivityController.php index b7a57682..7faf24dc 100644 --- a/app/Http/Controllers/Api/PreparedLocalActivityController.php +++ b/app/Http/Controllers/Api/PreparedLocalActivityController.php @@ -62,6 +62,10 @@ private function admit(Request $request, string $taskId, bool $recover): JsonRes if ($task === null) { return WorkerProtocol::json(['reason' => 'task_not_found'], 404); } + if (array_key_exists('cancellation_policy', $validated['descriptor']) + && ($response = self::cancellationPolicyRefusal($request, $task, $validated['descriptor']['cancellation_policy']))) { + return $response; + } $claim = PreparedLocalActivityPolicy::currentClaim($task); if ($claim === null) { return self::refused($validated['lease_owner'], ['worker_claim_capability']); @@ -75,6 +79,10 @@ private function admit(Request $request, string $taskId, bool $recover): JsonRes if (! WorkerPollFence::isCurrentForUpdate($claim)) { return WorkerProtocol::json(['reason' => 'stale_worker_registration'], 409); } + if (array_key_exists('cancellation_policy', $validated['descriptor']) + && ($response = self::cancellationPolicyRefusal($request, NamespaceWorkflowScope::task($namespace, $taskId), $validated['descriptor']['cancellation_policy']))) { + return $response; + } /** @var PreparedLocalActivityTaskBridge $bridge */ $bridge = app(WorkflowTaskBridge::class); $reply = $recover @@ -284,6 +292,27 @@ public static function unavailable(Request $request): ?JsonResponse return $reasons === [] ? null : self::refused((string) $request->input('lease_owner', ''), $reasons); } + public static function cancellationPolicyRefusal(Request $request, ?WorkflowTask $task, mixed $policy): ?JsonResponse + { + $reasons = PreparedLocalActivityPolicy::cancellationPolicyUnavailableReasons($task, $policy, WorkerProtocol::requestVersion($request)); + if ($reasons === []) { + return null; + } + + return WorkerProtocol::json([ + 'reason' => 'prepared_local_activity_cancellation_policy_not_supported', + 'recorded' => false, + 'worker_id' => $task?->lease_owner ?? (string) $request->input('lease_owner', ''), + 'requested_policy' => $policy, + 'supported_policies' => PreparedLocalActivityPolicy::cancellationPolicies(), + 'required_capability' => PreparedLocalActivityPolicy::CANCELLATION_POLICIES_CAPABILITY, + 'minimum_protocol_version' => PreparedLocalActivityPolicy::MINIMUM_PROTOCOL_VERSION, + 'requested_version' => WorkerProtocol::requestVersion($request), + 'unavailable' => $reasons, + 'remediation' => 'Use an installed backend with the requested local policy and a new protocol 1.20 claim issued with prepared_local_activity_cancellation_policies. Local abandon requires independently bounded remote work.', + ], 409); + } + /** @param list $reasons */ public static function refused(string $workerId, array $reasons, string $capability = PreparedLocalActivityPolicy::CAPABILITY): JsonResponse { diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index c9ca937e..1c4e00b4 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -230,6 +230,16 @@ public function register(Request $request): JsonResponse ]); $workerCapabilities = $this->nonEmptyStringArray($validated['capabilities'] ?? []); + if (in_array(PreparedLocalActivityPolicy::CANCELLATION_POLICIES_CAPABILITY, $workerCapabilities, true) + && ! PreparedLocalActivityPolicy::supportsCancellationPolicies($workerCapabilities, WorkerProtocol::requestVersion($request))) { + return WorkerProtocol::json([ + 'registered' => false, + 'reason' => 'prepared_local_activity_cancellation_policy_capability_mismatch', + 'required_capabilities' => [CooperativeCancellationPolicy::CAPABILITY, PreparedLocalActivityPolicy::CAPABILITY], + 'minimum_protocol_version' => PreparedLocalActivityPolicy::MINIMUM_PROTOCOL_VERSION, + 'requested_version' => WorkerProtocol::requestVersion($request), + ], 409); + } if (in_array(PreparedLocalActivityPolicy::GROUP_CAPABILITY, $workerCapabilities, true) && (! in_array(PreparedLocalActivityPolicy::CAPABILITY, $workerCapabilities, true) || ! CooperativeCancellationPolicy::supports($workerCapabilities, WorkerProtocol::requestVersion($request)))) { @@ -1727,6 +1737,10 @@ private function mutateWorkflowTaskCommands(Request $request, string $taskId, bo $this->validateWorkflowTaskCommandScopes($commands); + if ($response = $this->guardPreparedLocalCancellationPoliciesAvailable($request, (string) $namespace, $taskId, $commands)) { + return $response; + } + if ($response = $this->guardWorkflowTaskOwnership( $request, $namespace, @@ -1942,6 +1956,10 @@ function () use ( return $response; } + if ($response = $this->guardPreparedLocalCancellationPoliciesAvailable($request, (string) $namespace, $taskId, $commands, $claimedTask)) { + return $response; + } + if ($response = $this->guardChildCancellationPoliciesAvailable( $request, (string) $namespace, @@ -2180,6 +2198,22 @@ private function normalizePreparedLocalGroupCommands(array $commands, string $pr return $commands; } + /** @param array> $commands */ + private function guardPreparedLocalCancellationPoliciesAvailable(Request $request, string $namespace, string $taskId, array $commands, ?WorkflowTask $claimedTask = null): ?JsonResponse + { + foreach ($commands as $command) { + if (($command['type'] ?? null) !== 'prepare_local_activity' || ! array_key_exists('cancellation_policy', $command)) { + continue; + } + $task = $claimedTask ?? NamespaceWorkflowScope::task($namespace, $taskId); + if ($response = PreparedLocalActivityController::cancellationPolicyRefusal($request, $task, $command['cancellation_policy'])) { + return $response; + } + } + + return null; + } + /** @param array> $commands */ private function guardRemoteActivityCancellationPoliciesAvailable( Request $request, @@ -2983,9 +3017,9 @@ private function validateWorkflowTaskCommandScopes(array $commands): void } if ($this->hasCommandValue($command, 'cancellation_policy') - && ! in_array($type, ['start_child_workflow', 'schedule_activity'], true)) { + && ! in_array($type, ['start_child_workflow', 'schedule_activity', 'prepare_local_activity'], true)) { $errors["commands.{$index}.cancellation_policy"][] = - 'cancellation_policy is only supported for start_child_workflow or schedule_activity commands.'; + 'cancellation_policy is only supported for start_child_workflow, schedule_activity or prepared local Activity commands.'; } if ($this->hasCommandValue($command, 'non_retryable') diff --git a/app/Support/PreparedLocalActivityPolicy.php b/app/Support/PreparedLocalActivityPolicy.php index b7c23ddf..4d307870 100644 --- a/app/Support/PreparedLocalActivityPolicy.php +++ b/app/Support/PreparedLocalActivityPolicy.php @@ -5,7 +5,9 @@ use Workflow\V2\Contracts\PreparedLocalActivityGroupTaskBridge; use Workflow\V2\Contracts\PreparedLocalActivityTaskBridge; use Workflow\V2\Contracts\WorkflowTaskBridge; +use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Models\ActivityAttempt; +use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowTask; /** Candidate protocol 1.20 admission and original issued local authority. */ @@ -15,6 +17,8 @@ final class PreparedLocalActivityPolicy public const GROUP_CAPABILITY = 'prepared_local_activity_groups'; + public const CANCELLATION_POLICIES_CAPABILITY = 'prepared_local_activity_cancellation_policies'; + public const MINIMUM_PROTOCOL_VERSION = '1.20'; private const CLAIM_KEY = '_server_workflow_claim'; @@ -39,6 +43,72 @@ public static function groupsSupported(): bool && app(WorkflowTaskBridge::class) instanceof PreparedLocalActivityGroupTaskBridge; } + /** @return list */ + public static function cancellationPolicies(): array + { + $bridge = app(WorkflowTaskBridge::class); + if (! self::serverSupported() || ! method_exists($bridge, 'supportedLocalActivityCancellationPolicies') + || ! is_callable([$bridge, 'supportedLocalActivityCancellationPolicies'])) { + return []; + } + + $policies = $bridge->supportedLocalActivityCancellationPolicies(); + + return is_array($policies) + ? array_values(array_filter(['try_cancel', 'wait_cancellation_completed'], static fn (string $policy): bool => in_array($policy, $policies, true))) + : []; + } + + /** @param list $capabilities */ + public static function supportsCancellationPolicies(array $capabilities, ?string $protocolVersion): bool + { + return CooperativeCancellationPolicy::supports($capabilities, $protocolVersion) + && in_array(self::CAPABILITY, $capabilities, true) + && in_array(self::CANCELLATION_POLICIES_CAPABILITY, $capabilities, true); + } + + /** @return list */ + public static function cancellationPolicyUnavailableReasons(?WorkflowTask $task, mixed $policy, ?string $requestVersion): array + { + $reasons = self::unavailableReasons($requestVersion); + if (! in_array($policy, ['try_cancel', 'wait_cancellation_completed'], true)) { + $reasons[] = 'local_activity_cancellation_policy'; + } elseif (! in_array($policy, self::cancellationPolicies(), true)) { + $reasons[] = 'installed_runtime_local_activity_cancellation_policy'; + } + $claim = $task instanceof WorkflowTask ? self::currentClaim($task) : null; + if ($claim === null || ! self::supportsCancellationPolicies($claim['capabilities'] ?? [], $claim['protocol_version'] ?? null)) { + $reasons[] = 'worker_claim_cancellation_policy_capability'; + } + + return $reasons; + } + + /** + * Canonical admission determines replay compatibility. Registration cannot + * upgrade the authority of an existing claim, and a legacy worker cannot + * reinterpret an explicitly authored policy during takeover. + * + * @param list $capabilities + */ + public static function canReplayRun(string $runId, array $capabilities, ?string $protocolVersion): bool + { + $policies = self::cancellationPolicies(); + $workerSupported = self::supportsCancellationPolicies($capabilities, $protocolVersion); + if ($workerSupported && count($policies) === 2) { + return true; + } + $events = WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::ActivityScheduled) + ->where('payload->execution_mode', 'local') + ->whereNotNull('payload->activity->cancellation_policy'); + if ($workerSupported && $policies !== []) { + $events->whereNotIn('payload->activity->cancellation_policy', $policies); + } + + return ! $events->exists(); + } + /** @return list */ public static function groupUnavailableReasons(?string $requestVersion): array { diff --git a/app/Support/WorkerProtocol.php b/app/Support/WorkerProtocol.php index 08fad7f9..9488fafd 100644 --- a/app/Support/WorkerProtocol.php +++ b/app/Support/WorkerProtocol.php @@ -549,6 +549,7 @@ public static function serverCapabilities(): array 'cooperative_cancellation' => CooperativeCancellationPolicy::serverSupported(), 'prepared_local_activities' => PreparedLocalActivityPolicy::serverSupported(), 'prepared_local_activity_groups' => PreparedLocalActivityPolicy::groupsSupported(), + 'prepared_local_activity_cancellation_policies' => PreparedLocalActivityPolicy::cancellationPolicies(), 'activity_cancellation_acknowledgement' => CooperativeCancellationPolicy::serverSupported() && CooperativeCancellationPolicy::activityAcknowledgementBackendSupported(), 'local_activities' => [ diff --git a/app/Support/WorkflowTaskPoller.php b/app/Support/WorkflowTaskPoller.php index 9b222cb7..427e00dd 100644 --- a/app/Support/WorkflowTaskPoller.php +++ b/app/Support/WorkflowTaskPoller.php @@ -2339,11 +2339,12 @@ private function workerCanReplayRun( return false; } - return WorkflowMetadataCapabilityPolicy::canReplayRun( - $runId, - $capabilities, - $protocolVersion, - ); + return PreparedLocalActivityPolicy::canReplayRun($runId, $capabilities, $protocolVersion) + && WorkflowMetadataCapabilityPolicy::canReplayRun( + $runId, + $capabilities, + $protocolVersion, + ); } /** @param array $workflowDefinitionFingerprints */ diff --git a/docs/contracts/external-execution-surface.md b/docs/contracts/external-execution-surface.md index 341989df..26086d5b 100644 --- a/docs/contracts/external-execution-surface.md +++ b/docs/contracts/external-execution-surface.md @@ -102,6 +102,35 @@ the original receipt event and late receipt remains explicitly late. This report grants no renewed lease, application heartbeat, result authority or cleanup budget. It describes the worker's stopped callback, not reversal of external effects. Local activities require separate workflow claim authority. + +## Candidate prepared local cancellation policies + +An explicit prepared local Activity policy requires candidate protocol 1.20, +the installed bridge's `supportedLocalActivityCancellationPolicies()` list, +and `prepared_local_activity_cancellation_policies` on the original issued +workflow claim. Discovery reports the actual supported policy list in +`server_capabilities.prepared_local_activity_cancellation_policies`. Older +custom prepared bridges do not acquire this support from their optional role. +The list is empty at default protocol 1.19. + +`try_cancel` fences publication and releases the durable await without claiming +the callback has stopped. `wait_cancellation_completed` parks delivery until an +original-owner stopped-and-joined receipt or canonical callback outcome exists. +Both retain the original root request and immutable cleanup deadline. Omission +preserves historical TryCancel. Explicit null and local `abandon` are refused. +Independent work should use a remote Activity with a qualified bounded lifetime. + +The prepare, recover and atomic group endpoints reject unsupported policies +before resolving payloads or recording any sibling. Refusals identify the +requested policy, installed supported list, original worker claim, missing +capability and remediation. Re-registering a worker cannot upgrade its existing +claim. Replacement polling checks the explicit policy in canonical Scheduled +history before probing and again under the run lock. An incompatible worker +cannot reinterpret that policy through replay. + +SDK policy authoring, replay and physical callback supervision need separate +qualification. The installed backend's policy list does not establish those +SDK capabilities or change the default published protocol. Receipt writes are admitted during storage draining and refused when storage is fenced. SDK stop/join emission, local receipts and activity waiting policies still need connected qualification before this candidate is published. diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 475474c5..0eaaf13c 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "32" + version: "33" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -44,7 +44,11 @@ x-durable-workflow-cooperative-cancellation-contract: remote_activity_policy_default: try_cancel remote_activity_abandon_lifetime: original_finite_schedule_to_close_timeout unsupported_remote_activity_policy: { status: 409, reason: activity_cancellation_policy_not_supported, recorded: false } - local_activity_policy_authoring: refused_pending_portable_admission + local_activity_operation_policies: [try_cancel, wait_cancellation_completed] + local_activity_policy_admission: [server_protocol, request_protocol, immutable_worker_claim_capability, installed_runtime_local_policy_list] + local_activity_policy_worker_capability: prepared_local_activity_cancellation_policies + local_activity_policy_authoring: prepared_admission_only + unsupported_local_activity_policy: { status: 409, reason: prepared_local_activity_cancellation_policy_not_supported, recorded: false } local_activity_abandon: refused_pending_independent_callback_lifetime claim_proof: immutable_registration_snapshot_bound_to_task_run_owner_and_attempt request_against_incompatible_active_claim: { status: 409, reason: active_claim_cancellation_not_supported, history_appended: false } @@ -1380,6 +1384,11 @@ components: prepared_local_activity_groups: type: boolean description: Candidate atomic complete-all-group admission. Requires prepared local activities, protocol 1.20 and the actual bound PreparedLocalActivityGroupTaskBridge. Existing custom prepared bridges do not acquire this role. + prepared_local_activity_cancellation_policies: + type: array + uniqueItems: true + items: { type: string, enum: [try_cancel, wait_cancellation_completed] } + description: Explicit policies supported by the actual installed prepared-local bridge's optional supportedLocalActivityCancellationPolicies method. Empty at default protocol 1.19 or for older optional-role bridges. Workers need a new immutable claim with the separately named worker capability before admission or replay. supported_workflow_task_commands: type: array uniqueItems: true @@ -1555,6 +1564,9 @@ components: worker_sessions: "1.18" sticky_execution: "1.18" cooperative_cancellation: "1.20" + prepared_local_activities: "1.20" + prepared_local_activity_groups: "1.20" + prepared_local_activity_cancellation_policies: "1.20" capability_manifest: $ref: "#/components/schemas/PortableWorkerCapabilityManifest" WorkerHeartbeatRequest: @@ -1767,6 +1779,10 @@ components: - { type: object, required: [codec], properties: { codec: { const: avro } } } payload_codec: { type: string, const: avro } execution_mode: { type: string, const: local } + cancellation_policy: + type: string + enum: [try_cancel, wait_cancellation_completed] + description: Explicit candidate prepared-local policy. Requires the installed backend policy list and prepared_local_activity_cancellation_policies on the original issued claim. Omission retains historical TryCancel. Null and Abandon are refused before payload resolution or admission. start_to_close_timeout: { type: [integer, "null"], minimum: 1 } schedule_to_close_timeout: { type: [integer, "null"], minimum: 1 } heartbeat_timeout: { type: [integer, "null"], minimum: 1 } @@ -2463,7 +2479,7 @@ components: type: [string, "null"] enum: [try_cancel, wait_cancellation_completed, abandon, null] x-durable-workflow-minimum-protocol-version: "1.20" - description: Per-operation child or remote Activity cancellation policy. Absent or null retains child abandon and Activity try_cancel. Explicit remote policies require protocol 1.20, a capable immutable worker claim and an installed runtime implementation. Remote abandon requires a finite schedule_to_close_timeout. Explicit local Activity policies are not admitted. + description: Per-operation child, remote Activity or prepared local Activity cancellation policy. Absent or null retains child abandon and remote Activity try_cancel. Explicit remote policies require protocol 1.20, a capable immutable worker claim and an installed runtime implementation. Remote abandon requires a finite schedule_to_close_timeout. Prepared local descriptors accept only explicit TryCancel or WaitCancellationCompleted under their separate installed-backend and original-claim capability checks, including atomic group admission. Null and local Abandon are refused. allOf: - if: properties: diff --git a/tests/Source/PreparedLocalCancellationPolicyTest.php b/tests/Source/PreparedLocalCancellationPolicyTest.php new file mode 100644 index 00000000..2077c687 --- /dev/null +++ b/tests/Source/PreparedLocalCancellationPolicyTest.php @@ -0,0 +1,282 @@ + '1.20', + 'workflows.v2.workflow_task_lease_seconds' => 10, + 'workflows.v2.types.workflows' => ['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class], + ]); + WorkflowNamespace::query()->create(['name' => 'default', 'description' => 'Test', 'retention_days' => 30, 'status' => 'active']); + $this->assertSame(['try_cancel', 'wait_cancellation_completed'], PreparedLocalActivityPolicy::cancellationPolicies()); + } + + public static function policies(): array + { + return [['try_cancel'], ['wait_cancellation_completed']]; + } + + #[DataProvider('policies')] + public function test_explicit_policy_is_admitted_once_and_canonical_history_preserves_it(string $policy): void + { + $task = $this->claim(); + $descriptor = $this->descriptor($policy); + $first = $this->local($task, 'prepare', $descriptor)->assertOk()->assertJsonPath('prepared', true)->json(); + $before = WorkflowHistoryEvent::query()->count(); + $this->local($task, 'prepare', $descriptor)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('activity_attempt_id', $first['activity_attempt_id']); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertSame($policy, ActivityExecution::query()->sole()->activity_options['cancellation_policy']); + foreach ([HistoryEventType::ActivityScheduled, HistoryEventType::ActivityStarted] as $eventType) { + $event = WorkflowHistoryEvent::query()->where('event_type', $eventType)->sole(); + $this->assertSame('local', $event->payload['execution_mode']); + $this->assertSame($policy, $event->payload['activity']['cancellation_policy']); + } + $changed = $policy === 'try_cancel' ? 'wait_cancellation_completed' : 'try_cancel'; + $this->local($task, 'prepare', $this->descriptor($changed))->assertStatus(409)->assertJsonPath('reason', 'local_activity_preparation_mismatch'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_default_protocol_and_an_older_optional_role_do_not_advertise_policies(): void + { + config(['server.worker_protocol.version' => '1.19']); + $this->assertSame([], WorkerProtocol::serverCapabilities()['prepared_local_activity_cancellation_policies']); + config(['server.worker_protocol.version' => '1.20']); + $task = $this->claim(); + $older = \Mockery::mock(PreparedLocalActivityTaskBridge::class); + $this->app->instance(WorkflowTaskBridge::class, $older); + $this->assertTrue(PreparedLocalActivityPolicy::backendSupported()); + $this->assertSame([], WorkerProtocol::serverCapabilities()['prepared_local_activity_cancellation_policies']); + $this->local($task, 'prepare', $this->descriptor('try_cancel'))->assertStatus(409) + ->assertJsonPath('unavailable', ['installed_runtime_local_activity_cancellation_policy']); + $this->assertDatabaseCount('activity_executions', 0); + } + + #[DataProvider('policies')] + public function test_replay_requires_the_specific_policy_supported_by_the_installed_backend(string $policy): void + { + $task = $this->claim(); + $this->local($task, 'prepare', $this->descriptor($policy))->assertOk(); + $restricted = \Mockery::mock(PreparedLocalActivityTaskBridge::class.', '.LocalCancellationPolicyBridge::class); + $restricted->shouldReceive('supportedLocalActivityCancellationPolicies')->andReturn(['try_cancel']); + $this->app->instance(WorkflowTaskBridge::class, $restricted); + $this->assertSame(['try_cancel'], WorkerProtocol::serverCapabilities()['prepared_local_activity_cancellation_policies']); + $this->assertSame($policy === 'try_cancel', PreparedLocalActivityPolicy::canReplayRun($task['run_id'], $this->capabilities(), '1.20')); + $this->assertFalse(PreparedLocalActivityPolicy::canReplayRun($task['run_id'], $this->capabilities(false), '1.20')); + $this->local($task, 'recover', $this->descriptor('wait_cancellation_completed'))->assertStatus(409) + ->assertJsonPath('unavailable', ['installed_runtime_local_activity_cancellation_policy']); + } + + public function test_registration_requires_preparation_cooperation_and_protocol_1_20(): void + { + foreach ([['1.20', []], ['1.20', [CooperativeCancellationPolicy::CAPABILITY]], + ['1.20', [PreparedLocalActivityPolicy::CAPABILITY]], + ['1.19', [CooperativeCancellationPolicy::CAPABILITY, PreparedLocalActivityPolicy::CAPABILITY]]] as [$version, $capabilities]) { + $this->register('unsupported', [...$capabilities, PreparedLocalActivityPolicy::CANCELLATION_POLICIES_CAPABILITY], $version) + ->assertStatus(409)->assertJsonPath('registered', false) + ->assertJsonPath('reason', 'prepared_local_activity_cancellation_policy_capability_mismatch'); + } + $this->assertDatabaseCount('workflow_worker_registrations', 0); + } + + public function test_registration_changes_cannot_upgrade_an_existing_claim_or_resolve_its_payload(): void + { + $task = $this->claim(false); + $before = WorkflowHistoryEvent::query()->count(); + $this->register('original', $this->capabilities())->assertCreated(); + $descriptor = [...$this->descriptor('try_cancel'), 'arguments' => ['codec' => 'avro', 'invalid_reference' => true]]; + foreach (['prepare', 'recover'] as $operation) { + $this->local($task, $operation, $descriptor)->assertStatus(409) + ->assertJsonPath('reason', 'prepared_local_activity_cancellation_policy_not_supported') + ->assertJsonPath('worker_id', 'original')->assertJsonPath('requested_policy', 'try_cancel') + ->assertJsonPath('required_capability', PreparedLocalActivityPolicy::CANCELLATION_POLICIES_CAPABILITY) + ->assertJsonPath('unavailable', ['worker_claim_cancellation_policy_capability']); + } + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseCount('activity_executions', 0); + $this->assertDatabaseCount('activity_attempts', 0); + } + + public function test_omission_preserves_admission_for_an_older_prepared_claim(): void + { + $task = $this->claim(false); + $descriptor = $this->descriptor('try_cancel'); + unset($descriptor['cancellation_policy']); + $this->local($task, 'prepare', $descriptor)->assertOk()->assertJsonPath('prepared', true); + $this->assertArrayNotHasKey('cancellation_policy', ActivityExecution::query()->sole()->activity_options); + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $this->register('legacy', $this->capabilities(false))->assertCreated(); + $this->assertSame($task['task_id'], $this->poll('legacy')->assertOk()->json('task.task_id')); + } + + public static function invalidPolicies(): array + { + return [['abandon'], [null], ['unknown'], [true]]; + } + + #[DataProvider('invalidPolicies')] + public function test_invalid_or_abandoned_local_policy_is_refused_before_payload_and_admission(mixed $policy): void + { + $task = $this->claim(); + $before = WorkflowHistoryEvent::query()->count(); + $this->local($task, 'prepare', [...$this->descriptor($policy), 'schedule_to_close_timeout' => 30, + 'arguments' => ['codec' => 'avro', 'invalid_reference' => true]])->assertStatus(409) + ->assertJsonPath('recorded', false)->assertJsonPath('unavailable', ['local_activity_cancellation_policy']); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseCount('activity_executions', 0); + } + + #[DataProvider('policies')] + public function test_replacement_requires_the_recorded_policy_and_recovers_without_changing_its_lifetime(string $policy): void + { + $task = $this->claim(); + $descriptor = [...$this->descriptor($policy), 'schedule_to_close_timeout' => 60, + 'retry_policy' => ['max_attempts' => 2, 'backoff_seconds' => [2]]]; + $first = $this->local($task, 'prepare', $descriptor)->assertOk()->json(); + ActivityAttempt::query()->findOrFail($first['activity_attempt_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $this->register('legacy', $this->capabilities(false))->assertCreated(); + $history = WorkflowHistoryEvent::query()->count(); + $this->poll('legacy')->assertOk()->assertJsonPath('task', null); + // Poll maintenance records the expired lease's repair request even + // when this poller cannot receive or replay the resulting claim. + $this->assertSame($history + 1, WorkflowHistoryEvent::query()->count()); + $this->assertSame(HistoryEventType::RepairRequested, WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id'])->orderByDesc('sequence')->firstOrFail()->event_type); + $this->assertSame($task['workflow_task_attempt'], WorkflowTask::query()->findOrFail($task['task_id'])->attempt_count); + $this->register('replacement', $this->capabilities())->assertCreated(); + $replacement = $this->poll('replacement')->assertOk()->json('task'); + $this->assertSame($task['task_id'], $replacement['task_id']); + $this->assertSame($task['workflow_task_attempt'] + 1, $replacement['workflow_task_attempt']); + $recovered = $this->local($replacement, 'recover', $descriptor)->assertOk() + ->assertJsonPath('recovered', true)->assertJsonPath('claim_released', true)->assertJsonPath('callback_stop_state', 'unknown')->json(); + $this->local($replacement, 'recover', $descriptor)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('event_id', $recovered['event_id']); + $this->travel(2)->seconds(); + $retry = $this->poll('replacement')->assertOk()->json('task'); + $second = $this->local($retry, 'prepare', $descriptor, 'sdk-recovered')->assertOk()->assertJsonPath('attempt_number', 2)->json(); + $this->assertSame($first['schedule_to_close_deadline_at'], $second['schedule_to_close_deadline_at']); + $this->assertSame($policy, ActivityExecution::query()->sole()->activity_options['cancellation_policy']); + } + + #[DataProvider('policies')] + public function test_group_policy_is_checked_before_any_child_or_local_member_is_committed(string $policy): void + { + $task = $this->claim(false, true); + $before = WorkflowHistoryEvent::query()->count(); + $this->group($task, $policy)->assertStatus(409)->assertJsonPath('unavailable', ['worker_claim_cancellation_policy_capability']); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseCount('workflow_runs', 1); + $this->assertDatabaseCount('activity_executions', 0); + $this->assertArrayNotHasKey('portable_local_group_checkpoint', WorkflowTask::query()->findOrFail($task['task_id'])->payload); + } + + #[DataProvider('policies')] + public function test_capable_group_records_the_explicit_local_policy(string $policy): void + { + $task = $this->claim(true, true); + $this->group($task, $policy)->assertOk()->assertJsonPath('checkpointed', true); + $event = WorkflowHistoryEvent::query()->where('event_type', HistoryEventType::ActivityScheduled)->sole(); + $this->assertSame('local', $event->payload['execution_mode']); + $this->assertSame($policy, $event->payload['activity']['cancellation_policy']); + $this->assertDatabaseCount('workflow_runs', 2); + $this->assertDatabaseCount('activity_executions', 1); + } + + private function capabilities(bool $policy = true, bool $group = false): array + { + return [CooperativeCancellationPolicy::CAPABILITY, PreparedLocalActivityPolicy::CAPABILITY, + ...($policy ? [PreparedLocalActivityPolicy::CANCELLATION_POLICIES_CAPABILITY] : []), + ...($group ? [PreparedLocalActivityPolicy::GROUP_CAPABILITY] : [])]; + } + + private function claim(bool $policy = true, bool $group = false): array + { + $this->withHeaders(['X-Namespace' => 'default', 'X-Durable-Workflow-Control-Plane-Version' => '2'])->postJson('/api/workflows', [ + 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'local-policy', 'input' => ['Ada'], + ])->assertCreated(); + $this->register('original', $this->capabilities($policy, $group))->assertCreated(); + + return $this->poll('original')->assertOk()->json('task'); + } + + private function register(string $id, array $capabilities, string $version = '1.20'): TestResponse + { + return $this->withHeaders($this->headers($version))->postJson('/api/worker/register', [ + 'worker_id' => $id, 'task_queue' => 'local-policy', 'runtime' => 'php', + 'supported_workflow_types' => ['tests.external-greeting-workflow'], 'capabilities' => $capabilities, + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), 'max_concurrent_workflow_tasks' => 1, + 'process_metrics' => ['host' => 'test-worker', 'process_started_at' => now()->toISOString(), 'process_id' => $id === 'original' ? 10 : 20], + ]); + } + + private function poll(string $id): TestResponse + { + return $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', ['worker_id' => $id, 'task_queue' => 'local-policy']); + } + + private function descriptor(mixed $policy): array + { + return ['type' => 'record_local_activity', 'activity_type' => 'opaque-local', + 'arguments' => Serializer::serializeWithCodec('avro', ['Ada']), 'payload_codec' => 'avro', 'cancellation_policy' => $policy]; + } + + private function local(array $task, string $operation, array $descriptor, string $workerAttemptId = 'sdk-local-1'): TestResponse + { + return $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/{$operation}", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'sequence' => 1, 'worker_attempt_id' => $workerAttemptId, 'descriptor' => $descriptor, + ]); + } + + private function group(array $task, string $policy): TestResponse + { + return $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/checkpoint-group", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'checkpoint_id' => 'policy-group', 'start_sequence' => 1, 'commands' => [[ + 'type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', + 'arguments' => Serializer::serializeWithCodec('avro', ['child']), 'payload_codec' => 'avro', + ...ParallelChildGroup::itemMetadata(1, 2, 0, 'mixed'), + ], [...$this->descriptor($policy), 'type' => 'prepare_local_activity', + ...ParallelChildGroup::itemMetadata(1, 2, 1, 'mixed')]], + ]); + } + + private function headers(string $version = '1.20'): array + { + return ['X-Namespace' => 'default', WorkerProtocol::HEADER => $version]; + } +} + +interface LocalCancellationPolicyBridge +{ + public function supportedLocalActivityCancellationPolicies(): array; +} From ccede6dc298beaf9310d21627a477088fb080583 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 3 Oct 2026 06:06:29 +0000 Subject: [PATCH 30/57] Preserve scope admission and commit streams only for accepted tasks --- .github/workflows/phpunit-feature.yml | 2 +- app/Http/Controllers/Api/WorkerController.php | 70 ++++- .../WorkflowStreamCommandProcessor.php | 18 +- .../worker-protocol-api.openapi.yaml | 17 +- tests/Feature/WorkflowStreamsTest.php | 85 ++++++ .../Source/CancellationScopeAdmissionTest.php | 250 ++++++++++++++++++ 6 files changed, 433 insertions(+), 9 deletions(-) create mode 100644 tests/Source/CancellationScopeAdmissionTest.php diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index ce5ccb71..b1a0405f 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -340,7 +340,7 @@ jobs: composer install --no-interaction --no-progress --prefer-dist mv vendor/durable-workflow/workflow /tmp/locked-workflow-package cp -a prepared-workflow-source vendor/durable-workflow/workflow - php vendor/bin/phpunit tests/Feature/PreparedLocalActivityProtocolTest.php tests/Feature/HistoryRetentionTest.php tests/Unit/WorkerPollFenceTest.php tests/Source/CancellationCascadeDiagnosticsTest.php tests/Source/PreparedLocalCancellationPolicyTest.php tests/Feature/WorkflowDebugTest.php --fail-on-skipped --log-junit "/evidence/$DW_PREPARED_DATABASE.xml" --colors=never + php vendor/bin/phpunit tests/Feature/PreparedLocalActivityProtocolTest.php tests/Feature/HistoryRetentionTest.php tests/Unit/WorkerPollFenceTest.php tests/Source/CancellationCascadeDiagnosticsTest.php tests/Source/PreparedLocalCancellationPolicyTest.php tests/Source/CancellationScopeAdmissionTest.php tests/Feature/WorkflowStreamsTest.php tests/Feature/WorkflowDebugTest.php --fail-on-skipped --log-junit "/evidence/$DW_PREPARED_DATABASE.xml" --colors=never if [ "$DW_PREPARED_DATABASE" = sqlite ]; then php vendor/bin/phpunit tests/Feature/CooperativeCancellationProtocolTest.php tests/Feature/WorkflowWorkerProtocolTest.php tests/Feature/ActivityWorkerProtocolTest.php tests/Feature/SearchAttributeValueValidationTest.php tests/Feature/NamespaceDurableStateQuotaTest.php tests/Feature/RuntimeExternalPayloadTransportTest.php tests/Feature/RuntimePayloadCompletionUploadsTest.php tests/Unit/RuntimePayloadCompletionContextTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php --log-junit /evidence/affected-regression.xml --colors=never fi diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index 1c4e00b4..d107ccc8 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -54,6 +54,7 @@ use Illuminate\Support\Facades\Validator; use Illuminate\Support\Str; use Illuminate\Validation\ValidationException; +use Workflow\V2\Contracts\CancellationScopeAdmission; use Workflow\V2\Contracts\HistoryProjectionRole; use Workflow\V2\Contracts\PreparedLocalActivityGroupTaskBridge; use Workflow\V2\Contracts\PreparedLocalActivityTaskBridge; @@ -75,6 +76,7 @@ use Workflow\V2\Support\StickyExecution; use Workflow\V2\Support\WorkerProtocolVersion; use Workflow\V2\Support\WorkflowCommandNormalizer; +use Workflow\V2\Support\WorkflowStepHistory; use Workflow\V2\Support\WorkflowTaskOwnership; class WorkerController @@ -1552,6 +1554,7 @@ private function mutateWorkflowTaskCommands(Request $request, string $taskId, bo 'workflow_task_attempt' => ['required', 'integer', 'min:1'], 'commands' => ['required', 'array', 'min:1'], 'commands.*.type' => ['required', 'string'], + 'commands.*.cancellation_scope_id' => ['sometimes', 'string', 'min:1', 'max:255'], 'commands.*.result' => ['nullable'], 'commands.*.activity_type' => ['nullable', 'string'], 'commands.*.arguments' => ['nullable'], @@ -1760,6 +1763,11 @@ private function mutateWorkflowTaskCommands(Request $request, string $taskId, bo return $response; } + if ($response = $this->guardCancellationScopeAdmission($request, (string) $namespace, $taskId, + (int) $validated['workflow_task_attempt'], $commands)) { + return $response; + } + if ($response = $this->guardRemoteActivityCancellationPoliciesAvailable( $request, (string) $namespace, @@ -1985,12 +1993,13 @@ function () use ( } $this->authorizeServiceOperationReplays($request, (string) $namespace, $taskId, $commands); + if ($response = $this->guardCancellationScopeAdmission($request, (string) $namespace, $taskId, + (int) $validated['workflow_task_attempt'], $commands, true)) { + return $response; + } $commands = $this->canonicalizeWorkflowStreamPayloadCodecs($commands); - $commands = app(WorkflowStreamCommandProcessor::class)->process( - $taskId, - (string) $namespace, - $commands, - ); + $streamCommands = $commands; + $commands = app(WorkflowStreamCommandProcessor::class)->withoutDirectives($commands); $commands = $group ? $this->normalizePreparedLocalGroupCommands($commands, WorkerProtocol::requestVersion($request)) : WorkflowCommandNormalizer::normalize($commands, WorkerProtocol::requestVersion($request)); if ($checkpoint) { @@ -2012,6 +2021,9 @@ function () use ( } else { $outcome = $bridge->complete($taskId, $commands); } + if (($outcome[$checkpoint ? 'checkpointed' : 'completed'] ?? false) === true) { + app(WorkflowStreamCommandProcessor::class)->process($taskId, (string) $namespace, $streamCommands); + } if (($outcome['completed'] ?? false) === true && $claimedTask instanceof WorkflowTask && CooperativeCancellationPolicy::claimSupportsCancellation($claimedTask)) { @@ -2885,6 +2897,54 @@ private function guardConditionWaitOccurrenceIdentityAvailable( ], 409); } + /** @param list> $commands */ + private function guardCancellationScopeAdmission( + Request $request, + string $namespace, + string $taskId, + int $workflowTaskAttempt, + array $commands, + bool $lock = false, + ): ?JsonResponse { + if (! collect($commands)->contains(static fn (array $command): bool => array_key_exists('cancellation_scope_id', $command))) { + return null; + } + $reason = null; + $unavailable = []; + if (! WorkerProtocol::versionMeetsMinimum(WorkerProtocol::requestVersion($request), '1.20')) { + $reason = 'cancellation_scope_membership_unavailable'; + $unavailable[] = 'request_protocol'; + } + $bridge = app(WorkflowTaskBridge::class); + if (! $bridge instanceof CancellationScopeAdmission) { + $reason = 'cancellation_scope_membership_unavailable'; + $unavailable[] = 'installed_runtime_scope_admission'; + } + foreach ($commands as $command) { + if (array_key_exists('cancellation_scope_id', $command) + && ! in_array($command['type'], ['schedule_activity', 'start_timer', 'start_child_workflow', 'prepare_local_activity'], true)) { + $reason = 'invalid_cancellation_scope_command'; + } + } + if ($reason === null) { + $task = WorkflowTask::query()->whereKey($taskId)->where('namespace', $namespace)->first(); + $runQuery = WorkflowRun::query()->whereKey($task?->workflow_run_id)->where('namespace', $namespace); + $run = ($lock ? $runQuery->lockForUpdate() : $runQuery)->first(); + if ($run instanceof WorkflowRun) { + $reason = $bridge->validateCancellationScopeMembership($run, $commands, WorkflowStepHistory::nextDurableCommandSequence($run)); + } + } + if ($reason === null) { + return null; + } + + return WorkerProtocol::json([ + 'task_id' => $taskId, 'workflow_task_attempt' => $workflowTaskAttempt, 'outcome' => 'rejected', + 'recorded' => false, 'reason' => $reason, 'unavailable' => $unavailable, + 'requested_version' => WorkerProtocol::requestVersion($request), 'minimum_protocol_version' => '1.20', + ], 409); + } + /** * @param list> $commands */ diff --git a/app/Support/WorkflowStreamCommandProcessor.php b/app/Support/WorkflowStreamCommandProcessor.php index 9c4d88ed..bf10a0c9 100644 --- a/app/Support/WorkflowStreamCommandProcessor.php +++ b/app/Support/WorkflowStreamCommandProcessor.php @@ -7,11 +7,12 @@ use Workflow\V2\Models\WorkflowTask; /** - * Applies replay-safe Workflow Stream directives before task completion. + * Applies replay-safe Workflow Stream directives after successful admission. * * The directive rides a record_side_effect command so the append/close and * SideEffectRecorded event commit in one outer database transaction. The - * directive is stripped before the package command normalizer sees it. + * directive is stripped before the package command normalizer sees it. Effects + * are applied only after admission succeeds, inside the same outer transaction. */ final class WorkflowStreamCommandProcessor { @@ -19,6 +20,19 @@ public function __construct( private readonly WorkflowStreamService $streams, ) {} + /** + * @param list> $commands + * @return list> + */ + public function withoutDirectives(array $commands): array + { + foreach ($commands as &$command) { + unset($command['workflow_stream']); + } + + return $commands; + } + /** * @param list> $commands * @return list> diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 0eaaf13c..ce973580 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "33" + version: "34" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -23,6 +23,15 @@ x-durable-workflow-catalog-entry: worker_protocol_api x-durable-workflow-catalog-schema: durable-workflow.v2.platform-protocol-specs.catalog x-durable-workflow-catalog-version: 16 x-durable-workflow-evolution-rule: additive_minor_breaking_major +x-durable-workflow-cancellation-scope-admission: + minimum_protocol_version: "1.20" + status: unfrozen_candidate_membership_only + required_backend_role: CancellationScopeAdmission + scope_execution_capability: false + checks: [before_payload_resolution, under_run_lock_before_admission] + unknown_scope: { status: 409, reason: operation_scope_not_recorded, recorded: false } + unsupported_backend: { status: 409, reason: cancellation_scope_membership_unavailable, recorded: false } + stream_effects: successful_admission_only_in_same_outer_transaction x-durable-workflow-cooperative-cancellation-contract: minimum_protocol_version: "1.20" worker_capability: cooperative_cancellation @@ -2472,6 +2481,12 @@ components: required: [type] properties: type: { type: string } + cancellation_scope_id: + type: string + minLength: 1 + maxLength: 255 + x-durable-workflow-minimum-protocol-version: "1.20" + description: Optional exact-run canonical membership for schedule_activity, start_timer, start_child_workflow and prepare_local_activity commands in the unfrozen candidate. Requires an installed CancellationScopeAdmission backend. The named scope must already be recorded before admission. Invalid membership is refused before payload resolution and rechecked under the run lock. Omission preserves historical root membership. This field does not advertise scope execution or supervision. parent_close_policy: type: [string, "null"] description: Child parent-close policy. request_cancel retains legacy terminal cancellation. request_cancellation requests cooperative cleanup under protocol 1.20 and the original shared budget. diff --git a/tests/Feature/WorkflowStreamsTest.php b/tests/Feature/WorkflowStreamsTest.php index 531d77aa..0ec0f85f 100644 --- a/tests/Feature/WorkflowStreamsTest.php +++ b/tests/Feature/WorkflowStreamsTest.php @@ -553,6 +553,91 @@ public function test_worker_completion_commits_stream_append_with_side_effect_hi ->count()); } + public function test_timed_out_completion_records_timeout_without_committing_stream_output(): void + { + $task = $this->claimStreamCompletionTask(1); + $this->travel(2)->seconds(); + $this->withHeaders($this->workerHeaders())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [$this->streamCompletionCommand($task)], + ])->assertConflict()->assertJsonPath('recorded', false)->assertJsonPath('reason', 'run_timed_out'); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + $this->assertDatabaseCount('workflow_durable_streams', 0); + $this->assertDatabaseHas('workflow_history_events', ['workflow_run_id' => $task['run_id'], 'event_type' => 'WorkflowTimedOut']); + $this->assertDatabaseMissing('workflow_history_events', ['workflow_run_id' => $task['run_id'], 'event_type' => 'SideEffectRecorded']); + $this->assertSame('failed', WorkflowRun::query()->findOrFail($task['run_id'])->status->value); + } + + public function test_terminal_completion_commits_its_last_stream_item_and_close(): void + { + $task = $this->claimStreamCompletionTask(); + $close = $this->streamCompletionCommand($task); + $close['workflow_stream'] = [...$close['workflow_stream'], 'operation' => 'close', 'command_ordinal' => 1]; + unset($close['workflow_stream']['items']); + $this->withHeaders($this->workerHeaders())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [$this->streamCompletionCommand($task), $close, ['type' => 'complete_workflow', 'result' => Serializer::serializeWithCodec('avro', 'done')]], + ])->assertOk()->assertJsonPath('recorded', true); + $this->assertDatabaseCount('workflow_durable_stream_items', 1); + $this->assertSame('closed', WorkflowDurableStream::query()->sole()->status); + $this->assertSame('completed', WorkflowRun::query()->findOrFail($task['run_id'])->status->value); + } + + public function test_legacy_request_refuses_explicit_scope_before_stream_output(): void + { + $task = $this->claimStreamCompletionTask(); + $this->withHeaders($this->workerHeaders())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [$this->streamCompletionCommand($task), ['type' => 'start_timer', 'delay_seconds' => 10, 'cancellation_scope_id' => 'root']], + ])->assertConflict()->assertJsonPath('recorded', false)->assertJsonPath('reason', 'cancellation_scope_membership_unavailable'); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + $this->assertDatabaseCount('workflow_run_timers', 0); + } + + public function test_stream_failure_rolls_back_successful_native_admission(): void + { + $task = $this->claimStreamCompletionTask(); + $before = WorkflowRun::query()->findOrFail($task['run_id'])->status; + $command = $this->streamCompletionCommand($task); + $command['workflow_stream']['max_pending_items'] = 1; + $second = $command['workflow_stream']['items'][0]; + $second['idempotency_key'] = substr($second['idempotency_key'], 0, -1).'1'; + $command['workflow_stream']['items'][] = $second; + $this->withHeaders($this->workerHeaders())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [$command, ['type' => 'complete_workflow', 'result' => Serializer::serializeWithCodec('avro', 'done')]], + ])->assertStatus(429); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + $this->assertDatabaseMissing('workflow_history_events', ['workflow_run_id' => $task['run_id'], 'event_type' => 'SideEffectRecorded']); + $this->assertSame($before, WorkflowRun::query()->findOrFail($task['run_id'])->status); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + } + + private function claimStreamCompletionTask(?int $timeout = null): array + { + config()->set('workflows.v2.types.workflows', ['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class]); + $this->withHeaders($this->apiHeaders())->postJson('/api/workflows', [ + 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'stream-command-queue', + 'input' => ['codec' => 'avro', 'blob' => Serializer::serializeWithCodec('avro', ['Ada'])], + ...($timeout === null ? [] : ['run_timeout_seconds' => $timeout]), + ])->assertCreated(); + $this->registerWorker('stream-command-worker', 'stream-command-queue', supportedWorkflowTypes: ['tests.external-greeting-workflow']); + + return $this->withHeaders($this->workerHeaders())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'stream-command-worker', 'task_queue' => 'stream-command-queue', + ])->assertOk()->json('task'); + } + + private function streamCompletionCommand(array $task): array + { + $identity = $task['workflow_command_id'] ?: $task['task_id']; + + return ['type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', null), + 'workflow_stream' => ['operation' => 'append', 'stream_name' => 'tokens', 'command_identity' => $identity, + 'command_ordinal' => 0, 'items' => [['payload' => Serializer::serializeWithCodec('avro', ['token']), + 'payload_codec' => 'avro', 'idempotency_key' => "dw-stream:{$identity}:0:0"]]]]; + } + /** * @param list> $items */ diff --git a/tests/Source/CancellationScopeAdmissionTest.php b/tests/Source/CancellationScopeAdmissionTest.php new file mode 100644 index 00000000..5f8ef7c3 --- /dev/null +++ b/tests/Source/CancellationScopeAdmissionTest.php @@ -0,0 +1,250 @@ + '1.20', + 'workflows.v2.workflow_task_lease_seconds' => 10, + 'workflows.v2.types.workflows' => ['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class], + ]); + WorkflowNamespace::query()->create(['name' => 'default', 'retention_days' => 30, 'status' => 'active']); + $this->assertTrue(class_exists(CancellationScopeHistory::class)); + } + + public static function operations(): array + { + return [['schedule_activity'], ['start_timer'], ['start_child_workflow']]; + } + + #[DataProvider('operations')] + public function test_unknown_scope_is_refused_without_publishing_a_stream_item(string $type): void + { + $task = $this->claim(); + $before = WorkflowHistoryEvent::query()->count(); + $this->complete($task, 'unknown-scope', $type)->assertStatus(409) + ->assertJsonPath('recorded', false)->assertJsonPath('reason', 'operation_scope_not_recorded'); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + $this->assertDatabaseCount('workflow_durable_streams', 0); + $this->assertDatabaseCount('activity_executions', 0); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + #[DataProvider('operations')] + public function test_recorded_scope_reaches_durable_history_and_commits_the_stream(string $type): void + { + $task = $this->claim(); + $run = WorkflowRun::query()->findOrFail($task['run_id']); + $scope = CancellationScopeHistory::open($run, WorkflowTask::query()->findOrFail($task['task_id']), 1, '1.20'); + $scopeId = $scope->payload['scope_id']; + $this->complete($task, $scopeId, $type)->assertOk()->assertJsonPath('recorded', true); + $this->assertDatabaseCount('workflow_durable_stream_items', 1); + $eventType = match ($type) { + 'schedule_activity' => 'ActivityScheduled', 'start_timer' => 'TimerScheduled', 'start_child_workflow' => 'ChildWorkflowScheduled', + }; + $event = WorkflowHistoryEvent::query()->where('workflow_run_id', $run->id)->where('event_type', $eventType)->sole(); + $this->assertSame($scopeId, $type === 'schedule_activity' ? $event->payload['activity']['cancellation_scope_id'] : $event->payload['cancellation_scope_id']); + } + + public function test_invalid_scope_is_refused_before_resolving_a_payload(): void + { + $task = $this->claim(); + $this->complete($task, 'unknown-scope', arguments: ['codec' => 'avro', 'invalid_reference' => true]) + ->assertStatus(409)->assertJsonPath('reason', 'operation_scope_not_recorded'); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + $this->assertDatabaseCount('activity_executions', 0); + } + + public function test_a_runtime_without_the_optional_admission_role_cannot_silently_drop_membership(): void + { + $task = $this->claim(); + $current = app(WorkflowTaskBridge::class); + $older = \Mockery::mock(WorkflowTaskBridge::class); + $older->shouldReceive('status')->andReturnUsing(fn (string $id) => $current->status($id)); + $older->shouldNotReceive('complete'); + $this->app->instance(WorkflowTaskBridge::class, $older); + $this->complete($task, 'root')->assertStatus(409)->assertJsonPath('reason', 'cancellation_scope_membership_unavailable') + ->assertJsonPath('unavailable', ['installed_runtime_scope_admission']); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + } + + public function test_a_legacy_request_cannot_silently_drop_membership(): void + { + $task = $this->claim(); + $this->complete($task, 'root', version: '1.19')->assertStatus(409) + ->assertJsonPath('reason', 'cancellation_scope_membership_unavailable')->assertJsonPath('unavailable', ['request_protocol']); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + } + + public static function malformedScopes(): array + { + return [[null], [''], [true], [str_repeat('x', 256)]]; + } + + #[DataProvider('malformedScopes')] + public function test_malformed_membership_is_refused_before_effects(mixed $scope): void + { + $task = $this->claim(); + $this->complete($task, $scope)->assertStatus(422); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + $this->assertDatabaseCount('activity_executions', 0); + } + + public function test_scope_on_an_unrelated_command_is_refused_before_effects(): void + { + $task = $this->claim(); + $this->complete($task, 'root', 'record_side_effect')->assertStatus(409)->assertJsonPath('reason', 'invalid_cancellation_scope_command'); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + } + + public function test_admission_is_rechecked_under_the_run_lock_before_stream_effects(): void + { + $task = $this->claim(); + $current = app(WorkflowTaskBridge::class); + $bridge = \Mockery::mock(WorkflowTaskBridge::class.', '.CancellationScopeAdmission::class); + $bridge->shouldReceive('status')->andReturnUsing(fn (string $id) => $current->status($id)); + $bridge->shouldReceive('validateCancellationScopeMembership')->twice()->andReturn(null, 'operation_scope_not_recorded'); + $bridge->shouldNotReceive('complete'); + $this->app->instance(WorkflowTaskBridge::class, $bridge); + $this->complete($task, 'root')->assertStatus(409)->assertJsonPath('reason', 'operation_scope_not_recorded'); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + $this->assertDatabaseCount('activity_executions', 0); + } + + public static function groupScopes(): array + { + return [[true], [false]]; + } + + #[DataProvider('groupScopes')] + public function test_retained_prefix_rejects_unknown_scope_or_commits_stream_once(bool $valid): void + { + $task = $this->claim(); + $run = WorkflowRun::query()->findOrFail($task['run_id']); + $scope = CancellationScopeHistory::open($run, WorkflowTask::query()->findOrFail($task['task_id']), 1, '1.20')->payload['scope_id']; + $body = ['lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'checkpoint_id' => 'scope-prefix', 'start_sequence' => 2, + 'commands' => $this->commandBatch($task, $valid ? $scope : 'unknown-scope', 'start_timer')]; + $route = "/api/worker/workflow-tasks/{$task['task_id']}/local-activities/checkpoint"; + $response = $this->withHeaders($this->headers())->postJson($route, $body); + if ($valid) { + $response->assertOk()->assertJsonPath('checkpointed', true); + $before = WorkflowHistoryEvent::query()->count(); + $this->withHeaders($this->headers())->postJson($route, $body)->assertOk()->assertJsonPath('duplicate', true); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseCount('workflow_durable_stream_items', 1); + } else { + $response->assertStatus(409)->assertJsonPath('reason', 'operation_scope_not_recorded'); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + $this->assertDatabaseCount('workflow_run_timers', 0); + } + } + + #[DataProvider('groupScopes')] + public function test_atomic_group_preflights_local_membership_before_any_sibling(bool $valid): void + { + $task = $this->claim(); + $run = WorkflowRun::query()->findOrFail($task['run_id']); + $scope = CancellationScopeHistory::open($run, WorkflowTask::query()->findOrFail($task['task_id']), 1, '1.20')->payload['scope_id']; + $before = WorkflowHistoryEvent::query()->count(); + $response = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/checkpoint-group", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'checkpoint_id' => 'scope-group', 'start_sequence' => 2, 'commands' => [[ + 'type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', + 'arguments' => Serializer::serializeWithCodec('avro', ['child']), 'payload_codec' => 'avro', 'cancellation_scope_id' => $scope, + ...ParallelChildGroup::itemMetadata(2, 2, 0, 'mixed'), + ], [ + 'type' => 'prepare_local_activity', 'activity_type' => 'opaque-local', 'payload_codec' => 'avro', + 'arguments' => $valid ? Serializer::serializeWithCodec('avro', ['Ada']) : ['codec' => 'avro', 'invalid_reference' => true], + 'cancellation_scope_id' => $valid ? $scope : 'unknown-scope', + ...ParallelChildGroup::itemMetadata(2, 2, 1, 'mixed'), + ]], + ]); + if ($valid) { + $response->assertOk()->assertJsonPath('checkpointed', true); + $this->assertSame($scope, ActivityExecution::query()->sole()->activity_options['cancellation_scope_id']); + } else { + $response->assertStatus(409)->assertJsonPath('reason', 'local_activity_scope_not_recorded'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseCount('activity_executions', 0); + $this->assertDatabaseCount('workflow_child_calls', 0); + } + } + + private function claim(): array + { + $this->withHeaders(['X-Namespace' => 'default', 'X-Durable-Workflow-Control-Plane-Version' => '2']) + ->postJson('/api/workflows', ['workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'scope-admission', 'input' => ['Ada']])->assertCreated(); + $this->withHeaders($this->headers())->postJson('/api/worker/register', [ + 'worker_id' => 'scope-worker', 'task_queue' => 'scope-admission', 'runtime' => 'php', + 'supported_workflow_types' => ['tests.external-greeting-workflow'], + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY, PreparedLocalActivityPolicy::CAPABILITY, PreparedLocalActivityPolicy::GROUP_CAPABILITY], + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), 'max_concurrent_workflow_tasks' => 1, + 'process_metrics' => ['host' => 'test-worker', 'process_started_at' => now()->toISOString(), 'process_id' => 10], + ])->assertCreated(); + + return $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'scope-worker', 'task_queue' => 'scope-admission', + ])->assertOk()->json('task'); + } + + private function complete(array $task, mixed $scopeId, string $type = 'schedule_activity', mixed $arguments = null, string $version = '1.20') + { + return $this->withHeaders(['X-Namespace' => 'default', WorkerProtocol::HEADER => $version])->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => $this->commandBatch($task, $scopeId, $type, $arguments), + ]); + } + + private function commandBatch(array $task, mixed $scopeId, string $type, mixed $arguments = null): array + { + $identity = $task['workflow_command_id'] ?: $task['task_id']; + $operation = match ($type) { + 'schedule_activity' => ['activity_type' => 'opaque-activity', 'arguments' => $arguments ?? Serializer::serializeWithCodec('avro', ['Ada']), 'payload_codec' => 'avro'], + 'start_timer' => ['delay_seconds' => 60], + 'start_child_workflow' => ['workflow_type' => 'tests.external-greeting-workflow', 'arguments' => Serializer::serializeWithCodec('avro', ['child']), 'payload_codec' => 'avro'], + 'record_side_effect' => ['result' => Serializer::serializeWithCodec('avro', null)], + }; + + return [[ + 'type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', null), + 'workflow_stream' => ['operation' => 'append', 'stream_name' => 'tokens', 'command_identity' => $identity, + 'command_ordinal' => 0, 'items' => [['payload' => Serializer::serializeWithCodec('avro', ['token']), + 'payload_codec' => 'avro', 'idempotency_key' => "dw-stream:{$identity}:0:0"]]], + ], [ + 'type' => $type, ...$operation, 'cancellation_scope_id' => $scopeId, + ]]; + } + + private function headers(): array + { + return ['X-Namespace' => 'default', WorkerProtocol::HEADER => '1.20']; + } +} From 49ab258db7aee18730abe5e0bb2883d9f839dd02 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 3 Oct 2026 07:14:44 +0000 Subject: [PATCH 31/57] Preflight scoped single local activity preparation and recovery --- .../Api/PreparedLocalActivityController.php | 42 +++++ .../worker-protocol-api.openapi.yaml | 10 +- .../Source/CancellationScopeAdmissionTest.php | 145 +++++++++++++++++- 3 files changed, 193 insertions(+), 4 deletions(-) diff --git a/app/Http/Controllers/Api/PreparedLocalActivityController.php b/app/Http/Controllers/Api/PreparedLocalActivityController.php index 7faf24dc..2239fc38 100644 --- a/app/Http/Controllers/Api/PreparedLocalActivityController.php +++ b/app/Http/Controllers/Api/PreparedLocalActivityController.php @@ -21,12 +21,14 @@ use Illuminate\Http\Request; use Illuminate\Support\Facades\DB; use Illuminate\Validation\ValidationException; +use Workflow\V2\Contracts\CancellationScopeAdmission; use Workflow\V2\Contracts\PreparedLocalActivityTaskBridge; use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Exceptions\ExternalPayloadIntegrityException; use Workflow\V2\Exceptions\StructuralLimitExceededException; use Workflow\V2\Models\ActivityAttempt; use Workflow\V2\Models\WorkflowHistoryEvent; +use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; final class PreparedLocalActivityController @@ -57,6 +59,11 @@ private function admit(Request $request, string $taskId, bool $recover): JsonRes 'worker_attempt_id' => [$recover ? 'nullable' : 'required', 'string', 'max:255'], 'descriptor' => ['required', 'array'], ]); + // Validate membership separately so Laravel retains the complete + // opaque descriptor for Native's canonical descriptor validation. + $request->validate([ + 'descriptor.cancellation_scope_id' => ['sometimes', 'required', 'string', 'max:255'], + ]); $namespace = (string) $request->attributes->get('namespace'); $task = NamespaceWorkflowScope::task($namespace, $taskId); if ($task === null) { @@ -71,6 +78,9 @@ private function admit(Request $request, string $taskId, bool $recover): JsonRes return self::refused($validated['lease_owner'], ['worker_claim_capability']); } try { + if ($response = $this->cancellationScopeRefusal($request, $task, $validated['descriptor'], (int) $validated['sequence'])) { + return $response; + } $validated['descriptor'] = $this->resolvePayload($validated['descriptor'], 'arguments', 'descriptor', $namespace); $reply = $this->mutations->run(fn (): array|JsonResponse => DB::transaction(function () use ($request, $namespace, $taskId, $validated, $claim, $recover): array|JsonResponse { $quotaSnapshot = $this->quota->snapshotForMutation($namespace, self::quotaResources()); @@ -118,6 +128,38 @@ private function admit(Request $request, string $taskId, bool $recover): JsonRes return $this->reply($request, $reply); } + private function cancellationScopeRefusal(Request $request, WorkflowTask $task, array $descriptor, int $sequence): ?JsonResponse + { + if (! array_key_exists('cancellation_scope_id', $descriptor)) { + return null; + } + $bridge = app(WorkflowTaskBridge::class); + $unavailable = []; + if (! $bridge instanceof CancellationScopeAdmission) { + $reason = 'cancellation_scope_membership_unavailable'; + $unavailable[] = 'installed_runtime_scope_admission'; + } else { + $run = WorkflowRun::query()->whereKey($task->workflow_run_id)->where('namespace', $task->namespace)->first(); + if (! $run instanceof WorkflowRun) { + return WorkerProtocol::json(['reason' => 'task_not_found'], 404); + } + // Payload resolution follows this preflight. Native rechecks + // canonical membership under its attempt/execution/run/task locks. + $reason = $bridge->validateCancellationScopeMembership($run, [[ + 'type' => 'prepare_local_activity', 'cancellation_scope_id' => $descriptor['cancellation_scope_id'], + ]], $sequence); + } + if ($reason === null) { + return null; + } + + return WorkerProtocol::json([ + 'task_id' => $task->id, 'workflow_task_attempt' => (int) $request->input('workflow_task_attempt'), + 'outcome' => 'rejected', 'recorded' => false, 'reason' => $reason, 'unavailable' => $unavailable, + 'requested_version' => WorkerProtocol::requestVersion($request), 'minimum_protocol_version' => '1.20', + ], 409); + } + public function control(Request $request, string $taskId, string $attemptId): JsonResponse { return $this->attemptOperation($request, $taskId, $attemptId, 'control'); diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index ce973580..9d2a7c6b 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "34" + version: "35" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -30,6 +30,8 @@ x-durable-workflow-cancellation-scope-admission: scope_execution_capability: false checks: [before_payload_resolution, under_run_lock_before_admission] unknown_scope: { status: 409, reason: operation_scope_not_recorded, recorded: false } + unknown_local_scope: { status: 409, reason: local_activity_scope_not_recorded, recorded: false } + prepared_local_surfaces: [single_prepare, single_recover, atomic_group_checkpoint] unsupported_backend: { status: 409, reason: cancellation_scope_membership_unavailable, recorded: false } stream_effects: successful_admission_only_in_same_outer_transaction x-durable-workflow-cooperative-cancellation-contract: @@ -1788,6 +1790,12 @@ components: - { type: object, required: [codec], properties: { codec: { const: avro } } } payload_codec: { type: string, const: avro } execution_mode: { type: string, const: local } + cancellation_scope_id: + type: string + minLength: 1 + maxLength: 255 + x-durable-workflow-minimum-protocol-version: "1.20" + description: Optional exact-run canonical scope for prepared local admission and recovery. Requires the installed CancellationScopeAdmission backend. The scope must be recorded before this descriptor's sequence. Syntax and membership are checked before payload resolution, and Native validates canonical history under its existing admission locks. Omission retains root membership. This does not advertise scope execution or supervision. cancellation_policy: type: string enum: [try_cancel, wait_cancellation_completed] diff --git a/tests/Source/CancellationScopeAdmissionTest.php b/tests/Source/CancellationScopeAdmissionTest.php index 5f8ef7c3..7b27206b 100644 --- a/tests/Source/CancellationScopeAdmissionTest.php +++ b/tests/Source/CancellationScopeAdmissionTest.php @@ -4,6 +4,7 @@ use App\Models\WorkflowNamespace; use App\Support\CooperativeCancellationPolicy; +use App\Support\NamespaceExternalPayloadStorage; use App\Support\PreparedLocalActivityPolicy; use App\Support\WorkerProtocol; use Illuminate\Foundation\Testing\RefreshDatabase; @@ -13,7 +14,9 @@ use Tests\TestCase; use Workflow\Serializers\Serializer; use Workflow\V2\Contracts\CancellationScopeAdmission; +use Workflow\V2\Contracts\PreparedLocalActivityTaskBridge; use Workflow\V2\Contracts\WorkflowTaskBridge; +use Workflow\V2\Models\ActivityAttempt; use Workflow\V2\Models\ActivityExecution; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowRun; @@ -198,12 +201,148 @@ public function test_atomic_group_preflights_local_membership_before_any_sibling } } - private function claim(): array + public static function singleLocalOperations(): array + { + return [['prepare'], ['recover']]; + } + + #[DataProvider('singleLocalOperations')] + public function test_single_local_rejects_unknown_scope_before_payload_resolution(string $operation): void + { + $task = $this->claim(); + $storage = \Mockery::mock(NamespaceExternalPayloadStorage::class); + $storage->shouldNotReceive('driverFor'); + $this->app->instance(NamespaceExternalPayloadStorage::class, $storage); + $before = WorkflowHistoryEvent::query()->count(); + $this->singleLocal($task, $operation, 'unknown-scope', arguments: ['codec' => 'avro', 'invalid_reference' => true]) + ->assertConflict()->assertJsonPath('recorded', false)->assertJsonPath('reason', 'local_activity_scope_not_recorded'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseCount('activity_executions', 0); + $this->assertDatabaseCount('activity_attempts', 0); + } + + public static function malformedSingleLocalScopes(): array + { + $cases = []; + foreach (self::singleLocalOperations() as [$operation]) { + foreach (self::malformedScopes() as [$scope]) { + $cases[] = [$operation, $scope]; + } + } + + return $cases; + } + + #[DataProvider('malformedSingleLocalScopes')] + public function test_single_local_validates_scope_syntax_before_payload_resolution(string $operation, mixed $scope): void + { + $task = $this->claim(); + $this->singleLocal($task, $operation, $scope, arguments: ['codec' => 'avro', 'invalid_reference' => true]) + ->assertUnprocessable()->assertJsonValidationErrors('descriptor.cancellation_scope_id'); + $this->assertDatabaseCount('activity_executions', 0); + $this->assertDatabaseCount('activity_attempts', 0); + } + + #[DataProvider('singleLocalOperations')] + public function test_single_local_rejects_a_legacy_request_before_payload_resolution(string $operation): void + { + $task = $this->claim(); + $this->singleLocal($task, $operation, 'root', arguments: ['codec' => 'avro', 'invalid_reference' => true], version: '1.19') + ->assertConflict()->assertJsonPath('reason', 'prepared_local_activity_not_supported') + ->assertJsonPath('unavailable', ['request_protocol']); + $this->assertDatabaseCount('activity_executions', 0); + } + + #[DataProvider('singleLocalOperations')] + public function test_single_local_rejects_a_backend_without_scope_admission_before_payload_resolution(string $operation): void + { + $task = $this->claim(); + $older = \Mockery::mock(PreparedLocalActivityTaskBridge::class); + $older->shouldNotReceive('prepareLocalActivity'); + $older->shouldNotReceive('recoverLocalActivity'); + $this->app->instance(WorkflowTaskBridge::class, $older); + $this->singleLocal($task, $operation, 'root', arguments: ['codec' => 'avro', 'invalid_reference' => true]) + ->assertConflict()->assertJsonPath('reason', 'cancellation_scope_membership_unavailable') + ->assertJsonPath('unavailable', ['installed_runtime_scope_admission']); + $this->assertDatabaseCount('activity_executions', 0); + } + + #[DataProvider('singleLocalOperations')] + public function test_single_local_cannot_use_a_scope_before_its_recorded_sequence(string $operation): void + { + $task = $this->claim(); + $scope = CancellationScopeHistory::open(WorkflowRun::query()->findOrFail($task['run_id']), + WorkflowTask::query()->findOrFail($task['task_id']), 1, '1.20')->payload['scope_id']; + $this->singleLocal($task, $operation, $scope, sequence: 1, arguments: ['codec' => 'avro', 'invalid_reference' => true]) + ->assertConflict()->assertJsonPath('reason', 'local_activity_scope_not_recorded'); + $this->assertDatabaseCount('activity_executions', 0); + } + + #[DataProvider('singleLocalOperations')] + public function test_single_local_cannot_use_a_scope_from_another_run(string $operation): void + { + $other = $this->claim(); + $scope = CancellationScopeHistory::open(WorkflowRun::query()->findOrFail($other['run_id']), + WorkflowTask::query()->findOrFail($other['task_id']), 1, '1.20')->payload['scope_id']; + $task = $this->claim('other-scope-worker'); + $this->singleLocal($task, $operation, $scope, arguments: ['codec' => 'avro', 'invalid_reference' => true]) + ->assertConflict()->assertJsonPath('reason', 'local_activity_scope_not_recorded'); + $this->assertDatabaseCount('activity_executions', 0); + } + + public function test_single_local_records_scope_and_recovers_once_after_claim_replacement(): void + { + $task = $this->claim(); + $scope = CancellationScopeHistory::open(WorkflowRun::query()->findOrFail($task['run_id']), + WorkflowTask::query()->findOrFail($task['task_id']), 1, '1.20')->payload['scope_id']; + $first = $this->singleLocal($task, 'prepare', $scope)->assertOk()->assertJsonPath('prepared', true)->json(); + $before = WorkflowHistoryEvent::query()->count(); + $this->singleLocal($task, 'prepare', $scope)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('activity_attempt_id', $first['activity_attempt_id']); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertSame($scope, ActivityExecution::query()->sole()->activity_options['cancellation_scope_id']); + ActivityAttempt::query()->findOrFail($first['activity_attempt_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $replacement = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'scope-worker', 'task_queue' => 'scope-admission', + ])->assertOk()->json('task'); + $this->assertSame($task['task_id'], $replacement['task_id']); + $this->assertSame($task['workflow_task_attempt'] + 1, $replacement['workflow_task_attempt']); + $recovered = $this->singleLocal($replacement, 'recover', $scope)->assertOk()->assertJsonPath('recovered', true) + ->assertJsonPath('claim_released', true)->assertJsonPath('callback_stop_state', 'unknown')->json(); + $before = WorkflowHistoryEvent::query()->count(); + $this->singleLocal($replacement, 'recover', $scope)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('event_id', $recovered['event_id']); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertSame($scope, ActivityExecution::query()->sole()->activity_options['cancellation_scope_id']); + } + + public function test_single_local_omitted_scope_keeps_preparation_compatible(): void + { + $task = $this->claim(); + $this->singleLocal($task, 'prepare', sequence: 1, includeScope: false)->assertOk()->assertJsonPath('prepared', true); + $this->assertArrayNotHasKey('cancellation_scope_id', ActivityExecution::query()->sole()->activity_options); + } + + private function singleLocal(array $task, string $operation, mixed $scope = null, int $sequence = 2, mixed $arguments = null, string $version = '1.20', bool $includeScope = true) + { + return $this->withHeaders(['X-Namespace' => 'default', WorkerProtocol::HEADER => $version]) + ->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/{$operation}", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'worker_attempt_id' => 'scope-local-attempt', 'sequence' => $sequence, + 'descriptor' => ['type' => 'record_local_activity', 'activity_type' => 'opaque-local', 'payload_codec' => 'avro', + 'arguments' => $arguments ?? Serializer::serializeWithCodec('avro', ['Ada']), + 'retry_policy' => ['max_attempts' => 2, 'backoff_seconds' => [2]], + ...($includeScope ? ['cancellation_scope_id' => $scope] : [])], + ]); + } + + private function claim(string $worker = 'scope-worker'): array { $this->withHeaders(['X-Namespace' => 'default', 'X-Durable-Workflow-Control-Plane-Version' => '2']) ->postJson('/api/workflows', ['workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'scope-admission', 'input' => ['Ada']])->assertCreated(); $this->withHeaders($this->headers())->postJson('/api/worker/register', [ - 'worker_id' => 'scope-worker', 'task_queue' => 'scope-admission', 'runtime' => 'php', + 'worker_id' => $worker, 'task_queue' => 'scope-admission', 'runtime' => 'php', 'supported_workflow_types' => ['tests.external-greeting-workflow'], 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY, PreparedLocalActivityPolicy::CAPABILITY, PreparedLocalActivityPolicy::GROUP_CAPABILITY], 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), 'max_concurrent_workflow_tasks' => 1, @@ -211,7 +350,7 @@ private function claim(): array ])->assertCreated(); return $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ - 'worker_id' => 'scope-worker', 'task_queue' => 'scope-admission', + 'worker_id' => $worker, 'task_queue' => 'scope-admission', ])->assertOk()->json('task'); } From 6bc01e923a31055e2f9138bf6804add0f4935cdc Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 3 Oct 2026 09:39:14 +0000 Subject: [PATCH 32/57] Bind canonical cancellation scope opening to issued worker claims --- .../Api/CancellationScopeController.php | 93 ++++++++++ .../worker-protocol-api.openapi.yaml | 65 ++++++- routes/api.php | 2 + .../Source/CancellationScopeAdmissionTest.php | 161 ++++++++++++++++++ 4 files changed, 320 insertions(+), 1 deletion(-) create mode 100644 app/Http/Controllers/Api/CancellationScopeController.php diff --git a/app/Http/Controllers/Api/CancellationScopeController.php b/app/Http/Controllers/Api/CancellationScopeController.php new file mode 100644 index 00000000..9801354b --- /dev/null +++ b/app/Http/Controllers/Api/CancellationScopeController.php @@ -0,0 +1,93 @@ + false, 'reason' => 'cancellation_scope_requires_protocol_1_20'], 409); + } + $bridge = app(WorkflowTaskBridge::class); + if (! $bridge instanceof CancellationScopeTaskBridge) { + return WorkerProtocol::json(['opened' => false, 'reason' => 'cancellation_scope_opening_unavailable', + 'unavailable' => ['installed_runtime_scope_opening']], 409); + } + + $validated = $request->validate([ + 'lease_owner' => ['required', 'string', 'max:255'], + 'workflow_task_attempt' => ['required', 'integer', 'min:1'], + 'sequence' => ['required', 'integer', 'min:1'], + 'parent_scope_id' => ['sometimes', 'required', 'string', 'max:255'], + 'shield_parent' => ['sometimes', 'required', 'boolean'], + ]); + $namespace = (string) $request->attributes->get('namespace'); + $result = $this->mutations->run(fn (): array => DB::transaction(function () use ( + $namespace, $taskId, $validated, $version, $bridge, + ): array { + $refused = static fn (string $reason): array => ['opened' => false, 'reason' => $reason]; + $snapshot = NamespaceWorkflowScope::taskQuery($namespace)->find($taskId); + if (! $snapshot instanceof WorkflowTask) { + return $refused('task_not_found'); + } + + // Native scope history locks the run before its task. Keep that + // order while checking the namespace and immutable issued claim. + $run = WorkflowRun::query()->where('namespace', $namespace) + ->lockForUpdate()->find($snapshot->workflow_run_id); + if ($run === null || ! NamespaceWorkflowScope::workflowBound($namespace, $run->workflow_instance_id)) { + return $refused('task_not_found'); + } + $task = NamespaceWorkflowScope::taskQuery($namespace)->lockForUpdate()->find($taskId); + if (! $task instanceof WorkflowTask || $task->workflow_run_id !== $run->id) { + return $refused('task_not_found'); + } + if ($task->lease_owner !== $validated['lease_owner']) { + return $refused('lease_owner_mismatch'); + } + if ($task->attempt_count !== (int) $validated['workflow_task_attempt']) { + return $refused('workflow_task_attempt_mismatch'); + } + if (! CooperativeCancellationPolicy::claimSupportsCancellation($task)) { + return $refused('active_claim_cancellation_not_supported'); + } + $worker = WorkerRegistration::query()->where('namespace', $namespace) + ->where('worker_id', $validated['lease_owner'])->lockForUpdate()->first(); + if (! $worker instanceof WorkerRegistration || ! WorkerPollFence::isFresh($worker)) { + return $refused('stale_worker_registration'); + } + + return $bridge->openCancellationScope( + $taskId, $validated['lease_owner'], (int) $validated['workflow_task_attempt'], + (int) $validated['sequence'], $validated['parent_scope_id'] ?? 'root', + (bool) ($validated['shield_parent'] ?? false), $version, + ); + })); + + return WorkerProtocol::json($result, ($result['opened'] ?? false) ? 200 + : (($result['reason'] ?? null) === 'task_not_found' ? 404 : 409)); + } +} diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 9d2a7c6b..6728c269 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "35" + version: "36" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -34,6 +34,15 @@ x-durable-workflow-cancellation-scope-admission: prepared_local_surfaces: [single_prepare, single_recover, atomic_group_checkpoint] unsupported_backend: { status: 409, reason: cancellation_scope_membership_unavailable, recorded: false } stream_effects: successful_admission_only_in_same_outer_transaction +x-durable-workflow-cancellation-scope-opening: + minimum_protocol_version: "1.20" + status: unfrozen_candidate_opening_only + required_backend_role: CancellationScopeTaskBridge + scope_execution_capability: false + authority: [authenticated_namespace, original_capable_workflow_claim, fresh_worker_registration] + lock_order: [run, task, worker_registration] + claim_released: false + replay: identical_parent_and_shield_reuses_canonical_history_identity x-durable-workflow-cooperative-cancellation-contract: minimum_protocol_version: "1.20" worker_capability: cooperative_cancellation @@ -459,6 +468,33 @@ paths: "404": { $ref: "#/components/responses/WorkerError" } "409": { $ref: "#/components/responses/WorkerError" } "422": { $ref: "#/components/responses/WorkerError" } + /worker/workflow-tasks/{taskId}/cancellation-scopes/open: + post: + operationId: openWorkflowCancellationScope + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: cooperative_cancellation + description: >- + Unfrozen candidate admission. Commit a canonical scope before entering its body under the original issued workflow claim. + An identical retry reuses its history and scope identities without releasing or renewing the claim. + This requires the optional installed CancellationScopeTaskBridge and does not advertise scope execution or delivery support. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/CancellationScopeOpeningRequest" } + responses: + "200": + description: Canonical opening committed or identical replay reused. + content: + application/json: + schema: { $ref: "#/components/schemas/CancellationScopeOpeningResponse" } + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } /worker/workflow-tasks/{taskId}/local-activities/prepare: post: operationId: prepareLocalActivity @@ -1770,6 +1806,33 @@ components: operation_sequence: { type: "null" } operation_sequence_span: { type: "null" } reason: { type: string, enum: [cancellation_waiting_for_child, cancellation_waiting_for_activity] } + CancellationScopeOpeningRequest: + type: object + required: [lease_owner, workflow_task_attempt, sequence] + properties: + lease_owner: { type: string, minLength: 1, maxLength: 255 } + workflow_task_attempt: { type: integer, minimum: 1 } + sequence: { type: integer, minimum: 1 } + parent_scope_id: { type: string, minLength: 1, maxLength: 255, default: root } + shield_parent: { type: boolean, default: false } + CancellationScopeOpeningResponse: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - type: object + required: [opened, duplicate, claim_released, task_id, workflow_run_id, history_event_id, scope_id, parent_scope_id, shield_parent, sequence, created_task_ids, reason] + properties: + opened: { type: boolean, const: true } + duplicate: { type: boolean } + claim_released: { type: boolean, const: false } + task_id: { type: string, minLength: 1 } + workflow_run_id: { type: string, minLength: 1 } + history_event_id: { type: string, minLength: 1 } + scope_id: { type: string, minLength: 1 } + parent_scope_id: { type: string, minLength: 1 } + shield_parent: { type: boolean } + sequence: { type: integer, minimum: 1 } + created_task_ids: { type: array, maxItems: 0 } + reason: { type: "null" } PreparedLocalClaimRequest: type: object required: [lease_owner, workflow_task_attempt] diff --git a/routes/api.php b/routes/api.php index f029ef7d..8b6df7d5 100644 --- a/routes/api.php +++ b/routes/api.php @@ -3,6 +3,7 @@ use App\Http\Controllers\Api\ActivityController; use App\Http\Controllers\Api\ActivityTaskController; use App\Http\Controllers\Api\BridgeAdapterController; +use App\Http\Controllers\Api\CancellationScopeController; use App\Http\Controllers\Api\CooperativeCancellationController; use App\Http\Controllers\Api\DeploymentController; use App\Http\Controllers\Api\EmbeddedV2ImportController; @@ -235,6 +236,7 @@ Route::post('/workflow-tasks/{taskId}/history', [WorkerController::class, 'workflowTaskHistory']); Route::post('/workflow-tasks/{taskId}/heartbeat', [WorkerController::class, 'heartbeatWorkflowTask']); Route::post('/workflow-tasks/{taskId}/deliver-cancellation', [CooperativeCancellationController::class, 'deliver']); + Route::post('/workflow-tasks/{taskId}/cancellation-scopes/open', [CancellationScopeController::class, 'open']); Route::post('/workflow-tasks/{taskId}/local-activities/prepare', [PreparedLocalActivityController::class, 'prepare']); Route::post('/workflow-tasks/{taskId}/local-activities/checkpoint', [WorkerController::class, 'checkpointLocalActivityPrefix']); Route::post('/workflow-tasks/{taskId}/local-activities/checkpoint-group', [WorkerController::class, 'checkpointLocalActivityGroup']); diff --git a/tests/Source/CancellationScopeAdmissionTest.php b/tests/Source/CancellationScopeAdmissionTest.php index 7b27206b..a5606704 100644 --- a/tests/Source/CancellationScopeAdmissionTest.php +++ b/tests/Source/CancellationScopeAdmissionTest.php @@ -2,6 +2,7 @@ namespace Tests\Source; +use App\Models\WorkerRegistration; use App\Models\WorkflowNamespace; use App\Support\CooperativeCancellationPolicy; use App\Support\NamespaceExternalPayloadStorage; @@ -11,9 +12,11 @@ use Illuminate\Support\Facades\Queue; use PHPUnit\Framework\Attributes\DataProvider; use Tests\Fixtures\ExternalGreetingWorkflow; +use Tests\Support\OpenApiSchema; use Tests\TestCase; use Workflow\Serializers\Serializer; use Workflow\V2\Contracts\CancellationScopeAdmission; +use Workflow\V2\Contracts\CancellationScopeTaskBridge; use Workflow\V2\Contracts\PreparedLocalActivityTaskBridge; use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Models\ActivityAttempt; @@ -42,6 +45,164 @@ protected function setUp(): void $this->assertTrue(class_exists(CancellationScopeHistory::class)); } + public function test_scope_opening_records_once_and_preserves_the_live_claim(): void + { + $task = $this->claim(); + $claim = WorkflowTask::query()->findOrFail($task['task_id']); + $originalLease = $claim->lease_expires_at->toISOString(); + $before = WorkflowHistoryEvent::query()->count(); + $response = $this->openScope($task)->assertOk()->assertJsonPath('opened', true) + ->assertJsonPath('duplicate', false)->assertJsonPath('claim_released', false) + ->assertJsonPath('parent_scope_id', 'root')->assertJsonPath('shield_parent', false) + ->assertJsonPath('created_task_ids', []); + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/CancellationScopeOpeningResponse/allOf/1', json_decode($response->getContent(), flags: JSON_THROW_ON_ERROR)); + $opened = $response->json(); + $this->assertSame($before + 1, WorkflowHistoryEvent::query()->count()); + $this->openScope($task)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('history_event_id', $opened['history_event_id'])->assertJsonPath('scope_id', $opened['scope_id']); + $this->assertSame($before + 1, WorkflowHistoryEvent::query()->count()); + $this->assertSame('leased', $claim->refresh()->status->value); + $this->assertSame($originalLease, $claim->lease_expires_at->toISOString()); + $this->complete($task, $opened['scope_id'], 'start_timer')->assertOk()->assertJsonPath('recorded', true); + $this->assertSame($opened['scope_id'], WorkflowHistoryEvent::query()->where('event_type', 'TimerScheduled')->sole()->payload['cancellation_scope_id']); + } + + public function test_nested_shield_is_recorded_and_changed_replay_is_refused(): void + { + $task = $this->claim(); + $parent = $this->openScope($task)->assertOk()->json('scope_id'); + $nested = $this->openScope($task, ['sequence' => 2, 'parent_scope_id' => $parent, 'shield_parent' => true]) + ->assertOk()->assertJsonPath('parent_scope_id', $parent)->assertJsonPath('shield_parent', true)->json('scope_id'); + $this->assertNotSame($parent, $nested); + $before = WorkflowHistoryEvent::query()->count(); + $this->openScope($task, ['sequence' => 2, 'parent_scope_id' => $parent, 'shield_parent' => false]) + ->assertStatus(409)->assertJsonPath('reason', 'cancellation_scope_replay_mismatch'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_replacement_claim_reuses_the_scope_boundary_and_fences_the_old_attempt(): void + { + $task = $this->claim(); + $opened = $this->openScope($task)->assertOk()->json(); + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $replacement = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => $task['lease_owner'], 'task_queue' => 'scope-admission', + ])->assertOk()->json('task'); + $this->assertSame($task['task_id'], $replacement['task_id']); + $this->assertSame($task['workflow_task_attempt'] + 1, $replacement['workflow_task_attempt']); + $before = WorkflowHistoryEvent::query()->count(); + $this->openScope($replacement)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('history_event_id', $opened['history_event_id'])->assertJsonPath('scope_id', $opened['scope_id']); + $this->openScope($task)->assertStatus(409)->assertJsonPath('reason', 'workflow_task_attempt_mismatch'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public static function refusedScopeOpenings(): array + { + return [ + 'wrong owner' => [['lease_owner' => 'other-worker'], 'lease_owner_mismatch'], + 'wrong attempt' => [['workflow_task_attempt' => 99], 'workflow_task_attempt_mismatch'], + 'unknown parent' => [['parent_scope_id' => 'foreign-scope'], 'cancellation_scope_parent_not_recorded'], + 'future sequence' => [['sequence' => 2], 'cancellation_scope_sequence_mismatch'], + ]; + } + + #[DataProvider('refusedScopeOpenings')] + public function test_invalid_scope_opening_cannot_mutate_history(array $overrides, string $reason): void + { + $task = $this->claim(); + $before = WorkflowHistoryEvent::query()->count(); + $this->openScope($task, $overrides)->assertStatus(409)->assertJsonPath('opened', false)->assertJsonPath('reason', $reason); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_foreign_namespace_scope_opening_does_not_reveal_or_mutate_the_task(): void + { + $task = $this->claim(); + WorkflowNamespace::query()->create(['name' => 'other', 'retention_days' => 30, 'status' => 'active']); + $before = WorkflowHistoryEvent::query()->count(); + $this->openScope($task, namespace: 'other')->assertNotFound()->assertJsonPath('reason', 'task_not_found'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_stale_worker_cannot_open_a_scope_with_a_live_task_lease(): void + { + $task = $this->claim(); + WorkerRegistration::query()->where('worker_id', $task['lease_owner'])->update(['status' => 'inactive']); + $before = WorkflowHistoryEvent::query()->count(); + $this->openScope($task)->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_expired_claim_cannot_open_or_replay_a_scope(): void + { + $task = $this->claim(); + $this->openScope($task)->assertOk(); + $before = WorkflowHistoryEvent::query()->count(); + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $this->openScope($task)->assertStatus(409)->assertJsonPath('reason', 'cancellation_scope_workflow_claim_mismatch'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_incompatible_issued_claim_cannot_be_upgraded_by_current_registration(): void + { + $task = $this->claim(); + $claim = WorkflowTask::query()->findOrFail($task['task_id']); + $payload = $claim->payload; + $payload['_server_workflow_claim']['capabilities'] = []; + $claim->forceFill(['payload' => $payload])->save(); + $before = WorkflowHistoryEvent::query()->count(); + $this->openScope($task)->assertStatus(409)->assertJsonPath('reason', 'active_claim_cancellation_not_supported'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_scope_opening_rejects_legacy_protocol_and_missing_optional_backend(): void + { + $task = $this->claim(); + $this->openScope($task, version: '1.19')->assertStatus(409)->assertJsonPath('reason', 'cancellation_scope_requires_protocol_1_20'); + $this->assertInstanceOf(CancellationScopeTaskBridge::class, app(WorkflowTaskBridge::class)); + $this->app->instance(WorkflowTaskBridge::class, \Mockery::mock(WorkflowTaskBridge::class)); + $this->openScope($task)->assertStatus(409)->assertJsonPath('reason', 'cancellation_scope_opening_unavailable') + ->assertJsonPath('unavailable', ['installed_runtime_scope_opening']); + } + + public function test_scope_opening_requires_authenticated_worker_authority(): void + { + $task = $this->claim(); + config(['server.auth.driver' => 'token', 'server.auth.token' => 'scope-test-token']); + $before = WorkflowHistoryEvent::query()->count(); + $this->openScope($task)->assertUnauthorized(); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->withHeaders(['Authorization' => 'Bearer scope-test-token']); + $this->openScope($task)->assertOk()->assertJsonPath('opened', true); + } + + public static function malformedScopeOpenings(): array + { + return [ + [['sequence' => 0]], [['workflow_task_attempt' => null]], [['lease_owner' => '']], + [['parent_scope_id' => '']], [['parent_scope_id' => str_repeat('x', 256)]], [['shield_parent' => 'invalid']], + ]; + } + + #[DataProvider('malformedScopeOpenings')] + public function test_malformed_scope_opening_is_rejected_before_history(array $overrides): void + { + $task = $this->claim(); + $before = WorkflowHistoryEvent::query()->count(); + $this->openScope($task, $overrides)->assertStatus(422); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + private function openScope(array $task, array $overrides = [], string $version = '1.20', string $namespace = 'default') + { + return $this->withHeaders(['X-Namespace' => $namespace, WorkerProtocol::HEADER => $version]) + ->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/open", array_replace([ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], 'sequence' => 1, + ], $overrides)); + } + public static function operations(): array { return [['schedule_activity'], ['start_timer'], ['start_child_workflow']]; From d33d574c24592a41f7e0f8853c52b36ac62bd9af Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 3 Oct 2026 10:21:16 +0000 Subject: [PATCH 33/57] Checkpoint scope prefixes under cooperative workflow claims --- app/Http/Controllers/Api/WorkerController.php | 35 +++- .../Middleware/EnforceStorageAdmission.php | 2 + .../RuntimeExternalPayloadTransport.php | 10 ++ .../RuntimeExternalPayloadReference.php | 5 + .../RuntimePayloadCompletionContext.php | 6 +- app/Support/RuntimePayloadCompletionLease.php | 16 +- .../worker-protocol-api.openapi.yaml | 53 +++++- routes/api.php | 1 + .../Source/CancellationScopeAdmissionTest.php | 153 +++++++++++++++++- .../RuntimePayloadCompletionContextTest.php | 2 + 10 files changed, 270 insertions(+), 13 deletions(-) diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index d107ccc8..8570d163 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -55,6 +55,7 @@ use Illuminate\Support\Str; use Illuminate\Validation\ValidationException; use Workflow\V2\Contracts\CancellationScopeAdmission; +use Workflow\V2\Contracts\CancellationScopeTaskBridge; use Workflow\V2\Contracts\HistoryProjectionRole; use Workflow\V2\Contracts\PreparedLocalActivityGroupTaskBridge; use Workflow\V2\Contracts\PreparedLocalActivityTaskBridge; @@ -1524,7 +1525,23 @@ public function checkpointLocalActivityGroup(Request $request, string $taskId): return $this->mutateWorkflowTaskCommands($request, $taskId, true, true); } - private function mutateWorkflowTaskCommands(Request $request, string $taskId, bool $checkpoint = false, bool $group = false): JsonResponse + public function checkpointCancellationScopePrefix(Request $request, string $taskId): JsonResponse + { + if ($response = WorkerProtocol::rejectUnsupported($request)) { + return $response; + } + if (! WorkerProtocol::versionMeetsMinimum(WorkerProtocol::requestVersion($request), CooperativeCancellationPolicy::MINIMUM_PROTOCOL_VERSION)) { + return WorkerProtocol::json(['checkpointed' => false, 'reason' => 'cancellation_scope_checkpoint_requires_protocol_1_20'], 409); + } + if (! app(WorkflowTaskBridge::class) instanceof CancellationScopeTaskBridge) { + return WorkerProtocol::json(['checkpointed' => false, 'reason' => 'cancellation_scope_checkpoint_unavailable', + 'unavailable' => ['installed_runtime_scope_checkpoint']], 409); + } + + return $this->mutateWorkflowTaskCommands($request, $taskId, true, scopePrefix: true); + } + + private function mutateWorkflowTaskCommands(Request $request, string $taskId, bool $checkpoint = false, bool $group = false, bool $scopePrefix = false): JsonResponse { if ($response = WorkerProtocol::rejectUnsupported($request)) { return $response; @@ -1900,6 +1917,7 @@ function () use ( $checkpoint, $checkpointInput, $group, + $scopePrefix, ): array|JsonResponse { return DB::transaction(function () use ( $bridge, @@ -1913,6 +1931,7 @@ function () use ( $checkpoint, $checkpointInput, $group, + $scopePrefix, ): array|JsonResponse { $quotaSnapshot = $this->durableStateQuota->snapshotForMutation( (string) $namespace, @@ -2003,13 +2022,20 @@ function () use ( $commands = $group ? $this->normalizePreparedLocalGroupCommands($commands, WorkerProtocol::requestVersion($request)) : WorkflowCommandNormalizer::normalize($commands, WorkerProtocol::requestVersion($request)); if ($checkpoint) { - if (PreparedLocalActivityPolicy::currentClaim($claimedTask) === null + if ($scopePrefix) { + if (! $bridge instanceof CancellationScopeTaskBridge + || ! $claimedTask instanceof WorkflowTask + || ! CooperativeCancellationPolicy::claimSupportsCancellation($claimedTask)) { + return WorkerProtocol::json(['checkpointed' => false, 'reason' => 'active_claim_cancellation_not_supported'], 409); + } + } elseif (PreparedLocalActivityPolicy::currentClaim($claimedTask) === null || ! $bridge instanceof PreparedLocalActivityTaskBridge || ($group && (PreparedLocalActivityPolicy::currentGroupClaim($claimedTask) === null || ! $bridge instanceof PreparedLocalActivityGroupTaskBridge))) { throw new PreparedLocalActivityAdmissionRefused; } - $method = $group ? 'checkpointLocalActivityGroup' : 'checkpointLocalActivityPrefix'; + $method = $scopePrefix ? 'checkpointCancellationScopePrefix' + : ($group ? 'checkpointLocalActivityGroup' : 'checkpointLocalActivityPrefix'); $outcome = $bridge->$method( $taskId, $validated['lease_owner'], (int) $validated['workflow_task_attempt'], $checkpointInput['checkpoint_id'], (int) $checkpointInput['start_sequence'], $commands, @@ -2771,7 +2797,8 @@ private function persistTypedSearchAttributeHistoryIdentity( } else { $payload = WorkflowTask::query()->find($taskId)?->payload ?? []; $prefixEnd = max($payload['portable_local_checkpoint']['next_sequence'] ?? 0, - $payload['portable_local_group_checkpoint']['next_sequence'] ?? 0); + $payload['portable_local_group_checkpoint']['next_sequence'] ?? 0, + $payload['portable_scope_checkpoint']['next_sequence'] ?? 0); if (is_int($prefixEnd) && $prefixEnd > 0) { $eventQuery->where('payload->sequence', '>=', $prefixEnd); } diff --git a/app/Http/Middleware/EnforceStorageAdmission.php b/app/Http/Middleware/EnforceStorageAdmission.php index 16359388..908cd797 100644 --- a/app/Http/Middleware/EnforceStorageAdmission.php +++ b/app/Http/Middleware/EnforceStorageAdmission.php @@ -19,6 +19,8 @@ final class EnforceStorageAdmission 'WorkerController@completeWorkflowTask', 'WorkerController@checkpointLocalActivityPrefix', 'WorkerController@checkpointLocalActivityGroup', + 'WorkerController@checkpointCancellationScopePrefix', + 'CancellationScopeController@open', 'PreparedLocalActivityController@prepare', 'PreparedLocalActivityController@recover', 'PreparedLocalActivityController@control', diff --git a/app/Http/Middleware/RuntimeExternalPayloadTransport.php b/app/Http/Middleware/RuntimeExternalPayloadTransport.php index cb686f15..5e237f63 100644 --- a/app/Http/Middleware/RuntimeExternalPayloadTransport.php +++ b/app/Http/Middleware/RuntimeExternalPayloadTransport.php @@ -44,6 +44,13 @@ class RuntimeExternalPayloadTransport ['commands', '*', 'request_payload'], ['commands', '*', 'result'], ], + 'WorkerController@checkpointCancellationScopePrefix' => [ + ['commands', '*', 'arguments'], + ['commands', '*', 'entries'], + ['commands', '*', 'exception', 'details'], + ['commands', '*', 'request_payload'], + ['commands', '*', 'result'], + ], 'WorkerController@completeWorkflowTask' => [ ['commands', '*', 'arguments'], ['commands', '*', 'entries'], @@ -68,6 +75,9 @@ class RuntimeExternalPayloadTransport 'WorkerController@checkpointLocalActivityGroup' => [ ['commands', '*', 'workflow_stream', 'items', '*', 'payload_reference'], ], + 'WorkerController@checkpointCancellationScopePrefix' => [ + ['commands', '*', 'workflow_stream', 'items', '*', 'payload_reference'], + ], 'WorkerController@completeWorkflowTask' => [ ['commands', '*', 'workflow_stream', 'items', '*', 'payload_reference'], ], diff --git a/app/Support/RuntimeExternalPayloadReference.php b/app/Support/RuntimeExternalPayloadReference.php index d9575455..01c4c3a5 100644 --- a/app/Support/RuntimeExternalPayloadReference.php +++ b/app/Support/RuntimeExternalPayloadReference.php @@ -3,6 +3,8 @@ namespace App\Support; use InvalidArgumentException; +use Workflow\V2\Contracts\CancellationScopeTaskBridge; +use Workflow\V2\Contracts\WorkflowTaskBridge; final class RuntimeExternalPayloadReference { @@ -76,6 +78,9 @@ public static function transportManifest(): array 'schema' => RuntimePayloadCompletionContext::SCHEMA, 'prepared_schema' => PreparedLocalActivityPolicy::serverSupported() ? RuntimePayloadCompletionContext::PREPARED_SCHEMA : null, + 'scope_schema' => CooperativeCancellationPolicy::serverSupported() + && app(WorkflowTaskBridge::class) instanceof CancellationScopeTaskBridge + ? RuntimePayloadCompletionContext::PREPARED_SCHEMA : null, 'header' => RuntimePayloadCompletionContext::HEADER, 'use' => 'retry_draining_refusal_for_current_worker_completion_only', 'max_bytes_per_lease' => RuntimePayloadCompletionUploads::maxBytes(), diff --git a/app/Support/RuntimePayloadCompletionContext.php b/app/Support/RuntimePayloadCompletionContext.php index b87efcb7..80d678c1 100644 --- a/app/Support/RuntimePayloadCompletionContext.php +++ b/app/Support/RuntimePayloadCompletionContext.php @@ -41,7 +41,7 @@ public static function parse(string $header): self if ($prepared) { $keys[] = match ($value['operation'] ?? null) { 'local_activity_outcome' => 'activity_attempt_id', - 'local_activity_checkpoint', 'local_activity_group_checkpoint' => 'checkpoint_id', + 'local_activity_checkpoint', 'local_activity_group_checkpoint', 'cancellation_scope_checkpoint' => 'checkpoint_id', default => 'sequence', }; } @@ -56,7 +56,7 @@ public static function parse(string $header): self } if ($prepared && ($value['kind'] !== 'workflow' || ! match ($value['operation'] ?? null) { 'local_activity_outcome' => self::identifier($value['activity_attempt_id']), - 'local_activity_checkpoint', 'local_activity_group_checkpoint' => self::identifier($value['checkpoint_id']), + 'local_activity_checkpoint', 'local_activity_group_checkpoint', 'cancellation_scope_checkpoint' => self::identifier($value['checkpoint_id']), default => is_int($value['sequence']) && $value['sequence'] > 0, })) { throw self::invalid(); @@ -67,7 +67,7 @@ public static function parse(string $header): self throw self::invalid(); } $validSlot = $prepared ? match ($value['operation']) { - 'local_activity_checkpoint', 'local_activity_group_checkpoint' => self::workflowSlot($value['slot']), + 'local_activity_checkpoint', 'local_activity_group_checkpoint', 'cancellation_scope_checkpoint' => self::workflowSlot($value['slot']), 'local_activity_prepare', 'local_activity_recover' => $value['slot'] === ['descriptor', 'arguments'], 'local_activity_outcome' => $value['slot'] === ['report', 'result'], default => false, diff --git a/app/Support/RuntimePayloadCompletionLease.php b/app/Support/RuntimePayloadCompletionLease.php index 971a21fd..c821fd5b 100644 --- a/app/Support/RuntimePayloadCompletionLease.php +++ b/app/Support/RuntimePayloadCompletionLease.php @@ -5,6 +5,7 @@ use App\Models\WorkerRegistration; use Carbon\CarbonImmutable; use Workflow\V2\Contracts\ActivityTaskBridge; +use Workflow\V2\Contracts\CancellationScopeTaskBridge; use Workflow\V2\Contracts\PreparedLocalActivityTaskBridge; use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Enums\TaskType; @@ -108,9 +109,18 @@ private function workflow(string $namespace, RuntimePayloadCompletionContext $co if ($context->schema === RuntimePayloadCompletionContext::PREPARED_SCHEMA) { $task = $guard['task']; - $claim = PreparedLocalActivityPolicy::currentClaim($task); - if (! PreparedLocalActivityPolicy::serverSupported() || $claim === null || ! WorkerPollFence::isCurrent($claim)) { - throw self::rejected(); + if ($context->operation === 'cancellation_scope_checkpoint') { + if (! CooperativeCancellationPolicy::serverSupported() + || ! app(WorkflowTaskBridge::class) instanceof CancellationScopeTaskBridge + || ! CooperativeCancellationPolicy::claimSupportsCancellation($task)) { + throw self::rejected(); + } + $claim = null; + } else { + $claim = PreparedLocalActivityPolicy::currentClaim($task); + if (! PreparedLocalActivityPolicy::serverSupported() || $claim === null || ! WorkerPollFence::isCurrent($claim)) { + throw self::rejected(); + } } if ($context->operation === 'local_activity_group_checkpoint' && (! PreparedLocalActivityPolicy::groupsSupported() || PreparedLocalActivityPolicy::currentGroupClaim($task) === null)) { diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 6728c269..106d4c62 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "36" + version: "37" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -42,6 +42,18 @@ x-durable-workflow-cancellation-scope-opening: authority: [authenticated_namespace, original_capable_workflow_claim, fresh_worker_registration] lock_order: [run, task, worker_registration] claim_released: false + prefix_checkpoint: + path: /worker/workflow-tasks/{taskId}/cancellation-scopes/checkpoint + admission: same_original_cooperative_claim_without_local_activity_capability + history: replay_canonical_prefix_before_opening + receipt: immutable_latest_scope_checkpoint_on_this_claim + external_payload_completion: + discovery: namespace.external_payload_storage.transport.upload.completion_context.scope_schema + schema: durable-workflow.v2.payload-completion-context.v2 + operation: cancellation_scope_checkpoint + identity: checkpoint_id + slots: ordinary_workflow_command_payload_slots + allowance: shares_existing_workflow_claim_budget replay: identical_parent_and_shield_reuses_canonical_history_identity x-durable-workflow-cooperative-cancellation-contract: minimum_protocol_version: "1.20" @@ -495,6 +507,37 @@ paths: "404": { $ref: "#/components/responses/WorkerError" } "409": { $ref: "#/components/responses/WorkerError" } "422": { $ref: "#/components/responses/WorkerError" } + /worker/workflow-tasks/{taskId}/cancellation-scopes/checkpoint: + post: + operationId: checkpointCancellationScopePrefix + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: cooperative_cancellation + description: >- + Unfrozen candidate admission. Commit the ordinary command prefix preceding a scope through the same + validation, authorization and quota path as workflow completion, retaining the original claim. + Replay the returned canonical history before opening the scope. An identical retry reuses the original + checkpoint receipt. This requires the optional installed CancellationScopeTaskBridge. + No local Activity capability, callback admission or scope execution advertisement is implied. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/CancellationScopeCheckpointRequest" } + responses: + "200": + description: Canonical prefix committed or identical receipt reused. The claim remains leased. + content: + application/json: + schema: { $ref: "#/components/schemas/CancellationScopeCheckpointResponse" } + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "429": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } /worker/workflow-tasks/{taskId}/local-activities/prepare: post: operationId: prepareLocalActivity @@ -1833,6 +1876,14 @@ components: sequence: { type: integer, minimum: 1 } created_task_ids: { type: array, maxItems: 0 } reason: { type: "null" } + CancellationScopeCheckpointRequest: + description: Ordinary retained prefix before scope opening. Does not admit local callbacks. + allOf: + - $ref: "#/components/schemas/PreparedLocalCheckpointRequest" + CancellationScopeCheckpointResponse: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - $ref: "#/components/schemas/PreparedLocalCheckpointResult" PreparedLocalClaimRequest: type: object required: [lease_owner, workflow_task_attempt] diff --git a/routes/api.php b/routes/api.php index 8b6df7d5..60fa2274 100644 --- a/routes/api.php +++ b/routes/api.php @@ -237,6 +237,7 @@ Route::post('/workflow-tasks/{taskId}/heartbeat', [WorkerController::class, 'heartbeatWorkflowTask']); Route::post('/workflow-tasks/{taskId}/deliver-cancellation', [CooperativeCancellationController::class, 'deliver']); Route::post('/workflow-tasks/{taskId}/cancellation-scopes/open', [CancellationScopeController::class, 'open']); + Route::post('/workflow-tasks/{taskId}/cancellation-scopes/checkpoint', [WorkerController::class, 'checkpointCancellationScopePrefix']); Route::post('/workflow-tasks/{taskId}/local-activities/prepare', [PreparedLocalActivityController::class, 'prepare']); Route::post('/workflow-tasks/{taskId}/local-activities/checkpoint', [WorkerController::class, 'checkpointLocalActivityPrefix']); Route::post('/workflow-tasks/{taskId}/local-activities/checkpoint-group', [WorkerController::class, 'checkpointLocalActivityGroup']); diff --git a/tests/Source/CancellationScopeAdmissionTest.php b/tests/Source/CancellationScopeAdmissionTest.php index a5606704..237f00f1 100644 --- a/tests/Source/CancellationScopeAdmissionTest.php +++ b/tests/Source/CancellationScopeAdmissionTest.php @@ -2,15 +2,19 @@ namespace Tests\Source; +use App\Models\RuntimePayloadCompletionBudget; use App\Models\WorkerRegistration; use App\Models\WorkflowNamespace; use App\Support\CooperativeCancellationPolicy; use App\Support\NamespaceExternalPayloadStorage; use App\Support\PreparedLocalActivityPolicy; +use App\Support\RuntimePayloadCompletionContext; use App\Support\WorkerProtocol; use Illuminate\Foundation\Testing\RefreshDatabase; +use Illuminate\Support\Facades\File; use Illuminate\Support\Facades\Queue; use PHPUnit\Framework\Attributes\DataProvider; +use Tests\Feature\Concerns\StoragePressureFixture; use Tests\Fixtures\ExternalGreetingWorkflow; use Tests\Support\OpenApiSchema; use Tests\TestCase; @@ -31,6 +35,7 @@ final class CancellationScopeAdmissionTest extends TestCase { use RefreshDatabase; + use StoragePressureFixture; protected function setUp(): void { @@ -45,6 +50,150 @@ protected function setUp(): void $this->assertTrue(class_exists(CancellationScopeHistory::class)); } + public function test_scope_prefix_requires_no_local_activity_capability_and_replays_before_opening(): void + { + $task = $this->claim(capabilities: [CooperativeCancellationPolicy::CAPABILITY]); + $claim = WorkflowTask::query()->findOrFail($task['task_id']); + $originalLease = $claim->lease_expires_at->toISOString(); + $first = $this->scopePrefix($task)->assertOk()->assertJsonPath('checkpointed', true) + ->assertJsonPath('duplicate', false)->assertJsonPath('start_sequence', 1)->assertJsonPath('next_sequence', 2)->json(); + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/CancellationScopeCheckpointResponse/allOf/1', json_decode(json_encode($first, JSON_THROW_ON_ERROR), flags: JSON_THROW_ON_ERROR)); + $this->assertIsString($first['history_refresh_page_token']); + $before = WorkflowHistoryEvent::query()->count(); + $this->scopePrefix($task)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('fingerprint', $first['fingerprint']); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->openScope($task, ['sequence' => 2])->assertOk()->assertJsonPath('opened', true); + $this->assertSame($originalLease, $claim->refresh()->lease_expires_at->toISOString()); + $this->assertSame('leased', $claim->status->value); + $this->assertArrayHasKey('portable_scope_checkpoint', $claim->payload); + $this->assertArrayNotHasKey('portable_local_checkpoint', $claim->payload); + $this->assertSame(0, ActivityExecution::query()->count()); + } + + public static function refusedScopePrefixes(): array + { + return [ + 'wrong owner' => [['lease_owner' => 'other-worker'], 'lease_owner_mismatch'], + 'wrong attempt' => [['workflow_task_attempt' => 99], 'workflow_task_attempt_mismatch'], + 'future sequence' => [['start_sequence' => 2], 'cancellation_scope_checkpoint_sequence_mismatch'], + 'terminal command' => [['commands' => [['type' => 'complete_workflow', 'result' => Serializer::serializeWithCodec('avro', 'done')]]], 'invalid_cancellation_scope_checkpoint_commands'], + ]; + } + + #[DataProvider('refusedScopePrefixes')] + public function test_scope_prefix_refuses_before_committing_history(array $overrides, string $reason): void + { + $task = $this->claim(capabilities: [CooperativeCancellationPolicy::CAPABILITY]); + $before = WorkflowHistoryEvent::query()->count(); + $this->scopePrefix($task, $overrides)->assertStatus(409)->assertJsonPath('reason', $reason); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_scope_prefix_changed_retry_and_replaced_claim_are_fenced(): void + { + $task = $this->claim(capabilities: [CooperativeCancellationPolicy::CAPABILITY]); + $this->scopePrefix($task)->assertOk(); + $before = WorkflowHistoryEvent::query()->count(); + $this->scopePrefix($task, ['commands' => [['type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 'changed')]]]) + ->assertStatus(409)->assertJsonPath('reason', 'cancellation_scope_checkpoint_mismatch'); + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $replacement = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => $task['lease_owner'], 'task_queue' => 'scope-admission', + ])->assertOk()->json('task'); + $this->scopePrefix($task)->assertStatus(409)->assertJsonPath('reason', 'workflow_task_attempt_mismatch'); + $this->openScope($replacement, ['sequence' => 2])->assertOk()->assertJsonPath('opened', true); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'SideEffectRecorded')->count()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeOpened')->count()); + } + + public function test_scope_prefix_rejects_legacy_missing_backend_and_incompatible_issued_claim(): void + { + $task = $this->claim(capabilities: [CooperativeCancellationPolicy::CAPABILITY]); + $this->scopePrefix($task, version: '1.19')->assertStatus(409)->assertJsonPath('reason', 'cancellation_scope_checkpoint_requires_protocol_1_20'); + $bridge = app(WorkflowTaskBridge::class); + $this->app->instance(WorkflowTaskBridge::class, $this->createMock(WorkflowTaskBridge::class)); + $this->scopePrefix($task)->assertStatus(409)->assertJsonPath('reason', 'cancellation_scope_checkpoint_unavailable'); + $this->app->instance(WorkflowTaskBridge::class, $bridge); + $claim = WorkflowTask::query()->findOrFail($task['task_id']); + $payload = $claim->payload; + $payload['_server_workflow_claim']['capabilities'] = []; + $claim->forceFill(['payload' => $payload])->save(); + $this->scopePrefix($task)->assertStatus(409)->assertJsonPath('reason', 'active_claim_cancellation_not_supported'); + } + + public function test_scope_prefix_hides_other_namespaces_and_refuses_stale_workers(): void + { + $task = $this->claim(capabilities: [CooperativeCancellationPolicy::CAPABILITY]); + WorkflowNamespace::query()->create(['name' => 'other', 'retention_days' => 30, 'status' => 'active']); + $before = WorkflowHistoryEvent::query()->count(); + $this->scopePrefix($task, namespace: 'other')->assertNotFound(); + WorkerRegistration::query()->where('worker_id', $task['lease_owner'])->update(['last_heartbeat_at' => now()->subMinutes(10)]); + $this->scopePrefix($task)->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + private function scopePrefix(array $task, array $overrides = [], string $version = '1.20', string $namespace = 'default') + { + return $this->withHeaders(['X-Namespace' => $namespace, WorkerProtocol::HEADER => $version]) + ->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/checkpoint", array_replace([ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'checkpoint_id' => 'scope-prefix', 'start_sequence' => 1, + 'commands' => [['type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 'before-scope')]], + ], $overrides)); + } + + public function test_scope_prefix_uploads_during_draining_share_one_fenced_claim_allowance(): void + { + $task = $this->claim(capabilities: [CooperativeCancellationPolicy::CAPABILITY]); + $directory = storage_path('framework/testing/scope-prefix-uploads'); + WorkflowNamespace::query()->where('name', 'default')->update(['external_payload_storage' => [ + 'driver' => 'local', 'enabled' => true, 'threshold_bytes' => 32, + 'config' => ['uri' => 'file://'.$directory], + ]]); + $this->configureStoragePressure('draining'); + $payload = Serializer::serializeWithCodec('avro', str_repeat('before-scope', 200)); + $context = [ + 'schema' => RuntimePayloadCompletionContext::PREPARED_SCHEMA, 'kind' => 'workflow', + 'task_id' => $task['task_id'], 'attempt' => $task['workflow_task_attempt'], 'lease_owner' => $task['lease_owner'], + 'operation' => 'cancellation_scope_checkpoint', 'checkpoint_id' => 'scope-prefix', 'slot' => ['commands', 0, 'result'], + ]; + $upload = fn (array $value) => $this->call('POST', '/api/external-payloads/v1', [], [], [], [ + 'CONTENT_TYPE' => 'application/octet-stream', 'HTTP_X_NAMESPACE' => 'default', + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_CODEC' => 'avro', 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_SIZE' => (string) strlen($payload), + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_SHA256' => hash('sha256', $payload), + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_COMPLETION' => json_encode($value, JSON_THROW_ON_ERROR), + ], $payload); + try { + $reference = $upload($context)->assertCreated()->json('reference'); + $this->scopePrefix($task, ['commands' => [['type' => 'record_side_effect', 'result' => ['codec' => 'avro', 'external_payload' => $reference]]]]) + ->assertOk()->assertJsonPath('checkpointed', true); + $stored = WorkflowHistoryEvent::query()->where('event_type', 'SideEffectRecorded')->sole()->payload['result']; + $this->assertSame('avro', $stored['codec']); + $this->assertSame(hash('sha256', $payload), $stored['external_storage']['sha256']); + $fetch = $this->withHeaders([ + 'X-Namespace' => 'default', 'X-Durable-Workflow-Payload-Codec' => $reference['codec'], + 'X-Durable-Workflow-Payload-Size' => (string) $reference['size_bytes'], + 'X-Durable-Workflow-Payload-SHA256' => $reference['sha256'], + ])->get('/api/external-payloads/v1/'.$reference['reference_id'])->assertOk(); + $this->assertSame($payload, $fetch->streamedContent()); + $legacy = array_diff_key($context, ['checkpoint_id' => true]); + $legacy['schema'] = RuntimePayloadCompletionContext::SCHEMA; + $legacy['operation'] = 'complete'; + $upload($legacy)->assertCreated(); + $this->assertSame(1, RuntimePayloadCompletionBudget::query()->count()); + $this->openScope($task, ['sequence' => 2])->assertOk()->assertJsonPath('opened', true); + $claim = WorkflowTask::query()->findOrFail($task['task_id']); + $claim->forceFill(['lease_owner' => 'replacement', 'attempt_count' => $task['workflow_task_attempt'] + 1])->save(); + $context['checkpoint_id'] = 'new-prefix'; + $upload($context)->assertStatus(409); + } finally { + $this->removeStoragePressure(); + File::deleteDirectory($directory); + } + } + public function test_scope_opening_records_once_and_preserves_the_live_claim(): void { $task = $this->claim(); @@ -498,14 +647,14 @@ private function singleLocal(array $task, string $operation, mixed $scope = null ]); } - private function claim(string $worker = 'scope-worker'): array + private function claim(string $worker = 'scope-worker', ?array $capabilities = null): array { $this->withHeaders(['X-Namespace' => 'default', 'X-Durable-Workflow-Control-Plane-Version' => '2']) ->postJson('/api/workflows', ['workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'scope-admission', 'input' => ['Ada']])->assertCreated(); $this->withHeaders($this->headers())->postJson('/api/worker/register', [ 'worker_id' => $worker, 'task_queue' => 'scope-admission', 'runtime' => 'php', 'supported_workflow_types' => ['tests.external-greeting-workflow'], - 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY, PreparedLocalActivityPolicy::CAPABILITY, PreparedLocalActivityPolicy::GROUP_CAPABILITY], + 'capabilities' => $capabilities ?? [CooperativeCancellationPolicy::CAPABILITY, PreparedLocalActivityPolicy::CAPABILITY, PreparedLocalActivityPolicy::GROUP_CAPABILITY], 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), 'max_concurrent_workflow_tasks' => 1, 'process_metrics' => ['host' => 'test-worker', 'process_started_at' => now()->toISOString(), 'process_id' => 10], ])->assertCreated(); diff --git a/tests/Unit/RuntimePayloadCompletionContextTest.php b/tests/Unit/RuntimePayloadCompletionContextTest.php index 8d50e380..308f76f2 100644 --- a/tests/Unit/RuntimePayloadCompletionContextTest.php +++ b/tests/Unit/RuntimePayloadCompletionContextTest.php @@ -86,6 +86,8 @@ public function test_prepared_operations_and_legacy_completion_share_one_workflo foreach ([ ['local_activity_checkpoint', ['commands', 0, 'result'], ['checkpoint_id' => 'prefix-1']], ['local_activity_checkpoint', ['commands', 0, 'result'], ['checkpoint_id' => 'prefix-2']], + ['cancellation_scope_checkpoint', ['commands', 0, 'result'], ['checkpoint_id' => 'prefix-1']], + ['cancellation_scope_checkpoint', ['commands', 0, 'result'], ['checkpoint_id' => 'prefix-2']], ['local_activity_group_checkpoint', ['commands', 0, 'arguments'], ['checkpoint_id' => 'group-1']], ['local_activity_group_checkpoint', ['commands', 1, 'arguments'], ['checkpoint_id' => 'group-1']], ['local_activity_group_checkpoint', ['commands', 0, 'arguments'], ['checkpoint_id' => 'group-2']], From f92d0a7fb95d4bc597695bb75c39d4fa4a25e7ba Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 3 Oct 2026 10:49:02 +0000 Subject: [PATCH 34/57] Return a claim-bound history cursor after scope opening --- .../Api/CancellationScopeController.php | 7 +++ .../worker-protocol-api.openapi.yaml | 10 +++- .../Source/CancellationScopeAdmissionTest.php | 52 +++++++++++++++++++ 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/app/Http/Controllers/Api/CancellationScopeController.php b/app/Http/Controllers/Api/CancellationScopeController.php index 9801354b..89afb367 100644 --- a/app/Http/Controllers/Api/CancellationScopeController.php +++ b/app/Http/Controllers/Api/CancellationScopeController.php @@ -8,6 +8,7 @@ use App\Support\WorkerPollFence; use App\Support\WorkerProtocol; use App\Support\WorkerProtocolMutationRetrier; +use App\Support\WorkflowHistoryPageToken; use Illuminate\Http\JsonResponse; use Illuminate\Http\Request; use Illuminate\Support\Facades\DB; @@ -87,6 +88,12 @@ public function open(Request $request, string $taskId): JsonResponse ); })); + if (($result['opened'] ?? false) === true) { + $result['lease_owner'] = $validated['lease_owner']; + $result['workflow_task_attempt'] = (int) $validated['workflow_task_attempt']; + $result['history_refresh_page_token'] = WorkflowHistoryPageToken::encode(0); + } + return WorkerProtocol::json($result, ($result['opened'] ?? false) ? 200 : (($result['reason'] ?? null) === 'task_not_found' ? 404 : 409)); } diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 106d4c62..dc3c00fd 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "37" + version: "38" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -1862,19 +1862,25 @@ components: allOf: - $ref: "#/components/schemas/WorkerEnvelope" - type: object - required: [opened, duplicate, claim_released, task_id, workflow_run_id, history_event_id, scope_id, parent_scope_id, shield_parent, sequence, created_task_ids, reason] + required: [opened, duplicate, claim_released, task_id, workflow_run_id, workflow_task_attempt, lease_owner, history_event_id, scope_id, parent_scope_id, shield_parent, sequence, created_task_ids, history_refresh_page_token, reason] properties: opened: { type: boolean, const: true } duplicate: { type: boolean } claim_released: { type: boolean, const: false } task_id: { type: string, minLength: 1 } workflow_run_id: { type: string, minLength: 1 } + workflow_task_attempt: { type: integer, minimum: 1 } + lease_owner: { type: string, minLength: 1 } history_event_id: { type: string, minLength: 1 } scope_id: { type: string, minLength: 1 } parent_scope_id: { type: string, minLength: 1 } shield_parent: { type: boolean } sequence: { type: integer, minimum: 1 } created_task_ids: { type: array, maxItems: 0 } + history_refresh_page_token: + type: string + minLength: 1 + description: Opaque cursor from the original run's start. Fetch every page through the claim-bound history endpoint before entering the scope body. This cursor grants no claim authority and does not renew or release the task. reason: { type: "null" } CancellationScopeCheckpointRequest: description: Ordinary retained prefix before scope opening. Does not admit local callbacks. diff --git a/tests/Source/CancellationScopeAdmissionTest.php b/tests/Source/CancellationScopeAdmissionTest.php index 237f00f1..7c7c6ccf 100644 --- a/tests/Source/CancellationScopeAdmissionTest.php +++ b/tests/Source/CancellationScopeAdmissionTest.php @@ -217,6 +217,58 @@ public function test_scope_opening_records_once_and_preserves_the_live_claim(): $this->assertSame($opened['scope_id'], WorkflowHistoryEvent::query()->where('event_type', 'TimerScheduled')->sole()->payload['cancellation_scope_id']); } + public function test_scope_opening_refresh_cursor_replays_canonical_history_on_only_the_live_claim(): void + { + $task = $this->claim(capabilities: [CooperativeCancellationPolicy::CAPABILITY]); + $claim = WorkflowTask::query()->findOrFail($task['task_id']); + $originalLease = $claim->lease_expires_at->toISOString(); + $opened = $this->openScope($task)->assertOk() + ->assertJsonPath('lease_owner', $task['lease_owner']) + ->assertJsonPath('workflow_task_attempt', $task['workflow_task_attempt'])->json(); + $this->assertIsString($opened['history_refresh_page_token']); + $read = function (array $authority, string $token, string $namespace = 'default') use ($task) { + return $this->withHeaders(['X-Namespace' => $namespace, WorkerProtocol::HEADER => '1.20']) + ->postJson("/api/worker/workflow-tasks/{$task['task_id']}/history", [ + 'lease_owner' => $authority['lease_owner'], 'workflow_task_attempt' => $authority['workflow_task_attempt'], + 'next_history_page_token' => $token, 'history_page_size' => 1, + ]); + }; + $token = $opened['history_refresh_page_token']; + $events = []; + $seen = []; + do { + $this->assertArrayNotHasKey($token, $seen); + $seen[$token] = true; + $page = $read($task, $token)->assertOk()->json(); + $events = [...$events, ...$page['history_events']]; + $token = $page['next_history_page_token']; + } while ($token !== null); + $this->assertSame('StartAccepted', $events[0]['event_type']); + $this->assertContains('WorkflowStarted', array_column($events, 'event_type')); + $scope = array_values(array_filter($events, fn (array $event): bool => $event['event_type'] === 'CancellationScopeOpened')); + $this->assertCount(1, $scope); + $this->assertSame($opened['history_event_id'], $scope[0]['id']); + $this->assertSame($opened['scope_id'], $scope[0]['payload']['scope_id']); + $this->assertSame($originalLease, $claim->refresh()->lease_expires_at->toISOString()); + $this->assertSame('leased', $claim->status->value); + + WorkflowNamespace::query()->create(['name' => 'other', 'retention_days' => 30, 'status' => 'active']); + $read($task, $opened['history_refresh_page_token'], 'other')->assertNotFound(); + $read(array_replace($task, ['lease_owner' => 'other-worker']), $opened['history_refresh_page_token']) + ->assertStatus(409)->assertJsonPath('reason', 'lease_owner_mismatch'); + $claim->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $replacement = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => $task['lease_owner'], 'task_queue' => 'scope-admission', + ])->assertOk()->json('task'); + $read($task, $opened['history_refresh_page_token'])->assertStatus(409) + ->assertJsonPath('reason', 'workflow_task_attempt_mismatch'); + $replayed = $this->openScope($replacement)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('scope_id', $opened['scope_id'])->assertJsonPath('history_event_id', $opened['history_event_id']) + ->assertJsonPath('workflow_task_attempt', $replacement['workflow_task_attempt'])->json(); + $read($replacement, $replayed['history_refresh_page_token'])->assertOk(); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeOpened')->count()); + } + public function test_nested_shield_is_recorded_and_changed_replay_is_refused(): void { $task = $this->claim(); From b995b30ae7e327da6b2ccac1100b6058b2e0079e Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 3 Oct 2026 14:29:06 +0000 Subject: [PATCH 35/57] fix: observe original scoped activity cancellation receipts --- .github/workflows/phpunit-feature.yml | 2 +- .../Api/ActivityTaskController.php | 61 +++- tests/Feature/ActivityAttemptStatusTest.php | 28 ++ .../ScopedActivityCancellationReceiptTest.php | 285 ++++++++++++++++++ 4 files changed, 359 insertions(+), 17 deletions(-) create mode 100644 tests/Source/ScopedActivityCancellationReceiptTest.php diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index b1a0405f..f7a7cc1c 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -340,7 +340,7 @@ jobs: composer install --no-interaction --no-progress --prefer-dist mv vendor/durable-workflow/workflow /tmp/locked-workflow-package cp -a prepared-workflow-source vendor/durable-workflow/workflow - php vendor/bin/phpunit tests/Feature/PreparedLocalActivityProtocolTest.php tests/Feature/HistoryRetentionTest.php tests/Unit/WorkerPollFenceTest.php tests/Source/CancellationCascadeDiagnosticsTest.php tests/Source/PreparedLocalCancellationPolicyTest.php tests/Source/CancellationScopeAdmissionTest.php tests/Feature/WorkflowStreamsTest.php tests/Feature/WorkflowDebugTest.php --fail-on-skipped --log-junit "/evidence/$DW_PREPARED_DATABASE.xml" --colors=never + php vendor/bin/phpunit tests/Feature/PreparedLocalActivityProtocolTest.php tests/Feature/ActivityAttemptStatusTest.php tests/Feature/HistoryRetentionTest.php tests/Unit/WorkerPollFenceTest.php tests/Source/CancellationCascadeDiagnosticsTest.php tests/Source/PreparedLocalCancellationPolicyTest.php tests/Source/CancellationScopeAdmissionTest.php tests/Source/ScopedActivityCancellationReceiptTest.php tests/Feature/WorkflowStreamsTest.php tests/Feature/WorkflowDebugTest.php --fail-on-skipped --log-junit "/evidence/$DW_PREPARED_DATABASE.xml" --colors=never if [ "$DW_PREPARED_DATABASE" = sqlite ]; then php vendor/bin/phpunit tests/Feature/CooperativeCancellationProtocolTest.php tests/Feature/WorkflowWorkerProtocolTest.php tests/Feature/ActivityWorkerProtocolTest.php tests/Feature/SearchAttributeValueValidationTest.php tests/Feature/NamespaceDurableStateQuotaTest.php tests/Feature/RuntimeExternalPayloadTransportTest.php tests/Feature/RuntimePayloadCompletionUploadsTest.php tests/Unit/RuntimePayloadCompletionContextTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php --log-junit /evidence/affected-regression.xml --colors=never fi diff --git a/app/Http/Controllers/Api/ActivityTaskController.php b/app/Http/Controllers/Api/ActivityTaskController.php index 3452edb3..0fbe4cde 100644 --- a/app/Http/Controllers/Api/ActivityTaskController.php +++ b/app/Http/Controllers/Api/ActivityTaskController.php @@ -38,6 +38,8 @@ use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Support\ActivityCancellationAcknowledgement; +use Workflow\V2\Support\ActivityCancellationCompletion; +use Workflow\V2\Support\ActivityCancellationContext; use Workflow\V2\Support\ActivityRowLockOrder; use Workflow\V2\Support\CooperativeCancellationDelivery; use Workflow\V2\Support\WorkerProtocolVersion; @@ -617,33 +619,60 @@ private function activityCancellationReceipt( return null; } $run = WorkflowRun::query()->where('namespace', $namespace)->find($execution->workflow_run_id); - if ($run === null || ! is_string($run->cancellation_request_command_id)) { + if ($run === null) { return null; } - $context = CooperativeCancellationDelivery::context($run); - if ($context === null || $context->requestId !== $run->cancellation_request_command_id) { - return null; - } - $events = WorkflowHistoryEvent::query()->where('workflow_run_id', $run->id) - ->where('workflow_command_id', $context->requestId) - ->where('payload->activity_execution_id', $executionId) - ->whereIn('event_type', [HistoryEventType::ActivityCancelled->value, HistoryEventType::ActivityCancellationAcknowledged->value]) + // Observe this execution's original fence, even when no whole-run + // request exists or a later request owns the run's cleanup. Keep this + // frequent worker observation bounded to the relevant cancellation facts. + $events = $run->historyEvents()->where(function ($query) use ($run, $executionId): void { + $query->where(function ($query) use ($executionId): void { + $query->where('payload->activity_execution_id', $executionId) + ->whereIn('event_type', [HistoryEventType::ActivityCancelled->value, HistoryEventType::ActivityCancellationAcknowledged->value]); + })->orWhere(function ($query) use ($run): void { + $query->where('event_type', HistoryEventType::CooperativeCancellationRequested->value) + ->where('workflow_command_id', $run->cancellation_request_command_id); + }); + }) ->get(); - $cancelled = $events->first(fn (WorkflowHistoryEvent $event): bool => $event->event_type === HistoryEventType::ActivityCancelled - && ($event->payload['activity_attempt_id'] ?? null) === $attemptId); + $run->setRelation('historyEvents', $events); + $cancelled = $events->first(fn (WorkflowHistoryEvent $event): bool => $event->event_type === HistoryEventType::ActivityCancelled); $snapshot = $cancelled?->payload['activity_attempt'] ?? null; - if (! is_array($snapshot) || ($snapshot['id'] ?? null) !== $attemptId + if (! is_array($snapshot) || ($cancelled->payload['activity_attempt_id'] ?? null) !== $attemptId + || ($snapshot['id'] ?? null) !== $attemptId || ($snapshot['task_id'] ?? null) !== $taskId || ($snapshot['lease_owner'] ?? null) !== $leaseOwner || ($snapshot['activity_execution_id'] ?? null) !== $executionId || ($snapshot['status'] ?? null) !== 'cancelled') { return null; } - $ack = $events->first(fn (WorkflowHistoryEvent $event): bool => $event->event_type === HistoryEventType::ActivityCancellationAcknowledged - && ($event->payload['cancellation_history_event_id'] ?? null) === $cancelled->id - && ($event->payload['activity_attempt_id'] ?? null) === $attemptId); + $hasCanonicalContext = class_exists(ActivityCancellationContext::class) + && method_exists(ActivityCancellationContext::class, 'forEvent') + && method_exists(ActivityCancellationContext::class, 'rootRequestId'); + $context = $hasCanonicalContext + ? ActivityCancellationContext::forEvent($run, $cancelled) + : (array_key_exists('cancellation_scope', $cancelled->payload) ? null : CooperativeCancellationDelivery::context($run)); + if ($context === null || $cancelled->workflow_command_id !== $context->requestId) { + return null; + } + $rootRequestId = $hasCanonicalContext ? ActivityCancellationContext::rootRequestId($context) : $context->rootRequestId; + $ack = class_exists(ActivityCancellationCompletion::class) && method_exists(ActivityCancellationCompletion::class, 'stopReceipt') + ? ActivityCancellationCompletion::stopReceipt($run, $cancelled) + : $events->first(fn (WorkflowHistoryEvent $event): bool => $event->event_type === HistoryEventType::ActivityCancellationAcknowledged + && $event->sequence > $cancelled->sequence + && $event->workflow_command_id === $context->requestId + && ($event->payload['sequence'] ?? null) === ($cancelled->payload['sequence'] ?? null) + && ($event->payload['activity_execution_id'] ?? null) === $executionId + && ($event->payload['activity_attempt_id'] ?? null) === $attemptId + && ($event->payload['lease_owner'] ?? null) === $leaseOwner + && ($event->payload['cancellation_history_event_id'] ?? null) === $cancelled->id + && ($event->payload['request_id'] ?? null) === $context->requestId + && ($event->payload['root_request_id'] ?? null) === $rootRequestId + && ($event->payload['cleanup_deadline_at'] ?? null) === $context->deadline()->toISOString() + && ($event->payload['callback_state'] ?? null) === 'stopped' + && ($event->payload['evidence_source'] ?? null) === 'activity_worker'); return [ 'request_id' => $context->requestId, - 'root_request_id' => $context->rootRequestId, + 'root_request_id' => $rootRequestId, 'cleanup_deadline_at' => $context->deadline()->toISOString(), 'cancellation_history_event_id' => $cancelled->id, 'callback_state' => $ack === null ? 'unknown' : 'stopped', diff --git a/tests/Feature/ActivityAttemptStatusTest.php b/tests/Feature/ActivityAttemptStatusTest.php index 5e72887e..2abaacd3 100644 --- a/tests/Feature/ActivityAttemptStatusTest.php +++ b/tests/Feature/ActivityAttemptStatusTest.php @@ -331,6 +331,34 @@ public function test_stop_receipt_is_distinct_from_fencing_and_duplicate_keeps_o ->assertJsonPath('recorded', false); } + public static function mismatchedStopReceipts(): array + { + return [ + 'wrong owner' => ['lease_owner', 'other-owner'], + 'wrong root' => ['root_request_id', 'other-root'], + 'changed original deadline' => ['cleanup_deadline_at', '2099-01-01T00:00:00.000000Z'], + 'callback still running' => ['callback_state', 'running'], + 'wrong evidence source' => ['evidence_source', 'workflow_worker'], + ]; + } + + #[DataProvider('mismatchedStopReceipts')] + public function test_mismatched_stop_receipt_cannot_report_whole_run_callback_as_stopped(string $field, mixed $value): void + { + $this->requireAcknowledgementBackend(); + $task = $this->lease(capable: true); + $request = $this->cancelCooperatively($task); + $ack = $this->acknowledge($task, $request['request_id'])->assertOk()->json('history_event_id'); + $event = WorkflowHistoryEvent::query()->findOrFail($ack); + $payload = $event->payload; + $payload[$field] = $value; + $event->forceFill(['payload' => $payload])->save(); + $before = $this->snapshot($task); + $this->observe($task)->assertOk()->assertJsonPath('cancellation_acknowledgement.callback_state', 'unknown') + ->assertJsonPath('cancellation_acknowledgement.history_event_id', null); + $this->assertSame($before, $this->snapshot($task)); + } + public function test_late_stop_receipt_cannot_renew_original_deadline(): void { $this->requireAcknowledgementBackend(); diff --git a/tests/Source/ScopedActivityCancellationReceiptTest.php b/tests/Source/ScopedActivityCancellationReceiptTest.php new file mode 100644 index 00000000..8b572170 --- /dev/null +++ b/tests/Source/ScopedActivityCancellationReceiptTest.php @@ -0,0 +1,285 @@ + '1.20', + 'server.auth.driver' => 'token', + 'server.auth.role_tokens' => ['operator' => 'fixture-operator', 'worker' => 'fixture-worker'], + 'workflows.v2.types.workflows' => ['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class], + ]); + WorkflowNamespace::query()->create(['name' => 'default', 'retention_days' => 30, 'status' => 'active']); + $this->assertTrue(class_exists(ScopedActivityCancellation::class)); + $this->assertTrue(class_exists(ActivityCancellationContext::class)); + } + + public function test_scoped_receipt_and_original_owner_acknowledgement_leave_sibling_and_workflow_running(): void + { + [$workflowTask, $target, $sibling, $request, $root, $fence] = $this->scopedPair(); + $before = $this->snapshot($workflowTask, $target, $sibling); + $observed = $this->observe($target)->assertOk()->assertJsonPath('can_continue', false) + ->assertJsonPath('cancel_requested', true)->assertJsonPath('heartbeat_recorded', false) + ->assertJsonPath('cancellation_acknowledgement.callback_state', 'unknown') + ->assertJsonPath('cancellation_acknowledgement.request_id', $request->payload['request_id']) + ->assertJsonPath('cancellation_acknowledgement.root_request_id', $root->payload['request_id']) + ->assertJsonPath('cancellation_acknowledgement.cleanup_deadline_at', $fence['cleanup_deadline_at']) + ->assertJsonPath('cancellation_acknowledgement.cancellation_history_event_id', $fence['history_event_id']); + $this->assertSame($before, $this->snapshot($workflowTask, $target, $sibling)); + $this->assertNull(WorkflowRun::query()->findOrFail($target['run_id'])->cancellation_request_command_id); + $this->assertNotSame($root->payload['request_id'], $request->payload['request_id']); + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/ActivityCancellationReceipt', json_decode($observed->getContent(), flags: JSON_THROW_ON_ERROR)->cancellation_acknowledgement); + $this->observe($sibling)->assertOk()->assertJsonPath('can_continue', true) + ->assertJsonPath('cancel_requested', false)->assertJsonMissingPath('cancellation_acknowledgement'); + $this->withHeaders($this->headers())->postJson($this->path($target, 'complete'), $this->attempt($target) + ['result' => null]) + ->assertConflict()->assertJsonPath('recorded', false); + $this->withHeaders($this->headers())->postJson($this->path($target, 'fail'), $this->attempt($target) + ['failure' => ['message' => 'stale', 'non_retryable' => true]]) + ->assertConflict()->assertJsonPath('recorded', false); + $this->assertSame($before, $this->snapshot($workflowTask, $target, $sibling)); + + $ack = $this->acknowledge($target, $request->payload['request_id'])->assertOk() + ->assertJsonPath('acknowledged', true)->assertJsonPath('duplicate', false) + ->assertJsonPath('heartbeat_recorded', false)->json('history_event_id'); + $this->observe($target)->assertOk()->assertJsonPath('cancellation_acknowledgement.callback_state', 'stopped') + ->assertJsonPath('cancellation_acknowledgement.history_event_id', $ack) + ->assertJsonPath('cancellation_acknowledgement.received_after_deadline', false); + $this->acknowledge($target, $request->payload['request_id'])->assertOk() + ->assertJsonPath('duplicate', true)->assertJsonPath('history_event_id', $ack); + $after = $this->snapshot($workflowTask, $target, $sibling); + $this->assertSame($before[3] + 1, $after[3]); + unset($before[3], $after[3]); + $this->assertSame($before[2]['last_history_sequence'] + 1, $after[2]['last_history_sequence']); + foreach (['last_history_sequence', 'last_progress_at', 'updated_at'] as $field) { + unset($before[2][$field], $after[2][$field]); + } + $this->assertSame($before, $after); + $run = WorkflowRun::query()->findOrFail($target['run_id']); + $this->assertTrue(ActivityCancellationCompletion::resolved($run, $target['activity_execution_id'], CancellationScopeRequests::context($run, $fence['scope_id']))); + } + + public function test_later_whole_run_request_and_late_receipt_preserve_original_scoped_identity_and_deadline(): void + { + [, $target, , $request, $root, $fence] = $this->scopedPair(); + $this->travel(7)->seconds(); + $result = WorkflowStub::load($target['workflow_id'])->requestCancellation('later whole-run cleanup', 90); + $this->assertTrue($result->accepted()); + $run = WorkflowRun::query()->findOrFail($target['run_id']); + $this->assertNotSame($request->payload['request_id'], $run->cancellation_request_command_id); + $this->assertNotSame($root->payload['request_id'], $run->cancellation_request_command_id); + $this->observe($target)->assertOk()->assertJsonPath('cancellation_acknowledgement.request_id', $request->payload['request_id']) + ->assertJsonPath('cancellation_acknowledgement.root_request_id', $root->payload['request_id']) + ->assertJsonPath('cancellation_acknowledgement.cleanup_deadline_at', $fence['cleanup_deadline_at']); + $this->acknowledge($target, $run->cancellation_request_command_id)->assertConflict()->assertJsonPath('acknowledged', false); + $this->travel(24)->seconds(); + $ack = $this->acknowledge($target, $request->payload['request_id'])->assertOk()->json('history_event_id'); + $this->observe($target)->assertOk()->assertJsonPath('cancellation_acknowledgement.callback_state', 'stopped') + ->assertJsonPath('cancellation_acknowledgement.request_id', $request->payload['request_id']) + ->assertJsonPath('cancellation_acknowledgement.root_request_id', $root->payload['request_id']) + ->assertJsonPath('cancellation_acknowledgement.cleanup_deadline_at', $fence['cleanup_deadline_at']) + ->assertJsonPath('cancellation_acknowledgement.received_after_deadline', true); + $this->acknowledge($target, $request->payload['request_id'])->assertOk() + ->assertJsonPath('duplicate', true)->assertJsonPath('history_event_id', $ack); + $this->assertSame($request->id, CancellationScopeRequests::request($run->fresh(), $fence['scope_id'], '1.20', 300)->id); + $this->assertSame($run->cancellation_deadline_at->toISOString(), $run->fresh()->cancellation_deadline_at->toISOString()); + } + + public static function invalidAcknowledgements(): array + { + return [ + 'wrong original owner' => ['lease_owner', 'other-owner'], + 'wrong root' => ['root_request_id', 'other-root'], + 'wrong original request' => ['request_id', 'other-request'], + 'changed budget' => ['cleanup_deadline_at', '2099-01-01T00:00:00.000000Z'], + 'still running' => ['callback_state', 'running'], + 'wrong evidence source' => ['evidence_source', 'workflow_worker'], + 'wrong authored operation' => ['sequence', 99], + ]; + } + + #[DataProvider('invalidAcknowledgements')] + public function test_mismatched_stop_report_cannot_make_scoped_callback_look_stopped(string $field, mixed $value): void + { + [$workflowTask, $target, $sibling, $request] = $this->scopedPair(); + $ack = $this->acknowledge($target, $request->payload['request_id'])->assertOk()->json('history_event_id'); + $event = WorkflowHistoryEvent::query()->findOrFail($ack); + $payload = $event->payload; + $payload[$field] = $value; + $event->forceFill(['payload' => $payload])->save(); + $before = $this->snapshot($workflowTask, $target, $sibling); + $this->observe($target)->assertOk()->assertJsonPath('cancellation_acknowledgement.callback_state', 'unknown') + ->assertJsonPath('cancellation_acknowledgement.history_event_id', null); + $this->assertSame($before, $this->snapshot($workflowTask, $target, $sibling)); + } + + public static function invalidScopeFacts(): array + { + return [ + 'missing context' => ['cancellation_scope', null], + 'wrong run' => ['workflow_run_id', 'other-run'], + 'wrong accepted request' => ['request_id', 'other-request'], + 'wrong scope membership' => ['scope_id', 'other-scope'], + 'malformed request event ID' => ['request_history_event_id', []], + 'inflated authority budget' => ['authority_deadline_at', '2099-01-01T00:00:00.000000Z'], + ]; + } + + #[DataProvider('invalidScopeFacts')] + public function test_malformed_scoped_fact_never_falls_back_to_later_whole_run_request(string $field, mixed $value): void + { + [$workflowTask, $target, $sibling, $request, , $fence] = $this->scopedPair(); + $this->assertTrue(WorkflowStub::load($target['workflow_id'])->requestCancellation('later', 90)->accepted()); + $event = WorkflowHistoryEvent::query()->findOrFail($fence['history_event_id']); + $payload = $event->payload; + if ($field === 'cancellation_scope') { + $payload[$field] = $value; + } else { + $payload['cancellation_scope'][$field] = $value; + } + $event->forceFill(['payload' => $payload])->save(); + $before = $this->snapshot($workflowTask, $target, $sibling); + $this->observe($target)->assertOk()->assertJsonPath('can_continue', false) + ->assertJsonPath('cancel_requested', true)->assertJsonMissingPath('cancellation_acknowledgement'); + $this->acknowledge($target, $request->payload['request_id'])->assertConflict()->assertJsonPath('acknowledged', false); + $this->assertSame($before, $this->snapshot($workflowTask, $target, $sibling)); + } + + public function test_scoped_receipt_remains_bound_to_original_worker_role_owner_attempt_and_namespace(): void + { + [$workflowTask, $target, $sibling, $request] = $this->scopedPair(); + WorkflowNamespace::query()->create(['name' => 'other', 'retention_days' => 30, 'status' => 'active']); + $before = $this->snapshot($workflowTask, $target, $sibling); + $this->withHeaders($this->headers(role: 'operator'))->postJson($this->path($target), $this->attempt($target))->assertForbidden(); + $this->withHeaders($this->headers(role: 'operator'))->postJson($this->path($target, 'acknowledge-cancellation'), $this->attempt($target) + ['request_id' => $request->payload['request_id']])->assertForbidden(); + $this->withHeaders($this->headers(namespace: 'other'))->postJson($this->path($target), $this->attempt($target))->assertNotFound(); + $this->withHeaders($this->headers(namespace: 'other'))->postJson($this->path($target, 'acknowledge-cancellation'), $this->attempt($target) + ['request_id' => $request->payload['request_id']])->assertNotFound(); + $this->observe($target, ['lease_owner' => 'other-owner'])->assertConflict()->assertJsonPath('reason', 'lease_owner_mismatch'); + $this->acknowledge($target, $request->payload['request_id'], ['lease_owner' => 'other-owner'])->assertConflict(); + $this->observe($target, ['activity_attempt_id' => $sibling['activity_attempt_id']])->assertConflict()->assertJsonPath('reason', 'task_mismatch'); + $this->acknowledge($target, $request->payload['request_id'], ['activity_attempt_id' => $sibling['activity_attempt_id']])->assertConflict(); + $this->assertSame($before, $this->snapshot($workflowTask, $target, $sibling)); + } + + /** Canonical open/checkpoint/claims use HTTP. Scoped request/fence are still internal Native primitives. */ + private function scopedPair(): array + { + $this->withHeaders($this->headers(role: 'operator')) + ->postJson('/api/workflows', ['workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'scoped-receipt', 'input' => ['Ada']])->assertCreated(); + $this->withHeaders($this->headers())->postJson('/api/worker/register', [ + 'worker_id' => 'scope-owner', 'task_queue' => 'scoped-receipt', 'runtime' => 'php', + 'supported_workflow_types' => ['tests.external-greeting-workflow'], + 'supported_activity_types' => ['opaque-scoped-activity'], + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY], + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + 'max_concurrent_workflow_tasks' => 1, + 'process_metrics' => ['host' => 'fixture', 'process_started_at' => now()->toISOString(), 'process_id' => 10], + ])->assertCreated(); + $task = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'scope-owner', 'task_queue' => 'scoped-receipt', + ])->assertOk()->json('task'); + $scopes = []; + foreach (range(1, 3) as $sequence) { + $scopes[] = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/open", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'sequence' => $sequence, 'parent_scope_id' => $sequence === 2 ? $scopes[0] : 'root', + ])->assertOk()->assertJsonPath('opened', true)->json('scope_id'); + } + $commands = []; + foreach ([$scopes[1], $scopes[2]] as $scope) { + $commands[] = ['type' => 'schedule_activity', 'activity_type' => 'opaque-scoped-activity', + 'arguments' => Serializer::serializeWithCodec('avro', ['Ada']), 'payload_codec' => 'avro', + 'queue' => 'scoped-receipt', 'cancellation_scope_id' => $scope, + 'cancellation_policy' => 'wait_cancellation_completed', 'schedule_to_close_timeout' => 120]; + } + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/checkpoint", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'checkpoint_id' => 'scoped-activities', 'start_sequence' => 4, 'commands' => $commands, + ])->assertOk()->assertJsonPath('checkpointed', true); + $activities = []; + foreach (range(1, 2) as $_) { + $activities[] = $this->withHeaders($this->headers())->postJson('/api/worker/activity-tasks/poll', [ + 'worker_id' => 'scope-owner', 'task_queue' => 'scoped-receipt', + ])->assertOk()->json('task'); + } + usort($activities, fn (array $a, array $b): int => ActivityExecution::query()->findOrFail($a['activity_execution_id'])->sequence + <=> ActivityExecution::query()->findOrFail($b['activity_execution_id'])->sequence); + $run = WorkflowRun::query()->findOrFail($task['run_id']); + $root = CancellationScopeRequests::request($run, $scopes[0], '1.20', 30, 'original scoped cleanup'); + $request = CancellationScopeRequests::request($run, $scopes[1], '1.20', 300, parentScopeId: $scopes[0]); + $fence = ScopedActivityCancellation::fence($run, WorkflowTask::query()->findOrFail($task['task_id']), + $activities[0]['activity_execution_id'], $scopes[1], $request->payload['request_id'], '1.20'); + $this->assertTrue($fence['fenced']); + $this->assertTrue($fence['waiting_for_stop']); + + return [$task, $activities[0], $activities[1], $request, $root, $fence]; + } + + private function headers(string $role = 'worker', string $namespace = 'default'): array + { + return ['Authorization' => 'Bearer fixture-'.$role, 'X-Namespace' => $namespace, + 'X-Durable-Workflow-Control-Plane-Version' => '2', WorkerProtocol::HEADER => '1.20']; + } + + private function path(array $task, string $action = 'status'): string + { + return "/api/worker/activity-tasks/{$task['task_id']}/{$action}"; + } + + private function attempt(array $task): array + { + return ['activity_attempt_id' => $task['activity_attempt_id'], 'lease_owner' => $task['lease_owner']]; + } + + private function observe(array $task, array $overrides = []): TestResponse + { + return $this->withHeaders($this->headers())->postJson($this->path($task), array_replace($this->attempt($task), $overrides)); + } + + private function acknowledge(array $task, string $requestId, array $overrides = []): TestResponse + { + return $this->withHeaders($this->headers())->postJson($this->path($task, 'acknowledge-cancellation'), + array_replace($this->attempt($task) + ['request_id' => $requestId], $overrides)); + } + + private function snapshot(array $workflowTask, array $target, array $sibling): array + { + return [WorkflowTask::query()->findOrFail($workflowTask['task_id'])->getRawOriginal(), + array_map(fn (array $task): array => [ + ActivityExecution::query()->findOrFail($task['activity_execution_id'])->getRawOriginal(), + ActivityAttempt::query()->findOrFail($task['activity_attempt_id'])->getRawOriginal(), + WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(), + ], [$target, $sibling]), + WorkflowRun::query()->findOrFail($target['run_id'])->getRawOriginal(), + WorkflowHistoryEvent::query()->where('workflow_run_id', $target['run_id'])->count()]; + } +} From f8749cad86ea26de68c2d0acdeb8c97382a0db4c Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 3 Oct 2026 18:13:44 +0000 Subject: [PATCH 36/57] feat: expose authenticated scoped cancellation preparation and recording --- .github/workflows/phpunit-feature.yml | 2 +- .../Api/CancellationScopeController.php | 86 ++++++ .../Middleware/EnforceStorageAdmission.php | 2 + .../worker-protocol-api.openapi.yaml | 138 ++++++++- routes/api.php | 2 + .../CooperativeCancellationProtocolTest.php | 12 + .../Source/CancellationScopeDeliveryTest.php | 261 ++++++++++++++++++ .../ScopedActivityCancellationReceiptTest.php | 36 ++- 8 files changed, 535 insertions(+), 4 deletions(-) create mode 100644 tests/Source/CancellationScopeDeliveryTest.php diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index f7a7cc1c..7676dc1e 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -340,7 +340,7 @@ jobs: composer install --no-interaction --no-progress --prefer-dist mv vendor/durable-workflow/workflow /tmp/locked-workflow-package cp -a prepared-workflow-source vendor/durable-workflow/workflow - php vendor/bin/phpunit tests/Feature/PreparedLocalActivityProtocolTest.php tests/Feature/ActivityAttemptStatusTest.php tests/Feature/HistoryRetentionTest.php tests/Unit/WorkerPollFenceTest.php tests/Source/CancellationCascadeDiagnosticsTest.php tests/Source/PreparedLocalCancellationPolicyTest.php tests/Source/CancellationScopeAdmissionTest.php tests/Source/ScopedActivityCancellationReceiptTest.php tests/Feature/WorkflowStreamsTest.php tests/Feature/WorkflowDebugTest.php --fail-on-skipped --log-junit "/evidence/$DW_PREPARED_DATABASE.xml" --colors=never + php vendor/bin/phpunit tests/Feature/PreparedLocalActivityProtocolTest.php tests/Feature/ActivityAttemptStatusTest.php tests/Feature/HistoryRetentionTest.php tests/Unit/WorkerPollFenceTest.php tests/Source/CancellationCascadeDiagnosticsTest.php tests/Source/PreparedLocalCancellationPolicyTest.php tests/Source/CancellationScopeAdmissionTest.php tests/Source/ScopedActivityCancellationReceiptTest.php tests/Source/CancellationScopeDeliveryTest.php tests/Feature/WorkflowStreamsTest.php tests/Feature/WorkflowDebugTest.php --fail-on-skipped --log-junit "/evidence/$DW_PREPARED_DATABASE.xml" --colors=never if [ "$DW_PREPARED_DATABASE" = sqlite ]; then php vendor/bin/phpunit tests/Feature/CooperativeCancellationProtocolTest.php tests/Feature/WorkflowWorkerProtocolTest.php tests/Feature/ActivityWorkerProtocolTest.php tests/Feature/SearchAttributeValueValidationTest.php tests/Feature/NamespaceDurableStateQuotaTest.php tests/Feature/RuntimeExternalPayloadTransportTest.php tests/Feature/RuntimePayloadCompletionUploadsTest.php tests/Unit/RuntimePayloadCompletionContextTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php --log-junit /evidence/affected-regression.xml --colors=never fi diff --git a/app/Http/Controllers/Api/CancellationScopeController.php b/app/Http/Controllers/Api/CancellationScopeController.php index 89afb367..14d9e6b8 100644 --- a/app/Http/Controllers/Api/CancellationScopeController.php +++ b/app/Http/Controllers/Api/CancellationScopeController.php @@ -13,6 +13,7 @@ use Illuminate\Http\Request; use Illuminate\Support\Facades\DB; use Workflow\V2\Contracts\CancellationScopeTaskBridge; +use Workflow\V2\Contracts\PreparedCancellationScopeTaskBridge; use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; @@ -22,6 +23,91 @@ final class CancellationScopeController { public function __construct(private readonly WorkerProtocolMutationRetrier $mutations) {} + public function prepare(Request $request, string $taskId): JsonResponse + { + return $this->delivery($request, $taskId, true); + } + + public function deliver(Request $request, string $taskId): JsonResponse + { + return $this->delivery($request, $taskId, false); + } + + private function delivery(Request $request, string $taskId, bool $preparing): JsonResponse + { + if ($response = WorkerProtocol::rejectUnsupported($request)) { + return $response; + } + $version = WorkerProtocol::requestVersion($request); + $refused = static fn (string $reason): array => ['prepared' => false, 'delivered' => false, + 'claim_released' => false, 'task_id' => $taskId, 'created_task_ids' => [], 'reason' => $reason]; + if (! WorkerProtocol::versionMeetsMinimum($version, CooperativeCancellationPolicy::MINIMUM_PROTOCOL_VERSION)) { + return WorkerProtocol::json($refused('cancellation_scope_requires_protocol_1_20'), 409); + } + $bridge = app(WorkflowTaskBridge::class); + if (! $bridge instanceof PreparedCancellationScopeTaskBridge) { + return WorkerProtocol::json($refused('cancellation_scope_delivery_unavailable') + + ['unavailable' => ['installed_runtime_scope_preparation_delivery']], 409); + } + $validated = $request->validate([ + 'lease_owner' => ['required', 'string', 'max:255'], + 'workflow_task_attempt' => ['required', 'integer', 'min:1'], + 'scope_id' => ['required', 'string', 'max:255'], + 'request_id' => ['required', 'string', 'max:255'], + 'sequence' => ['required', 'integer', 'min:1'], + 'call_kind' => ['required', 'in:activity,local_activity,timer,condition,signal,child,parallel,selection_handle'], + 'sequence_span' => ['sometimes', 'required', 'integer', 'min:1'], + 'operation_sequence' => ['nullable', 'integer', 'min:1'], + 'operation_sequence_span' => ['sometimes', 'required', 'integer', 'min:1'], + ]); + $namespace = (string) $request->attributes->get('namespace'); + // Preparation owns its transaction and commits before Activity locks. + // Check tenant/issued capability/registration without an outer lock; + // Native rechecks the live owner and attempt under its run/task locks. + $result = $this->mutations->run(function () use ($namespace, $taskId, $validated, $version, $bridge, $preparing, $refused): array { + $task = NamespaceWorkflowScope::taskQuery($namespace)->find($taskId); + if (! $task instanceof WorkflowTask) { + return $refused('task_not_found'); + } + $run = WorkflowRun::query()->where('namespace', $namespace)->find($task->workflow_run_id); + if ($run === null || ! NamespaceWorkflowScope::workflowBound($namespace, $run->workflow_instance_id)) { + return $refused('task_not_found'); + } + if ($task->lease_owner !== $validated['lease_owner']) { + return $refused('lease_owner_mismatch'); + } + if ($task->attempt_count !== (int) $validated['workflow_task_attempt']) { + return $refused('workflow_task_attempt_mismatch'); + } + if (! CooperativeCancellationPolicy::claimSupportsCancellation($task)) { + return $refused('active_claim_cancellation_not_supported'); + } + $worker = WorkerRegistration::query()->where('namespace', $namespace) + ->where('worker_id', $validated['lease_owner'])->first(); + if (! $worker instanceof WorkerRegistration || ! WorkerPollFence::isFresh($worker)) { + return $refused('stale_worker_registration'); + } + $method = $preparing ? 'prepareCancellationScopeDelivery' : 'deliverCancellationScope'; + + return $bridge->$method($taskId, $validated['lease_owner'], (int) $validated['workflow_task_attempt'], + $validated['scope_id'], $validated['request_id'], (int) $validated['sequence'], $validated['call_kind'], + (int) ($validated['sequence_span'] ?? 1), isset($validated['operation_sequence']) ? (int) $validated['operation_sequence'] : null, + (int) ($validated['operation_sequence_span'] ?? 1), $version); + }); + if (($result['prepared'] ?? false) === true) { + $result['lease_owner'] = $validated['lease_owner']; + $result['workflow_task_attempt'] = (int) $validated['workflow_task_attempt']; + $result['history_refresh_page_token'] = WorkflowHistoryPageToken::encode(0); + } + $accepted = $preparing ? ($result['prepared'] ?? false) : ($result['delivered'] ?? false); + $pending = ! $preparing && ($result['prepared'] ?? false) === true + && ($result['reason'] ?? null) === 'cancellation_scope_activity_stop_not_acknowledged'; + $successful = ($accepted && ($result['reason'] ?? null) === null) || $pending; + + return WorkerProtocol::json($result, $successful ? 200 + : (($result['reason'] ?? null) === 'task_not_found' ? 404 : 409)); + } + public function open(Request $request, string $taskId): JsonResponse { if ($response = WorkerProtocol::rejectUnsupported($request)) { diff --git a/app/Http/Middleware/EnforceStorageAdmission.php b/app/Http/Middleware/EnforceStorageAdmission.php index 908cd797..f59a6c0d 100644 --- a/app/Http/Middleware/EnforceStorageAdmission.php +++ b/app/Http/Middleware/EnforceStorageAdmission.php @@ -21,6 +21,8 @@ final class EnforceStorageAdmission 'WorkerController@checkpointLocalActivityGroup', 'WorkerController@checkpointCancellationScopePrefix', 'CancellationScopeController@open', + 'CancellationScopeController@prepare', + 'CancellationScopeController@deliver', 'PreparedLocalActivityController@prepare', 'PreparedLocalActivityController@recover', 'PreparedLocalActivityController@control', diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index dc3c00fd..4d34b348 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "38" + version: "39" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -507,6 +507,65 @@ paths: "404": { $ref: "#/components/responses/WorkerError" } "409": { $ref: "#/components/responses/WorkerError" } "422": { $ref: "#/components/responses/WorkerError" } + /worker/workflow-tasks/{taskId}/cancellation-scopes/prepare: + post: + operationId: prepareWorkflowCancellationScopeDelivery + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: cooperative_cancellation + description: >- + Unfrozen candidate preparation. Authenticate the namespace, issued capable workflow claim and fresh registration. + The optional PreparedCancellationScopeTaskBridge independently commits the first context, call/range, + deadline and whole direct-scope Activity member set before operation effects. Retry reuses that preparation. + This route does not fence operations, stop callbacks or advertise scope execution. The claim remains leased. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/CancellationScopeDeliveryRequest" } + responses: + "200": + description: Original preparation committed or reused, without renewing or releasing the claim. + content: + application/json: + schema: { $ref: "#/components/schemas/CancellationScopeDeliveryResponse" } + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/cancellation-scopes/deliver: + post: + operationId: recordWorkflowCancellationScopeDelivery + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: cooperative_cancellation + description: >- + Unfrozen candidate recording. Recheck the original issued claim and require the retained preparation. + Record the delivery only after every original member's policy proof exists. A pending original-owner stop + receipt returns HTTP200 with delivered=false and the original prepared context. Missing operation actors + return HTTP409 with explicit unavailable diagnostics and no delivery marker. No effect dispatch or callback + supervision is implied. Refresh canonical history through the returned cursor before injecting cancellation. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/CancellationScopeDeliveryRequest" } + responses: + "200": + description: Original delivery recorded/reused or original-owner stop proof still pending. The claim remains leased. + content: + application/json: + schema: { $ref: "#/components/schemas/CancellationScopeDeliveryResponse" } + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } /worker/workflow-tasks/{taskId}/cancellation-scopes/checkpoint: post: operationId: checkpointCancellationScopePrefix @@ -1849,6 +1908,83 @@ components: operation_sequence: { type: "null" } operation_sequence_span: { type: "null" } reason: { type: string, enum: [cancellation_waiting_for_child, cancellation_waiting_for_activity] } + CancellationScopeDeliveryRequest: + allOf: + - $ref: "#/components/schemas/CooperativeCancellationDeliveryRequest" + - type: object + required: [scope_id] + properties: + scope_id: { type: string, minLength: 1, maxLength: 255 } + lease_owner: { type: string, minLength: 1, maxLength: 255 } + request_id: { type: string, minLength: 1, maxLength: 255 } + sequence_span: { type: integer, minimum: 1, default: 1 } + operation_sequence_span: { type: integer, minimum: 1, default: 1 } + CancellationScopeDeliveryResponse: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - type: object + required: [prepared, delivered, claim_released, task_id, created_task_ids, reason] + properties: + prepared: { type: boolean } + delivered: { type: boolean } + claim_released: { type: boolean, const: false } + task_id: { type: string, minLength: 1 } + created_task_ids: { type: array, maxItems: 0 } + reason: { type: [string, "null"] } + unavailable: { type: array, items: { type: string, minLength: 1 } } + if: + required: [prepared] + properties: + prepared: { const: true } + then: + required: [workflow_run_id, preparation_history_event_id, scope_id, request_id, cancellation, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span, authority_deadline_at, activity_members, lease_owner, workflow_task_attempt, history_refresh_page_token] + properties: + workflow_run_id: { type: string, minLength: 1 } + preparation_history_event_id: { type: string, minLength: 1 } + history_event_id: { type: string, minLength: 1 } + scope_id: { type: string, minLength: 1 } + request_id: { type: string, minLength: 1 } + cancellation: + type: object + required: [schema, root_context, lineage] + properties: + schema: { const: durable-workflow.scoped-cancellation-context/v1 } + root_context: + type: object + required: [request_id, cleanup_deadline_at] + properties: + request_id: { type: string, minLength: 1 } + cleanup_deadline_at: { type: string, format: date-time } + lineage: + type: array + minItems: 1 + items: + type: object + required: [request_id, workflow_instance_id, workflow_run_id, scope_id, cleanup_deadline_at] + properties: + request_id: { type: string, minLength: 1 } + workflow_instance_id: { type: string, minLength: 1 } + workflow_run_id: { type: string, minLength: 1 } + scope_id: { type: string, minLength: 1 } + cleanup_deadline_at: { type: string, format: date-time } + sequence: { type: integer, minimum: 1 } + call_kind: { type: string, enum: [activity, local_activity, timer, condition, signal, child, parallel, selection_handle] } + sequence_span: { type: integer, minimum: 1 } + operation_sequence: { type: [integer, "null"], minimum: 1 } + operation_sequence_span: { type: integer, minimum: 1 } + authority_deadline_at: { type: string, format: date-time } + activity_members: + type: array + items: + type: object + required: [sequence, activity_execution_id, descriptor_hash] + properties: + sequence: { type: integer, minimum: 1 } + activity_execution_id: { type: string, minLength: 1 } + descriptor_hash: { type: string, pattern: "^[a-f0-9]{64}$" } + lease_owner: { type: string, minLength: 1 } + workflow_task_attempt: { type: integer, minimum: 1 } + history_refresh_page_token: { type: string, minLength: 1 } CancellationScopeOpeningRequest: type: object required: [lease_owner, workflow_task_attempt, sequence] diff --git a/routes/api.php b/routes/api.php index 60fa2274..e8b87409 100644 --- a/routes/api.php +++ b/routes/api.php @@ -237,6 +237,8 @@ Route::post('/workflow-tasks/{taskId}/heartbeat', [WorkerController::class, 'heartbeatWorkflowTask']); Route::post('/workflow-tasks/{taskId}/deliver-cancellation', [CooperativeCancellationController::class, 'deliver']); Route::post('/workflow-tasks/{taskId}/cancellation-scopes/open', [CancellationScopeController::class, 'open']); + Route::post('/workflow-tasks/{taskId}/cancellation-scopes/prepare', [CancellationScopeController::class, 'prepare']); + Route::post('/workflow-tasks/{taskId}/cancellation-scopes/deliver', [CancellationScopeController::class, 'deliver']); Route::post('/workflow-tasks/{taskId}/cancellation-scopes/checkpoint', [WorkerController::class, 'checkpointCancellationScopePrefix']); Route::post('/workflow-tasks/{taskId}/local-activities/prepare', [PreparedLocalActivityController::class, 'prepare']); Route::post('/workflow-tasks/{taskId}/local-activities/checkpoint', [WorkerController::class, 'checkpointLocalActivityPrefix']); diff --git a/tests/Feature/CooperativeCancellationProtocolTest.php b/tests/Feature/CooperativeCancellationProtocolTest.php index 6cdbaadc..98efa029 100644 --- a/tests/Feature/CooperativeCancellationProtocolTest.php +++ b/tests/Feature/CooperativeCancellationProtocolTest.php @@ -59,6 +59,18 @@ public function test_request_before_claim_routes_only_to_capable_workers_and_kee $this->assertSame(1, $this->eventCount($runId, HistoryEventType::CooperativeCancellationRequested)); } + public function test_scoped_transport_refuses_an_installed_bridge_without_the_optional_role(): void + { + $this->app->instance(WorkflowTaskBridge::class, \Mockery::mock(WorkflowTaskBridge::class)); + $before = WorkflowHistoryEvent::query()->count(); + foreach (['prepare', 'deliver'] as $phase) { + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/absent/cancellation-scopes/{$phase}", []) + ->assertConflict()->assertJsonPath('reason', 'cancellation_scope_delivery_unavailable') + ->assertJsonPath('unavailable', ['installed_runtime_scope_preparation_delivery']); + } + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + public function test_active_claim_observes_request_on_heartbeat_and_cached_poll_without_losing_lease(): void { [$workflowId] = $this->start(); diff --git a/tests/Source/CancellationScopeDeliveryTest.php b/tests/Source/CancellationScopeDeliveryTest.php new file mode 100644 index 00000000..793f8c74 --- /dev/null +++ b/tests/Source/CancellationScopeDeliveryTest.php @@ -0,0 +1,261 @@ + '1.20', + 'server.auth.driver' => 'token', + 'server.auth.role_tokens' => ['operator' => 'fixture-operator', 'worker' => 'fixture-worker'], + 'workflows.v2.types.workflows' => ['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class]]); + WorkflowNamespace::query()->create(['name' => 'default', 'retention_days' => 30, 'status' => 'active']); + } + + public function test_preparation_and_delivery_reuse_first_boundary_without_releasing_or_renewing_claim(): void + { + [$task, $scope, $request] = $this->claim(); + $before = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + $prepared = $this->mutate('prepare', $task, $scope, $request)->assertOk() + ->assertJsonPath('prepared', true)->assertJsonPath('delivered', false) + ->assertJsonPath('claim_released', false)->assertJsonPath('activity_members', [])->json(); + $this->assertContract($prepared); + $this->mutate('prepare', $task, $scope, $request)->assertOk() + ->assertJsonPath('preparation_history_event_id', $prepared['preparation_history_event_id']) + ->assertJsonPath('cancellation', $prepared['cancellation']) + ->assertJsonPath('authority_deadline_at', $prepared['authority_deadline_at']); + $delivery = $this->mutate('deliver', $task, $scope, $request)->assertOk()->assertJsonPath('delivered', true)->json(); + $this->assertContract($delivery); + $this->mutate('deliver', $task, $scope, $request)->assertOk()->assertJsonPath('history_event_id', $delivery['history_event_id']); + $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDeliveryPrepared')->count()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDelivered')->count()); + $this->assertSame($prepared['preparation_history_event_id'], $delivery['preparation_history_event_id']); + } + + public static function phases(): array + { + return [['prepare'], ['deliver']]; + } + + #[DataProvider('phases')] + public function test_invalid_authority_and_namespace_are_refused_without_effects(string $phase): void + { + [$task, $scope, $request] = $this->claim(); + WorkflowNamespace::query()->create(['name' => 'other', 'retention_days' => 30, 'status' => 'active']); + $before = WorkflowHistoryEvent::query()->count(); + $original = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + $this->mutate($phase, $task, $scope, $request, ['lease_owner' => 'other'])->assertConflict()->assertJsonPath('reason', 'lease_owner_mismatch'); + $this->mutate($phase, $task, $scope, $request, ['workflow_task_attempt' => 99])->assertConflict()->assertJsonPath('reason', 'workflow_task_attempt_mismatch'); + $this->mutate($phase, $task, $scope, $request, namespace: 'other')->assertNotFound()->assertJsonPath('reason', 'task_not_found'); + $this->mutate($phase, $task, $scope, $request, role: 'operator')->assertForbidden(); + $this->mutate($phase, $task, $scope, $request, version: '1.19')->assertConflict()->assertJsonPath('reason', 'cancellation_scope_requires_protocol_1_20'); + $this->mutate($phase, $task, $scope, $request, version: '1.21')->assertBadRequest(); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertSame($original, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + } + + #[DataProvider('phases')] + public function test_claim_capability_and_fresh_registration_are_required(string $phase): void + { + [$task, $scope, $request] = $this->claim(); + $before = WorkflowHistoryEvent::query()->count(); + $claim = WorkflowTask::query()->findOrFail($task['task_id']); + $payload = $claim->payload; + $claim->forceFill(['payload' => []])->save(); + $this->mutate($phase, $task, $scope, $request)->assertConflict()->assertJsonPath('reason', 'active_claim_cancellation_not_supported'); + $claim->forceFill(['payload' => $payload])->save(); + WorkerRegistration::query()->where('worker_id', 'delivery-owner')->update(['last_heartbeat_at' => now()->subHour()]); + $this->mutate($phase, $task, $scope, $request)->assertConflict()->assertJsonPath('reason', 'stale_worker_registration'); + WorkerRegistration::query()->where('worker_id', 'delivery-owner')->update(['last_heartbeat_at' => now()]); + $claim->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $this->mutate($phase, $task, $scope, $request)->assertConflict()->assertJsonPath('reason', 'cancellation_scope_workflow_claim_mismatch'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + #[DataProvider('phases')] + public function test_missing_optional_backend_is_explicit(string $phase): void + { + [$task, $scope, $request] = $this->claim(); + $before = WorkflowHistoryEvent::query()->count(); + $this->app->instance(WorkflowTaskBridge::class, \Mockery::mock(WorkflowTaskBridge::class)); + $this->mutate($phase, $task, $scope, $request)->assertConflict()->assertJsonPath('reason', 'cancellation_scope_delivery_unavailable') + ->assertJsonPath('unavailable', ['installed_runtime_scope_preparation_delivery']); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_delivery_without_preparation_and_changed_retry_are_refused(): void + { + [$task, $scope, $request] = $this->claim(); + $this->mutate('deliver', $task, $scope, $request)->assertConflict()->assertJsonPath('reason', 'cancellation_scope_delivery_not_prepared'); + $prepared = $this->mutate('prepare', $task, $scope, $request)->assertOk()->json(); + $before = WorkflowHistoryEvent::query()->count(); + $this->mutate('prepare', $task, $scope, $request, ['sequence' => 3])->assertConflict(); + $this->mutate('deliver', $task, $scope, $request, ['sequence' => 3])->assertConflict(); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->mutate('prepare', $task, $scope, $request)->assertOk()->assertJsonPath('preparation_history_event_id', $prepared['preparation_history_event_id']); + } + + public function test_preparation_and_delivery_remain_available_during_storage_drain(): void + { + [$task, $scope, $request] = $this->claim(); + $this->configureStoragePressure('draining'); + try { + $this->mutate('prepare', $task, $scope, $request)->assertOk()->assertJsonPath('prepared', true); + $this->mutate('deliver', $task, $scope, $request)->assertOk()->assertJsonPath('delivered', true); + } finally { + $this->removeStoragePressure(); + } + } + + public static function malformed(): array + { + return [[['scope_id' => '']], [['request_id' => null]], [['sequence' => 0]], [['call_kind' => 'unknown']], + [['sequence_span' => 0]], [['operation_sequence' => 0]], [['workflow_task_attempt' => 0]]]; + } + + #[DataProvider('malformed')] + public function test_malformed_boundary_is_rejected_before_history(array $overrides): void + { + [$task, $scope, $request] = $this->claim(); + $before = WorkflowHistoryEvent::query()->count(); + foreach (['prepare', 'deliver'] as $phase) { + $this->mutate($phase, $task, $scope, $request, $overrides)->assertUnprocessable(); + } + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + #[DataProvider('phases')] + public function test_claim_replaced_after_server_checks_is_refused_by_native_before_commit(string $phase): void + { + [$task, $scope, $request] = $this->claim(); + if ($phase === 'deliver') { + $this->mutate('prepare', $task, $scope, $request)->assertOk(); + } + $before = WorkflowHistoryEvent::query()->count(); + $native = app(WorkflowTaskBridge::class); + $method = $phase === 'prepare' ? 'prepareCancellationScopeDelivery' : 'deliverCancellationScope'; + $bridge = \Mockery::mock(WorkflowTaskBridge::class.', '.PreparedCancellationScopeTaskBridge::class); + $bridge->shouldReceive($method)->once()->andReturnUsing(function (...$arguments) use ($task, $native, $method): array { + $this->assertSame(0, DB::connection()->transactionLevel()); + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_owner' => 'replacement'])->save(); + + return $native->$method(...$arguments); + }); + $this->app->instance(WorkflowTaskBridge::class, $bridge); + $this->mutate($phase, $task, $scope, $request)->assertConflict()->assertJsonPath('reason', 'cancellation_scope_workflow_claim_mismatch'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_replacement_claim_reuses_preparation_and_original_deadline(): void + { + [$task, $scope, $request] = $this->claim(); + $prepared = $this->mutate('prepare', $task, $scope, $request)->assertOk()->json(); + $this->travel(1)->seconds(); + $claim = WorkflowTask::query()->findOrFail($task['task_id']); + $claim->forceFill(['attempt_count' => $task['workflow_task_attempt'] + 1])->save(); + CooperativeCancellationPolicy::bindClaim($claim, CooperativeCancellationPolicy::workerSnapshot( + WorkerRegistration::query()->where('worker_id', 'delivery-owner')->sole(), '1.20')); + $this->mutate('prepare', $task, $scope, $request)->assertConflict()->assertJsonPath('reason', 'workflow_task_attempt_mismatch'); + $task['workflow_task_attempt']++; + $this->mutate('prepare', $task, $scope, $request)->assertOk() + ->assertJsonPath('preparation_history_event_id', $prepared['preparation_history_event_id']) + ->assertJsonPath('cancellation', $prepared['cancellation']) + ->assertJsonPath('authority_deadline_at', $prepared['authority_deadline_at']); + $this->mutate('deliver', $task, $scope, $request)->assertOk()->assertJsonPath('delivered', true); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDeliveryPrepared')->count()); + } + + public function test_missing_timer_actor_keeps_preparation_and_returns_explicit_diagnostic(): void + { + [$task, $scope, $request] = $this->claim(timer: true); + $prepared = $this->mutate('prepare', $task, $scope, $request, ['call_kind' => 'timer'])->assertOk()->json(); + $response = $this->mutate('deliver', $task, $scope, $request, ['call_kind' => 'timer'])->assertConflict() + ->assertJsonPath('delivered', false)->assertJsonPath('prepared', true) + ->assertJsonPath('reason', 'cancellation_scope_operation_delivery_unavailable') + ->assertJsonPath('unavailable', ['scoped_timer_delivery']) + ->assertJsonPath('preparation_history_event_id', $prepared['preparation_history_event_id'])->json(); + $this->assertContract($response); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDelivered')->count()); + $this->assertSame('pending', WorkflowTimer::query()->sole()->status->value); + } + + private function claim(bool $timer = false): array + { + $this->withHeaders($this->headers('operator'))->postJson('/api/workflows', [ + 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'scoped-delivery', 'input' => ['Ada'], + ])->assertCreated(); + $this->withHeaders($this->headers())->postJson('/api/worker/register', [ + 'worker_id' => 'delivery-owner', 'task_queue' => 'scoped-delivery', 'runtime' => 'php', + 'supported_workflow_types' => ['tests.external-greeting-workflow'], + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY], + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), 'max_concurrent_workflow_tasks' => 1, + ])->assertCreated(); + $task = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'delivery-owner', 'task_queue' => 'scoped-delivery', + ])->assertOk()->json('task'); + $scope = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/open", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], 'sequence' => 1, + ])->assertOk()->json('scope_id'); + if ($timer) { + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/checkpoint", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'checkpoint_id' => 'timer-prefix', 'start_sequence' => 2, + 'commands' => [['type' => 'start_timer', 'delay_seconds' => 60, 'cancellation_scope_id' => $scope]], + ])->assertOk()->assertJsonPath('checkpointed', true); + } + $request = CancellationScopeRequests::request(WorkflowRun::query()->findOrFail($task['run_id']), $scope, '1.20', 30); + + return [$task, $scope, $request->payload['request_id']]; + } + + private function mutate(string $phase, array $task, string $scope, string $request, array $overrides = [], + string $namespace = 'default', string $role = 'worker', string $version = '1.20') + { + return $this->withHeaders($this->headers($role, $namespace, $version)) + ->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/{$phase}", array_replace([ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'scope_id' => $scope, 'request_id' => $request, 'sequence' => 2, 'call_kind' => 'activity', + ], $overrides)); + } + + private function headers(string $role = 'worker', string $namespace = 'default', string $version = '1.20'): array + { + return ['Authorization' => 'Bearer fixture-'.$role, 'X-Namespace' => $namespace, + 'X-Durable-Workflow-Control-Plane-Version' => '2', WorkerProtocol::HEADER => $version]; + } + + private function assertContract(array $response): void + { + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/CancellationScopeDeliveryResponse/allOf/1', + json_decode(json_encode($response, JSON_THROW_ON_ERROR), flags: JSON_THROW_ON_ERROR)); + } +} diff --git a/tests/Source/ScopedActivityCancellationReceiptTest.php b/tests/Source/ScopedActivityCancellationReceiptTest.php index 8b572170..dc107ef9 100644 --- a/tests/Source/ScopedActivityCancellationReceiptTest.php +++ b/tests/Source/ScopedActivityCancellationReceiptTest.php @@ -5,7 +5,7 @@ use App\Models\WorkflowNamespace; use App\Support\CooperativeCancellationPolicy; use App\Support\WorkerProtocol; -use Illuminate\Foundation\Testing\RefreshDatabase; +use Illuminate\Foundation\Testing\DatabaseMigrations; use Illuminate\Support\Facades\Queue; use Illuminate\Testing\TestResponse; use PHPUnit\Framework\Attributes\DataProvider; @@ -27,7 +27,7 @@ /** Exact Native Source qualification. The HTTP receipt does not prove physical callback exit. */ final class ScopedActivityCancellationReceiptTest extends TestCase { - use RefreshDatabase; + use DatabaseMigrations; protected function setUp(): void { @@ -88,6 +88,34 @@ public function test_scoped_receipt_and_original_owner_acknowledgement_leave_sib $this->assertTrue(ActivityCancellationCompletion::resolved($run, $target['activity_execution_id'], CancellationScopeRequests::context($run, $fence['scope_id']))); } + public function test_delivery_waits_for_original_owner_receipt_and_preserves_the_prepared_frame(): void + { + [$task, $target, $sibling, $request, , $fence] = $this->scopedPair(); + $path = "/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/deliver"; + $body = ['lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'scope_id' => $fence['scope_id'], 'request_id' => $request->payload['request_id'], 'sequence' => 4, 'call_kind' => 'activity']; + $before = $this->snapshot($task, $target, $sibling); + $pending = $this->withHeaders($this->headers())->postJson($path, $body)->assertOk() + ->assertJsonPath('delivered', false)->assertJsonPath('prepared', true) + ->assertJsonPath('reason', 'cancellation_scope_activity_stop_not_acknowledged') + ->assertJsonPath('claim_released', false)->json(); + $this->assertSame($before, $this->snapshot($task, $target, $sibling)); + $ack = $this->acknowledge($target, $request->payload['request_id'])->assertOk()->json('history_event_id'); + $delivered = $this->withHeaders($this->headers())->postJson($path, $body)->assertOk() + ->assertJsonPath('delivered', true) + ->assertJsonPath('preparation_history_event_id', $pending['preparation_history_event_id']) + ->assertJsonPath('cancellation', $pending['cancellation']) + ->assertJsonPath('authority_deadline_at', $pending['authority_deadline_at'])->json(); + $this->withHeaders($this->headers())->postJson($path, $body)->assertOk()->assertJsonPath('history_event_id', $delivered['history_event_id']); + $this->assertSame($before[0], WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $this->assertSame($before[1][1], $this->snapshot($task, $target, $sibling)[1][1]); + $this->assertLessThan(WorkflowHistoryEvent::query()->findOrFail($delivered['history_event_id'])->sequence, + WorkflowHistoryEvent::query()->findOrFail($ack)->sequence); + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/CancellationScopeDeliveryResponse/allOf/1', + json_decode(json_encode($pending, JSON_THROW_ON_ERROR), flags: JSON_THROW_ON_ERROR)); + } + public function test_later_whole_run_request_and_late_receipt_preserve_original_scoped_identity_and_deadline(): void { [, $target, , $request, $root, $fence] = $this->scopedPair(); @@ -236,6 +264,10 @@ private function scopedPair(): array $run = WorkflowRun::query()->findOrFail($task['run_id']); $root = CancellationScopeRequests::request($run, $scopes[0], '1.20', 30, 'original scoped cleanup'); $request = CancellationScopeRequests::request($run, $scopes[1], '1.20', 300, parentScopeId: $scopes[0]); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/prepare", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'scope_id' => $scopes[1], 'request_id' => $request->payload['request_id'], 'sequence' => 4, 'call_kind' => 'activity', + ])->assertOk()->assertJsonPath('prepared', true)->assertJsonPath('delivered', false); $fence = ScopedActivityCancellation::fence($run, WorkflowTask::query()->findOrFail($task['task_id']), $activities[0]['activity_execution_id'], $scopes[1], $request->payload['request_id'], '1.20'); $this->assertTrue($fence['fenced']); From a9aeb06b75d0836734df7cbf88e13f9ef9d3a6f0 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 3 Oct 2026 18:43:48 +0000 Subject: [PATCH 37/57] test: qualify authenticated scoped activity dispatch through HTTP --- .../worker-protocol-api.openapi.yaml | 29 ++++++++++++++----- .../ScopedActivityCancellationReceiptTest.php | 16 ++++++++-- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 4d34b348..c733c4a0 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "39" + version: "40" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -538,16 +538,18 @@ paths: "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } /worker/workflow-tasks/{taskId}/cancellation-scopes/deliver: post: - operationId: recordWorkflowCancellationScopeDelivery + operationId: deliverWorkflowCancellationScope tags: [workflow-tasks] x-durable-workflow-minimum-protocol-version: "1.20" x-durable-workflow-worker-capability: cooperative_cancellation description: >- - Unfrozen candidate recording. Recheck the original issued claim and require the retained preparation. - Record the delivery only after every original member's policy proof exists. A pending original-owner stop - receipt returns HTTP200 with delivered=false and the original prepared context. Missing operation actors - return HTTP409 with explicit unavailable diagnostics and no delivery marker. No effect dispatch or callback - supervision is implied. Refresh canonical history through the returned cursor before injecting cancellation. + Unfrozen candidate dispatch. Validate the retained preparation's exact boundary and live claim before effects. + Dispatch every original direct-scope Activity through its policy actor, with independently committed progress + and claim rechecks. Retry after partial progress reuses the same members, context and deadline. Record delivery + only after every original member's policy proof exists. A pending original-owner stop receipt returns HTTP200 + with delivered=false. Missing operation actors return HTTP409 before Activity dispatch. Activity fences deny + stale publication and do not prove physical callback exit. Refresh canonical history through the returned + cursor before injecting cancellation. Scope execution remains unadvertised. parameters: - $ref: "#/components/parameters/WorkerProtocolVersionHeader" - $ref: "#/components/parameters/TaskIdPath" @@ -1932,6 +1934,19 @@ components: created_task_ids: { type: array, maxItems: 0 } reason: { type: [string, "null"] } unavailable: { type: array, items: { type: string, minLength: 1 } } + activity_cancellations: + type: array + description: Committed results for original prepared members. Partial progress may be present on a refusal. A fence is publication denial, and waiting_for_stop is the policy barrier rather than a physical exit assertion. + items: + type: object + required: [sequence, activity_execution_id, fenced, abandoned, waiting_for_stop, history_event_id] + properties: + sequence: { type: integer, minimum: 1 } + activity_execution_id: { type: string, minLength: 1 } + fenced: { type: boolean } + abandoned: { type: boolean } + waiting_for_stop: { type: boolean } + history_event_id: { type: [string, "null"], minLength: 1 } if: required: [prepared] properties: diff --git a/tests/Source/ScopedActivityCancellationReceiptTest.php b/tests/Source/ScopedActivityCancellationReceiptTest.php index dc107ef9..4e8a27d6 100644 --- a/tests/Source/ScopedActivityCancellationReceiptTest.php +++ b/tests/Source/ScopedActivityCancellationReceiptTest.php @@ -218,7 +218,7 @@ public function test_scoped_receipt_remains_bound_to_original_worker_role_owner_ $this->assertSame($before, $this->snapshot($workflowTask, $target, $sibling)); } - /** Canonical open/checkpoint/claims use HTTP. Scoped request/fence are still internal Native primitives. */ + /** Scope request remains internal; admission, preparation, dispatch and receipts use HTTP. */ private function scopedPair(): array { $this->withHeaders($this->headers(role: 'operator')) @@ -268,8 +268,18 @@ private function scopedPair(): array 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], 'scope_id' => $scopes[1], 'request_id' => $request->payload['request_id'], 'sequence' => 4, 'call_kind' => 'activity', ])->assertOk()->assertJsonPath('prepared', true)->assertJsonPath('delivered', false); - $fence = ScopedActivityCancellation::fence($run, WorkflowTask::query()->findOrFail($task['task_id']), - $activities[0]['activity_execution_id'], $scopes[1], $request->payload['request_id'], '1.20'); + $this->assertSame('running', ActivityExecution::query()->findOrFail($activities[0]['activity_execution_id'])->status->value); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'ActivityCancelled')->count()); + $response = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/deliver", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'scope_id' => $scopes[1], 'request_id' => $request->payload['request_id'], 'sequence' => 4, 'call_kind' => 'activity', + ])->assertOk()->assertJsonPath('prepared', true)->assertJsonPath('delivered', false) + ->assertJsonPath('reason', 'cancellation_scope_activity_stop_not_acknowledged'); + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/CancellationScopeDeliveryResponse/allOf/1', + json_decode($response->getContent(), flags: JSON_THROW_ON_ERROR)); + $fence = $response->json('activity_cancellations.0'); + $this->assertSame($activities[0]['activity_execution_id'], $fence['activity_execution_id']); $this->assertTrue($fence['fenced']); $this->assertTrue($fence['waiting_for_stop']); From fffd5f4fef19fc5021a92b2419b520791842f4d7 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 3 Oct 2026 19:30:54 +0000 Subject: [PATCH 38/57] Qualify scoped timer delivery through the worker API --- .../worker-protocol-api.openapi.yaml | 29 +++++++++++++-- .../Source/CancellationScopeDeliveryTest.php | 37 +++++++++++++++---- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index c733c4a0..3b76a0a9 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "40" + version: "41" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -544,10 +544,11 @@ paths: x-durable-workflow-worker-capability: cooperative_cancellation description: >- Unfrozen candidate dispatch. Validate the retained preparation's exact boundary and live claim before effects. - Dispatch every original direct-scope Activity through its policy actor, with independently committed progress + Dispatch every original direct-scope timer and Activity through its actor, with independently committed progress and claim rechecks. Retry after partial progress reuses the same members, context and deadline. Record delivery only after every original member's policy proof exists. A pending original-owner stop receipt returns HTTP200 - with delivered=false. Missing operation actors return HTTP409 before Activity dispatch. Activity fences deny + with delivered=false. Missing operation actors return HTTP409 before timer or Activity dispatch. Timer fences + prevent a late job from firing or creating a workflow resumption. Activity fences deny stale publication and do not prove physical callback exit. Refresh canonical history through the returned cursor before injecting cancellation. Scope execution remains unadvertised. parameters: @@ -1947,12 +1948,23 @@ components: abandoned: { type: boolean } waiting_for_stop: { type: boolean } history_event_id: { type: [string, "null"], minLength: 1 } + timer_cancellations: + type: array + description: Committed results for original prepared timers. A natural fire retains its outcome with fenced=false. A fence prevents a late fire or resumption. Partial progress may be present on a refusal. + items: + type: object + required: [sequence, timer_id, fenced, history_event_id] + properties: + sequence: { type: integer, minimum: 1 } + timer_id: { type: string, minLength: 1 } + fenced: { type: boolean } + history_event_id: { type: [string, "null"], minLength: 1 } if: required: [prepared] properties: prepared: { const: true } then: - required: [workflow_run_id, preparation_history_event_id, scope_id, request_id, cancellation, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span, authority_deadline_at, activity_members, lease_owner, workflow_task_attempt, history_refresh_page_token] + required: [workflow_run_id, preparation_history_event_id, scope_id, request_id, cancellation, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span, authority_deadline_at, activity_members, timer_members, lease_owner, workflow_task_attempt, history_refresh_page_token] properties: workflow_run_id: { type: string, minLength: 1 } preparation_history_event_id: { type: string, minLength: 1 } @@ -1997,6 +2009,15 @@ components: sequence: { type: integer, minimum: 1 } activity_execution_id: { type: string, minLength: 1 } descriptor_hash: { type: string, pattern: "^[a-f0-9]{64}$" } + timer_members: + type: array + items: + type: object + required: [sequence, timer_id, descriptor_hash] + properties: + sequence: { type: integer, minimum: 1 } + timer_id: { type: string, minLength: 1 } + descriptor_hash: { type: string, pattern: "^[a-f0-9]{64}$" } lease_owner: { type: string, minLength: 1 } workflow_task_attempt: { type: integer, minimum: 1 } history_refresh_page_token: { type: string, minLength: 1 } diff --git a/tests/Source/CancellationScopeDeliveryTest.php b/tests/Source/CancellationScopeDeliveryTest.php index 793f8c74..5fb738a4 100644 --- a/tests/Source/CancellationScopeDeliveryTest.php +++ b/tests/Source/CancellationScopeDeliveryTest.php @@ -16,6 +16,10 @@ use Tests\TestCase; use Workflow\V2\Contracts\PreparedCancellationScopeTaskBridge; use Workflow\V2\Contracts\WorkflowTaskBridge; +use Workflow\V2\Enums\TaskStatus; +use Workflow\V2\Enums\TaskType; +use Workflow\V2\Enums\TimerStatus; +use Workflow\V2\Jobs\RunTimerTask; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; @@ -193,18 +197,37 @@ public function test_replacement_claim_reuses_preparation_and_original_deadline( $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDeliveryPrepared')->count()); } - public function test_missing_timer_actor_keeps_preparation_and_returns_explicit_diagnostic(): void + public function test_timer_dispatch_reuses_original_preparation_and_fences_late_jobs(): void { [$task, $scope, $request] = $this->claim(timer: true); $prepared = $this->mutate('prepare', $task, $scope, $request, ['call_kind' => 'timer'])->assertOk()->json(); - $response = $this->mutate('deliver', $task, $scope, $request, ['call_kind' => 'timer'])->assertConflict() - ->assertJsonPath('delivered', false)->assertJsonPath('prepared', true) - ->assertJsonPath('reason', 'cancellation_scope_operation_delivery_unavailable') - ->assertJsonPath('unavailable', ['scoped_timer_delivery']) + $before = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + $timer = WorkflowTimer::query()->sole(); + $this->assertSame($timer->id, $prepared['timer_members'][0]['timer_id']); + $response = $this->mutate('deliver', $task, $scope, $request, ['call_kind' => 'timer'])->assertOk() + ->assertJsonPath('delivered', true)->assertJsonPath('prepared', true) + ->assertJsonPath('timer_cancellations.0.fenced', true) + ->assertJsonPath('timer_cancellations.0.timer_id', $timer->id) ->assertJsonPath('preparation_history_event_id', $prepared['preparation_history_event_id'])->json(); $this->assertContract($response); - $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDelivered')->count()); - $this->assertSame('pending', WorkflowTimer::query()->sole()->status->value); + $this->assertContract($prepared); + $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $duplicate = $this->mutate('deliver', $task, $scope, $request, ['call_kind' => 'timer'])->assertOk()->json(); + $this->assertSame($response['timer_cancellations'], $duplicate['timer_cancellations']); + $this->assertSame($response['history_event_id'], $duplicate['history_event_id']); + $this->assertSame($prepared['authority_deadline_at'], $duplicate['authority_deadline_at']); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDelivered')->count()); + $this->assertSame(TimerStatus::Cancelled, $timer->fresh()->status); + $timerTask = WorkflowTask::query()->where('task_type', TaskType::Timer)->sole(); + $timerTask->forceFill(['status' => TaskStatus::Ready, 'available_at' => now()->subSecond()])->save(); + $timer->refresh()->forceFill(['status' => TimerStatus::Pending])->save(); + $historyBefore = WorkflowHistoryEvent::query()->count(); + $this->app->call([new RunTimerTask($timerTask->id), 'handle']); + $this->assertSame($historyBefore, WorkflowHistoryEvent::query()->count()); + $this->assertSame(TaskStatus::Cancelled, $timerTask->fresh()->status); + $this->assertSame(TimerStatus::Cancelled, $timer->fresh()->status); + $this->assertSame(1, WorkflowTask::query()->where('task_type', TaskType::Workflow)->count()); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'TimerFired')->count()); } private function claim(bool $timer = false): array From 7cc5707c660fe51879388b0f02e01ceb98a940ef Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 3 Oct 2026 21:26:33 +0000 Subject: [PATCH 39/57] Admit scoped signal and condition waits through the worker API --- app/Http/Controllers/Api/WorkerController.php | 2 +- .../platform-protocol-specs/worker-protocol-api.openapi.yaml | 2 +- tests/Source/CancellationScopeAdmissionTest.php | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index 8570d163..93386a7f 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -2949,7 +2949,7 @@ private function guardCancellationScopeAdmission( } foreach ($commands as $command) { if (array_key_exists('cancellation_scope_id', $command) - && ! in_array($command['type'], ['schedule_activity', 'start_timer', 'start_child_workflow', 'prepare_local_activity'], true)) { + && ! in_array($command['type'], ['schedule_activity', 'start_timer', 'start_child_workflow', 'prepare_local_activity', 'open_signal_wait', 'open_condition_wait'], true)) { $reason = 'invalid_cancellation_scope_command'; } } diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 3b76a0a9..a72e3323 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -2786,7 +2786,7 @@ components: minLength: 1 maxLength: 255 x-durable-workflow-minimum-protocol-version: "1.20" - description: Optional exact-run canonical membership for schedule_activity, start_timer, start_child_workflow and prepare_local_activity commands in the unfrozen candidate. Requires an installed CancellationScopeAdmission backend. The named scope must already be recorded before admission. Invalid membership is refused before payload resolution and rechecked under the run lock. Omission preserves historical root membership. This field does not advertise scope execution or supervision. + description: Optional exact-run canonical membership for schedule_activity, start_timer, start_child_workflow, prepare_local_activity, open_signal_wait and open_condition_wait commands in the unfrozen candidate. Requires an installed CancellationScopeAdmission backend. The named scope must already be recorded before admission. Invalid membership is refused before payload resolution and rechecked under the run lock. Omission preserves historical root membership. Wait commands close a turn and cannot enter a retained-claim prefix. This field does not advertise scope execution or supervision. parent_close_policy: type: [string, "null"] description: Child parent-close policy. request_cancel retains legacy terminal cancellation. request_cancellation requests cooperative cleanup under protocol 1.20 and the original shared budget. diff --git a/tests/Source/CancellationScopeAdmissionTest.php b/tests/Source/CancellationScopeAdmissionTest.php index 7c7c6ccf..59c3d344 100644 --- a/tests/Source/CancellationScopeAdmissionTest.php +++ b/tests/Source/CancellationScopeAdmissionTest.php @@ -406,7 +406,7 @@ private function openScope(array $task, array $overrides = [], string $version = public static function operations(): array { - return [['schedule_activity'], ['start_timer'], ['start_child_workflow']]; + return [['schedule_activity'], ['start_timer'], ['start_child_workflow'], ['open_signal_wait'], ['open_condition_wait']]; } #[DataProvider('operations')] @@ -433,6 +433,7 @@ public function test_recorded_scope_reaches_durable_history_and_commits_the_stre $this->assertDatabaseCount('workflow_durable_stream_items', 1); $eventType = match ($type) { 'schedule_activity' => 'ActivityScheduled', 'start_timer' => 'TimerScheduled', 'start_child_workflow' => 'ChildWorkflowScheduled', + 'open_signal_wait' => 'SignalWaitOpened', 'open_condition_wait' => 'ConditionWaitOpened', }; $event = WorkflowHistoryEvent::query()->where('workflow_run_id', $run->id)->where('event_type', $eventType)->sole(); $this->assertSame($scopeId, $type === 'schedule_activity' ? $event->payload['activity']['cancellation_scope_id'] : $event->payload['cancellation_scope_id']); @@ -731,6 +732,8 @@ private function commandBatch(array $task, mixed $scopeId, string $type, mixed $ 'schedule_activity' => ['activity_type' => 'opaque-activity', 'arguments' => $arguments ?? Serializer::serializeWithCodec('avro', ['Ada']), 'payload_codec' => 'avro'], 'start_timer' => ['delay_seconds' => 60], 'start_child_workflow' => ['workflow_type' => 'tests.external-greeting-workflow', 'arguments' => Serializer::serializeWithCodec('avro', ['child']), 'payload_codec' => 'avro'], + 'open_signal_wait' => ['signal_name' => 'ready', 'timeout_seconds' => 5], + 'open_condition_wait' => ['condition_key' => 'ready', 'timeout_seconds' => 5], 'record_side_effect' => ['result' => Serializer::serializeWithCodec('avro', null)], }; From 2c4eb436e6233ba85743cb042302c2bf9f3d7dca Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 3 Oct 2026 22:22:07 +0000 Subject: [PATCH 40/57] Qualify scoped wait cancellation through the worker HTTP API --- .../worker-protocol-api.openapi.yaml | 28 ++++++++++- .../Source/CancellationScopeDeliveryTest.php | 49 +++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index a72e3323..b3c6a32a 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "41" + version: "42" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -1959,12 +1959,24 @@ components: timer_id: { type: string, minLength: 1 } fenced: { type: boolean } history_event_id: { type: [string, "null"], minLength: 1 } + wait_cancellations: + type: array + description: Original prepared signal and condition wait decisions. A cancellation and associated timeout fence commit atomically. Earlier natural outcomes retain cancelled=false. Each receipt keeps its original history identity through retry or replacement. + items: + type: object + required: [kind, sequence, wait_id, cancelled, history_event_id] + properties: + kind: { type: string, enum: [signal, condition] } + sequence: { type: integer, minimum: 1 } + wait_id: { type: string, minLength: 1 } + cancelled: { type: boolean } + history_event_id: { type: string, minLength: 1 } if: required: [prepared] properties: prepared: { const: true } then: - required: [workflow_run_id, preparation_history_event_id, scope_id, request_id, cancellation, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span, authority_deadline_at, activity_members, timer_members, lease_owner, workflow_task_attempt, history_refresh_page_token] + required: [workflow_run_id, preparation_history_event_id, scope_id, request_id, cancellation, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span, authority_deadline_at, activity_members, timer_members, wait_members, lease_owner, workflow_task_attempt, history_refresh_page_token] properties: workflow_run_id: { type: string, minLength: 1 } preparation_history_event_id: { type: string, minLength: 1 } @@ -2021,6 +2033,18 @@ components: lease_owner: { type: string, minLength: 1 } workflow_task_attempt: { type: integer, minimum: 1 } history_refresh_page_token: { type: string, minLength: 1 } + wait_members: + type: array + description: Preparation v3 freezes original wait occurrences, including untimed waits, and their timeout association without application payload bytes. + items: + type: object + required: [kind, sequence, wait_id, timer_id, descriptor_hash] + properties: + kind: { type: string, enum: [signal, condition] } + sequence: { type: integer, minimum: 1 } + wait_id: { type: string, minLength: 1 } + timer_id: { type: [string, "null"], minLength: 1 } + descriptor_hash: { type: string, pattern: "^[a-f0-9]{64}$" } CancellationScopeOpeningRequest: type: object required: [lease_owner, workflow_task_attempt, sequence] diff --git a/tests/Source/CancellationScopeDeliveryTest.php b/tests/Source/CancellationScopeDeliveryTest.php index 5fb738a4..c8c8cbea 100644 --- a/tests/Source/CancellationScopeDeliveryTest.php +++ b/tests/Source/CancellationScopeDeliveryTest.php @@ -230,6 +230,55 @@ public function test_timer_dispatch_reuses_original_preparation_and_fences_late_ $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'TimerFired')->count()); } + #[DataProvider('waits')] + public function test_scoped_wait_delivery_retains_original_membership_and_claim(string $kind, ?int $timeout): void + { + [$task, $scope, $request] = $this->claim(); + $run = WorkflowRun::query()->findOrFail($task['run_id']); + $command = array_filter(['type' => 'open_'.$kind.'_wait', 'cancellation_scope_id' => $scope, + 'timeout_seconds' => $timeout, ...($kind === 'signal' ? ['signal_name' => 'ready'] : ['condition_key' => 'ready'])], + static fn (mixed $value): bool => $value !== null); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [$command], + ])->assertOk(); + $this->assertSame(TaskStatus::Completed, WorkflowTask::query()->findOrFail($task['task_id'])->status); + $run->tasks()->create(['namespace' => $run->namespace, 'task_type' => TaskType::Workflow, 'status' => TaskStatus::Ready, + 'available_at' => now(), 'connection' => $run->connection, 'queue' => $run->queue, 'compatibility' => $run->compatibility]); + $task = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'delivery-owner', 'task_queue' => 'scoped-delivery', + ])->assertOk()->json('task'); + $before = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + $prepared = $this->mutate('prepare', $task, $scope, $request, ['call_kind' => $kind])->assertOk()->json(); + $this->assertContract($prepared); + $this->assertCount(1, $prepared['wait_members']); + $this->assertSame($kind, $prepared['wait_members'][0]['kind']); + $this->assertSame($timeout !== null, $prepared['wait_members'][0]['timer_id'] !== null); + $response = $this->mutate('deliver', $task, $scope, $request, ['call_kind' => $kind])->assertOk() + ->assertJsonPath('delivered', true)->assertJsonPath('wait_cancellations.0.cancelled', true)->json(); + $this->assertContract($response); + $duplicate = $this->mutate('deliver', $task, $scope, $request, ['call_kind' => $kind])->assertOk()->json(); + $this->assertSame($response['wait_cancellations'], $duplicate['wait_cancellations']); + $this->assertSame($response['history_event_id'], $duplicate['history_event_id']); + $this->assertSame($prepared['wait_members'], $duplicate['wait_members']); + $this->assertSame($prepared['authority_deadline_at'], $duplicate['authority_deadline_at']); + $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $marker = WorkflowHistoryEvent::query()->where('event_type', $kind === 'signal' ? 'SignalWaitCancelled' : 'ConditionWaitCancelled')->sole(); + $this->assertSame($prepared['preparation_history_event_id'], $marker->payload['cancellation_scope']['preparation_history_event_id']); + if ($timeout !== null) { + $this->assertSame(TimerStatus::Cancelled, $run->timers()->sole()->status); + $this->assertSame(TaskStatus::Cancelled, $run->tasks()->where('task_type', TaskType::Timer)->sole()->status); + } + } + + public static function waits(): iterable + { + foreach (['signal', 'condition'] as $kind) { + yield $kind.':untimed' => [$kind, null]; + yield $kind.':timed' => [$kind, 5]; + } + } + private function claim(bool $timer = false): array { $this->withHeaders($this->headers('operator'))->postJson('/api/workflows', [ From 6d2fb0ce35478eba47994228d40d8e834c5867be Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 3 Oct 2026 23:55:28 +0000 Subject: [PATCH 41/57] Expose frozen child targets in scope preparation responses --- .../worker-protocol-api.openapi.yaml | 20 +++++++-- .../Source/CancellationScopeDeliveryTest.php | 44 ++++++++++++++++++- 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index b3c6a32a..dfcd4768 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "42" + version: "43" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -1976,7 +1976,7 @@ components: properties: prepared: { const: true } then: - required: [workflow_run_id, preparation_history_event_id, scope_id, request_id, cancellation, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span, authority_deadline_at, activity_members, timer_members, wait_members, lease_owner, workflow_task_attempt, history_refresh_page_token] + required: [workflow_run_id, preparation_history_event_id, scope_id, request_id, cancellation, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span, authority_deadline_at, activity_members, timer_members, wait_members, child_members, lease_owner, workflow_task_attempt, history_refresh_page_token] properties: workflow_run_id: { type: string, minLength: 1 } preparation_history_event_id: { type: string, minLength: 1 } @@ -2035,7 +2035,7 @@ components: history_refresh_page_token: { type: string, minLength: 1 } wait_members: type: array - description: Preparation v3 freezes original wait occurrences, including untimed waits, and their timeout association without application payload bytes. + description: Preparation freezes original wait occurrences, including untimed waits, and their timeout association without application payload bytes. items: type: object required: [kind, sequence, wait_id, timer_id, descriptor_hash] @@ -2045,6 +2045,20 @@ components: wait_id: { type: string, minLength: 1 } timer_id: { type: [string, "null"], minLength: 1 } descriptor_hash: { type: string, pattern: "^[a-f0-9]{64}$" } + child_members: + type: array + description: Preparation v4 freezes child call identity, instance, original typed-history run target and cancellation policy. The snapshot grants no child effect or delivery authority. Child dispatch remains unavailable until its committed receipt barrier is implemented. + items: + type: object + additionalProperties: false + required: [sequence, child_call_id, child_workflow_instance_id, child_workflow_run_id, cancellation_policy, descriptor_hash] + properties: + sequence: { type: integer, minimum: 1 } + child_call_id: { type: string, minLength: 1 } + child_workflow_instance_id: { type: string, minLength: 1 } + child_workflow_run_id: { type: string, minLength: 1 } + cancellation_policy: { type: string, enum: [try_cancel, wait_cancellation_completed, abandon] } + descriptor_hash: { type: string, pattern: "^[a-f0-9]{64}$" } CancellationScopeOpeningRequest: type: object required: [lease_owner, workflow_task_attempt, sequence] diff --git a/tests/Source/CancellationScopeDeliveryTest.php b/tests/Source/CancellationScopeDeliveryTest.php index c8c8cbea..b5290069 100644 --- a/tests/Source/CancellationScopeDeliveryTest.php +++ b/tests/Source/CancellationScopeDeliveryTest.php @@ -279,7 +279,40 @@ public static function waits(): iterable } } - private function claim(bool $timer = false): array + #[DataProvider('childPolicies')] + public function test_child_preparation_freezes_original_targets_without_claiming_delivery(string $policy): void + { + [$task, $scope, $request] = $this->claim(childPolicy: $policy); + $before = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + $prepared = $this->mutate('prepare', $task, $scope, $request, ['call_kind' => 'child'])->assertOk()->json(); + $this->assertContract($prepared); + $this->assertCount(1, $prepared['child_members']); + $scheduled = WorkflowHistoryEvent::query()->where('event_type', 'ChildWorkflowScheduled')->sole(); + $member = $prepared['child_members'][0]; + $this->assertSame($scheduled->payload['child_call_id'], $member['child_call_id']); + $this->assertSame($scheduled->payload['child_workflow_instance_id'], $member['child_workflow_instance_id']); + $this->assertSame($scheduled->payload['child_workflow_run_id'], $member['child_workflow_run_id']); + $this->assertSame($policy, $member['cancellation_policy']); + $this->assertSame($prepared['child_members'], $this->mutate('prepare', $task, $scope, $request, + ['call_kind' => 'child'])->assertOk()->json('child_members')); + $historyBefore = WorkflowHistoryEvent::query()->count(); + $this->mutate('deliver', $task, $scope, $request, ['call_kind' => 'child'])->assertConflict() + ->assertJsonPath('reason', 'cancellation_scope_operation_delivery_unavailable') + ->assertJsonPath('unavailable', ['scoped_child_delivery']); + $this->assertSame($historyBefore, WorkflowHistoryEvent::query()->count()); + $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $this->assertNull(WorkflowRun::query()->findOrFail($member['child_workflow_run_id'])->cancellation_request_command_id); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDelivered')->count()); + } + + public static function childPolicies(): iterable + { + foreach (['try_cancel', 'wait_cancellation_completed', 'abandon'] as $policy) { + yield $policy => [$policy]; + } + } + + private function claim(bool $timer = false, ?string $childPolicy = null): array { $this->withHeaders($this->headers('operator'))->postJson('/api/workflows', [ 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'scoped-delivery', 'input' => ['Ada'], @@ -303,6 +336,15 @@ private function claim(bool $timer = false): array 'commands' => [['type' => 'start_timer', 'delay_seconds' => 60, 'cancellation_scope_id' => $scope]], ])->assertOk()->assertJsonPath('checkpointed', true); } + if ($childPolicy !== null) { + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/checkpoint", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'checkpoint_id' => 'child-prefix', 'start_sequence' => 2, + 'commands' => [['type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', + 'arguments' => \Workflow\Serializers\Serializer::serializeWithCodec('avro', ['child']), + 'payload_codec' => 'avro', 'cancellation_scope_id' => $scope, 'cancellation_policy' => $childPolicy]], + ])->assertOk()->assertJsonPath('checkpointed', true); + } $request = CancellationScopeRequests::request(WorkflowRun::query()->findOrFail($task['run_id']), $scope, '1.20', 30); return [$task, $scope, $request->payload['request_id']]; From 785093feff1b44e92126be6f350c0c9bdb834886 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sun, 4 Oct 2026 03:25:02 +0000 Subject: [PATCH 42/57] Describe and qualify portable scoped child cancellation receipts --- .../worker-protocol-api.openapi.yaml | 17 +++++- .../Source/CancellationScopeDeliveryTest.php | 59 ++++++++++++++++--- 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index dfcd4768..6dfbb2dd 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "43" + version: "44" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -1971,6 +1971,21 @@ components: wait_id: { type: string, minLength: 1 } cancelled: { type: boolean } history_event_id: { type: string, minLength: 1 } + child_cancellations: + type: array + description: Durable decisions for original prepared direct children. A cooperative request and its parent receipt commit atomically. TryCancel is ready after acceptance, WAIT requires a separate canonical child terminal resolution, and Abandon leaves the child untouched. Partial receipts survive retries and replacement claims without changing the original identity or deadline. Readiness does not assert physical callback exit. + items: + type: object + additionalProperties: false + required: [child_call_id, child_workflow_run_id, history_event_id, resolution_history_event_id, request_id, policy, ready] + properties: + child_call_id: { type: string, minLength: 1 } + child_workflow_run_id: { type: string, minLength: 1 } + history_event_id: { type: string, minLength: 1 } + resolution_history_event_id: { type: [string, "null"], minLength: 1 } + request_id: { type: [string, "null"], minLength: 1 } + policy: { type: string, enum: [try_cancel, wait_cancellation_completed, abandon] } + ready: { type: boolean } if: required: [prepared] properties: diff --git a/tests/Source/CancellationScopeDeliveryTest.php b/tests/Source/CancellationScopeDeliveryTest.php index b5290069..9f364d6d 100644 --- a/tests/Source/CancellationScopeDeliveryTest.php +++ b/tests/Source/CancellationScopeDeliveryTest.php @@ -14,6 +14,7 @@ use Tests\Fixtures\ExternalGreetingWorkflow; use Tests\Support\OpenApiSchema; use Tests\TestCase; +use Workflow\Serializers\Serializer; use Workflow\V2\Contracts\PreparedCancellationScopeTaskBridge; use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Enums\TaskStatus; @@ -25,6 +26,8 @@ use Workflow\V2\Models\WorkflowTask; use Workflow\V2\Models\WorkflowTimer; use Workflow\V2\Support\CancellationScopeRequests; +use Workflow\V2\Support\CooperativeCancellationDelivery; +use Workflow\V2\WorkflowStub; /** Source-only preparation commits outside the test's transaction. */ final class CancellationScopeDeliveryTest extends TestCase @@ -280,7 +283,7 @@ public static function waits(): iterable } #[DataProvider('childPolicies')] - public function test_child_preparation_freezes_original_targets_without_claiming_delivery(string $policy): void + public function test_child_delivery_reconciles_original_policy_receipts_without_releasing_the_host_claim(string $policy): void { [$task, $scope, $request] = $this->claim(childPolicy: $policy); $before = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); @@ -295,14 +298,56 @@ public function test_child_preparation_freezes_original_targets_without_claiming $this->assertSame($policy, $member['cancellation_policy']); $this->assertSame($prepared['child_members'], $this->mutate('prepare', $task, $scope, $request, ['call_kind' => 'child'])->assertOk()->json('child_members')); + $this->assertNull(WorkflowRun::query()->findOrFail($member['child_workflow_run_id'])->cancellation_request_command_id); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDelivered')->count()); + $response = $this->mutate('deliver', $task, $scope, $request, ['call_kind' => 'child']); + if ($policy === 'wait_cancellation_completed') { + $response->assertConflict()->assertJsonPath('delivered', false) + ->assertJsonPath('reason', 'cancellation_scope_child_completion_not_established'); + } else { + $response->assertOk()->assertJsonPath('delivered', true); + } + $delivery = $response->json(); + $this->assertContract($delivery); + $this->assertCount(1, $delivery['child_cancellations']); + $receipt = $delivery['child_cancellations'][0]; + $this->assertSame($member['child_call_id'], $receipt['child_call_id']); + $this->assertSame($member['child_workflow_run_id'], $receipt['child_workflow_run_id']); + $this->assertSame($policy, $receipt['policy']); + $target = WorkflowStub::loadRun($member['child_workflow_run_id']); + $context = CooperativeCancellationDelivery::context($target->run()->fresh()); + $this->assertSame($policy === 'abandon', $context === null); + $this->assertFalse($target->run()->fresh()->status->isTerminal()); + if ($context !== null) { + $this->assertSame($prepared['cancellation'], $context->scopeOrigin->toArray()); + $this->assertSame($prepared['authority_deadline_at'], $context->deadline()->toISOString()); + $this->assertSame($context->requestId, $receipt['request_id']); + } $historyBefore = WorkflowHistoryEvent::query()->count(); - $this->mutate('deliver', $task, $scope, $request, ['call_kind' => 'child'])->assertConflict() - ->assertJsonPath('reason', 'cancellation_scope_operation_delivery_unavailable') - ->assertJsonPath('unavailable', ['scoped_child_delivery']); + $duplicate = $this->mutate('deliver', $task, $scope, $request, ['call_kind' => 'child'])->json(); + $this->assertContract($duplicate); + $this->assertSame($receipt, $duplicate['child_cancellations'][0]); + $this->assertSame($delivery['authority_deadline_at'], $duplicate['authority_deadline_at']); $this->assertSame($historyBefore, WorkflowHistoryEvent::query()->count()); + if ($policy === 'wait_cancellation_completed') { + $this->assertFalse($receipt['ready']); + $this->assertNull($receipt['resolution_history_event_id']); + $target->attemptCancel('canonical terminal fixture'); + $resolved = $this->mutate('deliver', $task, $scope, $request, ['call_kind' => 'child']) + ->assertOk()->assertJsonPath('delivered', true)->json(); + $this->assertContract($resolved); + $this->assertSame($receipt['history_event_id'], $resolved['child_cancellations'][0]['history_event_id']); + $this->assertSame($receipt['request_id'], $resolved['child_cancellations'][0]['request_id']); + $this->assertTrue($resolved['child_cancellations'][0]['ready']); + $this->assertNotNull($resolved['child_cancellations'][0]['resolution_history_event_id']); + $this->assertSame($prepared['authority_deadline_at'], $resolved['authority_deadline_at']); + } else { + $this->assertTrue($receipt['ready']); + $this->assertSame($delivery['history_event_id'], $duplicate['history_event_id']); + } $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); - $this->assertNull(WorkflowRun::query()->findOrFail($member['child_workflow_run_id'])->cancellation_request_command_id); - $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDelivered')->count()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'ChildCancellationRequested')->count()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDelivered')->count()); } public static function childPolicies(): iterable @@ -341,7 +386,7 @@ private function claim(bool $timer = false, ?string $childPolicy = null): array 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], 'checkpoint_id' => 'child-prefix', 'start_sequence' => 2, 'commands' => [['type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', - 'arguments' => \Workflow\Serializers\Serializer::serializeWithCodec('avro', ['child']), + 'arguments' => Serializer::serializeWithCodec('avro', ['child']), 'payload_codec' => 'avro', 'cancellation_scope_id' => $scope, 'cancellation_policy' => $childPolicy]], ])->assertOk()->assertJsonPath('checkpointed', true); } From 13114fafac00a7ef8aadc639af0883333e6bed48 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sun, 4 Oct 2026 19:08:07 +0000 Subject: [PATCH 43/57] Preserve scoped local cleanup proof through Server admission --- .../Api/PreparedLocalActivityController.php | 2 + app/Http/Controllers/Api/WorkerController.php | 3 +- .../worker-protocol-api.openapi.yaml | 69 +++++-- .../Source/CancellationScopeDeliveryTest.php | 194 +++++++++++++++++- .../WorkerProtocolOpenApiContractTest.php | 36 ++++ 5 files changed, 285 insertions(+), 19 deletions(-) diff --git a/app/Http/Controllers/Api/PreparedLocalActivityController.php b/app/Http/Controllers/Api/PreparedLocalActivityController.php index 2239fc38..99c97f4b 100644 --- a/app/Http/Controllers/Api/PreparedLocalActivityController.php +++ b/app/Http/Controllers/Api/PreparedLocalActivityController.php @@ -147,6 +147,8 @@ private function cancellationScopeRefusal(Request $request, WorkflowTask $task, // canonical membership under its attempt/execution/run/task locks. $reason = $bridge->validateCancellationScopeMembership($run, [[ 'type' => 'prepare_local_activity', 'cancellation_scope_id' => $descriptor['cancellation_scope_id'], + ...(array_key_exists('cancellation_cleanup', $descriptor) + ? ['cancellation_cleanup' => $descriptor['cancellation_cleanup']] : []), ]], $sequence); } if ($reason === null) { diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index 93386a7f..22224ada 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -1704,7 +1704,8 @@ private function mutateWorkflowTaskCommands(Request $request, string $taskId, bo $topLevelRules = []; if ($group) { $completionRules['commands'] = ['required', 'array', 'min:1', 'max:100']; - $completionRules['commands.*.cancellation_cleanup'] = ['nullable', 'array:request_id,delivery_history_event_id']; + $completionRules['commands.*.cancellation_cleanup'] = ['nullable', 'array:scope_id,request_id,delivery_history_event_id']; + $completionRules['commands.*.cancellation_cleanup.scope_id'] = ['sometimes', 'required', 'string', 'max:255']; $completionRules['commands.*.cancellation_cleanup.request_id'] = ['required_with:commands.*.cancellation_cleanup', 'string', 'max:255']; $completionRules['commands.*.cancellation_cleanup.delivery_history_event_id'] = ['required_with:commands.*.cancellation_cleanup', 'string', 'max:255']; } diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 6dfbb2dd..79d0e57d 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "44" + version: "45" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -101,6 +101,18 @@ x-durable-workflow-prepared-local-activity-contract: renewal: atomic_workflow_and_local_attempt_leases_without_application_heartbeat application_heartbeat: canonical_progress_and_heartbeat_timeout_without_lease_renewal shielded_cleanup: canonical_request_and_delivery_identity_with_runtime_owned_original_deadline + scoped_cleanup: + status: unfrozen_candidate_prepared_local_only + proof: [scope_id, request_id, delivery_history_event_id] + authority: canonical_original_delivery_and_preparation_with_same_root_context + membership: original_scope_or_original_unshielded_descendant + snapshot: [scope_id, operation_scope_id, request_id, root_request_id, delivery_history_event_id, preparation_history_event_id, cleanup_deadline_at, authority_deadline_at] + admission: after_original_authored_delivery_range_and_before_payload_resolution_or_any_group_sibling + worker_deadline: forbidden + later_limits: may_shorten_current_authority_without_rewriting_snapshot + whole_run_cancellation: still_fences_scoped_cleanup + hosting_claim: renewable_interval_separate_from_subtree_deadline + scope_execution_capability: false cancellation: original_root_identity_and_deadline_without_releasing_cleanup_claim stop_acknowledgement: original_supervisor_reports_only_after_callback_is_stopped_and_joined expired_callback_stop: unknown_until_original_supervisor_reports_join @@ -2149,13 +2161,7 @@ components: schedule_to_close_timeout: { type: [integer, "null"], minimum: 1 } heartbeat_timeout: { type: [integer, "null"], minimum: 1 } cancellation_cleanup: - type: object - additionalProperties: false - required: [request_id, delivery_history_event_id] - description: Explicit shielded cleanup after canonical delivery. The runtime supplies the original root identity and deadline. - properties: - request_id: { type: string, minLength: 1 } - delivery_history_event_id: { type: string, minLength: 1 } + $ref: "#/components/schemas/PreparedLocalCleanupProof" parallel_group_id: { type: string, minLength: 1 } parallel_group_kind: { type: string, enum: [activity, mixed] } parallel_group_mode: { type: string, const: all } @@ -2235,15 +2241,46 @@ components: total: { type: [number, string], description: Nonnegative finite number or numeric string. } unit: { type: string, minLength: 1, maxLength: 64 } details: { type: object, maxProperties: 20 } + PreparedLocalCleanupProof: + description: Explicit cleanup after canonical delivery. A scoped proof names the original cancelled address, while cancellation_scope_id names the new operation's original scope or unshielded descendant. The runtime supplies the root identity and immutable budget. No worker-supplied deadline or snapshot field is accepted. + oneOf: + - type: object + additionalProperties: false + required: [request_id, delivery_history_event_id] + properties: + request_id: { type: string, minLength: 1 } + delivery_history_event_id: { type: string, minLength: 1 } + - type: object + additionalProperties: false + required: [scope_id, request_id, delivery_history_event_id] + properties: + scope_id: { type: string, minLength: 1, maxLength: 255 } + request_id: { type: string, minLength: 1 } + delivery_history_event_id: { type: string, minLength: 1 } PreparedLocalCleanupSnapshot: - type: [object, "null"] - additionalProperties: false - required: [request_id, root_request_id, delivery_history_event_id, cleanup_deadline_at] - properties: - request_id: { type: string, minLength: 1 } - root_request_id: { type: string, minLength: 1 } - delivery_history_event_id: { type: string, minLength: 1 } - cleanup_deadline_at: { type: string, format: date-time } + description: Original runtime authority. The four-field root snapshot remains unchanged. A scoped snapshot binds the original delivery and preparation, new operation membership and captured finite ceiling. Current ancestor or run limits may shorten authority without changing this snapshot. It grants no permission after expiry, terminal closure or whole-run cancellation. + oneOf: + - type: "null" + - type: object + additionalProperties: false + required: [request_id, root_request_id, delivery_history_event_id, cleanup_deadline_at] + properties: + request_id: { type: string, minLength: 1 } + root_request_id: { type: string, minLength: 1 } + delivery_history_event_id: { type: string, minLength: 1 } + cleanup_deadline_at: { type: string, format: date-time } + - type: object + additionalProperties: false + required: [scope_id, operation_scope_id, request_id, root_request_id, delivery_history_event_id, preparation_history_event_id, cleanup_deadline_at, authority_deadline_at] + properties: + scope_id: { type: string, minLength: 1 } + operation_scope_id: { type: string, minLength: 1 } + request_id: { type: string, minLength: 1 } + root_request_id: { type: string, minLength: 1 } + delivery_history_event_id: { type: string, minLength: 1 } + preparation_history_event_id: { type: string, minLength: 1 } + cleanup_deadline_at: { type: string, format: date-time } + authority_deadline_at: { type: string, format: date-time } PreparedLocalOutcomeRequest: allOf: - $ref: "#/components/schemas/PreparedLocalClaimRequest" diff --git a/tests/Source/CancellationScopeDeliveryTest.php b/tests/Source/CancellationScopeDeliveryTest.php index 9f364d6d..7333da29 100644 --- a/tests/Source/CancellationScopeDeliveryTest.php +++ b/tests/Source/CancellationScopeDeliveryTest.php @@ -5,6 +5,8 @@ use App\Models\WorkerRegistration; use App\Models\WorkflowNamespace; use App\Support\CooperativeCancellationPolicy; +use App\Support\NamespaceExternalPayloadStorage; +use App\Support\PreparedLocalActivityPolicy; use App\Support\WorkerProtocol; use Illuminate\Foundation\Testing\DatabaseMigrations; use Illuminate\Support\Facades\DB; @@ -21,12 +23,15 @@ use Workflow\V2\Enums\TaskType; use Workflow\V2\Enums\TimerStatus; use Workflow\V2\Jobs\RunTimerTask; +use Workflow\V2\Models\ActivityAttempt; +use Workflow\V2\Models\ActivityExecution; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; use Workflow\V2\Models\WorkflowTimer; use Workflow\V2\Support\CancellationScopeRequests; use Workflow\V2\Support\CooperativeCancellationDelivery; +use Workflow\V2\Support\ParallelChildGroup; use Workflow\V2\WorkflowStub; /** Source-only preparation commits outside the test's transaction. */ @@ -357,7 +362,191 @@ public static function childPolicies(): iterable } } - private function claim(bool $timer = false, ?string $childPolicy = null): array + public function test_scoped_cleanup_prepares_and_reuses_original_runtime_snapshot(): void + { + [$task, $scope, $request, $proof] = $this->cleanupClaim(); + $response = $this->cleanupLocal($task, 'prepare', $scope, $proof)->assertOk() + ->assertJsonPath('prepared', true)->assertJsonPath('duplicate', false); + $first = $response->json(); + $snapshot = $first['cancellation_cleanup']; + $this->assertCleanupContract($snapshot); + $this->assertSame($scope, $snapshot['scope_id']); + $this->assertSame($scope, $snapshot['operation_scope_id']); + $this->assertSame($request, $snapshot['request_id']); + $this->assertSame($proof['delivery_history_event_id'], $snapshot['delivery_history_event_id']); + $this->assertSame($snapshot['authority_deadline_at'], $first['schedule_to_close_deadline_at']); + $before = WorkflowHistoryEvent::query()->count(); + $duplicate = $this->cleanupLocal($task, 'prepare', $scope, $proof)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('activity_attempt_id', $first['activity_attempt_id'])->json(); + $duplicateSnapshot = $duplicate['cancellation_cleanup']; + ksort($snapshot); + ksort($duplicateSnapshot); + $this->assertSame($snapshot, $duplicateSnapshot); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertNull(WorkflowRun::query()->findOrFail($task['run_id'])->cancellation_request_command_id); + } + + public function test_scoped_cleanup_recovery_keeps_original_proof_and_refuses_stale_publication(): void + { + [$task, $scope, , $proof] = $this->cleanupClaim(); + $first = $this->cleanupLocal($task, 'prepare', $scope, $proof)->assertOk()->json(); + ActivityAttempt::query()->findOrFail($first['activity_attempt_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $replacement = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'delivery-owner', 'task_queue' => 'scoped-delivery', + ])->assertOk()->json('task'); + $this->assertSame($task['task_id'], $replacement['task_id']); + $this->assertSame($task['workflow_task_attempt'] + 1, $replacement['workflow_task_attempt']); + $recovery = $this->cleanupLocal($replacement, 'recover', $scope, $proof)->assertOk() + ->assertJsonPath('recovered', true)->assertJsonPath('claim_released', true) + ->assertJsonPath('callback_stop_state', 'unknown')->json(); + $snapshot = ActivityExecution::query()->findOrFail($first['activity_execution_id'])->activity_options['cancellation_cleanup']; + $this->assertCleanupContract($snapshot); + $expected = $first['cancellation_cleanup']; + ksort($expected); + ksort($snapshot); + $this->assertSame($expected, $snapshot); + $before = WorkflowHistoryEvent::query()->count(); + $this->cleanupLocal($replacement, 'recover', $scope, $proof)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('event_id', $recovery['event_id']); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/{$first['activity_attempt_id']}/outcome", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'report' => ['outcome' => 'completed', 'result' => Serializer::serializeWithCodec('avro', 'stale'), 'payload_codec' => 'avro'], + ])->assertConflict()->assertJsonPath('recorded', false); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'ActivityCompleted')->count()); + $retryTask = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'delivery-owner', 'task_queue' => 'scoped-delivery', + ])->assertOk()->json('task'); + $this->assertSame($recovery['created_task_ids'][0], $retryTask['task_id']); + $retryResponse = $this->cleanupLocal($retryTask, 'prepare', $scope, $proof, workerAttempt: 'replacement-cleanup-attempt'); + $this->assertSame(200, $retryResponse->status(), json_encode($retryResponse->json('reason'), JSON_THROW_ON_ERROR)); + $retry = $retryResponse->assertJsonPath('prepared', true)->assertJsonPath('attempt_number', 2)->json(); + $this->assertNotSame($first['activity_attempt_id'], $retry['activity_attempt_id']); + $retrySnapshot = $retry['cancellation_cleanup']; + ksort($retrySnapshot); + $this->assertSame($expected, $retrySnapshot); + $this->assertSame($first['schedule_to_close_deadline_at'], $retry['schedule_to_close_deadline_at']); + $body = [ + 'lease_owner' => $retryTask['lease_owner'], 'workflow_task_attempt' => $retryTask['workflow_task_attempt'], + 'report' => ['outcome' => 'completed', 'result' => Serializer::serializeWithCodec('avro', 'resumed'), 'payload_codec' => 'avro'], + ]; + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/PreparedLocalOutcomeRequest', + json_decode(json_encode($body, JSON_THROW_ON_ERROR), flags: JSON_THROW_ON_ERROR)); + $outcome = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$retryTask['task_id']}/local-activities/{$retry['activity_attempt_id']}/outcome", $body); + $this->assertSame(200, $outcome->status(), json_encode($outcome->json('reason'), JSON_THROW_ON_ERROR)); + $outcome->assertJsonPath('recorded', true)->assertJsonPath('claim_released', false); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDelivered')->count()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'ActivityCompleted')->count()); + $this->assertNull(WorkflowRun::query()->findOrFail($task['run_id'])->cancellation_request_command_id); + } + + public static function forgedCleanupProofs(): iterable + { + foreach (['prepare', 'recover'] as $operation) { + foreach (['scope_id', 'request_id', 'delivery_history_event_id', 'cleanup_deadline_at'] as $field) { + yield $operation.':'.$field => [$operation, $field]; + } + } + } + + #[DataProvider('forgedCleanupProofs')] + public function test_scoped_cleanup_rejects_forged_authority_before_payload_resolution(string $operation, string $field): void + { + [$task, $scope, , $proof] = $this->cleanupClaim(); + $proof[$field] = 'invented-authority'; + $storage = \Mockery::mock(NamespaceExternalPayloadStorage::class); + $storage->shouldNotReceive('driverFor'); + $this->app->instance(NamespaceExternalPayloadStorage::class, $storage); + $before = WorkflowHistoryEvent::query()->count(); + $this->cleanupLocal($task, $operation, $scope, $proof, ['codec' => 'avro', 'invalid_reference' => true]) + ->assertConflict()->assertJsonPath('recorded', false)->assertJsonPath('reason', 'operation_scope_cancellation_prepared'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertSame(0, ActivityExecution::query()->count()); + $this->assertSame(0, ActivityAttempt::query()->count()); + } + + public static function cleanupGroupValidity(): iterable + { + yield 'original proofs' => [false]; + yield 'second proof invalid' => [true]; + } + + #[DataProvider('cleanupGroupValidity')] + public function test_scoped_cleanup_group_validates_all_proofs_before_any_sibling(bool $invalidSecond): void + { + [$task, $scope, , $proof] = $this->cleanupClaim(); + $commands = []; + foreach ([0, 1] as $index) { + $commands[] = [...$this->cleanupDescriptor($scope, $proof), 'type' => 'prepare_local_activity', + ...ParallelChildGroup::itemMetadata(3, 2, $index, 'activity')]; + } + if ($invalidSecond) { + $commands[1]['cancellation_cleanup']['request_id'] = 'invented-authority'; + $commands[1]['arguments'] = ['codec' => 'avro', 'invalid_reference' => true]; + } + $before = WorkflowHistoryEvent::query()->count(); + $response = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/checkpoint-group", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'checkpoint_id' => 'scoped-cleanup-group', 'start_sequence' => 3, 'commands' => $commands, + ]); + if ($invalidSecond) { + $response->assertConflict()->assertJsonPath('recorded', false)->assertJsonPath('reason', 'operation_scope_cancellation_prepared'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertSame(0, ActivityExecution::query()->count()); + $this->assertSame(0, ActivityAttempt::query()->count()); + } else { + $response->assertOk()->assertJsonPath('checkpointed', true); + $this->assertCount(2, $response->json('local_activities')); + $this->assertSame(2, ActivityExecution::query()->count()); + foreach (ActivityExecution::query()->get() as $execution) { + $this->assertCleanupContract($execution->activity_options['cancellation_cleanup']); + $this->assertSame($proof['delivery_history_event_id'], $execution->activity_options['cancellation_cleanup']['delivery_history_event_id']); + } + } + } + + private function cleanupClaim(): array + { + [$task, $scope, $request] = $this->claim(prepared: true); + $this->mutate('prepare', $task, $scope, $request, ['call_kind' => 'local_activity'])->assertOk(); + $delivery = $this->mutate('deliver', $task, $scope, $request, ['call_kind' => 'local_activity'])->assertOk()->json(); + $proof = ['scope_id' => $scope, 'request_id' => $request, 'delivery_history_event_id' => $delivery['history_event_id']]; + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/PreparedLocalCleanupProof', (object) $proof); + + return [$task, $scope, $request, $proof]; + } + + private function cleanupDescriptor(string $scope, array $proof, mixed $arguments = null): array + { + return ['type' => 'record_local_activity', 'activity_type' => 'opaque-local', 'payload_codec' => 'avro', + 'arguments' => $arguments ?? Serializer::serializeWithCodec('avro', []), + 'retry_policy' => ['max_attempts' => 2, 'backoff_seconds' => [0]], + 'cancellation_scope_id' => $scope, 'cancellation_cleanup' => $proof]; + } + + private function cleanupLocal(array $task, string $operation, string $scope, array $proof, mixed $arguments = null, + string $workerAttempt = 'scoped-cleanup-attempt') + { + $body = [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'worker_attempt_id' => $workerAttempt, 'sequence' => 3, + 'descriptor' => $this->cleanupDescriptor($scope, $proof, $arguments), + ]; + + return $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/{$operation}", $body); + } + + private function assertCleanupContract(array $snapshot): void + { + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/PreparedLocalCleanupSnapshot', + json_decode(json_encode($snapshot, JSON_THROW_ON_ERROR), flags: JSON_THROW_ON_ERROR)); + } + + private function claim(bool $timer = false, ?string $childPolicy = null, bool $prepared = false): array { $this->withHeaders($this->headers('operator'))->postJson('/api/workflows', [ 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'scoped-delivery', 'input' => ['Ada'], @@ -365,7 +554,8 @@ private function claim(bool $timer = false, ?string $childPolicy = null): array $this->withHeaders($this->headers())->postJson('/api/worker/register', [ 'worker_id' => 'delivery-owner', 'task_queue' => 'scoped-delivery', 'runtime' => 'php', 'supported_workflow_types' => ['tests.external-greeting-workflow'], - 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY], + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY, + ...($prepared ? [PreparedLocalActivityPolicy::CAPABILITY, PreparedLocalActivityPolicy::GROUP_CAPABILITY] : [])], 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), 'max_concurrent_workflow_tasks' => 1, ])->assertCreated(); $task = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ diff --git a/tests/Unit/WorkerProtocolOpenApiContractTest.php b/tests/Unit/WorkerProtocolOpenApiContractTest.php index 3298f1ef..1a878c08 100644 --- a/tests/Unit/WorkerProtocolOpenApiContractTest.php +++ b/tests/Unit/WorkerProtocolOpenApiContractTest.php @@ -3,9 +3,11 @@ namespace Tests\Unit; use App\Support\WorkerProtocol; +use PHPUnit\Framework\AssertionFailedError; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Symfony\Component\Yaml\Yaml; +use Tests\Support\OpenApiSchema; class WorkerProtocolOpenApiContractTest extends TestCase { @@ -41,6 +43,40 @@ public function test_http_spec_publishes_the_runtime_negotiation_contract(): voi ); } + public function test_prepared_cleanup_contract_preserves_root_shapes_and_requires_scoped_authority(): void + { + $schema = OpenApiSchema::fromFile(dirname(__DIR__, 2).'/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml'); + $rootProof = ['request_id' => 'request', 'delivery_history_event_id' => 'delivery']; + $rootSnapshot = [...$rootProof, 'root_request_id' => 'root-request', 'cleanup_deadline_at' => '2026-10-04T00:00:30.000000Z']; + $scopedSnapshot = [...$rootSnapshot, 'scope_id' => 'scope', 'operation_scope_id' => 'child-scope', + 'preparation_history_event_id' => 'preparation', 'authority_deadline_at' => '2026-10-04T00:00:15.000000Z']; + $schema->assertReferenceMatches('#/components/schemas/PreparedLocalCleanupProof', (object) $rootProof); + $schema->assertReferenceMatches('#/components/schemas/PreparedLocalCleanupProof', (object) [...$rootProof, 'scope_id' => 'scope']); + $schema->assertReferenceMatches('#/components/schemas/PreparedLocalCleanupSnapshot', null); + $schema->assertReferenceMatches('#/components/schemas/PreparedLocalCleanupSnapshot', (object) $rootSnapshot); + $schema->assertReferenceMatches('#/components/schemas/PreparedLocalCleanupSnapshot', (object) $scopedSnapshot); + } + + public static function invalidPreparedCleanupAuthority(): iterable + { + $proof = ['scope_id' => 'scope', 'request_id' => 'request', 'delivery_history_event_id' => 'delivery']; + $snapshot = [...$proof, 'root_request_id' => 'root-request', 'operation_scope_id' => 'child-scope', + 'preparation_history_event_id' => 'preparation', 'cleanup_deadline_at' => '2026-10-04T00:00:30.000000Z', + 'authority_deadline_at' => '2026-10-04T00:00:15.000000Z']; + yield 'invented proof deadline' => ['PreparedLocalCleanupProof', [...$proof, 'cleanup_deadline_at' => 'later']]; + yield 'null scope' => ['PreparedLocalCleanupProof', [...$proof, 'scope_id' => null]]; + yield 'missing preparation' => ['PreparedLocalCleanupSnapshot', array_diff_key($snapshot, ['preparation_history_event_id' => true])]; + yield 'missing ceiling' => ['PreparedLocalCleanupSnapshot', array_diff_key($snapshot, ['authority_deadline_at' => true])]; + } + + #[DataProvider('invalidPreparedCleanupAuthority')] + public function test_prepared_cleanup_contract_refuses_incomplete_or_worker_invented_authority(string $name, array $value): void + { + $this->expectException(AssertionFailedError::class); + OpenApiSchema::fromFile(dirname(__DIR__, 2).'/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml') + ->assertReferenceMatches('#/components/schemas/'.$name, (object) $value); + } + public function test_http_and_stream_specs_publish_the_runtime_negotiation_floor(): void { $streamSpecPath = dirname(__DIR__, 2).'/resources/platform-protocol-specs/worker-protocol-stream.asyncapi.yaml'; From 6845e877fc04275a08f1c17806d72d82d223a995 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sun, 4 Oct 2026 20:49:24 +0000 Subject: [PATCH 44/57] Verify scoped delivery keeps a recoverable hosting claim --- .../Source/CancellationScopeDeliveryTest.php | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/tests/Source/CancellationScopeDeliveryTest.php b/tests/Source/CancellationScopeDeliveryTest.php index 7333da29..f150931c 100644 --- a/tests/Source/CancellationScopeDeliveryTest.php +++ b/tests/Source/CancellationScopeDeliveryTest.php @@ -9,6 +9,7 @@ use App\Support\PreparedLocalActivityPolicy; use App\Support\WorkerProtocol; use Illuminate\Foundation\Testing\DatabaseMigrations; +use Illuminate\Support\Carbon; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Queue; use PHPUnit\Framework\Attributes\DataProvider; @@ -51,7 +52,7 @@ protected function setUp(): void WorkflowNamespace::query()->create(['name' => 'default', 'retention_days' => 30, 'status' => 'active']); } - public function test_preparation_and_delivery_reuse_first_boundary_without_releasing_or_renewing_claim(): void + public function test_preparation_and_delivery_reuse_first_boundary_with_recoverable_original_claim(): void { [$task, $scope, $request] = $this->claim(); $before = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); @@ -63,10 +64,11 @@ public function test_preparation_and_delivery_reuse_first_boundary_without_relea ->assertJsonPath('preparation_history_event_id', $prepared['preparation_history_event_id']) ->assertJsonPath('cancellation', $prepared['cancellation']) ->assertJsonPath('authority_deadline_at', $prepared['authority_deadline_at']); + $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); $delivery = $this->mutate('deliver', $task, $scope, $request)->assertOk()->assertJsonPath('delivered', true)->json(); $this->assertContract($delivery); $this->mutate('deliver', $task, $scope, $request)->assertOk()->assertJsonPath('history_event_id', $delivery['history_event_id']); - $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $this->assertRecoverableHostingClaim($before, $task['task_id']); $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDeliveryPrepared')->count()); $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDelivered')->count()); $this->assertSame($prepared['preparation_history_event_id'], $delivery['preparation_history_event_id']); @@ -219,7 +221,7 @@ public function test_timer_dispatch_reuses_original_preparation_and_fences_late_ ->assertJsonPath('preparation_history_event_id', $prepared['preparation_history_event_id'])->json(); $this->assertContract($response); $this->assertContract($prepared); - $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $this->assertRecoverableHostingClaim($before, $task['task_id']); $duplicate = $this->mutate('deliver', $task, $scope, $request, ['call_kind' => 'timer'])->assertOk()->json(); $this->assertSame($response['timer_cancellations'], $duplicate['timer_cancellations']); $this->assertSame($response['history_event_id'], $duplicate['history_event_id']); @@ -270,7 +272,7 @@ public function test_scoped_wait_delivery_retains_original_membership_and_claim( $this->assertSame($response['history_event_id'], $duplicate['history_event_id']); $this->assertSame($prepared['wait_members'], $duplicate['wait_members']); $this->assertSame($prepared['authority_deadline_at'], $duplicate['authority_deadline_at']); - $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $this->assertRecoverableHostingClaim($before, $task['task_id']); $marker = WorkflowHistoryEvent::query()->where('event_type', $kind === 'signal' ? 'SignalWaitCancelled' : 'ConditionWaitCancelled')->sole(); $this->assertSame($prepared['preparation_history_event_id'], $marker->payload['cancellation_scope']['preparation_history_event_id']); if ($timeout !== null) { @@ -350,7 +352,7 @@ public function test_child_delivery_reconciles_original_policy_receipts_without_ $this->assertTrue($receipt['ready']); $this->assertSame($delivery['history_event_id'], $duplicate['history_event_id']); } - $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $this->assertRecoverableHostingClaim($before, $task['task_id']); $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'ChildCancellationRequested')->count()); $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDelivered')->count()); } @@ -601,6 +603,17 @@ private function headers(string $role = 'worker', string $namespace = 'default', 'X-Durable-Workflow-Control-Plane-Version' => '2', WorkerProtocol::HEADER => $version]; } + private function assertRecoverableHostingClaim(array $before, string $taskId): void + { + $claim = WorkflowTask::query()->findOrFail($taskId); + $after = $claim->getRawOriginal(); + $this->assertTrue($claim->lease_expires_at->gt(now())); + $this->assertTrue($claim->lease_expires_at->lte(now()->addSeconds(10))); + $this->assertTrue($claim->lease_expires_at->lt(Carbon::parse($before['lease_expires_at'], 'UTC'))); + unset($before['lease_expires_at'], $before['updated_at'], $after['lease_expires_at'], $after['updated_at']); + $this->assertSame($before, $after); + } + private function assertContract(array $response): void { OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) From 30063a6dc1cc19a1630699a95edd1669ef0fcd44 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sun, 4 Oct 2026 21:04:05 +0000 Subject: [PATCH 45/57] Verify the stop-receipt hosting lease with shared source assertions --- tests/Source/CancellationScopeDeliveryTest.php | 12 ------------ .../Source/ScopedActivityCancellationReceiptTest.php | 2 +- tests/TestCase.php | 12 ++++++++++++ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/Source/CancellationScopeDeliveryTest.php b/tests/Source/CancellationScopeDeliveryTest.php index f150931c..69d1f2cc 100644 --- a/tests/Source/CancellationScopeDeliveryTest.php +++ b/tests/Source/CancellationScopeDeliveryTest.php @@ -9,7 +9,6 @@ use App\Support\PreparedLocalActivityPolicy; use App\Support\WorkerProtocol; use Illuminate\Foundation\Testing\DatabaseMigrations; -use Illuminate\Support\Carbon; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Queue; use PHPUnit\Framework\Attributes\DataProvider; @@ -603,17 +602,6 @@ private function headers(string $role = 'worker', string $namespace = 'default', 'X-Durable-Workflow-Control-Plane-Version' => '2', WorkerProtocol::HEADER => $version]; } - private function assertRecoverableHostingClaim(array $before, string $taskId): void - { - $claim = WorkflowTask::query()->findOrFail($taskId); - $after = $claim->getRawOriginal(); - $this->assertTrue($claim->lease_expires_at->gt(now())); - $this->assertTrue($claim->lease_expires_at->lte(now()->addSeconds(10))); - $this->assertTrue($claim->lease_expires_at->lt(Carbon::parse($before['lease_expires_at'], 'UTC'))); - unset($before['lease_expires_at'], $before['updated_at'], $after['lease_expires_at'], $after['updated_at']); - $this->assertSame($before, $after); - } - private function assertContract(array $response): void { OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) diff --git a/tests/Source/ScopedActivityCancellationReceiptTest.php b/tests/Source/ScopedActivityCancellationReceiptTest.php index 4e8a27d6..97e3c573 100644 --- a/tests/Source/ScopedActivityCancellationReceiptTest.php +++ b/tests/Source/ScopedActivityCancellationReceiptTest.php @@ -107,7 +107,7 @@ public function test_delivery_waits_for_original_owner_receipt_and_preserves_the ->assertJsonPath('cancellation', $pending['cancellation']) ->assertJsonPath('authority_deadline_at', $pending['authority_deadline_at'])->json(); $this->withHeaders($this->headers())->postJson($path, $body)->assertOk()->assertJsonPath('history_event_id', $delivered['history_event_id']); - $this->assertSame($before[0], WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $this->assertRecoverableHostingClaim($before[0], $task['task_id']); $this->assertSame($before[1][1], $this->snapshot($task, $target, $sibling)[1][1]); $this->assertLessThan(WorkflowHistoryEvent::query()->findOrFail($delivered['history_event_id'])->sequence, WorkflowHistoryEvent::query()->findOrFail($ack)->sequence); diff --git a/tests/TestCase.php b/tests/TestCase.php index 5fcfb67b..bfb3aa1b 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -4,6 +4,7 @@ use App\Support\WorkerProtocol; use Illuminate\Foundation\Testing\TestCase as BaseTestCase; +use Illuminate\Support\Carbon; use Workflow\V2\Models\WorkflowInstance; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowRunSummary; @@ -71,6 +72,17 @@ protected function portableWorkerAffinityRefusalManifest(): array ]); } + protected function assertRecoverableHostingClaim(array $before, string $taskId): void + { + $claim = WorkflowTask::query()->findOrFail($taskId); + $after = $claim->getRawOriginal(); + $this->assertTrue($claim->lease_expires_at->gt(now())); + $this->assertTrue($claim->lease_expires_at->lte(now()->addSeconds(10))); + $this->assertTrue($claim->lease_expires_at->lt(Carbon::parse($before['lease_expires_at'], 'UTC'))); + unset($before['lease_expires_at'], $before['updated_at'], $after['lease_expires_at'], $after['updated_at']); + $this->assertSame($before, $after); + } + private function pollingCachePath(): string { return sys_get_temp_dir().'/dw-server-test-polling-'.getmypid(); From 4429d5815f71410aeeaa9e250253f1b799336240 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sun, 4 Oct 2026 21:55:37 +0000 Subject: [PATCH 46/57] Qualify recoverable scoped preparation through HTTP --- .../Source/CancellationScopeDeliveryTest.php | 23 +++++++++++-------- .../ScopedActivityCancellationReceiptTest.php | 8 ++++++- tests/TestCase.php | 6 +++-- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/tests/Source/CancellationScopeDeliveryTest.php b/tests/Source/CancellationScopeDeliveryTest.php index 69d1f2cc..177140dd 100644 --- a/tests/Source/CancellationScopeDeliveryTest.php +++ b/tests/Source/CancellationScopeDeliveryTest.php @@ -32,6 +32,7 @@ use Workflow\V2\Support\CancellationScopeRequests; use Workflow\V2\Support\CooperativeCancellationDelivery; use Workflow\V2\Support\ParallelChildGroup; +use Workflow\V2\TaskWatchdog; use Workflow\V2\WorkflowStub; /** Source-only preparation commits outside the test's transaction. */ @@ -59,11 +60,12 @@ public function test_preparation_and_delivery_reuse_first_boundary_with_recovera ->assertJsonPath('prepared', true)->assertJsonPath('delivered', false) ->assertJsonPath('claim_released', false)->assertJsonPath('activity_members', [])->json(); $this->assertContract($prepared); + $this->assertRecoverableHostingClaim($before, $task['task_id'], shortened: true); $this->mutate('prepare', $task, $scope, $request)->assertOk() ->assertJsonPath('preparation_history_event_id', $prepared['preparation_history_event_id']) ->assertJsonPath('cancellation', $prepared['cancellation']) ->assertJsonPath('authority_deadline_at', $prepared['authority_deadline_at']); - $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $this->assertRecoverableHostingClaim($before, $task['task_id']); $delivery = $this->mutate('deliver', $task, $scope, $request)->assertOk()->assertJsonPath('delivered', true)->json(); $this->assertContract($delivery); $this->mutate('deliver', $task, $scope, $request)->assertOk()->assertJsonPath('history_event_id', $delivery['history_event_id']); @@ -191,18 +193,21 @@ public function test_replacement_claim_reuses_preparation_and_original_deadline( { [$task, $scope, $request] = $this->claim(); $prepared = $this->mutate('prepare', $task, $scope, $request)->assertOk()->json(); - $this->travel(1)->seconds(); - $claim = WorkflowTask::query()->findOrFail($task['task_id']); - $claim->forceFill(['attempt_count' => $task['workflow_task_attempt'] + 1])->save(); - CooperativeCancellationPolicy::bindClaim($claim, CooperativeCancellationPolicy::workerSnapshot( - WorkerRegistration::query()->where('worker_id', 'delivery-owner')->sole(), '1.20')); + $this->travel(11)->seconds(); + $repair = TaskWatchdog::runPass(respectThrottle: false, runIds: [$task['run_id']]); + $this->assertSame([], $repair['existing_task_failures']); + $this->assertSame(1, $repair['repaired_existing_tasks']); + $replacement = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'delivery-owner', 'task_queue' => 'scoped-delivery', + ])->assertOk()->json('task'); + $this->assertSame($task['task_id'], $replacement['task_id']); + $this->assertSame($task['workflow_task_attempt'] + 1, $replacement['workflow_task_attempt']); $this->mutate('prepare', $task, $scope, $request)->assertConflict()->assertJsonPath('reason', 'workflow_task_attempt_mismatch'); - $task['workflow_task_attempt']++; - $this->mutate('prepare', $task, $scope, $request)->assertOk() + $this->mutate('prepare', $replacement, $scope, $request)->assertOk() ->assertJsonPath('preparation_history_event_id', $prepared['preparation_history_event_id']) ->assertJsonPath('cancellation', $prepared['cancellation']) ->assertJsonPath('authority_deadline_at', $prepared['authority_deadline_at']); - $this->mutate('deliver', $task, $scope, $request)->assertOk()->assertJsonPath('delivered', true); + $this->mutate('deliver', $replacement, $scope, $request)->assertOk()->assertJsonPath('delivered', true); $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDeliveryPrepared')->count()); } diff --git a/tests/Source/ScopedActivityCancellationReceiptTest.php b/tests/Source/ScopedActivityCancellationReceiptTest.php index 97e3c573..bf6a411d 100644 --- a/tests/Source/ScopedActivityCancellationReceiptTest.php +++ b/tests/Source/ScopedActivityCancellationReceiptTest.php @@ -99,7 +99,13 @@ public function test_delivery_waits_for_original_owner_receipt_and_preserves_the ->assertJsonPath('delivered', false)->assertJsonPath('prepared', true) ->assertJsonPath('reason', 'cancellation_scope_activity_stop_not_acknowledged') ->assertJsonPath('claim_released', false)->json(); - $this->assertSame($before, $this->snapshot($task, $target, $sibling)); + $this->assertRecoverableHostingClaim($before[0], $task['task_id']); + $afterPending = $this->snapshot($task, $target, $sibling); + $beforePending = $before; + foreach (['lease_expires_at', 'updated_at'] as $field) { + unset($beforePending[0][$field], $afterPending[0][$field]); + } + $this->assertSame($beforePending, $afterPending); $ack = $this->acknowledge($target, $request->payload['request_id'])->assertOk()->json('history_event_id'); $delivered = $this->withHeaders($this->headers())->postJson($path, $body)->assertOk() ->assertJsonPath('delivered', true) diff --git a/tests/TestCase.php b/tests/TestCase.php index bfb3aa1b..d4597676 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -72,13 +72,15 @@ protected function portableWorkerAffinityRefusalManifest(): array ]); } - protected function assertRecoverableHostingClaim(array $before, string $taskId): void + protected function assertRecoverableHostingClaim(array $before, string $taskId, bool $shortened = false): void { $claim = WorkflowTask::query()->findOrFail($taskId); $after = $claim->getRawOriginal(); $this->assertTrue($claim->lease_expires_at->gt(now())); $this->assertTrue($claim->lease_expires_at->lte(now()->addSeconds(10))); - $this->assertTrue($claim->lease_expires_at->lt(Carbon::parse($before['lease_expires_at'], 'UTC'))); + if ($shortened) { + $this->assertTrue($claim->lease_expires_at->lt(Carbon::parse($before['lease_expires_at'], 'UTC'))); + } unset($before['lease_expires_at'], $before['updated_at'], $after['lease_expires_at'], $after['updated_at']); $this->assertSame($before, $after); } From d0d7e43c1e7ec23dd1307dc68d6df84d9ba1aa96 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sun, 4 Oct 2026 22:20:47 +0000 Subject: [PATCH 47/57] Allow the complete source database pairing to finish --- .github/workflows/phpunit-feature.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index 7676dc1e..b924c6a8 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -263,7 +263,7 @@ jobs: needs: [action-policy, preflight] if: ${{ github.event_name == 'workflow_dispatch' && inputs.prepared_local_workflow_commit != '' }} runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 services: mysql: image: mysql@sha256:679e7e924f38a3cbb62a3d7df32924b83f7321a602d3f9f967c01b3df18495d6 From 92e9f12866be51e86d35b043597af3e705ff8122 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sun, 4 Oct 2026 23:35:20 +0000 Subject: [PATCH 48/57] Qualify accepted scoped cancellation recovery through HTTP --- .../Source/CancellationScopeDeliveryTest.php | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/tests/Source/CancellationScopeDeliveryTest.php b/tests/Source/CancellationScopeDeliveryTest.php index 177140dd..46337d1e 100644 --- a/tests/Source/CancellationScopeDeliveryTest.php +++ b/tests/Source/CancellationScopeDeliveryTest.php @@ -60,7 +60,7 @@ public function test_preparation_and_delivery_reuse_first_boundary_with_recovera ->assertJsonPath('prepared', true)->assertJsonPath('delivered', false) ->assertJsonPath('claim_released', false)->assertJsonPath('activity_members', [])->json(); $this->assertContract($prepared); - $this->assertRecoverableHostingClaim($before, $task['task_id'], shortened: true); + $this->assertRecoverableHostingClaim($before, $task['task_id']); $this->mutate('prepare', $task, $scope, $request)->assertOk() ->assertJsonPath('preparation_history_event_id', $prepared['preparation_history_event_id']) ->assertJsonPath('cancellation', $prepared['cancellation']) @@ -80,6 +80,47 @@ public static function phases(): array return [['prepare'], ['deliver']]; } + public function test_accepted_unprepared_request_recovers_through_watchdog_and_http_poll(): void + { + [$task, $scope, $request] = $this->claim(); + $accepted = WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeRequested')->sole(); + $before = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + $historyBefore = WorkflowHistoryEvent::query()->count(); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDeliveryPrepared')->count()); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDelivered')->count()); + $this->travel(11)->seconds(); + $this->mutate('prepare', $task, $scope, $request)->assertConflict() + ->assertJsonPath('reason', 'cancellation_scope_workflow_claim_mismatch'); + $this->assertSame($historyBefore, WorkflowHistoryEvent::query()->count()); + $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $repair = TaskWatchdog::runPass(respectThrottle: false, runIds: [$task['run_id']]); + $this->assertSame([], $repair['existing_task_failures']); + $this->assertSame(1, $repair['repaired_existing_tasks']); + $replacement = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'delivery-owner', 'task_queue' => 'scoped-delivery', + ])->assertOk()->json('task'); + $this->assertSame($task['task_id'], $replacement['task_id']); + $this->assertSame($task['workflow_task_attempt'] + 1, $replacement['workflow_task_attempt']); + $run = WorkflowRun::query()->findOrFail($task['run_id']); + $replacementBefore = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + $duplicate = CancellationScopeRequests::request($run, $scope, '1.20', 300); + $this->assertSame($accepted->id, $duplicate->id); + $this->assertEquals($accepted->payload, $duplicate->payload); + $this->assertSame($replacementBefore, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $this->mutate('prepare', $task, $scope, $request)->assertConflict() + ->assertJsonPath('reason', 'workflow_task_attempt_mismatch'); + $prepared = $this->mutate('prepare', $replacement, $scope, $request)->assertOk()->json(); + $this->assertContract($prepared); + $this->assertSame($request, $prepared['request_id']); + $this->assertSame($accepted->payload['cancellation']['root_context']['cleanup_deadline_at'], $prepared['authority_deadline_at']); + $this->assertNull($run->fresh()->cancellation_request_command_id); + $this->assertRecoverableHostingClaim($replacementBefore, $task['task_id']); + $this->mutate('deliver', $replacement, $scope, $request)->assertOk() + ->assertJsonPath('delivered', true)->assertJsonPath('preparation_history_event_id', $prepared['preparation_history_event_id']); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeRequested')->count()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDeliveryPrepared')->count()); + } + #[DataProvider('phases')] public function test_invalid_authority_and_namespace_are_refused_without_effects(string $phase): void { @@ -586,7 +627,9 @@ private function claim(bool $timer = false, ?string $childPolicy = null, bool $p 'payload_codec' => 'avro', 'cancellation_scope_id' => $scope, 'cancellation_policy' => $childPolicy]], ])->assertOk()->assertJsonPath('checkpointed', true); } + $beforeRequest = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); $request = CancellationScopeRequests::request(WorkflowRun::query()->findOrFail($task['run_id']), $scope, '1.20', 30); + $this->assertRecoverableHostingClaim($beforeRequest, $task['task_id'], shortened: true); return [$task, $scope, $request->payload['request_id']]; } From 95806ae21511db513548a7064d34912cfea115fd Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Mon, 5 Oct 2026 21:06:28 +0000 Subject: [PATCH 49/57] Prepare scoped local cancellation before guarded publication --- .../Api/PreparedLocalActivityController.php | 26 ++++++- .../Source/CancellationScopeDeliveryTest.php | 71 +++++++++++++++++-- 2 files changed, 92 insertions(+), 5 deletions(-) diff --git a/app/Http/Controllers/Api/PreparedLocalActivityController.php b/app/Http/Controllers/Api/PreparedLocalActivityController.php index 99c97f4b..e6ef5eae 100644 --- a/app/Http/Controllers/Api/PreparedLocalActivityController.php +++ b/app/Http/Controllers/Api/PreparedLocalActivityController.php @@ -22,11 +22,13 @@ use Illuminate\Support\Facades\DB; use Illuminate\Validation\ValidationException; use Workflow\V2\Contracts\CancellationScopeAdmission; +use Workflow\V2\Contracts\PreparedCancellationScopeTaskBridge; use Workflow\V2\Contracts\PreparedLocalActivityTaskBridge; use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Exceptions\ExternalPayloadIntegrityException; use Workflow\V2\Exceptions\StructuralLimitExceededException; use Workflow\V2\Models\ActivityAttempt; +use Workflow\V2\Models\ActivityExecution; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; @@ -205,6 +207,26 @@ private function attemptOperation(Request $request, string $taskId, string $atte return self::refused($validated['lease_owner'], ['original_prepared_local_claim']); } try { + $bridge = app(WorkflowTaskBridge::class); + if ($operation !== 'acknowledge' && $bridge instanceof PreparedCancellationScopeTaskBridge) { + $execution = ActivityExecution::query()->where('workflow_run_id', $task->workflow_run_id) + ->find($attempt->activity_execution_id); + if ($execution instanceof ActivityExecution + && ($execution->activity_options['cancellation_scope_id'] ?? 'root') !== 'root' + && ! array_key_exists('cancellation_cleanup', $execution->activity_options)) { + // Cancellation preparation commits before the fence actor. + // This stop-only observation grants no renewal, heartbeat + // or publication authority. Keep those mutations behind + // the registration and quota locks below. + $control = $this->mutations->run(fn (): array => $bridge->controlLocalActivity( + $attemptId, $validated['lease_owner'], (int) $validated['workflow_task_attempt'], + false, WorkerProtocol::requestVersion($request) + )); + if ($operation !== 'outcome' && isset($control['cancellation_scope'])) { + return $this->reply($request, $control); + } + } + } if ($operation === 'outcome' && ($validated['report']['outcome'] ?? null) === 'completed') { $validated['report'] = $this->resolvePayload($validated['report'], 'result', 'report', $namespace); } @@ -398,7 +420,9 @@ private function reply(Request $request, array|JsonResponse $reply): JsonRespons return BackendLockPressure::workerOperationResponse($request, true); } $reason = $reply['reason'] ?? null; - $ok = $reason === null || isset($reply['cancellation_request']); + $ok = $reason === null || isset($reply['cancellation_request']) + || (($reply['fenced'] ?? false) === true && is_array($reply['cancellation_scope'] ?? null) + && in_array($reason, ['cancellation_scope_requested', 'cancellation_scope_deadline_expired'], true)); return WorkerProtocol::json($reply, $ok ? 200 : (in_array($reason, ['task_not_found', 'activity_attempt_not_found'], true) ? 404 : 409)); } diff --git a/tests/Source/CancellationScopeDeliveryTest.php b/tests/Source/CancellationScopeDeliveryTest.php index 46337d1e..47ef9687 100644 --- a/tests/Source/CancellationScopeDeliveryTest.php +++ b/tests/Source/CancellationScopeDeliveryTest.php @@ -433,6 +433,64 @@ public function test_scoped_cleanup_prepares_and_reuses_original_runtime_snapsho $this->assertNull(WorkflowRun::query()->findOrFail($task['run_id'])->cancellation_request_command_id); } + #[DataProvider('localControlRequests')] + public function test_running_local_control_commits_original_scope_preparation_before_fencing(string $operation, bool $runRequest, bool $staleRegistration): void + { + [$task, $scope] = $this->claim(prepared: true, requestCancellation: false); + $path = "/api/worker/workflow-tasks/{$task['task_id']}/local-activities"; + $owner = ['lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt']]; + $local = $this->withHeaders($this->headers())->postJson($path.'/prepare', [ + ...$owner, 'sequence' => 2, 'worker_attempt_id' => 'running-scoped-local', + 'descriptor' => ['type' => 'record_local_activity', 'activity_type' => 'opaque-local', + 'arguments' => Serializer::serializeWithCodec('avro', []), 'payload_codec' => 'avro', + 'cancellation_scope_id' => $scope], + ])->assertOk()->json(); + $run = WorkflowRun::query()->findOrFail($task['run_id']); + if ($runRequest) { + WorkflowStub::loadRun($run->id)->requestCancellation('finish the run', 30); + } else { + CancellationScopeRequests::request($run, $scope, '1.20', 30); + } + $request = CancellationScopeRequests::context($run->fresh(), $scope); + if ($staleRegistration) { + WorkerRegistration::query()->where('worker_id', 'delivery-owner') + ->update(['last_heartbeat_at' => now()->subHour()]); + } + $before = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + $response = $this->withHeaders($this->headers())->postJson($path.'/'.$local['activity_attempt_id'].'/'.$operation, + [...$owner, 'renew_lease' => true, 'progress' => ['message' => 'must not record']]); + $response->assertOk()->assertJsonPath('active', false)->assertJsonPath('renewed', false) + ->assertJsonPath('heartbeat_recorded', false)->assertJsonPath('fenced', true) + ->assertJsonPath('reason', 'cancellation_scope_requested') + ->assertJsonPath('cancellation_scope.cancellation', $request->toArray()); + $this->assertIsString($response->json('history_refresh_page_token')); + $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $preparation = WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDeliveryPrepared')->sole(); + $fence = WorkflowHistoryEvent::query()->where('event_type', 'ActivityCancelled')->sole(); + $this->assertLessThan($fence->sequence, $preparation->sequence); + $this->assertSame($request->requestId, $fence->workflow_command_id); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'ActivityHeartbeatRecorded')->count()); + $this->withHeaders($this->headers())->postJson($path.'/'.$local['activity_attempt_id'].'/acknowledge-cancellation', + [...$owner, 'request_id' => $request->requestId])->assertOk()->assertJsonPath('acknowledged', true); + $this->withHeaders($this->headers())->postJson($path.'/'.$local['activity_attempt_id'].'/outcome', + [...$owner, 'report' => ['outcome' => 'completed', 'result' => Serializer::serializeWithCodec('avro', 'stale'), + 'payload_codec' => 'avro']])->assertConflict()->assertJsonPath('recorded', false); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'ActivityCompleted')->count()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDeliveryPrepared')->count()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'ActivityCancelled')->count()); + } + + public static function localControlRequests(): iterable + { + foreach (['control', 'heartbeat'] as $operation) { + foreach ([false, true] as $runRequest) { + foreach ([false, true] as $staleRegistration) { + yield $operation.':'.($runRequest ? 'run' : 'scope').':'.($staleRegistration ? 'stale' : 'fresh') => [$operation, $runRequest, $staleRegistration]; + } + } + } + } + public function test_scoped_cleanup_recovery_keeps_original_proof_and_refuses_stale_publication(): void { [$task, $scope, , $proof] = $this->cleanupClaim(); @@ -593,7 +651,7 @@ private function assertCleanupContract(array $snapshot): void json_decode(json_encode($snapshot, JSON_THROW_ON_ERROR), flags: JSON_THROW_ON_ERROR)); } - private function claim(bool $timer = false, ?string $childPolicy = null, bool $prepared = false): array + private function claim(bool $timer = false, ?string $childPolicy = null, bool $prepared = false, bool $requestCancellation = true): array { $this->withHeaders($this->headers('operator'))->postJson('/api/workflows', [ 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'scoped-delivery', 'input' => ['Ada'], @@ -628,10 +686,15 @@ private function claim(bool $timer = false, ?string $childPolicy = null, bool $p ])->assertOk()->assertJsonPath('checkpointed', true); } $beforeRequest = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); - $request = CancellationScopeRequests::request(WorkflowRun::query()->findOrFail($task['run_id']), $scope, '1.20', 30); - $this->assertRecoverableHostingClaim($beforeRequest, $task['task_id'], shortened: true); + $request = $requestCancellation + ? CancellationScopeRequests::request(WorkflowRun::query()->findOrFail($task['run_id']), $scope, '1.20', 30) : null; + if ($requestCancellation) { + $this->assertRecoverableHostingClaim($beforeRequest, $task['task_id'], shortened: true); + } else { + $this->assertSame($beforeRequest, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + } - return [$task, $scope, $request->payload['request_id']]; + return [$task, $scope, $request?->payload['request_id']]; } private function mutate(string $phase, array $task, string $scope, string $request, array $overrides = [], From 4e95bdd39ab8c11825dde48a7f302bebf0fcbbd0 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Mon, 5 Oct 2026 22:15:35 +0000 Subject: [PATCH 50/57] Compare scoped cancellation context independently of JSON object order --- tests/Source/CancellationScopeDeliveryTest.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/Source/CancellationScopeDeliveryTest.php b/tests/Source/CancellationScopeDeliveryTest.php index 47ef9687..f05092d4 100644 --- a/tests/Source/CancellationScopeDeliveryTest.php +++ b/tests/Source/CancellationScopeDeliveryTest.php @@ -461,8 +461,11 @@ public function test_running_local_control_commits_original_scope_preparation_be [...$owner, 'renew_lease' => true, 'progress' => ['message' => 'must not record']]); $response->assertOk()->assertJsonPath('active', false)->assertJsonPath('renewed', false) ->assertJsonPath('heartbeat_recorded', false)->assertJsonPath('fenced', true) - ->assertJsonPath('reason', 'cancellation_scope_requested') - ->assertJsonPath('cancellation_scope.cancellation', $request->toArray()); + ->assertJsonPath('reason', 'cancellation_scope_requested'); + $this->assertSame( + $request->toArray(), + $request::fromArray($response->json('cancellation_scope.cancellation'))->toArray(), + ); $this->assertIsString($response->json('history_refresh_page_token')); $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); $preparation = WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDeliveryPrepared')->sole(); From 741920d6c4756123ec268ab8b5dedc4be72bfa80 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 01:26:55 +0000 Subject: [PATCH 51/57] fix: retain scoped cleanup timer proof through HTTP completion --- app/Http/Controllers/Api/WorkerController.php | 8 +-- .../worker-protocol-api.openapi.yaml | 4 ++ .../Source/CancellationScopeDeliveryTest.php | 63 +++++++++++++++++++ 3 files changed, 71 insertions(+), 4 deletions(-) diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index 22224ada..a9fd6eac 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -1572,6 +1572,10 @@ private function mutateWorkflowTaskCommands(Request $request, string $taskId, bo 'commands' => ['required', 'array', 'min:1'], 'commands.*.type' => ['required', 'string'], 'commands.*.cancellation_scope_id' => ['sometimes', 'string', 'min:1', 'max:255'], + 'commands.*.cancellation_cleanup' => ['sometimes', 'required', 'array:scope_id,request_id,delivery_history_event_id'], + 'commands.*.cancellation_cleanup.scope_id' => ['sometimes', 'required', 'string', 'max:255'], + 'commands.*.cancellation_cleanup.request_id' => ['required_with:commands.*.cancellation_cleanup', 'string', 'max:255'], + 'commands.*.cancellation_cleanup.delivery_history_event_id' => ['required_with:commands.*.cancellation_cleanup', 'string', 'max:255'], 'commands.*.result' => ['nullable'], 'commands.*.activity_type' => ['nullable', 'string'], 'commands.*.arguments' => ['nullable'], @@ -1704,10 +1708,6 @@ private function mutateWorkflowTaskCommands(Request $request, string $taskId, bo $topLevelRules = []; if ($group) { $completionRules['commands'] = ['required', 'array', 'min:1', 'max:100']; - $completionRules['commands.*.cancellation_cleanup'] = ['nullable', 'array:scope_id,request_id,delivery_history_event_id']; - $completionRules['commands.*.cancellation_cleanup.scope_id'] = ['sometimes', 'required', 'string', 'max:255']; - $completionRules['commands.*.cancellation_cleanup.request_id'] = ['required_with:commands.*.cancellation_cleanup', 'string', 'max:255']; - $completionRules['commands.*.cancellation_cleanup.delivery_history_event_id'] = ['required_with:commands.*.cancellation_cleanup', 'string', 'max:255']; } $commandRules = ['commands' => ['required', 'array', 'min:1']]; foreach ($completionRules as $field => $rules) { diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 79d0e57d..07daf03b 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -2877,6 +2877,10 @@ components: maxLength: 255 x-durable-workflow-minimum-protocol-version: "1.20" description: Optional exact-run canonical membership for schedule_activity, start_timer, start_child_workflow, prepare_local_activity, open_signal_wait and open_condition_wait commands in the unfrozen candidate. Requires an installed CancellationScopeAdmission backend. The named scope must already be recorded before admission. Invalid membership is refused before payload resolution and rechecked under the run lock. Omission preserves historical root membership. Wait commands close a turn and cannot enter a retained-claim prefix. This field does not advertise scope execution or supervision. + cancellation_cleanup: + $ref: "#/components/schemas/PreparedLocalCleanupProof" + x-durable-workflow-minimum-protocol-version: "1.20" + description: Original scope, request and committed delivery references for a shielded scoped start_timer or prepared local cleanup operation. The installed runtime derives immutable deadline authority from canonical history. A timer at or beyond the remaining authority ceiling is refused. This unfrozen candidate field never grants a new cleanup budget. parent_close_policy: type: [string, "null"] description: Child parent-close policy. request_cancel retains legacy terminal cancellation. request_cancellation requests cooperative cleanup under protocol 1.20 and the original shared budget. diff --git a/tests/Source/CancellationScopeDeliveryTest.php b/tests/Source/CancellationScopeDeliveryTest.php index f05092d4..4241c85c 100644 --- a/tests/Source/CancellationScopeDeliveryTest.php +++ b/tests/Source/CancellationScopeDeliveryTest.php @@ -615,6 +615,69 @@ public function test_scoped_cleanup_group_validates_all_proofs_before_any_siblin } } + public function test_completion_preserves_original_scoped_timer_cleanup_proof_after_metadata_prefix(): void + { + [$task, $scope, , $proof] = $this->cleanupClaim(); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [ + ['type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 'cleanup entry')], + ['type' => 'start_timer', 'delay_seconds' => 1, 'cancellation_scope_id' => $scope, + 'cancellation_cleanup' => $proof], + ], + ])->assertOk(); + $timer = WorkflowHistoryEvent::query()->where('event_type', 'TimerScheduled')->sole(); + $this->assertSame(4, $timer->payload['sequence']); + $this->assertSame($scope, $timer->payload['cancellation_scope_id']); + $snapshot = $timer->payload['cancellation_cleanup']; + $this->assertSame($proof['request_id'], $snapshot['request_id']); + $this->assertSame($proof['delivery_history_event_id'], $snapshot['delivery_history_event_id']); + $delivery = WorkflowHistoryEvent::query()->findOrFail($proof['delivery_history_event_id']); + $this->assertSame($delivery->payload['authority_deadline_at'], $snapshot['authority_deadline_at']); + $this->assertSame($delivery->payload['cancellation']['root_context']['cleanup_deadline_at'], $snapshot['cleanup_deadline_at']); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'SideEffectRecorded')->count()); + $this->assertSame(TimerStatus::Pending, WorkflowTimer::query()->sole()->status); + } + + public static function invalidTimerCleanupProofs(): iterable + { + foreach (['null', 'empty', 'missing-scope', 'new-deadline', 'different-request', 'too-long'] as $mutation) { + yield $mutation => [$mutation]; + } + } + + #[DataProvider('invalidTimerCleanupProofs')] + public function test_completion_refuses_invalid_timer_cleanup_before_any_prefix_commit(string $mutation): void + { + [$task, $scope, , $proof] = $this->cleanupClaim(); + $command = ['type' => 'start_timer', 'delay_seconds' => 1, 'cancellation_scope_id' => $scope, + 'cancellation_cleanup' => $proof]; + if ($mutation === 'null') { $command['cancellation_cleanup'] = null; } + elseif ($mutation === 'empty') { $command['cancellation_cleanup'] = []; } + elseif ($mutation === 'missing-scope') { unset($command['cancellation_cleanup']['scope_id']); } + elseif ($mutation === 'new-deadline') { $command['cancellation_cleanup']['cleanup_deadline_at'] = now()->addHour()->toIso8601String(); } + elseif ($mutation === 'different-request') { $command['cancellation_cleanup']['request_id'] = 'invented-request'; } + else { $command['delay_seconds'] = 31; } + $before = WorkflowHistoryEvent::query()->count(); + $response = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [ + ['type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 'must not commit')], + $command, + ], + ]); + if (in_array($mutation, ['missing-scope', 'different-request', 'too-long'], true)) { + $response->assertConflict()->assertJsonPath('recorded', false)->assertJsonPath('reason', + $mutation === 'too-long' ? 'cancellation_scope_cleanup_timer_exceeds_deadline' + : 'cancellation_scope_cleanup_authority_mismatch'); + } else { + $response->assertUnprocessable(); + } + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertSame(0, WorkflowTimer::query()->count()); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + } + private function cleanupClaim(): array { [$task, $scope, $request] = $this->claim(prepared: true); From c350304c73ca80804a39c6f9b6a92a23ce6decbb Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 01:32:04 +0000 Subject: [PATCH 52/57] docs: version the scoped cleanup timer HTTP contract --- .../platform-protocol-specs/worker-protocol-api.openapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 07daf03b..4f3dc46f 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "45" + version: "46" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: From dfff248300a6f243071a16db5ac940148fc0a034 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 05:46:17 +0000 Subject: [PATCH 53/57] Retain bounded MySQL lock diagnostics for the existing canary failure --- .github/workflows/server-perf.yml | 1 + scripts/perf/mysql-lock-diagnostics.sql | 14 ++++++++++++++ scripts/perf/run-server-soak.sh | 17 +++++++++++++++++ 3 files changed, 32 insertions(+) create mode 100644 scripts/perf/mysql-lock-diagnostics.sql diff --git a/.github/workflows/server-perf.yml b/.github/workflows/server-perf.yml index 0714b93a..42e0048e 100644 --- a/.github/workflows/server-perf.yml +++ b/.github/workflows/server-perf.yml @@ -258,6 +258,7 @@ jobs: DW_PERF_MAX_HEALTH_LATENCY_SECONDS: "3" DW_PERF_READINESS_DIAGNOSTICS: "1" DW_PERF_CONTROL_PLANE_INTERVAL_SECONDS: "5" + DW_PERF_MYSQL_LOCK_DIAGNOSTICS: "1" DW_PERF_MAX_CONTROL_PLANE_LATENCY_SECONDS: "5" DW_PERF_WORKFLOW_VERSION: ${{ steps.workflow.outputs.ref }} DW_PERF_MAX_SERVER_MEMORY_MB: "768" diff --git a/scripts/perf/mysql-lock-diagnostics.sql b/scripts/perf/mysql-lock-diagnostics.sql new file mode 100644 index 00000000..3797bdba --- /dev/null +++ b/scripts/perf/mysql-lock-diagnostics.sql @@ -0,0 +1,14 @@ +SELECT NOW(6) AS captured_at, @@innodb_print_all_deadlocks AS deadlock_logging; +SHOW ENGINE INNODB STATUS; +SELECT THREAD_ID, EVENT_ID, CURRENT_SCHEMA, SQL_TEXT, MYSQL_ERRNO, + RETURNED_SQLSTATE, MESSAGE_TEXT, TIMER_WAIT, LOCK_TIME, ROWS_AFFECTED +FROM performance_schema.events_statements_history_long +WHERE MYSQL_ERRNO IN (1205, 1213) +ORDER BY THREAD_ID, EVENT_ID +LIMIT 100; +SELECT ENGINE_TRANSACTION_ID, THREAD_ID, OBJECT_SCHEMA, OBJECT_NAME, + INDEX_NAME, LOCK_TYPE, LOCK_MODE, LOCK_STATUS +FROM performance_schema.data_locks +WHERE OBJECT_SCHEMA = 'durable_workflow' +LIMIT 100; +SELECT * FROM performance_schema.data_lock_waits LIMIT 100; diff --git a/scripts/perf/run-server-soak.sh b/scripts/perf/run-server-soak.sh index 9ca3b46f..db99e913 100755 --- a/scripts/perf/run-server-soak.sh +++ b/scripts/perf/run-server-soak.sh @@ -33,6 +33,7 @@ POLL_TIMEOUT="${DW_PERF_POLL_TIMEOUT:-1}" POLL_INTERVAL_MS="${DW_PERF_POLL_INTERVAL_MS:-50}" POLL_SIGNAL_CHECK_INTERVAL_MS="${DW_PERF_POLL_SIGNAL_CHECK_INTERVAL_MS:-25}" READINESS_DIAGNOSTICS="${DW_PERF_READINESS_DIAGNOSTICS:-0}" +MYSQL_LOCK_DIAGNOSTICS="${DW_PERF_MYSQL_LOCK_DIAGNOSTICS:-0}" PUBLISHED_SERVER_IMAGE="${DW_PERF_PUBLISHED_SERVER_IMAGE:-}" HTTP_VARIANT="${DW_PERF_HTTP_VARIANT:-apache}" PREBUILT_HTTP_IMAGE_ID="${DW_PERF_PREBUILT_HTTP_IMAGE_ID:-}" @@ -72,6 +73,10 @@ if [[ "$READINESS_DIAGNOSTICS" != 0 && "$READINESS_DIAGNOSTICS" != 1 ]]; then echo "DW_PERF_READINESS_DIAGNOSTICS must be 0 or 1." >&2 exit 2 fi +if [[ "$MYSQL_LOCK_DIAGNOSTICS" != 0 && "$MYSQL_LOCK_DIAGNOSTICS" != 1 ]]; then + echo "DW_PERF_MYSQL_LOCK_DIAGNOSTICS must be 0 or 1." >&2 + exit 2 +fi if [[ -n "$PUBLISHED_SERVER_IMAGE" \ && ! "$PUBLISHED_SERVER_IMAGE" =~ ^durableworkflow/server@sha256:[0-9a-f]{64}$ ]]; then echo "DW_PERF_PUBLISHED_SERVER_IMAGE must be an exact durableworkflow/server sha256 digest." >&2 @@ -317,6 +322,12 @@ fi cleanup() { local status=$? + if [[ "$MYSQL_LOCK_DIAGNOSTICS" == 1 ]]; then + timeout 15s docker exec -i -e MYSQL_PWD=root "${PROJECT}-mysql-1" \ + mysql --user=root --batch \ + < "$ROOT_DIR/scripts/perf/mysql-lock-diagnostics.sql" \ + > "$ARTIFACT_DIR/mysql-lock-diagnostics.txt" 2>&1 || true + fi docker logs "${PROJECT}-server-1" > "$ARTIFACT_DIR/server.log" 2>&1 || true if [[ "$HTTP_VARIANT" == nginx-fpm || "$HTTP_VARIANT" == apache-event-fpm ]]; then docker logs "${PROJECT}-fpm-1" > "$ARTIFACT_DIR/fpm.log" 2>&1 || true @@ -496,6 +507,12 @@ if [ "$setup_status" -ne 0 ]; then exit "$setup_status" fi +if [[ "$MYSQL_LOCK_DIAGNOSTICS" == 1 ]]; then + timeout 15s docker exec -e MYSQL_PWD=root "${PROJECT}-mysql-1" \ + mysql --user=root --execute="SET GLOBAL innodb_print_all_deadlocks=ON; UPDATE performance_schema.setup_consumers SET ENABLED='YES' WHERE NAME='events_statements_history_long';" \ + > "$ARTIFACT_DIR/mysql-lock-diagnostics-setup.txt" 2>&1 +fi + if [[ -n "$PUBLISHED_SERVER_IMAGE" ]]; then for service in bootstrap server worker scheduler; do actual_image_id="$(docker inspect "${PROJECT}-${service}-1" --format '{{.Image}}')" From f6ceeceee9751892961ef1e56cac3e4e249e0adc Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 06:18:37 +0000 Subject: [PATCH 54/57] Lock workflow runs before worker task mutations and repair --- .github/workflows/phpunit-feature.yml | 2 +- .../Api/CooperativeCancellationController.php | 2 +- app/Http/Controllers/Api/WorkerController.php | 59 +++-- app/Support/NamespaceWorkflowScope.php | 21 ++ tests/Feature/WorkflowTaskRepairRaceTest.php | 235 ++++++++++++++++++ 5 files changed, 290 insertions(+), 29 deletions(-) create mode 100644 tests/Feature/WorkflowTaskRepairRaceTest.php diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index b924c6a8..66ef02c7 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -187,7 +187,7 @@ jobs: --health-cmd='mysqladmin --silent --host=127.0.0.1 ping' \ --health-interval=1s --health-start-period=20s --health-retries=30 \ mysql@sha256:679e7e924f38a3cbb62a3d7df32924b83f7321a602d3f9f967c01b3df18495d6 \ - --disable-log-bin --performance-schema=OFF \ + --disable-log-bin --performance-schema=ON \ --innodb-buffer-pool-size=64M --innodb-redo-log-capacity=64M \ --innodb-log-buffer-size=8M --table-open-cache=256 --max-connections=40 docker run --detach \ diff --git a/app/Http/Controllers/Api/CooperativeCancellationController.php b/app/Http/Controllers/Api/CooperativeCancellationController.php index 4e85cb8c..7aa4c92a 100644 --- a/app/Http/Controllers/Api/CooperativeCancellationController.php +++ b/app/Http/Controllers/Api/CooperativeCancellationController.php @@ -183,7 +183,7 @@ public function deliver(Request $request, string $taskId): JsonResponse ): array { // Keep the ownership check and engine delivery under the same // task lock. A reclaim cannot replace the checked attempt. - $task = NamespaceWorkflowScope::taskQuery($namespace)->lockForUpdate()->find($taskId); + $task = NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId); if (! $task instanceof WorkflowTask) { return ['delivered' => false, 'reason' => 'task_not_found']; } diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index a9fd6eac..526a3974 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -1947,10 +1947,7 @@ function () use ( ->where('worker_id', $validated['lease_owner']) ->lockForUpdate() ->first(); - $claimedTask = NamespaceWorkflowScope::taskQuery((string) $namespace) - ->whereKey($taskId) - ->lockForUpdate() - ->first(); + $claimedTask = NamespaceWorkflowScope::lockTaskForMutation((string) $namespace, $taskId); if ($response = $this->guardWorkflowTaskOwnership( $request, @@ -3658,7 +3655,7 @@ public function heartbeatWorkflowTask(Request $request, string $taskId): JsonRes fn (): array|JsonResponse => DB::transaction(function () use ($request, $namespace, $taskId, $validated, $bridge): array|JsonResponse { // Ownership must remain valid until renewal commits. A // check before this lock can acknowledge a reclaimed lease. - NamespaceWorkflowScope::taskQuery($namespace)->lockForUpdate()->find($taskId); + NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId); if ($response = $this->guardWorkflowTaskOwnership( $request, $namespace, @@ -3745,11 +3742,15 @@ public function failWorkflowTask(Request $request, string $taskId): JsonResponse if ($this->workflowTaskFailureWaitsForHistory($validated['failure'])) { try { $outcome = $this->storageMutations->run( - fn (): array => $this->acknowledgeWorkflowTaskWaitingForHistory( - $namespace, - $taskId, - $validated['failure'], - ), + fn (): array|JsonResponse => DB::transaction(function () use ($request, $namespace, $taskId, $validated): array|JsonResponse { + NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId); + if ($response = $this->guardWorkflowTaskOwnership($request, $namespace, $taskId, + (int) $validated['workflow_task_attempt'], $validated['lease_owner'])) { + return $response; + } + + return $this->acknowledgeWorkflowTaskWaitingForHistory($namespace, $taskId, $validated['failure']); + }), ); } catch (\Throwable $exception) { if (! BackendLockPressure::is($exception)) { @@ -3759,6 +3760,10 @@ public function failWorkflowTask(Request $request, string $taskId): JsonResponse return BackendLockPressure::workerOperationResponse($request, false); } + if ($outcome instanceof JsonResponse) { + return $outcome; + } + return WorkerProtocol::json([ 'task_id' => $taskId, 'workflow_task_attempt' => (int) $validated['workflow_task_attempt'], @@ -3774,16 +3779,24 @@ public function failWorkflowTask(Request $request, string $taskId): JsonResponse try { $outcome = $this->storageMutations->run(function () use ( $bridge, + $request, $namespace, $taskId, $validated, - ): array { + ): array|JsonResponse { return DB::transaction(function () use ( $bridge, + $request, $namespace, $taskId, $validated, - ): array { + ): array|JsonResponse { + NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId); + if ($response = $this->guardWorkflowTaskOwnership($request, $namespace, $taskId, + (int) $validated['workflow_task_attempt'], $validated['lease_owner'])) { + return $response; + } + $outcome = $bridge->fail($taskId, $validated['failure']); $nextTaskId = is_string($outcome['next_task_id'] ?? null) ? $outcome['next_task_id'] @@ -3822,6 +3835,10 @@ public function failWorkflowTask(Request $request, string $taskId): JsonResponse return BackendLockPressure::workerOperationResponse($request, false); } + if ($outcome instanceof JsonResponse) { + return $outcome; + } + $nextTaskId = is_string($outcome['next_task_id'] ?? null) ? $outcome['next_task_id'] : null; @@ -3915,11 +3932,7 @@ private function acknowledgeWorkflowTaskWaitingForHistory(string $namespace, str &$createdTaskIds, ): array { /** @var WorkflowTask|null $task */ - $task = WorkflowTask::query() - ->lockForUpdate() - ->whereKey($taskId) - ->where('namespace', $namespace) - ->first(); + $task = NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId); if (! $task instanceof WorkflowTask) { return ['recorded' => false, 'reason' => 'task_not_found', 'next_task_id' => null]; @@ -4039,11 +4052,7 @@ private function recordWorkflowTaskFailureIdentity( ): void { DB::transaction(function () use ($namespace, $taskId, $failure, $replayBlocked): void { /** @var WorkflowTask|null $task */ - $task = WorkflowTask::query() - ->lockForUpdate() - ->whereKey($taskId) - ->where('namespace', $namespace) - ->first(); + $task = NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId); if (! $task instanceof WorkflowTask || $task->task_type !== TaskType::Workflow @@ -4086,11 +4095,7 @@ private function createRetryWorkflowTask(string $namespace, string $failedTaskId { return DB::transaction(function () use ($namespace, $failedTaskId): ?string { /** @var WorkflowTask|null $failedTask */ - $failedTask = WorkflowTask::query() - ->lockForUpdate() - ->whereKey($failedTaskId) - ->where('namespace', $namespace) - ->first(); + $failedTask = NamespaceWorkflowScope::lockTaskForMutation($namespace, $failedTaskId); if (! $failedTask instanceof WorkflowTask || $failedTask->task_type !== TaskType::Workflow diff --git a/app/Support/NamespaceWorkflowScope.php b/app/Support/NamespaceWorkflowScope.php index e245caf7..3716ab6a 100644 --- a/app/Support/NamespaceWorkflowScope.php +++ b/app/Support/NamespaceWorkflowScope.php @@ -86,4 +86,25 @@ public static function task(string $namespace, string $taskId): ?WorkflowTask ->whereKey($taskId) ->first(); } + + /** Acquire these locks inside the caller's mutation transaction. */ + public static function lockTaskForMutation(string $namespace, string $taskId): ?WorkflowTask + { + // Repair and cancellation admission lock the run before its tasks. + // An unlocked lookup discovers the immutable run ID without holding a + // task lock while waiting for that run. Revalidate it under the lock. + $runId = self::taskQuery($namespace)->whereKey($taskId)->value('workflow_run_id'); + if (! is_string($runId)) { + return null; + } + + $run = WorkflowRun::query()->where('namespace', $namespace) + ->whereKey($runId)->lockForUpdate()->first(); + if (! $run instanceof WorkflowRun) { + return null; + } + + return self::taskQuery($namespace)->whereKey($taskId) + ->where('workflow_run_id', $runId)->lockForUpdate()->first(); + } } diff --git a/tests/Feature/WorkflowTaskRepairRaceTest.php b/tests/Feature/WorkflowTaskRepairRaceTest.php new file mode 100644 index 00000000..cb6058ef --- /dev/null +++ b/tests/Feature/WorkflowTaskRepairRaceTest.php @@ -0,0 +1,235 @@ +markTestSkipped('Set '.$prefix.'HOST to a disposable database test server.'); + } + if (! function_exists('pcntl_fork') || ! function_exists('posix_kill') || ! function_exists('stream_socket_pair')) { + $this->markTestSkipped('Process control and local sockets are required.'); + } + $port = getenv($prefix.'PORT') ?: ($driver === 'pgsql' ? '5432' : '3306'); + $user = getenv($prefix.'USER') ?: ($driver === 'pgsql' ? 'postgres' : 'root'); + $password = getenv($prefix.'PASSWORD') ?: ''; + $this->databaseAdmin = [$driver.':host='.$host.';port='.$port.($driver === 'pgsql' ? ';dbname=postgres' : ''), $user, $password]; + $this->databaseName = 'dw_task_repair_'.bin2hex(random_bytes(6)); + $admin = new PDO(...$this->databaseAdmin); + $admin->exec('CREATE DATABASE '.$this->databaseName); + unset($admin); + config(['database.default' => $driver, 'database.connections.'.$driver.'.host' => $host, + 'database.connections.'.$driver.'.port' => $port, 'database.connections.'.$driver.'.database' => $this->databaseName, + 'database.connections.'.$driver.'.username' => $user, 'database.connections.'.$driver.'.password' => $password, + 'database.connections.'.$driver.'.url' => null]); + DB::purge($driver); + $this->artisan('migrate:fresh', ['--force' => true])->assertExitCode(0); + Queue::fake(); + $this->configureWorkflowTypes(['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class]); + $this->createNamespace('default'); + $this->registerWorker('repair-race-worker', 'repair-race'); + } + + protected function tearDown(): void + { + if ($this->databaseAdmin !== null) { + DB::disconnect(); + $admin = new PDO(...$this->databaseAdmin); + $admin->exec('DROP DATABASE '.$this->databaseName); + } + parent::tearDown(); + } + + public static function mutations(): array + { + $cases = []; + foreach (['mysql', 'pgsql'] as $driver) { + foreach (['heartbeat', 'complete', 'fail', 'waiting_history'] as $operation) { + $cases[$driver.' '.$operation] = [$driver, $operation]; + } + } + + return $cases; + } + + #[DataProvider('mutations')] + public function test_repair_can_lock_tasks_while_a_worker_mutation_waits_for_the_run(string $driver, string $operation): void + { + $this->initializeDatabase($driver); + $runId = $this->postJson('/api/workflows', [ + 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'repair-race', 'input' => ['Ada'], + ], $this->apiHeaders())->assertCreated()->json('run_id'); + $claim = $this->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'repair-race-worker', 'task_queue' => 'repair-race', 'poll_request_id' => 'claim', + ], $this->workerHeaders())->assertOk()->json('task'); + $this->assertIsArray($claim); + $actors = []; + DB::purge(); + try { + $actors['repair'] = $this->forkActor('repair', $runId, $claim); + $this->readMessage($actors['repair']); + fwrite($actors['repair']['socket'], "go\n"); + $this->assertTrue($this->readMessage($actors['repair'])['run_locked']); + $actors['worker'] = $this->forkActor($operation, $runId, $claim); + $worker = $this->readMessage($actors['worker']); + fwrite($actors['worker']['socket'], "go\n"); + $this->awaitRunWait($worker['connection_id']); + + // Repair holds the run, and the real HTTP request is waiting for + // it. Its task must remain available to repair, rather than form + // the opposite half of the captured task/run deadlock. + DB::transaction(function () use ($claim): void { + $task = WorkflowTask::query()->whereKey($claim['task_id'])->lock('for update nowait')->firstOrFail(); + $this->assertSame('repair-race-worker', $task->lease_owner); + $this->assertSame($claim['workflow_task_attempt'], $task->attempt_count); + }); + + fwrite($actors['repair']['socket'], "continue\n"); + $this->assertTrue($this->finishActor($actors['repair'])['accepted']); + unset($actors['repair']); + $result = $this->finishActor($actors['worker']); + unset($actors['worker']); + $this->assertSame(200, $result['status'], json_encode($result)); + $this->assertTrue($result['body'][$operation === 'heartbeat' ? 'renewed' : 'recorded']); + $this->assertSame(match ($operation) { + 'complete' => 'completed', 'waiting_history' => 'waiting', default => 'pending', + }, + WorkflowRun::query()->findOrFail($runId)->status->value); + } finally { + foreach ($actors as $actor) { + posix_kill($actor['pid'], SIGKILL); + pcntl_waitpid($actor['pid'], $status); + fclose($actor['socket']); + } + DB::purge(); + DB::reconnect(); + } + } + + /** @return array{pid: int, socket: resource} */ + private function forkActor(string $operation, string $runId, array $claim): array + { + $sockets = stream_socket_pair(STREAM_PF_UNIX, STREAM_SOCK_STREAM, STREAM_IPPROTO_IP); + $this->assertIsArray($sockets); + $pid = pcntl_fork(); + $this->assertNotSame(-1, $pid); + if ($pid === 0) { + fclose($sockets[0]); + stream_set_timeout($sockets[1], 15); + $ok = false; + try { + DB::reconnect(); + $idQuery = DB::connection()->getDriverName() === 'mysql' + ? 'SELECT CONNECTION_ID() AS id' : 'SELECT pg_backend_pid() AS id'; + fwrite($sockets[1], json_encode(['connection_id' => (int) DB::selectOne($idQuery)->id]).PHP_EOL); + if (fgets($sockets[1]) !== "go\n") { + throw new RuntimeException('Actor was not released.'); + } + if ($operation === 'repair') { + $paused = false; + DB::listen(static function (QueryExecuted $query) use (&$paused, $sockets): void { + if (! $paused && str_contains($query->sql, 'workflow_runs') + && str_contains(strtolower($query->sql), 'for update')) { + $paused = true; + fwrite($sockets[1], json_encode(['run_locked' => true]).PHP_EOL); + if (fgets($sockets[1]) !== "continue\n") { + throw new RuntimeException('Repair was not released.'); + } + } + }); + $result = ['accepted' => WorkflowStub::loadRun($runId)->repair()->accepted()]; + } else { + $body = ['lease_owner' => 'repair-race-worker', + 'workflow_task_attempt' => $claim['workflow_task_attempt']]; + if ($operation === 'complete') { + $body['commands'] = [['type' => 'complete_workflow', + 'result' => Serializer::serializeWithCodec('avro', 'done')]]; + } elseif (in_array($operation, ['fail', 'waiting_history'], true)) { + $body['failure'] = ['message' => 'Synthetic worker failure', + 'type' => $operation === 'waiting_history' ? 'WorkflowTaskWaitingForHistory' : 'TestFailure']; + } + $endpoint = $operation === 'waiting_history' ? 'fail' : $operation; + $response = $this->postJson('/api/worker/workflow-tasks/'.$claim['task_id'].'/'.$endpoint, + $body, $this->workerHeaders()); + $result = ['status' => $response->status(), 'body' => array_intersect_key($response->json(), + array_flip(['renewed', 'recorded', 'reason', 'outcome', 'run_status', 'errors']))]; + } + fwrite($sockets[1], json_encode(['result' => $result], JSON_THROW_ON_ERROR).PHP_EOL); + $ok = true; + } catch (Throwable $error) { + fwrite($sockets[1], json_encode(['error' => $error::class.': '.$error->getMessage()]).PHP_EOL); + } finally { + DB::disconnect(); + fclose($sockets[1]); + } + exit($ok ? 0 : 1); + } + fclose($sockets[1]); + stream_set_timeout($sockets[0], 15); + + return ['pid' => $pid, 'socket' => $sockets[0]]; + } + + private function awaitRunWait(int $connectionId): void + { + $query = DB::connection()->getDriverName() === 'mysql' + ? 'SELECT COUNT(*) AS waiting FROM performance_schema.data_lock_waits w JOIN performance_schema.threads t ON t.THREAD_ID = w.REQUESTING_THREAD_ID WHERE t.PROCESSLIST_ID = ?' + : 'SELECT COUNT(*) AS waiting FROM pg_locks WHERE pid = ? AND NOT granted'; + $until = microtime(true) + 5; + do { + if ((int) DB::selectOne($query, [$connectionId])->waiting > 0) { + $this->addToAssertionCount(1); + + return; + } + usleep(10_000); + } while (microtime(true) < $until); + $this->fail('Worker mutation did not wait on the run lock, connection '.$connectionId.'.'); + } + + private function readMessage(array $actor): array + { + $line = fgets($actor['socket']); + $this->assertIsString($line, 'Actor did not respond.'); + $message = json_decode($line, true, flags: JSON_THROW_ON_ERROR); + $this->assertArrayNotHasKey('error', $message, $message['error'] ?? ''); + + return $message; + } + + private function finishActor(array $actor): array + { + $message = $this->readMessage($actor); + pcntl_waitpid($actor['pid'], $status); + $this->assertTrue(pcntl_wifexited($status)); + $this->assertSame(0, pcntl_wexitstatus($status)); + fclose($actor['socket']); + + return $message['result']; + } +} From 56524277fc43b30d8d53b6ed5ad5ab686f5b4188 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 06:18:38 +0000 Subject: [PATCH 55/57] Prepare Server cancellation RC with published Native package --- app/Support/ClientCompatibility.php | 2 +- app/Support/WorkerProtocol.php | 2 +- composer.json | 4 ++-- composer.lock | 16 ++++++------- docker-compose.dedicated-matching.yml | 4 ++-- docker-compose.memo-rolling.yml | 4 ++-- docker-compose.published.yml | 4 ++-- docker-compose.small-cluster.yml | 2 +- docker-compose.yml | 8 +++---- k8s/README.md | 10 ++++---- k8s/helm/durable-workflow/Chart.yaml | 6 ++--- k8s/helm/durable-workflow/README.md | 2 +- .../ci/existing-secrets-values.yaml | 2 +- .../ci/ingress-and-hpa-values.yaml | 2 +- .../ci/inline-secrets-values.yaml | 2 +- .../durable-workflow/templates/_helpers.tpl | 2 +- k8s/helm/durable-workflow/values.yaml | 2 +- k8s/helm/examples/values-dev.yaml | 2 +- .../values-external-secrets-operator.yaml | 2 +- .../values-production-existing-secrets.yaml | 2 +- k8s/migration-job.yaml | 2 +- k8s/scheduler-cronjob.yaml | 2 +- k8s/secret.yaml | 2 +- k8s/server-deployment.yaml | 2 +- k8s/worker-deployment.yaml | 2 +- .../worker-protocol-api.openapi.yaml | 23 +++++++++++-------- .../worker-protocol-stream.asyncapi.yaml | 6 ++--- resources/release/source-release.json | 4 ++-- scripts/ci/helm_chart_release.py | 15 ++++++++---- scripts/ci/sync-source-release.mjs | 6 ++--- scripts/ci/test-source-release.mjs | 4 ++-- scripts/ci/test_helm_chart_release.py | 13 +++++++++++ scripts/k8s-kind-smoke.sh | 2 +- .../Feature/ClusterInfoCompatibilityTest.php | 6 ++--- .../CooperativeCancellationProtocolTest.php | 4 ++-- 35 files changed, 99 insertions(+), 74 deletions(-) diff --git a/app/Support/ClientCompatibility.php b/app/Support/ClientCompatibility.php index 73621847..33bd6773 100644 --- a/app/Support/ClientCompatibility.php +++ b/app/Support/ClientCompatibility.php @@ -13,7 +13,7 @@ final class ClientCompatibility private const SUPPORTED_SDK_VERSIONS = [ 'php' => self::STABLE_2_X, 'python' => self::STABLE_2_X, - 'rust' => self::STABLE_2_X, + 'rust' => '>=2.0.0,<4.0.0', 'cli' => self::STABLE_2_X, ]; diff --git a/app/Support/WorkerProtocol.php b/app/Support/WorkerProtocol.php index 9488fafd..ac73109a 100644 --- a/app/Support/WorkerProtocol.php +++ b/app/Support/WorkerProtocol.php @@ -19,7 +19,7 @@ class WorkerProtocol * here. WorkflowPackageApiFloor asserts the installed package still * provides the companion protocol helpers for this version. */ - public const VERSION = '1.19'; + public const VERSION = '1.20'; public const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION = '1.18'; diff --git a/composer.json b/composer.json index 5d1184bc..6fe92567 100644 --- a/composer.json +++ b/composer.json @@ -6,7 +6,7 @@ "require": { "php": "^8.2", "apache/avro": "^1.12", - "durable-workflow/workflow": "2.3.4", + "durable-workflow/workflow": "2.4.0-rc.1", "laravel/framework": "^13.30", "laravel/tinker": "^3.0", "league/flysystem-aws-s3-v3": "^3.35.3" @@ -48,7 +48,7 @@ }, "extra": { "durable-workflow": { - "product-train": "2.4.40" + "product-train": "2.5.0-rc.1" }, "laravel": { "dont-discover": [] diff --git a/composer.lock b/composer.lock index 9b2dcdac..dd0dd5f3 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "cf02d93da18a4a641b24c9242ac9ad9d", + "content-hash": "fe2189c084f56d517dbc310e3212b474", "packages": [ { "name": "apache/avro", @@ -655,16 +655,16 @@ }, { "name": "durable-workflow/workflow", - "version": "2.3.4", + "version": "2.4.0-rc.1", "source": { "type": "git", "url": "https://github.com/durable-workflow/workflow.git", - "reference": "e66d22482541ddcaa03964fe006538a7340083fe" + "reference": "dd23e5e0632165a23b9d1e6a8b67dbdb798d6213" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/durable-workflow/workflow/zipball/e66d22482541ddcaa03964fe006538a7340083fe", - "reference": "e66d22482541ddcaa03964fe006538a7340083fe", + "url": "https://api.github.com/repos/durable-workflow/workflow/zipball/dd23e5e0632165a23b9d1e6a8b67dbdb798d6213", + "reference": "dd23e5e0632165a23b9d1e6a8b67dbdb798d6213", "shasum": "" }, "require": { @@ -697,7 +697,7 @@ "dev-main": "2.0.x-dev" }, "durable-workflow": { - "product-train": "2.3.4", + "product-train": "2.4.0-rc.1", "laravel-embedded-upgrade-contract": "resources/laravel-embedded-upgrade-contract.json", "laravel-dependency-security-policy": "resources/laravel-dependency-security-policy.json" } @@ -724,9 +724,9 @@ "description": "Embedded durable workflow runtime and orchestration engine for Laravel applications.", "support": { "issues": "https://github.com/durable-workflow/workflow/issues", - "source": "https://github.com/durable-workflow/workflow/tree/2.3.4" + "source": "https://github.com/durable-workflow/workflow/tree/2.4.0-rc.1" }, - "time": "2026-10-05T05:54:54+00:00" + "time": "2026-10-06T05:57:49+00:00" }, { "name": "egulias/email-validator", diff --git a/docker-compose.dedicated-matching.yml b/docker-compose.dedicated-matching.yml index 861849aa..35e30de4 100644 --- a/docker-compose.dedicated-matching.yml +++ b/docker-compose.dedicated-matching.yml @@ -32,13 +32,13 @@ name: durable-workflow-server # daemon reports `shape: dedicated`. # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.4.40}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.5.0-rc.1}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.4.40}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.5.0-rc.1}} APP_DEBUG: ${APP_DEBUG:-false} DB_CONNECTION: mysql DB_HOST: mysql diff --git a/docker-compose.memo-rolling.yml b/docker-compose.memo-rolling.yml index a8bba672..41d7b47e 100644 --- a/docker-compose.memo-rolling.yml +++ b/docker-compose.memo-rolling.yml @@ -49,14 +49,14 @@ services: command: ["server-bootstrap"] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.4.40} + APP_VERSION: ${APP_VERSION:-2.5.0-rc.1} successor: image: ${DW_MEMO_SUCCESSOR_IMAGE:-durable-workflow/server-memo-rolling:local} ports: !override [] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.4.40} + APP_VERSION: ${APP_VERSION:-2.5.0-rc.1} DW_SERVER_ID: memo-successor DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node diff --git a/docker-compose.published.yml b/docker-compose.published.yml index 202d5e3f..9e8cc46d 100644 --- a/docker-compose.published.yml +++ b/docker-compose.published.yml @@ -1,13 +1,13 @@ name: durable-workflow-server # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.4.40}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.5.0-rc.1}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.4.40}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.5.0-rc.1}} APP_DEBUG: ${APP_DEBUG:-false} LOG_CHANNEL: ${LOG_CHANNEL:-stderr} LOG_LEVEL: ${LOG_LEVEL:-info} diff --git a/docker-compose.small-cluster.yml b/docker-compose.small-cluster.yml index 196cc85d..c02d87df 100644 --- a/docker-compose.small-cluster.yml +++ b/docker-compose.small-cluster.yml @@ -12,7 +12,7 @@ x-server-build: &server-build x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: testing - APP_VERSION: ${APP_VERSION:-2.4.40} + APP_VERSION: ${APP_VERSION:-2.5.0-rc.1} APP_DEBUG: "false" DW_SERVER_KEY: ${DW_SERVER_KEY:-base64:5Zt4nUhlCm3DD0nLXZJQdHiwPfb56yGo9gNV/g3jYbY=} DB_CONNECTION: ${DW_SMALL_CLUSTER_DB:-mysql} diff --git a/docker-compose.yml b/docker-compose.yml index 1db96882..6b30d5b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node - APP_VERSION: "${APP_VERSION:-2.4.40}" + APP_VERSION: "${APP_VERSION:-2.5.0-rc.1}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -62,7 +62,7 @@ services: APP_NAME: "Durable Workflow Server" APP_ENV: local DW_SERVER_KEY: "${DW_SERVER_KEY:-}" - APP_VERSION: "${APP_VERSION:-2.4.40}" + APP_VERSION: "${APP_VERSION:-2.5.0-rc.1}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -124,7 +124,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: worker_node - APP_VERSION: "${APP_VERSION:-2.4.40}" + APP_VERSION: "${APP_VERSION:-2.5.0-rc.1}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 @@ -178,7 +178,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: scheduler_node - APP_VERSION: "${APP_VERSION:-2.4.40}" + APP_VERSION: "${APP_VERSION:-2.5.0-rc.1}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 diff --git a/k8s/README.md b/k8s/README.md index 898034bd..adad2a79 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -13,7 +13,7 @@ The checked-in manifests are synchronized with the repository's stable source release and pin its Docker Hub tag: ```text -durableworkflow/server:2.4.40 +durableworkflow/server:2.5.0-rc.1 ``` Before production use, patch every workload image to the exact published tag or @@ -21,15 +21,15 @@ digest you intend to run: ```bash kubectl set image -n durable-workflow deploy/durable-workflow-server \ - server=durableworkflow/server:2.4.40 + server=durableworkflow/server:2.5.0-rc.1 kubectl set image -n durable-workflow deploy/durable-workflow-worker \ - worker=durableworkflow/server:2.4.40 + worker=durableworkflow/server:2.5.0-rc.1 kubectl set image -n durable-workflow cronjob/durable-workflow-scheduler \ - scheduler=durableworkflow/server:2.4.40 + scheduler=durableworkflow/server:2.5.0-rc.1 ``` GitHub Container Registry publishes the same release line at -`ghcr.io/durable-workflow/server:2.4.40`. Digest pinning is preferred for strict +`ghcr.io/durable-workflow/server:2.5.0-rc.1`. Digest pinning is preferred for strict change control. The manifests expect you to provide: diff --git a/k8s/helm/durable-workflow/Chart.yaml b/k8s/helm/durable-workflow/Chart.yaml index a538308a..3eccc158 100644 --- a/k8s/helm/durable-workflow/Chart.yaml +++ b/k8s/helm/durable-workflow/Chart.yaml @@ -5,11 +5,11 @@ type: application # The chart's own semver version. Bumped on every chart release; treated as # independent of the server image version (appVersion). Breaking-change rules # for this version live in docs/helm-upgrading.md alongside the chart. -version: 0.1.136 +version: 0.1.137-rc.1 # The immutable Durable Workflow Server identity this chart release packages. # The onboarding default in values.yaml and appVersion are generated from the # checked-in source release record. -appVersion: "2.4.40" +appVersion: "2.5.0-rc.1" kubeVersion: ">=1.27.0-0" home: https://durable-workflow.github.io/docs/2.0/deployment sources: @@ -30,7 +30,7 @@ annotations: # exact commit that most recently changed the packaged chart. org.opencontainers.image.source: https://github.com/durable-workflow/server dev.durable-workflow.source-revision: "unreleased" - dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.4.40" + dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.5.0-rc.1" artifacthub.io/license: MIT artifacthub.io/category: integration-delivery # Free-form changelog for the current chart release shown by Artifact Hub. diff --git a/k8s/helm/durable-workflow/README.md b/k8s/helm/durable-workflow/README.md index 9385acd2..c7e27703 100644 --- a/k8s/helm/durable-workflow/README.md +++ b/k8s/helm/durable-workflow/README.md @@ -63,7 +63,7 @@ helm install durable-workflow ./k8s/helm/durable-workflow \ ```yaml image: - tag: "2.4.40" + tag: "2.5.0-rc.1" # Pin a digest in production: # digest: "sha256:abc123..." # memoPayloadStorage: "raw-json-v1" # Required for a digest or custom image. diff --git a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml index 1df00c27..562b31f4 100644 --- a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml @@ -1,7 +1,7 @@ # CI fixture: GitOps / externally-managed-secret path. The chart consumes # existing Secrets and renders no Secret resources of its own. image: - tag: "2.4.40" + tag: "2.5.0-rc.1" externalDatabase: connection: pgsql diff --git a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml index d80a3ac7..bd1272c2 100644 --- a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml +++ b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml @@ -1,6 +1,6 @@ # CI fixture: ingress + autoscaling enabled. Exercises optional templates. image: - tag: "2.4.40" + tag: "2.5.0-rc.1" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml index 954bec0c..5034a46f 100644 --- a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml @@ -2,7 +2,7 @@ # chart's render path is exercised end-to-end. Real deployments should use # existingSecret instead. image: - tag: "2.4.40" + tag: "2.5.0-rc.1" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/templates/_helpers.tpl b/k8s/helm/durable-workflow/templates/_helpers.tpl index 3ef03ef5..61571caf 100644 --- a/k8s/helm/durable-workflow/templates/_helpers.tpl +++ b/k8s/helm/durable-workflow/templates/_helpers.tpl @@ -88,7 +88,7 @@ resolved by an explicit capability declaration or an existing workload marker. {{- define "durable-workflow.memoPayloadStorageForImage" -}} {{- $image := toString . -}} {{- $normalized := regexReplaceAll "^index\\.docker\\.io/" $image "docker.io/" -}} -{{- if eq $normalized "docker.io/durableworkflow/server:2.4.40" -}} +{{- if eq $normalized "docker.io/durableworkflow/server:2.5.0-rc.1" -}} dual-v1 {{- else if regexMatch "^docker\\.io/durableworkflow/server:2\\.0\\.0-rc\\.[0-9]+$" $normalized -}} {{- $releaseCandidate := atoi (regexFind "[0-9]+$" $normalized) -}} diff --git a/k8s/helm/durable-workflow/values.yaml b/k8s/helm/durable-workflow/values.yaml index d8bbfd43..d7ef6d23 100644 --- a/k8s/helm/durable-workflow/values.yaml +++ b/k8s/helm/durable-workflow/values.yaml @@ -21,7 +21,7 @@ image: registry: docker.io repository: durableworkflow/server # Generated by scripts/ci/sync-source-release.mjs. Do not edit this default. - tag: "2.4.40" + tag: "2.5.0-rc.1" # Optional digest pin. When set, takes precedence over tag for change control. # Example: "sha256:abc123..." digest: "" diff --git a/k8s/helm/examples/values-dev.yaml b/k8s/helm/examples/values-dev.yaml index a6bc2170..51a2f947 100644 --- a/k8s/helm/examples/values-dev.yaml +++ b/k8s/helm/examples/values-dev.yaml @@ -3,7 +3,7 @@ # shape in production. image: - tag: "2.4.40" + tag: "2.5.0-rc.1" externalDatabase: connection: mysql diff --git a/k8s/helm/examples/values-external-secrets-operator.yaml b/k8s/helm/examples/values-external-secrets-operator.yaml index 2be8597e..e4b0863f 100644 --- a/k8s/helm/examples/values-external-secrets-operator.yaml +++ b/k8s/helm/examples/values-external-secrets-operator.yaml @@ -5,7 +5,7 @@ # concern. image: - tag: "2.4.40" + tag: "2.5.0-rc.1" externalDatabase: connection: pgsql diff --git a/k8s/helm/examples/values-production-existing-secrets.yaml b/k8s/helm/examples/values-production-existing-secrets.yaml index 0504d984..6f5305ca 100644 --- a/k8s/helm/examples/values-production-existing-secrets.yaml +++ b/k8s/helm/examples/values-production-existing-secrets.yaml @@ -10,7 +10,7 @@ image: repository: durable-workflow/server # Pin a digest in production for change-control auditability. digest: "" # e.g. "sha256:abc123..." - tag: "2.4.40" + tag: "2.5.0-rc.1" externalDatabase: connection: pgsql diff --git a/k8s/migration-job.yaml b/k8s/migration-job.yaml index ce6fd2ff..0ad2e66c 100644 --- a/k8s/migration-job.yaml +++ b/k8s/migration-job.yaml @@ -13,7 +13,7 @@ spec: restartPolicy: OnFailure containers: - name: migrate - image: durableworkflow/server:2.4.40 + image: durableworkflow/server:2.5.0-rc.1 command: ["server-entrypoint"] args: ["server-bootstrap"] envFrom: diff --git a/k8s/scheduler-cronjob.yaml b/k8s/scheduler-cronjob.yaml index 1406e115..adbabb55 100644 --- a/k8s/scheduler-cronjob.yaml +++ b/k8s/scheduler-cronjob.yaml @@ -24,7 +24,7 @@ spec: restartPolicy: Never containers: - name: scheduler - image: durableworkflow/server:2.4.40 + image: durableworkflow/server:2.5.0-rc.1 command: ["server-entrypoint"] args: - sh diff --git a/k8s/secret.yaml b/k8s/secret.yaml index e80d1202..9fc185d3 100644 --- a/k8s/secret.yaml +++ b/k8s/secret.yaml @@ -12,7 +12,7 @@ metadata: app.kubernetes.io/name: durable-workflow data: APP_NAME: "Durable Workflow Server" - APP_VERSION: "2.4.40" + APP_VERSION: "2.5.0-rc.1" APP_ENV: production APP_DEBUG: "false" DB_CONNECTION: mysql diff --git a/k8s/server-deployment.yaml b/k8s/server-deployment.yaml index f962149e..bae6b72c 100644 --- a/k8s/server-deployment.yaml +++ b/k8s/server-deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: server - image: durableworkflow/server:2.4.40 + image: durableworkflow/server:2.5.0-rc.1 ports: - containerPort: 8080 name: http diff --git a/k8s/worker-deployment.yaml b/k8s/worker-deployment.yaml index f6b9935b..6a796b4e 100644 --- a/k8s/worker-deployment.yaml +++ b/k8s/worker-deployment.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: worker - image: durableworkflow/server:2.4.40 + image: durableworkflow/server:2.5.0-rc.1 command: ["server-entrypoint"] args: ["php", "artisan", "queue:work", "--sleep=1", "--tries=3", "--max-time=3600"] envFrom: diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 4f3dc46f..9daaaf58 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "46" + version: "47" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -14,9 +14,9 @@ info: from the same major whose minor is less than or equal to N. x-durable-workflow-worker-protocol-negotiation: advertised_version_path: worker_protocol.version - default_advertised_version: "1.19" + default_advertised_version: "1.20" request_header_rule: same_major_and_minor_less_than_or_equal_to_advertised - accepted_request_versions_by_default: ["1.0", "1.1", "1.2", "1.3", "1.4", "1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13", "1.14", "1.15", "1.16", "1.17", "1.18", "1.19"] + accepted_request_versions_by_default: ["1.0", "1.1", "1.2", "1.3", "1.4", "1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13", "1.14", "1.15", "1.16", "1.17", "1.18", "1.19", "1.20"] response_version: advertised_version fail_closed_on: [missing_header, malformed_version, different_major, minor_greater_than_advertised] x-durable-workflow-catalog-entry: worker_protocol_api @@ -56,6 +56,7 @@ x-durable-workflow-cancellation-scope-opening: allowance: shares_existing_workflow_claim_budget replay: identical_parent_and_shield_reuses_canonical_history_identity x-durable-workflow-cooperative-cancellation-contract: + status: release_candidate minimum_protocol_version: "1.20" worker_capability: cooperative_cancellation request_operation: POST /workflows/{workflowId}/request-cancellation @@ -90,7 +91,7 @@ x-durable-workflow-cooperative-cancellation-contract: cleanup_completion: cancelled_with_original_request_id terminal_cancel_and_terminate: unchanged x-durable-workflow-prepared-local-activity-contract: - status: candidate + status: release_candidate minimum_protocol_version: "1.20" worker_capabilities: [cooperative_cancellation, prepared_local_activities] backend_role: Workflow\V2\Contracts\PreparedLocalActivityTaskBridge @@ -495,6 +496,7 @@ paths: /worker/workflow-tasks/{taskId}/cancellation-scopes/open: post: operationId: openWorkflowCancellationScope + x-durable-workflow-experimental: true tags: [workflow-tasks] x-durable-workflow-minimum-protocol-version: "1.20" x-durable-workflow-worker-capability: cooperative_cancellation @@ -522,6 +524,7 @@ paths: /worker/workflow-tasks/{taskId}/cancellation-scopes/prepare: post: operationId: prepareWorkflowCancellationScopeDelivery + x-durable-workflow-experimental: true tags: [workflow-tasks] x-durable-workflow-minimum-protocol-version: "1.20" x-durable-workflow-worker-capability: cooperative_cancellation @@ -551,6 +554,7 @@ paths: /worker/workflow-tasks/{taskId}/cancellation-scopes/deliver: post: operationId: deliverWorkflowCancellationScope + x-durable-workflow-experimental: true tags: [workflow-tasks] x-durable-workflow-minimum-protocol-version: "1.20" x-durable-workflow-worker-capability: cooperative_cancellation @@ -584,6 +588,7 @@ paths: /worker/workflow-tasks/{taskId}/cancellation-scopes/checkpoint: post: operationId: checkpointCancellationScopePrefix + x-durable-workflow-experimental: true tags: [workflow-tasks] x-durable-workflow-minimum-protocol-version: "1.20" x-durable-workflow-worker-capability: cooperative_cancellation @@ -1538,21 +1543,21 @@ components: properties: cooperative_cancellation: type: boolean - description: Explicit support for claim-bound cooperative request delivery, requiring worker protocol 1.20 or newer in the same major version. False at the default protocol 1.19. + description: Explicit support for claim-bound cooperative request delivery, requiring worker protocol 1.20 or newer in the same major version. Workers must opt in on an immutable claim. activity_cancellation_acknowledgement: type: boolean - description: Original remote callback-stop receipts require an installed runtime primitive and worker protocol 1.20. False for the default protocol and for runtimes without that primitive. + description: Original remote callback-stop receipts require an installed runtime primitive and worker protocol 1.20. False for earlier protocols and runtimes without that primitive. prepared_local_activities: type: boolean - description: Candidate preparation, prefix checkpoint, supervisor control, application heartbeat, outcome, recovery and joined-stop receipts. Requires protocol 1.20 and the actual bound PreparedLocalActivityTaskBridge role including heartbeat support. False at the default protocol 1.19. + description: Preparation, prefix checkpoint, supervisor control, application heartbeat, outcome, recovery and joined-stop receipts. Requires protocol 1.20 and the actual bound PreparedLocalActivityTaskBridge role including heartbeat support. False for earlier protocols or an absent backend role. prepared_local_activity_groups: type: boolean - description: Candidate atomic complete-all-group admission. Requires prepared local activities, protocol 1.20 and the actual bound PreparedLocalActivityGroupTaskBridge. Existing custom prepared bridges do not acquire this role. + description: Atomic complete-all-group admission. Requires prepared local activities, protocol 1.20 and the actual bound PreparedLocalActivityGroupTaskBridge. Existing custom prepared bridges do not acquire this role. prepared_local_activity_cancellation_policies: type: array uniqueItems: true items: { type: string, enum: [try_cancel, wait_cancellation_completed] } - description: Explicit policies supported by the actual installed prepared-local bridge's optional supportedLocalActivityCancellationPolicies method. Empty at default protocol 1.19 or for older optional-role bridges. Workers need a new immutable claim with the separately named worker capability before admission or replay. + description: Explicit policies supported by the actual installed prepared-local bridge's optional supportedLocalActivityCancellationPolicies method. Empty for earlier protocols or older optional-role bridges. Workers need a new immutable claim with the separately named worker capability before admission or replay. supported_workflow_task_commands: type: array uniqueItems: true diff --git a/resources/platform-protocol-specs/worker-protocol-stream.asyncapi.yaml b/resources/platform-protocol-specs/worker-protocol-stream.asyncapi.yaml index 147f82bb..d4300794 100644 --- a/resources/platform-protocol-specs/worker-protocol-stream.asyncapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-stream.asyncapi.yaml @@ -2,7 +2,7 @@ asyncapi: 2.6.0 id: durable-workflow.v2.worker-protocol-stream info: title: Durable Workflow worker-protocol long-poll stream semantics - version: "13" + version: "14" description: > Normative AsyncAPI description of the public worker-protocol stream semantics. Durable uses HTTP long-poll request/response rather than a @@ -12,9 +12,9 @@ info: events, and terminal completion/failure events close the lease. x-durable-workflow-worker-protocol-negotiation: advertised_version_path: worker_protocol.version - default_advertised_version: "1.19" + default_advertised_version: "1.20" request_header_rule: same_major_and_minor_less_than_or_equal_to_advertised - accepted_request_versions_by_default: ["1.0", "1.1", "1.2", "1.3", "1.4", "1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13", "1.14", "1.15", "1.16", "1.17", "1.18", "1.19"] + accepted_request_versions_by_default: ["1.0", "1.1", "1.2", "1.3", "1.4", "1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13", "1.14", "1.15", "1.16", "1.17", "1.18", "1.19", "1.20"] response_version: advertised_version fail_closed_on: [missing_header, malformed_version, different_major, minor_greater_than_advertised] x-durable-workflow-catalog-entry: worker_protocol_stream diff --git a/resources/release/source-release.json b/resources/release/source-release.json index a2ed198e..72d9ffc1 100644 --- a/resources/release/source-release.json +++ b/resources/release/source-release.json @@ -1,9 +1,9 @@ { "schema": "durable-workflow.server.source-release/v1", "server": { - "version": "2.4.40" + "version": "2.5.0-rc.1" }, "helm_chart": { - "version": "0.1.136" + "version": "0.1.137-rc.1" } } diff --git a/scripts/ci/helm_chart_release.py b/scripts/ci/helm_chart_release.py index 486a438a..aa801419 100644 --- a/scripts/ci/helm_chart_release.py +++ b/scripts/ci/helm_chart_release.py @@ -187,11 +187,18 @@ def validate_source(chart_path: Path = DEFAULT_CHART_PATH) -> dict[str, Any]: return metadata -def semver_key(version: str) -> tuple[int, int, int]: +def semver_key(version: str) -> tuple[int, int, int, int, int]: match = SEMVER_PATTERN.fullmatch(version) - if match is None or match.group(4): - raise ReleaseError(f"chart version must be a stable numeric SemVer: {version}") - return tuple(int(match.group(index)) for index in range(1, 4)) + if match is None: + raise ReleaseError(f"chart version must be an exact SemVer: {version}") + stage_order, number = 3, 0 + if match.group(4): + prerelease = re.fullmatch(r"(alpha|beta|rc)\.(0|[1-9][0-9]*)", match.group(4)) + if prerelease is None: + raise ReleaseError(f"unsupported chart prerelease: {version}") + stage_order = {"alpha": 0, "beta": 1, "rc": 2}[prerelease.group(1)] + number = int(prerelease.group(2)) + return (int(match.group(1)), int(match.group(2)), int(match.group(3)), stage_order, number) def git_output(arguments: list[str]) -> str: diff --git a/scripts/ci/sync-source-release.mjs b/scripts/ci/sync-source-release.mjs index 511c012d..13a9089a 100644 --- a/scripts/ci/sync-source-release.mjs +++ b/scripts/ci/sync-source-release.mjs @@ -10,7 +10,7 @@ const defaultRepositoryRoot = resolve(fileURLToPath(new URL('../..', import.meta const schema = 'durable-workflow.server.source-release/v1'; const prerelease = String.raw`(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)-(?:alpha|beta|rc)\.(?:0|[1-9]\d*)`; const stableVersion = String.raw`(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)`; -const serverVersion = String.raw`(?:${stableVersion}|${prerelease})`; +const serverVersion = String.raw`(?:${prerelease}|${stableVersion})`; const remediation = 'node scripts/ci/sync-source-release.mjs --write'; function fail(message) { @@ -52,7 +52,7 @@ export async function sourceRelease(repositoryRoot = defaultRepositoryRoot) { ), chartVersion: exact( object(record.helm_chart, 'source release helm_chart').version, - stableVersion, + serverVersion, 'source release helm_chart.version', ), }; @@ -265,7 +265,7 @@ const consumers = [ render(source, release) { let rendered = replaceExactly( source, - new RegExp(`^(version:\\s*)${stableVersion}\\s*$`, 'm'), + new RegExp(`^(version:\\s*)${serverVersion}\\s*$`, 'm'), (_match, prefix) => `${prefix}${release.chartVersion}`, 1, this.path, diff --git a/scripts/ci/test-source-release.mjs b/scripts/ci/test-source-release.mjs index b0aa4ca8..efb47a1a 100644 --- a/scripts/ci/test-source-release.mjs +++ b/scripts/ci/test-source-release.mjs @@ -99,7 +99,7 @@ test('a stable server source identity fans out through current release consumers const manifestPath = join(temporaryRoot, 'resources/release/source-release.json'); const manifest = JSON.parse(await readFile(manifestPath, 'utf8')); manifest.server.version = '2.0.0'; - manifest.helm_chart.version = nextPatch(manifest.helm_chart.version); + manifest.helm_chart.version = nextServerRelease(manifest.helm_chart.version); await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); const write = runGenerator(temporaryRoot, ['--write']); @@ -127,7 +127,7 @@ test('a simulated next release fans out without rewriting history or retaining s const previousServer = manifest.server.version; const previousChart = manifest.helm_chart.version; const nextServer = nextServerRelease(previousServer); - const nextChart = nextPatch(previousChart); + const nextChart = nextServerRelease(previousChart); manifest.server.version = nextServer; manifest.helm_chart.version = nextChart; await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); diff --git a/scripts/ci/test_helm_chart_release.py b/scripts/ci/test_helm_chart_release.py index c2d5928d..89c24885 100644 --- a/scripts/ci/test_helm_chart_release.py +++ b/scripts/ci/test_helm_chart_release.py @@ -448,6 +448,19 @@ def test_chart_qualification_uses_only_source_controlled_tool_versions( for caller in ("helm-chart-validation.yml", "helm-chart-release.yml"): self.assertIn('- "uv.toml"', workflow_source(caller)) + def test_chart_prerelease_order_preserves_stable_precedence(self) -> None: + versions = [ + "0.1.136", "0.1.137-alpha.1", "0.1.137-alpha.2", "0.1.137-beta.1", + "0.1.137-rc.1", "0.1.137-rc.2", "0.1.137", "0.1.138-rc.1", + ] + for earlier, later in zip(versions, versions[1:]): + with self.subTest(earlier=earlier, later=later): + self.assertLess(RELEASE.semver_key(earlier), RELEASE.semver_key(later)) + for invalid in ["latest", "0.1.137-rc.01", "0.1.137-preview.1"]: + with self.subTest(invalid=invalid): + with self.assertRaises(RELEASE.ReleaseError): + RELEASE.semver_key(invalid) + def test_current_source_has_synchronized_public_identity(self) -> None: metadata = RELEASE.validate_source() source_release = RELEASE.source_release_metadata() diff --git a/scripts/k8s-kind-smoke.sh b/scripts/k8s-kind-smoke.sh index b7a88bd2..30fc9394 100755 --- a/scripts/k8s-kind-smoke.sh +++ b/scripts/k8s-kind-smoke.sh @@ -7,7 +7,7 @@ cluster="${K8S_SMOKE_CLUSTER:-durable-workflow-server-smoke}" image="${K8S_SMOKE_IMAGE:-durableworkflow/server:k8s-smoke}" # Generated by scripts/ci/sync-source-release.mjs so the smoke replaces the # same default shipped by the public manifests. -manifest_image="durableworkflow/server:2.4.40" +manifest_image="durableworkflow/server:2.5.0-rc.1" kind_node_image="${K8S_SMOKE_KIND_NODE_IMAGE:-kindest/node:v1.29.4}" artifact_dir="${K8S_SMOKE_ARTIFACT_DIR:-/tmp/durable-workflow-k8s-kind-smoke-artifacts}" rendered_dir="${artifact_dir}/rendered-manifests" diff --git a/tests/Feature/ClusterInfoCompatibilityTest.php b/tests/Feature/ClusterInfoCompatibilityTest.php index 3bed9ef9..051d1f39 100644 --- a/tests/Feature/ClusterInfoCompatibilityTest.php +++ b/tests/Feature/ClusterInfoCompatibilityTest.php @@ -57,7 +57,7 @@ public function test_default_cluster_info_keeps_compatibility_preflight_bounded( ->assertJsonPath('client_compatibility.authority', 'protocol_manifests') ->assertJsonPath('capabilities.workflow_tasks', true) ->assertJsonPath('platform_protocol_specs.schema', PlatformProtocolSpecs::SCHEMA) - ->assertJsonPath('surface_stability_contract.version', 4) + ->assertJsonPath('surface_stability_contract.version', 5) ->assertJsonPath('activity_runtime_contract.version', 1) ->assertJsonPath('topology.schema', ServerTopology::SCHEMA) ->assertJsonMissingPath('worker_fleet') @@ -707,7 +707,7 @@ public function test_cluster_info_publishes_the_canonical_surface_stability_cont $response = $this->getJson('/api/cluster/info')->assertOk(); $response - ->assertJsonPath('surface_stability_contract.version', 4) + ->assertJsonPath('surface_stability_contract.version', 5) ->assertJsonPath( 'surface_stability_contract.surface_families.official_sdks.package_compatibility.rust_sdk.package', 'durable-workflow', @@ -1038,7 +1038,7 @@ public function test_cluster_info_names_protocol_manifests_as_client_compatibili ->assertJsonPath('client_compatibility.clients.cli.supported_versions', '>=2.0.0,<3.0.0') ->assertJsonPath('client_compatibility.clients.sdk-php.supported_versions', '>=2.0.0,<3.0.0') ->assertJsonPath('client_compatibility.clients.sdk-python.supported_versions', '>=2.0.0,<3.0.0') - ->assertJsonPath('client_compatibility.clients.sdk-rust.supported_versions', '>=2.0.0,<3.0.0'); + ->assertJsonPath('client_compatibility.clients.sdk-rust.supported_versions', '>=2.0.0,<4.0.0'); foreach ($response->json('supported_sdk_versions') as $supportedVersions) { $this->assertStringNotContainsString( diff --git a/tests/Feature/CooperativeCancellationProtocolTest.php b/tests/Feature/CooperativeCancellationProtocolTest.php index 98efa029..a762f18a 100644 --- a/tests/Feature/CooperativeCancellationProtocolTest.php +++ b/tests/Feature/CooperativeCancellationProtocolTest.php @@ -285,7 +285,7 @@ public function test_request_is_unavailable_until_the_server_protocol_supports_i ->assertJsonPath('reason', 'cooperative_cancellation_not_supported'); } - public function test_discovery_matches_request_support_and_preserves_the_default_protocol(): void + public function test_discovery_matches_request_support_and_defaults_to_the_cooperative_protocol(): void { foreach (['1.19' => false, '1.20' => true, '2.0' => false, 'malformed' => false] as $version => $supported) { config(['server.worker_protocol.version' => $version]); @@ -296,7 +296,7 @@ public function test_discovery_matches_request_support_and_preserves_the_default $this->assertSame($supported, CooperativeCancellationPolicy::serverSupported()); } - $this->assertSame('1.19', WorkerProtocol::VERSION); + $this->assertSame('1.20', WorkerProtocol::VERSION); } public function test_waiting_timer_is_interrupted_only_when_delivery_is_committed(): void From 94b62637daf4f008127e8351f392f111a775b622 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 06:31:52 +0000 Subject: [PATCH 56/57] Align protocol schema and fail closed on invalid task namespace bindings --- app/Http/Controllers/Api/WorkerController.php | 15 +++++++-- .../worker-protocol-api.openapi.yaml | 10 +++--- tests/Feature/WorkflowStreamsTest.php | 3 +- tests/Feature/WorkflowWorkerProtocolTest.php | 33 +++++++++++++++++++ 4 files changed, 52 insertions(+), 9 deletions(-) diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index 526a3974..15de2096 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -1948,6 +1948,9 @@ function () use ( ->lockForUpdate() ->first(); $claimedTask = NamespaceWorkflowScope::lockTaskForMutation((string) $namespace, $taskId); + if (! $claimedTask instanceof WorkflowTask) { + return WorkerProtocol::json(['reason' => 'task_not_found'], 404); + } if ($response = $this->guardWorkflowTaskOwnership( $request, @@ -3655,7 +3658,9 @@ public function heartbeatWorkflowTask(Request $request, string $taskId): JsonRes fn (): array|JsonResponse => DB::transaction(function () use ($request, $namespace, $taskId, $validated, $bridge): array|JsonResponse { // Ownership must remain valid until renewal commits. A // check before this lock can acknowledge a reclaimed lease. - NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId); + if (! NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId) instanceof WorkflowTask) { + return WorkerProtocol::json(['reason' => 'task_not_found'], 404); + } if ($response = $this->guardWorkflowTaskOwnership( $request, $namespace, @@ -3743,7 +3748,9 @@ public function failWorkflowTask(Request $request, string $taskId): JsonResponse try { $outcome = $this->storageMutations->run( fn (): array|JsonResponse => DB::transaction(function () use ($request, $namespace, $taskId, $validated): array|JsonResponse { - NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId); + if (! NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId) instanceof WorkflowTask) { + return WorkerProtocol::json(['reason' => 'task_not_found'], 404); + } if ($response = $this->guardWorkflowTaskOwnership($request, $namespace, $taskId, (int) $validated['workflow_task_attempt'], $validated['lease_owner'])) { return $response; @@ -3791,7 +3798,9 @@ public function failWorkflowTask(Request $request, string $taskId): JsonResponse $taskId, $validated, ): array|JsonResponse { - NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId); + if (! NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId) instanceof WorkflowTask) { + return WorkerProtocol::json(['reason' => 'task_not_found'], 404); + } if ($response = $this->guardWorkflowTaskOwnership($request, $namespace, $taskId, (int) $validated['workflow_task_attempt'], $validated['lease_owner'])) { return $response; diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 9daaaf58..63cbbbef 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1159,8 +1159,8 @@ components: in: header required: true description: > - Worker-selected protocol version. The default server advertises 1.19 - and accepts same-major versions whose minor is no newer than 19. + Worker-selected protocol version. The default server advertises 1.20 + and accepts same-major versions whose minor is no newer than 20. Missing, malformed, different-major, and ahead-of-server values fail closed. schema: { $ref: "#/components/schemas/AcceptedWorkerProtocolRequestVersion" } TaskIdPath: @@ -1341,14 +1341,14 @@ components: schemas: AcceptedWorkerProtocolRequestVersion: type: string - enum: ["1.0", "1.1", "1.2", "1.3", "1.4", "1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13", "1.14", "1.15", "1.16", "1.17", "1.18", "1.19"] + enum: ["1.0", "1.1", "1.2", "1.3", "1.4", "1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13", "1.14", "1.15", "1.16", "1.17", "1.18", "1.19", "1.20"] description: > Request versions accepted by a server using the default advertised - version 1.19. If deployment configuration changes the advertised + version 1.20. If deployment configuration changes the advertised version to 1.N, the accepted set is 1.0 through 1.N. AdvertisedWorkerProtocolVersion: type: string - const: "1.19" + const: "1.20" description: Default worker protocol version advertised by responses implementing the Durable Workflow 2.0 protocol contract. WorkerEnvelope: type: object diff --git a/tests/Feature/WorkflowStreamsTest.php b/tests/Feature/WorkflowStreamsTest.php index 0ec0f85f..3411be2e 100644 --- a/tests/Feature/WorkflowStreamsTest.php +++ b/tests/Feature/WorkflowStreamsTest.php @@ -8,6 +8,7 @@ use App\Models\WorkflowDurableStreamItem; use App\Models\WorkflowNamespace; use App\Support\RuntimeExternalPayloadRegistry; +use App\Support\WorkerProtocol; use App\Support\WorkflowStreamCommandProcessor; use App\Support\WorkflowStreamService; use Illuminate\Foundation\Testing\RefreshDatabase; @@ -586,7 +587,7 @@ public function test_terminal_completion_commits_its_last_stream_item_and_close( public function test_legacy_request_refuses_explicit_scope_before_stream_output(): void { $task = $this->claimStreamCompletionTask(); - $this->withHeaders($this->workerHeaders())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + $this->withHeaders(array_replace($this->workerHeaders(), [WorkerProtocol::HEADER => '1.19']))->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], 'commands' => [$this->streamCompletionCommand($task), ['type' => 'start_timer', 'delay_seconds' => 10, 'cancellation_scope_id' => 'root']], ])->assertConflict()->assertJsonPath('recorded', false)->assertJsonPath('reason', 'cancellation_scope_membership_unavailable'); diff --git a/tests/Feature/WorkflowWorkerProtocolTest.php b/tests/Feature/WorkflowWorkerProtocolTest.php index 101f7286..d21718a9 100644 --- a/tests/Feature/WorkflowWorkerProtocolTest.php +++ b/tests/Feature/WorkflowWorkerProtocolTest.php @@ -53,6 +53,39 @@ protected function setUp(): void ]); } + public function test_worker_mutations_refuse_a_task_whose_run_is_outside_its_namespace(): void + { + Queue::fake(); + $this->configureWorkflowTypes(); + $this->createNamespace('default', 'Default'); + $this->createNamespace('other', 'Other'); + $runId = $this->postJson('/api/workflows', [ + 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'namespace-lock', 'input' => ['Ada'], + ], $this->apiHeaders())->assertCreated()->json('run_id'); + $this->registerWorker('namespace-lock-worker', 'namespace-lock'); + $claim = $this->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'namespace-lock-worker', 'task_queue' => 'namespace-lock', + ], $this->workerHeaders())->assertOk()->json('task'); + $this->assertIsArray($claim); + WorkflowRun::query()->whereKey($runId)->update(['namespace' => 'other']); + $taskBefore = WorkflowTask::query()->findOrFail($claim['task_id'])->getRawOriginal(); + $historyBefore = WorkflowHistoryEvent::query()->where('workflow_run_id', $runId)->count(); + + foreach (['heartbeat', 'complete', 'fail'] as $operation) { + $body = ['lease_owner' => $claim['lease_owner'], 'workflow_task_attempt' => $claim['workflow_task_attempt']]; + if ($operation === 'complete') { + $body['commands'] = [['type' => 'complete_workflow', 'result' => Serializer::serializeWithCodec('avro', 'refuse')]]; + } elseif ($operation === 'fail') { + $body['failure'] = ['message' => 'Refuse cross-namespace mutation']; + } + $this->postJson('/api/worker/workflow-tasks/'.$claim['task_id'].'/'.$operation, $body, $this->workerHeaders()) + ->assertNotFound()->assertJsonPath('reason', 'task_not_found'); + } + $this->assertSame($taskBefore, WorkflowTask::query()->findOrFail($claim['task_id'])->getRawOriginal()); + $this->assertSame($historyBefore, WorkflowHistoryEvent::query()->where('workflow_run_id', $runId)->count()); + $this->assertSame('pending', WorkflowRun::query()->findOrFail($runId)->status->value); + } + public function test_it_starts_workflows_and_completes_workflow_tasks_through_the_external_worker_protocol(): void { Queue::fake(); From deb8be3a1e7b98e38a12ec5579850bdd1ef442e5 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 07:06:15 +0000 Subject: [PATCH 57/57] Consume published scheduler correction and preserve worker refusal diagnostics --- app/Http/Controllers/Api/WorkerController.php | 25 +++++++++++-------- composer.json | 2 +- composer.lock | 16 ++++++------ scripts/perf/mysql-lock-diagnostics.sql | 4 ++- scripts/perf/run-server-soak.sh | 4 +-- tests/Feature/WorkflowWorkerProtocolTest.php | 5 +++- 6 files changed, 33 insertions(+), 23 deletions(-) diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index 15de2096..f9c7111c 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -1949,7 +1949,7 @@ function () use ( ->first(); $claimedTask = NamespaceWorkflowScope::lockTaskForMutation((string) $namespace, $taskId); if (! $claimedTask instanceof WorkflowTask) { - return WorkerProtocol::json(['reason' => 'task_not_found'], 404); + return $this->workflowTaskNotFound($taskId, (int) $validated['workflow_task_attempt']); } if ($response = $this->guardWorkflowTaskOwnership( @@ -3659,7 +3659,7 @@ public function heartbeatWorkflowTask(Request $request, string $taskId): JsonRes // Ownership must remain valid until renewal commits. A // check before this lock can acknowledge a reclaimed lease. if (! NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId) instanceof WorkflowTask) { - return WorkerProtocol::json(['reason' => 'task_not_found'], 404); + return $this->workflowTaskNotFound($taskId, (int) $validated['workflow_task_attempt']); } if ($response = $this->guardWorkflowTaskOwnership( $request, @@ -3749,7 +3749,7 @@ public function failWorkflowTask(Request $request, string $taskId): JsonResponse $outcome = $this->storageMutations->run( fn (): array|JsonResponse => DB::transaction(function () use ($request, $namespace, $taskId, $validated): array|JsonResponse { if (! NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId) instanceof WorkflowTask) { - return WorkerProtocol::json(['reason' => 'task_not_found'], 404); + return $this->workflowTaskNotFound($taskId, (int) $validated['workflow_task_attempt']); } if ($response = $this->guardWorkflowTaskOwnership($request, $namespace, $taskId, (int) $validated['workflow_task_attempt'], $validated['lease_owner'])) { @@ -3799,7 +3799,7 @@ public function failWorkflowTask(Request $request, string $taskId): JsonResponse $validated, ): array|JsonResponse { if (! NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId) instanceof WorkflowTask) { - return WorkerProtocol::json(['reason' => 'task_not_found'], 404); + return $this->workflowTaskNotFound($taskId, (int) $validated['workflow_task_attempt']); } if ($response = $this->guardWorkflowTaskOwnership($request, $namespace, $taskId, (int) $validated['workflow_task_attempt'], $validated['lease_owner'])) { @@ -4599,6 +4599,16 @@ private static function decodeHistoryPageToken(?string $token): ?int return WorkflowHistoryPageToken::decode($token); } + private function workflowTaskNotFound(string $taskId, int $workflowTaskAttempt): JsonResponse + { + return WorkerProtocol::json([ + 'task_id' => $taskId, + 'workflow_task_attempt' => $workflowTaskAttempt, + 'error' => 'Workflow task not found.', + 'reason' => 'task_not_found', + ], 404); + } + /** * Guard workflow task ownership and lease validity. * @@ -4659,12 +4669,7 @@ private function guardWorkflowTaskOwnership( // Convert package-level outcomes to HTTP responses return match ($result['reason']) { - 'task_not_found' => WorkerProtocol::json([ - 'task_id' => $taskId, - 'workflow_task_attempt' => $workflowTaskAttempt, - 'error' => 'Workflow task not found.', - 'reason' => 'task_not_found', - ], 404), + 'task_not_found' => $this->workflowTaskNotFound($taskId, $workflowTaskAttempt), 'task_not_leased' => WorkerProtocol::json([ 'task_id' => $taskId, diff --git a/composer.json b/composer.json index 6fe92567..1b80a60a 100644 --- a/composer.json +++ b/composer.json @@ -6,7 +6,7 @@ "require": { "php": "^8.2", "apache/avro": "^1.12", - "durable-workflow/workflow": "2.4.0-rc.1", + "durable-workflow/workflow": "2.4.0-rc.2", "laravel/framework": "^13.30", "laravel/tinker": "^3.0", "league/flysystem-aws-s3-v3": "^3.35.3" diff --git a/composer.lock b/composer.lock index dd0dd5f3..fe613a14 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "fe2189c084f56d517dbc310e3212b474", + "content-hash": "1305e1110d7cbb355934e8ce8cd29c8a", "packages": [ { "name": "apache/avro", @@ -655,16 +655,16 @@ }, { "name": "durable-workflow/workflow", - "version": "2.4.0-rc.1", + "version": "2.4.0-rc.2", "source": { "type": "git", "url": "https://github.com/durable-workflow/workflow.git", - "reference": "dd23e5e0632165a23b9d1e6a8b67dbdb798d6213" + "reference": "200a3c0ffa1e90557619221458deb3516b8cf955" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/durable-workflow/workflow/zipball/dd23e5e0632165a23b9d1e6a8b67dbdb798d6213", - "reference": "dd23e5e0632165a23b9d1e6a8b67dbdb798d6213", + "url": "https://api.github.com/repos/durable-workflow/workflow/zipball/200a3c0ffa1e90557619221458deb3516b8cf955", + "reference": "200a3c0ffa1e90557619221458deb3516b8cf955", "shasum": "" }, "require": { @@ -697,7 +697,7 @@ "dev-main": "2.0.x-dev" }, "durable-workflow": { - "product-train": "2.4.0-rc.1", + "product-train": "2.4.0-rc.2", "laravel-embedded-upgrade-contract": "resources/laravel-embedded-upgrade-contract.json", "laravel-dependency-security-policy": "resources/laravel-dependency-security-policy.json" } @@ -724,9 +724,9 @@ "description": "Embedded durable workflow runtime and orchestration engine for Laravel applications.", "support": { "issues": "https://github.com/durable-workflow/workflow/issues", - "source": "https://github.com/durable-workflow/workflow/tree/2.4.0-rc.1" + "source": "https://github.com/durable-workflow/workflow/tree/2.4.0-rc.2" }, - "time": "2026-10-06T05:57:49+00:00" + "time": "2026-10-06T07:03:04+00:00" }, { "name": "egulias/email-validator", diff --git a/scripts/perf/mysql-lock-diagnostics.sql b/scripts/perf/mysql-lock-diagnostics.sql index 3797bdba..3abf676d 100644 --- a/scripts/perf/mysql-lock-diagnostics.sql +++ b/scripts/perf/mysql-lock-diagnostics.sql @@ -1,4 +1,6 @@ -SELECT NOW(6) AS captured_at, @@innodb_print_all_deadlocks AS deadlock_logging; +SELECT NOW(6) AS captured_at, @@innodb_print_all_deadlocks AS deadlock_logging, + @@log_error_verbosity AS error_log_verbosity; +SHOW GLOBAL STATUS LIKE 'Innodb_deadlocks'; SHOW ENGINE INNODB STATUS; SELECT THREAD_ID, EVENT_ID, CURRENT_SCHEMA, SQL_TEXT, MYSQL_ERRNO, RETURNED_SQLSTATE, MESSAGE_TEXT, TIMER_WAIT, LOCK_TIME, ROWS_AFFECTED diff --git a/scripts/perf/run-server-soak.sh b/scripts/perf/run-server-soak.sh index db99e913..f188037d 100755 --- a/scripts/perf/run-server-soak.sh +++ b/scripts/perf/run-server-soak.sh @@ -324,7 +324,7 @@ cleanup() { if [[ "$MYSQL_LOCK_DIAGNOSTICS" == 1 ]]; then timeout 15s docker exec -i -e MYSQL_PWD=root "${PROJECT}-mysql-1" \ - mysql --user=root --batch \ + mysql --user=root --batch --raw \ < "$ROOT_DIR/scripts/perf/mysql-lock-diagnostics.sql" \ > "$ARTIFACT_DIR/mysql-lock-diagnostics.txt" 2>&1 || true fi @@ -509,7 +509,7 @@ fi if [[ "$MYSQL_LOCK_DIAGNOSTICS" == 1 ]]; then timeout 15s docker exec -e MYSQL_PWD=root "${PROJECT}-mysql-1" \ - mysql --user=root --execute="SET GLOBAL innodb_print_all_deadlocks=ON; UPDATE performance_schema.setup_consumers SET ENABLED='YES' WHERE NAME='events_statements_history_long';" \ + mysql --user=root --execute="SET GLOBAL log_error_verbosity=3; SET GLOBAL innodb_print_all_deadlocks=ON; UPDATE performance_schema.setup_consumers SET ENABLED='YES' WHERE NAME='events_statements_history_long';" \ > "$ARTIFACT_DIR/mysql-lock-diagnostics-setup.txt" 2>&1 fi diff --git a/tests/Feature/WorkflowWorkerProtocolTest.php b/tests/Feature/WorkflowWorkerProtocolTest.php index d21718a9..42512931 100644 --- a/tests/Feature/WorkflowWorkerProtocolTest.php +++ b/tests/Feature/WorkflowWorkerProtocolTest.php @@ -79,7 +79,10 @@ public function test_worker_mutations_refuse_a_task_whose_run_is_outside_its_nam $body['failure'] = ['message' => 'Refuse cross-namespace mutation']; } $this->postJson('/api/worker/workflow-tasks/'.$claim['task_id'].'/'.$operation, $body, $this->workerHeaders()) - ->assertNotFound()->assertJsonPath('reason', 'task_not_found'); + ->assertNotFound()->assertJsonPath('reason', 'task_not_found') + ->assertJsonPath('error', 'Workflow task not found.') + ->assertJsonPath('task_id', $claim['task_id']) + ->assertJsonPath('workflow_task_attempt', $claim['workflow_task_attempt']); } $this->assertSame($taskBefore, WorkflowTask::query()->findOrFail($claim['task_id'])->getRawOriginal()); $this->assertSame($historyBefore, WorkflowHistoryEvent::query()->where('workflow_run_id', $runId)->count());