diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index ad15f156..66ef02c7 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -11,6 +11,10 @@ on: description: Full Git commit SHA to classify regression-corpus changes against required: true type: string + prepared_local_workflow_commit: + description: Optional full Native commit for candidate prepared-local source qualification. Does not replace locked-package CI. + required: false + type: string permissions: contents: read @@ -183,7 +187,7 @@ jobs: --health-cmd='mysqladmin --silent --host=127.0.0.1 ping' \ --health-interval=1s --health-start-period=20s --health-retries=30 \ mysql@sha256:679e7e924f38a3cbb62a3d7df32924b83f7321a602d3f9f967c01b3df18495d6 \ - --disable-log-bin --performance-schema=OFF \ + --disable-log-bin --performance-schema=ON \ --innodb-buffer-pool-size=64M --innodb-redo-log-capacity=64M \ --innodb-log-buffer-size=8M --table-open-cache=256 --max-connections=40 docker run --detach \ @@ -253,3 +257,103 @@ jobs: test "$PREFLIGHT_RESULT" = success test "$FEATURE_RESULT" = success echo "Server repository contracts and feature tests succeeded." + + prepared-local-source: + name: Candidate cancellation and prepared-local source qualification + needs: [action-policy, preflight] + if: ${{ github.event_name == 'workflow_dispatch' && inputs.prepared_local_workflow_commit != '' }} + runs-on: ubuntu-latest + timeout-minutes: 30 + services: + mysql: + image: mysql@sha256:679e7e924f38a3cbb62a3d7df32924b83f7321a602d3f9f967c01b3df18495d6 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + MYSQL_DATABASE: prepared_local_test + MYSQL_ROOT_HOST: "%" + ports: ["3317:3306"] + options: >- + --health-cmd="mysqladmin --silent --host=127.0.0.1 ping" + --health-interval=2s --health-timeout=5s --health-retries=30 + postgres: + image: postgres@sha256:f1c3376c26f2609ab9f29f71f824103fe2fcd8ee0346485cb6122a4f93df6f94 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: prepared_local_test + ports: ["5439:5432"] + options: >- + --health-cmd="pg_isready --host=127.0.0.1 --username=postgres" + --health-interval=2s --health-timeout=5s --health-retries=30 + env: + DW_PREPARED_WORKFLOW_COMMIT: ${{ inputs.prepared_local_workflow_commit }} + steps: + - name: Require immutable candidate source + run: | + set -euo pipefail + [[ "$DW_PREPARED_WORKFLOW_COMMIT" =~ ^[0-9a-f]{40}$ ]] + [[ ! "$DW_PREPARED_WORKFLOW_COMMIT" =~ ^0+$ ]] + - name: Check out Server candidate + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Check out exact Native candidate + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: durable-workflow/workflow + ref: ${{ inputs.prepared_local_workflow_commit }} + path: prepared-workflow-source + persist-credentials: false + - name: Record source binding and qualify all supported databases + run: | + set -euo pipefail + test "$(git -C prepared-workflow-source rev-parse HEAD)" = "$DW_PREPARED_WORKFLOW_COMMIT" + mkdir -p prepared-local-evidence + chmod 0777 prepared-local-evidence + { + echo "Candidate source qualification only. No image, release or SDK conformance claim." + echo "Server commit: $(git rev-parse HEAD)" + echo "Native source overlay: $DW_PREPARED_WORKFLOW_COMMIT" + echo "Locked package baseline retained by ordinary feature CI." + } > prepared-local-evidence/provenance.txt + docker build --target base -t dw-server-prepared-local-base . + tar --exclude=vendor --exclude='*/vendor' --exclude=build --exclude='*/build' \ + --exclude=prepared-local-evidence -cf ../prepared-local-source.tar . + qualification_status=0 + for database in sqlite mysql pgsql; do + database_name=prepared_local_test + database_user=root + database_port=3317 + if [ "$database" = sqlite ]; then database_name=:memory:; fi + if [ "$database" = pgsql ]; then database_user=postgres; database_port=5439; fi + if ! docker run --rm --init --network host \ + --user 1000:1000 \ + --tmpfs /app:rw,uid=1000,gid=1000,mode=0755 \ + --volume "$PWD/../prepared-local-source.tar:/source.tar:ro" \ + --volume "$PWD/prepared-local-evidence:/evidence" \ + --env COMPOSER_HOME=/tmp/dw-prepared-composer \ + --env DB_CONNECTION="$database" --env DB_DATABASE="$database_name" \ + --env DB_HOST=127.0.0.1 --env DB_PORT="$database_port" \ + --env DB_USERNAME="$database_user" --env DB_PASSWORD= \ + --env DW_PREPARED_DATABASE="$database" \ + --workdir /app --entrypoint sh dw-server-prepared-local-base -ec ' + tar --no-same-owner -xf /source.tar -C /app + composer install --no-interaction --no-progress --prefer-dist + mv vendor/durable-workflow/workflow /tmp/locked-workflow-package + cp -a prepared-workflow-source vendor/durable-workflow/workflow + php vendor/bin/phpunit tests/Feature/PreparedLocalActivityProtocolTest.php tests/Feature/ActivityAttemptStatusTest.php tests/Feature/HistoryRetentionTest.php tests/Unit/WorkerPollFenceTest.php tests/Source/CancellationCascadeDiagnosticsTest.php tests/Source/PreparedLocalCancellationPolicyTest.php tests/Source/CancellationScopeAdmissionTest.php tests/Source/ScopedActivityCancellationReceiptTest.php tests/Source/CancellationScopeDeliveryTest.php tests/Feature/WorkflowStreamsTest.php tests/Feature/WorkflowDebugTest.php --fail-on-skipped --log-junit "/evidence/$DW_PREPARED_DATABASE.xml" --colors=never + if [ "$DW_PREPARED_DATABASE" = sqlite ]; then + php vendor/bin/phpunit tests/Feature/CooperativeCancellationProtocolTest.php tests/Feature/WorkflowWorkerProtocolTest.php tests/Feature/ActivityWorkerProtocolTest.php tests/Feature/SearchAttributeValueValidationTest.php tests/Feature/NamespaceDurableStateQuotaTest.php tests/Feature/RuntimeExternalPayloadTransportTest.php tests/Feature/RuntimePayloadCompletionUploadsTest.php tests/Unit/RuntimePayloadCompletionContextTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php --log-junit /evidence/affected-regression.xml --colors=never + fi + ' > "prepared-local-evidence/$database.log" 2>&1; then + qualification_status=1 + fi + done + test "$qualification_status" = 0 + - name: Retain exact source qualification evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: candidate-prepared-local-source + path: prepared-local-evidence + if-no-files-found: error + retention-days: 90 diff --git a/.github/workflows/server-perf.yml b/.github/workflows/server-perf.yml index 0714b93a..42e0048e 100644 --- a/.github/workflows/server-perf.yml +++ b/.github/workflows/server-perf.yml @@ -258,6 +258,7 @@ jobs: DW_PERF_MAX_HEALTH_LATENCY_SECONDS: "3" DW_PERF_READINESS_DIAGNOSTICS: "1" DW_PERF_CONTROL_PLANE_INTERVAL_SECONDS: "5" + DW_PERF_MYSQL_LOCK_DIAGNOSTICS: "1" DW_PERF_MAX_CONTROL_PLANE_LATENCY_SECONDS: "5" DW_PERF_WORKFLOW_VERSION: ${{ steps.workflow.outputs.ref }} DW_PERF_MAX_SERVER_MEMORY_MB: "768" diff --git a/app/Http/Controllers/Api/ActivityTaskController.php b/app/Http/Controllers/Api/ActivityTaskController.php index ae4ca4f0..0fbe4cde 100644 --- a/app/Http/Controllers/Api/ActivityTaskController.php +++ b/app/Http/Controllers/Api/ActivityTaskController.php @@ -8,6 +8,7 @@ use App\Support\ActivityTaskPoller; use App\Support\AvroPayloadEnvelopeResolver; use App\Support\BackendLockPressure; +use App\Support\CooperativeCancellationPolicy; use App\Support\ExternalExecutorConfigContract; use App\Support\ExternalPayloadEnvelopeService; use App\Support\ExternalPayloadStorageUnavailable; @@ -22,15 +23,25 @@ use App\Support\WorkerProtocol; use App\Support\WorkerProtocolMutationRetrier; use App\Support\WorkerSessionRegistry; +use Carbon\CarbonImmutable; use Carbon\CarbonInterface; use Illuminate\Http\JsonResponse; use Illuminate\Http\Request; +use Illuminate\Support\Facades\DB; use Illuminate\Validation\Rule; use Illuminate\Validation\ValidationException; use InvalidArgumentException; use Workflow\V2\Contracts\ActivityTaskBridge as ActivityTaskBridgeContract; +use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Exceptions\ExternalPayloadIntegrityException; use Workflow\V2\Models\ActivityExecution; +use Workflow\V2\Models\WorkflowHistoryEvent; +use Workflow\V2\Models\WorkflowRun; +use Workflow\V2\Support\ActivityCancellationAcknowledgement; +use Workflow\V2\Support\ActivityCancellationCompletion; +use Workflow\V2\Support\ActivityCancellationContext; +use Workflow\V2\Support\ActivityRowLockOrder; +use Workflow\V2\Support\CooperativeCancellationDelivery; use Workflow\V2\Support\WorkerProtocolVersion; class ActivityTaskController @@ -119,6 +130,7 @@ public function poll(Request $request): JsonResponse $request, ), timeoutSeconds: $timeoutSeconds, + protocolVersion: WorkerProtocol::requestVersion($request), ); } catch (\Throwable $exception) { if ($exception instanceof InvalidArgumentException @@ -414,6 +426,262 @@ public function fail(Request $request, string $taskId): JsonResponse ], $stopStatus), $this->outcomeStatus($outcome['reason'])); } + /** Observe an owned attempt without renewing its lease or recording progress. */ + public function status(Request $request, string $taskId): JsonResponse + { + if ($response = WorkerProtocol::rejectUnsupported($request)) { + return $response; + } + + if (! CooperativeCancellationPolicy::serverSupported() + || ! WorkerProtocol::versionMeetsMinimum(WorkerProtocol::requestVersion($request), '1.20')) { + return WorkerProtocol::json([ + 'error' => 'Activity attempt observation requires worker protocol 1.20.', + 'reason' => 'activity_attempt_status_unavailable', + 'minimum_protocol_version' => '1.20', + ], 422); + } + + $validated = $request->validate([ + 'activity_attempt_id' => ['required', 'string'], + 'lease_owner' => ['required', 'string'], + ]); + $status = null; + try { + if ($response = $this->guardAttemptOwnership( + $request->attributes->get('namespace'), + $taskId, + $validated['activity_attempt_id'], + $validated['lease_owner'], + $status, + )) { + return $response; + } + $deadlines = $this->executionDeadlines($status['activity_execution_id'] ?? null) ?? []; + $workerSession = $this->workerSessions->workerSessionForExecution($status['activity_execution_id'] ?? null); + $cancellation = $status['cancel_requested'] === true + ? $this->activityCancellationReceipt( + (string) $request->attributes->get('namespace'), + $taskId, + $validated['activity_attempt_id'], + $validated['lease_owner'], + $status['activity_execution_id'] ?? null, + ) + : null; + } catch (\Throwable $exception) { + if (! BackendLockPressure::is($exception)) { + throw $exception; + } + + return BackendLockPressure::workerOperationResponse($request, false); + } + + if ($status['can_continue'] === true) { + $expiresAt = $status['lease_expires_at'] ?? null; + if (! is_string($expiresAt) || ! CarbonImmutable::parse($expiresAt)->isAfter(now())) { + $status['can_continue'] = false; + $status['reason'] = 'lease_expired'; + } + foreach (['heartbeat', 'start_to_close', 'schedule_to_close'] as $kind) { + if ($status['can_continue'] === true && isset($deadlines[$kind]) + && ! CarbonImmutable::parse($deadlines[$kind])->isAfter(now())) { + $status['can_continue'] = false; + $status['reason'] = $kind.'_timeout'; + } + } + if ($status['can_continue'] === true && $workerSession !== null) { + $sessionLease = $workerSession['lease_expires_at'] ?? null; + $sessionTtl = $workerSession['ttl_expires_at'] ?? null; + if (($workerSession['status'] ?? null) !== 'active' + || ($workerSession['lease_owner'] ?? null) !== $validated['lease_owner'] + || ! is_string($sessionLease) || ! CarbonImmutable::parse($sessionLease)->isAfter(now()) + || ! is_string($sessionTtl) || ! CarbonImmutable::parse($sessionTtl)->isAfter(now())) { + $status['can_continue'] = false; + $status['reason'] = 'worker_session_unavailable'; + } + } + } + + return WorkerProtocol::json([ + 'task_id' => $taskId, + 'activity_attempt_id' => $validated['activity_attempt_id'], + 'lease_owner' => $status['lease_owner'], + 'can_continue' => $status['can_continue'], + 'cancel_requested' => $status['cancel_requested'], + 'reason' => $status['reason'], + 'heartbeat_recorded' => false, + 'lease_expires_at' => $status['lease_expires_at'], + 'last_heartbeat_at' => $status['last_heartbeat_at'], + 'run_status' => $status['run_status'], + 'run_closed_reason' => $status['run_closed_reason'] ?? null, + 'run_closed_at' => $status['run_closed_at'] ?? null, + 'activity_status' => $status['activity_status'], + 'attempt_status' => $status['attempt_status'], + 'task_status' => $status['task_status'], + 'deadlines' => $deadlines === [] ? null : $deadlines, + 'worker_session' => $workerSession, + ...($cancellation === null ? [] : ['cancellation_acknowledgement' => $cancellation]), + ]); + } + + /** Record an original owner's report after its remote callback has stopped and joined. */ + public function acknowledgeCancellation(Request $request, string $taskId): JsonResponse + { + if ($response = WorkerProtocol::rejectUnsupported($request)) { + return $response; + } + if (! CooperativeCancellationPolicy::serverSupported() + || ! WorkerProtocol::versionMeetsMinimum(WorkerProtocol::requestVersion($request), '1.20')) { + return WorkerProtocol::json([ + 'acknowledged' => false, + 'reason' => 'activity_cancellation_acknowledgement_not_supported', + 'minimum_protocol_version' => '1.20', + 'unavailable' => ['worker_protocol'], + ], 422); + } + $validated = $request->validate([ + 'activity_attempt_id' => ['required', 'string', 'max:255'], + 'lease_owner' => ['required', 'string', 'max:255'], + 'request_id' => ['required', 'string', 'max:255'], + ]); + $namespace = (string) $request->attributes->get('namespace'); + try { + if ($response = $this->guardAttemptOwnership( + $namespace, $taskId, $validated['activity_attempt_id'], $validated['lease_owner'], + )) { + return $response; + } + if (! CooperativeCancellationPolicy::activityAcknowledgementBackendSupported()) { + return $this->activityAcknowledgementUnavailable($validated['lease_owner'], ['installed_runtime_activity_acknowledgement']); + } + + return $this->storageMutations->run(fn (): JsonResponse => DB::transaction(function () use ($namespace, $taskId, $validated): JsonResponse { + // Acquire the package's activity lock order before checking the + // immutable claim and calling the nested package transaction. + $rows = ActivityRowLockOrder::lockForAttempt($validated['activity_attempt_id']); + $attempt = $rows['attempt']; + if ($attempt === null || $attempt->workflow_task_id !== $taskId) { + return WorkerProtocol::json(['acknowledged' => false, 'reason' => 'activity_attempt_not_found'], 404); + } + $run = WorkflowRun::query()->where('namespace', $namespace)->lockForUpdate()->find($attempt->workflow_run_id); + $task = NamespaceWorkflowScope::taskQuery($namespace)->lockForUpdate()->find($taskId); + if ($run === null || $task === null || $task->workflow_run_id !== $run->id) { + return WorkerProtocol::json(['acknowledged' => false, 'reason' => 'task_not_found'], 404); + } + if (! CooperativeCancellationPolicy::claimSupportsCancellation($task)) { + return $this->activityAcknowledgementUnavailable($validated['lease_owner'], ['worker_claim_capability']); + } + $outcome = ActivityCancellationAcknowledgement::recordStopped( + $validated['activity_attempt_id'], $validated['lease_owner'], $validated['request_id'], + ); + + return WorkerProtocol::json([ + 'task_id' => $taskId, + 'activity_attempt_id' => $validated['activity_attempt_id'], + 'lease_owner' => $validated['lease_owner'], + 'request_id' => $validated['request_id'], + ...$outcome, + 'heartbeat_recorded' => false, + ], $outcome['acknowledged'] ? 200 : ($outcome['reason'] === 'activity_attempt_not_found' ? 404 : 409)); + })); + } catch (\Throwable $exception) { + if (! BackendLockPressure::is($exception)) { + throw $exception; + } + + return BackendLockPressure::workerOperationResponse($request, false); + } + } + + /** @param list $unavailable */ + private function activityAcknowledgementUnavailable(string $workerId, array $unavailable): JsonResponse + { + return WorkerProtocol::json([ + 'acknowledged' => false, + 'reason' => 'activity_cancellation_acknowledgement_not_supported', + 'required_capability' => CooperativeCancellationPolicy::CAPABILITY, + 'minimum_protocol_version' => '1.20', + 'worker_id' => $workerId, + 'unavailable' => $unavailable, + 'remediation' => 'Use a runtime with activity stop acknowledgements and an original claim from a cooperative worker on protocol 1.20 or newer.', + ], 409); + } + + /** @return array|null */ + private function activityCancellationReceipt( + string $namespace, string $taskId, string $attemptId, string $leaseOwner, ?string $executionId, + ): ?array { + if (! CooperativeCancellationPolicy::activityAcknowledgementBackendSupported() || $executionId === null) { + return null; + } + $execution = ActivityExecution::query()->find($executionId); + if ($execution === null || $execution->current_attempt_id !== $attemptId) { + return null; + } + $run = WorkflowRun::query()->where('namespace', $namespace)->find($execution->workflow_run_id); + if ($run === null) { + return null; + } + // Observe this execution's original fence, even when no whole-run + // request exists or a later request owns the run's cleanup. Keep this + // frequent worker observation bounded to the relevant cancellation facts. + $events = $run->historyEvents()->where(function ($query) use ($run, $executionId): void { + $query->where(function ($query) use ($executionId): void { + $query->where('payload->activity_execution_id', $executionId) + ->whereIn('event_type', [HistoryEventType::ActivityCancelled->value, HistoryEventType::ActivityCancellationAcknowledged->value]); + })->orWhere(function ($query) use ($run): void { + $query->where('event_type', HistoryEventType::CooperativeCancellationRequested->value) + ->where('workflow_command_id', $run->cancellation_request_command_id); + }); + }) + ->get(); + $run->setRelation('historyEvents', $events); + $cancelled = $events->first(fn (WorkflowHistoryEvent $event): bool => $event->event_type === HistoryEventType::ActivityCancelled); + $snapshot = $cancelled?->payload['activity_attempt'] ?? null; + if (! is_array($snapshot) || ($cancelled->payload['activity_attempt_id'] ?? null) !== $attemptId + || ($snapshot['id'] ?? null) !== $attemptId + || ($snapshot['task_id'] ?? null) !== $taskId || ($snapshot['lease_owner'] ?? null) !== $leaseOwner + || ($snapshot['activity_execution_id'] ?? null) !== $executionId || ($snapshot['status'] ?? null) !== 'cancelled') { + return null; + } + $hasCanonicalContext = class_exists(ActivityCancellationContext::class) + && method_exists(ActivityCancellationContext::class, 'forEvent') + && method_exists(ActivityCancellationContext::class, 'rootRequestId'); + $context = $hasCanonicalContext + ? ActivityCancellationContext::forEvent($run, $cancelled) + : (array_key_exists('cancellation_scope', $cancelled->payload) ? null : CooperativeCancellationDelivery::context($run)); + if ($context === null || $cancelled->workflow_command_id !== $context->requestId) { + return null; + } + $rootRequestId = $hasCanonicalContext ? ActivityCancellationContext::rootRequestId($context) : $context->rootRequestId; + $ack = class_exists(ActivityCancellationCompletion::class) && method_exists(ActivityCancellationCompletion::class, 'stopReceipt') + ? ActivityCancellationCompletion::stopReceipt($run, $cancelled) + : $events->first(fn (WorkflowHistoryEvent $event): bool => $event->event_type === HistoryEventType::ActivityCancellationAcknowledged + && $event->sequence > $cancelled->sequence + && $event->workflow_command_id === $context->requestId + && ($event->payload['sequence'] ?? null) === ($cancelled->payload['sequence'] ?? null) + && ($event->payload['activity_execution_id'] ?? null) === $executionId + && ($event->payload['activity_attempt_id'] ?? null) === $attemptId + && ($event->payload['lease_owner'] ?? null) === $leaseOwner + && ($event->payload['cancellation_history_event_id'] ?? null) === $cancelled->id + && ($event->payload['request_id'] ?? null) === $context->requestId + && ($event->payload['root_request_id'] ?? null) === $rootRequestId + && ($event->payload['cleanup_deadline_at'] ?? null) === $context->deadline()->toISOString() + && ($event->payload['callback_state'] ?? null) === 'stopped' + && ($event->payload['evidence_source'] ?? null) === 'activity_worker'); + + return [ + 'request_id' => $context->requestId, + 'root_request_id' => $rootRequestId, + 'cleanup_deadline_at' => $context->deadline()->toISOString(), + 'cancellation_history_event_id' => $cancelled->id, + 'callback_state' => $ack === null ? 'unknown' : 'stopped', + 'history_event_id' => $ack?->id, + 'acknowledged_at' => $ack?->payload['acknowledged_at'] ?? null, + 'received_after_deadline' => $ack?->payload['received_after_deadline'] ?? null, + ]; + } + /** * Heartbeat an in-progress activity task. * @@ -686,6 +954,7 @@ private function guardAttemptOwnership( string $taskId, string $attemptId, string $leaseOwner, + ?array &$status = null, ): ?JsonResponse { $task = NamespaceWorkflowScope::task($namespace, $taskId); diff --git a/app/Http/Controllers/Api/CancellationScopeController.php b/app/Http/Controllers/Api/CancellationScopeController.php new file mode 100644 index 00000000..14d9e6b8 --- /dev/null +++ b/app/Http/Controllers/Api/CancellationScopeController.php @@ -0,0 +1,186 @@ +delivery($request, $taskId, true); + } + + public function deliver(Request $request, string $taskId): JsonResponse + { + return $this->delivery($request, $taskId, false); + } + + private function delivery(Request $request, string $taskId, bool $preparing): JsonResponse + { + if ($response = WorkerProtocol::rejectUnsupported($request)) { + return $response; + } + $version = WorkerProtocol::requestVersion($request); + $refused = static fn (string $reason): array => ['prepared' => false, 'delivered' => false, + 'claim_released' => false, 'task_id' => $taskId, 'created_task_ids' => [], 'reason' => $reason]; + if (! WorkerProtocol::versionMeetsMinimum($version, CooperativeCancellationPolicy::MINIMUM_PROTOCOL_VERSION)) { + return WorkerProtocol::json($refused('cancellation_scope_requires_protocol_1_20'), 409); + } + $bridge = app(WorkflowTaskBridge::class); + if (! $bridge instanceof PreparedCancellationScopeTaskBridge) { + return WorkerProtocol::json($refused('cancellation_scope_delivery_unavailable') + + ['unavailable' => ['installed_runtime_scope_preparation_delivery']], 409); + } + $validated = $request->validate([ + 'lease_owner' => ['required', 'string', 'max:255'], + 'workflow_task_attempt' => ['required', 'integer', 'min:1'], + 'scope_id' => ['required', 'string', 'max:255'], + 'request_id' => ['required', 'string', 'max:255'], + 'sequence' => ['required', 'integer', 'min:1'], + 'call_kind' => ['required', 'in:activity,local_activity,timer,condition,signal,child,parallel,selection_handle'], + 'sequence_span' => ['sometimes', 'required', 'integer', 'min:1'], + 'operation_sequence' => ['nullable', 'integer', 'min:1'], + 'operation_sequence_span' => ['sometimes', 'required', 'integer', 'min:1'], + ]); + $namespace = (string) $request->attributes->get('namespace'); + // Preparation owns its transaction and commits before Activity locks. + // Check tenant/issued capability/registration without an outer lock; + // Native rechecks the live owner and attempt under its run/task locks. + $result = $this->mutations->run(function () use ($namespace, $taskId, $validated, $version, $bridge, $preparing, $refused): array { + $task = NamespaceWorkflowScope::taskQuery($namespace)->find($taskId); + if (! $task instanceof WorkflowTask) { + return $refused('task_not_found'); + } + $run = WorkflowRun::query()->where('namespace', $namespace)->find($task->workflow_run_id); + if ($run === null || ! NamespaceWorkflowScope::workflowBound($namespace, $run->workflow_instance_id)) { + return $refused('task_not_found'); + } + if ($task->lease_owner !== $validated['lease_owner']) { + return $refused('lease_owner_mismatch'); + } + if ($task->attempt_count !== (int) $validated['workflow_task_attempt']) { + return $refused('workflow_task_attempt_mismatch'); + } + if (! CooperativeCancellationPolicy::claimSupportsCancellation($task)) { + return $refused('active_claim_cancellation_not_supported'); + } + $worker = WorkerRegistration::query()->where('namespace', $namespace) + ->where('worker_id', $validated['lease_owner'])->first(); + if (! $worker instanceof WorkerRegistration || ! WorkerPollFence::isFresh($worker)) { + return $refused('stale_worker_registration'); + } + $method = $preparing ? 'prepareCancellationScopeDelivery' : 'deliverCancellationScope'; + + return $bridge->$method($taskId, $validated['lease_owner'], (int) $validated['workflow_task_attempt'], + $validated['scope_id'], $validated['request_id'], (int) $validated['sequence'], $validated['call_kind'], + (int) ($validated['sequence_span'] ?? 1), isset($validated['operation_sequence']) ? (int) $validated['operation_sequence'] : null, + (int) ($validated['operation_sequence_span'] ?? 1), $version); + }); + if (($result['prepared'] ?? false) === true) { + $result['lease_owner'] = $validated['lease_owner']; + $result['workflow_task_attempt'] = (int) $validated['workflow_task_attempt']; + $result['history_refresh_page_token'] = WorkflowHistoryPageToken::encode(0); + } + $accepted = $preparing ? ($result['prepared'] ?? false) : ($result['delivered'] ?? false); + $pending = ! $preparing && ($result['prepared'] ?? false) === true + && ($result['reason'] ?? null) === 'cancellation_scope_activity_stop_not_acknowledged'; + $successful = ($accepted && ($result['reason'] ?? null) === null) || $pending; + + return WorkerProtocol::json($result, $successful ? 200 + : (($result['reason'] ?? null) === 'task_not_found' ? 404 : 409)); + } + + public function open(Request $request, string $taskId): JsonResponse + { + if ($response = WorkerProtocol::rejectUnsupported($request)) { + return $response; + } + + $version = WorkerProtocol::requestVersion($request); + if (! WorkerProtocol::versionMeetsMinimum($version, CooperativeCancellationPolicy::MINIMUM_PROTOCOL_VERSION)) { + return WorkerProtocol::json(['opened' => false, 'reason' => 'cancellation_scope_requires_protocol_1_20'], 409); + } + $bridge = app(WorkflowTaskBridge::class); + if (! $bridge instanceof CancellationScopeTaskBridge) { + return WorkerProtocol::json(['opened' => false, 'reason' => 'cancellation_scope_opening_unavailable', + 'unavailable' => ['installed_runtime_scope_opening']], 409); + } + + $validated = $request->validate([ + 'lease_owner' => ['required', 'string', 'max:255'], + 'workflow_task_attempt' => ['required', 'integer', 'min:1'], + 'sequence' => ['required', 'integer', 'min:1'], + 'parent_scope_id' => ['sometimes', 'required', 'string', 'max:255'], + 'shield_parent' => ['sometimes', 'required', 'boolean'], + ]); + $namespace = (string) $request->attributes->get('namespace'); + $result = $this->mutations->run(fn (): array => DB::transaction(function () use ( + $namespace, $taskId, $validated, $version, $bridge, + ): array { + $refused = static fn (string $reason): array => ['opened' => false, 'reason' => $reason]; + $snapshot = NamespaceWorkflowScope::taskQuery($namespace)->find($taskId); + if (! $snapshot instanceof WorkflowTask) { + return $refused('task_not_found'); + } + + // Native scope history locks the run before its task. Keep that + // order while checking the namespace and immutable issued claim. + $run = WorkflowRun::query()->where('namespace', $namespace) + ->lockForUpdate()->find($snapshot->workflow_run_id); + if ($run === null || ! NamespaceWorkflowScope::workflowBound($namespace, $run->workflow_instance_id)) { + return $refused('task_not_found'); + } + $task = NamespaceWorkflowScope::taskQuery($namespace)->lockForUpdate()->find($taskId); + if (! $task instanceof WorkflowTask || $task->workflow_run_id !== $run->id) { + return $refused('task_not_found'); + } + if ($task->lease_owner !== $validated['lease_owner']) { + return $refused('lease_owner_mismatch'); + } + if ($task->attempt_count !== (int) $validated['workflow_task_attempt']) { + return $refused('workflow_task_attempt_mismatch'); + } + if (! CooperativeCancellationPolicy::claimSupportsCancellation($task)) { + return $refused('active_claim_cancellation_not_supported'); + } + $worker = WorkerRegistration::query()->where('namespace', $namespace) + ->where('worker_id', $validated['lease_owner'])->lockForUpdate()->first(); + if (! $worker instanceof WorkerRegistration || ! WorkerPollFence::isFresh($worker)) { + return $refused('stale_worker_registration'); + } + + return $bridge->openCancellationScope( + $taskId, $validated['lease_owner'], (int) $validated['workflow_task_attempt'], + (int) $validated['sequence'], $validated['parent_scope_id'] ?? 'root', + (bool) ($validated['shield_parent'] ?? false), $version, + ); + })); + + if (($result['opened'] ?? false) === true) { + $result['lease_owner'] = $validated['lease_owner']; + $result['workflow_task_attempt'] = (int) $validated['workflow_task_attempt']; + $result['history_refresh_page_token'] = WorkflowHistoryPageToken::encode(0); + } + + return WorkerProtocol::json($result, ($result['opened'] ?? false) ? 200 + : (($result['reason'] ?? null) === 'task_not_found' ? 404 : 409)); + } +} diff --git a/app/Http/Controllers/Api/CooperativeCancellationController.php b/app/Http/Controllers/Api/CooperativeCancellationController.php new file mode 100644 index 00000000..7aa4c92a --- /dev/null +++ b/app/Http/Controllers/Api/CooperativeCancellationController.php @@ -0,0 +1,219 @@ + 'This Server does not support cooperative cancellation requests.', + 'reason' => 'cooperative_cancellation_not_supported', + 'minimum_protocol_version' => CooperativeCancellationPolicy::MINIMUM_PROTOCOL_VERSION, + ], 409); + } + + $namespace = (string) $request->attributes->get('namespace'); + if (! NamespaceWorkflowScope::workflowBound($namespace, $workflowId)) { + return ControlPlaneProtocol::jsonForRequest($request, ['message' => 'Workflow not found.', 'reason' => 'instance_not_found'], 404); + } + + $validated = $request->validate([ + 'reason' => ['nullable', 'string', 'max:1000'], + 'cleanup_timeout_seconds' => ['nullable', 'integer', 'min:1', 'max:3600'], + ]); + + $result = $this->controlMutations->run(fn (): array => DB::transaction(function () use ( + $request, $namespace, $workflowId, $runId, $validated, + ): array { + $instance = WorkflowInstance::query()->where('namespace', $namespace) + ->lockForUpdate()->find($workflowId); + if (! $instance instanceof WorkflowInstance) { + return ['reason' => 'instance_not_found', 'http_status' => 404]; + } + + $run = NamespaceWorkflowScope::runQuery($namespace, $workflowId) + ->lockForUpdate()->find($runId ?? $instance->current_run_id); + if (! $run instanceof WorkflowRun) { + return ['reason' => 'run_not_found', 'http_status' => 404]; + } + if ($run->id !== $instance->current_run_id) { + return ['reason' => 'selected_run_not_current', 'http_status' => 409]; + } + if (LegacyV1Projection::isProjectedRun($run)) { + return ['reason' => 'legacy_v1_operation_not_supported', 'http_status' => 409]; + } + + // Duplicates keep the original identity and deadline even after + // cleanup ended or the worker registration changed. + if (($pending = CooperativeCancellationPolicy::pending($run)) !== null) { + return [ + 'workflow_id' => $workflowId, + 'run_id' => $run->id, + 'accepted' => true, + 'duplicate' => true, + 'run_status' => $run->status->value, + 'cancellation_request' => $pending, + 'http_status' => 200, + ]; + } + if ($run->status->isTerminal()) { + return ['reason' => 'run_not_active', 'run_status' => $run->status->value, 'http_status' => 409]; + } + + $claims = NamespaceWorkflowScope::taskQuery($namespace) + ->where('workflow_tasks.workflow_run_id', $run->id) + ->where('workflow_tasks.task_type', TaskType::Workflow->value) + ->where('workflow_tasks.status', TaskStatus::Leased->value) + ->where('workflow_tasks.lease_expires_at', '>', now()) + ->lockForUpdate()->get(); + foreach ($claims as $claim) { + if (! CooperativeCancellationPolicy::claimSupportsCancellation($claim)) { + return [ + 'reason' => 'active_claim_cancellation_not_supported', + 'task_id' => $claim->id, + 'workflow_task_attempt' => $claim->attempt_count, + 'http_status' => 409, + ]; + } + } + + $command = WorkflowStub::loadSelection($workflowId, $runId, $namespace) + ->withCommandContext($this->contexts->make($request, $workflowId, 'request_cancellation')) + ->attemptRequestCancellation( + $validated['reason'] ?? null, + $validated['cleanup_timeout_seconds'] ?? 600, + ); + $run->refresh(); + + return [ + 'workflow_id' => $workflowId, + 'run_id' => $run->id, + 'accepted' => $command->accepted(), + 'duplicate' => false, + 'run_status' => $run->status->value, + 'cancellation_request' => CooperativeCancellationPolicy::pending($run), + 'reason' => $command->rejectionReason(), + 'http_status' => $command->accepted() ? 202 : 409, + ]; + }), allBackends: true); + + if (($result['accepted'] ?? false) === true) { + app(LongPollSignalStore::class)->signalWorkflowTaskQueuesForWorkflow($workflowId, $namespace); + } + $status = $result['http_status']; + unset($result['http_status']); + if (($result['accepted'] ?? false) !== true) { + $result['message'] = match ($result['reason'] ?? null) { + 'active_claim_cancellation_not_supported' => 'The active workflow-task claim cannot perform cooperative cancellation. Retry after a capable worker claims the task.', + 'selected_run_not_current' => 'The selected run is no longer the current run.', + 'run_not_active' => 'The workflow run is already closed.', + 'legacy_v1_operation_not_supported' => 'Imported legacy workflows do not support cooperative cancellation.', + 'run_not_found' => 'Workflow run not found.', + default => 'Workflow not found.', + }; + } + + return ControlPlaneProtocol::jsonForRequest($request, $result, $status); + } + + public function deliver(Request $request, string $taskId): JsonResponse + { + if ($response = WorkerProtocol::rejectUnsupported($request)) { + return $response; + } + if (! WorkerProtocol::versionMeetsMinimum( + WorkerProtocol::requestVersion($request), CooperativeCancellationPolicy::MINIMUM_PROTOCOL_VERSION, + )) { + return WorkerProtocol::json(['reason' => 'cooperative_cancellation_not_supported'], 409); + } + + $validated = $request->validate([ + 'lease_owner' => ['required', 'string'], + 'workflow_task_attempt' => ['required', 'integer', 'min:1'], + 'request_id' => ['required', 'string'], + 'sequence' => ['required', 'integer', 'min:1'], + 'call_kind' => ['required', 'string', 'in:activity,local_activity,timer,condition,signal,child,parallel,selection_handle'], + 'sequence_span' => ['nullable', 'integer', 'min:1'], + 'operation_sequence' => ['nullable', 'integer', 'min:1'], + 'operation_sequence_span' => ['nullable', 'integer', 'min:1'], + ]); + $namespace = (string) $request->attributes->get('namespace'); + $bridge = app(WorkflowTaskBridge::class); + if (! $bridge instanceof CooperativeWorkflowTaskBridge) { + return WorkerProtocol::json(['reason' => 'cooperative_cancellation_not_supported'], 409); + } + + $result = $this->workerMutations->run(fn (): array => DB::transaction(function () use ( + $namespace, $taskId, $validated, $bridge, + ): array { + // Keep the ownership check and engine delivery under the same + // task lock. A reclaim cannot replace the checked attempt. + $task = NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId); + if (! $task instanceof WorkflowTask) { + return ['delivered' => false, 'reason' => 'task_not_found']; + } + if ($task->lease_owner !== $validated['lease_owner']) { + return ['delivered' => false, 'reason' => 'lease_owner_mismatch']; + } + if ($task->attempt_count !== (int) $validated['workflow_task_attempt']) { + return ['delivered' => false, 'reason' => 'workflow_task_attempt_mismatch']; + } + if (! CooperativeCancellationPolicy::claimSupportsCancellation($task)) { + return ['delivered' => false, 'reason' => 'active_claim_cancellation_not_supported']; + } + + $worker = WorkerRegistration::query()->where('namespace', $namespace) + ->where('worker_id', $validated['lease_owner'])->first(); + if (! $worker instanceof WorkerRegistration || ! WorkerPollFence::isFresh($worker)) { + return ['delivered' => false, 'reason' => 'stale_worker_registration']; + } + + return $bridge->deliverCancellation( + $taskId, $validated['request_id'], (int) $validated['sequence'], $validated['call_kind'], + (int) ($validated['sequence_span'] ?? 1), + isset($validated['operation_sequence']) ? (int) $validated['operation_sequence'] : null, + (int) ($validated['operation_sequence_span'] ?? 1), + ); + })); + + return WorkerProtocol::json($result, (($result['delivered'] ?? false) + || (($result['claim_released'] ?? null) === true && in_array($result['reason'] ?? null, + ['cancellation_waiting_for_child', 'cancellation_waiting_for_activity'], true))) ? 200 + : (($result['reason'] ?? null) === 'task_not_found' ? 404 : 409)); + } +} diff --git a/app/Http/Controllers/Api/PreparedLocalActivityController.php b/app/Http/Controllers/Api/PreparedLocalActivityController.php new file mode 100644 index 00000000..e6ef5eae --- /dev/null +++ b/app/Http/Controllers/Api/PreparedLocalActivityController.php @@ -0,0 +1,429 @@ +admit($request, $taskId, false); + } + + public function recover(Request $request, string $taskId): JsonResponse + { + return $this->admit($request, $taskId, true); + } + + private function admit(Request $request, string $taskId, bool $recover): JsonResponse + { + if ($response = self::unavailable($request)) { + return $response; + } + $validated = $request->validate([ + ...self::claimRules(), + 'sequence' => ['required', 'integer', 'min:1'], + 'worker_attempt_id' => [$recover ? 'nullable' : 'required', 'string', 'max:255'], + 'descriptor' => ['required', 'array'], + ]); + // Validate membership separately so Laravel retains the complete + // opaque descriptor for Native's canonical descriptor validation. + $request->validate([ + 'descriptor.cancellation_scope_id' => ['sometimes', 'required', 'string', 'max:255'], + ]); + $namespace = (string) $request->attributes->get('namespace'); + $task = NamespaceWorkflowScope::task($namespace, $taskId); + if ($task === null) { + return WorkerProtocol::json(['reason' => 'task_not_found'], 404); + } + if (array_key_exists('cancellation_policy', $validated['descriptor']) + && ($response = self::cancellationPolicyRefusal($request, $task, $validated['descriptor']['cancellation_policy']))) { + return $response; + } + $claim = PreparedLocalActivityPolicy::currentClaim($task); + if ($claim === null) { + return self::refused($validated['lease_owner'], ['worker_claim_capability']); + } + try { + if ($response = $this->cancellationScopeRefusal($request, $task, $validated['descriptor'], (int) $validated['sequence'])) { + return $response; + } + $validated['descriptor'] = $this->resolvePayload($validated['descriptor'], 'arguments', 'descriptor', $namespace); + $reply = $this->mutations->run(fn (): array|JsonResponse => DB::transaction(function () use ($request, $namespace, $taskId, $validated, $claim, $recover): array|JsonResponse { + $quotaSnapshot = $this->quota->snapshotForMutation($namespace, self::quotaResources()); + // Registration precedes Native's attempt/execution/run/task + // locks. Do not lock the hosting task before entering Native. + if (! WorkerPollFence::isCurrentForUpdate($claim)) { + return WorkerProtocol::json(['reason' => 'stale_worker_registration'], 409); + } + if (array_key_exists('cancellation_policy', $validated['descriptor']) + && ($response = self::cancellationPolicyRefusal($request, NamespaceWorkflowScope::task($namespace, $taskId), $validated['descriptor']['cancellation_policy']))) { + return $response; + } + /** @var PreparedLocalActivityTaskBridge $bridge */ + $bridge = app(WorkflowTaskBridge::class); + $reply = $recover + ? $bridge->recoverLocalActivity($taskId, $validated['lease_owner'], (int) $validated['workflow_task_attempt'], (int) $validated['sequence'], $validated['descriptor'], WorkerProtocol::requestVersion($request)) + : $bridge->prepareLocalActivity($taskId, $validated['lease_owner'], (int) $validated['workflow_task_attempt'], (int) $validated['sequence'], $validated['worker_attempt_id'], $validated['descriptor'], WorkerProtocol::requestVersion($request)); + if (! $recover && ($reply['prepared'] ?? false)) { + $task = NamespaceWorkflowScope::taskQuery($namespace)->lockForUpdate()->find($taskId); + $attemptId = $reply['activity_attempt_id'] ?? null; + $attempt = is_string($attemptId) ? ActivityAttempt::query()->find($attemptId) : null; + if (! $task instanceof WorkflowTask || ! $attempt instanceof ActivityAttempt + || ! PreparedLocalActivityPolicy::remember($task, $attempt, $claim)) { + throw new PreparedLocalActivityAdmissionRefused; + } + } + + $this->quota->assertNoIncreasePastLimit($quotaSnapshot); + + return $reply; + })); + } catch (PreparedLocalActivityAdmissionRefused) { + return self::refused($validated['lease_owner'], ['original_claim_persistence']); + } catch (\Throwable $exception) { + if ($response = $this->payloadFailure($exception)) { + return $response; + } + if (! BackendLockPressure::is($exception)) { + throw $exception; + } + + return BackendLockPressure::workerOperationResponse($request, false); + } + + return $this->reply($request, $reply); + } + + private function cancellationScopeRefusal(Request $request, WorkflowTask $task, array $descriptor, int $sequence): ?JsonResponse + { + if (! array_key_exists('cancellation_scope_id', $descriptor)) { + return null; + } + $bridge = app(WorkflowTaskBridge::class); + $unavailable = []; + if (! $bridge instanceof CancellationScopeAdmission) { + $reason = 'cancellation_scope_membership_unavailable'; + $unavailable[] = 'installed_runtime_scope_admission'; + } else { + $run = WorkflowRun::query()->whereKey($task->workflow_run_id)->where('namespace', $task->namespace)->first(); + if (! $run instanceof WorkflowRun) { + return WorkerProtocol::json(['reason' => 'task_not_found'], 404); + } + // Payload resolution follows this preflight. Native rechecks + // canonical membership under its attempt/execution/run/task locks. + $reason = $bridge->validateCancellationScopeMembership($run, [[ + 'type' => 'prepare_local_activity', 'cancellation_scope_id' => $descriptor['cancellation_scope_id'], + ...(array_key_exists('cancellation_cleanup', $descriptor) + ? ['cancellation_cleanup' => $descriptor['cancellation_cleanup']] : []), + ]], $sequence); + } + if ($reason === null) { + return null; + } + + return WorkerProtocol::json([ + 'task_id' => $task->id, 'workflow_task_attempt' => (int) $request->input('workflow_task_attempt'), + 'outcome' => 'rejected', 'recorded' => false, 'reason' => $reason, 'unavailable' => $unavailable, + 'requested_version' => WorkerProtocol::requestVersion($request), 'minimum_protocol_version' => '1.20', + ], 409); + } + + public function control(Request $request, string $taskId, string $attemptId): JsonResponse + { + return $this->attemptOperation($request, $taskId, $attemptId, 'control'); + } + + public function outcome(Request $request, string $taskId, string $attemptId): JsonResponse + { + return $this->attemptOperation($request, $taskId, $attemptId, 'outcome'); + } + + public function heartbeat(Request $request, string $taskId, string $attemptId): JsonResponse + { + return $this->attemptOperation($request, $taskId, $attemptId, 'heartbeat'); + } + + public function acknowledgeCancellation(Request $request, string $taskId, string $attemptId): JsonResponse + { + return $this->attemptOperation($request, $taskId, $attemptId, 'acknowledge'); + } + + private function attemptOperation(Request $request, string $taskId, string $attemptId, string $operation): JsonResponse + { + if ($response = self::unavailable($request)) { + return $response; + } + $validated = $request->validate([ + ...self::claimRules(), + 'renew_lease' => ['nullable', 'boolean'], + 'report' => [$operation === 'outcome' ? 'required' : 'nullable', 'array'], + 'request_id' => [$operation === 'acknowledge' ? 'required' : 'nullable', 'string', 'max:255'], + 'progress' => ['nullable', 'array'], + ]); + $namespace = (string) $request->attributes->get('namespace'); + $task = NamespaceWorkflowScope::task($namespace, $taskId); + $attempt = ActivityAttempt::query()->where('workflow_task_id', $taskId)->find($attemptId); + if ($task === null || $attempt === null || $attempt->workflow_run_id !== $task->workflow_run_id) { + return WorkerProtocol::json(['reason' => 'activity_attempt_not_found'], 404); + } + $claim = PreparedLocalActivityPolicy::originalClaim($task, $attempt, $validated['lease_owner'], (int) $validated['workflow_task_attempt']); + if ($claim === null) { + return self::refused($validated['lease_owner'], ['original_prepared_local_claim']); + } + try { + $bridge = app(WorkflowTaskBridge::class); + if ($operation !== 'acknowledge' && $bridge instanceof PreparedCancellationScopeTaskBridge) { + $execution = ActivityExecution::query()->where('workflow_run_id', $task->workflow_run_id) + ->find($attempt->activity_execution_id); + if ($execution instanceof ActivityExecution + && ($execution->activity_options['cancellation_scope_id'] ?? 'root') !== 'root' + && ! array_key_exists('cancellation_cleanup', $execution->activity_options)) { + // Cancellation preparation commits before the fence actor. + // This stop-only observation grants no renewal, heartbeat + // or publication authority. Keep those mutations behind + // the registration and quota locks below. + $control = $this->mutations->run(fn (): array => $bridge->controlLocalActivity( + $attemptId, $validated['lease_owner'], (int) $validated['workflow_task_attempt'], + false, WorkerProtocol::requestVersion($request) + )); + if ($operation !== 'outcome' && isset($control['cancellation_scope'])) { + return $this->reply($request, $control); + } + } + } + if ($operation === 'outcome' && ($validated['report']['outcome'] ?? null) === 'completed') { + $validated['report'] = $this->resolvePayload($validated['report'], 'result', 'report', $namespace); + } + $reply = $this->mutations->run(fn (): array => DB::transaction(function () use ($request, $validated, $claim, $attemptId, $operation): array { + $quotaSnapshot = $this->quota->snapshotForMutation($claim['namespace'], self::quotaResources()); + /** @var PreparedLocalActivityTaskBridge $bridge */ + $bridge = app(WorkflowTaskBridge::class); + $version = WorkerProtocol::requestVersion($request); + $epoch = (int) $validated['workflow_task_attempt']; + $owner = $validated['lease_owner']; + if ($operation === 'acknowledge') { + // A dead, drained or replaced registration may still have + // a surviving supervisor reporting its actual joined stop. + $reply = $bridge->acknowledgeLocalActivityCancellation($attemptId, $owner, $validated['request_id'], $epoch, $version); + $this->quota->assertNoIncreasePastLimit($quotaSnapshot); + + return $reply; + } + $current = WorkerPollFence::isCurrentForUpdate($claim); + if ($operation === 'outcome') { + // Lost responses may still read their immutable receipt. + // Cancellation may fence, but a stale registration cannot + // publish a fresh result under an unreclaimed task epoch. + if (! $current && ! WorkflowHistoryEvent::query() + ->where('workflow_run_id', $claim['run_id']) + ->where('payload->activity_attempt_id', $attemptId) + ->whereNotNull('payload->local_outcome')->exists() + ) { + // A stale registration can observe and fence its old + // cancelled callback. It cannot publish shielded + // cleanup merely because cancellation was accepted. + $control = $bridge->controlLocalActivity($attemptId, $owner, $epoch, false, $version); + if (! in_array($control['reason'] ?? null, ['cancellation_requested', 'cancellation_deadline_expired'], true)) { + return ['recorded' => false, 'reason' => 'stale_worker_registration']; + } + } + $reply = $bridge->recordLocalActivityOutcome($attemptId, $owner, $epoch, $validated['report'], $version); + $this->quota->assertNoIncreasePastLimit($quotaSnapshot); + + return $reply; + } + $reply = $operation === 'heartbeat' && $current + ? $bridge->heartbeatLocalActivity($attemptId, $owner, $epoch, $validated['progress'] ?? [], $version) + : $bridge->controlLocalActivity($attemptId, $owner, $epoch, $current && ($validated['renew_lease'] ?? false), $version); + if (! $current && ($reply['active'] ?? false)) { + $reply = [...$reply, 'active' => false, 'renewed' => false, 'heartbeat_recorded' => false, + 'stop_required' => true, 'reason' => 'stale_worker_registration']; + } + $this->quota->assertNoIncreasePastLimit($quotaSnapshot); + + return $reply; + })); + } catch (\Throwable $exception) { + if ($response = $this->payloadFailure($exception)) { + return $response; + } + if (! BackendLockPressure::is($exception)) { + throw $exception; + } + + return BackendLockPressure::workerOperationResponse($request, false); + } + + return $this->reply($request, $reply); + } + + /** @return array> */ + private static function claimRules(): array + { + return ['lease_owner' => ['required', 'string', 'max:255'], 'workflow_task_attempt' => ['required', 'integer', 'min:1']]; + } + + /** @return list */ + private static function quotaResources(): array + { + return [NamespaceDurableStateQuota::WORKFLOW_HISTORY_EVENTS, NamespaceDurableStateQuota::WORKFLOW_TASKS, + NamespaceDurableStateQuota::PENDING_WORKFLOW_TASKS]; + } + + /** @param array $payload + * @return array + */ + private function resolvePayload(array $payload, string $key, string $prefix, string $namespace): array + { + if (array_key_exists('payload_codec', $payload)) { + try { + $payload['payload_codec'] = PayloadCodecContract::canonicalize($payload['payload_codec']); + } catch (\InvalidArgumentException $exception) { + throw ValidationException::withMessages([$prefix.'.payload_codec' => [$exception->getMessage()]]); + } + } + $value = $payload[$key] ?? null; + if (is_string($value)) { + AvroPayloadEnvelopeResolver::assertSerializedPayload($value, $prefix.'.'.$key); + } elseif (is_array($value)) { + $resolved = AvroPayloadEnvelopeResolver::resolveCommandPayloadWithCodec( + $value, $prefix.'.'.$key, app(NamespaceExternalPayloadStorage::class)->driverFor($namespace), + ); + $payload[$key] = $resolved['payload']; + $payload['payload_codec'] ??= $resolved['codec']; + } + + return $payload; + } + + private function payloadFailure(\Throwable $exception): ?JsonResponse + { + if ($exception instanceof StructuralLimitExceededException) { + return WorkerProtocol::json(['recorded' => false, 'reason' => 'structural_limit_exceeded', + 'limit_kind' => $exception->limitKind->value, 'current_value' => $exception->currentValue, + 'configured_limit' => $exception->configuredLimit], 422); + } + if ($exception instanceof ExternalPayloadIntegrityException || $exception instanceof ExternalPayloadStorageUnavailable) { + return WorkerProtocol::json(['recorded' => false, 'reason' => $exception instanceof ExternalPayloadIntegrityException + ? 'external_payload_integrity_failed' : 'external_payload_storage_unavailable'], + $exception instanceof ExternalPayloadIntegrityException ? 422 : 503); + } + + return null; + } + + public static function unavailable(Request $request): ?JsonResponse + { + if ($response = WorkerProtocol::rejectUnsupported($request)) { + return $response; + } + $reasons = PreparedLocalActivityPolicy::unavailableReasons(WorkerProtocol::requestVersion($request)); + + return $reasons === [] ? null : self::refused((string) $request->input('lease_owner', ''), $reasons); + } + + public static function cancellationPolicyRefusal(Request $request, ?WorkflowTask $task, mixed $policy): ?JsonResponse + { + $reasons = PreparedLocalActivityPolicy::cancellationPolicyUnavailableReasons($task, $policy, WorkerProtocol::requestVersion($request)); + if ($reasons === []) { + return null; + } + + return WorkerProtocol::json([ + 'reason' => 'prepared_local_activity_cancellation_policy_not_supported', + 'recorded' => false, + 'worker_id' => $task?->lease_owner ?? (string) $request->input('lease_owner', ''), + 'requested_policy' => $policy, + 'supported_policies' => PreparedLocalActivityPolicy::cancellationPolicies(), + 'required_capability' => PreparedLocalActivityPolicy::CANCELLATION_POLICIES_CAPABILITY, + 'minimum_protocol_version' => PreparedLocalActivityPolicy::MINIMUM_PROTOCOL_VERSION, + 'requested_version' => WorkerProtocol::requestVersion($request), + 'unavailable' => $reasons, + 'remediation' => 'Use an installed backend with the requested local policy and a new protocol 1.20 claim issued with prepared_local_activity_cancellation_policies. Local abandon requires independently bounded remote work.', + ], 409); + } + + /** @param list $reasons */ + public static function refused(string $workerId, array $reasons, string $capability = PreparedLocalActivityPolicy::CAPABILITY): JsonResponse + { + return WorkerProtocol::json([ + 'reason' => $capability === PreparedLocalActivityPolicy::GROUP_CAPABILITY + ? 'prepared_local_activity_groups_not_supported' : 'prepared_local_activity_not_supported', + 'required_capability' => $capability, + 'minimum_protocol_version' => PreparedLocalActivityPolicy::MINIMUM_PROTOCOL_VERSION, + 'worker_id' => $workerId, + 'unavailable' => $reasons, + 'remediation' => 'Use a prepared-local backend and a claim issued to a capable protocol 1.20 worker.', + ], 409); + } + + /** @param array|JsonResponse $reply */ + private function reply(Request $request, array|JsonResponse $reply): JsonResponse + { + if ($reply instanceof JsonResponse) { + return $reply; + } + // This is a cursor, not authority. History paging still requires the + // current namespace, lease owner and workflow claim epoch. + $reply['history_refresh_page_token'] = WorkflowHistoryPageToken::encode(0); + try { + $this->mutations->run(function () use ($reply): void { + app(ServiceModeTimerDispatcher::class)->dispatchCreatedTaskIds($reply['created_task_ids'] ?? []); + foreach ($reply['created_task_ids'] ?? [] as $taskId) { + $task = WorkflowTask::query()->find($taskId); + if ($task instanceof WorkflowTask) { + app(LongPollSignalStore::class)->signalTask($task); + } + } + }); + } catch (\Throwable $exception) { + if (! BackendLockPressure::is($exception)) { + throw $exception; + } + + return BackendLockPressure::workerOperationResponse($request, true); + } + $reason = $reply['reason'] ?? null; + $ok = $reason === null || isset($reply['cancellation_request']) + || (($reply['fenced'] ?? false) === true && is_array($reply['cancellation_scope'] ?? null) + && in_array($reason, ['cancellation_scope_requested', 'cancellation_scope_deadline_expired'], true)); + + return WorkerProtocol::json($reply, $ok ? 200 : (in_array($reason, ['task_not_found', 'activity_attempt_not_found'], true) ? 404 : 409)); + } +} diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index 822e6915..f9c7111c 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -9,6 +9,7 @@ use App\Support\AvroPayloadEnvelopeResolver; use App\Support\BackendLockPressure; use App\Support\CachedPollTaskKindConflict; +use App\Support\CooperativeCancellationPolicy; use App\Support\ExternalPayloadStorageUnavailable; use App\Support\HistoryRetentionEnforcer; use App\Support\LongPollCapacityExhaustedException; @@ -19,6 +20,8 @@ use App\Support\NamespaceWorkflowScope; use App\Support\PayloadCodecContract; use App\Support\PollRequestTaskKindsConflict; +use App\Support\PreparedLocalActivityAdmissionRefused; +use App\Support\PreparedLocalActivityPolicy; use App\Support\QueryTaskQueueUnavailableException; use App\Support\RouteAuthorizationResource; use App\Support\RuntimeExternalPayloadAudit; @@ -36,6 +39,7 @@ use App\Support\WorkerProtocol; use App\Support\WorkerProtocolMutationRetrier; use App\Support\WorkerTerminalEventAttribution; +use App\Support\WorkflowHistoryPageToken; use App\Support\WorkflowMetadataCapabilityPolicy; use App\Support\WorkflowQueryTaskBroker; use App\Support\WorkflowStreamCommandProcessor; @@ -50,7 +54,11 @@ use Illuminate\Support\Facades\Validator; use Illuminate\Support\Str; use Illuminate\Validation\ValidationException; +use Workflow\V2\Contracts\CancellationScopeAdmission; +use Workflow\V2\Contracts\CancellationScopeTaskBridge; use Workflow\V2\Contracts\HistoryProjectionRole; +use Workflow\V2\Contracts\PreparedLocalActivityGroupTaskBridge; +use Workflow\V2\Contracts\PreparedLocalActivityTaskBridge; use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Enums\ActivityAttemptStatus; use Workflow\V2\Enums\HistoryEventType; @@ -65,9 +73,11 @@ use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowServiceCall; use Workflow\V2\Models\WorkflowTask; +use Workflow\V2\Support\PortableLocalActivityPreparation; use Workflow\V2\Support\StickyExecution; use Workflow\V2\Support\WorkerProtocolVersion; use Workflow\V2\Support\WorkflowCommandNormalizer; +use Workflow\V2\Support\WorkflowStepHistory; use Workflow\V2\Support\WorkflowTaskOwnership; class WorkerController @@ -223,6 +233,47 @@ public function register(Request $request): JsonResponse ]); $workerCapabilities = $this->nonEmptyStringArray($validated['capabilities'] ?? []); + if (in_array(PreparedLocalActivityPolicy::CANCELLATION_POLICIES_CAPABILITY, $workerCapabilities, true) + && ! PreparedLocalActivityPolicy::supportsCancellationPolicies($workerCapabilities, WorkerProtocol::requestVersion($request))) { + return WorkerProtocol::json([ + 'registered' => false, + 'reason' => 'prepared_local_activity_cancellation_policy_capability_mismatch', + 'required_capabilities' => [CooperativeCancellationPolicy::CAPABILITY, PreparedLocalActivityPolicy::CAPABILITY], + 'minimum_protocol_version' => PreparedLocalActivityPolicy::MINIMUM_PROTOCOL_VERSION, + 'requested_version' => WorkerProtocol::requestVersion($request), + ], 409); + } + if (in_array(PreparedLocalActivityPolicy::GROUP_CAPABILITY, $workerCapabilities, true) + && (! in_array(PreparedLocalActivityPolicy::CAPABILITY, $workerCapabilities, true) + || ! CooperativeCancellationPolicy::supports($workerCapabilities, WorkerProtocol::requestVersion($request)))) { + return WorkerProtocol::json([ + 'registered' => false, + 'reason' => 'prepared_local_activity_group_capability_mismatch', + 'required_capabilities' => [CooperativeCancellationPolicy::CAPABILITY, PreparedLocalActivityPolicy::CAPABILITY], + 'minimum_protocol_version' => PreparedLocalActivityPolicy::MINIMUM_PROTOCOL_VERSION, + 'requested_version' => WorkerProtocol::requestVersion($request), + ], 409); + } + if (in_array(PreparedLocalActivityPolicy::CAPABILITY, $workerCapabilities, true) + && ! CooperativeCancellationPolicy::supports($workerCapabilities, WorkerProtocol::requestVersion($request))) { + return WorkerProtocol::json([ + 'registered' => false, + 'reason' => 'prepared_local_activity_capability_mismatch', + 'required_capability' => CooperativeCancellationPolicy::CAPABILITY, + 'minimum_protocol_version' => PreparedLocalActivityPolicy::MINIMUM_PROTOCOL_VERSION, + 'requested_version' => WorkerProtocol::requestVersion($request), + ], 409); + } + if (in_array(CooperativeCancellationPolicy::CAPABILITY, $workerCapabilities, true) + && ! CooperativeCancellationPolicy::supports($workerCapabilities, WorkerProtocol::requestVersion($request))) { + return WorkerProtocol::json([ + 'registered' => false, + 'reason' => 'cooperative_cancellation_protocol_mismatch', + 'minimum_protocol_version' => CooperativeCancellationPolicy::MINIMUM_PROTOCOL_VERSION, + 'requested_version' => WorkerProtocol::requestVersion($request), + ], 409); + } + $capabilityManifest = $this->portableWorkerCapabilityManifest( is_array($validated['capability_manifest'] ?? null) ? $validated['capability_manifest'] : [], ); @@ -1434,6 +1485,63 @@ public function workflowTaskHistory(Request $request, string $taskId): JsonRespo * Complete a claimed workflow task with commands emitted by an external worker. */ public function completeWorkflowTask(Request $request, string $taskId): JsonResponse + { + return $this->mutateWorkflowTaskCommands($request, $taskId); + } + + public function checkpointLocalActivityPrefix(Request $request, string $taskId): JsonResponse + { + if ($response = PreparedLocalActivityController::unavailable($request)) { + return $response; + } + $task = NamespaceWorkflowScope::task((string) $request->attributes->get('namespace'), $taskId); + if ($task === null) { + return WorkerProtocol::json(['reason' => 'task_not_found'], 404); + } + if (PreparedLocalActivityPolicy::currentClaim($task) === null) { + return PreparedLocalActivityController::refused((string) $request->input('lease_owner', ''), ['worker_claim_capability']); + } + + return $this->mutateWorkflowTaskCommands($request, $taskId, true); + } + + public function checkpointLocalActivityGroup(Request $request, string $taskId): JsonResponse + { + if ($response = WorkerProtocol::rejectUnsupported($request)) { + return $response; + } + $reasons = PreparedLocalActivityPolicy::groupUnavailableReasons(WorkerProtocol::requestVersion($request)); + if ($reasons !== []) { + return PreparedLocalActivityController::refused((string) $request->input('lease_owner', ''), $reasons, PreparedLocalActivityPolicy::GROUP_CAPABILITY); + } + $task = NamespaceWorkflowScope::task((string) $request->attributes->get('namespace'), $taskId); + if ($task === null) { + return WorkerProtocol::json(['reason' => 'task_not_found'], 404); + } + if (PreparedLocalActivityPolicy::currentGroupClaim($task) === null) { + return PreparedLocalActivityController::refused((string) $request->input('lease_owner', ''), ['worker_claim_capability'], PreparedLocalActivityPolicy::GROUP_CAPABILITY); + } + + return $this->mutateWorkflowTaskCommands($request, $taskId, true, true); + } + + public function checkpointCancellationScopePrefix(Request $request, string $taskId): JsonResponse + { + if ($response = WorkerProtocol::rejectUnsupported($request)) { + return $response; + } + if (! WorkerProtocol::versionMeetsMinimum(WorkerProtocol::requestVersion($request), CooperativeCancellationPolicy::MINIMUM_PROTOCOL_VERSION)) { + return WorkerProtocol::json(['checkpointed' => false, 'reason' => 'cancellation_scope_checkpoint_requires_protocol_1_20'], 409); + } + if (! app(WorkflowTaskBridge::class) instanceof CancellationScopeTaskBridge) { + return WorkerProtocol::json(['checkpointed' => false, 'reason' => 'cancellation_scope_checkpoint_unavailable', + 'unavailable' => ['installed_runtime_scope_checkpoint']], 409); + } + + return $this->mutateWorkflowTaskCommands($request, $taskId, true, scopePrefix: true); + } + + private function mutateWorkflowTaskCommands(Request $request, string $taskId, bool $checkpoint = false, bool $group = false, bool $scopePrefix = false): JsonResponse { if ($response = WorkerProtocol::rejectUnsupported($request)) { return $response; @@ -1441,6 +1549,14 @@ public function completeWorkflowTask(Request $request, string $taskId): JsonResp $namespace = $request->attributes->get('namespace'); + $checkpointInput = $checkpoint ? $request->validate([ + 'checkpoint_id' => ['required', 'string', 'max:255'], + 'start_sequence' => ['required', 'integer', 'min:1'], + 'message_stream_cursors' => ['prohibited'], + 'message_stream_waits' => ['prohibited'], + 'sticky_cache' => ['prohibited'], + ]) : []; + $messageStreamCompletion = $request->validate([ 'message_stream_cursors' => ['nullable', 'array', 'max:100'], 'message_stream_cursors.*.stream_name' => ['required', 'string', 'max:128'], @@ -1455,6 +1571,11 @@ public function completeWorkflowTask(Request $request, string $taskId): JsonResp 'workflow_task_attempt' => ['required', 'integer', 'min:1'], 'commands' => ['required', 'array', 'min:1'], 'commands.*.type' => ['required', 'string'], + 'commands.*.cancellation_scope_id' => ['sometimes', 'string', 'min:1', 'max:255'], + 'commands.*.cancellation_cleanup' => ['sometimes', 'required', 'array:scope_id,request_id,delivery_history_event_id'], + 'commands.*.cancellation_cleanup.scope_id' => ['sometimes', 'required', 'string', 'max:255'], + 'commands.*.cancellation_cleanup.request_id' => ['required_with:commands.*.cancellation_cleanup', 'string', 'max:255'], + 'commands.*.cancellation_cleanup.delivery_history_event_id' => ['required_with:commands.*.cancellation_cleanup', 'string', 'max:255'], 'commands.*.result' => ['nullable'], 'commands.*.activity_type' => ['nullable', 'string'], 'commands.*.arguments' => ['nullable'], @@ -1523,6 +1644,7 @@ public function completeWorkflowTask(Request $request, string $taskId): JsonResp 'commands.*.entries' => ['nullable', 'array'], 'commands.*.non_retryable' => ['nullable', 'boolean'], 'commands.*.parent_close_policy' => ['nullable', 'string'], + 'commands.*.cancellation_policy' => ['nullable', 'string', 'in:try_cancel,wait_cancellation_completed,abandon'], 'commands.*.condition_key' => ['nullable', 'string'], 'commands.*.condition_definition_fingerprint' => ['nullable', 'string'], 'commands.*.condition_wait_occurrence_id' => ['nullable', 'string'], @@ -1584,6 +1706,9 @@ public function completeWorkflowTask(Request $request, string $taskId): JsonResp ]; $topLevelRules = []; + if ($group) { + $completionRules['commands'] = ['required', 'array', 'min:1', 'max:100']; + } $commandRules = ['commands' => ['required', 'array', 'min:1']]; foreach ($completionRules as $field => $rules) { if (str_starts_with($field, 'commands.*.')) { @@ -1611,7 +1736,11 @@ public function completeWorkflowTask(Request $request, string $taskId): JsonResp } foreach ($chunkValidator->validated()['commands'] as $index => $command) { - $validatedCommands[$index] = $command; + // Prepared descriptors have a closed Native grammar. Keep + // original keys so transport validation cannot silently + // discard routing, fabricated reports or unknown fields. + $validatedCommands[$index] = $group && ($command['type'] ?? null) === 'prepare_local_activity' + ? $chunk[$index] : $command; } } } @@ -1623,11 +1752,16 @@ public function completeWorkflowTask(Request $request, string $taskId): JsonResp $validated['commands'] = $validatedCommands; $commands = $this->normalizeWorkflowTaskCommandIntegerFields($validated['commands']); - $commands = WorkflowCommandNormalizer::preflightParallelMetadata($commands); + $commands = $group ? $this->preflightPreparedLocalGroupMetadata($commands) + : WorkflowCommandNormalizer::preflightParallelMetadata($commands); $commands = $this->applyWorkerSessionRoutingDefaults($commands); $this->validateWorkflowTaskCommandScopes($commands); + if ($response = $this->guardPreparedLocalCancellationPoliciesAvailable($request, (string) $namespace, $taskId, $commands)) { + return $response; + } + if ($response = $this->guardWorkflowTaskOwnership( $request, $namespace, @@ -1647,6 +1781,31 @@ public function completeWorkflowTask(Request $request, string $taskId): JsonResp return $response; } + if ($response = $this->guardCancellationScopeAdmission($request, (string) $namespace, $taskId, + (int) $validated['workflow_task_attempt'], $commands)) { + return $response; + } + + if ($response = $this->guardRemoteActivityCancellationPoliciesAvailable( + $request, + (string) $namespace, + $taskId, + (int) $validated['workflow_task_attempt'], + $commands, + )) { + return $response; + } + + if ($response = $this->guardChildCancellationPoliciesAvailable( + $request, + (string) $namespace, + $taskId, + (int) $validated['workflow_task_attempt'], + $commands, + )) { + return $response; + } + $commands = $this->authorizeServiceOperationCommands($request, (string) $namespace, $commands); if ($response = $this->guardWorkerSessionCommandsAvailable( @@ -1756,6 +1915,10 @@ function () use ( $validated, $messageStreamCursors, $messageStreamWaits, + $checkpoint, + $checkpointInput, + $group, + $scopePrefix, ): array|JsonResponse { return DB::transaction(function () use ( $bridge, @@ -1766,6 +1929,10 @@ function () use ( $validated, $messageStreamCursors, $messageStreamWaits, + $checkpoint, + $checkpointInput, + $group, + $scopePrefix, ): array|JsonResponse { $quotaSnapshot = $this->durableStateQuota->snapshotForMutation( (string) $namespace, @@ -1780,10 +1947,10 @@ function () use ( ->where('worker_id', $validated['lease_owner']) ->lockForUpdate() ->first(); - NamespaceWorkflowScope::taskQuery((string) $namespace) - ->whereKey($taskId) - ->lockForUpdate() - ->first(); + $claimedTask = NamespaceWorkflowScope::lockTaskForMutation((string) $namespace, $taskId); + if (! $claimedTask instanceof WorkflowTask) { + return $this->workflowTaskNotFound($taskId, (int) $validated['workflow_task_attempt']); + } if ($response = $this->guardWorkflowTaskOwnership( $request, @@ -1806,6 +1973,32 @@ function () use ( return $response; } + if ($response = $this->guardRemoteActivityCancellationPoliciesAvailable( + $request, + (string) $namespace, + $taskId, + (int) $validated['workflow_task_attempt'], + $commands, + $claimedTask, + )) { + return $response; + } + + if ($response = $this->guardPreparedLocalCancellationPoliciesAvailable($request, (string) $namespace, $taskId, $commands, $claimedTask)) { + return $response; + } + + if ($response = $this->guardChildCancellationPoliciesAvailable( + $request, + (string) $namespace, + $taskId, + (int) $validated['workflow_task_attempt'], + $commands, + $claimedTask, + )) { + return $response; + } + if ($response = $this->guardPortableWorkerAffinityCompletion( $request, (string) $namespace, @@ -1820,17 +2013,52 @@ function () use ( } $this->authorizeServiceOperationReplays($request, (string) $namespace, $taskId, $commands); + if ($response = $this->guardCancellationScopeAdmission($request, (string) $namespace, $taskId, + (int) $validated['workflow_task_attempt'], $commands, true)) { + return $response; + } $commands = $this->canonicalizeWorkflowStreamPayloadCodecs($commands); $streamCommands = $commands; $commands = app(WorkflowStreamCommandProcessor::class)->withoutDirectives($commands); - $commands = WorkflowCommandNormalizer::normalize( - $commands, - WorkerProtocol::requestVersion($request), - ); - $outcome = $bridge->complete($taskId, $commands); - if (($outcome['completed'] ?? false) === true) { + $commands = $group ? $this->normalizePreparedLocalGroupCommands($commands, WorkerProtocol::requestVersion($request)) + : WorkflowCommandNormalizer::normalize($commands, WorkerProtocol::requestVersion($request)); + if ($checkpoint) { + if ($scopePrefix) { + if (! $bridge instanceof CancellationScopeTaskBridge + || ! $claimedTask instanceof WorkflowTask + || ! CooperativeCancellationPolicy::claimSupportsCancellation($claimedTask)) { + return WorkerProtocol::json(['checkpointed' => false, 'reason' => 'active_claim_cancellation_not_supported'], 409); + } + } elseif (PreparedLocalActivityPolicy::currentClaim($claimedTask) === null + || ! $bridge instanceof PreparedLocalActivityTaskBridge + || ($group && (PreparedLocalActivityPolicy::currentGroupClaim($claimedTask) === null + || ! $bridge instanceof PreparedLocalActivityGroupTaskBridge))) { + throw new PreparedLocalActivityAdmissionRefused; + } + $method = $scopePrefix ? 'checkpointCancellationScopePrefix' + : ($group ? 'checkpointLocalActivityGroup' : 'checkpointLocalActivityPrefix'); + $outcome = $bridge->$method( + $taskId, $validated['lease_owner'], (int) $validated['workflow_task_attempt'], + $checkpointInput['checkpoint_id'], (int) $checkpointInput['start_sequence'], $commands, + WorkerProtocol::requestVersion($request), + ); + // Shared post-processing must not release this claim + // or attribute a terminal event to a prefix receipt. + $outcome['completed'] = false; + } else { + $outcome = $bridge->complete($taskId, $commands); + } + if (($outcome[$checkpoint ? 'checkpointed' : 'completed'] ?? false) === true) { app(WorkflowStreamCommandProcessor::class)->process($taskId, (string) $namespace, $streamCommands); } + if (($outcome['completed'] ?? false) === true + && $claimedTask instanceof WorkflowTask + && CooperativeCancellationPolicy::claimSupportsCancellation($claimedTask)) { + $successor = CooperativeCancellationPolicy::resumeUndeliveredRequest($claimedTask->refresh()); + if ($successor instanceof WorkflowTask) { + $outcome['created_task_ids'][] = $successor->id; + } + } $this->applyStickyCacheClaim( $taskId, $validated['lease_owner'], @@ -1875,6 +2103,9 @@ function () use ( 'current_value' => $e->currentValue, 'configured_limit' => $e->configuredLimit, ], 422); + } catch (PreparedLocalActivityAdmissionRefused) { + return PreparedLocalActivityController::refused($validated['lease_owner'], ['worker_claim_capability'], + $group ? PreparedLocalActivityPolicy::GROUP_CAPABILITY : PreparedLocalActivityPolicy::CAPABILITY); } catch (ExternalPayloadStorageUnavailable $exception) { return $this->externalPayloadFailure($taskId, (int) $validated['workflow_task_attempt'], $exception, 503); } catch (StreamFullException $exception) { @@ -1948,6 +2179,13 @@ function () use ( return BackendLockPressure::workerOperationResponse($request, true); } + if ($checkpoint) { + unset($outcome['completed']); + $outcome['history_refresh_page_token'] = WorkflowHistoryPageToken::encode(0); + + return WorkerProtocol::json($outcome, $this->workflowOutcomeStatus($outcome['reason'])); + } + return WorkerProtocol::json([ 'task_id' => $taskId, 'workflow_task_attempt' => (int) $validated['workflow_task_attempt'], @@ -1960,9 +2198,151 @@ function () use ( ], $this->workflowOutcomeStatus($outcome['reason'])); } - /** - * @param list> $commands - */ + /** @param array> $commands */ + private function preflightPreparedLocalGroupMetadata(array $commands): array + { + $locals = []; + foreach ($commands as $index => $command) { + if (($command['type'] ?? null) === 'prepare_local_activity') { + $locals[] = $index; + $commands[$index]['type'] = 'schedule_activity'; + } + } + $commands = WorkflowCommandNormalizer::preflightParallelMetadata($commands); + foreach ($locals as $index) { + $commands[$index]['type'] = 'prepare_local_activity'; + } + + return $commands; + } + + /** @param array> $commands */ + private function normalizePreparedLocalGroupCommands(array $commands, string $protocolVersion): array + { + $locals = []; + foreach ($commands as $index => $command) { + if (($command['type'] ?? null) !== 'prepare_local_activity') { + continue; + } + $descriptor = PortableLocalActivityPreparation::normalizeDescriptor([...$command, 'type' => 'record_local_activity']); + $locals[$index] = [...$descriptor, 'type' => 'prepare_local_activity']; + unset($descriptor['execution_mode'], $descriptor['cancellation_cleanup']); + $commands[$index] = [...$descriptor, 'type' => 'schedule_activity']; + } + $commands = WorkflowCommandNormalizer::normalize($commands, $protocolVersion); + foreach ($locals as $index => $descriptor) { + $commands[$index] = $descriptor; + } + + return $commands; + } + + /** @param array> $commands */ + private function guardPreparedLocalCancellationPoliciesAvailable(Request $request, string $namespace, string $taskId, array $commands, ?WorkflowTask $claimedTask = null): ?JsonResponse + { + foreach ($commands as $command) { + if (($command['type'] ?? null) !== 'prepare_local_activity' || ! array_key_exists('cancellation_policy', $command)) { + continue; + } + $task = $claimedTask ?? NamespaceWorkflowScope::task($namespace, $taskId); + if ($response = PreparedLocalActivityController::cancellationPolicyRefusal($request, $task, $command['cancellation_policy'])) { + return $response; + } + } + + return null; + } + + /** @param array> $commands */ + private function guardRemoteActivityCancellationPoliciesAvailable( + Request $request, + string $namespace, + string $taskId, + int $workflowTaskAttempt, + array $commands, + ?WorkflowTask $claimedTask = null, + ): ?JsonResponse { + $usesPolicy = false; + foreach ($commands as $command) { + if (($command['type'] ?? null) === 'schedule_activity' + && $this->hasCommandValue($command, 'cancellation_policy')) { + $usesPolicy = true; + break; + } + } + if (! $usesPolicy) { + return null; + } + $task = $claimedTask ?? NamespaceWorkflowScope::taskQuery($namespace)->whereKey($taskId)->first(); + $unavailable = CooperativeCancellationPolicy::remoteActivityPolicyUnavailableReasons( + $task, + WorkerProtocol::requestVersion($request), + ); + if ($unavailable === []) { + return null; + } + + return WorkerProtocol::json([ + 'task_id' => $taskId, + 'workflow_task_attempt' => $workflowTaskAttempt, + 'worker_id' => $task?->lease_owner, + 'outcome' => 'rejected', + 'recorded' => false, + 'reason' => 'activity_cancellation_policy_not_supported', + 'required_capability' => CooperativeCancellationPolicy::CAPABILITY, + 'minimum_protocol_version' => CooperativeCancellationPolicy::MINIMUM_PROTOCOL_VERSION, + 'requested_version' => WorkerProtocol::requestVersion($request), + 'unavailable' => $unavailable, + 'remediation' => 'Use a runtime with remote activity cancellation policies and a cooperative worker claim on protocol 1.20 or newer.', + ], 409); + } + + /** @param array> $commands */ + private function guardChildCancellationPoliciesAvailable( + Request $request, + string $namespace, + string $taskId, + int $workflowTaskAttempt, + array $commands, + ?WorkflowTask $claimedTask = null, + ): ?JsonResponse { + $usesPolicy = false; + foreach ($commands as $command) { + if (($command['type'] ?? null) === 'start_child_workflow' + && (($command['parent_close_policy'] ?? null) === 'request_cancellation' + || in_array($command['cancellation_policy'] ?? null, ['try_cancel', 'wait_cancellation_completed'], true))) { + $usesPolicy = true; + break; + } + } + if (! $usesPolicy) { + return null; + } + $task = $claimedTask ?? NamespaceWorkflowScope::taskQuery($namespace)->whereKey($taskId)->first(); + $unavailable = CooperativeCancellationPolicy::childPolicyUnavailableReasons( + $task, + WorkerProtocol::requestVersion($request), + ); + if ($unavailable === []) { + return null; + } + + return WorkerProtocol::json([ + 'task_id' => $taskId, + 'workflow_task_attempt' => $workflowTaskAttempt, + 'worker_id' => $task?->lease_owner, + 'outcome' => 'rejected', + 'recorded' => false, + 'reason' => 'child_cancellation_policy_not_supported', + 'required_capability' => CooperativeCancellationPolicy::CAPABILITY, + 'minimum_protocol_version' => CooperativeCancellationPolicy::MINIMUM_PROTOCOL_VERSION, + 'requested_version' => WorkerProtocol::requestVersion($request), + 'unavailable' => $unavailable, + 'remediation' => 'Use a runtime with child cancellation policies and a claim from a cooperative worker on protocol 1.20 or newer.', + ], 409); + } + + /** @param list> $commands */ private function guardWorkerSessionCommandsAvailable( Request $request, string $taskId, @@ -2396,7 +2776,7 @@ private function persistTypedSearchAttributeHistoryIdentity( array $commands, array $outcome, ): void { - if (($outcome['completed'] ?? false) !== true) { + if (($outcome['completed'] ?? false) !== true && ($outcome['checkpointed'] ?? false) !== true) { return; } @@ -2409,11 +2789,22 @@ private function persistTypedSearchAttributeHistoryIdentity( return; } - $events = WorkflowHistoryEvent::query() + $eventQuery = WorkflowHistoryEvent::query() ->where('workflow_task_id', $taskId) - ->where('event_type', HistoryEventType::SearchAttributesUpserted->value) - ->orderBy('sequence') - ->get(); + ->where('event_type', HistoryEventType::SearchAttributesUpserted->value); + if (($outcome['checkpointed'] ?? false) === true) { + $eventQuery->where('payload->sequence', '>=', $outcome['start_sequence']) + ->where('payload->sequence', '<', $outcome['next_sequence']); + } else { + $payload = WorkflowTask::query()->find($taskId)?->payload ?? []; + $prefixEnd = max($payload['portable_local_checkpoint']['next_sequence'] ?? 0, + $payload['portable_local_group_checkpoint']['next_sequence'] ?? 0, + $payload['portable_scope_checkpoint']['next_sequence'] ?? 0); + if (is_int($prefixEnd) && $prefixEnd > 0) { + $eventQuery->where('payload->sequence', '>=', $prefixEnd); + } + } + $events = $eventQuery->orderBy('sequence')->get(); if ($events->count() !== count($upserts)) { throw new \RuntimeException('Typed search-attribute commands did not produce a one-to-one history event set.'); @@ -2534,6 +2925,54 @@ private function guardConditionWaitOccurrenceIdentityAvailable( ], 409); } + /** @param list> $commands */ + private function guardCancellationScopeAdmission( + Request $request, + string $namespace, + string $taskId, + int $workflowTaskAttempt, + array $commands, + bool $lock = false, + ): ?JsonResponse { + if (! collect($commands)->contains(static fn (array $command): bool => array_key_exists('cancellation_scope_id', $command))) { + return null; + } + $reason = null; + $unavailable = []; + if (! WorkerProtocol::versionMeetsMinimum(WorkerProtocol::requestVersion($request), '1.20')) { + $reason = 'cancellation_scope_membership_unavailable'; + $unavailable[] = 'request_protocol'; + } + $bridge = app(WorkflowTaskBridge::class); + if (! $bridge instanceof CancellationScopeAdmission) { + $reason = 'cancellation_scope_membership_unavailable'; + $unavailable[] = 'installed_runtime_scope_admission'; + } + foreach ($commands as $command) { + if (array_key_exists('cancellation_scope_id', $command) + && ! in_array($command['type'], ['schedule_activity', 'start_timer', 'start_child_workflow', 'prepare_local_activity', 'open_signal_wait', 'open_condition_wait'], true)) { + $reason = 'invalid_cancellation_scope_command'; + } + } + if ($reason === null) { + $task = WorkflowTask::query()->whereKey($taskId)->where('namespace', $namespace)->first(); + $runQuery = WorkflowRun::query()->whereKey($task?->workflow_run_id)->where('namespace', $namespace); + $run = ($lock ? $runQuery->lockForUpdate() : $runQuery)->first(); + if ($run instanceof WorkflowRun) { + $reason = $bridge->validateCancellationScopeMembership($run, $commands, WorkflowStepHistory::nextDurableCommandSequence($run)); + } + } + if ($reason === null) { + return null; + } + + return WorkerProtocol::json([ + 'task_id' => $taskId, 'workflow_task_attempt' => $workflowTaskAttempt, 'outcome' => 'rejected', + 'recorded' => false, 'reason' => $reason, 'unavailable' => $unavailable, + 'requested_version' => WorkerProtocol::requestVersion($request), 'minimum_protocol_version' => '1.20', + ], 409); + } + /** * @param list> $commands */ @@ -2638,7 +3077,7 @@ private function validateWorkflowTaskCommandScopes(array $commands): void } if ($this->hasCommandValue($command, 'retry_policy') - && ! in_array($type, ['schedule_activity', 'record_local_activity', 'start_child_workflow'], true) + && ! in_array($type, ['schedule_activity', 'record_local_activity', 'prepare_local_activity', 'start_child_workflow'], true) ) { $errors["commands.{$index}.retry_policy"][] = 'retry_policy is only supported for schedule_activity and start_child_workflow commands.'; @@ -2646,7 +3085,7 @@ private function validateWorkflowTaskCommandScopes(array $commands): void foreach (['start_to_close_timeout', 'schedule_to_start_timeout', 'schedule_to_close_timeout', 'heartbeat_timeout'] as $field) { if ($this->hasCommandValue($command, $field) - && ! in_array($type, ['schedule_activity', 'record_local_activity'], true) + && ! in_array($type, ['schedule_activity', 'record_local_activity', 'prepare_local_activity'], true) ) { $errors["commands.{$index}.{$field}"][] = "{$field} is only supported for schedule_activity commands."; @@ -2665,6 +3104,12 @@ private function validateWorkflowTaskCommandScopes(array $commands): void } } + if ($this->hasCommandValue($command, 'cancellation_policy') + && ! in_array($type, ['start_child_workflow', 'schedule_activity', 'prepare_local_activity'], true)) { + $errors["commands.{$index}.cancellation_policy"][] = + 'cancellation_policy is only supported for start_child_workflow, schedule_activity or prepared local Activity commands.'; + } + if ($this->hasCommandValue($command, 'non_retryable') && ! in_array($type, ['fail_workflow', 'fail_update', 'record_local_activity'], true) ) { @@ -2696,7 +3141,7 @@ private function validateWorkflowTaskCommandScopes(array $commands): void $this->validateActivityTimeoutEnvelope($command, $index, $errors); } - if ($type === 'record_local_activity') { + if (in_array($type, ['record_local_activity', 'prepare_local_activity'], true)) { $this->validateActivityTimeoutEnvelope($command, $index, $errors); } @@ -2978,7 +3423,8 @@ private function resolveWorkflowTaskCommandPayloadReferences(array $commands, st } } - $payloadFields = WorkflowCommandNormalizer::payloadEnvelopeFields()[$commandType] ?? []; + $payloadFields = $commandType === 'prepare_local_activity' ? ['arguments'] + : (WorkflowCommandNormalizer::payloadEnvelopeFields()[$commandType] ?? []); foreach ($payloadFields as $field) { if (is_string($command[$field] ?? null)) { AvroPayloadEnvelopeResolver::assertSerializedPayload($command[$field], "commands.{$index}.{$field}"); @@ -2996,6 +3442,13 @@ private function resolveWorkflowTaskCommandPayloadReferences(array $commands, st retainExternal: in_array($commandType, ['complete_workflow', 'schedule_activity'], true), ); + if ($commandType === 'prepare_local_activity') { + $commands[$index][$field] = $resolved['payload']; + $commands[$index]['payload_codec'] ??= $resolved['codec']; + + continue; + } + if ($resolved['codec'] === null) { $commands[$index][$field] = $resolved['payload']; @@ -3205,7 +3658,9 @@ public function heartbeatWorkflowTask(Request $request, string $taskId): JsonRes fn (): array|JsonResponse => DB::transaction(function () use ($request, $namespace, $taskId, $validated, $bridge): array|JsonResponse { // Ownership must remain valid until renewal commits. A // check before this lock can acknowledge a reclaimed lease. - NamespaceWorkflowScope::taskQuery($namespace)->lockForUpdate()->find($taskId); + if (! NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId) instanceof WorkflowTask) { + return $this->workflowTaskNotFound($taskId, (int) $validated['workflow_task_attempt']); + } if ($response = $this->guardWorkflowTaskOwnership( $request, $namespace, @@ -3216,7 +3671,16 @@ public function heartbeatWorkflowTask(Request $request, string $taskId): JsonRes return $response; } - return $bridge->heartbeat($taskId); + $status = $bridge->heartbeat($taskId); + if (($status['renewed'] ?? false) === true) { + $task = NamespaceWorkflowScope::task($namespace, $taskId); + if ($task?->run instanceof WorkflowRun + && ($pending = CooperativeCancellationPolicy::pending($task->run)) !== null) { + $status['cancellation_request'] = $pending; + } + } + + return $status; }), ); } catch (\Throwable $exception) { @@ -3244,6 +3708,7 @@ public function heartbeatWorkflowTask(Request $request, string $taskId): JsonRes 'run_status' => $status['run_status'], 'task_status' => $status['task_status'], 'reason' => $status['reason'], + ...(isset($status['cancellation_request']) ? ['cancellation_request' => $status['cancellation_request']] : []), ], $this->workflowOutcomeStatus($status['reason'])); } @@ -3282,11 +3747,17 @@ public function failWorkflowTask(Request $request, string $taskId): JsonResponse if ($this->workflowTaskFailureWaitsForHistory($validated['failure'])) { try { $outcome = $this->storageMutations->run( - fn (): array => $this->acknowledgeWorkflowTaskWaitingForHistory( - $namespace, - $taskId, - $validated['failure'], - ), + fn (): array|JsonResponse => DB::transaction(function () use ($request, $namespace, $taskId, $validated): array|JsonResponse { + if (! NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId) instanceof WorkflowTask) { + return $this->workflowTaskNotFound($taskId, (int) $validated['workflow_task_attempt']); + } + if ($response = $this->guardWorkflowTaskOwnership($request, $namespace, $taskId, + (int) $validated['workflow_task_attempt'], $validated['lease_owner'])) { + return $response; + } + + return $this->acknowledgeWorkflowTaskWaitingForHistory($namespace, $taskId, $validated['failure']); + }), ); } catch (\Throwable $exception) { if (! BackendLockPressure::is($exception)) { @@ -3296,6 +3767,10 @@ public function failWorkflowTask(Request $request, string $taskId): JsonResponse return BackendLockPressure::workerOperationResponse($request, false); } + if ($outcome instanceof JsonResponse) { + return $outcome; + } + return WorkerProtocol::json([ 'task_id' => $taskId, 'workflow_task_attempt' => (int) $validated['workflow_task_attempt'], @@ -3311,16 +3786,26 @@ public function failWorkflowTask(Request $request, string $taskId): JsonResponse try { $outcome = $this->storageMutations->run(function () use ( $bridge, + $request, $namespace, $taskId, $validated, - ): array { + ): array|JsonResponse { return DB::transaction(function () use ( $bridge, + $request, $namespace, $taskId, $validated, - ): array { + ): array|JsonResponse { + if (! NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId) instanceof WorkflowTask) { + return $this->workflowTaskNotFound($taskId, (int) $validated['workflow_task_attempt']); + } + if ($response = $this->guardWorkflowTaskOwnership($request, $namespace, $taskId, + (int) $validated['workflow_task_attempt'], $validated['lease_owner'])) { + return $response; + } + $outcome = $bridge->fail($taskId, $validated['failure']); $nextTaskId = is_string($outcome['next_task_id'] ?? null) ? $outcome['next_task_id'] @@ -3359,6 +3844,10 @@ public function failWorkflowTask(Request $request, string $taskId): JsonResponse return BackendLockPressure::workerOperationResponse($request, false); } + if ($outcome instanceof JsonResponse) { + return $outcome; + } + $nextTaskId = is_string($outcome['next_task_id'] ?? null) ? $outcome['next_task_id'] : null; @@ -3452,11 +3941,7 @@ private function acknowledgeWorkflowTaskWaitingForHistory(string $namespace, str &$createdTaskIds, ): array { /** @var WorkflowTask|null $task */ - $task = WorkflowTask::query() - ->lockForUpdate() - ->whereKey($taskId) - ->where('namespace', $namespace) - ->first(); + $task = NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId); if (! $task instanceof WorkflowTask) { return ['recorded' => false, 'reason' => 'task_not_found', 'next_task_id' => null]; @@ -3576,11 +4061,7 @@ private function recordWorkflowTaskFailureIdentity( ): void { DB::transaction(function () use ($namespace, $taskId, $failure, $replayBlocked): void { /** @var WorkflowTask|null $task */ - $task = WorkflowTask::query() - ->lockForUpdate() - ->whereKey($taskId) - ->where('namespace', $namespace) - ->first(); + $task = NamespaceWorkflowScope::lockTaskForMutation($namespace, $taskId); if (! $task instanceof WorkflowTask || $task->task_type !== TaskType::Workflow @@ -3623,11 +4104,7 @@ private function createRetryWorkflowTask(string $namespace, string $failedTaskId { return DB::transaction(function () use ($namespace, $failedTaskId): ?string { /** @var WorkflowTask|null $failedTask */ - $failedTask = WorkflowTask::query() - ->lockForUpdate() - ->whereKey($failedTaskId) - ->where('namespace', $namespace) - ->first(); + $failedTask = NamespaceWorkflowScope::lockTaskForMutation($namespace, $failedTaskId); if (! $failedTask instanceof WorkflowTask || $failedTask->task_type !== TaskType::Workflow @@ -4114,22 +4591,22 @@ private function formatTaskHistoryPagination(?array $task): ?array private static function encodeHistoryPageToken(int $sequence): string { - return base64_encode((string) $sequence); + return WorkflowHistoryPageToken::encode($sequence); } private static function decodeHistoryPageToken(?string $token): ?int { - if (! is_string($token) || trim($token) === '') { - return null; - } - - $decoded = base64_decode($token, true); - - if (! is_string($decoded) || ! ctype_digit($decoded)) { - return null; - } + return WorkflowHistoryPageToken::decode($token); + } - return (int) $decoded; + private function workflowTaskNotFound(string $taskId, int $workflowTaskAttempt): JsonResponse + { + return WorkerProtocol::json([ + 'task_id' => $taskId, + 'workflow_task_attempt' => $workflowTaskAttempt, + 'error' => 'Workflow task not found.', + 'reason' => 'task_not_found', + ], 404); } /** @@ -4192,12 +4669,7 @@ private function guardWorkflowTaskOwnership( // Convert package-level outcomes to HTTP responses return match ($result['reason']) { - 'task_not_found' => WorkerProtocol::json([ - 'task_id' => $taskId, - 'workflow_task_attempt' => $workflowTaskAttempt, - 'error' => 'Workflow task not found.', - 'reason' => 'task_not_found', - ], 404), + 'task_not_found' => $this->workflowTaskNotFound($taskId, $workflowTaskAttempt), 'task_not_leased' => WorkerProtocol::json([ 'task_id' => $taskId, diff --git a/app/Http/Middleware/EnforceStorageAdmission.php b/app/Http/Middleware/EnforceStorageAdmission.php index 4e9baf79..f59a6c0d 100644 --- a/app/Http/Middleware/EnforceStorageAdmission.php +++ b/app/Http/Middleware/EnforceStorageAdmission.php @@ -17,6 +17,18 @@ final class EnforceStorageAdmission 'WorkerController@heartbeat', 'WorkerController@heartbeatWorkflowTask', 'WorkerController@completeWorkflowTask', + 'WorkerController@checkpointLocalActivityPrefix', + 'WorkerController@checkpointLocalActivityGroup', + 'WorkerController@checkpointCancellationScopePrefix', + 'CancellationScopeController@open', + 'CancellationScopeController@prepare', + 'CancellationScopeController@deliver', + 'PreparedLocalActivityController@prepare', + 'PreparedLocalActivityController@recover', + 'PreparedLocalActivityController@control', + 'PreparedLocalActivityController@heartbeat', + 'PreparedLocalActivityController@outcome', + 'PreparedLocalActivityController@acknowledgeCancellation', 'WorkerController@failWorkflowTask', 'WorkerController@completeQueryTask', 'WorkerController@failQueryTask', @@ -25,6 +37,7 @@ final class EnforceStorageAdmission 'ActivityTaskController@complete', 'ActivityTaskController@fail', 'ActivityTaskController@heartbeat', + 'ActivityTaskController@acknowledgeCancellation', 'WorkerSessionController@heartbeat', ]; @@ -58,6 +71,7 @@ public function handle(Request $request, Closure $next): Response } if ($snapshot['state'] === 'normal' || $request->isMethodSafe() || $action === 'WorkerController@workflowTaskHistory' + || $action === 'ActivityTaskController@status' || ($snapshot['state'] === 'draining' && in_array($action, self::DRAIN_ACTIONS, true))) { return $next($request); } diff --git a/app/Http/Middleware/RuntimeExternalPayloadTransport.php b/app/Http/Middleware/RuntimeExternalPayloadTransport.php index 1c7d017d..5e237f63 100644 --- a/app/Http/Middleware/RuntimeExternalPayloadTransport.php +++ b/app/Http/Middleware/RuntimeExternalPayloadTransport.php @@ -27,6 +27,30 @@ class RuntimeExternalPayloadTransport 'ScheduleController@update' => [['action', 'input']], 'ServiceCatalogController@executeOperation' => [['arguments']], 'WorkerController@completeQueryTask' => [['result_envelope']], + 'PreparedLocalActivityController@prepare' => [['descriptor', 'arguments']], + 'PreparedLocalActivityController@recover' => [['descriptor', 'arguments']], + 'PreparedLocalActivityController@outcome' => [['report', 'result']], + 'WorkerController@checkpointLocalActivityPrefix' => [ + ['commands', '*', 'arguments'], + ['commands', '*', 'entries'], + ['commands', '*', 'exception', 'details'], + ['commands', '*', 'request_payload'], + ['commands', '*', 'result'], + ], + 'WorkerController@checkpointLocalActivityGroup' => [ + ['commands', '*', 'arguments'], + ['commands', '*', 'entries'], + ['commands', '*', 'exception', 'details'], + ['commands', '*', 'request_payload'], + ['commands', '*', 'result'], + ], + 'WorkerController@checkpointCancellationScopePrefix' => [ + ['commands', '*', 'arguments'], + ['commands', '*', 'entries'], + ['commands', '*', 'exception', 'details'], + ['commands', '*', 'request_payload'], + ['commands', '*', 'result'], + ], 'WorkerController@completeWorkflowTask' => [ ['commands', '*', 'arguments'], ['commands', '*', 'entries'], @@ -45,6 +69,15 @@ class RuntimeExternalPayloadTransport /** @var array>> */ private const INCOMING_REFERENCE_PATHS = [ + 'WorkerController@checkpointLocalActivityPrefix' => [ + ['commands', '*', 'workflow_stream', 'items', '*', 'payload_reference'], + ], + 'WorkerController@checkpointLocalActivityGroup' => [ + ['commands', '*', 'workflow_stream', 'items', '*', 'payload_reference'], + ], + 'WorkerController@checkpointCancellationScopePrefix' => [ + ['commands', '*', 'workflow_stream', 'items', '*', 'payload_reference'], + ], 'WorkerController@completeWorkflowTask' => [ ['commands', '*', 'workflow_stream', 'items', '*', 'payload_reference'], ], diff --git a/app/Support/ActivityTaskPoller.php b/app/Support/ActivityTaskPoller.php index d2b7e77e..277ea853 100644 --- a/app/Support/ActivityTaskPoller.php +++ b/app/Support/ActivityTaskPoller.php @@ -46,6 +46,7 @@ public function poll( array $supportedActivityTypes = [], bool $workerSessionsAvailable = true, ?int $timeoutSeconds = null, + ?string $protocolVersion = null, ): array { $pollRequestId = $this->nonEmptyString($pollRequestId); @@ -87,6 +88,7 @@ public function poll( supportedActivityTypes: $supportedActivityTypes, workerSessionsAvailable: $workerSessionsAvailable, timeoutSeconds: $timeoutSeconds, + protocolVersion: $protocolVersion, ); } @@ -100,6 +102,7 @@ public function poll( supportedActivityTypes: $supportedActivityTypes, workerSessionsAvailable: $workerSessionsAvailable, timeoutSeconds: $timeoutSeconds, + protocolVersion: $protocolVersion, ); } @@ -117,6 +120,7 @@ private function coordinatedPoll( array $supportedActivityTypes = [], bool $workerSessionsAvailable = true, ?int $timeoutSeconds = null, + ?string $protocolVersion = null, ): array { $workerPollFence = WorkerPollFence::snapshot($worker); @@ -161,6 +165,7 @@ private function coordinatedPoll( supportedActivityTypes: $supportedActivityTypes, workerSessionsAvailable: $workerSessionsAvailable, timeoutSeconds: $timeoutSeconds, + protocolVersion: $protocolVersion, ); } @@ -380,6 +385,7 @@ private function runCoordinatedPollLeader( array $supportedActivityTypes = [], bool $workerSessionsAvailable = true, ?int $timeoutSeconds = null, + ?string $protocolVersion = null, ): array { try { $task = $this->performPoll( @@ -392,6 +398,7 @@ private function runCoordinatedPollLeader( supportedActivityTypes: $supportedActivityTypes, workerSessionsAvailable: $workerSessionsAvailable, timeoutSeconds: $timeoutSeconds, + protocolVersion: $protocolVersion, ); } catch (\Throwable $exception) { $this->pollRequests->forgetPending( @@ -432,6 +439,7 @@ private function performPoll( array $supportedActivityTypes = [], bool $workerSessionsAvailable = true, ?int $timeoutSeconds = null, + ?string $protocolVersion = null, ): array { $limit = max(10, max(1, (int) config('server.polling.max_tasks_per_poll', 1)) * 10); $nextProbeAt = null; @@ -440,7 +448,7 @@ private function performPoll( 'poll_status' => 'empty', 'next_probe_at' => null, ]; - $workerPollFence = WorkerPollFence::snapshot($worker); + $workerPollFence = CooperativeCancellationPolicy::workerSnapshot($worker, $protocolVersion); $crossKindWake = WorkerProtocol::supportsCrossKindPollWake($worker); @@ -932,6 +940,7 @@ private function claimReadyTask( $taskId, $leaseOwner, $pollRequestId, + $workerPollFence, ); return $claim; diff --git a/app/Support/ClientCompatibility.php b/app/Support/ClientCompatibility.php index 73621847..33bd6773 100644 --- a/app/Support/ClientCompatibility.php +++ b/app/Support/ClientCompatibility.php @@ -13,7 +13,7 @@ final class ClientCompatibility private const SUPPORTED_SDK_VERSIONS = [ 'php' => self::STABLE_2_X, 'python' => self::STABLE_2_X, - 'rust' => self::STABLE_2_X, + 'rust' => '>=2.0.0,<4.0.0', 'cli' => self::STABLE_2_X, ]; diff --git a/app/Support/CooperativeCancellationPolicy.php b/app/Support/CooperativeCancellationPolicy.php new file mode 100644 index 00000000..86bea5db --- /dev/null +++ b/app/Support/CooperativeCancellationPolicy.php @@ -0,0 +1,204 @@ +supportsRemoteActivityCancellationPolicies() === true; + } + + /** @return list */ + public static function remoteActivityPolicyUnavailableReasons(?WorkflowTask $task, ?string $requestVersion): array + { + $reasons = []; + if (! self::serverSupported()) { + $reasons[] = 'server_protocol'; + } + if (! WorkerProtocol::versionMeetsMinimum($requestVersion, self::MINIMUM_PROTOCOL_VERSION)) { + $reasons[] = 'request_protocol'; + } + if ($task === null || ! self::claimSupportsCancellation($task)) { + $reasons[] = 'worker_claim_capability'; + } + if (! self::remoteActivityPolicyBackendSupported()) { + $reasons[] = 'installed_runtime_activity_policy'; + } + + return $reasons; + } + + /** @return list */ + public static function childPolicyUnavailableReasons(?WorkflowTask $task, ?string $requestVersion): array + { + $reasons = []; + if (! self::serverSupported()) { + $reasons[] = 'server_protocol'; + } + if (! WorkerProtocol::versionMeetsMinimum($requestVersion, self::MINIMUM_PROTOCOL_VERSION)) { + $reasons[] = 'request_protocol'; + } + if ($task === null || ! self::claimSupportsCancellation($task)) { + $reasons[] = 'worker_claim_capability'; + } + if (! self::childPolicyBackendSupported()) { + $reasons[] = 'installed_runtime_child_policy'; + } + + return $reasons; + } + + /** @return array */ + public static function workerSnapshot(WorkerRegistration $worker, ?string $protocolVersion): array + { + return [ + ...WorkerPollFence::snapshot($worker), + 'registration_id' => $worker->getKey(), + 'protocol_version' => $protocolVersion, + 'capabilities' => is_array($worker->capabilities) ? $worker->capabilities : [], + ]; + } + + /** @param array $snapshot */ + public static function bindClaim(WorkflowTask $task, array $snapshot): void + { + $payload = is_array($task->payload) ? $task->payload : []; + $payload[self::CLAIM_KEY] = [ + ...$snapshot, + 'task_id' => $task->id, + 'run_id' => $task->workflow_run_id, + 'lease_owner' => $task->lease_owner, + 'attempt' => $task->attempt_count, + ]; + $task->forceFill(['payload' => $payload])->save(); + } + + public static function claimSupportsCancellation(WorkflowTask $task): bool + { + $claim = $task->payload[self::CLAIM_KEY] ?? null; + + return is_array($claim) + && ($claim['task_id'] ?? null) === $task->id + && ($claim['run_id'] ?? null) === $task->workflow_run_id + && ($claim['namespace'] ?? null) === $task->namespace + && ($claim['worker_id'] ?? null) === $task->lease_owner + && ($claim['lease_owner'] ?? null) === $task->lease_owner + && ($claim['attempt'] ?? null) === $task->attempt_count + && self::supports($claim['capabilities'] ?? [], $claim['protocol_version'] ?? null); + } + + /** @param list $capabilities */ + public static function supports(array $capabilities, ?string $protocolVersion): bool + { + return in_array(self::CAPABILITY, $capabilities, true) + && WorkerProtocol::versionMeetsMinimum($protocolVersion, self::MINIMUM_PROTOCOL_VERSION); + } + + /** + * Keep an undelivered request runnable when a worker commits the commands + * preceding its cancellation boundary. Call inside the fenced completion + * transaction so the prefix and its successor commit together. + */ + public static function resumeUndeliveredRequest(WorkflowTask $completedTask): ?WorkflowTask + { + if ($completedTask->status !== TaskStatus::Completed + || ! self::claimSupportsCancellation($completedTask)) { + return null; + } + + $run = WorkflowRun::query()->where('namespace', $completedTask->namespace) + ->lockForUpdate()->find($completedTask->workflow_run_id); + if (! $run instanceof WorkflowRun + || $run->status->isTerminal() + || ! is_string($run->cancellation_request_command_id) + || $run->cancellation_delivery_sequence !== null + || $run->cancellation_deadline_at === null + || ! $run->cancellation_deadline_at->isFuture()) { + return null; + } + + if (WorkflowTask::query()->where('workflow_run_id', $run->id) + ->where('task_type', TaskType::Workflow->value) + ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value])->exists()) { + return null; + } + + return WorkflowTask::query()->create([ + 'workflow_run_id' => $run->id, + 'namespace' => $run->namespace, + 'task_type' => TaskType::Workflow->value, + 'status' => TaskStatus::Ready->value, + 'available_at' => now(), + 'payload' => [ + 'resume_source_kind' => 'cancellation_request', + 'resume_source_id' => $run->cancellation_request_command_id, + 'workflow_command_id' => $run->cancellation_request_command_id, + ], + 'connection' => $run->connection, + 'queue' => $run->queue, + 'compatibility' => $run->compatibility, + ]); + } + + /** @return array|null */ + public static function pending(WorkflowRun $run): ?array + { + if (! is_string($run->cancellation_request_command_id)) { + return null; + } + + return [ + 'request_id' => $run->cancellation_request_command_id, + 'requested_at' => $run->cancellation_requested_at?->toISOString(), + 'cleanup_deadline_at' => $run->cancellation_deadline_at?->toISOString(), + 'delivery_sequence' => $run->cancellation_delivery_sequence, + 'delivered_at' => $run->cancellation_delivered_at?->toISOString(), + 'history_refresh_page_token' => WorkflowHistoryPageToken::encode(0), + ]; + } +} diff --git a/app/Support/DetachedActivityRetentionHold.php b/app/Support/DetachedActivityRetentionHold.php new file mode 100644 index 00000000..c86e92ce --- /dev/null +++ b/app/Support/DetachedActivityRetentionHold.php @@ -0,0 +1,39 @@ + $runIds */ + public static function forRuns(string $namespace, array $runIds): ?string + { + $runs = WorkflowRun::query()->where('namespace', $namespace)->whereIn('id', $runIds)->get(); + foreach ($runs as $run) { + if (method_exists(WorkflowRunRetentionCleanup::class, 'retentionHoldReason')) { + $reason = WorkflowRunRetentionCleanup::retentionHoldReason($run); + if ($reason !== null) { + return $reason; + } + + continue; + } + // A downgraded backend must preserve newer detached work and its + // payloads even when it cannot execute that policy itself. + $schedules = WorkflowHistoryEvent::query()->where('workflow_run_id', $run->id) + ->where('event_type', HistoryEventType::ActivityScheduled->value) + ->where('payload->activity->cancellation_policy', 'abandon')->get(); + if ($schedules->isNotEmpty()) { + // Restore a compatible backend to evaluate its canonical + // outcome contract. Do not parse newer history with old enums. + return 'activity_policy_backend_unsupported'; + } + } + + return null; + } +} diff --git a/app/Support/ExternalPayloadRetentionCleanup.php b/app/Support/ExternalPayloadRetentionCleanup.php index f6f3ca7e..426673bd 100644 --- a/app/Support/ExternalPayloadRetentionCleanup.php +++ b/app/Support/ExternalPayloadRetentionCleanup.php @@ -59,6 +59,10 @@ public function deleteForRuns( array_map(static fn (mixed $itemId): int => (int) $itemId, $releasedInboundStreamItemIds), static fn (int $itemId): bool => $itemId > 0, ))); + $reason = DetachedActivityRetentionHold::forRuns($namespace, $runIds); + if ($reason !== null) { + return ['found' => 0, 'deleted' => 0, 'blocked' => true, 'reason' => $reason]; + } $references = $this->referencesForRuns($namespace, $runIds); $this->collectPayloadColumn( WorkflowInboundStreamItem::query()->whereIn('id', $releasedInboundStreamItemIds), diff --git a/app/Support/HistoryRetentionEnforcer.php b/app/Support/HistoryRetentionEnforcer.php index c2497cc1..50ca7bf2 100644 --- a/app/Support/HistoryRetentionEnforcer.php +++ b/app/Support/HistoryRetentionEnforcer.php @@ -200,6 +200,11 @@ public static function pruneRun(string $namespace, string $runId): array return self::skippedRetentionResult('run_archived'); } + $reason = DetachedActivityRetentionHold::forRuns($namespace, [$runId]); + if ($reason !== null) { + return self::skippedRetentionResult($reason); + } + $messageStreams = app(MessageStreamRetentionCleanup::class); $streamPlan = $messageStreams->planForRun( $namespace, diff --git a/app/Support/NamespaceWorkflowScope.php b/app/Support/NamespaceWorkflowScope.php index e245caf7..3716ab6a 100644 --- a/app/Support/NamespaceWorkflowScope.php +++ b/app/Support/NamespaceWorkflowScope.php @@ -86,4 +86,25 @@ public static function task(string $namespace, string $taskId): ?WorkflowTask ->whereKey($taskId) ->first(); } + + /** Acquire these locks inside the caller's mutation transaction. */ + public static function lockTaskForMutation(string $namespace, string $taskId): ?WorkflowTask + { + // Repair and cancellation admission lock the run before its tasks. + // An unlocked lookup discovers the immutable run ID without holding a + // task lock while waiting for that run. Revalidate it under the lock. + $runId = self::taskQuery($namespace)->whereKey($taskId)->value('workflow_run_id'); + if (! is_string($runId)) { + return null; + } + + $run = WorkflowRun::query()->where('namespace', $namespace) + ->whereKey($runId)->lockForUpdate()->first(); + if (! $run instanceof WorkflowRun) { + return null; + } + + return self::taskQuery($namespace)->whereKey($taskId) + ->where('workflow_run_id', $runId)->lockForUpdate()->first(); + } } diff --git a/app/Support/PollRequestLeaseBinding.php b/app/Support/PollRequestLeaseBinding.php index c901dced..7cfcd5f1 100644 --- a/app/Support/PollRequestLeaseBinding.php +++ b/app/Support/PollRequestLeaseBinding.php @@ -71,11 +71,13 @@ public function ensureUnbound( } } + /** @param array $workerClaim */ public function bindClaimedTask( string $namespace, string $taskId, string $leaseOwner, ?string $pollRequestId, + array $workerClaim = [], ): void { /** @var WorkflowTask|null $task */ $task = NamespaceWorkflowScope::taskQuery($namespace) @@ -100,5 +102,9 @@ public function bindClaimedTask( $task->forceFill([ 'payload' => $payload === [] ? null : $payload, ])->save(); + + if ($workerClaim !== []) { + CooperativeCancellationPolicy::bindClaim($task, $workerClaim); + } } } diff --git a/app/Support/PreparedLocalActivityAdmissionRefused.php b/app/Support/PreparedLocalActivityAdmissionRefused.php new file mode 100644 index 00000000..a08d75e2 --- /dev/null +++ b/app/Support/PreparedLocalActivityAdmissionRefused.php @@ -0,0 +1,6 @@ + */ + public static function cancellationPolicies(): array + { + $bridge = app(WorkflowTaskBridge::class); + if (! self::serverSupported() || ! method_exists($bridge, 'supportedLocalActivityCancellationPolicies') + || ! is_callable([$bridge, 'supportedLocalActivityCancellationPolicies'])) { + return []; + } + + $policies = $bridge->supportedLocalActivityCancellationPolicies(); + + return is_array($policies) + ? array_values(array_filter(['try_cancel', 'wait_cancellation_completed'], static fn (string $policy): bool => in_array($policy, $policies, true))) + : []; + } + + /** @param list $capabilities */ + public static function supportsCancellationPolicies(array $capabilities, ?string $protocolVersion): bool + { + return CooperativeCancellationPolicy::supports($capabilities, $protocolVersion) + && in_array(self::CAPABILITY, $capabilities, true) + && in_array(self::CANCELLATION_POLICIES_CAPABILITY, $capabilities, true); + } + + /** @return list */ + public static function cancellationPolicyUnavailableReasons(?WorkflowTask $task, mixed $policy, ?string $requestVersion): array + { + $reasons = self::unavailableReasons($requestVersion); + if (! in_array($policy, ['try_cancel', 'wait_cancellation_completed'], true)) { + $reasons[] = 'local_activity_cancellation_policy'; + } elseif (! in_array($policy, self::cancellationPolicies(), true)) { + $reasons[] = 'installed_runtime_local_activity_cancellation_policy'; + } + $claim = $task instanceof WorkflowTask ? self::currentClaim($task) : null; + if ($claim === null || ! self::supportsCancellationPolicies($claim['capabilities'] ?? [], $claim['protocol_version'] ?? null)) { + $reasons[] = 'worker_claim_cancellation_policy_capability'; + } + + return $reasons; + } + + /** + * Canonical admission determines replay compatibility. Registration cannot + * upgrade the authority of an existing claim, and a legacy worker cannot + * reinterpret an explicitly authored policy during takeover. + * + * @param list $capabilities + */ + public static function canReplayRun(string $runId, array $capabilities, ?string $protocolVersion): bool + { + $policies = self::cancellationPolicies(); + $workerSupported = self::supportsCancellationPolicies($capabilities, $protocolVersion); + if ($workerSupported && count($policies) === 2) { + return true; + } + $events = WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::ActivityScheduled) + ->where('payload->execution_mode', 'local') + ->whereNotNull('payload->activity->cancellation_policy'); + if ($workerSupported && $policies !== []) { + $events->whereNotIn('payload->activity->cancellation_policy', $policies); + } + + return ! $events->exists(); + } + + /** @return list */ + public static function groupUnavailableReasons(?string $requestVersion): array + { + $reasons = self::unavailableReasons($requestVersion); + if (! interface_exists(PreparedLocalActivityGroupTaskBridge::class) + || ! app(WorkflowTaskBridge::class) instanceof PreparedLocalActivityGroupTaskBridge) { + $reasons[] = 'installed_runtime_prepared_local_activity_groups'; + } + + return $reasons; + } + + /** @return list */ + public static function unavailableReasons(?string $requestVersion): array + { + $reasons = []; + if (! CooperativeCancellationPolicy::serverSupported()) { + $reasons[] = 'server_protocol'; + } + if (! WorkerProtocol::versionMeetsMinimum($requestVersion, self::MINIMUM_PROTOCOL_VERSION)) { + $reasons[] = 'request_protocol'; + } + if (! self::backendSupported()) { + $reasons[] = 'installed_runtime_prepared_local_activity'; + } + + return $reasons; + } + + /** @return array|null */ + public static function currentClaim(WorkflowTask $task): ?array + { + $claim = $task->payload[self::CLAIM_KEY] ?? null; + if (! is_array($claim) || ! CooperativeCancellationPolicy::claimSupportsCancellation($task) + || ! in_array(self::CAPABILITY, $claim['capabilities'] ?? [], true)) { + return null; + } + + return $claim; + } + + /** @return array|null */ + public static function currentGroupClaim(WorkflowTask $task): ?array + { + $claim = self::currentClaim($task); + + return $claim !== null && in_array(self::GROUP_CAPABILITY, $claim['capabilities'] ?? [], true) ? $claim : null; + } + + /** + * Persist only after Native preparation succeeds, in the same transaction. + * Native's attempt/execution/run/task locks must already be held. This + * receipt is issued authority, not a second lease or attempt tracker. + * + * @param array $claim + */ + public static function remember(WorkflowTask $task, ActivityAttempt $attempt, array $claim): bool + { + if (self::currentClaim($task) !== $claim || $attempt->workflow_task_id !== $task->id + || $attempt->workflow_run_id !== $task->workflow_run_id + || $attempt->lease_owner !== ($claim['lease_owner'] ?? null) + || ! is_string($attempt->worker_attempt_id) || $attempt->worker_attempt_id === '') { + return false; + } + $payload = is_array($task->payload) ? $task->payload : []; + $receipts = $payload[self::LOCAL_CLAIMS_KEY] ?? []; + if (! is_array($receipts)) { + return false; + } + $receipt = [ + 'schema' => 'durable-workflow.server.prepared-local-authority/v1', + 'claim' => $claim, + 'activity_execution_id' => $attempt->activity_execution_id, + 'activity_attempt_id' => $attempt->id, + 'worker_attempt_id' => $attempt->worker_attempt_id, + ]; + if (array_key_exists($attempt->id, $receipts)) { + // JSON objects may be reordered by MySQL. The validated original + // claim and current claim are both read from the same task payload. + return self::originalClaim($task, $attempt, $claim['lease_owner'], $claim['attempt']) === $claim; + } + $receipts[$attempt->id] = $receipt; + $payload[self::LOCAL_CLAIMS_KEY] = $receipts; + $task->forceFill(['payload' => $payload])->save(); + + return true; + } + + /** + * Takeover may replace the current claim. Stop observation and receipts + * still use the immutable claim that admitted this particular callback. + * Native separately validates its canonical Started event and authority. + * + * @return array|null + */ + public static function originalClaim( + WorkflowTask $task, + ActivityAttempt $attempt, + string $leaseOwner, + int $workflowTaskAttempt, + ): ?array { + $receipt = $task->payload[self::LOCAL_CLAIMS_KEY][$attempt->id] ?? null; + $claim = is_array($receipt) ? ($receipt['claim'] ?? null) : null; + if (! is_array($claim) + || ($receipt['schema'] ?? null) !== 'durable-workflow.server.prepared-local-authority/v1' + || ($receipt['activity_execution_id'] ?? null) !== $attempt->activity_execution_id + || ($receipt['activity_attempt_id'] ?? null) !== $attempt->id + || ($receipt['worker_attempt_id'] ?? null) !== $attempt->worker_attempt_id + || $attempt->workflow_task_id !== $task->id || $attempt->workflow_run_id !== $task->workflow_run_id + || $attempt->lease_owner !== $leaseOwner || $workflowTaskAttempt < 1 + || ($claim['task_id'] ?? null) !== $task->id || ($claim['run_id'] ?? null) !== $task->workflow_run_id + || ($claim['namespace'] ?? null) !== $task->namespace + || ($claim['worker_id'] ?? null) !== $leaseOwner || ($claim['lease_owner'] ?? null) !== $leaseOwner + || ($claim['attempt'] ?? null) !== $workflowTaskAttempt + || ! CooperativeCancellationPolicy::supports($claim['capabilities'] ?? [], $claim['protocol_version'] ?? null) + || ! in_array(self::CAPABILITY, $claim['capabilities'] ?? [], true)) { + return null; + } + + return $claim; + } +} diff --git a/app/Support/RuntimeExternalPayloadReference.php b/app/Support/RuntimeExternalPayloadReference.php index 91931776..01c4c3a5 100644 --- a/app/Support/RuntimeExternalPayloadReference.php +++ b/app/Support/RuntimeExternalPayloadReference.php @@ -3,6 +3,8 @@ namespace App\Support; use InvalidArgumentException; +use Workflow\V2\Contracts\CancellationScopeTaskBridge; +use Workflow\V2\Contracts\WorkflowTaskBridge; final class RuntimeExternalPayloadReference { @@ -74,6 +76,11 @@ public static function transportManifest(): array 'idempotency' => 'content_addressed_per_namespace', 'completion_context' => [ 'schema' => RuntimePayloadCompletionContext::SCHEMA, + 'prepared_schema' => PreparedLocalActivityPolicy::serverSupported() + ? RuntimePayloadCompletionContext::PREPARED_SCHEMA : null, + 'scope_schema' => CooperativeCancellationPolicy::serverSupported() + && app(WorkflowTaskBridge::class) instanceof CancellationScopeTaskBridge + ? RuntimePayloadCompletionContext::PREPARED_SCHEMA : null, 'header' => RuntimePayloadCompletionContext::HEADER, 'use' => 'retry_draining_refusal_for_current_worker_completion_only', 'max_bytes_per_lease' => RuntimePayloadCompletionUploads::maxBytes(), diff --git a/app/Support/RuntimePayloadCompletionContext.php b/app/Support/RuntimePayloadCompletionContext.php index a38a0de8..80d678c1 100644 --- a/app/Support/RuntimePayloadCompletionContext.php +++ b/app/Support/RuntimePayloadCompletionContext.php @@ -10,6 +10,8 @@ public const SCHEMA = 'durable-workflow.v2.payload-completion-context.v1'; + public const PREPARED_SCHEMA = 'durable-workflow.v2.payload-completion-context.v2'; + /** @param list $slot */ private function __construct( public string $kind, @@ -18,6 +20,10 @@ private function __construct( public string $leaseOwner, public string $operation, public array $slot, + public ?int $sequence = null, + public ?string $activityAttemptId = null, + public string $schema = self::SCHEMA, + public ?string $checkpointId = null, ) {} public static function parse(string $header): self @@ -31,21 +37,41 @@ public static function parse(string $header): self throw self::invalid(); } $keys = ['schema', 'kind', 'task_id', 'attempt', 'lease_owner', 'operation', 'slot']; + $prepared = is_array($value) && ($value['schema'] ?? null) === self::PREPARED_SCHEMA; + if ($prepared) { + $keys[] = match ($value['operation'] ?? null) { + 'local_activity_outcome' => 'activity_attempt_id', + 'local_activity_checkpoint', 'local_activity_group_checkpoint', 'cancellation_scope_checkpoint' => 'checkpoint_id', + default => 'sequence', + }; + } if (! is_array($value) || count($value) !== count($keys) || array_diff($keys, array_keys($value)) !== [] - || ($value['schema'] ?? null) !== self::SCHEMA + || ! in_array($value['schema'] ?? null, [self::SCHEMA, self::PREPARED_SCHEMA], true) || ! in_array($value['kind'] ?? null, ['activity', 'workflow', 'query'], true) || ! is_string($value['operation']) || ! self::identifier($value['task_id']) || ! self::identifier($value['lease_owner']) || ! is_array($value['slot']) || ! array_is_list($value['slot'])) { throw self::invalid(); } + if ($prepared && ($value['kind'] !== 'workflow' || ! match ($value['operation'] ?? null) { + 'local_activity_outcome' => self::identifier($value['activity_attempt_id']), + 'local_activity_checkpoint', 'local_activity_group_checkpoint', 'cancellation_scope_checkpoint' => self::identifier($value['checkpoint_id']), + default => is_int($value['sequence']) && $value['sequence'] > 0, + })) { + throw self::invalid(); + } if ($value['kind'] === 'activity' ? ! self::identifier($value['attempt']) : (! is_int($value['attempt']) || $value['attempt'] < 1)) { throw self::invalid(); } - $validSlot = match ($value['kind'].'.'.($value['operation'] ?? '')) { + $validSlot = $prepared ? match ($value['operation']) { + 'local_activity_checkpoint', 'local_activity_group_checkpoint', 'cancellation_scope_checkpoint' => self::workflowSlot($value['slot']), + 'local_activity_prepare', 'local_activity_recover' => $value['slot'] === ['descriptor', 'arguments'], + 'local_activity_outcome' => $value['slot'] === ['report', 'result'], + default => false, + } : match ($value['kind'].'.'.($value['operation'] ?? '')) { 'activity.complete' => $value['slot'] === ['result'], 'activity.fail' => $value['slot'] === ['failure', 'details'], 'query.complete' => $value['slot'] === ['result_envelope'], @@ -57,7 +83,8 @@ public static function parse(string $header): self } return new self($value['kind'], $value['task_id'], $value['attempt'], - $value['lease_owner'], $value['operation'], $value['slot']); + $value['lease_owner'], $value['operation'], $value['slot'], + $value['sequence'] ?? null, $value['activity_attempt_id'] ?? null, $value['schema'], $value['checkpoint_id'] ?? null); } public function scope(string $namespace): string @@ -69,14 +96,22 @@ public function scope(string $namespace): string public function slotIdentity(): string { - return hash('sha256', json_encode([$this->operation, $this->slot], JSON_THROW_ON_ERROR)); + $identity = [$this->operation, $this->slot]; + if ($this->schema === self::PREPARED_SCHEMA) { + $identity[] = $this->checkpointId ?? $this->sequence ?? $this->activityAttemptId; + } + + return hash('sha256', json_encode($identity, JSON_THROW_ON_ERROR)); } public function toArray(): array { - return ['schema' => self::SCHEMA, 'kind' => $this->kind, 'task_id' => $this->taskId, + return ['schema' => $this->schema, 'kind' => $this->kind, 'task_id' => $this->taskId, 'attempt' => $this->attempt, 'lease_owner' => $this->leaseOwner, - 'operation' => $this->operation, 'slot' => $this->slot]; + 'operation' => $this->operation, 'slot' => $this->slot, + ...($this->sequence === null ? [] : ['sequence' => $this->sequence]), + ...($this->activityAttemptId === null ? [] : ['activity_attempt_id' => $this->activityAttemptId]), + ...($this->checkpointId === null ? [] : ['checkpoint_id' => $this->checkpointId])]; } private static function identifier(mixed $value): bool diff --git a/app/Support/RuntimePayloadCompletionLease.php b/app/Support/RuntimePayloadCompletionLease.php index 27022196..c821fd5b 100644 --- a/app/Support/RuntimePayloadCompletionLease.php +++ b/app/Support/RuntimePayloadCompletionLease.php @@ -5,7 +5,12 @@ use App\Models\WorkerRegistration; use Carbon\CarbonImmutable; use Workflow\V2\Contracts\ActivityTaskBridge; +use Workflow\V2\Contracts\CancellationScopeTaskBridge; +use Workflow\V2\Contracts\PreparedLocalActivityTaskBridge; +use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Enums\TaskType; +use Workflow\V2\Models\ActivityAttempt; +use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Support\WorkflowTaskOwnership; final class RuntimePayloadCompletionLease @@ -102,6 +107,81 @@ private function workflow(string $namespace, RuntimePayloadCompletionContext $co throw self::rejected(); } + if ($context->schema === RuntimePayloadCompletionContext::PREPARED_SCHEMA) { + $task = $guard['task']; + if ($context->operation === 'cancellation_scope_checkpoint') { + if (! CooperativeCancellationPolicy::serverSupported() + || ! app(WorkflowTaskBridge::class) instanceof CancellationScopeTaskBridge + || ! CooperativeCancellationPolicy::claimSupportsCancellation($task)) { + throw self::rejected(); + } + $claim = null; + } else { + $claim = PreparedLocalActivityPolicy::currentClaim($task); + if (! PreparedLocalActivityPolicy::serverSupported() || $claim === null || ! WorkerPollFence::isCurrent($claim)) { + throw self::rejected(); + } + } + if ($context->operation === 'local_activity_group_checkpoint' + && (! PreparedLocalActivityPolicy::groupsSupported() || PreparedLocalActivityPolicy::currentGroupClaim($task) === null)) { + throw self::rejected(); + } + $expires = [$guard['status']['lease_expires_at'] ?? null]; + $run = WorkflowRun::query()->find($task->workflow_run_id); + if (! $run instanceof WorkflowRun) { + throw self::rejected(); + } + if ($run->cancellation_request_command_id !== null) { + if ($run->cancellation_deadline_at === null) { + throw self::rejected(); + } + $expires[] = $run->cancellation_deadline_at->toISOString(); + } + if ($context->operation === 'local_activity_outcome') { + $attempt = ActivityAttempt::query()->where('workflow_task_id', $task->id)->find($context->activityAttemptId); + if (! $attempt instanceof ActivityAttempt + || PreparedLocalActivityPolicy::originalClaim($task, $attempt, $context->leaseOwner, $context->attempt) !== $claim) { + throw self::rejected(); + } + /** @var PreparedLocalActivityTaskBridge $bridge */ + $bridge = app(WorkflowTaskBridge::class); + $control = $bridge->controlLocalActivity($attempt->id, $context->leaseOwner, $context->attempt, false, + PreparedLocalActivityPolicy::MINIMUM_PROTOCOL_VERSION); + if (($control['active'] ?? null) !== true + || ($control['activity_attempt_id'] ?? null) !== $attempt->id + || ($control['workflow_task_id'] ?? null) !== $task->id + || ($control['workflow_task_attempt'] ?? null) !== $context->attempt + || ($control['lease_owner'] ?? null) !== $context->leaseOwner + || ($control['renewed'] ?? null) !== false) { + throw self::rejected(); + } + foreach (['lease_expires_at', 'workflow_lease_expires_at', 'start_to_close_deadline_at', + 'schedule_to_close_deadline_at', 'heartbeat_deadline_at'] as $field) { + if (! array_key_exists($field, $control) + || (str_contains($field, 'lease_expires_at') && (! is_string($control[$field]) || $control[$field] === ''))) { + throw self::rejected(); + } + if (($control[$field] ?? null) !== null) { + $expires[] = $control[$field]; + } + } + } + $earliest = null; + foreach ($expires as $expiry) { + if (! is_string($expiry) || $expiry === '') { + throw self::rejected(); + } + try { + $deadline = CarbonImmutable::parse($expiry); + } catch (\Exception) { + throw self::rejected(); + } + $earliest = $earliest === null || $deadline->lt($earliest) ? $deadline : $earliest; + } + + return $earliest?->toISOString(); + } + return $guard['status']['lease_expires_at'] ?? null; } diff --git a/app/Support/WorkerPollFence.php b/app/Support/WorkerPollFence.php index 6e5a6b7d..4d3c3daf 100644 --- a/app/Support/WorkerPollFence.php +++ b/app/Support/WorkerPollFence.php @@ -92,6 +92,11 @@ private static function matchesCurrent(array $snapshot, bool $lockForUpdate): bo : []; if ($processIdentity !== [] || $currentProcessIdentity !== []) { + // MySQL may reorder keys when a claim is persisted as JSON. + // Preserve strict field/value checks without depending on order. + ksort($processIdentity); + ksort($currentProcessIdentity); + return $processIdentity === $currentProcessIdentity; } diff --git a/app/Support/WorkerProtocol.php b/app/Support/WorkerProtocol.php index 9f67d032..ac73109a 100644 --- a/app/Support/WorkerProtocol.php +++ b/app/Support/WorkerProtocol.php @@ -19,7 +19,7 @@ class WorkerProtocol * here. WorkflowPackageApiFloor asserts the installed package still * provides the companion protocol helpers for this version. */ - public const VERSION = '1.19'; + public const VERSION = '1.20'; public const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION = '1.18'; @@ -413,6 +413,8 @@ public static function workflowMemoUpdateSemantics(): array * query_task_timeouts: array{control_plane_timeout_seconds: int, lease_timeout_seconds: int, lease_grace_seconds: int}, * activity_retry_policy: bool, * activity_timeouts: bool, + * cooperative_cancellation: bool, + * activity_cancellation_acknowledgement: bool, * local_activities: array, * worker_session_verbs: list, * worker_sessions: array, @@ -544,6 +546,12 @@ public static function serverCapabilities(): array ], 'activity_retry_policy' => true, 'activity_timeouts' => true, + 'cooperative_cancellation' => CooperativeCancellationPolicy::serverSupported(), + 'prepared_local_activities' => PreparedLocalActivityPolicy::serverSupported(), + 'prepared_local_activity_groups' => PreparedLocalActivityPolicy::groupsSupported(), + 'prepared_local_activity_cancellation_policies' => PreparedLocalActivityPolicy::cancellationPolicies(), + 'activity_cancellation_acknowledgement' => CooperativeCancellationPolicy::serverSupported() + && CooperativeCancellationPolicy::activityAcknowledgementBackendSupported(), 'local_activities' => [ ...WorkerProtocolVersion::localActivitySemantics(), 'supported' => $portableWorkerAffinitySupported, diff --git a/app/Support/WorkflowHistoryPageToken.php b/app/Support/WorkflowHistoryPageToken.php new file mode 100644 index 00000000..dc2b0a06 --- /dev/null +++ b/app/Support/WorkflowHistoryPageToken.php @@ -0,0 +1,24 @@ + $recentFailures, 'latest_workflow_task_failure' => $latestWorkflowTaskFailure, 'compatibility' => $this->compatibility($namespace, $run, $summary, $taskQueue), + 'cancellation_cascade_supported' => class_exists(CancellationCascadeView::class), + 'cancellation_cascade' => class_exists(CancellationCascadeView::class) && $run->namespace === $namespace + ? CancellationCascadeView::forRun($run) + : null, ]; $payload['findings'] = $this->findings($payload); @@ -884,10 +889,13 @@ private function findings(array $payload): array } if ((bool) data_get($payload, 'execution.task_problem', false)) { + $historical = data_get($payload, 'execution.task_problem_badge.code') === 'history'; $findings[] = [ - 'severity' => 'warning', - 'code' => 'task_problem', - 'message' => 'The run summary has a task problem flag.', + 'severity' => $historical ? 'info' : 'warning', + 'code' => $historical ? 'task_recovery_history' : 'task_problem', + 'message' => $historical + ? 'The run previously needed workflow-task repair or replay recovery.' + : 'The run summary has a task problem flag.', ]; } diff --git a/app/Support/WorkflowTaskPoller.php b/app/Support/WorkflowTaskPoller.php index 3152bcb5..427e00dd 100644 --- a/app/Support/WorkflowTaskPoller.php +++ b/app/Support/WorkflowTaskPoller.php @@ -458,10 +458,10 @@ private function performPoll( 'poll_status' => 'empty', 'next_probe_at' => null, ]; - $workerPollFence = [ - ...WorkerPollFence::snapshot($worker), - 'protocol_version' => WorkerProtocol::requestVersion($request), - ]; + $workerPollFence = CooperativeCancellationPolicy::workerSnapshot( + $worker, + WorkerProtocol::requestVersion($request), + ); $supportsQueryTasks = in_array('workflow', $taskKinds, true) && $this->cache->available() && $this->queryTasks->workerSupportsQueryTasks($namespace, $worker); @@ -1070,6 +1070,16 @@ private function claimReadyTask( return ['claimed' => false, 'reason' => 'stale_worker_registration']; } + // Request admission takes the same run lock before inspecting + // leases. Either it observes this claim's immutable proof, or + // this claim observes the newly accepted request. + $run = WorkflowRun::query()->where('namespace', $namespace) + ->lockForUpdate()->find($runId); + + if (! $run instanceof WorkflowRun) { + return ['claimed' => false, 'reason' => 'run_not_found']; + } + if (! $this->workerCanReplayRun( $namespace, $leaseOwner, @@ -1096,6 +1106,7 @@ private function claimReadyTask( $taskId, $leaseOwner, $pollRequestId, + $workerPollFence, ); } @@ -1912,6 +1923,11 @@ private function refreshCachedTaskPayload(string $namespace, ?array $task): ?arr $payload['lease_expires_at'] = $workflowTask->lease_expires_at?->toJSON() ?? ($payload['lease_expires_at'] ?? null); + if ($workflowTask->run instanceof WorkflowRun + && ($pending = CooperativeCancellationPolicy::pending($workflowTask->run)) !== null) { + $payload['cancellation_request'] = $pending; + } + return $payload; } @@ -2062,6 +2078,11 @@ private function taskPayload( $payload = array_merge($payload, $this->workflowTaskResumeContext($namespace, (string) $claim['task_id'])); + $run = WorkflowRun::query()->where('namespace', $namespace)->find($claim['workflow_run_id']); + if ($run instanceof WorkflowRun && ($pending = CooperativeCancellationPolicy::pending($run)) !== null) { + $payload['cancellation_request'] = $pending; + } + // Include pagination metadata when history was fetched via // historyPayloadPaginated() so the controller can build page tokens. if (array_key_exists('has_more', $history)) { @@ -2312,11 +2333,18 @@ private function workerCanReplayRun( )) : []; - return WorkflowMetadataCapabilityPolicy::canReplayRun( - $runId, - $capabilities, - $protocolVersion, - ); + $run = WorkflowRun::query()->where('namespace', $namespace)->find($runId); + if ($run instanceof WorkflowRun && is_string($run->cancellation_request_command_id) + && ! CooperativeCancellationPolicy::supports($capabilities, $protocolVersion)) { + return false; + } + + return PreparedLocalActivityPolicy::canReplayRun($runId, $capabilities, $protocolVersion) + && WorkflowMetadataCapabilityPolicy::canReplayRun( + $runId, + $capabilities, + $protocolVersion, + ); } /** @param array $workflowDefinitionFingerprints */ diff --git a/composer.json b/composer.json index 5d1184bc..1b80a60a 100644 --- a/composer.json +++ b/composer.json @@ -6,7 +6,7 @@ "require": { "php": "^8.2", "apache/avro": "^1.12", - "durable-workflow/workflow": "2.3.4", + "durable-workflow/workflow": "2.4.0-rc.2", "laravel/framework": "^13.30", "laravel/tinker": "^3.0", "league/flysystem-aws-s3-v3": "^3.35.3" @@ -48,7 +48,7 @@ }, "extra": { "durable-workflow": { - "product-train": "2.4.40" + "product-train": "2.5.0-rc.1" }, "laravel": { "dont-discover": [] diff --git a/composer.lock b/composer.lock index 9b2dcdac..fe613a14 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "cf02d93da18a4a641b24c9242ac9ad9d", + "content-hash": "1305e1110d7cbb355934e8ce8cd29c8a", "packages": [ { "name": "apache/avro", @@ -655,16 +655,16 @@ }, { "name": "durable-workflow/workflow", - "version": "2.3.4", + "version": "2.4.0-rc.2", "source": { "type": "git", "url": "https://github.com/durable-workflow/workflow.git", - "reference": "e66d22482541ddcaa03964fe006538a7340083fe" + "reference": "200a3c0ffa1e90557619221458deb3516b8cf955" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/durable-workflow/workflow/zipball/e66d22482541ddcaa03964fe006538a7340083fe", - "reference": "e66d22482541ddcaa03964fe006538a7340083fe", + "url": "https://api.github.com/repos/durable-workflow/workflow/zipball/200a3c0ffa1e90557619221458deb3516b8cf955", + "reference": "200a3c0ffa1e90557619221458deb3516b8cf955", "shasum": "" }, "require": { @@ -697,7 +697,7 @@ "dev-main": "2.0.x-dev" }, "durable-workflow": { - "product-train": "2.3.4", + "product-train": "2.4.0-rc.2", "laravel-embedded-upgrade-contract": "resources/laravel-embedded-upgrade-contract.json", "laravel-dependency-security-policy": "resources/laravel-dependency-security-policy.json" } @@ -724,9 +724,9 @@ "description": "Embedded durable workflow runtime and orchestration engine for Laravel applications.", "support": { "issues": "https://github.com/durable-workflow/workflow/issues", - "source": "https://github.com/durable-workflow/workflow/tree/2.3.4" + "source": "https://github.com/durable-workflow/workflow/tree/2.4.0-rc.2" }, - "time": "2026-10-05T05:54:54+00:00" + "time": "2026-10-06T07:03:04+00:00" }, { "name": "egulias/email-validator", diff --git a/docker-compose.dedicated-matching.yml b/docker-compose.dedicated-matching.yml index 861849aa..35e30de4 100644 --- a/docker-compose.dedicated-matching.yml +++ b/docker-compose.dedicated-matching.yml @@ -32,13 +32,13 @@ name: durable-workflow-server # daemon reports `shape: dedicated`. # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.4.40}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.5.0-rc.1}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.4.40}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.5.0-rc.1}} APP_DEBUG: ${APP_DEBUG:-false} DB_CONNECTION: mysql DB_HOST: mysql diff --git a/docker-compose.memo-rolling.yml b/docker-compose.memo-rolling.yml index a8bba672..41d7b47e 100644 --- a/docker-compose.memo-rolling.yml +++ b/docker-compose.memo-rolling.yml @@ -49,14 +49,14 @@ services: command: ["server-bootstrap"] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.4.40} + APP_VERSION: ${APP_VERSION:-2.5.0-rc.1} successor: image: ${DW_MEMO_SUCCESSOR_IMAGE:-durable-workflow/server-memo-rolling:local} ports: !override [] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.4.40} + APP_VERSION: ${APP_VERSION:-2.5.0-rc.1} DW_SERVER_ID: memo-successor DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node diff --git a/docker-compose.published.yml b/docker-compose.published.yml index 202d5e3f..9e8cc46d 100644 --- a/docker-compose.published.yml +++ b/docker-compose.published.yml @@ -1,13 +1,13 @@ name: durable-workflow-server # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.4.40}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.5.0-rc.1}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.4.40}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.5.0-rc.1}} APP_DEBUG: ${APP_DEBUG:-false} LOG_CHANNEL: ${LOG_CHANNEL:-stderr} LOG_LEVEL: ${LOG_LEVEL:-info} diff --git a/docker-compose.small-cluster.yml b/docker-compose.small-cluster.yml index 196cc85d..c02d87df 100644 --- a/docker-compose.small-cluster.yml +++ b/docker-compose.small-cluster.yml @@ -12,7 +12,7 @@ x-server-build: &server-build x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: testing - APP_VERSION: ${APP_VERSION:-2.4.40} + APP_VERSION: ${APP_VERSION:-2.5.0-rc.1} APP_DEBUG: "false" DW_SERVER_KEY: ${DW_SERVER_KEY:-base64:5Zt4nUhlCm3DD0nLXZJQdHiwPfb56yGo9gNV/g3jYbY=} DB_CONNECTION: ${DW_SMALL_CLUSTER_DB:-mysql} diff --git a/docker-compose.yml b/docker-compose.yml index 1db96882..6b30d5b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node - APP_VERSION: "${APP_VERSION:-2.4.40}" + APP_VERSION: "${APP_VERSION:-2.5.0-rc.1}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -62,7 +62,7 @@ services: APP_NAME: "Durable Workflow Server" APP_ENV: local DW_SERVER_KEY: "${DW_SERVER_KEY:-}" - APP_VERSION: "${APP_VERSION:-2.4.40}" + APP_VERSION: "${APP_VERSION:-2.5.0-rc.1}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -124,7 +124,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: worker_node - APP_VERSION: "${APP_VERSION:-2.4.40}" + APP_VERSION: "${APP_VERSION:-2.5.0-rc.1}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 @@ -178,7 +178,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: scheduler_node - APP_VERSION: "${APP_VERSION:-2.4.40}" + APP_VERSION: "${APP_VERSION:-2.5.0-rc.1}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 diff --git a/docs/contracts/external-execution-surface.md b/docs/contracts/external-execution-surface.md index a12b72d5..26086d5b 100644 --- a/docs/contracts/external-execution-surface.md +++ b/docs/contracts/external-execution-surface.md @@ -77,3 +77,60 @@ Stable adjacent contract docs live in: - `docs/contracts/bridge-adapters.md` - `docs/contracts/external-task-input.md` - `docs/contracts/external-task-result.md` + +## Candidate cooperative activity stop receipts + +The cooperative cancellation draft adds remote callback-stop receipts at worker +protocol 1.20. The default remains 1.19 and the candidate contract is not frozen. +The installed Native runtime must provide the acknowledgment primitive. Discovery +advertises `server_capabilities.activity_cancellation_acknowledgement` only when +both conditions hold. + +`POST /api/worker/activity-tasks/{taskId}/status` remains read-only. After canonical +cooperative cancellation it reports the original request, root identity, immutable +cleanup deadline and cancellation history event. `callback_state: unknown` means +that the worker has not reported an actual stop. A cancelled durable row or +expired lease does not make that state `stopped`. + +The original cooperative worker may post `activity_attempt_id`, `lease_owner` +and `request_id` to `/acknowledge-cancellation` after stopping and joining the +remote callback. The Server checks the namespace, task, original claim capability, +attempt and canonical cancellation snapshot in a fenced transaction. A later +registration or request protocol cannot upgrade an old claim. Duplicates return +the original receipt event and late receipt remains explicitly late. + +This report grants no renewed lease, application heartbeat, result authority or +cleanup budget. It describes the worker's stopped callback, not reversal of +external effects. Local activities require separate workflow claim authority. + +## Candidate prepared local cancellation policies + +An explicit prepared local Activity policy requires candidate protocol 1.20, +the installed bridge's `supportedLocalActivityCancellationPolicies()` list, +and `prepared_local_activity_cancellation_policies` on the original issued +workflow claim. Discovery reports the actual supported policy list in +`server_capabilities.prepared_local_activity_cancellation_policies`. Older +custom prepared bridges do not acquire this support from their optional role. +The list is empty at default protocol 1.19. + +`try_cancel` fences publication and releases the durable await without claiming +the callback has stopped. `wait_cancellation_completed` parks delivery until an +original-owner stopped-and-joined receipt or canonical callback outcome exists. +Both retain the original root request and immutable cleanup deadline. Omission +preserves historical TryCancel. Explicit null and local `abandon` are refused. +Independent work should use a remote Activity with a qualified bounded lifetime. + +The prepare, recover and atomic group endpoints reject unsupported policies +before resolving payloads or recording any sibling. Refusals identify the +requested policy, installed supported list, original worker claim, missing +capability and remediation. Re-registering a worker cannot upgrade its existing +claim. Replacement polling checks the explicit policy in canonical Scheduled +history before probing and again under the run lock. An incompatible worker +cannot reinterpret that policy through replay. + +SDK policy authoring, replay and physical callback supervision need separate +qualification. The installed backend's policy list does not establish those +SDK capabilities or change the default published protocol. +Receipt writes are admitted during storage draining and refused when storage is +fenced. SDK stop/join emission, local receipts and activity waiting policies still +need connected qualification before this candidate is published. diff --git a/docs/contracts/external-payload-storage.md b/docs/contracts/external-payload-storage.md index c3cdba3b..1b93286c 100644 --- a/docs/contracts/external-payload-storage.md +++ b/docs/contracts/external-payload-storage.md @@ -220,6 +220,41 @@ must not replace the last verified recovery point. A restored database may contain the historical hold; release it after validating the restored copy or allow its original deadline to expire. +## Payload completion while storage drains + +An existing worker claim can retry a draining upload refusal once using the +completion schema and header advertised in namespace discovery. Every slot of +the same claim shares one byte allowance and a maximum of 128 slots. New client +work remains subject to ordinary storage admission. + +The prepared local activity candidate on explicit protocol 1.20 additionally +advertises `completion_context.prepared_schema` as +`durable-workflow.v2.payload-completion-context.v2`. Its exact header object +contains `schema`, `kind: workflow`, `task_id`, the original workflow `attempt`, +`lease_owner`, `operation` and `slot`, plus one operation identity: + +| Operation | Identity | Payload slot | +| --- | --- | --- | +| `local_activity_checkpoint` | `checkpoint_id` | Ordinary command payload slot | +| `local_activity_prepare` | `sequence` | `["descriptor", "arguments"]` | +| `local_activity_recover` | `sequence` | `["descriptor", "arguments"]` | +| `local_activity_outcome` | `activity_attempt_id` | `["report", "result"]` | + +All prepared operations and legacy completion uploads share the original +workflow claim's allowance. A new sequence, checkpoint or activity attempt +does not reset it. Prepared uploads require the original issued registration, +current workflow epoch and live authority. Results additionally require the +matching admitted activity attempt and its live execution deadlines. Accepted +cancellation fences results from ordinary callbacks. Shielded cleanup keeps +the original root deadline. + +An upload does not renew workflow or activity leases, application heartbeats, +or cancellation deadlines. An exact stored reference can be read after the +claim closes without rewriting its object or extending retention. Changed +bytes still require live authority. Fenced or stale storage admission refuses +both ordinary and completion uploads. Default published protocol 1.19 keeps +its existing completion schema. + ## Typed outcomes and retryability The transport returns a stable diff --git a/docs/server-reference.md b/docs/server-reference.md index 06a7e61c..704d1caf 100644 --- a/docs/server-reference.md +++ b/docs/server-reference.md @@ -638,6 +638,31 @@ metadata by default; add `include_last_event_payload=true` to include at most a 4 KiB JSON preview. Use the history endpoints when a full replay/debug archive is needed. +The candidate cooperative-cancellation runtime adds `cancellation_cascade_supported` +and `cancellation_cascade` to both debug responses. A supported runtime returns +`null` when the selected run has no cooperative request. An older installed +runtime reports support as `false`. This read-only view does not require worker +protocol 1.20 to inspect already recorded evidence. + +The `durable-workflow.cancellation-cascade/v1` view joins the original root +request, its immutable cleanup deadline, each visible run's local request and +lifecycle, child propagation, activity fencing and matching stop receipts, +cleanup lease recovery, and the recorded terminal cleanup outcome. Selecting a +historical run keeps that run selected. Every related run is checked against +the namespace before its identity or history is returned. Activity fencing +alone does not prove that its callback stopped. Recovery records identify the +old and replacement grants without guessing why a worker lost its lease. + +The view inspects at most 20 runs, 100 relationship edges, 128 recent relevant +history events per run and 512 recent history events in total. Original request +events are loaded separately so clipping recent history does not change the +root budget. Each request text field is capped at 8192 bytes. Missing, +inaccessible, inconsistent or clipped evidence is explained in `findings` and +sets `inspection_complete` to `false`. `inspection_complete` describes the +evidence inventory, not whether cleanup succeeded. Use the history endpoints +for evidence outside these bounds. This candidate contract is not a claim +about the currently published Server image. + ### History - `GET /api/workflows/{id}/runs/{runId}/history` — Get event history - `GET /api/workflows/{id}/runs/{runId}/history/export` — Export replay bundle diff --git a/docs/worker-activity-observation.md b/docs/worker-activity-observation.md new file mode 100644 index 00000000..f29c3ba7 --- /dev/null +++ b/docs/worker-activity-observation.md @@ -0,0 +1,36 @@ +# Candidate remote activity observation + +Worker protocol1.20 adds `POST /api/worker/activity-tasks/{taskId}/status`. +This remains a source candidate. The default released protocol is1.19. + +Send the namespace and worker credential used to claim the task, the1.20 +protocol header, and the exact `activity_attempt_id` and `lease_owner` in JSON. +The response reports whether the observed attempt can continue, its stop reason, +lease expiry, user heartbeat time, execution deadlines and any required worker +session. Namespace, task, attempt and owner mismatches refuse the observation. + +Observation does not renew the activity lease, worker registration or required +session. It does not reset the application's heartbeat deadline or write +history. The activity retains the existing five-minute lease. Authored user +heartbeats retain their existing progress and renewal behavior. Required +sessions also retain their own lease and TTL. A worker must keep the separate +registration heartbeat current while it executes work. + +An expired lease, heartbeat or execution deadline refuses continuation before +the timeout scanner runs. A required session must be active, owned by the same +worker and within its lease and TTL. The read remains available under draining +or fenced storage admission. Backend lock pressure returns retryable503 with +the attempted fence and does not grant permission to continue. + +An accepted cooperative request remains pending until the workflow records its +canonical delivery. Pending acceptance alone does not cancel a remote activity. +Once delivery cancels the activity, observation returns `cancel_requested=true` +and `can_continue=false`. Terminal cancel and terminate retain their existing +behavior. Workers must stop unsafe callback execution and check the attempt +again before encoding or publication. A prior successful observation is not an +ownership reservation, and completion/failure still validate the actual fence. + +Stopping an activity process cannot undo external effects. Safe retries still +require idempotency or reconciliation. This endpoint supplies the observation +contract for connected SDK qualification and does not by itself prove callback +supervision, graceful shutdown or cold replacement in any SDK. diff --git a/k8s/README.md b/k8s/README.md index 898034bd..adad2a79 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -13,7 +13,7 @@ The checked-in manifests are synchronized with the repository's stable source release and pin its Docker Hub tag: ```text -durableworkflow/server:2.4.40 +durableworkflow/server:2.5.0-rc.1 ``` Before production use, patch every workload image to the exact published tag or @@ -21,15 +21,15 @@ digest you intend to run: ```bash kubectl set image -n durable-workflow deploy/durable-workflow-server \ - server=durableworkflow/server:2.4.40 + server=durableworkflow/server:2.5.0-rc.1 kubectl set image -n durable-workflow deploy/durable-workflow-worker \ - worker=durableworkflow/server:2.4.40 + worker=durableworkflow/server:2.5.0-rc.1 kubectl set image -n durable-workflow cronjob/durable-workflow-scheduler \ - scheduler=durableworkflow/server:2.4.40 + scheduler=durableworkflow/server:2.5.0-rc.1 ``` GitHub Container Registry publishes the same release line at -`ghcr.io/durable-workflow/server:2.4.40`. Digest pinning is preferred for strict +`ghcr.io/durable-workflow/server:2.5.0-rc.1`. Digest pinning is preferred for strict change control. The manifests expect you to provide: diff --git a/k8s/helm/durable-workflow/Chart.yaml b/k8s/helm/durable-workflow/Chart.yaml index a538308a..3eccc158 100644 --- a/k8s/helm/durable-workflow/Chart.yaml +++ b/k8s/helm/durable-workflow/Chart.yaml @@ -5,11 +5,11 @@ type: application # The chart's own semver version. Bumped on every chart release; treated as # independent of the server image version (appVersion). Breaking-change rules # for this version live in docs/helm-upgrading.md alongside the chart. -version: 0.1.136 +version: 0.1.137-rc.1 # The immutable Durable Workflow Server identity this chart release packages. # The onboarding default in values.yaml and appVersion are generated from the # checked-in source release record. -appVersion: "2.4.40" +appVersion: "2.5.0-rc.1" kubeVersion: ">=1.27.0-0" home: https://durable-workflow.github.io/docs/2.0/deployment sources: @@ -30,7 +30,7 @@ annotations: # exact commit that most recently changed the packaged chart. org.opencontainers.image.source: https://github.com/durable-workflow/server dev.durable-workflow.source-revision: "unreleased" - dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.4.40" + dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.5.0-rc.1" artifacthub.io/license: MIT artifacthub.io/category: integration-delivery # Free-form changelog for the current chart release shown by Artifact Hub. diff --git a/k8s/helm/durable-workflow/README.md b/k8s/helm/durable-workflow/README.md index 9385acd2..c7e27703 100644 --- a/k8s/helm/durable-workflow/README.md +++ b/k8s/helm/durable-workflow/README.md @@ -63,7 +63,7 @@ helm install durable-workflow ./k8s/helm/durable-workflow \ ```yaml image: - tag: "2.4.40" + tag: "2.5.0-rc.1" # Pin a digest in production: # digest: "sha256:abc123..." # memoPayloadStorage: "raw-json-v1" # Required for a digest or custom image. diff --git a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml index 1df00c27..562b31f4 100644 --- a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml @@ -1,7 +1,7 @@ # CI fixture: GitOps / externally-managed-secret path. The chart consumes # existing Secrets and renders no Secret resources of its own. image: - tag: "2.4.40" + tag: "2.5.0-rc.1" externalDatabase: connection: pgsql diff --git a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml index d80a3ac7..bd1272c2 100644 --- a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml +++ b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml @@ -1,6 +1,6 @@ # CI fixture: ingress + autoscaling enabled. Exercises optional templates. image: - tag: "2.4.40" + tag: "2.5.0-rc.1" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml index 954bec0c..5034a46f 100644 --- a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml @@ -2,7 +2,7 @@ # chart's render path is exercised end-to-end. Real deployments should use # existingSecret instead. image: - tag: "2.4.40" + tag: "2.5.0-rc.1" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/templates/_helpers.tpl b/k8s/helm/durable-workflow/templates/_helpers.tpl index 3ef03ef5..61571caf 100644 --- a/k8s/helm/durable-workflow/templates/_helpers.tpl +++ b/k8s/helm/durable-workflow/templates/_helpers.tpl @@ -88,7 +88,7 @@ resolved by an explicit capability declaration or an existing workload marker. {{- define "durable-workflow.memoPayloadStorageForImage" -}} {{- $image := toString . -}} {{- $normalized := regexReplaceAll "^index\\.docker\\.io/" $image "docker.io/" -}} -{{- if eq $normalized "docker.io/durableworkflow/server:2.4.40" -}} +{{- if eq $normalized "docker.io/durableworkflow/server:2.5.0-rc.1" -}} dual-v1 {{- else if regexMatch "^docker\\.io/durableworkflow/server:2\\.0\\.0-rc\\.[0-9]+$" $normalized -}} {{- $releaseCandidate := atoi (regexFind "[0-9]+$" $normalized) -}} diff --git a/k8s/helm/durable-workflow/values.yaml b/k8s/helm/durable-workflow/values.yaml index d8bbfd43..d7ef6d23 100644 --- a/k8s/helm/durable-workflow/values.yaml +++ b/k8s/helm/durable-workflow/values.yaml @@ -21,7 +21,7 @@ image: registry: docker.io repository: durableworkflow/server # Generated by scripts/ci/sync-source-release.mjs. Do not edit this default. - tag: "2.4.40" + tag: "2.5.0-rc.1" # Optional digest pin. When set, takes precedence over tag for change control. # Example: "sha256:abc123..." digest: "" diff --git a/k8s/helm/examples/values-dev.yaml b/k8s/helm/examples/values-dev.yaml index a6bc2170..51a2f947 100644 --- a/k8s/helm/examples/values-dev.yaml +++ b/k8s/helm/examples/values-dev.yaml @@ -3,7 +3,7 @@ # shape in production. image: - tag: "2.4.40" + tag: "2.5.0-rc.1" externalDatabase: connection: mysql diff --git a/k8s/helm/examples/values-external-secrets-operator.yaml b/k8s/helm/examples/values-external-secrets-operator.yaml index 2be8597e..e4b0863f 100644 --- a/k8s/helm/examples/values-external-secrets-operator.yaml +++ b/k8s/helm/examples/values-external-secrets-operator.yaml @@ -5,7 +5,7 @@ # concern. image: - tag: "2.4.40" + tag: "2.5.0-rc.1" externalDatabase: connection: pgsql diff --git a/k8s/helm/examples/values-production-existing-secrets.yaml b/k8s/helm/examples/values-production-existing-secrets.yaml index 0504d984..6f5305ca 100644 --- a/k8s/helm/examples/values-production-existing-secrets.yaml +++ b/k8s/helm/examples/values-production-existing-secrets.yaml @@ -10,7 +10,7 @@ image: repository: durable-workflow/server # Pin a digest in production for change-control auditability. digest: "" # e.g. "sha256:abc123..." - tag: "2.4.40" + tag: "2.5.0-rc.1" externalDatabase: connection: pgsql diff --git a/k8s/migration-job.yaml b/k8s/migration-job.yaml index ce6fd2ff..0ad2e66c 100644 --- a/k8s/migration-job.yaml +++ b/k8s/migration-job.yaml @@ -13,7 +13,7 @@ spec: restartPolicy: OnFailure containers: - name: migrate - image: durableworkflow/server:2.4.40 + image: durableworkflow/server:2.5.0-rc.1 command: ["server-entrypoint"] args: ["server-bootstrap"] envFrom: diff --git a/k8s/scheduler-cronjob.yaml b/k8s/scheduler-cronjob.yaml index 1406e115..adbabb55 100644 --- a/k8s/scheduler-cronjob.yaml +++ b/k8s/scheduler-cronjob.yaml @@ -24,7 +24,7 @@ spec: restartPolicy: Never containers: - name: scheduler - image: durableworkflow/server:2.4.40 + image: durableworkflow/server:2.5.0-rc.1 command: ["server-entrypoint"] args: - sh diff --git a/k8s/secret.yaml b/k8s/secret.yaml index e80d1202..9fc185d3 100644 --- a/k8s/secret.yaml +++ b/k8s/secret.yaml @@ -12,7 +12,7 @@ metadata: app.kubernetes.io/name: durable-workflow data: APP_NAME: "Durable Workflow Server" - APP_VERSION: "2.4.40" + APP_VERSION: "2.5.0-rc.1" APP_ENV: production APP_DEBUG: "false" DB_CONNECTION: mysql diff --git a/k8s/server-deployment.yaml b/k8s/server-deployment.yaml index f962149e..bae6b72c 100644 --- a/k8s/server-deployment.yaml +++ b/k8s/server-deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: server - image: durableworkflow/server:2.4.40 + image: durableworkflow/server:2.5.0-rc.1 ports: - containerPort: 8080 name: http diff --git a/k8s/worker-deployment.yaml b/k8s/worker-deployment.yaml index f6b9935b..6a796b4e 100644 --- a/k8s/worker-deployment.yaml +++ b/k8s/worker-deployment.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: worker - image: durableworkflow/server:2.4.40 + image: durableworkflow/server:2.5.0-rc.1 command: ["server-entrypoint"] args: ["php", "artisan", "queue:work", "--sleep=1", "--tries=3", "--max-time=3600"] envFrom: diff --git a/resources/platform-protocol-specs/control-plane-api.openapi.yaml b/resources/platform-protocol-specs/control-plane-api.openapi.yaml index ff6aee5b..8b5ae1ca 100644 --- a/resources/platform-protocol-specs/control-plane-api.openapi.yaml +++ b/resources/platform-protocol-specs/control-plane-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.control-plane-api - version: "10" + version: "11" summary: Durable Workflow control-plane HTTP+JSON API description: > Normative OpenAPI specification for the standalone Durable Workflow @@ -318,6 +318,47 @@ paths: requestBody: { $ref: "#/components/requestBodies/ReasonObject" } responses: "202": { $ref: "#/components/responses/ControlPlaneEnvelope" } + /workflows/{workflowId}/request-cancellation: + post: + tags: [workflows] + operationId: requestWorkflowCancellation + summary: Request cooperative cancellation with bounded workflow cleanup. + x-durable-workflow-minimum-worker-protocol-version: "1.20" + parameters: + - $ref: "#/components/parameters/ControlPlaneVersionHeader" + - $ref: "#/components/parameters/WorkflowIdPath" + requestBody: + required: false + content: + application/json: + schema: { $ref: "#/components/schemas/CooperativeCancellationAdmissionRequest" } + responses: + "200": { $ref: "#/components/responses/CooperativeCancellationAdmission" } + "202": { $ref: "#/components/responses/CooperativeCancellationAdmission" } + "404": { $ref: "#/components/responses/ControlPlaneError" } + "409": { $ref: "#/components/responses/ControlPlaneError" } + "422": { $ref: "#/components/responses/ControlPlaneError" } + /workflows/{workflowId}/runs/{runId}/request-cancellation: + post: + tags: [workflows] + operationId: requestWorkflowRunCancellation + summary: Request cooperative cancellation of the selected current run. + x-durable-workflow-minimum-worker-protocol-version: "1.20" + parameters: + - $ref: "#/components/parameters/ControlPlaneVersionHeader" + - $ref: "#/components/parameters/WorkflowIdPath" + - $ref: "#/components/parameters/RunIdPath" + requestBody: + required: false + content: + application/json: + schema: { $ref: "#/components/schemas/CooperativeCancellationAdmissionRequest" } + responses: + "200": { $ref: "#/components/responses/CooperativeCancellationAdmission" } + "202": { $ref: "#/components/responses/CooperativeCancellationAdmission" } + "404": { $ref: "#/components/responses/ControlPlaneError" } + "409": { $ref: "#/components/responses/ControlPlaneError" } + "422": { $ref: "#/components/responses/ControlPlaneError" } /workflows/{workflowId}/terminate: post: tags: [workflows] @@ -900,6 +941,14 @@ components: content: application/json: schema: { $ref: "#/components/schemas/JsonObject" } + CooperativeCancellationAdmission: + description: The original request and immutable cleanup deadline, including duplicate acceptance. + headers: + X-Durable-Workflow-Control-Plane-Version: + schema: { type: string, const: "2" } + content: + application/json: + schema: { $ref: "#/components/schemas/CooperativeCancellationAdmission" } ControlPlaneEnvelope: description: Successful control-plane response. headers: @@ -933,6 +982,25 @@ components: application/json: schema: { $ref: "#/components/schemas/ScheduleListError" } schemas: + CooperativeCancellationAdmissionRequest: + type: object + additionalProperties: true + properties: + reason: { type: [string, "null"], maxLength: 1000 } + cleanup_timeout_seconds: { type: [integer, "null"], minimum: 1, maximum: 3600, default: 600 } + CooperativeCancellationAdmission: + allOf: + - $ref: "#/components/schemas/ControlPlaneEnvelope" + - type: object + required: [workflow_id, run_id, accepted, duplicate, run_status, cancellation_request] + properties: + workflow_id: { type: string, minLength: 1 } + run_id: { type: string, minLength: 1 } + accepted: { type: boolean, const: true } + duplicate: { type: boolean } + run_status: { type: string } + cancellation_request: + $ref: "./worker-protocol-api.openapi.yaml#/components/schemas/CooperativeCancellationRequest" JsonObject: type: object additionalProperties: true diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 33f9e658..63cbbbef 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "23" + version: "47" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -14,15 +14,133 @@ info: from the same major whose minor is less than or equal to N. x-durable-workflow-worker-protocol-negotiation: advertised_version_path: worker_protocol.version - default_advertised_version: "1.19" + default_advertised_version: "1.20" request_header_rule: same_major_and_minor_less_than_or_equal_to_advertised - accepted_request_versions_by_default: ["1.0", "1.1", "1.2", "1.3", "1.4", "1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13", "1.14", "1.15", "1.16", "1.17", "1.18", "1.19"] + accepted_request_versions_by_default: ["1.0", "1.1", "1.2", "1.3", "1.4", "1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13", "1.14", "1.15", "1.16", "1.17", "1.18", "1.19", "1.20"] response_version: advertised_version fail_closed_on: [missing_header, malformed_version, different_major, minor_greater_than_advertised] x-durable-workflow-catalog-entry: worker_protocol_api x-durable-workflow-catalog-schema: durable-workflow.v2.platform-protocol-specs.catalog x-durable-workflow-catalog-version: 16 x-durable-workflow-evolution-rule: additive_minor_breaking_major +x-durable-workflow-cancellation-scope-admission: + minimum_protocol_version: "1.20" + status: unfrozen_candidate_membership_only + required_backend_role: CancellationScopeAdmission + scope_execution_capability: false + checks: [before_payload_resolution, under_run_lock_before_admission] + unknown_scope: { status: 409, reason: operation_scope_not_recorded, recorded: false } + unknown_local_scope: { status: 409, reason: local_activity_scope_not_recorded, recorded: false } + prepared_local_surfaces: [single_prepare, single_recover, atomic_group_checkpoint] + unsupported_backend: { status: 409, reason: cancellation_scope_membership_unavailable, recorded: false } + stream_effects: successful_admission_only_in_same_outer_transaction +x-durable-workflow-cancellation-scope-opening: + minimum_protocol_version: "1.20" + status: unfrozen_candidate_opening_only + required_backend_role: CancellationScopeTaskBridge + scope_execution_capability: false + authority: [authenticated_namespace, original_capable_workflow_claim, fresh_worker_registration] + lock_order: [run, task, worker_registration] + claim_released: false + prefix_checkpoint: + path: /worker/workflow-tasks/{taskId}/cancellation-scopes/checkpoint + admission: same_original_cooperative_claim_without_local_activity_capability + history: replay_canonical_prefix_before_opening + receipt: immutable_latest_scope_checkpoint_on_this_claim + external_payload_completion: + discovery: namespace.external_payload_storage.transport.upload.completion_context.scope_schema + schema: durable-workflow.v2.payload-completion-context.v2 + operation: cancellation_scope_checkpoint + identity: checkpoint_id + slots: ordinary_workflow_command_payload_slots + allowance: shares_existing_workflow_claim_budget + replay: identical_parent_and_shield_reuses_canonical_history_identity +x-durable-workflow-cooperative-cancellation-contract: + status: release_candidate + minimum_protocol_version: "1.20" + worker_capability: cooperative_cancellation + request_operation: POST /workflows/{workflowId}/request-cancellation + observation_field: cancellation_request + observation_surfaces: [workflow_task_claim, successful_workflow_task_heartbeat] + delivery_operation: POST /worker/workflow-tasks/{taskId}/deliver-cancellation + canonical_history_event: CooperativeCancellationDelivered + delivery_identity: [request_id, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span] + pending_child_delivery: { status: 200, delivered: false, reason: cancellation_waiting_for_child, claim_released: true, canonical_delivery_recorded: false, resume_source: child_terminal_history } + pending_activity_delivery: { status: 200, delivered: false, reason: cancellation_waiting_for_activity, claim_released: true, canonical_delivery_recorded: false, resume_source: canonical_callback_outcome_or_original_stop_receipt_or_fenced_never_started_operation } + child_operation_policies: [try_cancel, wait_cancellation_completed, abandon] + cooperative_parent_close_policy: request_cancellation + legacy_terminal_parent_close_policy: request_cancel + child_policy_admission: [server_protocol, request_protocol, immutable_worker_claim_capability, installed_runtime_child_policy] + unsupported_child_policy: { status: 409, reason: child_cancellation_policy_not_supported, recorded: false } + remote_activity_operation_policies: [try_cancel, wait_cancellation_completed, abandon] + remote_activity_policy_admission: [server_protocol, request_protocol, immutable_worker_claim_capability, installed_runtime_activity_policy] + remote_activity_policy_default: try_cancel + remote_activity_abandon_lifetime: original_finite_schedule_to_close_timeout + unsupported_remote_activity_policy: { status: 409, reason: activity_cancellation_policy_not_supported, recorded: false } + local_activity_operation_policies: [try_cancel, wait_cancellation_completed] + local_activity_policy_admission: [server_protocol, request_protocol, immutable_worker_claim_capability, installed_runtime_local_policy_list] + local_activity_policy_worker_capability: prepared_local_activity_cancellation_policies + local_activity_policy_authoring: prepared_admission_only + unsupported_local_activity_policy: { status: 409, reason: prepared_local_activity_cancellation_policy_not_supported, recorded: false } + local_activity_abandon: refused_pending_independent_callback_lifetime + claim_proof: immutable_registration_snapshot_bound_to_task_run_owner_and_attempt + request_against_incompatible_active_claim: { status: 409, reason: active_claim_cancellation_not_supported, history_appended: false } + request_claim_race: serialized_on_workflow_run_lock + prior_completed_calls: replay_normally_before_canonical_delivery_boundary + cleanup_deadline: immutable_original_request_deadline + cleanup_completion: cancelled_with_original_request_id + terminal_cancel_and_terminate: unchanged +x-durable-workflow-prepared-local-activity-contract: + status: release_candidate + minimum_protocol_version: "1.20" + worker_capabilities: [cooperative_cancellation, prepared_local_activities] + backend_role: Workflow\V2\Contracts\PreparedLocalActivityTaskBridge + advertised_support: configured_protocol_and_actual_bound_optional_role + preparation: canonical_scheduled_and_started_commit_before_callback_admission + prefix: ordinary_command_validation_authorization_and_quota_with_retained_claim + supervisor: original_issued_claim_bound_to_namespace_task_run_owner_epoch_and_attempt + renewal: atomic_workflow_and_local_attempt_leases_without_application_heartbeat + application_heartbeat: canonical_progress_and_heartbeat_timeout_without_lease_renewal + shielded_cleanup: canonical_request_and_delivery_identity_with_runtime_owned_original_deadline + scoped_cleanup: + status: unfrozen_candidate_prepared_local_only + proof: [scope_id, request_id, delivery_history_event_id] + authority: canonical_original_delivery_and_preparation_with_same_root_context + membership: original_scope_or_original_unshielded_descendant + snapshot: [scope_id, operation_scope_id, request_id, root_request_id, delivery_history_event_id, preparation_history_event_id, cleanup_deadline_at, authority_deadline_at] + admission: after_original_authored_delivery_range_and_before_payload_resolution_or_any_group_sibling + worker_deadline: forbidden + later_limits: may_shorten_current_authority_without_rewriting_snapshot + whole_run_cancellation: still_fences_scoped_cleanup + hosting_claim: renewable_interval_separate_from_subtree_deadline + scope_execution_capability: false + cancellation: original_root_identity_and_deadline_without_releasing_cleanup_claim + stop_acknowledgement: original_supervisor_reports_only_after_callback_is_stopped_and_joined + expired_callback_stop: unknown_until_original_supervisor_reports_join + retry: durable_workflow_task_backoff_and_original_total_deadline + outcome: immutable_duplicate_receipt_and_stale_publication_refusal + history_refresh_page_token: opaque_cursor_from_first_event_with_current_claim_authority_still_required + storage_draining: existing_issued_claim_may_finish_and_report_join + external_payload_completion: + discovery: namespace.external_payload_storage.transport.upload.completion_context.prepared_schema + header: X-Durable-Workflow-Payload-Completion + schema: durable-workflow.v2.payload-completion-context.v2 + required: [schema, kind, task_id, attempt, lease_owner, operation, slot] + kind: workflow + attempt: original_workflow_task_attempt + operations: + local_activity_checkpoint: { identity: checkpoint_id, slots: ordinary_workflow_command_payload_slots } + local_activity_group_checkpoint: { identity: checkpoint_id, slots: ordinary_workflow_command_payload_slots } + local_activity_prepare: { identity: sequence, slots: [[descriptor, arguments]] } + local_activity_recover: { identity: sequence, slots: [[descriptor, arguments]] } + local_activity_outcome: { identity: activity_attempt_id, slots: [[report, result]] } + exact_shape: operation_identity_is_required_and_other_extra_fields_are_refused + admission: current_original_issued_prepared_claim_and_unexpired_execution_and_root_budget + allowance: shared_with_legacy_v1_completion_across_every_operation_and_slot_of_one_workflow_claim + duplicate: exact_stored_reference_may_be_read_after_claim_closes_without_object_or_retention_mutation + lifetime: no_workflow_or_activity_lease_or_application_heartbeat_or_cleanup_deadline_renewal + storage_fenced: every_local_mutation_endpoint_refused_before_handler + published_protocol_1_19: unchanged x-durable-workflow-message-streams-contract: discovery_path: /cluster/info#/message_streams_contract minimum_protocol_version: "1.15" @@ -342,11 +460,406 @@ paths: required: false content: application/json: - schema: { $ref: "#/components/schemas/TaskHeartbeatRequest" } + schema: { $ref: "#/components/schemas/WorkflowTaskHeartbeatRequest" } responses: - "200": { $ref: "#/components/responses/WorkerEnvelope" } + "200": + description: Lease renewal with optional durable cancellation observation. + content: + application/json: + schema: { $ref: "#/components/schemas/WorkflowTaskHeartbeatResponse" } + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/deliver-cancellation: + post: + operationId: deliverWorkflowCancellation + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: cooperative_cancellation + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/CooperativeCancellationDeliveryRequest" } + responses: + "200": + description: Canonical delivery committed, an identical acknowledgment retried, or child cancellation completion still pending. + content: + application/json: + schema: { $ref: "#/components/schemas/CooperativeCancellationDeliveryResponse" } + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + /worker/workflow-tasks/{taskId}/cancellation-scopes/open: + post: + operationId: openWorkflowCancellationScope + x-durable-workflow-experimental: true + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: cooperative_cancellation + description: >- + Unfrozen candidate admission. Commit a canonical scope before entering its body under the original issued workflow claim. + An identical retry reuses its history and scope identities without releasing or renewing the claim. + This requires the optional installed CancellationScopeTaskBridge and does not advertise scope execution or delivery support. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/CancellationScopeOpeningRequest" } + responses: + "200": + description: Canonical opening committed or identical replay reused. + content: + application/json: + schema: { $ref: "#/components/schemas/CancellationScopeOpeningResponse" } + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + /worker/workflow-tasks/{taskId}/cancellation-scopes/prepare: + post: + operationId: prepareWorkflowCancellationScopeDelivery + x-durable-workflow-experimental: true + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: cooperative_cancellation + description: >- + Unfrozen candidate preparation. Authenticate the namespace, issued capable workflow claim and fresh registration. + The optional PreparedCancellationScopeTaskBridge independently commits the first context, call/range, + deadline and whole direct-scope Activity member set before operation effects. Retry reuses that preparation. + This route does not fence operations, stop callbacks or advertise scope execution. The claim remains leased. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/CancellationScopeDeliveryRequest" } + responses: + "200": + description: Original preparation committed or reused, without renewing or releasing the claim. + content: + application/json: + schema: { $ref: "#/components/schemas/CancellationScopeDeliveryResponse" } + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/cancellation-scopes/deliver: + post: + operationId: deliverWorkflowCancellationScope + x-durable-workflow-experimental: true + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: cooperative_cancellation + description: >- + Unfrozen candidate dispatch. Validate the retained preparation's exact boundary and live claim before effects. + Dispatch every original direct-scope timer and Activity through its actor, with independently committed progress + and claim rechecks. Retry after partial progress reuses the same members, context and deadline. Record delivery + only after every original member's policy proof exists. A pending original-owner stop receipt returns HTTP200 + with delivered=false. Missing operation actors return HTTP409 before timer or Activity dispatch. Timer fences + prevent a late job from firing or creating a workflow resumption. Activity fences deny + stale publication and do not prove physical callback exit. Refresh canonical history through the returned + cursor before injecting cancellation. Scope execution remains unadvertised. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/CancellationScopeDeliveryRequest" } + responses: + "200": + description: Original delivery recorded/reused or original-owner stop proof still pending. The claim remains leased. + content: + application/json: + schema: { $ref: "#/components/schemas/CancellationScopeDeliveryResponse" } + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/cancellation-scopes/checkpoint: + post: + operationId: checkpointCancellationScopePrefix + x-durable-workflow-experimental: true + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: cooperative_cancellation + description: >- + Unfrozen candidate admission. Commit the ordinary command prefix preceding a scope through the same + validation, authorization and quota path as workflow completion, retaining the original claim. + Replay the returned canonical history before opening the scope. An identical retry reuses the original + checkpoint receipt. This requires the optional installed CancellationScopeTaskBridge. + No local Activity capability, callback admission or scope execution advertisement is implied. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/CancellationScopeCheckpointRequest" } + responses: + "200": + description: Canonical prefix committed or identical receipt reused. The claim remains leased. + content: + application/json: + schema: { $ref: "#/components/schemas/CancellationScopeCheckpointResponse" } + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "429": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/local-activities/prepare: + post: + operationId: prepareLocalActivity + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: prepared_local_activities + description: >- + Commit callback admission before application code starts. Retry an identical preparation with the same workflow claim and SDK attempt nonce. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/PreparedLocalPrepareRequest" } + responses: + "200": + description: The durable prepare result. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - $ref: "#/components/schemas/PreparedLocalPreparationResult" + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "429": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/local-activities/checkpoint: + post: + operationId: checkpointLocalActivityPrefix + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: prepared_local_activities + description: >- + Commit the preceding command prefix using ordinary completion validation, authorization and quota. Retain the hosting claim. Checkpoint commit grants no callback admission. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/PreparedLocalCheckpointRequest" } + responses: + "200": + description: The durable checkpoint result. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - $ref: "#/components/schemas/PreparedLocalCheckpointResult" + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "429": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/local-activities/checkpoint-group: + post: + operationId: checkpointLocalActivityGroup + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: prepared_local_activity_groups + description: >- + Atomically commit every member of a complete all group, including prepare_local_activity descriptors without outcomes. Local executions are pending without attempts until separate preparation. Retain the original workflow claim, total deadlines and immutable group receipt. Partial and selection groups are refused. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/PreparedLocalGroupCheckpointRequest" } + responses: + "200": + description: The durable complete-group checkpoint result. Callback execution is not admitted. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - $ref: "#/components/schemas/PreparedLocalGroupCheckpointResult" + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "429": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/local-activities/recover: + post: + operationId: recoverLocalActivity + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: prepared_local_activities + description: >- + A replacement workflow claim recovers an expired original local attempt. Durable recovery records callback stop as unknown and preserves the original total timeout budget. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/PreparedLocalRecoverRequest" } + responses: + "200": + description: The durable recover result. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - $ref: "#/components/schemas/PreparedLocalRecoveryResult" + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "429": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/local-activities/{attemptId}/control: + post: + operationId: controlLocalActivity + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: prepared_local_activities + description: >- + Observe the original local attempt and optionally renew both workflow and attempt leases atomically. No application heartbeat is recorded. Accepted cancellation returns its original identity and deadline even after workflow claim takeover. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + - $ref: "#/components/parameters/PreparedLocalAttemptIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/PreparedLocalControlRequest" } + responses: + "200": + description: The durable control result. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - $ref: "#/components/schemas/PreparedLocalControlResult" "404": { $ref: "#/components/responses/WorkerError" } "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "429": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/local-activities/{attemptId}/heartbeat: + post: + operationId: heartbeatLocalActivity + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: prepared_local_activities + description: >- + Record an application heartbeat and bounded progress under the original prepared claim. Update the heartbeat timeout without renewing either lease or moving fixed execution or root cleanup deadlines. Stale workers can observe cancellation but cannot record a heartbeat. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + - $ref: "#/components/parameters/PreparedLocalAttemptIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/PreparedLocalHeartbeatRequest" } + responses: + "200": + description: The durable heartbeat or stop result. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - $ref: "#/components/schemas/PreparedLocalControlResult" + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "429": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/local-activities/{attemptId}/outcome: + post: + operationId: recordLocalActivityOutcome + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: prepared_local_activities + description: >- + Commit an original callback outcome under its issued authority. An identical committed receipt remains readable after takeover. A stale worker cannot publish a fresh result. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + - $ref: "#/components/parameters/PreparedLocalAttemptIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/PreparedLocalOutcomeRequest" } + responses: + "200": + description: The durable outcome result. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - $ref: "#/components/schemas/PreparedLocalOutcomeResult" + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "429": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/workflow-tasks/{taskId}/local-activities/{attemptId}/acknowledge-cancellation: + post: + operationId: acknowledgeLocalActivityCancellation + tags: [workflow-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + x-durable-workflow-worker-capability: prepared_local_activities + description: >- + Report only after the original supervisor has stopped and joined the callback. A surviving supervisor may report after registration drain or claim takeover without gaining result or lease authority. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + - $ref: "#/components/parameters/PreparedLocalAttemptIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/PreparedLocalStopRequest" } + responses: + "200": + description: The durable acknowledge-cancellation result. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - $ref: "#/components/schemas/PreparedLocalStopResult" + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "429": { $ref: "#/components/responses/WorkerError" } "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } /worker/workflow-tasks/{taskId}/complete: post: @@ -511,6 +1024,78 @@ paths: - $ref: "#/components/schemas/DrainingPoll" "429": { $ref: "#/components/responses/WorkerPollBackpressure" } "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/activity-tasks/{taskId}/status: + post: + operationId: observeActivityTask + tags: [activity-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + description: > + Observe an activity attempt using its namespace, task, attempt and owner + fence. This call never renews an attempt lease, worker session or user + heartbeat and never writes history. An expired lease or execution/heartbeat + deadline refuses continuation even before timeout repair. A required + worker session must remain active, owned and within its lease and TTL. + Observation + remains available under draining or fenced storage admission. User + heartbeats retain the existing ownership + renewal contract. A successful observation is not a reservation for later + publication, which must independently validate the same attempt fence. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/ActivityTaskStatusRequest" } + responses: + "200": + description: Observed continuation or stop state, with no ownership or progress renewal. + content: + application/json: + schema: { $ref: "#/components/schemas/ActivityTaskStatusResponse" } + "400": { $ref: "#/components/responses/WorkerError" } + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } + /worker/activity-tasks/{taskId}/acknowledge-cancellation: + post: + operationId: acknowledgeActivityCancellation + tags: [activity-tasks] + x-durable-workflow-minimum-protocol-version: "1.20" + description: > + Report that the original remote callback has stopped and been joined. + Requires the immutable cooperative worker claim, original task, attempt, + owner and local cancellation request, plus matching canonical cancellation + history. Cancelled rows, lease expiry and a disconnected worker are not + stop reports. Duplicates return the original receipt event. This report + never renews leases, records application heartbeats, restores publication + authority or extends the cleanup deadline. Late receipt remains late. + A worker report does not undo external effects. Local activities require + separate workflow claim authority and cannot use this endpoint. Installed + runtimes without this primitive return explicit capability diagnostics. + Available during draining storage admission. Fenced storage refuses writes. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/ActivityCancellationAcknowledgementRequest" } + responses: + "200": + description: Original callback-stop receipt, including an idempotent duplicate. + content: + application/json: + schema: { $ref: "#/components/schemas/ActivityCancellationAcknowledgementResponse" } + "400": { $ref: "#/components/responses/WorkerError" } + "403": { $ref: "#/components/responses/WorkerError" } + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } /worker/activity-tasks/{taskId}/heartbeat: post: operationId: heartbeatActivityTask @@ -574,8 +1159,8 @@ components: in: header required: true description: > - Worker-selected protocol version. The default server advertises 1.19 - and accepts same-major versions whose minor is no newer than 19. + Worker-selected protocol version. The default server advertises 1.20 + and accepts same-major versions whose minor is no newer than 20. Missing, malformed, different-major, and ahead-of-server values fail closed. schema: { $ref: "#/components/schemas/AcceptedWorkerProtocolRequestVersion" } TaskIdPath: @@ -583,6 +1168,12 @@ components: in: path required: true schema: { type: string, minLength: 1 } + PreparedLocalAttemptIdPath: + name: attemptId + in: path + required: true + description: Canonical durable attempt ID returned by preparation, distinct from worker_attempt_id. + schema: { type: string, minLength: 1 } SessionIdPath: name: sessionId in: path @@ -750,14 +1341,14 @@ components: schemas: AcceptedWorkerProtocolRequestVersion: type: string - enum: ["1.0", "1.1", "1.2", "1.3", "1.4", "1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13", "1.14", "1.15", "1.16", "1.17", "1.18", "1.19"] + enum: ["1.0", "1.1", "1.2", "1.3", "1.4", "1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13", "1.14", "1.15", "1.16", "1.17", "1.18", "1.19", "1.20"] description: > Request versions accepted by a server using the default advertised - version 1.19. If deployment configuration changes the advertised + version 1.20. If deployment configuration changes the advertised version to 1.N, the accepted set is 1.0 through 1.N. AdvertisedWorkerProtocolVersion: type: string - const: "1.19" + const: "1.20" description: Default worker protocol version advertised by responses implementing the Durable Workflow 2.0 protocol contract. WorkerEnvelope: type: object @@ -950,6 +1541,23 @@ components: additionalProperties: true required: [supported_workflow_task_commands, workflow_task_poll_request_idempotency, workflow_memo_updates, message_streams, typed_search_attributes, condition_wait_occurrence_identity, update_validation_tasks, synchronous_update_validation, local_activities, sticky_execution] properties: + cooperative_cancellation: + type: boolean + description: Explicit support for claim-bound cooperative request delivery, requiring worker protocol 1.20 or newer in the same major version. Workers must opt in on an immutable claim. + activity_cancellation_acknowledgement: + type: boolean + description: Original remote callback-stop receipts require an installed runtime primitive and worker protocol 1.20. False for earlier protocols and runtimes without that primitive. + prepared_local_activities: + type: boolean + description: Preparation, prefix checkpoint, supervisor control, application heartbeat, outcome, recovery and joined-stop receipts. Requires protocol 1.20 and the actual bound PreparedLocalActivityTaskBridge role including heartbeat support. False for earlier protocols or an absent backend role. + prepared_local_activity_groups: + type: boolean + description: Atomic complete-all-group admission. Requires prepared local activities, protocol 1.20 and the actual bound PreparedLocalActivityGroupTaskBridge. Existing custom prepared bridges do not acquire this role. + prepared_local_activity_cancellation_policies: + type: array + uniqueItems: true + items: { type: string, enum: [try_cancel, wait_cancellation_completed] } + description: Explicit policies supported by the actual installed prepared-local bridge's optional supportedLocalActivityCancellationPolicies method. Empty for earlier protocols or older optional-role bridges. Workers need a new immutable claim with the separately named worker capability before admission or replay. supported_workflow_task_commands: type: array uniqueItems: true @@ -1124,6 +1732,10 @@ components: local_activities: "1.18" worker_sessions: "1.18" sticky_execution: "1.18" + cooperative_cancellation: "1.20" + prepared_local_activities: "1.20" + prepared_local_activity_groups: "1.20" + prepared_local_activity_cancellation_policies: "1.20" capability_manifest: $ref: "#/components/schemas/PortableWorkerCapabilityManifest" WorkerHeartbeatRequest: @@ -1255,6 +1867,577 @@ components: lease_owner: { type: string, minLength: 1 } lease_expires_at: { type: string, format: date-time } history_events: { type: array, items: { $ref: "#/components/schemas/WorkerHistoryEvent" } } + cancellation_request: + $ref: "#/components/schemas/CooperativeCancellationRequest" + x-durable-workflow-minimum-protocol-version: "1.20" + CooperativeCancellationRequest: + type: object + additionalProperties: false + required: [request_id, requested_at, cleanup_deadline_at, delivery_sequence, delivered_at, history_refresh_page_token] + properties: + request_id: { type: string, minLength: 1 } + requested_at: { type: string, format: date-time } + cleanup_deadline_at: { type: string, format: date-time } + delivery_sequence: { type: [integer, "null"], minimum: 1 } + delivered_at: { type: [string, "null"], format: date-time } + history_refresh_page_token: + type: string + minLength: 1 + description: Server-issued token for the first bounded history page. Reload canonical history after delivery using this token and the same task ownership fence. + CooperativeCancellationDeliveryRequest: + type: object + additionalProperties: true + required: [lease_owner, workflow_task_attempt, request_id, sequence, call_kind] + properties: + lease_owner: { type: string, minLength: 1 } + workflow_task_attempt: { type: integer, minimum: 1 } + request_id: { type: string, minLength: 1 } + sequence: { type: integer, minimum: 1 } + call_kind: { type: string, enum: [activity, local_activity, timer, condition, signal, child, parallel, selection_handle] } + sequence_span: { type: [integer, "null"], minimum: 1, default: 1 } + operation_sequence: { type: [integer, "null"], minimum: 1 } + operation_sequence_span: { type: [integer, "null"], minimum: 1, default: 1 } + CooperativeCancellationDeliveryResponse: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - oneOf: + - type: object + required: [delivered, task_id, workflow_run_id, request_id, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span, reason] + properties: + delivered: { type: boolean, const: true } + task_id: { type: string, minLength: 1 } + workflow_run_id: { type: string, minLength: 1 } + request_id: { type: string, minLength: 1 } + sequence: { type: integer, minimum: 1 } + call_kind: { type: string, enum: [activity, local_activity, timer, condition, signal, child, parallel, selection_handle] } + sequence_span: { type: integer, minimum: 1 } + operation_sequence: { type: [integer, "null"], minimum: 1 } + operation_sequence_span: { type: integer, minimum: 1 } + reason: { type: "null" } + - type: object + required: [delivered, task_id, workflow_run_id, request_id, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span, reason, claim_released] + properties: + delivered: { type: boolean, const: false } + claim_released: { type: boolean, const: true } + task_id: { type: string, minLength: 1 } + workflow_run_id: { type: string, minLength: 1 } + request_id: { type: "null" } + sequence: { type: "null" } + call_kind: { type: "null" } + sequence_span: { type: "null" } + operation_sequence: { type: "null" } + operation_sequence_span: { type: "null" } + reason: { type: string, enum: [cancellation_waiting_for_child, cancellation_waiting_for_activity] } + CancellationScopeDeliveryRequest: + allOf: + - $ref: "#/components/schemas/CooperativeCancellationDeliveryRequest" + - type: object + required: [scope_id] + properties: + scope_id: { type: string, minLength: 1, maxLength: 255 } + lease_owner: { type: string, minLength: 1, maxLength: 255 } + request_id: { type: string, minLength: 1, maxLength: 255 } + sequence_span: { type: integer, minimum: 1, default: 1 } + operation_sequence_span: { type: integer, minimum: 1, default: 1 } + CancellationScopeDeliveryResponse: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - type: object + required: [prepared, delivered, claim_released, task_id, created_task_ids, reason] + properties: + prepared: { type: boolean } + delivered: { type: boolean } + claim_released: { type: boolean, const: false } + task_id: { type: string, minLength: 1 } + created_task_ids: { type: array, maxItems: 0 } + reason: { type: [string, "null"] } + unavailable: { type: array, items: { type: string, minLength: 1 } } + activity_cancellations: + type: array + description: Committed results for original prepared members. Partial progress may be present on a refusal. A fence is publication denial, and waiting_for_stop is the policy barrier rather than a physical exit assertion. + items: + type: object + required: [sequence, activity_execution_id, fenced, abandoned, waiting_for_stop, history_event_id] + properties: + sequence: { type: integer, minimum: 1 } + activity_execution_id: { type: string, minLength: 1 } + fenced: { type: boolean } + abandoned: { type: boolean } + waiting_for_stop: { type: boolean } + history_event_id: { type: [string, "null"], minLength: 1 } + timer_cancellations: + type: array + description: Committed results for original prepared timers. A natural fire retains its outcome with fenced=false. A fence prevents a late fire or resumption. Partial progress may be present on a refusal. + items: + type: object + required: [sequence, timer_id, fenced, history_event_id] + properties: + sequence: { type: integer, minimum: 1 } + timer_id: { type: string, minLength: 1 } + fenced: { type: boolean } + history_event_id: { type: [string, "null"], minLength: 1 } + wait_cancellations: + type: array + description: Original prepared signal and condition wait decisions. A cancellation and associated timeout fence commit atomically. Earlier natural outcomes retain cancelled=false. Each receipt keeps its original history identity through retry or replacement. + items: + type: object + required: [kind, sequence, wait_id, cancelled, history_event_id] + properties: + kind: { type: string, enum: [signal, condition] } + sequence: { type: integer, minimum: 1 } + wait_id: { type: string, minLength: 1 } + cancelled: { type: boolean } + history_event_id: { type: string, minLength: 1 } + child_cancellations: + type: array + description: Durable decisions for original prepared direct children. A cooperative request and its parent receipt commit atomically. TryCancel is ready after acceptance, WAIT requires a separate canonical child terminal resolution, and Abandon leaves the child untouched. Partial receipts survive retries and replacement claims without changing the original identity or deadline. Readiness does not assert physical callback exit. + items: + type: object + additionalProperties: false + required: [child_call_id, child_workflow_run_id, history_event_id, resolution_history_event_id, request_id, policy, ready] + properties: + child_call_id: { type: string, minLength: 1 } + child_workflow_run_id: { type: string, minLength: 1 } + history_event_id: { type: string, minLength: 1 } + resolution_history_event_id: { type: [string, "null"], minLength: 1 } + request_id: { type: [string, "null"], minLength: 1 } + policy: { type: string, enum: [try_cancel, wait_cancellation_completed, abandon] } + ready: { type: boolean } + if: + required: [prepared] + properties: + prepared: { const: true } + then: + required: [workflow_run_id, preparation_history_event_id, scope_id, request_id, cancellation, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span, authority_deadline_at, activity_members, timer_members, wait_members, child_members, lease_owner, workflow_task_attempt, history_refresh_page_token] + properties: + workflow_run_id: { type: string, minLength: 1 } + preparation_history_event_id: { type: string, minLength: 1 } + history_event_id: { type: string, minLength: 1 } + scope_id: { type: string, minLength: 1 } + request_id: { type: string, minLength: 1 } + cancellation: + type: object + required: [schema, root_context, lineage] + properties: + schema: { const: durable-workflow.scoped-cancellation-context/v1 } + root_context: + type: object + required: [request_id, cleanup_deadline_at] + properties: + request_id: { type: string, minLength: 1 } + cleanup_deadline_at: { type: string, format: date-time } + lineage: + type: array + minItems: 1 + items: + type: object + required: [request_id, workflow_instance_id, workflow_run_id, scope_id, cleanup_deadline_at] + properties: + request_id: { type: string, minLength: 1 } + workflow_instance_id: { type: string, minLength: 1 } + workflow_run_id: { type: string, minLength: 1 } + scope_id: { type: string, minLength: 1 } + cleanup_deadline_at: { type: string, format: date-time } + sequence: { type: integer, minimum: 1 } + call_kind: { type: string, enum: [activity, local_activity, timer, condition, signal, child, parallel, selection_handle] } + sequence_span: { type: integer, minimum: 1 } + operation_sequence: { type: [integer, "null"], minimum: 1 } + operation_sequence_span: { type: integer, minimum: 1 } + authority_deadline_at: { type: string, format: date-time } + activity_members: + type: array + items: + type: object + required: [sequence, activity_execution_id, descriptor_hash] + properties: + sequence: { type: integer, minimum: 1 } + activity_execution_id: { type: string, minLength: 1 } + descriptor_hash: { type: string, pattern: "^[a-f0-9]{64}$" } + timer_members: + type: array + items: + type: object + required: [sequence, timer_id, descriptor_hash] + properties: + sequence: { type: integer, minimum: 1 } + timer_id: { type: string, minLength: 1 } + descriptor_hash: { type: string, pattern: "^[a-f0-9]{64}$" } + lease_owner: { type: string, minLength: 1 } + workflow_task_attempt: { type: integer, minimum: 1 } + history_refresh_page_token: { type: string, minLength: 1 } + wait_members: + type: array + description: Preparation freezes original wait occurrences, including untimed waits, and their timeout association without application payload bytes. + items: + type: object + required: [kind, sequence, wait_id, timer_id, descriptor_hash] + properties: + kind: { type: string, enum: [signal, condition] } + sequence: { type: integer, minimum: 1 } + wait_id: { type: string, minLength: 1 } + timer_id: { type: [string, "null"], minLength: 1 } + descriptor_hash: { type: string, pattern: "^[a-f0-9]{64}$" } + child_members: + type: array + description: Preparation v4 freezes child call identity, instance, original typed-history run target and cancellation policy. The snapshot grants no child effect or delivery authority. Child dispatch remains unavailable until its committed receipt barrier is implemented. + items: + type: object + additionalProperties: false + required: [sequence, child_call_id, child_workflow_instance_id, child_workflow_run_id, cancellation_policy, descriptor_hash] + properties: + sequence: { type: integer, minimum: 1 } + child_call_id: { type: string, minLength: 1 } + child_workflow_instance_id: { type: string, minLength: 1 } + child_workflow_run_id: { type: string, minLength: 1 } + cancellation_policy: { type: string, enum: [try_cancel, wait_cancellation_completed, abandon] } + descriptor_hash: { type: string, pattern: "^[a-f0-9]{64}$" } + CancellationScopeOpeningRequest: + type: object + required: [lease_owner, workflow_task_attempt, sequence] + properties: + lease_owner: { type: string, minLength: 1, maxLength: 255 } + workflow_task_attempt: { type: integer, minimum: 1 } + sequence: { type: integer, minimum: 1 } + parent_scope_id: { type: string, minLength: 1, maxLength: 255, default: root } + shield_parent: { type: boolean, default: false } + CancellationScopeOpeningResponse: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - type: object + required: [opened, duplicate, claim_released, task_id, workflow_run_id, workflow_task_attempt, lease_owner, history_event_id, scope_id, parent_scope_id, shield_parent, sequence, created_task_ids, history_refresh_page_token, reason] + properties: + opened: { type: boolean, const: true } + duplicate: { type: boolean } + claim_released: { type: boolean, const: false } + task_id: { type: string, minLength: 1 } + workflow_run_id: { type: string, minLength: 1 } + workflow_task_attempt: { type: integer, minimum: 1 } + lease_owner: { type: string, minLength: 1 } + history_event_id: { type: string, minLength: 1 } + scope_id: { type: string, minLength: 1 } + parent_scope_id: { type: string, minLength: 1 } + shield_parent: { type: boolean } + sequence: { type: integer, minimum: 1 } + created_task_ids: { type: array, maxItems: 0 } + history_refresh_page_token: + type: string + minLength: 1 + description: Opaque cursor from the original run's start. Fetch every page through the claim-bound history endpoint before entering the scope body. This cursor grants no claim authority and does not renew or release the task. + reason: { type: "null" } + CancellationScopeCheckpointRequest: + description: Ordinary retained prefix before scope opening. Does not admit local callbacks. + allOf: + - $ref: "#/components/schemas/PreparedLocalCheckpointRequest" + CancellationScopeCheckpointResponse: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - $ref: "#/components/schemas/PreparedLocalCheckpointResult" + PreparedLocalClaimRequest: + type: object + required: [lease_owner, workflow_task_attempt] + properties: + lease_owner: { type: string, minLength: 1, maxLength: 255 } + workflow_task_attempt: { type: integer, minimum: 1 } + PreparedLocalDescriptor: + type: object + additionalProperties: false + required: [type, activity_type, arguments] + properties: + type: { type: string, const: record_local_activity } + activity_type: { type: string, minLength: 1 } + arguments: + description: Encoded Avro or a standard Avro envelope resolved by this namespace's configured runtime storage. + oneOf: + - { type: string, minLength: 1 } + - { type: object, required: [codec], properties: { codec: { const: avro } } } + payload_codec: { type: string, const: avro } + execution_mode: { type: string, const: local } + cancellation_scope_id: + type: string + minLength: 1 + maxLength: 255 + x-durable-workflow-minimum-protocol-version: "1.20" + description: Optional exact-run canonical scope for prepared local admission and recovery. Requires the installed CancellationScopeAdmission backend. The scope must be recorded before this descriptor's sequence. Syntax and membership are checked before payload resolution, and Native validates canonical history under its existing admission locks. Omission retains root membership. This does not advertise scope execution or supervision. + cancellation_policy: + type: string + enum: [try_cancel, wait_cancellation_completed] + description: Explicit candidate prepared-local policy. Requires the installed backend policy list and prepared_local_activity_cancellation_policies on the original issued claim. Omission retains historical TryCancel. Null and Abandon are refused before payload resolution or admission. + start_to_close_timeout: { type: [integer, "null"], minimum: 1 } + schedule_to_close_timeout: { type: [integer, "null"], minimum: 1 } + heartbeat_timeout: { type: [integer, "null"], minimum: 1 } + cancellation_cleanup: + $ref: "#/components/schemas/PreparedLocalCleanupProof" + parallel_group_id: { type: string, minLength: 1 } + parallel_group_kind: { type: string, enum: [activity, mixed] } + parallel_group_mode: { type: string, const: all } + parallel_group_base_sequence: { type: integer, minimum: 1 } + parallel_group_size: { type: integer, minimum: 1 } + parallel_group_index: { type: integer, minimum: 0 } + parallel_group_path: + type: array + minItems: 1 + items: { type: object } + retry_policy: + type: [object, "null"] + properties: + max_attempts: { type: [integer, "null"], minimum: 1 } + backoff_seconds: { type: array, items: { type: integer, minimum: 0 } } + non_retryable_error_types: { type: array, items: { type: string } } + PreparedLocalRecoverRequest: + allOf: + - $ref: "#/components/schemas/PreparedLocalClaimRequest" + - type: object + required: [sequence, descriptor] + properties: + sequence: { type: integer, minimum: 1 } + descriptor: { $ref: "#/components/schemas/PreparedLocalDescriptor" } + PreparedLocalPrepareRequest: + allOf: + - $ref: "#/components/schemas/PreparedLocalRecoverRequest" + - type: object + required: [worker_attempt_id] + properties: + worker_attempt_id: + type: string + minLength: 1 + maxLength: 255 + description: Stable SDK nonce for this callback admission. It is distinct from the durable activity_attempt_id. + PreparedLocalCheckpointRequest: + allOf: + - $ref: "#/components/schemas/PreparedLocalClaimRequest" + - type: object + required: [checkpoint_id, start_sequence, commands] + properties: + checkpoint_id: { type: string, minLength: 1, maxLength: 255 } + start_sequence: { type: integer, minimum: 1 } + commands: + type: array + maxItems: 100 + description: Nonterminal prefix commands. Turn-closing waits, local outcome reports and selection cancellation are excluded. Nonempty completion-only message cursor, wait and sticky cache metadata are rejected. + items: { $ref: "#/components/schemas/WorkflowCommand" } + PreparedLocalGroupCheckpointRequest: + allOf: + - $ref: "#/components/schemas/PreparedLocalCheckpointRequest" + - type: object + properties: + commands: + type: array + minItems: 1 + maxItems: 100 + description: Complete all groups with at least one prepare_local_activity. Local descriptors use the PreparedLocalDescriptor fields with type prepare_local_activity and an authored parallel group path. They contain no outcome, result, routing or attempt reports. Cleanup proof is validated before any sibling is created. + items: { $ref: "#/components/schemas/WorkflowCommand" } + PreparedLocalControlRequest: + allOf: + - $ref: "#/components/schemas/PreparedLocalClaimRequest" + - type: object + properties: + renew_lease: { type: boolean, default: false } + PreparedLocalHeartbeatRequest: + allOf: + - $ref: "#/components/schemas/PreparedLocalClaimRequest" + - type: object + properties: + progress: + type: [object, "null"] + additionalProperties: false + properties: + message: { type: string, minLength: 1, maxLength: 280 } + current: { type: [number, string], description: Nonnegative finite number or numeric string. } + total: { type: [number, string], description: Nonnegative finite number or numeric string. } + unit: { type: string, minLength: 1, maxLength: 64 } + details: { type: object, maxProperties: 20 } + PreparedLocalCleanupProof: + description: Explicit cleanup after canonical delivery. A scoped proof names the original cancelled address, while cancellation_scope_id names the new operation's original scope or unshielded descendant. The runtime supplies the root identity and immutable budget. No worker-supplied deadline or snapshot field is accepted. + oneOf: + - type: object + additionalProperties: false + required: [request_id, delivery_history_event_id] + properties: + request_id: { type: string, minLength: 1 } + delivery_history_event_id: { type: string, minLength: 1 } + - type: object + additionalProperties: false + required: [scope_id, request_id, delivery_history_event_id] + properties: + scope_id: { type: string, minLength: 1, maxLength: 255 } + request_id: { type: string, minLength: 1 } + delivery_history_event_id: { type: string, minLength: 1 } + PreparedLocalCleanupSnapshot: + description: Original runtime authority. The four-field root snapshot remains unchanged. A scoped snapshot binds the original delivery and preparation, new operation membership and captured finite ceiling. Current ancestor or run limits may shorten authority without changing this snapshot. It grants no permission after expiry, terminal closure or whole-run cancellation. + oneOf: + - type: "null" + - type: object + additionalProperties: false + required: [request_id, root_request_id, delivery_history_event_id, cleanup_deadline_at] + properties: + request_id: { type: string, minLength: 1 } + root_request_id: { type: string, minLength: 1 } + delivery_history_event_id: { type: string, minLength: 1 } + cleanup_deadline_at: { type: string, format: date-time } + - type: object + additionalProperties: false + required: [scope_id, operation_scope_id, request_id, root_request_id, delivery_history_event_id, preparation_history_event_id, cleanup_deadline_at, authority_deadline_at] + properties: + scope_id: { type: string, minLength: 1 } + operation_scope_id: { type: string, minLength: 1 } + request_id: { type: string, minLength: 1 } + root_request_id: { type: string, minLength: 1 } + delivery_history_event_id: { type: string, minLength: 1 } + preparation_history_event_id: { type: string, minLength: 1 } + cleanup_deadline_at: { type: string, format: date-time } + authority_deadline_at: { type: string, format: date-time } + PreparedLocalOutcomeRequest: + allOf: + - $ref: "#/components/schemas/PreparedLocalClaimRequest" + - type: object + required: [report] + properties: + report: + oneOf: + - type: object + additionalProperties: false + required: [outcome, result] + properties: + outcome: { const: completed } + result: + oneOf: + - { type: string, minLength: 1 } + - { type: object, required: [codec], properties: { codec: { const: avro } } } + payload_codec: { const: avro } + - type: object + additionalProperties: false + required: [outcome, message] + properties: + outcome: { const: failed } + message: { type: string } + exception_type: { type: string, minLength: 1, maxLength: 255 } + non_retryable: { type: boolean, default: false } + - type: object + additionalProperties: false + required: [outcome] + properties: + outcome: { const: timed_out } + PreparedLocalStopRequest: + allOf: + - $ref: "#/components/schemas/PreparedLocalClaimRequest" + - type: object + required: [request_id] + properties: + request_id: { type: string, minLength: 1, maxLength: 255 } + PreparedLocalPreparationResult: + type: object + required: [prepared, duplicate, reason, activity_execution_id, activity_attempt_id, worker_attempt_id, attempt_number, workflow_task_id, workflow_task_attempt, lease_owner, lease_expires_at, server_time] + properties: + prepared: { const: true } + duplicate: { type: boolean } + reason: { type: "null" } + activity_execution_id: { type: string, minLength: 1 } + activity_attempt_id: { type: string, minLength: 1 } + worker_attempt_id: { type: string, minLength: 1 } + attempt_number: { type: integer, minimum: 1 } + workflow_task_id: { type: string, minLength: 1 } + workflow_task_attempt: { type: integer, minimum: 1 } + lease_owner: { type: string, minLength: 1 } + lease_expires_at: { type: string, format: date-time } + server_time: { type: string, format: date-time } + start_to_close_deadline_at: { type: [string, "null"], format: date-time } + schedule_to_close_deadline_at: { type: [string, "null"], format: date-time } + heartbeat_deadline_at: { type: [string, "null"], format: date-time } + cancellation_cleanup: { $ref: "#/components/schemas/PreparedLocalCleanupSnapshot" } + PreparedLocalCheckpointResult: + type: object + required: [checkpointed, duplicate, reason, checkpoint_id, task_id, workflow_run_id, workflow_task_attempt, start_sequence, next_sequence, fingerprint, created_task_ids, recorded_at] + properties: + checkpointed: { const: true } + duplicate: { type: boolean } + reason: { type: "null" } + checkpoint_id: { type: string, minLength: 1 } + task_id: { type: string, minLength: 1 } + workflow_run_id: { type: string, minLength: 1 } + workflow_task_attempt: { type: integer, minimum: 1 } + start_sequence: { type: integer, minimum: 1 } + next_sequence: { type: integer, minimum: 1 } + fingerprint: { type: string, pattern: "^[0-9a-f]{64}$" } + created_task_ids: { type: array, items: { type: string } } + recorded_at: { type: string, format: date-time } + PreparedLocalGroupCheckpointResult: + allOf: + - $ref: "#/components/schemas/PreparedLocalCheckpointResult" + - type: object + required: [local_activities] + properties: + local_activities: + type: array + minItems: 1 + maxItems: 100 + items: + type: object + additionalProperties: false + required: [sequence, activity_execution_id] + properties: + sequence: { type: integer, minimum: 1 } + activity_execution_id: { type: string, minLength: 1 } + PreparedLocalOutcomeResult: + type: object + required: [recorded, duplicate, reason, event_id, event_type, workflow_run_id, workflow_task_id, workflow_task_attempt, activity_execution_id, activity_attempt_id, worker_attempt_id, recorded_at, claim_released, created_task_ids] + properties: + recorded: { const: true } + duplicate: { type: boolean } + reason: { type: "null" } + event_id: { type: string, minLength: 1 } + event_type: { type: string, enum: [ActivityCompleted, ActivityFailed, ActivityTimedOut, ActivityRetryScheduled] } + workflow_run_id: { type: string, minLength: 1 } + workflow_task_id: { type: string, minLength: 1 } + workflow_task_attempt: { type: integer, minimum: 1 } + activity_execution_id: { type: string, minLength: 1 } + activity_attempt_id: { type: string, minLength: 1 } + worker_attempt_id: { type: string, minLength: 1 } + recorded_at: { type: string, format: date-time } + claim_released: { type: boolean } + created_task_ids: { type: array, items: { type: string } } + PreparedLocalRecoveryResult: + type: object + required: [recovered, duplicate, reason, event_id, event_type, callback_stop_state, claim_released, created_task_ids] + properties: + recovered: { const: true } + duplicate: { type: boolean } + reason: { type: "null" } + event_id: { type: string, minLength: 1 } + event_type: { type: string, enum: [ActivityFailed, ActivityTimedOut, ActivityRetryScheduled] } + callback_stop_state: { const: unknown } + claim_released: { type: boolean } + created_task_ids: { type: array, items: { type: string } } + PreparedLocalControlResult: + type: object + required: [active, renewed, stop_required, reason, activity_execution_id, activity_attempt_id, workflow_task_id, workflow_task_attempt, server_time] + properties: + active: { type: boolean } + renewed: { type: boolean } + stop_required: { type: boolean } + reason: { type: [string, "null"] } + activity_execution_id: { type: string, minLength: 1 } + activity_attempt_id: { type: string, minLength: 1 } + workflow_task_id: { type: string, minLength: 1 } + workflow_task_attempt: { type: integer, minimum: 1 } + server_time: { type: string, format: date-time } + heartbeat_recorded: { type: boolean } + heartbeat_history_event_id: { type: [string, "null"] } + cancellation_cleanup: { $ref: "#/components/schemas/PreparedLocalCleanupSnapshot" } + cancellation_request: + type: object + required: [request_id, root_request_id, cleanup_deadline_at] + properties: + request_id: { type: string, minLength: 1 } + root_request_id: { type: string, minLength: 1 } + cleanup_deadline_at: { type: string, format: date-time } + PreparedLocalStopResult: + type: object + required: [acknowledged, duplicate, reason, history_event_id] + properties: + acknowledged: { const: true } + duplicate: { type: boolean } + reason: { type: "null" } + history_event_id: { type: string, minLength: 1 } WorkerHistoryEvent: type: object additionalProperties: true @@ -1509,15 +2692,125 @@ components: HistoryPageRequest: type: object additionalProperties: true + required: [lease_owner, workflow_task_attempt, next_history_page_token] + properties: + lease_owner: { type: string, minLength: 1 } + workflow_task_attempt: { type: integer, minimum: 1 } + next_history_page_token: { type: string, minLength: 1 } + history_page_size: { type: [integer, "null"], minimum: 1 } + accept_history_encoding: { type: [string, "null"] } + ActivityTaskStatusRequest: + type: object + additionalProperties: true + required: [activity_attempt_id, lease_owner] properties: - page_token: { type: [string, "null"] } - page_size: { type: integer, minimum: 1 } + activity_attempt_id: { type: string, minLength: 1 } + lease_owner: { type: string, minLength: 1 } + ActivityCancellationAcknowledgementRequest: + type: object + additionalProperties: true + required: [activity_attempt_id, lease_owner, request_id] + properties: + activity_attempt_id: { type: string, minLength: 1, maxLength: 255 } + lease_owner: { type: string, minLength: 1, maxLength: 255 } + request_id: { type: string, minLength: 1, maxLength: 255 } + ActivityCancellationAcknowledgementResponse: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - type: object + required: [task_id, activity_attempt_id, lease_owner, request_id, acknowledged, duplicate, reason, history_event_id, heartbeat_recorded] + properties: + task_id: { type: string, minLength: 1 } + activity_attempt_id: { type: string, minLength: 1 } + lease_owner: { type: string, minLength: 1 } + request_id: { type: string, minLength: 1 } + acknowledged: { type: boolean, const: true } + duplicate: { type: boolean } + reason: { type: "null" } + history_event_id: { type: string, minLength: 1 } + heartbeat_recorded: { type: boolean, const: false } + ActivityCancellationReceipt: + type: object + description: > + Canonical cooperative request and callback-stop receipt. Unknown means no + original worker stop report has been recorded, even when publication is + already fenced. The deadline is the original immutable cleanup budget. + required: [request_id, root_request_id, cleanup_deadline_at, cancellation_history_event_id, callback_state, history_event_id, acknowledged_at, received_after_deadline] + properties: + request_id: { type: string, minLength: 1 } + root_request_id: { type: string, minLength: 1 } + cleanup_deadline_at: { type: string, format: date-time } + cancellation_history_event_id: { type: string, minLength: 1 } + callback_state: { type: string, enum: [unknown, stopped] } + history_event_id: { type: [string, "null"] } + acknowledged_at: { type: [string, "null"], format: date-time } + received_after_deadline: { type: [boolean, "null"] } + ActivityTaskStatusResponse: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - type: object + required: [task_id, activity_attempt_id, lease_owner, can_continue, cancel_requested, reason, heartbeat_recorded, lease_expires_at, last_heartbeat_at, run_status, activity_status, attempt_status, task_status] + properties: + task_id: { type: string, minLength: 1 } + activity_attempt_id: { type: string, minLength: 1 } + lease_owner: { type: string, minLength: 1 } + can_continue: { type: boolean } + cancel_requested: { type: boolean } + reason: { type: [string, "null"] } + heartbeat_recorded: { type: boolean, const: false } + lease_expires_at: { type: [string, "null"], format: date-time } + last_heartbeat_at: { type: [string, "null"], format: date-time } + run_status: { type: [string, "null"] } + run_closed_reason: { type: [string, "null"] } + run_closed_at: { type: [string, "null"], format: date-time } + activity_status: { type: [string, "null"] } + attempt_status: { type: [string, "null"] } + task_status: { type: [string, "null"] } + cancellation_acknowledgement: + $ref: "#/components/schemas/ActivityCancellationReceipt" + x-durable-workflow-minimum-protocol-version: "1.20" + deadlines: + type: [object, "null"] + description: Existing execution deadlines. Null when no deadline is configured. + properties: + schedule_to_start: { type: string, format: date-time } + start_to_close: { type: string, format: date-time } + schedule_to_close: { type: string, format: date-time } + heartbeat: { type: string, format: date-time } + worker_session: + type: [object, "null"] + additionalProperties: true + description: Existing required-session observation, never a renewal. TaskHeartbeatRequest: type: object additionalProperties: true properties: worker_id: { type: string } progress: true + WorkflowTaskHeartbeatRequest: + type: object + additionalProperties: true + required: [lease_owner, workflow_task_attempt] + properties: + lease_owner: { type: string, minLength: 1 } + workflow_task_attempt: { type: integer, minimum: 1 } + WorkflowTaskHeartbeatResponse: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - type: object + required: [task_id, workflow_task_attempt, lease_owner, renewed, lease_expires_at, run_status, task_status, reason] + properties: + task_id: { type: string, minLength: 1 } + workflow_task_attempt: { type: integer, minimum: 1 } + lease_owner: { type: string, minLength: 1 } + renewed: { type: boolean } + lease_expires_at: { type: [string, "null"], format: date-time } + run_status: { type: [string, "null"] } + task_status: { type: [string, "null"] } + reason: { type: [string, "null"] } + cancellation_request: + $ref: "#/components/schemas/CooperativeCancellationRequest" + x-durable-workflow-minimum-protocol-version: "1.20" WorkflowTaskCompleteRequest: type: object additionalProperties: true @@ -1583,6 +2876,24 @@ components: required: [type] properties: type: { type: string } + cancellation_scope_id: + type: string + minLength: 1 + maxLength: 255 + x-durable-workflow-minimum-protocol-version: "1.20" + description: Optional exact-run canonical membership for schedule_activity, start_timer, start_child_workflow, prepare_local_activity, open_signal_wait and open_condition_wait commands in the unfrozen candidate. Requires an installed CancellationScopeAdmission backend. The named scope must already be recorded before admission. Invalid membership is refused before payload resolution and rechecked under the run lock. Omission preserves historical root membership. Wait commands close a turn and cannot enter a retained-claim prefix. This field does not advertise scope execution or supervision. + cancellation_cleanup: + $ref: "#/components/schemas/PreparedLocalCleanupProof" + x-durable-workflow-minimum-protocol-version: "1.20" + description: Original scope, request and committed delivery references for a shielded scoped start_timer or prepared local cleanup operation. The installed runtime derives immutable deadline authority from canonical history. A timer at or beyond the remaining authority ceiling is refused. This unfrozen candidate field never grants a new cleanup budget. + parent_close_policy: + type: [string, "null"] + description: Child parent-close policy. request_cancel retains legacy terminal cancellation. request_cancellation requests cooperative cleanup under protocol 1.20 and the original shared budget. + cancellation_policy: + type: [string, "null"] + enum: [try_cancel, wait_cancellation_completed, abandon, null] + x-durable-workflow-minimum-protocol-version: "1.20" + description: Per-operation child, remote Activity or prepared local Activity cancellation policy. Absent or null retains child abandon and remote Activity try_cancel. Explicit remote policies require protocol 1.20, a capable immutable worker claim and an installed runtime implementation. Remote abandon requires a finite schedule_to_close_timeout. Prepared local descriptors accept only explicit TryCancel or WaitCancellationCompleted under their separate installed-backend and original-claim capability checks, including atomic group admission. Null and local Abandon are refused. allOf: - if: properties: @@ -1667,6 +2978,15 @@ components: outcome: { const: timed_out } then: required: [timeout_kind] + - if: + properties: + type: { const: schedule_activity } + cancellation_policy: { const: abandon } + required: [type, cancellation_policy] + then: + required: [schedule_to_close_timeout] + properties: + schedule_to_close_timeout: { type: integer, minimum: 1 } LocalActivityAttemptReport: type: object additionalProperties: false diff --git a/resources/platform-protocol-specs/worker-protocol-stream.asyncapi.yaml b/resources/platform-protocol-specs/worker-protocol-stream.asyncapi.yaml index c933ad78..d4300794 100644 --- a/resources/platform-protocol-specs/worker-protocol-stream.asyncapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-stream.asyncapi.yaml @@ -2,7 +2,7 @@ asyncapi: 2.6.0 id: durable-workflow.v2.worker-protocol-stream info: title: Durable Workflow worker-protocol long-poll stream semantics - version: "12" + version: "14" description: > Normative AsyncAPI description of the public worker-protocol stream semantics. Durable uses HTTP long-poll request/response rather than a @@ -12,15 +12,47 @@ info: events, and terminal completion/failure events close the lease. x-durable-workflow-worker-protocol-negotiation: advertised_version_path: worker_protocol.version - default_advertised_version: "1.19" + default_advertised_version: "1.20" request_header_rule: same_major_and_minor_less_than_or_equal_to_advertised - accepted_request_versions_by_default: ["1.0", "1.1", "1.2", "1.3", "1.4", "1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13", "1.14", "1.15", "1.16", "1.17", "1.18", "1.19"] + accepted_request_versions_by_default: ["1.0", "1.1", "1.2", "1.3", "1.4", "1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13", "1.14", "1.15", "1.16", "1.17", "1.18", "1.19", "1.20"] response_version: advertised_version fail_closed_on: [missing_header, malformed_version, different_major, minor_greater_than_advertised] x-durable-workflow-catalog-entry: worker_protocol_stream x-durable-workflow-catalog-schema: durable-workflow.v2.platform-protocol-specs.catalog x-durable-workflow-catalog-version: 16 x-durable-workflow-evolution-rule: additive_minor_breaking_major +x-durable-workflow-cooperative-cancellation-contract: + minimum_protocol_version: "1.20" + worker_capability: cooperative_cancellation + request: + path: /workflows/{workflowId}/request-cancellation + run_remains_active: true + duplicates: return_original_request_and_deadline + incompatible_active_claim: reject_without_history_or_command + request_claim_race: serialized_on_workflow_run_lock + observation: + field: cancellation_request + surfaces: [workflow_task_claim, successful_workflow_task_heartbeat] + identity: [request_id, requested_at, cleanup_deadline_at] + history_refresh_page_token: server_issued_first_bounded_history_page + observation_alone: does_not_throw_before_authored_delivery_boundary + delivery: + path: /worker/workflow-tasks/{taskId}/deliver-cancellation + lease_fences: [namespace, task_id, lease_owner, workflow_task_attempt, unexpired_lease] + capability_proof: immutable_registration_snapshot_at_claim + history_event: CooperativeCancellationDelivered + retry_identity: [request_id, sequence, call_kind, sequence_span, operation_sequence, operation_sequence_span] + prior_completed_calls: replay_normally + marker_retry: exactly_one_canonical_event + workflow_lease: retained_for_bounded_cleanup + affected_open_wait: cancelled_on_committed_delivery + cleanup: + deadline: original_immutable_cleanup_deadline_at + cold_restart: replay_canonical_delivery_and_completed_cleanup_work + completion_outcome: cancelled_with_original_request_id + expiry_or_termination: revoke_remaining_authority + external_effects: at_least_once_with_application_idempotency_or_reconciliation + terminal_cancel_and_terminate: unchanged x-durable-workflow-message-streams-contract: discovery_path: /cluster/info#/message_streams_contract minimum_protocol_version: "1.15" diff --git a/resources/release/source-release.json b/resources/release/source-release.json index a2ed198e..72d9ffc1 100644 --- a/resources/release/source-release.json +++ b/resources/release/source-release.json @@ -1,9 +1,9 @@ { "schema": "durable-workflow.server.source-release/v1", "server": { - "version": "2.4.40" + "version": "2.5.0-rc.1" }, "helm_chart": { - "version": "0.1.136" + "version": "0.1.137-rc.1" } } diff --git a/routes/api.php b/routes/api.php index 77e400ca..e8b87409 100644 --- a/routes/api.php +++ b/routes/api.php @@ -3,6 +3,8 @@ use App\Http\Controllers\Api\ActivityController; use App\Http\Controllers\Api\ActivityTaskController; use App\Http\Controllers\Api\BridgeAdapterController; +use App\Http\Controllers\Api\CancellationScopeController; +use App\Http\Controllers\Api\CooperativeCancellationController; use App\Http\Controllers\Api\DeploymentController; use App\Http\Controllers\Api\EmbeddedV2ImportController; use App\Http\Controllers\Api\HealthController; @@ -10,6 +12,7 @@ use App\Http\Controllers\Api\LegacyV1ProjectionController; use App\Http\Controllers\Api\MessageStreamController; use App\Http\Controllers\Api\NamespaceController; +use App\Http\Controllers\Api\PreparedLocalActivityController; use App\Http\Controllers\Api\RuntimeCredentialController; use App\Http\Controllers\Api\RuntimeExternalPayloadController; use App\Http\Controllers\Api\ScheduleController; @@ -140,6 +143,7 @@ Route::post('/{workflowId}/query/{queryName}', [WorkflowController::class, 'query']); Route::post('/{workflowId}/update/{updateName}', [WorkflowController::class, 'update']); Route::post('/{workflowId}/cancel', [WorkflowController::class, 'cancel']); + Route::post('/{workflowId}/request-cancellation', [CooperativeCancellationController::class, 'request']); Route::post('/{workflowId}/terminate', [WorkflowController::class, 'terminate']); Route::post('/{workflowId}/repair', [WorkflowController::class, 'repair']); Route::post('/{workflowId}/archive', [WorkflowController::class, 'archive']); @@ -149,6 +153,7 @@ Route::post('/{workflowId}/runs/{runId}/query/{queryName}', [WorkflowController::class, 'queryRun']); Route::post('/{workflowId}/runs/{runId}/update/{updateName}', [WorkflowController::class, 'updateRun']); Route::post('/{workflowId}/runs/{runId}/cancel', [WorkflowController::class, 'cancelRun']); + Route::post('/{workflowId}/runs/{runId}/request-cancellation', [CooperativeCancellationController::class, 'request']); Route::post('/{workflowId}/runs/{runId}/terminate', [WorkflowController::class, 'terminateRun']); Route::post('/{workflowId}/runs/{runId}/repair', [WorkflowController::class, 'repairRun']); Route::post('/{workflowId}/runs/{runId}/redrive', [WorkflowController::class, 'redriveRun']); @@ -230,6 +235,19 @@ Route::post('/workflow-tasks/poll', [WorkerController::class, 'pollWorkflowTasks']); Route::post('/workflow-tasks/{taskId}/history', [WorkerController::class, 'workflowTaskHistory']); Route::post('/workflow-tasks/{taskId}/heartbeat', [WorkerController::class, 'heartbeatWorkflowTask']); + Route::post('/workflow-tasks/{taskId}/deliver-cancellation', [CooperativeCancellationController::class, 'deliver']); + Route::post('/workflow-tasks/{taskId}/cancellation-scopes/open', [CancellationScopeController::class, 'open']); + Route::post('/workflow-tasks/{taskId}/cancellation-scopes/prepare', [CancellationScopeController::class, 'prepare']); + Route::post('/workflow-tasks/{taskId}/cancellation-scopes/deliver', [CancellationScopeController::class, 'deliver']); + Route::post('/workflow-tasks/{taskId}/cancellation-scopes/checkpoint', [WorkerController::class, 'checkpointCancellationScopePrefix']); + Route::post('/workflow-tasks/{taskId}/local-activities/prepare', [PreparedLocalActivityController::class, 'prepare']); + Route::post('/workflow-tasks/{taskId}/local-activities/checkpoint', [WorkerController::class, 'checkpointLocalActivityPrefix']); + Route::post('/workflow-tasks/{taskId}/local-activities/checkpoint-group', [WorkerController::class, 'checkpointLocalActivityGroup']); + Route::post('/workflow-tasks/{taskId}/local-activities/recover', [PreparedLocalActivityController::class, 'recover']); + Route::post('/workflow-tasks/{taskId}/local-activities/{attemptId}/control', [PreparedLocalActivityController::class, 'control']); + Route::post('/workflow-tasks/{taskId}/local-activities/{attemptId}/heartbeat', [PreparedLocalActivityController::class, 'heartbeat']); + Route::post('/workflow-tasks/{taskId}/local-activities/{attemptId}/outcome', [PreparedLocalActivityController::class, 'outcome']); + Route::post('/workflow-tasks/{taskId}/local-activities/{attemptId}/acknowledge-cancellation', [PreparedLocalActivityController::class, 'acknowledgeCancellation']); Route::post('/workflow-tasks/{taskId}/complete', [WorkerController::class, 'completeWorkflowTask']); Route::post('/workflow-tasks/{taskId}/fail', [WorkerController::class, 'failWorkflowTask']); @@ -245,6 +263,8 @@ // Activity tasks (long-poll) Route::post('/activity-tasks/poll', [ActivityTaskController::class, 'poll']); + Route::post('/activity-tasks/{taskId}/status', [ActivityTaskController::class, 'status']); + Route::post('/activity-tasks/{taskId}/acknowledge-cancellation', [ActivityTaskController::class, 'acknowledgeCancellation']); Route::post('/activity-tasks/{taskId}/complete', [ActivityTaskController::class, 'complete']); Route::post('/activity-tasks/{taskId}/fail', [ActivityTaskController::class, 'fail']); Route::post('/activity-tasks/{taskId}/heartbeat', [ActivityTaskController::class, 'heartbeat']); diff --git a/scripts/ci/helm_chart_release.py b/scripts/ci/helm_chart_release.py index 486a438a..aa801419 100644 --- a/scripts/ci/helm_chart_release.py +++ b/scripts/ci/helm_chart_release.py @@ -187,11 +187,18 @@ def validate_source(chart_path: Path = DEFAULT_CHART_PATH) -> dict[str, Any]: return metadata -def semver_key(version: str) -> tuple[int, int, int]: +def semver_key(version: str) -> tuple[int, int, int, int, int]: match = SEMVER_PATTERN.fullmatch(version) - if match is None or match.group(4): - raise ReleaseError(f"chart version must be a stable numeric SemVer: {version}") - return tuple(int(match.group(index)) for index in range(1, 4)) + if match is None: + raise ReleaseError(f"chart version must be an exact SemVer: {version}") + stage_order, number = 3, 0 + if match.group(4): + prerelease = re.fullmatch(r"(alpha|beta|rc)\.(0|[1-9][0-9]*)", match.group(4)) + if prerelease is None: + raise ReleaseError(f"unsupported chart prerelease: {version}") + stage_order = {"alpha": 0, "beta": 1, "rc": 2}[prerelease.group(1)] + number = int(prerelease.group(2)) + return (int(match.group(1)), int(match.group(2)), int(match.group(3)), stage_order, number) def git_output(arguments: list[str]) -> str: diff --git a/scripts/ci/sync-source-release.mjs b/scripts/ci/sync-source-release.mjs index 511c012d..13a9089a 100644 --- a/scripts/ci/sync-source-release.mjs +++ b/scripts/ci/sync-source-release.mjs @@ -10,7 +10,7 @@ const defaultRepositoryRoot = resolve(fileURLToPath(new URL('../..', import.meta const schema = 'durable-workflow.server.source-release/v1'; const prerelease = String.raw`(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)-(?:alpha|beta|rc)\.(?:0|[1-9]\d*)`; const stableVersion = String.raw`(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)`; -const serverVersion = String.raw`(?:${stableVersion}|${prerelease})`; +const serverVersion = String.raw`(?:${prerelease}|${stableVersion})`; const remediation = 'node scripts/ci/sync-source-release.mjs --write'; function fail(message) { @@ -52,7 +52,7 @@ export async function sourceRelease(repositoryRoot = defaultRepositoryRoot) { ), chartVersion: exact( object(record.helm_chart, 'source release helm_chart').version, - stableVersion, + serverVersion, 'source release helm_chart.version', ), }; @@ -265,7 +265,7 @@ const consumers = [ render(source, release) { let rendered = replaceExactly( source, - new RegExp(`^(version:\\s*)${stableVersion}\\s*$`, 'm'), + new RegExp(`^(version:\\s*)${serverVersion}\\s*$`, 'm'), (_match, prefix) => `${prefix}${release.chartVersion}`, 1, this.path, diff --git a/scripts/ci/test-source-release.mjs b/scripts/ci/test-source-release.mjs index b0aa4ca8..efb47a1a 100644 --- a/scripts/ci/test-source-release.mjs +++ b/scripts/ci/test-source-release.mjs @@ -99,7 +99,7 @@ test('a stable server source identity fans out through current release consumers const manifestPath = join(temporaryRoot, 'resources/release/source-release.json'); const manifest = JSON.parse(await readFile(manifestPath, 'utf8')); manifest.server.version = '2.0.0'; - manifest.helm_chart.version = nextPatch(manifest.helm_chart.version); + manifest.helm_chart.version = nextServerRelease(manifest.helm_chart.version); await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); const write = runGenerator(temporaryRoot, ['--write']); @@ -127,7 +127,7 @@ test('a simulated next release fans out without rewriting history or retaining s const previousServer = manifest.server.version; const previousChart = manifest.helm_chart.version; const nextServer = nextServerRelease(previousServer); - const nextChart = nextPatch(previousChart); + const nextChart = nextServerRelease(previousChart); manifest.server.version = nextServer; manifest.helm_chart.version = nextChart; await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); diff --git a/scripts/ci/test_helm_chart_release.py b/scripts/ci/test_helm_chart_release.py index c2d5928d..89c24885 100644 --- a/scripts/ci/test_helm_chart_release.py +++ b/scripts/ci/test_helm_chart_release.py @@ -448,6 +448,19 @@ def test_chart_qualification_uses_only_source_controlled_tool_versions( for caller in ("helm-chart-validation.yml", "helm-chart-release.yml"): self.assertIn('- "uv.toml"', workflow_source(caller)) + def test_chart_prerelease_order_preserves_stable_precedence(self) -> None: + versions = [ + "0.1.136", "0.1.137-alpha.1", "0.1.137-alpha.2", "0.1.137-beta.1", + "0.1.137-rc.1", "0.1.137-rc.2", "0.1.137", "0.1.138-rc.1", + ] + for earlier, later in zip(versions, versions[1:]): + with self.subTest(earlier=earlier, later=later): + self.assertLess(RELEASE.semver_key(earlier), RELEASE.semver_key(later)) + for invalid in ["latest", "0.1.137-rc.01", "0.1.137-preview.1"]: + with self.subTest(invalid=invalid): + with self.assertRaises(RELEASE.ReleaseError): + RELEASE.semver_key(invalid) + def test_current_source_has_synchronized_public_identity(self) -> None: metadata = RELEASE.validate_source() source_release = RELEASE.source_release_metadata() diff --git a/scripts/k8s-kind-smoke.sh b/scripts/k8s-kind-smoke.sh index b7a88bd2..30fc9394 100755 --- a/scripts/k8s-kind-smoke.sh +++ b/scripts/k8s-kind-smoke.sh @@ -7,7 +7,7 @@ cluster="${K8S_SMOKE_CLUSTER:-durable-workflow-server-smoke}" image="${K8S_SMOKE_IMAGE:-durableworkflow/server:k8s-smoke}" # Generated by scripts/ci/sync-source-release.mjs so the smoke replaces the # same default shipped by the public manifests. -manifest_image="durableworkflow/server:2.4.40" +manifest_image="durableworkflow/server:2.5.0-rc.1" kind_node_image="${K8S_SMOKE_KIND_NODE_IMAGE:-kindest/node:v1.29.4}" artifact_dir="${K8S_SMOKE_ARTIFACT_DIR:-/tmp/durable-workflow-k8s-kind-smoke-artifacts}" rendered_dir="${artifact_dir}/rendered-manifests" diff --git a/scripts/perf/mysql-lock-diagnostics.sql b/scripts/perf/mysql-lock-diagnostics.sql new file mode 100644 index 00000000..3abf676d --- /dev/null +++ b/scripts/perf/mysql-lock-diagnostics.sql @@ -0,0 +1,16 @@ +SELECT NOW(6) AS captured_at, @@innodb_print_all_deadlocks AS deadlock_logging, + @@log_error_verbosity AS error_log_verbosity; +SHOW GLOBAL STATUS LIKE 'Innodb_deadlocks'; +SHOW ENGINE INNODB STATUS; +SELECT THREAD_ID, EVENT_ID, CURRENT_SCHEMA, SQL_TEXT, MYSQL_ERRNO, + RETURNED_SQLSTATE, MESSAGE_TEXT, TIMER_WAIT, LOCK_TIME, ROWS_AFFECTED +FROM performance_schema.events_statements_history_long +WHERE MYSQL_ERRNO IN (1205, 1213) +ORDER BY THREAD_ID, EVENT_ID +LIMIT 100; +SELECT ENGINE_TRANSACTION_ID, THREAD_ID, OBJECT_SCHEMA, OBJECT_NAME, + INDEX_NAME, LOCK_TYPE, LOCK_MODE, LOCK_STATUS +FROM performance_schema.data_locks +WHERE OBJECT_SCHEMA = 'durable_workflow' +LIMIT 100; +SELECT * FROM performance_schema.data_lock_waits LIMIT 100; diff --git a/scripts/perf/run-server-soak.sh b/scripts/perf/run-server-soak.sh index 9ca3b46f..f188037d 100755 --- a/scripts/perf/run-server-soak.sh +++ b/scripts/perf/run-server-soak.sh @@ -33,6 +33,7 @@ POLL_TIMEOUT="${DW_PERF_POLL_TIMEOUT:-1}" POLL_INTERVAL_MS="${DW_PERF_POLL_INTERVAL_MS:-50}" POLL_SIGNAL_CHECK_INTERVAL_MS="${DW_PERF_POLL_SIGNAL_CHECK_INTERVAL_MS:-25}" READINESS_DIAGNOSTICS="${DW_PERF_READINESS_DIAGNOSTICS:-0}" +MYSQL_LOCK_DIAGNOSTICS="${DW_PERF_MYSQL_LOCK_DIAGNOSTICS:-0}" PUBLISHED_SERVER_IMAGE="${DW_PERF_PUBLISHED_SERVER_IMAGE:-}" HTTP_VARIANT="${DW_PERF_HTTP_VARIANT:-apache}" PREBUILT_HTTP_IMAGE_ID="${DW_PERF_PREBUILT_HTTP_IMAGE_ID:-}" @@ -72,6 +73,10 @@ if [[ "$READINESS_DIAGNOSTICS" != 0 && "$READINESS_DIAGNOSTICS" != 1 ]]; then echo "DW_PERF_READINESS_DIAGNOSTICS must be 0 or 1." >&2 exit 2 fi +if [[ "$MYSQL_LOCK_DIAGNOSTICS" != 0 && "$MYSQL_LOCK_DIAGNOSTICS" != 1 ]]; then + echo "DW_PERF_MYSQL_LOCK_DIAGNOSTICS must be 0 or 1." >&2 + exit 2 +fi if [[ -n "$PUBLISHED_SERVER_IMAGE" \ && ! "$PUBLISHED_SERVER_IMAGE" =~ ^durableworkflow/server@sha256:[0-9a-f]{64}$ ]]; then echo "DW_PERF_PUBLISHED_SERVER_IMAGE must be an exact durableworkflow/server sha256 digest." >&2 @@ -317,6 +322,12 @@ fi cleanup() { local status=$? + if [[ "$MYSQL_LOCK_DIAGNOSTICS" == 1 ]]; then + timeout 15s docker exec -i -e MYSQL_PWD=root "${PROJECT}-mysql-1" \ + mysql --user=root --batch --raw \ + < "$ROOT_DIR/scripts/perf/mysql-lock-diagnostics.sql" \ + > "$ARTIFACT_DIR/mysql-lock-diagnostics.txt" 2>&1 || true + fi docker logs "${PROJECT}-server-1" > "$ARTIFACT_DIR/server.log" 2>&1 || true if [[ "$HTTP_VARIANT" == nginx-fpm || "$HTTP_VARIANT" == apache-event-fpm ]]; then docker logs "${PROJECT}-fpm-1" > "$ARTIFACT_DIR/fpm.log" 2>&1 || true @@ -496,6 +507,12 @@ if [ "$setup_status" -ne 0 ]; then exit "$setup_status" fi +if [[ "$MYSQL_LOCK_DIAGNOSTICS" == 1 ]]; then + timeout 15s docker exec -e MYSQL_PWD=root "${PROJECT}-mysql-1" \ + mysql --user=root --execute="SET GLOBAL log_error_verbosity=3; SET GLOBAL innodb_print_all_deadlocks=ON; UPDATE performance_schema.setup_consumers SET ENABLED='YES' WHERE NAME='events_statements_history_long';" \ + > "$ARTIFACT_DIR/mysql-lock-diagnostics-setup.txt" 2>&1 +fi + if [[ -n "$PUBLISHED_SERVER_IMAGE" ]]; then for service in bootstrap server worker scheduler; do actual_image_id="$(docker inspect "${PROJECT}-${service}-1" --format '{{.Image}}')" diff --git a/tests/Feature/ActivityAttemptStatusTest.php b/tests/Feature/ActivityAttemptStatusTest.php new file mode 100644 index 00000000..2abaacd3 --- /dev/null +++ b/tests/Feature/ActivityAttemptStatusTest.php @@ -0,0 +1,539 @@ + '1.20']); + WorkflowNamespace::query()->create([ + 'name' => 'default', 'description' => 'Test', 'retention_days' => 30, 'status' => 'active', + ]); + } + + protected function tearDown(): void + { + $this->removeStoragePressure(); + parent::tearDown(); + } + + public function test_observation_preserves_user_progress_lease_and_registration(): void + { + $task = $this->lease(); + $execution = ActivityExecution::query()->findOrFail($task['activity_execution_id']); + $execution->forceFill(['retry_policy' => ['heartbeat_timeout' => 10]])->save(); + $this->withHeaders($this->headers())->postJson($this->path($task, 'heartbeat'), $this->fence($task) + [ + 'message' => 'authored progress', 'current' => 1, 'total' => 3, + ])->assertOk()->assertJsonPath('heartbeat_recorded', true); + $before = $this->snapshot($task); + $this->travel(2)->seconds(); + $response = $this->observe($task)->assertOk() + ->assertJsonPath('can_continue', true) + ->assertJsonPath('cancel_requested', false) + ->assertJsonPath('heartbeat_recorded', false) + ->assertJsonPath('task_id', $task['task_id']) + ->assertJsonPath('activity_attempt_id', $task['activity_attempt_id']) + ->assertJsonPath('lease_owner', $task['lease_owner']); + $this->assertSame($before, $this->snapshot($task)); + OpenApiSchema::fromFile(base_path('resources/platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/ActivityTaskStatusResponse/allOf/1', json_decode($response->getContent(), flags: JSON_THROW_ON_ERROR)); + } + + public function test_pending_request_does_not_cancel_before_canonical_delivery(): void + { + $task = $this->lease(); + $this->withHeaders($this->headers())->postJson("/api/workflows/{$task['workflow_id']}/request-cancellation", []) + ->assertStatus(202)->assertJsonPath('accepted', true); + $before = $this->snapshot($task); + $this->observe($task)->assertOk()->assertJsonPath('can_continue', true) + ->assertJsonPath('cancel_requested', false)->assertJsonPath('heartbeat_recorded', false); + $this->assertSame($before, $this->snapshot($task)); + } + + public function test_required_session_is_observed_without_renewal(): void + { + $task = $this->lease(); + $session = $this->requireSession($task); + $before = $session->fresh()->getRawOriginal(); + $this->travel(2)->seconds(); + $this->observe($task)->assertOk()->assertJsonPath('can_continue', true) + ->assertJsonPath('worker_session.session_id', 'required-session') + ->assertJsonPath('worker_session.lease_owner', $task['lease_owner']); + $this->assertSame($before, $session->fresh()->getRawOriginal()); + } + + public static function sessionProvider(): array + { + return [['lease_expires_at'], ['ttl_expires_at'], ['lease_owner'], ['status'], ['absent']]; + } + + #[DataProvider('sessionProvider')] + public function test_required_session_loss_refuses_continuation_without_repair(string $field): void + { + $task = $this->lease(); + $session = $this->requireSession($task); + if ($field === 'absent') { + $session->delete(); + } else { + $value = match ($field) { + 'lease_owner' => 'replacement-worker', + 'status' => 'closed', + default => now(), + }; + $session->forceFill([$field => $value])->save(); + } + $before = WorkerSessionLease::query()->get()->map->getRawOriginal()->all(); + $this->observe($task)->assertOk()->assertJsonPath('can_continue', false) + ->assertJsonPath('reason', 'worker_session_unavailable'); + $this->assertSame($before, WorkerSessionLease::query()->get()->map->getRawOriginal()->all()); + } + + public function test_operator_credentials_cannot_observe_worker_attempts(): void + { + $task = $this->lease(); + config(['server.auth.driver' => 'token', 'server.auth.role_tokens' => [ + 'operator' => 'fixture-operator', 'worker' => 'fixture-worker', + ]]); + $this->withHeaders($this->headers() + ['Authorization' => 'Bearer fixture-operator']) + ->postJson($this->path($task), $this->fence($task))->assertForbidden()->assertJsonPath('reason', 'forbidden'); + $this->withHeaders($this->headers() + ['Authorization' => 'Bearer fixture-worker']) + ->postJson($this->path($task), $this->fence($task))->assertOk()->assertJsonPath('can_continue', true); + } + + public function test_expired_lease_cannot_continue_before_repair_runs(): void + { + $task = $this->lease(); + $before = $this->snapshot($task); + $this->travel(5)->minutes(); + $this->observe($task)->assertOk()->assertJsonPath('can_continue', false) + ->assertJsonPath('reason', 'lease_expired')->assertJsonPath('heartbeat_recorded', false); + $this->assertSame($before, $this->snapshot($task)); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + } + + public static function deadlineProvider(): array + { + return [ + ['heartbeat_deadline_at', 'heartbeat_timeout'], + ['close_deadline_at', 'start_to_close_timeout'], + ['schedule_to_close_deadline_at', 'schedule_to_close_timeout'], + ]; + } + + #[DataProvider('deadlineProvider')] + public function test_expired_application_deadline_is_not_hidden_by_observation(string $field, string $reason): void + { + $task = $this->lease(); + ActivityExecution::query()->findOrFail($task['activity_execution_id']) + ->forceFill([$field => now()])->save(); + $before = $this->snapshot($task); + $this->observe($task)->assertOk()->assertJsonPath('can_continue', false) + ->assertJsonPath('reason', $reason)->assertJsonPath('heartbeat_recorded', false); + $this->assertSame($before, $this->snapshot($task)); + $this->assertSame(ActivityStatus::Running, ActivityExecution::query()->findOrFail($task['activity_execution_id'])->status); + } + + public function test_canonical_activity_cancellation_stops_without_another_history_event(): void + { + $task = $this->lease(); + $execution = ActivityExecution::query()->findOrFail($task['activity_execution_id']); + ActivityCancellation::record( + WorkflowRun::query()->findOrFail($task['run_id']), $execution, + WorkflowTask::query()->findOrFail($task['task_id']), + ); + $before = $this->snapshot($task); + $this->observe($task)->assertOk()->assertJsonPath('can_continue', false) + ->assertJsonPath('cancel_requested', true)->assertJsonPath('heartbeat_recorded', false); + $this->assertSame($before, $this->snapshot($task)); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id']) + ->where('event_type', HistoryEventType::ActivityCancelled->value)->count()); + } + + public static function closureProvider(): array + { + return [['cancel', 'run_cancelled'], ['terminate', 'run_terminated']]; + } + + #[DataProvider('closureProvider')] + public function test_terminal_close_is_observed_without_progress(string $action, string $reason): void + { + $task = $this->lease(); + $this->withHeaders($this->headers())->postJson("/api/workflows/{$task['workflow_id']}/{$action}", []) + ->assertSuccessful(); + $before = $this->snapshot($task); + $this->observe($task)->assertOk()->assertJsonPath('can_continue', false) + ->assertJsonPath('cancel_requested', true)->assertJsonPath('reason', $reason); + $this->assertSame($before, $this->snapshot($task)); + } + + public function test_namespace_task_attempt_and_owner_fences_are_required(): void + { + $task = $this->lease(); + $other = $this->lease('other'); + $this->observe($task, ['lease_owner' => 'different'])->assertStatus(409) + ->assertJsonPath('reason', 'lease_owner_mismatch'); + $this->observe($task, ['activity_attempt_id' => $other['activity_attempt_id']])->assertStatus(409) + ->assertJsonPath('reason', 'task_mismatch'); + $this->observe($task, ['activity_attempt_id' => 'absent'])->assertNotFound() + ->assertJsonPath('reason', 'attempt_not_found'); + WorkflowNamespace::query()->create([ + 'name' => 'isolated', 'description' => 'Other', 'retention_days' => 30, 'status' => 'active', + ]); + $this->observe($task, namespace: 'isolated')->assertNotFound()->assertJsonPath('reason', 'task_not_found'); + $this->withHeaders($this->headers())->postJson($this->path($task), [])->assertStatus(422) + ->assertJsonValidationErrors(['activity_attempt_id', 'lease_owner']); + } + + public function test_closed_old_attempt_cannot_continue_after_reclaim(): void + { + $task = $this->lease(); + $model = WorkflowTask::query()->findOrFail($task['task_id']); + $model->forceFill(['status' => TaskStatus::Ready, 'lease_owner' => null, 'lease_expires_at' => null])->save(); + $replacement = app(ActivityTaskBridge::class)->claim($model->id, 'replacement-worker'); + $this->assertNotNull($replacement); + $this->assertNotSame($task['activity_attempt_id'], $replacement['activity_attempt_id']); + $before = $this->snapshot($task); + $this->observe($task)->assertOk()->assertJsonPath('can_continue', false); + $this->assertSame($before, $this->snapshot($task)); + } + + public function test_lock_pressure_does_not_claim_permission_to_continue(): void + { + $task = $this->lease(); + $this->mock(ActivityTaskBridge::class, static function (MockInterface $mock): void { + $mock->shouldReceive('status')->andThrow(new BackendLockPressureException('database is locked')); + $mock->shouldNotReceive('heartbeat'); + }); + $this->observe($task)->assertStatus(503)->assertHeader('Retry-After', '1') + ->assertJsonPath('reason', 'backend_lock_pressure')->assertJsonPath('recorded', false) + ->assertJsonPath('activity_attempt_id', $task['activity_attempt_id']) + ->assertJsonMissingPath('can_continue'); + } + + public function test_read_is_available_when_storage_is_draining_or_fenced(): void + { + $task = $this->lease(); + $this->configureStoragePressure(); + $before = $this->snapshot($task); + foreach (['draining', 'fenced'] as $state) { + $this->observeStoragePressure($state); + $this->observe($task)->assertOk()->assertJsonPath('can_continue', true) + ->assertJsonPath('heartbeat_recorded', false); + } + $this->assertSame($before, $this->snapshot($task)); + } + + public function test_default_and_old_protocols_do_not_enable_observation(): void + { + $task = $this->lease(); + config(['server.worker_protocol.version' => '1.19']); + $this->observe($task)->assertStatus(400)->assertJsonPath('reason', 'unsupported_protocol_version'); + $old = $this->headers(); + $old[WorkerProtocol::HEADER] = '1.19'; + $this->withHeaders($old)->postJson($this->path($task), $this->fence($task)) + ->assertStatus(422)->assertJsonPath('reason', 'activity_attempt_status_unavailable'); + config(['server.worker_protocol.version' => '1.20']); + $this->withHeaders($old)->postJson($this->path($task), $this->fence($task)) + ->assertStatus(422)->assertJsonPath('reason', 'activity_attempt_status_unavailable'); + } + + public function test_stop_acknowledgement_requires_a_capable_installed_runtime(): void + { + $this->assertSame(CooperativeCancellationPolicy::activityAcknowledgementBackendSupported(), WorkerProtocol::serverCapabilities()['activity_cancellation_acknowledgement']); + config(['server.worker_protocol.version' => '1.19']); + $this->assertFalse(WorkerProtocol::serverCapabilities()['activity_cancellation_acknowledgement']); + config(['server.worker_protocol.version' => '1.20']); + $task = $this->lease(capable: true); + $response = $this->acknowledge($task, 'not-yet-requested'); + if (! CooperativeCancellationPolicy::activityAcknowledgementBackendSupported()) { + $response->assertStatus(409)->assertJsonPath('acknowledged', false) + ->assertJsonPath('unavailable', ['installed_runtime_activity_acknowledgement']); + } else { + $response->assertStatus(409)->assertJsonPath('reason', 'cancellation_request_mismatch'); + } + } + + public function test_stop_receipt_is_distinct_from_fencing_and_duplicate_keeps_original_history(): void + { + $this->requireAcknowledgementBackend(); + $task = $this->lease(capable: true); + $request = $this->cancelCooperatively($task); + $before = $this->snapshot($task); + $observed = $this->observe($task)->assertOk()->assertJsonPath('can_continue', false) + ->assertJsonPath('cancellation_acknowledgement.callback_state', 'unknown') + ->assertJsonPath('cancellation_acknowledgement.request_id', $request['request_id']) + ->assertJsonPath('cancellation_acknowledgement.root_request_id', $request['request_id']) + ->assertJsonPath('cancellation_acknowledgement.cleanup_deadline_at', $request['cleanup_deadline_at']); + $this->assertSame($before, $this->snapshot($task)); + $response = $this->acknowledge($task, $request['request_id'])->assertOk() + ->assertJsonPath('acknowledged', true)->assertJsonPath('duplicate', false) + ->assertJsonPath('heartbeat_recorded', false); + $eventId = $response->json('history_event_id'); + $schema = OpenApiSchema::fromFile(base_path('resources/platform-protocol-specs/worker-protocol-api.openapi.yaml')); + $schema->assertReferenceMatches('#/components/schemas/ActivityCancellationAcknowledgementResponse/allOf/1', json_decode($response->getContent(), flags: JSON_THROW_ON_ERROR)); + $schema->assertReferenceMatches('#/components/schemas/ActivityCancellationReceipt', json_decode($observed->getContent(), flags: JSON_THROW_ON_ERROR)->cancellation_acknowledgement); + $this->withHeaders($this->headers())->getJson("/api/workflows/{$task['workflow_id']}/runs/{$task['run_id']}")->assertOk(); + $this->withHeaders($this->headers())->getJson("/api/workflows/{$task['workflow_id']}/runs/{$task['run_id']}/debug")->assertOk(); + $this->withHeaders($this->headers())->postJson($this->path($task, 'complete'), $this->fence($task) + ['result' => null]) + ->assertStatus(409)->assertJsonPath('recorded', false); + $this->withHeaders($this->headers())->postJson($this->path($task, 'fail'), $this->fence($task) + ['failure' => ['message' => 'stale callback failure', 'non_retryable' => true]]) + ->assertStatus(409)->assertJsonPath('recorded', false); + $this->assertSame($before[4] + 1, $this->snapshot($task)[4]); + $this->travel(40)->seconds(); + $this->acknowledge($task, $request['request_id'])->assertOk() + ->assertJsonPath('duplicate', true)->assertJsonPath('history_event_id', $eventId); + $this->observe($task)->assertOk()->assertJsonPath('cancellation_acknowledgement.callback_state', 'stopped') + ->assertJsonPath('cancellation_acknowledgement.history_event_id', $eventId) + ->assertJsonPath('cancellation_acknowledgement.received_after_deadline', false) + ->assertJsonPath('cancellation_acknowledgement.cleanup_deadline_at', $request['cleanup_deadline_at']); + $after = $this->snapshot($task); + $this->assertSame($before[4] + 1, $after[4]); + unset($before[4], $after[4]); + $this->assertSame($before, $after); + $this->withHeaders($this->headers())->postJson($this->path($task, 'complete'), $this->fence($task) + ['result' => null]) + ->assertJsonPath('recorded', false); + } + + public static function mismatchedStopReceipts(): array + { + return [ + 'wrong owner' => ['lease_owner', 'other-owner'], + 'wrong root' => ['root_request_id', 'other-root'], + 'changed original deadline' => ['cleanup_deadline_at', '2099-01-01T00:00:00.000000Z'], + 'callback still running' => ['callback_state', 'running'], + 'wrong evidence source' => ['evidence_source', 'workflow_worker'], + ]; + } + + #[DataProvider('mismatchedStopReceipts')] + public function test_mismatched_stop_receipt_cannot_report_whole_run_callback_as_stopped(string $field, mixed $value): void + { + $this->requireAcknowledgementBackend(); + $task = $this->lease(capable: true); + $request = $this->cancelCooperatively($task); + $ack = $this->acknowledge($task, $request['request_id'])->assertOk()->json('history_event_id'); + $event = WorkflowHistoryEvent::query()->findOrFail($ack); + $payload = $event->payload; + $payload[$field] = $value; + $event->forceFill(['payload' => $payload])->save(); + $before = $this->snapshot($task); + $this->observe($task)->assertOk()->assertJsonPath('cancellation_acknowledgement.callback_state', 'unknown') + ->assertJsonPath('cancellation_acknowledgement.history_event_id', null); + $this->assertSame($before, $this->snapshot($task)); + } + + public function test_late_stop_receipt_cannot_renew_original_deadline(): void + { + $this->requireAcknowledgementBackend(); + $task = $this->lease(capable: true); + $request = $this->cancelCooperatively($task); + $this->travel(31)->seconds(); + $this->acknowledge($task, $request['request_id'])->assertOk()->assertJsonPath('acknowledged', true); + $this->observe($task)->assertOk()->assertJsonPath('cancellation_acknowledgement.received_after_deadline', true) + ->assertJsonPath('cancellation_acknowledgement.cleanup_deadline_at', $request['cleanup_deadline_at']); + $this->assertSame($request['cleanup_deadline_at'], WorkflowRun::query()->findOrFail($task['run_id'])->cancellation_deadline_at->toISOString()); + } + + public function test_changed_registration_cannot_upgrade_or_erase_original_activity_claim(): void + { + $this->requireAcknowledgementBackend(); + foreach ([false, true] as $originallyCapable) { + $task = $this->lease($originallyCapable ? 'capable' : 'legacy', capable: $originallyCapable); + $request = $this->cancelCooperatively($task); + WorkerRegistration::query()->where('worker_id', $task['lease_owner'])->firstOrFail()->forceFill([ + 'capabilities' => $originallyCapable ? [] : [CooperativeCancellationPolicy::CAPABILITY], + ])->save(); + $response = $this->acknowledge($task, $request['request_id']); + if ($originallyCapable) { + $response->assertOk()->assertJsonPath('acknowledged', true); + } else { + $response->assertStatus(409)->assertJsonPath('unavailable', ['worker_claim_capability']); + } + } + } + + public function test_acknowledgement_requires_original_request_owner_task_attempt_and_namespace(): void + { + $this->requireAcknowledgementBackend(); + $task = $this->lease(capable: true); + $other = $this->lease('other', capable: true); + $request = $this->cancelCooperatively($task); + $before = $this->snapshot($task); + $this->acknowledge($task, 'different')->assertStatus(409)->assertJsonPath('reason', 'cancellation_request_mismatch'); + $this->acknowledge($task, $request['request_id'], ['lease_owner' => 'different'])->assertStatus(409)->assertJsonPath('reason', 'lease_owner_mismatch'); + $this->acknowledge($task, $request['request_id'], ['activity_attempt_id' => $other['activity_attempt_id']]) + ->assertStatus(409)->assertJsonPath('reason', 'task_mismatch'); + WorkflowNamespace::query()->create(['name' => 'isolated', 'description' => 'Other', 'retention_days' => 30, 'status' => 'active']); + $this->withHeaders($this->headers('isolated'))->postJson($this->path($task, 'acknowledge-cancellation'), + $this->fence($task) + ['request_id' => $request['request_id']])->assertNotFound(); + $this->assertSame($before, $this->snapshot($task)); + } + + public function test_new_request_protocol_cannot_upgrade_a_legacy_activity_claim(): void + { + $this->requireAcknowledgementBackend(); + $task = $this->lease(capable: true, protocolVersion: '1.19'); + $request = $this->cancelCooperatively($task); + $before = $this->snapshot($task); + $this->acknowledge($task, $request['request_id'])->assertStatus(409) + ->assertJsonPath('unavailable', ['worker_claim_capability']); + $this->assertSame($before, $this->snapshot($task)); + } + + public function test_acknowledgement_accepts_drain_but_fenced_storage_preserves_unknown_stop_state(): void + { + $this->requireAcknowledgementBackend(); + $task = $this->lease(capable: true); + $request = $this->cancelCooperatively($task); + $this->configureStoragePressure(); + $this->observeStoragePressure('fenced'); + $before = $this->snapshot($task); + $this->acknowledge($task, $request['request_id'])->assertStatus(503)->assertJsonPath('request_admitted', false); + $this->observe($task)->assertOk()->assertJsonPath('cancellation_acknowledgement.callback_state', 'unknown'); + $this->assertSame($before, $this->snapshot($task)); + $this->observeStoragePressure('draining'); + $this->acknowledge($task, $request['request_id'])->assertOk()->assertJsonPath('acknowledged', true); + } + + public function test_old_protocol_and_operator_role_cannot_acknowledge_callbacks(): void + { + $task = $this->lease(capable: true); + $old = $this->headers(); + $old[WorkerProtocol::HEADER] = '1.19'; + $this->withHeaders($old)->postJson($this->path($task, 'acknowledge-cancellation'), $this->fence($task) + ['request_id' => 'request']) + ->assertStatus(422)->assertJsonPath('reason', 'activity_cancellation_acknowledgement_not_supported'); + config(['server.auth.driver' => 'token', 'server.auth.role_tokens' => ['operator' => 'fixture-operator', 'worker' => 'fixture-worker']]); + $this->withHeaders($this->headers() + ['Authorization' => 'Bearer fixture-operator']) + ->postJson($this->path($task, 'acknowledge-cancellation'), $this->fence($task) + ['request_id' => 'request']) + ->assertForbidden()->assertJsonPath('reason', 'forbidden'); + } + + private function requireAcknowledgementBackend(): void + { + if (! CooperativeCancellationPolicy::activityAcknowledgementBackendSupported()) { + $this->markTestSkipped('Requires the candidate Native acknowledgement primitive for source qualification.'); + } + } + + private function cancelCooperatively(array $task): array + { + $result = WorkflowStub::load($task['workflow_id'])->requestCancellation('cleanup', 30); + $this->assertTrue($result->accepted()); + $run = WorkflowRun::query()->findOrFail($task['run_id']); + $ready = $run->tasks()->where('task_type', 'workflow')->where('status', 'ready')->sole(); + $bridge = app(WorkflowTaskBridge::class); + $claim = $bridge->claimStatus($ready->id, 'cleanup-owner'); + $this->assertTrue($claim['claimed']); + $this->assertTrue($bridge->deliverCancellation($ready->id, $run->cancellation_request_command_id, 1, 'activity')['delivered']); + + return ['request_id' => $run->cancellation_request_command_id, 'cleanup_deadline_at' => $run->cancellation_deadline_at->toISOString()]; + } + + private function acknowledge(array $task, string $requestId, array $overrides = []): TestResponse + { + return $this->withHeaders($this->headers())->postJson($this->path($task, 'acknowledge-cancellation'), + array_replace($this->fence($task) + ['request_id' => $requestId], $overrides)); + } + + private function lease(string $suffix = 'one', bool $capable = false, string $protocolVersion = '1.20'): array + { + $workflow = WorkflowStub::make(ExternalGreetingWorkflow::class, 'status-'.$suffix); + $start = $workflow->start('Ada'); + NamespaceWorkflowScope::bind('default', $workflow->id(), ExternalGreetingWorkflow::class); + $id = WorkflowTask::query()->where('workflow_run_id', $start->runId()) + ->where('task_type', 'workflow')->where('status', 'ready')->value('id'); + (new RunWorkflowTask($id))->handle(app(WorkflowExecutor::class)); + WorkerRegistration::query()->updateOrCreate(['worker_id' => 'status-worker-'.$suffix, 'namespace' => 'default'], [ + 'task_queue' => 'external-activities', 'runtime' => 'php', + 'supported_activity_types' => ['tests.external-greeting-activity'], + 'capabilities' => $capable ? [CooperativeCancellationPolicy::CAPABILITY] : [], + 'last_heartbeat_at' => now(), 'status' => 'active', + ]); + + return $this->withHeaders(array_replace($this->headers(), [WorkerProtocol::HEADER => $protocolVersion]))->postJson('/api/worker/activity-tasks/poll', [ + 'worker_id' => 'status-worker-'.$suffix, 'task_queue' => 'external-activities', + ])->assertOk()->json('task'); + } + + private function requireSession(array $task): WorkerSessionLease + { + ActivityExecution::query()->findOrFail($task['activity_execution_id']) + ->forceFill(['activity_options' => ['worker_session' => [ + 'session_id' => 'required-session', 'lease_seconds' => 120, + ]]])->save(); + + return WorkerSessionLease::query()->create([ + 'namespace' => 'default', 'session_id' => 'required-session', 'queue' => 'external-activities', + 'status' => 'active', 'lease_owner' => $task['lease_owner'], 'lease_expires_at' => now()->addSeconds(120), + 'ttl_expires_at' => now()->addMinutes(30), 'last_heartbeat_at' => now(), + ]); + } + + private function headers(string $namespace = 'default'): array + { + return ['X-Namespace' => $namespace, 'X-Durable-Workflow-Control-Plane-Version' => '2', WorkerProtocol::HEADER => '1.20']; + } + + private function path(array $task, string $action = 'status'): string + { + return "/api/worker/activity-tasks/{$task['task_id']}/{$action}"; + } + + private function fence(array $task): array + { + return ['activity_attempt_id' => $task['activity_attempt_id'], 'lease_owner' => $task['lease_owner']]; + } + + private function observe(array $task, array $overrides = [], string $namespace = 'default'): TestResponse + { + return $this->withHeaders($this->headers($namespace))->postJson($this->path($task), array_replace($this->fence($task), $overrides)); + } + + private function snapshot(array $task): array + { + return [ + ActivityExecution::query()->findOrFail($task['activity_execution_id'])->getRawOriginal(), + ActivityAttempt::query()->findOrFail($task['activity_attempt_id'])->getRawOriginal(), + WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(), + WorkerRegistration::query()->where('worker_id', $task['lease_owner'])->firstOrFail()->getRawOriginal(), + WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id'])->count(), + ]; + } +} diff --git a/tests/Feature/ClusterInfoCompatibilityTest.php b/tests/Feature/ClusterInfoCompatibilityTest.php index 3bed9ef9..051d1f39 100644 --- a/tests/Feature/ClusterInfoCompatibilityTest.php +++ b/tests/Feature/ClusterInfoCompatibilityTest.php @@ -57,7 +57,7 @@ public function test_default_cluster_info_keeps_compatibility_preflight_bounded( ->assertJsonPath('client_compatibility.authority', 'protocol_manifests') ->assertJsonPath('capabilities.workflow_tasks', true) ->assertJsonPath('platform_protocol_specs.schema', PlatformProtocolSpecs::SCHEMA) - ->assertJsonPath('surface_stability_contract.version', 4) + ->assertJsonPath('surface_stability_contract.version', 5) ->assertJsonPath('activity_runtime_contract.version', 1) ->assertJsonPath('topology.schema', ServerTopology::SCHEMA) ->assertJsonMissingPath('worker_fleet') @@ -707,7 +707,7 @@ public function test_cluster_info_publishes_the_canonical_surface_stability_cont $response = $this->getJson('/api/cluster/info')->assertOk(); $response - ->assertJsonPath('surface_stability_contract.version', 4) + ->assertJsonPath('surface_stability_contract.version', 5) ->assertJsonPath( 'surface_stability_contract.surface_families.official_sdks.package_compatibility.rust_sdk.package', 'durable-workflow', @@ -1038,7 +1038,7 @@ public function test_cluster_info_names_protocol_manifests_as_client_compatibili ->assertJsonPath('client_compatibility.clients.cli.supported_versions', '>=2.0.0,<3.0.0') ->assertJsonPath('client_compatibility.clients.sdk-php.supported_versions', '>=2.0.0,<3.0.0') ->assertJsonPath('client_compatibility.clients.sdk-python.supported_versions', '>=2.0.0,<3.0.0') - ->assertJsonPath('client_compatibility.clients.sdk-rust.supported_versions', '>=2.0.0,<3.0.0'); + ->assertJsonPath('client_compatibility.clients.sdk-rust.supported_versions', '>=2.0.0,<4.0.0'); foreach ($response->json('supported_sdk_versions') as $supportedVersions) { $this->assertStringNotContainsString( diff --git a/tests/Feature/CooperativeCancellationProtocolTest.php b/tests/Feature/CooperativeCancellationProtocolTest.php new file mode 100644 index 00000000..a762f18a --- /dev/null +++ b/tests/Feature/CooperativeCancellationProtocolTest.php @@ -0,0 +1,695 @@ + '1.20', + 'workflows.v2.types.workflows' => ['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class], + ]); + WorkflowNamespace::query()->create([ + 'name' => 'default', 'description' => 'Test', 'retention_days' => 30, 'status' => 'active', + ]); + } + + public function test_request_before_claim_routes_only_to_capable_workers_and_keeps_original_deadline(): void + { + [$workflowId, $runId] = $this->start(); + $this->register('old', false); + $this->register('new', true); + $accepted = $this->requestCancellation($workflowId, ['cleanup_timeout_seconds' => 120]); + $accepted->assertStatus(202)->assertJsonPath('accepted', true)->assertJsonPath('run_id', $runId); + $original = $accepted->json('cancellation_request'); + $this->assertFalse(WorkflowRun::query()->findOrFail($runId)->status->isTerminal()); + $this->poll('old', '1.19')->assertOk()->assertJsonPath('task', null); + $this->poll('new')->assertOk()->assertJsonPath('task.cancellation_request', $original); + $this->travel(10)->seconds(); + $this->requestCancellation($workflowId, ['cleanup_timeout_seconds' => 3600]) + ->assertOk()->assertJsonPath('duplicate', true)->assertJsonPath('cancellation_request', $original); + $this->assertSame(1, $this->eventCount($runId, HistoryEventType::CooperativeCancellationRequested)); + } + + public function test_scoped_transport_refuses_an_installed_bridge_without_the_optional_role(): void + { + $this->app->instance(WorkflowTaskBridge::class, \Mockery::mock(WorkflowTaskBridge::class)); + $before = WorkflowHistoryEvent::query()->count(); + foreach (['prepare', 'deliver'] as $phase) { + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/absent/cancellation-scopes/{$phase}", []) + ->assertConflict()->assertJsonPath('reason', 'cancellation_scope_delivery_unavailable') + ->assertJsonPath('unavailable', ['installed_runtime_scope_preparation_delivery']); + } + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_active_claim_observes_request_on_heartbeat_and_cached_poll_without_losing_lease(): void + { + [$workflowId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new', pollRequestId: 'repeat')->json('task'); + $accepted = $this->requestCancellation($workflowId)->assertStatus(202); + $pending = $accepted->json('cancellation_request'); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/heartbeat", [ + 'lease_owner' => 'new', 'workflow_task_attempt' => $task['workflow_task_attempt'], + ])->assertOk()->assertJsonPath('renewed', true)->assertJsonPath('cancellation_request', $pending); + $this->poll('new', pollRequestId: 'repeat')->assertOk() + ->assertJsonPath('task.task_id', $task['task_id'])->assertJsonPath('task.cancellation_request', $pending); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + } + + public function test_request_refuses_incompatible_claim_without_partial_command_or_history(): void + { + [$workflowId, $runId] = $this->start(); + $this->register('old', false); + $this->poll('old', '1.19')->assertOk(); + $before = WorkflowHistoryEvent::query()->where('workflow_run_id', $runId)->count(); + $this->requestCancellation($workflowId)->assertStatus(409) + ->assertJsonPath('reason', 'active_claim_cancellation_not_supported'); + $this->assertNull(WorkflowRun::query()->findOrFail($runId)->cancellation_request_command_id); + $this->assertSame($before, WorkflowHistoryEvent::query()->where('workflow_run_id', $runId)->count()); + } + + public function test_mutating_registration_cannot_upgrade_an_old_claim(): void + { + [$workflowId] = $this->start(); + $this->register('same-id', false); + $this->poll('same-id', '1.19')->assertOk(); + WorkerRegistration::query()->where('worker_id', 'same-id')->firstOrFail()->forceFill([ + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY], + ])->save(); + $this->requestCancellation($workflowId)->assertStatus(409) + ->assertJsonPath('reason', 'active_claim_cancellation_not_supported'); + } + + public function test_mutating_registration_does_not_erase_a_capable_claims_original_proof(): void + { + [$workflowId] = $this->start(); + $this->register('same-id', true); + $task = $this->poll('same-id')->json('task'); + WorkerRegistration::query()->where('worker_id', 'same-id')->firstOrFail() + ->forceFill(['capabilities' => []])->save(); + $this->requestCancellation($workflowId)->assertStatus(202); + $this->assertTrue(CooperativeCancellationPolicy::claimSupportsCancellation( + WorkflowTask::query()->findOrFail($task['task_id']), + )); + } + + public function test_delivery_retry_records_one_marker_and_keeps_cleanup_authority(): void + { + [$workflowId, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $requestId = $this->requestCancellation($workflowId)->json('cancellation_request.request_id'); + $this->deliver($task, $requestId)->assertOk()->assertJsonPath('delivered', true)->assertJsonPath('sequence', 1); + $this->deliver($task, $requestId)->assertOk()->assertJsonPath('delivered', true); + $this->assertSame(1, $this->eventCount($runId, HistoryEventType::CooperativeCancellationDelivered)); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + $this->deliver($task, $requestId, ['call_kind' => 'timer'])->assertStatus(409) + ->assertJsonPath('reason', 'cancellation_delivery_mismatch'); + } + + #[DataProvider('pendingCancellationKinds')] + public function test_acknowledgement_pending_is_a_valid_reply_with_explicit_claim_release(string $kind, string $reason): void + { + [$workflowId, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $requestId = $this->requestCancellation($workflowId)->json('cancellation_request.request_id'); + $pending = ['delivered' => false, 'task_id' => $task['task_id'], 'workflow_run_id' => $runId, + 'request_id' => null, 'sequence' => null, 'call_kind' => null, 'sequence_span' => null, + 'operation_sequence' => null, 'operation_sequence_span' => null, + 'reason' => $reason, 'claim_released' => true]; + $bridge = \Mockery::mock(CooperativeWorkflowTaskBridge::class); + $bridge->shouldReceive('deliverCancellation')->once()->with( + $task['task_id'], $requestId, 1, $kind, 1, null, 1, + )->andReturn($pending); + $this->app->instance(WorkflowTaskBridge::class, $bridge); + + $response = $this->deliver($task, $requestId, ['call_kind' => $kind])->assertOk() + ->assertJsonPath('delivered', false)->assertJsonPath('reason', $reason) + ->assertJsonPath('claim_released', true); + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/CooperativeCancellationDeliveryResponse/allOf/1', json_decode($response->getContent(), flags: JSON_THROW_ON_ERROR)); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::CooperativeCancellationDelivered)); + } + + public static function pendingCancellationKinds(): array + { + return ['child' => ['child', 'cancellation_waiting_for_child'], + 'remote activity' => ['activity', 'cancellation_waiting_for_activity'], + 'local activity' => ['local_activity', 'cancellation_waiting_for_activity']]; + } + + public static function cancellationRequestTiming(): array + { + return ['before claim' => [true], 'after claim' => [false]]; + } + + #[DataProvider('cancellationRequestTiming')] + public function test_prefix_completion_keeps_pending_cancellation_deliverable(bool $beforeClaim): void + { + [$workflowId, $runId] = $this->start(); + $this->register('original', true); + $requestId = $beforeClaim + ? $this->requestCancellation($workflowId)->assertStatus(202)->json('cancellation_request.request_id') + : null; + $task = $this->poll('original')->assertOk()->json('task'); + $requestId ??= $this->requestCancellation($workflowId)->assertStatus(202)->json('cancellation_request.request_id'); + + $completed = $this->complete($task, [ + ['type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 7)], + ['type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 8)], + ])->assertOk()->assertJsonPath('recorded', true)->assertJsonPath('run_status', 'waiting'); + $this->assertCount(1, $completed->json('created_task_ids')); + $this->assertSame(2, $this->eventCount($runId, HistoryEventType::SideEffectRecorded)); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::CooperativeCancellationDelivered)); + $this->assertSame(1, WorkflowTask::query()->where('workflow_run_id', $runId) + ->where('task_type', 'workflow')->whereIn('status', ['ready', 'leased'])->count()); + $successor = WorkflowTask::query()->findOrFail($completed->json('created_task_ids.0')); + $this->assertSame($requestId, $successor->payload['resume_source_id']); + $this->assertSame('cancellation_request', $successor->payload['resume_source_kind']); + + // A repeated completion cannot publish a second successor or prefix. + $this->complete($task, [ + ['type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 7)], + ])->assertStatus(409); + $this->assertSame(2, WorkflowTask::query()->where('workflow_run_id', $runId) + ->where('task_type', 'workflow')->count()); + $this->assertSame(2, $this->eventCount($runId, HistoryEventType::SideEffectRecorded)); + + $this->register('successor', true); + $resumed = $this->poll('successor')->assertOk()->json('task'); + $this->assertSame($successor->id, $resumed['task_id']); + $this->assertSame($requestId, $resumed['cancellation_request']['request_id']); + $this->deliver($resumed, $requestId, ['sequence' => 3])->assertOk()->assertJsonPath('delivered', true); + $this->complete($resumed, [['type' => 'complete_workflow']])->assertOk() + ->assertJsonPath('run_status', 'cancelled')->assertJsonPath('created_task_ids', []); + $this->assertSame(1, $this->eventCount($runId, HistoryEventType::CooperativeCancellationDelivered)); + $this->assertSame(0, WorkflowTask::query()->where('workflow_run_id', $runId) + ->where('task_type', 'workflow')->whereIn('status', ['ready', 'leased'])->count()); + } + + public function test_delivery_fences_the_actual_claim_attempt_owner_and_request(): void + { + [$workflowId, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $requestId = $this->requestCancellation($workflowId)->json('cancellation_request.request_id'); + $this->deliver($task, $requestId, ['lease_owner' => 'other'])->assertStatus(409) + ->assertJsonPath('reason', 'lease_owner_mismatch'); + $this->deliver($task, $requestId, ['workflow_task_attempt' => 2])->assertStatus(409) + ->assertJsonPath('reason', 'workflow_task_attempt_mismatch'); + $this->deliver($task, 'different')->assertStatus(409)->assertJsonPath('reason', 'cancellation_request_mismatch'); + $this->deliver($task, $requestId, ['sequence' => 9])->assertStatus(409) + ->assertJsonPath('reason', 'cancellation_delivery_sequence_mismatch'); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::CooperativeCancellationDelivered)); + } + + public function test_worker_reloads_canonical_delivery_after_acknowledgment_loss_with_the_observed_token(): void + { + [$workflowId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $pending = $this->requestCancellation($workflowId)->assertStatus(202)->json('cancellation_request'); + $this->deliver($task, $pending['request_id'])->assertOk(); + // The old task payload predates the request and marker. Read their + // real canonical history rather than constructing an event locally. + $history = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/history", [ + 'lease_owner' => 'new', + 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'next_history_page_token' => $pending['history_refresh_page_token'], + ])->assertOk(); + $markers = array_values(array_filter($history->json('history_events'), static fn ($event) => $event['event_type'] === HistoryEventType::CooperativeCancellationDelivered->value)); + $this->assertCount(1, $markers); + $this->assertSame($pending['request_id'], $markers[0]['workflow_command_id']); + $this->assertSame(1, $markers[0]['payload']['sequence']); + $this->assertNotContains(HistoryEventType::CooperativeCancellationDelivered->value, array_column($task['history_events'], 'event_type')); + } + + public function test_cleanup_deadline_revokes_delivery_authority(): void + { + [$workflowId, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $requestId = $this->requestCancellation($workflowId, ['cleanup_timeout_seconds' => 1]) + ->json('cancellation_request.request_id'); + $this->travel(2)->seconds(); + $this->deliver($task, $requestId)->assertStatus(409)->assertJsonPath('reason', 'run_cancelled'); + $this->assertSame('cancelled', WorkflowRun::query()->findOrFail($runId)->status->value); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::CooperativeCancellationDelivered)); + } + + public function test_terminal_cancel_keeps_its_existing_semantics_and_request_is_separate(): void + { + [$workflowId, $runId] = $this->start(); + $this->withHeaders($this->controlHeaders())->postJson("/api/workflows/{$workflowId}/cancel", [])->assertOk(); + $this->requestCancellation($workflowId)->assertStatus(409)->assertJsonPath('reason', 'run_not_active'); + $this->assertSame('cancelled', WorkflowRun::query()->findOrFail($runId)->status->value); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::CooperativeCancellationRequested)); + } + + public function test_request_is_unavailable_until_the_server_protocol_supports_it(): void + { + [$workflowId] = $this->start(); + config(['server.worker_protocol.version' => '1.19']); + $this->requestCancellation($workflowId)->assertStatus(409) + ->assertJsonPath('reason', 'cooperative_cancellation_not_supported'); + } + + public function test_discovery_matches_request_support_and_defaults_to_the_cooperative_protocol(): void + { + foreach (['1.19' => false, '1.20' => true, '2.0' => false, 'malformed' => false] as $version => $supported) { + config(['server.worker_protocol.version' => $version]); + $this->withHeaders($this->controlHeaders())->getJson('/api/cluster/info') + ->assertOk() + ->assertJsonPath('worker_protocol.version', $version) + ->assertJsonPath('worker_protocol.server_capabilities.cooperative_cancellation', $supported); + $this->assertSame($supported, CooperativeCancellationPolicy::serverSupported()); + } + + $this->assertSame('1.20', WorkerProtocol::VERSION); + } + + public function test_waiting_timer_is_interrupted_only_when_delivery_is_committed(): void + { + [$workflowId, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $this->complete($task, [['type' => 'start_timer', 'delay_seconds' => 300]]) + ->assertOk()->assertJsonPath('run_status', 'waiting'); + $requestId = $this->requestCancellation($workflowId)->assertStatus(202)->json('cancellation_request.request_id'); + $this->assertSame('pending', WorkflowTimer::query()->where('workflow_run_id', $runId)->firstOrFail()->status->value); + $resumed = $this->poll('new')->assertOk()->json('task'); + $this->deliver($resumed, $requestId, ['call_kind' => 'timer'])->assertOk(); + $this->assertSame('cancelled', WorkflowTimer::query()->where('workflow_run_id', $runId)->firstOrFail()->status->value); + $this->assertSame(1, $this->eventCount($runId, HistoryEventType::TimerCancelled)); + $this->complete($resumed, [['type' => 'complete_workflow']])->assertOk()->assertJsonPath('run_status', 'cancelled'); + $event = WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::WorkflowCancelled->value)->firstOrFail(); + $this->assertSame($requestId, $event->workflow_command_id); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::WorkflowCompleted)); + } + + public function test_prior_committed_result_cannot_become_the_cancellation_boundary(): void + { + [$workflowId, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $run = WorkflowRun::query()->findOrFail($runId); + // A completed timer represents a durable call resolved before the request. + WorkflowHistoryEvent::record($run, HistoryEventType::TimerScheduled, [ + 'sequence' => 1, 'timer_id' => 'completed-timer', 'delay_seconds' => 1, + ]); + WorkflowHistoryEvent::record($run, HistoryEventType::TimerFired, [ + 'sequence' => 1, 'timer_id' => 'completed-timer', + ]); + $requestId = $this->requestCancellation($workflowId)->json('cancellation_request.request_id'); + $this->deliver($task, $requestId, ['call_kind' => 'timer'])->assertStatus(409) + ->assertJsonPath('reason', 'cancellation_delivery_not_eligible'); + $this->deliver($task, $requestId, ['sequence' => 2])->assertOk()->assertJsonPath('sequence', 2); + $this->assertSame(1, $this->eventCount($runId, HistoryEventType::CooperativeCancellationDelivered)); + } + + public function test_termination_during_cleanup_revokes_late_delivery_and_completion(): void + { + [$workflowId, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $requestId = $this->requestCancellation($workflowId)->json('cancellation_request.request_id'); + $this->deliver($task, $requestId)->assertOk(); + $this->withHeaders($this->controlHeaders())->postJson("/api/workflows/{$workflowId}/terminate", [])->assertOk(); + $this->deliver($task, $requestId)->assertStatus(409)->assertJsonPath('reason', 'task_not_leased'); + $this->complete($task, [['type' => 'complete_workflow']])->assertStatus(409)->assertJsonPath('can_continue', false); + $this->assertSame('terminated', WorkflowRun::query()->findOrFail($runId)->status->value); + $this->assertSame(1, $this->eventCount($runId, HistoryEventType::WorkflowTerminated)); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::WorkflowCompleted)); + } + + public function test_claim_proof_cannot_be_reused_for_another_attempt(): void + { + [$workflowId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + WorkflowTask::query()->findOrFail($task['task_id'])->increment('attempt_count'); + $this->requestCancellation($workflowId)->assertStatus(409) + ->assertJsonPath('reason', 'active_claim_cancellation_not_supported'); + } + + public function test_real_reregistration_requires_a_new_claim_and_fences_the_old_attempt(): void + { + [$workflowId] = $this->start(); + $this->register('same-id', false); + $oldTask = $this->poll('same-id', '1.19')->json('task'); + $this->register('same-id', true, '2026-09-30T00:01:00Z'); + $requestId = $this->requestCancellation($workflowId)->assertStatus(202)->json('cancellation_request.request_id'); + $newTask = $this->poll('same-id')->assertOk()->json('task'); + $this->assertSame($oldTask['task_id'], $newTask['task_id']); + $this->assertSame(2, $newTask['workflow_task_attempt']); + $this->deliver($oldTask, $requestId)->assertStatus(409) + ->assertJsonPath('reason', 'workflow_task_attempt_mismatch'); + $this->deliver($newTask, $requestId)->assertOk(); + } + + public function test_cold_reclaim_preserves_delivery_and_can_commit_bounded_cleanup(): void + { + [$workflowId, $runId] = $this->start(); + $this->register('same-id', true); + $task = $this->poll('same-id')->json('task'); + $requestId = $this->requestCancellation($workflowId)->json('cancellation_request.request_id'); + $this->deliver($task, $requestId)->assertOk(); + // The process dies after the marker commits, before the acknowledgment + // or any cleanup submission. Re-registration is the real recovery path. + $this->register('same-id', true, '2026-09-30T00:01:00Z'); + $cold = $this->poll('same-id')->assertOk()->json('task'); + $this->assertSame(2, $cold['workflow_task_attempt']); + $markers = array_values(array_filter($cold['history_events'], static fn ($event) => $event['event_type'] === HistoryEventType::CooperativeCancellationDelivered->value)); + $this->assertCount(1, $markers); + $this->assertSame(1, $markers[0]['payload']['sequence']); + $this->deliver($cold, $requestId)->assertOk(); + $this->complete($cold, [['type' => 'start_timer', 'delay_seconds' => 1]]) + ->assertOk()->assertJsonPath('run_status', 'waiting'); + $timerTask = WorkflowTask::query()->where('workflow_run_id', $runId)->where('task_type', 'timer') + ->where('status', TaskStatus::Ready->value)->firstOrFail(); + $this->travel(2)->seconds(); + (new RunTimerTask($timerTask->id))->handle(); + $resumed = $this->poll('same-id')->assertOk()->json('task'); + $this->complete($resumed, [['type' => 'complete_workflow']])->assertOk()->assertJsonPath('run_status', 'cancelled'); + $this->assertSame(1, $this->eventCount($runId, HistoryEventType::CooperativeCancellationDelivered)); + $this->assertSame(1, $this->eventCount($runId, HistoryEventType::TimerScheduled)); + $this->assertSame(1, $this->eventCount($runId, HistoryEventType::WorkflowCancelled)); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::WorkflowCompleted)); + } + + public function test_registration_cannot_advertise_cooperation_below_its_protocol_floor(): void + { + $this->withHeaders($this->headers('1.19'))->postJson('/api/worker/register', [ + 'worker_id' => 'invalid', 'task_queue' => 'cooperative', 'runtime' => 'php', + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY], + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + ])->assertStatus(409)->assertJsonPath('reason', 'cooperative_cancellation_protocol_mismatch'); + $this->assertFalse(WorkerRegistration::query()->where('worker_id', 'invalid')->exists()); + } + + public function test_new_admission_observation_and_delivery_fields_match_the_versioned_schemas(): void + { + [$workflowId] = $this->start(); + $this->register('new', true); + $accepted = $this->requestCancellation($workflowId)->assertStatus(202); + $specDirectory = dirname(__DIR__, 2).'/resources/platform-protocol-specs/'; + OpenApiSchema::fromFile($specDirectory.'control-plane-api.openapi.yaml') + ->assertReferenceMatches('#/components/schemas/CooperativeCancellationAdmission', json_decode($accepted->getContent(), flags: JSON_THROW_ON_ERROR)); + $workerSpec = OpenApiSchema::fromFile($specDirectory.'worker-protocol-api.openapi.yaml'); + $poll = $this->poll('new')->assertOk(); + $task = $poll->json('task'); + $workerSpec->assertReferenceMatches('#/components/schemas/WorkflowTask', json_decode($poll->getContent(), flags: JSON_THROW_ON_ERROR)->task); + $heartbeat = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/heartbeat", [ + 'lease_owner' => 'new', 'workflow_task_attempt' => $task['workflow_task_attempt'], + ])->assertOk(); + $workerSpec->assertReferenceMatches('#/components/schemas/WorkflowTaskHeartbeatResponse/allOf/1', json_decode($heartbeat->getContent(), flags: JSON_THROW_ON_ERROR)); + $delivery = $this->deliver($task, $accepted->json('cancellation_request.request_id'))->assertOk(); + $workerSpec->assertReferenceMatches('#/components/schemas/CooperativeCancellationDeliveryResponse/allOf/1', json_decode($delivery->getContent(), flags: JSON_THROW_ON_ERROR)); + } + + public function test_run_target_and_namespace_are_checked_before_request_admission(): void + { + [$workflowId, $runId] = $this->start(); + $this->withHeaders($this->controlHeaders())->postJson( + "/api/workflows/{$workflowId}/runs/missing/request-cancellation", [], + )->assertNotFound()->assertJsonPath('reason', 'run_not_found'); + $accepted = $this->withHeaders($this->controlHeaders())->postJson( + "/api/workflows/{$workflowId}/runs/{$runId}/request-cancellation", [], + )->assertStatus(202); + $this->assertSame($runId, $accepted->json('run_id')); + $this->requestCancellation('missing')->assertNotFound()->assertJsonPath('reason', 'instance_not_found'); + } + + public function test_child_policy_is_preserved_or_reports_the_missing_installed_runtime(): void + { + [, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $response = $this->complete($task, [[ + 'type' => 'start_child_workflow', + 'workflow_type' => 'tests.external-greeting-workflow', + 'arguments' => Serializer::serialize(['child']), + 'cancellation_policy' => 'wait_cancellation_completed', + 'parent_close_policy' => 'request_cancellation', + ]]); + if (! CooperativeCancellationPolicy::childPolicyBackendSupported()) { + $response->assertStatus(409)->assertJsonPath('reason', 'child_cancellation_policy_not_supported') + ->assertJsonPath('unavailable', ['installed_runtime_child_policy'])->assertJsonPath('recorded', false); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::ChildWorkflowScheduled)); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + + return; + } + $response->assertOk()->assertJsonPath('outcome', 'completed'); + $event = WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::ChildWorkflowScheduled)->sole(); + $this->assertSame('wait_cancellation_completed', $event->payload['cancellation_policy']); + $this->assertSame('request_cancellation', $event->payload['parent_close_policy']); + $this->assertSame('request_cancellation', WorkflowLink::query() + ->where('parent_workflow_run_id', $runId)->where('link_type', 'child_workflow')->sole()->parent_close_policy); + } + + #[DataProvider('remoteActivityPolicies')] + public function test_remote_activity_policy_is_canonical_or_reports_the_missing_installed_runtime(string $policy): void + { + [, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $response = $this->complete($task, [['type' => 'schedule_activity', + 'activity_type' => 'tests.remote', 'arguments' => Serializer::serialize(['work']), + 'cancellation_policy' => $policy, 'schedule_to_close_timeout' => 180]]); + if (! CooperativeCancellationPolicy::remoteActivityPolicyBackendSupported()) { + $response->assertStatus(409)->assertJsonPath('reason', 'activity_cancellation_policy_not_supported') + ->assertJsonPath('unavailable', ['installed_runtime_activity_policy'])->assertJsonPath('recorded', false); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::ActivityScheduled)); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + + return; + } + $response->assertOk()->assertJsonPath('outcome', 'completed'); + $event = WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::ActivityScheduled)->sole(); + $this->assertSame($policy, $event->payload['activity']['cancellation_policy']); + if ($policy === 'abandon') { + $this->assertNotNull($event->payload['activity']['schedule_to_close_deadline_at']); + } + } + + public static function remoteActivityPolicies(): array + { + return [['try_cancel'], ['wait_cancellation_completed'], ['abandon']]; + } + + public function test_remote_activity_policy_cannot_upgrade_an_old_claim_from_a_changed_registration(): void + { + [, $runId] = $this->start(); + $this->register('old', false); + $task = $this->poll('old', '1.19')->json('task'); + WorkerRegistration::query()->where('worker_id', 'old')->sole()->forceFill([ + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY], + ])->save(); + $this->complete($task, [['type' => 'schedule_activity', 'activity_type' => 'tests.remote', + 'cancellation_policy' => 'wait_cancellation_completed']])->assertStatus(409) + ->assertJsonPath('reason', 'activity_cancellation_policy_not_supported')->assertJsonPath('worker_id', 'old') + ->assertJsonPath('recorded', false)->assertJsonFragment(['worker_claim_capability']); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::ActivityScheduled)); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + } + + public function test_remote_abandon_requires_a_total_lifetime_before_scheduling(): void + { + [, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $response = $this->complete($task, [['type' => 'schedule_activity', 'activity_type' => 'tests.remote', + 'cancellation_policy' => 'abandon']]); + if (CooperativeCancellationPolicy::remoteActivityPolicyBackendSupported()) { + $response->assertStatus(422)->assertJsonValidationErrors('commands.0.schedule_to_close_timeout'); + } else { + $response->assertStatus(409)->assertJsonPath('reason', 'activity_cancellation_policy_not_supported'); + } + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::ActivityScheduled)); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + } + + public function test_remote_activity_policy_refuses_completion_on_the_legacy_protocol(): void + { + [, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $this->withHeaders($this->headers('1.19'))->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => 'new', 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [['type' => 'schedule_activity', 'activity_type' => 'tests.remote', + 'cancellation_policy' => 'try_cancel']], + ])->assertStatus(409)->assertJsonPath('reason', 'activity_cancellation_policy_not_supported') + ->assertJsonFragment(['request_protocol'])->assertJsonPath('recorded', false); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::ActivityScheduled)); + } + + public function test_old_claim_cannot_acquire_child_policy_support_from_a_changed_registration(): void + { + [, $runId] = $this->start(); + $this->register('old', false); + $task = $this->poll('old', '1.19')->json('task'); + WorkerRegistration::query()->where('worker_id', 'old')->sole()->forceFill([ + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY], + ])->save(); + $this->complete($task, [[ + 'type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', + 'cancellation_policy' => 'try_cancel', + ]])->assertStatus(409)->assertJsonPath('reason', 'child_cancellation_policy_not_supported') + ->assertJsonPath('worker_id', 'old')->assertJsonPath('recorded', false) + ->assertJsonFragment(['worker_claim_capability']); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::ChildWorkflowScheduled)); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + } + + public function test_capable_claim_still_requires_a_cooperative_completion_protocol(): void + { + [, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + $this->withHeaders($this->headers('1.19')) + ->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [[ + 'type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', + 'parent_close_policy' => 'request_cancellation', + ]], + ])->assertStatus(409)->assertJsonPath('reason', 'child_cancellation_policy_not_supported') + ->assertJsonFragment(['request_protocol']); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::ChildWorkflowScheduled)); + } + + public function test_invalid_or_misplaced_child_cancellation_policy_is_rejected_before_completion(): void + { + [, $runId] = $this->start(); + $this->register('new', true); + $task = $this->poll('new')->json('task'); + foreach ([ + ['type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', 'cancellation_policy' => 'unknown'], + ['type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', 'cancellation_policy' => []], + ['type' => 'start_timer', 'delay_seconds' => 1, 'cancellation_policy' => 'try_cancel'], + ] as $command) { + $this->complete($task, [$command])->assertStatus(422) + ->assertJsonValidationErrors('commands.0.cancellation_policy'); + } + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::ChildWorkflowScheduled)); + $this->assertSame(0, $this->eventCount($runId, HistoryEventType::TimerScheduled)); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + } + + public function test_legacy_terminal_parent_close_policy_keeps_its_old_protocol_path(): void + { + [, $runId] = $this->start(); + $this->register('old', false); + $task = $this->poll('old', '1.19')->json('task'); + $this->withHeaders($this->headers('1.19')) + ->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [[ + 'type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', + 'parent_close_policy' => 'request_cancel', + ]], + ])->assertOk()->assertJsonPath('outcome', 'completed'); + $event = WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::ChildWorkflowScheduled)->sole(); + $this->assertSame('request_cancel', $event->payload['parent_close_policy']); + } + + private function complete(array $task, array $commands): TestResponse + { + return $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => $commands, + ]); + } + + private function start(): array + { + $response = $this->withHeaders($this->controlHeaders())->postJson('/api/workflows', [ + 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'cooperative', 'input' => ['Ada'], + ])->assertCreated(); + + return [$response->json('workflow_id'), $response->json('run_id')]; + } + + private function register(string $workerId, bool $capable, string $processStart = '2026-09-30T00:00:00Z'): void + { + $this->withHeaders($this->headers($capable ? '1.20' : '1.19'))->postJson('/api/worker/register', [ + 'worker_id' => $workerId, + 'task_queue' => 'cooperative', 'runtime' => 'php', 'supported_workflow_types' => ['tests.external-greeting-workflow'], + 'capabilities' => $capable ? [CooperativeCancellationPolicy::CAPABILITY] : [], + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + 'max_concurrent_workflow_tasks' => 1, + 'process_metrics' => ['host' => 'test-worker', 'process_started_at' => $processStart, 'process_id' => 10], + ])->assertCreated(); + } + + private function poll(string $workerId, string $version = '1.20', ?string $pollRequestId = null): TestResponse + { + return $this->withHeaders($this->headers($version))->postJson('/api/worker/workflow-tasks/poll', array_filter([ + 'worker_id' => $workerId, 'task_queue' => 'cooperative', 'poll_request_id' => $pollRequestId, + ], static fn ($value) => $value !== null)); + } + + private function requestCancellation(string $workflowId, array $body = []): TestResponse + { + return $this->withHeaders($this->controlHeaders())->postJson("/api/workflows/{$workflowId}/request-cancellation", $body); + } + + private function deliver(array $task, string $requestId, array $overrides = []): TestResponse + { + return $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/deliver-cancellation", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'request_id' => $requestId, 'sequence' => 1, 'call_kind' => 'activity', ...$overrides, + ]); + } + + private function eventCount(string $runId, HistoryEventType $type): int + { + return WorkflowHistoryEvent::query()->where('workflow_run_id', $runId)->where('event_type', $type->value)->count(); + } + + private function headers(string $version = '1.20'): array + { + return ['X-Namespace' => 'default', WorkerProtocol::HEADER => $version]; + } + + private function controlHeaders(): array + { + return ['X-Namespace' => 'default', 'X-Durable-Workflow-Control-Plane-Version' => '2']; + } +} diff --git a/tests/Feature/CooperativeCancellationRaceTest.php b/tests/Feature/CooperativeCancellationRaceTest.php new file mode 100644 index 00000000..4de41bd8 --- /dev/null +++ b/tests/Feature/CooperativeCancellationRaceTest.php @@ -0,0 +1,274 @@ +assertIsString($path); + $this->databasePath = $path; + config([ + 'database.default' => 'sqlite', + 'database.connections.sqlite.database' => $path, + 'database.connections.sqlite.busy_timeout' => 100, + 'database.connections.sqlite.journal_mode' => 'WAL', + 'database.connections.sqlite.transaction_mode' => 'IMMEDIATE', + 'cache.default' => 'array', + 'server.worker_protocol.version' => '1.20', + 'workflows.v2.types.workflows' => ['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class], + ]); + DB::purge('sqlite'); + $this->artisan('migrate:fresh', ['--force' => true])->assertExitCode(0); + Queue::fake(); + WorkflowNamespace::query()->create([ + 'name' => 'default', 'description' => 'Test', 'retention_days' => 30, 'status' => 'active', + ]); + } + + protected function tearDown(): void + { + DB::disconnect('sqlite'); + foreach ([$this->databasePath, $this->databasePath.'-wal', $this->databasePath.'-shm'] as $path) { + if (is_file($path)) { + unlink($path); + } + } + parent::tearDown(); + } + + public static function workerCapabilities(): array + { + return ['old worker' => [false], 'cooperative worker' => [true]]; + } + + public function test_heartbeat_cannot_acknowledge_an_old_claim_after_concurrent_reclaim(): void + { + $clock = now()->startOfSecond(); + $this->travelTo($clock); + config(['workflows.v2.workflow_task_lease_seconds' => 1]); + $start = $this->withHeaders($this->controlHeaders())->postJson('/api/workflows', [ + 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'heartbeat-race', 'input' => ['Ada'], + ])->assertCreated(); + $runId = $start->json('run_id'); + foreach (['original', 'replacement'] as $workerId) { + WorkerRegistration::query()->create([ + 'namespace' => 'default', 'worker_id' => $workerId, 'task_queue' => 'heartbeat-race', + 'runtime' => 'php', 'supported_workflow_types' => ['tests.external-greeting-workflow'], + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY], + 'max_concurrent_workflow_tasks' => 1, 'last_heartbeat_at' => now(), 'status' => 'active', + ]); + } + $pending = $this->withHeaders($this->controlHeaders()) + ->postJson('/api/workflows/'.$start->json('workflow_id').'/request-cancellation', []) + ->assertStatus(202)->json('cancellation_request'); + $claim = $this->withHeaders([ + 'X-Namespace' => 'default', 'X-Durable-Workflow-Protocol-Version' => '1.20', + ])->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'original', 'task_queue' => 'heartbeat-race', 'poll_request_id' => 'original-poll', + ])->assertOk()->json('task'); + $this->assertIsArray($claim); + + $children = []; + try { + foreach (['heartbeat' => 'original', 'poll' => 'replacement'] as $operation => $workerId) { + $process = proc_open([ + PHP_BINARY, dirname(__DIR__).'/Fixtures/CooperativeCancellationRaceRequest.php', + $this->databasePath, $operation, $claim['task_id'], $workerId, 'heartbeat-race', '1.20', + $clock->toIso8601String(), (string) $claim['workflow_task_attempt'], + ], [0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + $this->assertIsResource($process); + stream_set_timeout($pipes[1], 10); + $children[$operation] = ['process' => $process, 'pipes' => $pipes]; + $this->assertSame("ready\n", fgets($pipes[1])); + } + fwrite($children['heartbeat']['pipes'][0], "go\n"); + $this->assertSame("checked\n", fgets($children['heartbeat']['pipes'][1])); + // The original guard passed before expiry. Advance both actual + // HTTP kernels beyond expiry and let Native try a replacement + // claim while renewal is paused, without editing any lease row. + fwrite($children['poll']['pipes'][0], "go\n"); + $read = [$children['poll']['pipes'][1]]; + $write = $except = null; + stream_select($read, $write, $except, 1); + fwrite($children['heartbeat']['pipes'][0], "go\n"); + + $results = []; + foreach ($children as $operation => $child) { + fclose($child['pipes'][0]); + $output = stream_get_contents($child['pipes'][1]); + $this->assertFalse(stream_get_meta_data($child['pipes'][1])['timed_out'], 'Heartbeat race timed out.'); + $results[$operation] = json_decode($output, true, flags: JSON_THROW_ON_ERROR); + } + $heartbeat = $results['heartbeat']; + $task = WorkflowTask::query()->findOrFail($claim['task_id']); + $trace = json_encode([ + 'heartbeat_status' => $heartbeat['status'], + 'heartbeat' => array_intersect_key($heartbeat['body'], array_flip([ + 'task_id', 'lease_owner', 'workflow_task_attempt', 'renewed', 'reason', + ])), + 'replacement_claim' => is_array($results['poll']['body']['task'] ?? null) + ? array_intersect_key($results['poll']['body']['task'], array_flip([ + 'task_id', 'lease_owner', 'workflow_task_attempt', + ])) : null, + ]); + $this->assertContains($heartbeat['status'], [200, 409], $trace); + $this->assertSame(200, $results['poll']['status'], json_encode($results['poll'])); + if ($heartbeat['status'] === 200) { + $this->assertTrue($heartbeat['body']['renewed']); + $this->assertSame($task->lease_owner, $heartbeat['body']['lease_owner'], $trace); + $this->assertSame($task->attempt_count, $heartbeat['body']['workflow_task_attempt']); + $this->assertSame('original', $task->lease_owner); + $this->assertSame($claim['workflow_task_attempt'], $task->attempt_count); + $this->assertNull($results['poll']['body']['task']); + $this->assertSame($pending, $heartbeat['body']['cancellation_request']); + } else { + $this->assertSame('lease_owner_mismatch', $heartbeat['body']['reason'], $trace); + $this->assertFalse($heartbeat['body']['renewed'] ?? false); + $this->assertSame('replacement', $task->lease_owner); + $this->assertSame($claim['workflow_task_attempt'] + 1, $task->attempt_count); + $this->assertSame($claim['task_id'], $results['poll']['body']['task']['task_id']); + $this->assertSame($task->lease_owner, $results['poll']['body']['task']['lease_owner']); + $this->assertSame($task->attempt_count, $results['poll']['body']['task']['workflow_task_attempt']); + $this->assertSame($pending, $results['poll']['body']['task']['cancellation_request']); + } + $this->assertSame(1, WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::CooperativeCancellationRequested->value)->count()); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered->value)->count()); + } finally { + foreach ($children as $child) { + if (proc_get_status($child['process'])['running']) { + proc_terminate($child['process']); + } + foreach ($child['pipes'] as $pipe) { + if (is_resource($pipe)) { + fclose($pipe); + } + } + proc_close($child['process']); + } + } + } + + #[DataProvider('workerCapabilities')] + public function test_concurrent_request_and_claim_cannot_accept_an_incompatible_lease(bool $capable): void + { + // Repeat real process races. Deterministic before/after cases live in + // CooperativeCancellationProtocolTest; this covers overlapping writes. + for ($iteration = 1; $iteration <= 3; $iteration++) { + $workerId = 'race-worker-'.$iteration; + $queue = 'race-'.$iteration; + $start = $this->withHeaders($this->controlHeaders())->postJson('/api/workflows', [ + 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => $queue, 'input' => ['Ada'], + ])->assertCreated(); + $workflowId = $start->json('workflow_id'); + $runId = $start->json('run_id'); + WorkerRegistration::query()->create([ + 'namespace' => 'default', 'worker_id' => $workerId, 'task_queue' => $queue, + 'runtime' => 'php', 'supported_workflow_types' => ['tests.external-greeting-workflow'], + 'capabilities' => $capable ? [CooperativeCancellationPolicy::CAPABILITY] : [], + 'max_concurrent_workflow_tasks' => 1, 'last_heartbeat_at' => now(), 'status' => 'active', + ]); + + [$request, $poll] = $this->race($workflowId, $workerId, $queue, $capable ? '1.20' : '1.19'); + foreach ([$request, $poll] as $result) { + foreach (array_slice($result['attempts'], 0, -1) as $retry) { + $this->assertSame(['status' => 503, 'reason' => 'backend_lock_pressure'], $retry); + } + } + $this->assertSame(200, $poll['status'], json_encode($poll)); + $this->assertContains($request['status'], [202, 409], json_encode($request)); + $run = WorkflowRun::query()->findOrFail($runId); + $task = $poll['body']['task']; + $this->assertSame($request['status'] === 202 ? 1 : 0, WorkflowHistoryEvent::query() + ->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::CooperativeCancellationRequested->value) + ->count()); + + if ($request['status'] === 202) { + $this->assertNotNull($run->cancellation_request_command_id); + if ($capable) { + $this->assertIsArray($task); + $this->assertTrue(CooperativeCancellationPolicy::claimSupportsCancellation( + WorkflowTask::query()->findOrFail($task['task_id']), + )); + } else { + $this->assertNull($task); + } + } else { + $this->assertFalse($capable); + $this->assertSame('active_claim_cancellation_not_supported', $request['body']['reason']); + $this->assertNull($run->cancellation_request_command_id); + $this->assertIsArray($task); + } + } + } + + /** @return list> */ + private function race(string $workflowId, string $workerId, string $queue, string $protocol): array + { + $children = []; + try { + foreach (['request', 'poll'] as $operation) { + $process = proc_open([ + PHP_BINARY, dirname(__DIR__).'/Fixtures/CooperativeCancellationRaceRequest.php', + $this->databasePath, $operation, $workflowId, $workerId, $queue, $protocol, + ], [0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + $this->assertIsResource($process); + stream_set_timeout($pipes[1], 10); + $children[] = ['process' => $process, 'pipes' => $pipes]; + } + foreach ($children as $child) { + $this->assertSame("ready\n", fgets($child['pipes'][1])); + } + foreach ($children as $child) { + fwrite($child['pipes'][0], "go\n"); + fclose($child['pipes'][0]); + } + + $results = []; + foreach ($children as $child) { + $output = stream_get_contents($child['pipes'][1]); + $this->assertFalse(stream_get_meta_data($child['pipes'][1])['timed_out'], 'Concurrent request timed out.'); + $results[] = json_decode($output, true, flags: JSON_THROW_ON_ERROR); + } + + return $results; + } finally { + foreach ($children as $child) { + if (proc_get_status($child['process'])['running']) { + proc_terminate($child['process']); + } + foreach ($child['pipes'] as $pipe) { + if (is_resource($pipe)) { + fclose($pipe); + } + } + proc_close($child['process']); + } + } + } + + private function controlHeaders(): array + { + return ['X-Namespace' => 'default', 'X-Durable-Workflow-Control-Plane-Version' => '2']; + } +} diff --git a/tests/Feature/HistoryRetentionTest.php b/tests/Feature/HistoryRetentionTest.php index b30616b5..4401ac39 100644 --- a/tests/Feature/HistoryRetentionTest.php +++ b/tests/Feature/HistoryRetentionTest.php @@ -8,6 +8,7 @@ use App\Models\WorkflowInboundStream; use App\Models\WorkflowInboundStreamItem; use App\Models\WorkflowNamespace; +use App\Support\ExternalPayloadRetentionCleanup; use App\Support\MessageStreamService; use App\Support\NamespaceWorkflowScope; use App\Support\RuntimeExternalPayloadRegistry; @@ -35,6 +36,7 @@ use Workflow\V2\Models\WorkflowSearchAttribute; use Workflow\V2\Models\WorkflowTask; use Workflow\V2\Support\ExternalPayloads; +use Workflow\V2\Support\WorkflowRunRetentionCleanup; use Workflow\V2\WorkflowStub; class HistoryRetentionTest extends TestCase @@ -42,6 +44,78 @@ class HistoryRetentionTest extends TestCase use RefreshDatabase; use ServerTestHelpers; + public function test_retention_holds_detached_activity_history_and_payload_bytes_until_it_closes(): void + { + Queue::fake(); + config(['filesystems.disks.detached-retention-payloads' => [ + 'driver' => 'local', 'root' => storage_path('framework/testing/detached-retention-payloads'), + ]]); + Storage::fake('detached-retention-payloads'); + $this->createNamespace('default'); + WorkflowNamespace::where('name', 'default')->update(['external_payload_storage' => [ + 'driver' => 's3', 'enabled' => true, 'config' => [ + 'disk' => 'detached-retention-payloads', 'bucket' => 'dw-payloads', 'prefix' => 'retention/', + ], + ]]); + $expectedReason = method_exists(WorkflowRunRetentionCleanup::class, 'retentionHoldReason') + ? 'detached_activity_still_open' : 'activity_policy_backend_unsupported'; + foreach ([ActivityStatus::Pending, ActivityStatus::Running] as $status) { + $runId = $this->createExpiredClosedRun('default', 'wf-detached-retention-'.$status->value); + $run = WorkflowRun::findOrFail($runId); + $payload = "independent detached bytes\0".$status->value; + $key = 'retention/avro/'.substr(hash('sha256', $payload), 0, 2).'/'.hash('sha256', $payload); + Storage::disk('detached-retention-payloads')->put($key, $payload); + $execution = ActivityExecution::query()->create([ + 'workflow_run_id' => $runId, 'sequence' => 999, + 'activity_class' => 'detached-fixture-activity', 'activity_type' => 'detached-fixture-activity', + 'status' => $status, 'arguments' => $this->storedExternalPayloadReference('s3://dw-payloads/'.$key, $payload), + 'activity_options' => ['cancellation_policy' => 'abandon'], + 'schedule_to_close_deadline_at' => now()->addMinutes(3), + ]); + WorkflowHistoryEvent::query()->create([ + 'workflow_run_id' => $runId, 'sequence' => ($run->historyEvents()->max('sequence') ?? 0) + 1, + 'event_type' => HistoryEventType::ActivityScheduled, + 'payload' => ['sequence' => 999, 'activity_execution_id' => $execution->id, + 'activity' => ['id' => $execution->id, 'cancellation_policy' => 'abandon']], + 'recorded_at' => now(), + ]); + $historyCount = $run->historyEvents()->count(); + $this->withHeaders($this->apiHeaders())->postJson('/api/system/retention/pass', ['run_ids' => [$runId]]) + ->assertOk()->assertJsonPath('pruned', 0)->assertJsonPath('skipped', 1) + ->assertJsonPath('results.0.reason', $expectedReason); + $direct = app(ExternalPayloadRetentionCleanup::class)->deleteForRun('default', $runId); + $this->assertTrue($direct['blocked']); + $this->assertSame($expectedReason, $direct['reason']); + $this->assertSame(0, $direct['deleted']); + $this->assertSame($payload, Storage::disk('detached-retention-payloads')->get($key)); + $this->assertSame($historyCount, $run->historyEvents()->count()); + $this->assertNull($run->refresh()->details_pruned_at); + $execution->forceFill(['status' => ActivityStatus::Completed, 'closed_at' => now()])->save(); + $this->withHeaders($this->apiHeaders())->postJson('/api/system/retention/pass', ['run_ids' => [$runId]]) + ->assertOk()->assertJsonPath('pruned', 0)->assertJsonPath('results.0.reason', $expectedReason); + $this->assertSame($payload, Storage::disk('detached-retention-payloads')->get($key)); + WorkflowHistoryEvent::query()->create([ + 'workflow_run_id' => $runId, 'sequence' => $run->historyEvents()->max('sequence') + 1, + 'event_type' => HistoryEventType::ActivityCancelled, + 'payload' => ['sequence' => 999, 'activity_execution_id' => $execution->id, 'activity_attempt_id' => null], + 'recorded_at' => now(), + ]); + $execution->forceFill(['status' => ActivityStatus::Cancelled])->save(); + if (! method_exists(WorkflowRunRetentionCleanup::class, 'retentionHoldReason')) { + $this->withHeaders($this->apiHeaders())->postJson('/api/system/retention/pass', ['run_ids' => [$runId]]) + ->assertOk()->assertJsonPath('pruned', 0)->assertJsonPath('results.0.reason', $expectedReason); + $this->assertSame($payload, Storage::disk('detached-retention-payloads')->get($key)); + + continue; + } + $this->withHeaders($this->apiHeaders())->postJson('/api/system/retention/pass', ['run_ids' => [$runId]]) + ->assertOk()->assertJsonPath('pruned', 1)->assertJsonPath('results.0.external_payloads_deleted', 1); + Storage::disk('detached-retention-payloads')->assertMissing($key); + $this->assertSame(0, ActivityExecution::query()->whereKey($execution->id)->count()); + $this->assertNull(WorkflowRunSummary::find($runId)); + } + } + // ── Retention Status Endpoint ────────────────────────────────── public function test_retention_status_returns_empty_when_no_expired_runs(): void diff --git a/tests/Feature/PreparedLocalActivityProtocolTest.php b/tests/Feature/PreparedLocalActivityProtocolTest.php new file mode 100644 index 00000000..a0731149 --- /dev/null +++ b/tests/Feature/PreparedLocalActivityProtocolTest.php @@ -0,0 +1,1143 @@ + '1.20', + 'workflows.v2.workflow_task_lease_seconds' => 10, + 'workflows.v2.types.workflows' => ['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class], + ]); + WorkflowNamespace::query()->create([ + 'name' => 'default', 'description' => 'Test', 'retention_days' => 30, 'status' => 'active', + ]); + } + + protected function tearDown(): void + { + $this->removeStoragePressure(); + if ($this->payloadDirectory !== null) { + File::deleteDirectory($this->payloadDirectory); + } + parent::tearDown(); + } + + public function test_default_protocol_does_not_advertise_or_admit_prepared_callbacks(): void + { + config(['server.worker_protocol.version' => '1.19']); + $this->assertNull(RuntimeExternalPayloadReference::transportManifest()['upload']['completion_context']['prepared_schema']); + $this->withHeaders($this->headers('1.19'))->postJson('/api/worker/workflow-tasks/missing/local-activities/prepare', []) + ->assertStatus(409)->assertJsonPath('reason', 'prepared_local_activity_not_supported') + ->assertJsonPath('server_capabilities.prepared_local_activities', false) + ->assertJsonPath('unavailable', fn ($reasons) => in_array('server_protocol', $reasons, true) && in_array('request_protocol', $reasons, true)); + } + + public function test_actual_bound_bridge_must_supply_the_optional_role(): void + { + $this->app->instance(WorkflowTaskBridge::class, \Mockery::mock(WorkflowTaskBridge::class)); + $this->assertNull(RuntimeExternalPayloadReference::transportManifest()['upload']['completion_context']['prepared_schema']); + $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/missing/local-activities/prepare', []) + ->assertStatus(409)->assertJsonPath('reason', 'prepared_local_activity_not_supported') + ->assertJsonPath('server_capabilities.prepared_local_activities', false) + ->assertJsonFragment(['installed_runtime_prepared_local_activity']); + } + + public function test_prepared_capability_requires_cooperation_and_protocol_1_20(): void + { + foreach (['1.19', '1.20'] as $version) { + $this->withHeaders($this->headers($version))->postJson('/api/worker/register', [ + 'worker_id' => 'incorrect', 'task_queue' => 'prepared', 'runtime' => 'php', + 'capabilities' => [PreparedLocalActivityPolicy::CAPABILITY], + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + ])->assertStatus(409)->assertJsonPath('reason', 'prepared_local_activity_capability_mismatch'); + } + $this->assertSame(0, WorkerRegistration::query()->count()); + } + + public function test_preparation_records_admission_before_callback_and_duplicate_keeps_authority(): void + { + $task = $this->claim(); + $response = $this->prepare($task); + $this->assertSame(200, $response->status(), json_encode($response->json('reason'), JSON_THROW_ON_ERROR)); + $prepared = $response->assertJsonPath('prepared', true)->json(); + $attempt = ActivityAttempt::query()->findOrFail($prepared['activity_attempt_id']); + $this->assertNotSame('sdk-local-1', $attempt->id); + $this->assertSame('sdk-local-1', $attempt->worker_attempt_id); + $this->assertSame(1, $attempt->attempt_number); + $this->assertSame($task['task_id'], $attempt->workflow_task_id); + $hosting = WorkflowTask::query()->findOrFail($task['task_id']); + $this->assertSame(TaskStatus::Leased, $hosting->status); + $this->assertNotNull(PreparedLocalActivityPolicy::originalClaim($hosting, $attempt, $task['lease_owner'], $task['workflow_task_attempt'])); + $before = $attempt->getAttributes(); + $this->travel(2)->seconds(); + $this->prepare($task)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('activity_attempt_id', $attempt->id) + ->assertJsonPath('lease_expires_at', $prepared['lease_expires_at']); + $this->assertSame($before, $attempt->refresh()->getAttributes()); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityScheduled)); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityStarted)); + $this->assertSame(0, WorkflowTask::query()->where('task_type', 'activity')->count()); + } + + public function test_claim_cannot_be_upgraded_by_later_registration_edit(): void + { + $task = $this->claim(prepared: false); + WorkerRegistration::query()->where('worker_id', 'original')->sole()->forceFill([ + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY, PreparedLocalActivityPolicy::CAPABILITY], + ])->save(); + $this->prepare($task)->assertStatus(409)->assertJsonFragment(['worker_claim_capability']); + $this->assertSame(0, ActivityAttempt::query()->count()); + } + + public function test_malformed_and_non_avro_arguments_are_refused_before_admission(): void + { + $task = $this->claim(); + $fixture = json_decode(file_get_contents(base_path('tests/Fixtures/CodecRegression/worker-malformed-serialized-payload.json')), true, flags: JSON_THROW_ON_ERROR); + $badBytes = base64_decode($fixture['framing']['wire_base64'], true); + $this->prepare($task, ['descriptor' => $this->descriptor(['arguments' => $badBytes])]) + ->assertStatus(422)->assertJsonValidationErrors('descriptor.arguments'); + $this->prepare($task, ['descriptor' => $this->descriptor(['payload_codec' => 'json'])]) + ->assertStatus(422)->assertJsonValidationErrors('descriptor.payload_codec'); + $this->assertSame(0, ActivityAttempt::query()->count()); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityScheduled)); + } + + public function test_inline_envelopes_preserve_exact_arguments_and_result_bytes(): void + { + $task = $this->claim(); + $input = Serializer::serializeWithCodec('avro', ['Ada', ['nested' => true]]); + $prepared = $this->prepare($task, ['descriptor' => $this->descriptor(['arguments' => ['codec' => 'avro', 'blob' => $input]])]) + ->assertOk()->json(); + $this->assertSame($input, ActivityExecution::query()->findOrFail($prepared['activity_execution_id'])->arguments); + $result = Serializer::serializeWithCodec('avro', ['binary' => AvroBinaryValue::fromBytes("\0\xff")]); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => [ + 'outcome' => 'completed', 'result' => ['codec' => 'avro', 'blob' => $result], + ]])->assertOk()->assertJsonPath('recorded', true); + $event = WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id']) + ->where('event_type', HistoryEventType::ActivityCompleted)->sole(); + $this->assertSame($result, $event->payload['result']); + } + + public function test_external_arguments_and_result_references_are_namespace_bound(): void + { + $task = $this->claim(); + $this->payloadDirectory = sys_get_temp_dir().'/dw-prepared-payload-'.getmypid(); + foreach (['default', 'other'] as $namespace) { + WorkflowNamespace::query()->updateOrCreate(['name' => $namespace], [ + 'description' => 'Fixture', 'retention_days' => 30, 'status' => 'active', + 'external_payload_storage' => ['driver' => 'local', 'enabled' => true, 'threshold_bytes' => 4096, + 'config' => ['uri' => 'file://'.$this->payloadDirectory.'/'.$namespace]], + ]); + } + $arguments = Serializer::serializeWithCodec('avro', ['external arguments']); + $foreign = $this->uploadPayload($arguments, 'other'); + $this->prepare($task, ['descriptor' => $this->descriptor(['arguments' => ['codec' => 'avro', 'external_payload' => $foreign]])]) + ->assertStatus(404)->assertJsonPath('reason', 'external_payload_not_found'); + $this->assertSame(0, ActivityAttempt::query()->count()); + $reference = $this->uploadPayload($arguments, 'default'); + $prepared = $this->prepare($task, ['descriptor' => $this->descriptor(['arguments' => ['codec' => 'avro', 'external_payload' => $reference]])]) + ->assertOk()->json(); + $this->assertSame($arguments, ActivityExecution::query()->findOrFail($prepared['activity_execution_id'])->arguments); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => [ + 'outcome' => 'completed', 'result' => ['codec' => 'avro', 'external_payload' => $foreign], + ]])->assertStatus(404)->assertJsonPath('reason', 'external_payload_not_found'); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + $result = Serializer::serializeWithCodec('avro', ['external result']); + $resultReference = $this->uploadPayload($result, 'default'); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => [ + 'outcome' => 'completed', 'result' => ['codec' => 'avro', 'external_payload' => $resultReference], + ]])->assertOk()->assertJsonPath('recorded', true); + $this->assertSame($result, ActivityExecution::query()->findOrFail($prepared['activity_execution_id'])->result); + } + + public function test_checkpoint_commits_prefix_keeps_claim_and_reuses_completion_validation(): void + { + $task = $this->claim(); + $claim = WorkflowTask::query()->findOrFail($task['task_id']); + $expiry = $claim->lease_expires_at->toISOString(); + $body = ['checkpoint_id' => 'prefix-1', 'start_sequence' => 1, 'commands' => [ + ['type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 7)], + ]]; + $this->localRequest($task, 'checkpoint', $body)->assertOk()->assertJsonPath('checkpointed', true) + ->assertJsonPath('next_sequence', 2); + $this->localRequest($task, 'checkpoint', $body)->assertOk()->assertJsonPath('duplicate', true); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::SideEffectRecorded)); + $this->assertSame(TaskStatus::Leased, $claim->refresh()->status); + $this->assertSame($expiry, $claim->lease_expires_at->toISOString()); + $this->localRequest($task, 'checkpoint', [...$body, 'commands' => [['type' => 'start_timer', 'delay_seconds' => 2]]]) + ->assertStatus(409)->assertJsonPath('reason', 'local_activity_checkpoint_mismatch'); + $this->localRequest($task, 'checkpoint', [...$body, 'commands' => [[ + 'type' => 'start_timer', 'delay_seconds' => 1, 'cancellation_policy' => 'try_cancel', + ]]])->assertStatus(422)->assertJsonValidationErrors('commands.0.cancellation_policy'); + $this->localRequest($task, 'checkpoint', [...$body, 'commands' => [['type' => 'complete_workflow']]]) + ->assertStatus(409)->assertJsonPath('reason', 'invalid_local_activity_checkpoint_commands'); + $this->prepare($task, ['sequence' => 2])->assertOk()->assertJsonPath('prepared', true); + } + + public function test_checkpoint_rejects_completion_only_metadata(): void + { + $task = $this->claim(); + $this->localRequest($task, 'checkpoint', ['checkpoint_id' => 'one', 'start_sequence' => 1, 'commands' => [], + 'message_stream_cursors' => [['stream_name' => 'input', 'cursor' => 1]], + ])->assertStatus(422)->assertJsonValidationErrors('message_stream_cursors'); + $this->assertArrayNotHasKey('portable_local_checkpoint', WorkflowTask::query()->findOrFail($task['task_id'])->payload); + } + + public function test_sequential_checkpoints_keep_typed_search_history_identity(): void + { + SearchAttributeDefinition::query()->create(['namespace' => 'default', 'name' => 'Tier', 'type' => 'keyword']); + $task = $this->claim(extraCapabilities: ['typed_search_attributes']); + foreach ([1 => 'basic', 2 => 'pro'] as $sequence => $value) { + $body = ['checkpoint_id' => "search-{$sequence}", 'start_sequence' => $sequence, 'commands' => [[ + 'type' => 'upsert_search_attributes', 'attributes' => ['Tier' => $value], 'attribute_types' => ['Tier' => 'keyword'], + ]]]; + $this->localRequest($task, 'checkpoint', $body)->assertOk()->assertJsonPath('checkpointed', true); + $this->localRequest($task, 'checkpoint', $body)->assertOk()->assertJsonPath('duplicate', true); + } + $events = WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id']) + ->where('event_type', HistoryEventType::SearchAttributesUpserted)->orderBy('sequence')->get(); + $this->assertCount(2, $events); + foreach ($events as $event) { + $this->assertSame(['Tier' => 'keyword'], $event->payload['attribute_types']); + } + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [ + ['type' => 'upsert_search_attributes', 'attributes' => ['Tier' => 'enterprise'], 'attribute_types' => ['Tier' => 'keyword']], + ['type' => 'complete_workflow'], + ], + ])->assertOk()->assertJsonPath('recorded', true)->assertJsonPath('run_status', 'completed'); + $final = WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id']) + ->where('event_type', HistoryEventType::SearchAttributesUpserted)->orderByDesc('sequence')->firstOrFail(); + $this->assertSame(3, $final->payload['sequence']); + $this->assertSame(['Tier' => 'keyword'], $final->payload['attribute_types']); + } + + public function test_supervisor_renews_both_leases_without_application_heartbeats(): void + { + $task = $this->claim(); + $prepared = $this->prepare($task)->assertOk()->json(); + $attempt = ActivityAttempt::query()->findOrFail($prepared['activity_attempt_id']); + $heartbeat = $attempt->last_heartbeat_at->toISOString(); + $execution = ActivityExecution::query()->findOrFail($prepared['activity_execution_id'])->getAttributes(); + $this->travel(4)->seconds(); + $this->localRequest($task, "{$attempt->id}/control", ['renew_lease' => true])->assertOk() + ->assertJsonPath('active', true)->assertJsonPath('renewed', true); + $this->assertSame($heartbeat, $attempt->refresh()->last_heartbeat_at->toISOString()); + $this->assertSame($execution, ActivityExecution::query()->findOrFail($prepared['activity_execution_id'])->getAttributes()); + $this->assertSame($attempt->lease_expires_at->toISOString(), WorkflowTask::query()->findOrFail($task['task_id'])->lease_expires_at->toISOString()); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityStarted)); + } + + public function test_same_claim_history_refresh_and_completion_do_not_repeat_a_prepared_callback(): void + { + $task = $this->claim(); + $this->localRequest($task, 'checkpoint', ['checkpoint_id' => 'first', 'start_sequence' => 1, 'commands' => [[ + 'type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 7), + ]]])->assertOk(); + $prepared = $this->prepare($task, ['sequence' => 2])->assertOk()->json(); + $receipt = $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => $this->success()])->assertOk()->json(); + $history = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/history", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'next_history_page_token' => $receipt['history_refresh_page_token'], + ])->assertOk()->json('history_events'); + $this->assertContains('SideEffectRecorded', array_column($history, 'event_type')); + $this->assertContains('ActivityCompleted', array_column($history, 'event_type')); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [['type' => 'complete_workflow', 'result' => $this->success()['result'], 'payload_codec' => 'avro']], + ])->assertOk()->assertJsonPath('recorded', true)->assertJsonPath('run_status', 'completed'); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityScheduled)); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityStarted)); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::SideEffectRecorded)); + $this->assertSame(TaskStatus::Completed, WorkflowTask::query()->findOrFail($task['task_id'])->status); + } + + public function test_application_heartbeat_records_progress_without_renewing_either_lease(): void + { + $task = $this->claim(); + $prepared = $this->prepare($task, ['descriptor' => $this->descriptor([ + 'heartbeat_timeout' => 3, 'start_to_close_timeout' => 60, 'schedule_to_close_timeout' => 120, + ])])->assertOk()->json(); + $lease = $prepared['lease_expires_at']; + $this->travel(2)->seconds(); + $heartbeat = $this->localRequest($task, "{$prepared['activity_attempt_id']}/heartbeat", ['progress' => ['message' => 'working']]) + ->assertOk()->assertJsonPath('active', true)->assertJsonPath('heartbeat_recorded', true) + ->assertJsonPath('renewed', false)->json(); + $this->assertSame($lease, $heartbeat['lease_expires_at']); + $this->assertSame($lease, $heartbeat['workflow_lease_expires_at']); + $this->assertSame($prepared['start_to_close_deadline_at'], $heartbeat['start_to_close_deadline_at']); + $this->assertSame($prepared['schedule_to_close_deadline_at'], $heartbeat['schedule_to_close_deadline_at']); + $this->assertNotSame($prepared['heartbeat_deadline_at'], $heartbeat['heartbeat_deadline_at']); + $event = WorkflowHistoryEvent::query()->whereKey($heartbeat['heartbeat_history_event_id'])->firstOrFail(); + $this->assertSame(['message' => 'working'], $event->payload['progress']); + $this->assertSame($task['run_id'], $event->workflow_run_id); + $this->travel(2)->seconds(); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/control", ['renew_lease' => true])->assertOk()->assertJsonPath('active', true); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityHeartbeatRecorded)); + } + + public function test_expired_or_stale_claims_cannot_record_application_heartbeats(): void + { + $task = $this->claim(); + $prepared = $this->prepare($task, ['descriptor' => $this->descriptor(['heartbeat_timeout' => 3])])->assertOk()->json(); + WorkerRegistration::query()->where('worker_id', 'original')->sole()->forceFill(['status' => 'draining'])->save(); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/heartbeat", ['progress' => ['message' => 'stale']]) + ->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration')->assertJsonPath('heartbeat_recorded', false); + WorkerRegistration::query()->where('worker_id', 'original')->sole()->forceFill(['status' => 'active'])->save(); + $this->travel(3)->seconds(); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/heartbeat")->assertStatus(409) + ->assertJsonPath('reason', 'local_activity_deadline_expired')->assertJsonPath('heartbeat_recorded', false); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityHeartbeatRecorded)); + } + + public function test_shielded_cleanup_is_bound_to_canonical_delivery_and_original_budget(): void + { + [$task, $descriptor, $accepted] = $this->cleanupClaim(); + $this->prepare($task, ['sequence' => 2])->assertStatus(409)->assertJsonPath('reason', 'cancellation_requested'); + $this->localRequest($task, 'checkpoint', ['checkpoint_id' => 'cleanup-prefix', 'start_sequence' => 2, 'commands' => [[ + 'type' => 'record_side_effect', 'marker_id' => 'cleanup', 'result' => $this->success()['result'], 'payload_codec' => 'avro', + ]]])->assertOk()->assertJsonPath('next_sequence', 3); + $response = $this->prepare($task, ['sequence' => 3, 'descriptor' => $descriptor]); + $this->assertSame(200, $response->status(), json_encode($response->json('reason'), JSON_THROW_ON_ERROR)); + $prepared = $response->assertOk() + ->assertJsonPath('cancellation_cleanup.request_id', $accepted['request_id']) + ->assertJsonPath('cancellation_cleanup.root_request_id', $accepted['request_id']) + ->assertJsonPath('cancellation_cleanup.cleanup_deadline_at', $accepted['cleanup_deadline_at'])->json(); + $this->assertSame($accepted['cleanup_deadline_at'], $prepared['schedule_to_close_deadline_at']); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/heartbeat", ['progress' => ['message' => 'cleaning']]) + ->assertOk()->assertJsonPath('active', true)->assertJsonPath('heartbeat_recorded', true); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/control", ['renew_lease' => true])->assertOk()->assertJsonPath('active', true); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => $this->success()])->assertOk()->assertJsonPath('recorded', true); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [['type' => 'complete_workflow']], + ])->assertOk()->assertJsonPath('run_status', 'cancelled'); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::CooperativeCancellationDelivered)); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + } + + public function test_stale_registration_cannot_publish_cleanup_despite_accepted_cancellation(): void + { + [$task, $descriptor] = $this->cleanupClaim(); + $response = $this->prepare($task, ['sequence' => 2, 'descriptor' => $descriptor]); + $this->assertSame(200, $response->status(), json_encode($response->json('reason'), JSON_THROW_ON_ERROR)); + $prepared = $response->assertOk()->json(); + $attempt = $prepared['activity_attempt_id']; + $before = ActivityAttempt::query()->findOrFail($attempt)->getAttributes(); + WorkerRegistration::query()->where('worker_id', 'original')->sole()->forceFill(['status' => 'draining'])->save(); + $this->localRequest($task, "{$attempt}/outcome", ['report' => $this->success()])->assertStatus(409) + ->assertJsonPath('reason', 'stale_worker_registration')->assertJsonPath('recorded', false); + $this->localRequest($task, "{$attempt}/heartbeat")->assertStatus(409)->assertJsonPath('heartbeat_recorded', false); + $this->assertSame($before, ActivityAttempt::query()->findOrFail($attempt)->getAttributes()); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityHeartbeatRecorded)); + } + + public function test_changed_registration_cannot_renew_or_publish_a_fresh_result(): void + { + $task = $this->claim(); + $attempt = $this->prepare($task)->assertOk()->json('activity_attempt_id'); + WorkerRegistration::query()->where('worker_id', 'original')->sole()->forceFill(['status' => 'draining'])->save(); + $this->localRequest($task, "{$attempt}/control", ['renew_lease' => true])->assertStatus(409) + ->assertJsonPath('active', false)->assertJsonPath('renewed', false)->assertJsonPath('stop_required', true) + ->assertJsonPath('reason', 'stale_worker_registration'); + $this->localRequest($task, "{$attempt}/outcome", ['report' => $this->success()])->assertStatus(409) + ->assertJsonPath('recorded', false)->assertJsonPath('reason', 'stale_worker_registration'); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + } + + public function test_success_receipt_is_idempotent_after_takeover_and_changed_result_is_refused(): void + { + $task = $this->claim(); + $attempt = $this->prepare($task)->assertOk()->json('activity_attempt_id'); + $first = $this->localRequest($task, "{$attempt}/outcome", ['report' => $this->success()])->assertOk() + ->assertJsonPath('recorded', true)->assertJsonPath('claim_released', false)->json(); + $this->replaceClaim($task); + $this->localRequest($task, "{$attempt}/outcome", ['report' => $this->success()])->assertOk() + ->assertJsonPath('duplicate', true)->assertJsonPath('event_id', $first['event_id']); + $this->localRequest($task, "{$attempt}/outcome", ['report' => $this->success('changed')])->assertStatus(409) + ->assertJsonPath('reason', 'local_activity_outcome_mismatch'); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + } + + public function test_failure_schedules_one_durable_retry_and_retains_the_total_deadline(): void + { + $task = $this->claim(); + $descriptor = $this->descriptor(['schedule_to_close_timeout' => 60, 'retry_policy' => [ + 'max_attempts' => 2, 'backoff_seconds' => [2], + ]]); + $first = $this->prepare($task, ['descriptor' => $descriptor])->assertOk()->json(); + $failure = ['outcome' => 'failed', 'message' => 'retry me', 'exception_type' => \RuntimeException::class]; + $result = $this->localRequest($task, "{$first['activity_attempt_id']}/outcome", ['report' => $failure]) + ->assertOk()->assertJsonPath('recorded', true)->assertJsonPath('claim_released', true) + ->assertJsonPath('event_type', 'ActivityRetryScheduled')->json(); + $this->localRequest($task, "{$first['activity_attempt_id']}/outcome", ['report' => $failure]) + ->assertOk()->assertJsonPath('duplicate', true)->assertJsonPath('event_id', $result['event_id']); + $this->assertCount(1, $result['created_task_ids']); + $this->assertSame(TaskStatus::Completed, WorkflowTask::query()->findOrFail($task['task_id'])->status); + $this->assertSame(0, WorkflowTask::query()->where('task_type', 'activity')->count()); + $this->poll('original')->assertOk()->assertJsonPath('task', null); + $this->travel(2)->seconds(); + $retry = $this->poll('original')->assertOk()->json('task'); + $this->assertSame($result['created_task_ids'][0], $retry['task_id']); + $second = $this->prepare($retry, ['worker_attempt_id' => 'sdk-local-2', 'descriptor' => $descriptor]) + ->assertOk()->assertJsonPath('attempt_number', 2)->json(); + $this->assertNotSame($first['activity_attempt_id'], $second['activity_attempt_id']); + $this->assertSame($first['schedule_to_close_deadline_at'], $second['schedule_to_close_deadline_at']); + $this->localRequest($retry, "{$second['activity_attempt_id']}/outcome", ['report' => $this->success()]) + ->assertOk()->assertJsonPath('recorded', true); + } + + public function test_cold_recovery_records_unknown_stop_then_retries_under_original_budget(): void + { + $task = $this->claim(); + $descriptor = $this->descriptor(['schedule_to_close_timeout' => 60, 'retry_policy' => [ + 'max_attempts' => 2, 'backoff_seconds' => [2], + ]]); + $first = $this->prepare($task, ['descriptor' => $descriptor])->assertOk()->json(); + ActivityAttempt::query()->findOrFail($first['activity_attempt_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $replacement = $this->replaceClaim($task); + $body = ['sequence' => 1, 'descriptor' => $descriptor]; + $recovered = $this->localRequest($replacement, 'recover', $body)->assertOk() + ->assertJsonPath('recovered', true)->assertJsonPath('claim_released', true) + ->assertJsonPath('callback_stop_state', 'unknown')->json(); + $this->localRequest($replacement, 'recover', $body)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('event_id', $recovered['event_id']); + $this->localRequest($task, "{$first['activity_attempt_id']}/outcome", ['report' => $this->success()]) + ->assertStatus(409)->assertJsonPath('reason', 'stale_activity_attempt'); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityCancellationAcknowledged)); + $this->travel(2)->seconds(); + $retry = $this->poll('replacement')->assertOk()->json('task'); + $this->assertSame($recovered['created_task_ids'][0], $retry['task_id']); + $second = $this->prepare($retry, ['worker_attempt_id' => 'sdk-recovered', 'descriptor' => $descriptor]) + ->assertOk()->assertJsonPath('attempt_number', 2)->json(); + $this->assertSame($first['schedule_to_close_deadline_at'], $second['schedule_to_close_deadline_at']); + $this->assertSame(0, WorkflowTask::query()->where('task_type', 'activity')->count()); + } + + public function test_accepted_cancellation_refuses_publication_without_releasing_cleanup_claim(): void + { + $task = $this->claim(); + $attempt = $this->prepare($task)->assertOk()->json('activity_attempt_id'); + $this->withHeaders($this->controlHeaders())->postJson("/api/workflows/{$task['workflow_id']}/request-cancellation", []) + ->assertStatus(202); + $this->localRequest($task, "{$attempt}/outcome", ['report' => $this->success()])->assertStatus(409) + ->assertJsonPath('recorded', false)->assertJsonPath('fenced', true)->assertJsonPath('reason', 'cancellation_requested'); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityCancelled)); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityCancellationAcknowledged)); + } + + public function test_original_supervisor_observes_cancellation_and_records_join_after_takeover(): void + { + $task = $this->claim(); + $attempt = $this->prepare($task)->assertOk()->json('activity_attempt_id'); + $replacement = $this->replaceClaim($task); + $accepted = $this->withHeaders($this->controlHeaders())->postJson("/api/workflows/{$task['workflow_id']}/request-cancellation", [ + 'cleanup_timeout_seconds' => 30, + ])->assertStatus(202)->json('cancellation_request'); + $this->localRequest($task, "{$attempt}/control", ['renew_lease' => true])->assertOk() + ->assertJsonPath('active', false)->assertJsonPath('renewed', false) + ->assertJsonPath('stop_required', true)->assertJsonPath('cancellation_request.request_id', $accepted['request_id']) + ->assertJsonPath('cancellation_request.cleanup_deadline_at', $accepted['cleanup_deadline_at']); + $this->localRequest($task, "{$attempt}/acknowledge-cancellation", ['request_id' => $accepted['request_id']]) + ->assertOk()->assertJsonPath('acknowledged', true); + $this->localRequest($task, "{$attempt}/acknowledge-cancellation", ['request_id' => $accepted['request_id']]) + ->assertOk()->assertJsonPath('duplicate', true); + $current = WorkflowTask::query()->findOrFail($task['task_id']); + $this->assertSame(TaskStatus::Leased, $current->status); + $this->assertSame($replacement['lease_owner'], $current->lease_owner); + $this->assertSame($replacement['workflow_task_attempt'], $current->attempt_count); + } + + public function test_wrong_owner_epoch_namespace_and_unknown_attempt_cannot_act(): void + { + $task = $this->claim(); + $attempt = $this->prepare($task)->assertOk()->json('activity_attempt_id'); + foreach ([['lease_owner' => 'different'], ['workflow_task_attempt' => 99]] as $wrong) { + $this->localRequest($task, "{$attempt}/control", $wrong)->assertStatus(409) + ->assertJsonFragment(['original_prepared_local_claim']); + } + WorkflowNamespace::query()->create(['name' => 'other', 'description' => 'Other', 'retention_days' => 30, 'status' => 'active']); + $this->withHeaders([...$this->headers(), 'X-Namespace' => 'other']) + ->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/{$attempt}/control", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + ])->assertStatus(404)->assertJsonPath('reason', 'activity_attempt_not_found'); + $this->localRequest($task, 'missing/control')->assertStatus(404); + $this->withHeaders($this->headers('1.19'))->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/{$attempt}/control", []) + ->assertStatus(409)->assertJsonFragment(['request_protocol']); + } + + public function test_quota_refusal_rolls_back_admission_and_original_authority_receipt(): void + { + $task = $this->claim(); + $before = WorkflowHistoryEvent::query()->count(); + config(['server.namespace_durable_state.limits' => ['max_workflow_history_events' => $before + 1]]); + $this->prepare($task)->assertStatus(429)->assertJsonPath('reason', 'namespace_workflow_history_events_exhausted'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertSame(0, ActivityAttempt::query()->count()); + $this->assertArrayNotHasKey('_server_prepared_local_activity_claims', WorkflowTask::query()->findOrFail($task['task_id'])->payload); + } + + public function test_draining_allows_an_existing_claim_to_commit_prefix_prepare_and_finish(): void + { + $task = $this->claim(); + $this->configureStoragePressure('draining'); + $this->localRequest($task, 'checkpoint', ['checkpoint_id' => 'drain', 'start_sequence' => 1, 'commands' => [[ + 'type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 7), + ]]])->assertOk()->assertJsonPath('checkpointed', true); + $prepared = $this->prepare($task, ['sequence' => 2])->assertOk()->json(); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/control", ['renew_lease' => true]) + ->assertOk()->assertJsonPath('renewed', true); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/heartbeat", ['progress' => ['message' => 'draining']]) + ->assertOk()->assertJsonPath('heartbeat_recorded', true)->assertJsonPath('renewed', false); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => $this->success()]) + ->assertOk()->assertJsonPath('recorded', true); + } + + public function test_draining_preserves_original_cancellation_observation_and_join_receipt(): void + { + $task = $this->claim(); + $attempt = $this->prepare($task)->assertOk()->json('activity_attempt_id'); + $pending = $this->withHeaders($this->controlHeaders())->postJson("/api/workflows/{$task['workflow_id']}/request-cancellation", []) + ->assertStatus(202)->json('cancellation_request'); + $this->configureStoragePressure('draining'); + $this->localRequest($task, "{$attempt}/control")->assertOk()->assertJsonPath('stop_required', true) + ->assertJsonPath('cancellation_request.request_id', $pending['request_id']); + $this->localRequest($task, "{$attempt}/acknowledge-cancellation", ['request_id' => $pending['request_id']]) + ->assertOk()->assertJsonPath('acknowledged', true); + } + + public function test_draining_external_prefix_arguments_recovery_and_result_complete_without_extending_authority(): void + { + $task = $this->claim(); + $this->enableCompletionPayloads(); + $this->withHeaders($this->controlHeaders())->getJson('/api/cluster/info')->assertOk() + ->assertJsonPath('namespace.external_payload_storage.transport.upload.completion_context.prepared_schema', RuntimePayloadCompletionContext::PREPARED_SCHEMA); + $before = WorkflowTask::query()->findOrFail($task['task_id'])->lease_expires_at->toISOString(); + $this->configureStoragePressure('draining'); + $prefix = Serializer::serializeWithCodec('avro', str_repeat('prefix', 30)); + $prefixReference = $this->completionUpload($prefix, $this->completionContext($task, 'checkpoint', 'prefix-1')) + ->assertCreated()->json('reference'); + $this->localRequest($task, 'checkpoint', ['checkpoint_id' => 'prefix-1', 'start_sequence' => 1, 'commands' => [[ + 'type' => 'record_side_effect', 'result' => ['codec' => 'avro', 'external_payload' => $prefixReference], + ]]])->assertOk(); + $arguments = Serializer::serializeWithCodec('avro', [str_repeat('arguments', 30)]); + $argumentReference = $this->completionUpload($arguments, $this->completionContext($task, 'prepare', 2)) + ->assertCreated()->json('reference'); + $prepared = $this->prepare($task, ['sequence' => 2, 'descriptor' => $this->descriptor([ + 'arguments' => ['codec' => 'avro', 'external_payload' => $argumentReference], + ])])->assertOk()->json(); + $this->completionUpload($arguments, $this->completionContext($task, 'recover', 2))->assertCreated(); + $result = Serializer::serializeWithCodec('avro', str_repeat('result', 30)); + $context = $this->completionContext($task, 'outcome', $prepared['activity_attempt_id']); + $reference = $this->completionUpload($result, $context)->assertCreated()->json('reference'); + $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->lease_expires_at->toISOString()); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => [ + 'outcome' => 'completed', 'result' => ['codec' => 'avro', 'external_payload' => $reference], + ]])->assertOk()->assertJsonPath('recorded', true); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [['type' => 'complete_workflow']], + ])->assertOk()->assertJsonPath('run_status', 'completed'); + $this->assertSame($result, ExternalPayloads::resolveStoredPayload(ActivityExecution::query()->findOrFail($prepared['activity_execution_id'])->result, 'avro', 'default')); + $this->assertDatabaseCount('runtime_payload_completion_budgets', 1); + $budget = RuntimePayloadCompletionBudget::query()->sole(); + $this->assertCount(4, $budget->slots); + $this->assertCount(3, $budget->objects); + $rows = RuntimeExternalPayload::query()->orderBy('id')->get()->map->getAttributes()->all(); + $budgetBefore = $budget->getAttributes(); + $this->assertSame($reference, RuntimeExternalPayloadReference::validate($this->completionUpload($result, $context)->assertCreated()->json('reference'))); + $this->assertSame($rows, RuntimeExternalPayload::query()->orderBy('id')->get()->map->getAttributes()->all()); + $this->assertSame($budgetBefore, $budget->refresh()->getAttributes()); + $this->completionUpload('new bytes', $context)->assertStatus(409)->assertJsonPath('reason', 'external_payload_completion_lease_rejected'); + } + + public function test_prepared_operations_cannot_multiply_the_legacy_workflow_byte_allowance(): void + { + $task = $this->claim(); + $this->enableCompletionPayloads(); + config(['server.external_payload_transport.completion_max_bytes' => 8]); + $this->configureStoragePressure('draining'); + $legacy = ['schema' => RuntimePayloadCompletionContext::SCHEMA, 'kind' => 'workflow', + 'task_id' => $task['task_id'], 'attempt' => $task['workflow_task_attempt'], 'lease_owner' => $task['lease_owner'], + 'operation' => 'complete', 'slot' => ['commands', 0, 'result']]; + $this->completionUpload('aaaa', $legacy)->assertCreated(); + $this->completionUpload('bbbb', $this->completionContext($task, 'prepare', 1))->assertCreated(); + foreach ([$this->completionContext($task, 'prepare', 2), $this->completionContext($task, 'checkpoint', 'prefix-2')] as $context) { + $this->completionUpload('cccc', $context)->assertStatus(503)->assertJsonPath('request_admitted', false); + } + $this->assertDatabaseCount('runtime_external_payloads', 2); + $this->assertDatabaseCount('runtime_payload_completion_budgets', 1); + $budget = RuntimePayloadCompletionBudget::query()->sole(); + $this->assertCount(2, $budget->slots); + $this->assertSame(8, array_sum($budget->objects)); + } + + public function test_prepared_upload_refuses_unissued_claim_protocol_namespace_and_attempt(): void + { + $task = $this->claim(); + $this->enableCompletionPayloads(); + $prepared = $this->prepare($task)->assertOk()->json(); + $this->configureStoragePressure('draining'); + $context = $this->completionContext($task, 'prepare', 2); + foreach ([['attempt' => 99], ['lease_owner' => 'different']] as $changes) { + $this->completionUpload('bytes', array_replace($context, $changes))->assertStatus(409); + } + $this->completionUpload('bytes', $context, 'other')->assertStatus(409); + $this->completionUpload('bytes', $this->completionContext($task, 'outcome', 'unknown'))->assertStatus(409); + config(['server.worker_protocol.version' => '1.19']); + $this->completionUpload('bytes', $context)->assertStatus(409); + config(['server.worker_protocol.version' => '1.20']); + $this->observeStoragePressure('normal'); + $this->register('original'); + $this->observeStoragePressure('draining'); + foreach ([$context, $this->completionContext($task, 'outcome', $prepared['activity_attempt_id'])] as $value) { + $this->completionUpload('bytes', $value)->assertStatus(409); + } + $this->assertDatabaseCount('runtime_external_payloads', 0); + $this->assertDatabaseCount('runtime_payload_completion_budgets', 0); + } + + public function test_cooperative_only_worker_cannot_claim_prepared_upload_authority(): void + { + $task = $this->claim(false); + $this->enableCompletionPayloads(); + $this->configureStoragePressure('draining'); + $this->completionUpload('bytes', $this->completionContext($task, 'prepare', 1)) + ->assertStatus(409)->assertJsonPath('reason', 'external_payload_completion_lease_rejected'); + $this->assertDatabaseCount('runtime_external_payloads', 0); + $this->assertDatabaseCount('runtime_payload_completion_budgets', 0); + } + + public function test_accepted_cancellation_fences_old_callback_result_upload(): void + { + $task = $this->claim(); + $this->enableCompletionPayloads(); + $prepared = $this->prepare($task)->assertOk()->json(); + $this->withHeaders($this->controlHeaders())->postJson("/api/workflows/{$task['workflow_id']}/request-cancellation", []) + ->assertStatus(202); + $this->configureStoragePressure('draining'); + $this->completionUpload('bytes', $this->completionContext($task, 'outcome', $prepared['activity_attempt_id'])) + ->assertStatus(409)->assertJsonPath('reason', 'external_payload_completion_lease_rejected'); + $this->assertDatabaseCount('runtime_external_payloads', 0); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + } + + public function test_cleanup_upload_does_not_extend_original_root_budget_or_heartbeat(): void + { + [$task, $descriptor, $accepted] = $this->cleanupClaim(); + $this->enableCompletionPayloads(); + $prepared = $this->prepare($task, ['sequence' => 2, 'descriptor' => $descriptor])->assertOk()->json(); + $this->configureStoragePressure('draining'); + $attempt = ActivityAttempt::query()->findOrFail($prepared['activity_attempt_id']); + $before = $attempt->getAttributes(); + $context = $this->completionContext($task, 'outcome', $attempt->id); + $result = Serializer::serializeWithCodec('avro', str_repeat('cleanup', 30)); + $reference = $this->completionUpload($result, $context)->assertCreated()->json('reference'); + $this->assertSame($before, $attempt->refresh()->getAttributes()); + $this->localRequest($task, "{$attempt->id}/outcome", ['report' => [ + 'outcome' => 'completed', 'result' => ['codec' => 'avro', 'external_payload' => $reference], + ]])->assertOk()->assertJsonPath('recorded', true); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [['type' => 'complete_workflow']], + ])->assertOk()->assertJsonPath('run_status', 'cancelled'); + $this->assertSame($accepted['cleanup_deadline_at'], $prepared['cancellation_cleanup']['cleanup_deadline_at']); + } + + public function test_result_upload_is_fenced_by_fixed_activity_deadline_even_with_live_workflow_lease(): void + { + $task = $this->claim(); + $this->enableCompletionPayloads(); + $prepared = $this->prepare($task, ['descriptor' => $this->descriptor(['start_to_close_timeout' => 1])])->assertOk()->json(); + $this->configureStoragePressure('draining'); + $this->travel(2)->seconds(); + $this->observeStoragePressure('draining'); + $this->assertTrue(WorkflowTask::query()->findOrFail($task['task_id'])->lease_expires_at->isFuture()); + $this->completionUpload('bytes', $this->completionContext($task, 'outcome', $prepared['activity_attempt_id'])) + ->assertStatus(409)->assertJsonPath('reason', 'external_payload_completion_lease_rejected'); + $this->assertDatabaseCount('runtime_external_payloads', 0); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + } + + public function test_expired_root_budget_refuses_new_preparation_upload_with_live_workflow_lease(): void + { + $task = $this->claim(); + $this->enableCompletionPayloads(); + $this->withHeaders($this->controlHeaders())->postJson("/api/workflows/{$task['workflow_id']}/request-cancellation", [ + 'cleanup_timeout_seconds' => 1, + ])->assertStatus(202); + $this->configureStoragePressure('draining'); + $this->travel(2)->seconds(); + $this->observeStoragePressure('draining'); + $this->assertTrue(WorkflowTask::query()->findOrFail($task['task_id'])->lease_expires_at->isFuture()); + $this->completionUpload('bytes', $this->completionContext($task, 'prepare', 2)) + ->assertStatus(409)->assertJsonPath('reason', 'external_payload_completion_lease_rejected'); + $this->assertDatabaseCount('runtime_external_payloads', 0); + $this->assertDatabaseCount('runtime_payload_completion_budgets', 0); + } + + public function test_storage_fence_refuses_all_local_mutations_without_changing_leases_or_history(): void + { + $task = $this->claim(); + $attemptId = $this->prepare($task)->assertOk()->json('activity_attempt_id'); + $attempt = ActivityAttempt::query()->findOrFail($attemptId); + $before = $attempt->getAttributes(); + $history = WorkflowHistoryEvent::query()->count(); + $this->configureStoragePressure('fenced'); + foreach (['prepare', 'recover', 'checkpoint', 'checkpoint-group', "{$attemptId}/control", "{$attemptId}/heartbeat", "{$attemptId}/outcome", "{$attemptId}/acknowledge-cancellation"] as $path) { + $this->localRequest($task, $path)->assertStatus(503)->assertJsonPath('request_admitted', false); + } + $this->assertSame($before, $attempt->refresh()->getAttributes()); + $this->assertSame($history, WorkflowHistoryEvent::query()->count()); + } + + private function cleanupClaim(bool $group = false): array + { + $task = $group ? $this->groupClaim() : $this->claim(); + $accepted = $this->withHeaders($this->controlHeaders())->postJson("/api/workflows/{$task['workflow_id']}/request-cancellation", [ + 'cleanup_timeout_seconds' => 30, + ])->assertStatus(202)->json('cancellation_request'); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/deliver-cancellation", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'request_id' => $accepted['request_id'], 'sequence' => 1, 'call_kind' => 'timer', + ])->assertOk()->assertJsonPath('delivered', true); + $delivery = WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id']) + ->where('event_type', HistoryEventType::CooperativeCancellationDelivered)->sole(); + + return [$task, $this->descriptor(['cancellation_cleanup' => [ + 'request_id' => $accepted['request_id'], 'delivery_history_event_id' => $delivery->id, + ]]), $accepted]; + } + + public function test_group_capability_requires_preparation_cooperation_and_protocol(): void + { + foreach ([['1.19', [CooperativeCancellationPolicy::CAPABILITY, PreparedLocalActivityPolicy::CAPABILITY]], + ['1.20', [CooperativeCancellationPolicy::CAPABILITY]], ['1.20', [PreparedLocalActivityPolicy::CAPABILITY]]] as [$version, $capabilities]) { + $this->withHeaders($this->headers($version))->postJson('/api/worker/register', [ + 'worker_id' => 'incorrect-group', 'task_queue' => 'prepared', 'runtime' => 'php', + 'capabilities' => [...$capabilities, PreparedLocalActivityPolicy::GROUP_CAPABILITY], + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + ])->assertStatus(409)->assertJsonPath('reason', 'prepared_local_activity_group_capability_mismatch'); + } + $this->assertSame(0, WorkerRegistration::query()->count()); + } + + public function test_default_protocol_refuses_group_admission_with_capability_diagnostics(): void + { + config(['server.worker_protocol.version' => '1.19']); + $this->withHeaders($this->headers('1.19'))->postJson('/api/worker/workflow-tasks/missing/local-activities/checkpoint-group', []) + ->assertStatus(409)->assertJsonPath('reason', 'prepared_local_activity_groups_not_supported') + ->assertJsonPath('required_capability', PreparedLocalActivityPolicy::GROUP_CAPABILITY) + ->assertJsonPath('server_capabilities.prepared_local_activity_groups', false); + } + + public function test_existing_custom_prepared_bridge_does_not_acquire_group_capability(): void + { + $task = $this->groupClaim(); + $this->app->instance(WorkflowTaskBridge::class, \Mockery::mock(PreparedLocalActivityTaskBridge::class)); + $this->localRequest($task, 'checkpoint-group', $this->groupBody())->assertStatus(409) + ->assertJsonFragment(['installed_runtime_prepared_local_activity_groups']) + ->assertJsonPath('server_capabilities.prepared_local_activity_groups', false); + $this->assertDatabaseCount('activity_executions', 0); + } + + public function test_child_and_local_group_commit_before_callback_preparation_and_duplicate_is_immutable(): void + { + $task = $this->groupClaim(); + $hosting = WorkflowTask::query()->findOrFail($task['task_id']); + $expiry = $hosting->lease_expires_at->toISOString(); + $body = $this->groupBody(); + $this->localRequest($task, 'checkpoint-group', $body)->assertOk()->assertJsonPath('checkpointed', true) + ->assertJsonPath('next_sequence', 3)->assertJsonCount(1, 'local_activities'); + $this->assertDatabaseCount('workflow_runs', 2); + $this->assertDatabaseCount('activity_attempts', 0); + $execution = ActivityExecution::query()->sole(); + $this->assertSame('pending', $execution->status->value); + $this->assertSame(0, $execution->attempt_count); + $before = $execution->getAttributes(); + $events = WorkflowHistoryEvent::query()->count(); + $this->travel(2)->seconds(); + $this->localRequest($task, 'checkpoint-group', $body)->assertOk()->assertJsonPath('duplicate', true); + $this->assertSame($before, $execution->refresh()->getAttributes()); + $this->assertSame($events, WorkflowHistoryEvent::query()->count()); + $this->assertSame($expiry, $hosting->refresh()->lease_expires_at->toISOString()); + $prepared = $this->prepare($task, ['sequence' => 2, 'descriptor' => [...$body['commands'][1], 'type' => 'record_local_activity']]) + ->assertOk()->assertJsonPath('activity_execution_id', $execution->id)->json(); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => $this->success()]) + ->assertOk()->assertJsonPath('recorded', true)->assertJsonPath('claim_released', false); + $this->assertSame(1, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + } + + public function test_group_claim_cannot_be_upgraded_after_issue_and_is_namespace_bound(): void + { + $task = $this->claim(); + WorkerRegistration::query()->where('worker_id', 'original')->sole()->forceFill([ + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY, PreparedLocalActivityPolicy::CAPABILITY, PreparedLocalActivityPolicy::GROUP_CAPABILITY], + ])->save(); + $this->localRequest($task, 'checkpoint-group', $this->groupBody())->assertStatus(409)->assertJsonFragment(['worker_claim_capability']); + WorkflowNamespace::query()->create(['name' => 'other', 'description' => 'Other', 'retention_days' => 30, 'status' => 'active']); + $this->withHeaders([...$this->headers(), 'X-Namespace' => 'other']) + ->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/checkpoint-group", $this->groupBody()) + ->assertNotFound()->assertJsonPath('reason', 'task_not_found'); + $this->assertDatabaseCount('workflow_runs', 1); + $this->assertDatabaseCount('activity_executions', 0); + } + + public function test_partial_group_and_terminal_report_do_not_create_any_sibling(): void + { + $task = $this->groupClaim(); + $body = $this->groupBody(); + $before = WorkflowHistoryEvent::query()->count(); + $this->localRequest($task, 'checkpoint-group', [...$body, 'commands' => [$body['commands'][1]]]) + ->assertStatus(409)->assertJsonPath('reason', 'invalid_local_activity_checkpoint_commands'); + $this->localRequest($task, 'checkpoint-group', [...$body, 'commands' => [...$body['commands'], ['type' => 'complete_workflow']]]) + ->assertStatus(409)->assertJsonPath('reason', 'invalid_local_activity_checkpoint_commands'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseCount('workflow_runs', 1); + $this->assertDatabaseCount('activity_executions', 0); + } + + public function test_group_quota_refusal_rolls_back_child_local_and_receipt(): void + { + $task = $this->groupClaim(); + $before = WorkflowHistoryEvent::query()->count(); + config(['server.namespace_durable_state.limits' => ['max_workflow_history_events' => $before + 1]]); + $this->localRequest($task, 'checkpoint-group', $this->groupBody())->assertStatus(429); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseCount('workflow_runs', 1); + $this->assertDatabaseCount('activity_executions', 0); + $this->assertArrayNotHasKey('portable_local_group_checkpoint', WorkflowTask::query()->findOrFail($task['task_id'])->payload); + } + + public function test_local_group_descriptor_cannot_hide_unknown_fields_routing_or_a_fabricated_outcome(): void + { + $task = $this->groupClaim(); + $before = WorkflowHistoryEvent::query()->count(); + foreach ([['unknown_field' => true], ['queue' => 'remote'], ['outcome' => 'completed']] as $changes) { + $body = $this->groupBody(); + $body['commands'][1] = [...$body['commands'][1], ...$changes]; + $this->localRequest($task, 'checkpoint-group', $body)->assertStatus(422); + } + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseCount('workflow_runs', 1); + $this->assertDatabaseCount('activity_executions', 0); + } + + public function test_draining_group_uploads_and_admission_share_one_original_claim_budget(): void + { + $task = $this->groupClaim(); + $this->enableCompletionPayloads(); + $this->configureStoragePressure('draining'); + $expiry = WorkflowTask::query()->findOrFail($task['task_id'])->lease_expires_at->toISOString(); + $body = $this->groupBody(); + $contexts = []; + foreach ([0, 1] as $index) { + $contexts[$index] = ['schema' => RuntimePayloadCompletionContext::PREPARED_SCHEMA, 'kind' => 'workflow', + 'task_id' => $task['task_id'], 'attempt' => $task['workflow_task_attempt'], 'lease_owner' => $task['lease_owner'], + 'operation' => 'local_activity_group_checkpoint', 'checkpoint_id' => $body['checkpoint_id'], + 'slot' => ['commands', $index, 'arguments']]; + $bytes = $body['commands'][$index]['arguments']; + $reference = $this->completionUpload($bytes, $contexts[$index])->assertCreated()->json('reference'); + $body['commands'][$index]['arguments'] = ['codec' => 'avro', 'external_payload' => $reference]; + } + $this->localRequest($task, 'checkpoint-group', $body)->assertOk()->assertJsonPath('checkpointed', true); + $this->assertSame($expiry, WorkflowTask::query()->findOrFail($task['task_id'])->lease_expires_at->toISOString()); + $this->assertDatabaseCount('runtime_payload_completion_budgets', 1); + $this->assertCount(2, RuntimePayloadCompletionBudget::query()->sole()->slots); + $prepared = $this->prepare($task, ['sequence' => 2, 'descriptor' => [...$body['commands'][1], 'type' => 'record_local_activity']]) + ->assertOk()->json(); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => $this->success()])->assertOk(); + } + + public function test_prepared_claim_without_group_capability_cannot_upload_group_payloads(): void + { + $task = $this->claim(); + $this->enableCompletionPayloads(); + $this->configureStoragePressure('draining'); + $context = ['schema' => RuntimePayloadCompletionContext::PREPARED_SCHEMA, 'kind' => 'workflow', + 'task_id' => $task['task_id'], 'attempt' => $task['workflow_task_attempt'], 'lease_owner' => $task['lease_owner'], + 'operation' => 'local_activity_group_checkpoint', 'checkpoint_id' => 'group-1', 'slot' => ['commands', 1, 'arguments']]; + $this->completionUpload('bytes', $context)->assertStatus(409)->assertJsonPath('reason', 'external_payload_completion_lease_rejected'); + $this->assertDatabaseCount('runtime_external_payloads', 0); + } + + public function test_replacement_can_prepare_the_committed_group_without_inventing_an_old_attempt(): void + { + $task = $this->groupClaim(); + $body = $this->groupBody(); + $body['commands'][0]['queue'] = 'children'; + $this->localRequest($task, 'checkpoint-group', $body)->assertOk(); + $execution = ActivityExecution::query()->sole(); + $deadline = $execution->schedule_to_close_deadline_at->toISOString(); + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $this->register('replacement', extraCapabilities: [PreparedLocalActivityPolicy::GROUP_CAPABILITY]); + $replacement = $this->poll('replacement')->assertOk()->json('task'); + $this->assertSame($task['task_id'], $replacement['task_id']); + $this->assertSame($task['workflow_task_attempt'] + 1, $replacement['workflow_task_attempt']); + $this->prepare($task, ['sequence' => 2, 'descriptor' => [...$body['commands'][1], 'type' => 'record_local_activity']])->assertStatus(409); + $prepared = $this->prepare($replacement, ['sequence' => 2, 'descriptor' => [...$body['commands'][1], 'type' => 'record_local_activity']]) + ->assertOk()->assertJsonPath('activity_execution_id', $execution->id)->assertJsonPath('attempt_number', 1) + ->assertJsonPath('workflow_task_attempt', $replacement['workflow_task_attempt'])->json(); + $this->assertSame($deadline, $prepared['schedule_to_close_deadline_at']); + $this->assertDatabaseCount('activity_attempts', 1); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityCancellationAcknowledged)); + $this->assertSame(0, $this->eventCount($task, HistoryEventType::ActivityRetryScheduled)); + } + + public function test_cleanup_group_members_preserve_the_one_original_root_budget(): void + { + [$task, $descriptor, $accepted] = $this->cleanupClaim(group: true); + $commands = []; + foreach ([0, 1] as $index) { + $commands[] = [...$descriptor, 'type' => 'prepare_local_activity', + ...ParallelChildGroup::itemMetadata(2, 2, $index, 'activity')]; + } + $this->localRequest($task, 'checkpoint-group', ['checkpoint_id' => 'cleanup-group', 'start_sequence' => 2, 'commands' => $commands]) + ->assertOk()->assertJsonCount(2, 'local_activities'); + $this->assertDatabaseCount('activity_attempts', 0); + $this->travel(3)->seconds(); + foreach ($commands as $index => $command) { + $prepared = $this->prepare($task, ['sequence' => $index + 2, 'worker_attempt_id' => 'cleanup-'.$index, + 'descriptor' => [...$command, 'type' => 'record_local_activity']])->assertOk() + ->assertJsonPath('cancellation_cleanup.root_request_id', $accepted['request_id']) + ->assertJsonPath('cancellation_cleanup.cleanup_deadline_at', $accepted['cleanup_deadline_at'])->json(); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => $this->success()])->assertOk(); + } + $this->assertSame(1, $this->eventCount($task, HistoryEventType::CooperativeCancellationDelivered)); + $this->assertSame(2, $this->eventCount($task, HistoryEventType::ActivityCompleted)); + } + + public function test_typed_search_history_identity_survives_a_group_checkpoint_then_completion(): void + { + if (! PreparedLocalActivityPolicy::groupsSupported()) { + $this->markTestSkipped('Prepared-group source binding required.'); + } + SearchAttributeDefinition::query()->create(['namespace' => 'default', 'name' => 'Tier', 'type' => 'keyword']); + $task = $this->claim(extraCapabilities: [PreparedLocalActivityPolicy::GROUP_CAPABILITY, 'typed_search_attributes']); + $body = $this->groupBody(); + foreach ([0, 1] as $index) { + $body['commands'][$index] = [...$body['commands'][$index], ...ParallelChildGroup::itemMetadata(2, 2, $index, 'mixed')]; + } + array_unshift($body['commands'], ['type' => 'upsert_search_attributes', 'attributes' => ['Tier' => 'basic'], 'attribute_types' => ['Tier' => 'keyword']]); + $this->localRequest($task, 'checkpoint-group', $body)->assertOk()->assertJsonPath('next_sequence', 4); + $prepared = $this->prepare($task, ['sequence' => 3, 'descriptor' => [...$body['commands'][2], 'type' => 'record_local_activity']])->assertOk()->json(); + $this->localRequest($task, "{$prepared['activity_attempt_id']}/outcome", ['report' => $this->success()])->assertOk(); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [['type' => 'upsert_search_attributes', 'attributes' => ['Tier' => 'pro'], 'attribute_types' => ['Tier' => 'keyword']], + ['type' => 'complete_workflow']], + ])->assertOk()->assertJsonPath('run_status', 'completed'); + $events = WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id']) + ->where('event_type', HistoryEventType::SearchAttributesUpserted)->orderBy('sequence')->get(); + $this->assertCount(2, $events); + $this->assertSame([1, 4], $events->map(fn ($event) => $event->payload['sequence'])->all()); + foreach ($events as $event) { + $this->assertSame(['Tier' => 'keyword'], $event->payload['attribute_types']); + } + } + + private function groupClaim(): array + { + if (! PreparedLocalActivityPolicy::groupsSupported()) { + $this->markTestSkipped('Prepared-group source binding required.'); + } + + return $this->claim(extraCapabilities: [PreparedLocalActivityPolicy::GROUP_CAPABILITY]); + } + + private function groupBody(): array + { + return ['checkpoint_id' => 'group-1', 'start_sequence' => 1, 'commands' => [[ + 'type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', + 'arguments' => Serializer::serializeWithCodec('avro', ['child']), 'payload_codec' => 'avro', + ...ParallelChildGroup::itemMetadata(1, 2, 0, 'mixed'), + ], [...$this->descriptor(['schedule_to_close_timeout' => 30]), 'type' => 'prepare_local_activity', + ...ParallelChildGroup::itemMetadata(1, 2, 1, 'mixed')]]]; + } + + private function claim(bool $prepared = true, array $extraCapabilities = []): array + { + if (! PreparedLocalActivityPolicy::backendSupported()) { + $this->markTestSkipped('Prepared-local source binding required. Published protocol 1.19 refusal is covered separately.'); + } + $this->withHeaders($this->controlHeaders())->postJson('/api/workflows', [ + 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'prepared', 'input' => ['Ada'], + ])->assertCreated(); + $this->register('original', $prepared, $extraCapabilities); + + return $this->poll('original')->assertOk()->json('task'); + } + + private function register(string $id, bool $prepared = true, array $extraCapabilities = []): void + { + $this->withHeaders($this->headers())->postJson('/api/worker/register', [ + 'worker_id' => $id, 'task_queue' => 'prepared', 'runtime' => 'php', + 'supported_workflow_types' => ['tests.external-greeting-workflow'], + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY, ...($prepared ? [PreparedLocalActivityPolicy::CAPABILITY] : []), ...$extraCapabilities], + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + 'max_concurrent_workflow_tasks' => 1, + 'process_metrics' => ['host' => 'test-worker', 'process_started_at' => now()->toISOString(), 'process_id' => $id === 'original' ? 10 : 20], + ])->assertCreated(); + } + + private function poll(string $id): TestResponse + { + return $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', ['worker_id' => $id, 'task_queue' => 'prepared']); + } + + private function replaceClaim(array $task): array + { + // Exercise the real poll takeover using an expired workflow claim. + // Keep the prepared callback lease live to test original stop authority. + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $this->register('replacement'); + $replacement = $this->poll('replacement')->assertOk()->json('task'); + $this->assertSame($task['task_id'], $replacement['task_id']); + $this->assertSame($task['workflow_task_attempt'] + 1, $replacement['workflow_task_attempt']); + + return $replacement; + } + + private function prepare(array $task, array $overrides = []): TestResponse + { + return $this->localRequest($task, 'prepare', ['sequence' => 1, 'worker_attempt_id' => 'sdk-local-1', 'descriptor' => $this->descriptor(), ...$overrides]); + } + + private function descriptor(array $overrides = []): array + { + return ['type' => 'record_local_activity', 'activity_type' => 'opaque-local', + 'arguments' => Serializer::serializeWithCodec('avro', ['Ada']), 'payload_codec' => 'avro', ...$overrides]; + } + + private function success(string $value = 'done'): array + { + return ['outcome' => 'completed', 'result' => Serializer::serializeWithCodec('avro', $value), 'payload_codec' => 'avro']; + } + + private function localRequest(array $task, string $path, array $body = []): TestResponse + { + $response = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/{$path}", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], ...$body, + ]); + if ($response->status() === 200) { + $operation = substr($path, strrpos('/'.$path, '/')); + $schema = match ($operation) { + 'prepare' => 'PreparedLocalPreparationResult', 'recover' => 'PreparedLocalRecoveryResult', + 'checkpoint' => 'PreparedLocalCheckpointResult', 'checkpoint-group' => 'PreparedLocalGroupCheckpointResult', + 'control', 'heartbeat' => 'PreparedLocalControlResult', + 'outcome' => 'PreparedLocalOutcomeResult', 'acknowledge-cancellation' => 'PreparedLocalStopResult', + }; + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/'.$schema, json_decode($response->getContent(), flags: JSON_THROW_ON_ERROR)); + } + + return $response; + } + + private function uploadPayload(string $payload, string $namespace): array + { + return $this->call('POST', '/api/external-payloads/v1', [], [], [], [ + 'CONTENT_TYPE' => 'application/octet-stream', 'HTTP_X_NAMESPACE' => $namespace, + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_CODEC' => 'avro', + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_SIZE' => (string) strlen($payload), + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_SHA256' => hash('sha256', $payload), + ], $payload)->assertCreated()->json('reference'); + } + + private function enableCompletionPayloads(): void + { + $this->payloadDirectory = sys_get_temp_dir().'/dw-prepared-completion-'.getmypid(); + foreach (['default', 'other'] as $namespace) { + WorkflowNamespace::query()->updateOrCreate(['name' => $namespace], [ + 'description' => 'Fixture', 'retention_days' => 30, 'status' => 'active', + 'external_payload_storage' => ['driver' => 'local', 'enabled' => true, 'threshold_bytes' => 32, + 'config' => ['uri' => 'file://'.$this->payloadDirectory.'/'.$namespace]], + ]); + } + } + + private function completionContext(array $task, string $operation, int|string $identity): array + { + return ['schema' => RuntimePayloadCompletionContext::PREPARED_SCHEMA, 'kind' => 'workflow', + 'task_id' => $task['task_id'], 'attempt' => $task['workflow_task_attempt'], 'lease_owner' => $task['lease_owner'], + 'operation' => 'local_activity_'.$operation, + 'slot' => match ($operation) { + 'checkpoint' => ['commands', 0, 'result'], 'outcome' => ['report', 'result'], default => ['descriptor', 'arguments'], + }, + match ($operation) { + 'checkpoint' => 'checkpoint_id', 'outcome' => 'activity_attempt_id', default => 'sequence', + } => $identity]; + } + + private function completionUpload(string $bytes, array $context, string $namespace = 'default'): TestResponse + { + return $this->call('POST', '/api/external-payloads/v1', [], [], [], [ + 'CONTENT_TYPE' => 'application/octet-stream', 'HTTP_X_NAMESPACE' => $namespace, + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_CODEC' => 'avro', + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_SIZE' => (string) strlen($bytes), + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_SHA256' => hash('sha256', $bytes), + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_COMPLETION' => json_encode($context, JSON_THROW_ON_ERROR), + ], $bytes); + } + + private function eventCount(array $task, HistoryEventType $type): int + { + return WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id'])->where('event_type', $type)->count(); + } + + private function headers(string $version = '1.20'): array + { + return ['X-Namespace' => 'default', WorkerProtocol::HEADER => $version]; + } + + private function controlHeaders(): array + { + return ['X-Namespace' => 'default', 'X-Durable-Workflow-Control-Plane-Version' => '2']; + } +} diff --git a/tests/Feature/WorkerProtocolVersionCoverageTest.php b/tests/Feature/WorkerProtocolVersionCoverageTest.php index 467eb4be..2fbbc5ee 100644 --- a/tests/Feature/WorkerProtocolVersionCoverageTest.php +++ b/tests/Feature/WorkerProtocolVersionCoverageTest.php @@ -42,12 +42,15 @@ public static function workerEndpointProvider(): array 'workflow-tasks.poll' => ['method' => 'post', 'path' => '/api/worker/workflow-tasks/poll'], 'workflow-tasks.history' => ['method' => 'post', 'path' => '/api/worker/workflow-tasks/task-1/history'], 'workflow-tasks.heartbeat' => ['method' => 'post', 'path' => '/api/worker/workflow-tasks/task-1/heartbeat'], + 'workflow-tasks.deliver-cancellation' => ['method' => 'post', 'path' => '/api/worker/workflow-tasks/task-1/deliver-cancellation'], 'workflow-tasks.complete' => ['method' => 'post', 'path' => '/api/worker/workflow-tasks/task-1/complete'], 'workflow-tasks.fail' => ['method' => 'post', 'path' => '/api/worker/workflow-tasks/task-1/fail'], 'query-tasks.poll' => ['method' => 'post', 'path' => '/api/worker/query-tasks/poll'], 'query-tasks.complete' => ['method' => 'post', 'path' => '/api/worker/query-tasks/task-1/complete'], 'query-tasks.fail' => ['method' => 'post', 'path' => '/api/worker/query-tasks/task-1/fail'], 'activity-tasks.poll' => ['method' => 'post', 'path' => '/api/worker/activity-tasks/poll'], + 'activity-tasks.status' => ['method' => 'post', 'path' => '/api/worker/activity-tasks/task-1/status'], + 'activity-tasks.acknowledge-cancellation' => ['method' => 'post', 'path' => '/api/worker/activity-tasks/task-1/acknowledge-cancellation'], 'activity-tasks.complete' => ['method' => 'post', 'path' => '/api/worker/activity-tasks/task-1/complete'], 'activity-tasks.fail' => ['method' => 'post', 'path' => '/api/worker/activity-tasks/task-1/fail'], 'activity-tasks.heartbeat' => ['method' => 'post', 'path' => '/api/worker/activity-tasks/task-1/heartbeat'], diff --git a/tests/Feature/WorkflowDebugTest.php b/tests/Feature/WorkflowDebugTest.php index b1931beb..4826cde8 100644 --- a/tests/Feature/WorkflowDebugTest.php +++ b/tests/Feature/WorkflowDebugTest.php @@ -5,6 +5,7 @@ use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Queue; +use PHPUnit\Framework\Attributes\DataProvider; use Tests\Feature\Concerns\ServerTestHelpers; use Tests\Fixtures\AwaitApprovalWorkflow; use Tests\TestCase; @@ -21,6 +22,8 @@ use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; +use Workflow\V2\Support\CancellationCascadeView; +use Workflow\V2\Support\RunSummaryProjector; use Workflow\V2\Support\WorkerCompatibilityFleet; class WorkflowDebugTest extends TestCase @@ -40,6 +43,85 @@ protected function setUp(): void ]); } + #[DataProvider('recoveredTerminalOutcomes')] + public function test_recovered_tasks_are_informational_history_after_a_run_closes(string $outcome): void + { + $workflowId = 'debug-recovered-'.$outcome; + [$runId, $task, $workerId, $attempt] = $this->diagnosticTask($workflowId); + $task->forceFill(['repair_count' => 1])->save(); + + if ($outcome === 'cancelled') { + $this->postJson("/api/workflows/{$workflowId}/cancel", [], $this->controlPlaneHeadersWithWorkerProtocol()) + ->assertOk(); + } else { + $this->postJson("/api/worker/workflow-tasks/{$task->id}/complete", [ + 'lease_owner' => $workerId, + 'workflow_task_attempt' => $attempt, + 'commands' => [['type' => 'complete_workflow']], + ], $this->workerHeaders())->assertOk()->assertJsonPath('run_status', 'completed'); + } + + $this->getJson("/api/workflows/{$workflowId}/debug", $this->controlPlaneHeadersWithWorkerProtocol()) + ->assertOk() + ->assertJsonPath('run_id', $runId) + ->assertJsonPath('diagnostic_status', 'terminal') + ->assertJsonPath('execution.status', $outcome) + ->assertJsonPath('execution.task_problem', true) + ->assertJsonPath('execution.task_problem_badge.code', 'history') + ->assertJsonPath('findings.0.code', 'task_recovery_history') + ->assertJsonPath('findings.0.severity', 'info') + ->assertJsonPath('findings.0.message', 'The run previously needed workflow-task repair or replay recovery.') + ->assertJsonMissing(['code' => 'task_problem']); + } + + public static function recoveredTerminalOutcomes(): array + { + return ['completed' => ['completed'], 'cancelled' => ['cancelled']]; + } + + public function test_active_replay_failure_keeps_its_warning_and_error(): void + { + $workflowId = 'debug-active-replay'; + [$runId, $task] = $this->diagnosticTask($workflowId); + $task->forceFill([ + 'status' => TaskStatus::Failed, + 'payload' => [...($task->payload ?? []), 'replay_blocked' => true], + 'last_error' => 'Recorded workflow commands changed.', + ])->save(); + RunSummaryProjector::project(WorkflowRun::query()->findOrFail($runId)); + + $debug = $this->getJson("/api/workflows/{$workflowId}/debug", $this->controlPlaneHeadersWithWorkerProtocol()) + ->assertOk() + ->assertJsonPath('diagnostic_status', 'needs_attention') + ->assertJsonPath('execution.task_problem_badge.code', 'replay_blocked'); + $findings = collect($debug->json('findings'))->keyBy('code'); + $this->assertSame('warning', $findings['task_problem']['severity']); + $this->assertSame('error', $findings['workflow_replay_blocked']['severity']); + $this->assertFalse($findings->has('task_recovery_history')); + } + + private function diagnosticTask(string $workflowId): array + { + $workerId = $workflowId.'-worker'; + $queue = $workflowId.'-queue'; + $this->registerWorker($workerId, $queue, supportedWorkflowTypes: ['tests.await-approval-workflow']); + $started = $this->postJson('/api/workflows', [ + 'workflow_id' => $workflowId, + 'workflow_type' => 'tests.await-approval-workflow', + 'task_queue' => $queue, + ], $this->controlPlaneHeadersWithWorkerProtocol())->assertCreated(); + $polled = $this->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => $workerId, 'task_queue' => $queue, + ], $this->workerHeaders())->assertOk(); + + return [ + $started->json('run_id'), + WorkflowTask::query()->findOrFail($polled->json('task.task_id')), + $workerId, + (int) $polled->json('task.workflow_task_attempt'), + ]; + } + public function test_it_aggregates_a_one_shot_workflow_debug_diagnostic(): void { $this->registerWorker( @@ -86,6 +168,8 @@ public function test_it_aggregates_a_one_shot_workflow_debug_diagnostic(): void ->assertJsonPath('workflow_id', 'wf-debug') ->assertJsonPath('run_id', $runId) ->assertJsonPath('namespace', 'default') + ->assertJsonPath('cancellation_cascade_supported', class_exists(CancellationCascadeView::class)) + ->assertJsonPath('cancellation_cascade', null) ->assertJsonPath('diagnostic_status', 'pending_work') ->assertJsonPath('execution.status', 'pending') ->assertJsonPath('execution.task_queue', 'debug-queue') diff --git a/tests/Feature/WorkflowStreamsTest.php b/tests/Feature/WorkflowStreamsTest.php index 55d45c54..3411be2e 100644 --- a/tests/Feature/WorkflowStreamsTest.php +++ b/tests/Feature/WorkflowStreamsTest.php @@ -8,6 +8,7 @@ use App\Models\WorkflowDurableStreamItem; use App\Models\WorkflowNamespace; use App\Support\RuntimeExternalPayloadRegistry; +use App\Support\WorkerProtocol; use App\Support\WorkflowStreamCommandProcessor; use App\Support\WorkflowStreamService; use Illuminate\Foundation\Testing\RefreshDatabase; @@ -583,6 +584,17 @@ public function test_terminal_completion_commits_its_last_stream_item_and_close( $this->assertSame('completed', WorkflowRun::query()->findOrFail($task['run_id'])->status->value); } + public function test_legacy_request_refuses_explicit_scope_before_stream_output(): void + { + $task = $this->claimStreamCompletionTask(); + $this->withHeaders(array_replace($this->workerHeaders(), [WorkerProtocol::HEADER => '1.19']))->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [$this->streamCompletionCommand($task), ['type' => 'start_timer', 'delay_seconds' => 10, 'cancellation_scope_id' => 'root']], + ])->assertConflict()->assertJsonPath('recorded', false)->assertJsonPath('reason', 'cancellation_scope_membership_unavailable'); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + $this->assertDatabaseCount('workflow_run_timers', 0); + } + public function test_stream_failure_rolls_back_successful_native_admission(): void { $task = $this->claimStreamCompletionTask(); diff --git a/tests/Feature/WorkflowTaskRepairRaceTest.php b/tests/Feature/WorkflowTaskRepairRaceTest.php new file mode 100644 index 00000000..cb6058ef --- /dev/null +++ b/tests/Feature/WorkflowTaskRepairRaceTest.php @@ -0,0 +1,235 @@ +markTestSkipped('Set '.$prefix.'HOST to a disposable database test server.'); + } + if (! function_exists('pcntl_fork') || ! function_exists('posix_kill') || ! function_exists('stream_socket_pair')) { + $this->markTestSkipped('Process control and local sockets are required.'); + } + $port = getenv($prefix.'PORT') ?: ($driver === 'pgsql' ? '5432' : '3306'); + $user = getenv($prefix.'USER') ?: ($driver === 'pgsql' ? 'postgres' : 'root'); + $password = getenv($prefix.'PASSWORD') ?: ''; + $this->databaseAdmin = [$driver.':host='.$host.';port='.$port.($driver === 'pgsql' ? ';dbname=postgres' : ''), $user, $password]; + $this->databaseName = 'dw_task_repair_'.bin2hex(random_bytes(6)); + $admin = new PDO(...$this->databaseAdmin); + $admin->exec('CREATE DATABASE '.$this->databaseName); + unset($admin); + config(['database.default' => $driver, 'database.connections.'.$driver.'.host' => $host, + 'database.connections.'.$driver.'.port' => $port, 'database.connections.'.$driver.'.database' => $this->databaseName, + 'database.connections.'.$driver.'.username' => $user, 'database.connections.'.$driver.'.password' => $password, + 'database.connections.'.$driver.'.url' => null]); + DB::purge($driver); + $this->artisan('migrate:fresh', ['--force' => true])->assertExitCode(0); + Queue::fake(); + $this->configureWorkflowTypes(['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class]); + $this->createNamespace('default'); + $this->registerWorker('repair-race-worker', 'repair-race'); + } + + protected function tearDown(): void + { + if ($this->databaseAdmin !== null) { + DB::disconnect(); + $admin = new PDO(...$this->databaseAdmin); + $admin->exec('DROP DATABASE '.$this->databaseName); + } + parent::tearDown(); + } + + public static function mutations(): array + { + $cases = []; + foreach (['mysql', 'pgsql'] as $driver) { + foreach (['heartbeat', 'complete', 'fail', 'waiting_history'] as $operation) { + $cases[$driver.' '.$operation] = [$driver, $operation]; + } + } + + return $cases; + } + + #[DataProvider('mutations')] + public function test_repair_can_lock_tasks_while_a_worker_mutation_waits_for_the_run(string $driver, string $operation): void + { + $this->initializeDatabase($driver); + $runId = $this->postJson('/api/workflows', [ + 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'repair-race', 'input' => ['Ada'], + ], $this->apiHeaders())->assertCreated()->json('run_id'); + $claim = $this->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'repair-race-worker', 'task_queue' => 'repair-race', 'poll_request_id' => 'claim', + ], $this->workerHeaders())->assertOk()->json('task'); + $this->assertIsArray($claim); + $actors = []; + DB::purge(); + try { + $actors['repair'] = $this->forkActor('repair', $runId, $claim); + $this->readMessage($actors['repair']); + fwrite($actors['repair']['socket'], "go\n"); + $this->assertTrue($this->readMessage($actors['repair'])['run_locked']); + $actors['worker'] = $this->forkActor($operation, $runId, $claim); + $worker = $this->readMessage($actors['worker']); + fwrite($actors['worker']['socket'], "go\n"); + $this->awaitRunWait($worker['connection_id']); + + // Repair holds the run, and the real HTTP request is waiting for + // it. Its task must remain available to repair, rather than form + // the opposite half of the captured task/run deadlock. + DB::transaction(function () use ($claim): void { + $task = WorkflowTask::query()->whereKey($claim['task_id'])->lock('for update nowait')->firstOrFail(); + $this->assertSame('repair-race-worker', $task->lease_owner); + $this->assertSame($claim['workflow_task_attempt'], $task->attempt_count); + }); + + fwrite($actors['repair']['socket'], "continue\n"); + $this->assertTrue($this->finishActor($actors['repair'])['accepted']); + unset($actors['repair']); + $result = $this->finishActor($actors['worker']); + unset($actors['worker']); + $this->assertSame(200, $result['status'], json_encode($result)); + $this->assertTrue($result['body'][$operation === 'heartbeat' ? 'renewed' : 'recorded']); + $this->assertSame(match ($operation) { + 'complete' => 'completed', 'waiting_history' => 'waiting', default => 'pending', + }, + WorkflowRun::query()->findOrFail($runId)->status->value); + } finally { + foreach ($actors as $actor) { + posix_kill($actor['pid'], SIGKILL); + pcntl_waitpid($actor['pid'], $status); + fclose($actor['socket']); + } + DB::purge(); + DB::reconnect(); + } + } + + /** @return array{pid: int, socket: resource} */ + private function forkActor(string $operation, string $runId, array $claim): array + { + $sockets = stream_socket_pair(STREAM_PF_UNIX, STREAM_SOCK_STREAM, STREAM_IPPROTO_IP); + $this->assertIsArray($sockets); + $pid = pcntl_fork(); + $this->assertNotSame(-1, $pid); + if ($pid === 0) { + fclose($sockets[0]); + stream_set_timeout($sockets[1], 15); + $ok = false; + try { + DB::reconnect(); + $idQuery = DB::connection()->getDriverName() === 'mysql' + ? 'SELECT CONNECTION_ID() AS id' : 'SELECT pg_backend_pid() AS id'; + fwrite($sockets[1], json_encode(['connection_id' => (int) DB::selectOne($idQuery)->id]).PHP_EOL); + if (fgets($sockets[1]) !== "go\n") { + throw new RuntimeException('Actor was not released.'); + } + if ($operation === 'repair') { + $paused = false; + DB::listen(static function (QueryExecuted $query) use (&$paused, $sockets): void { + if (! $paused && str_contains($query->sql, 'workflow_runs') + && str_contains(strtolower($query->sql), 'for update')) { + $paused = true; + fwrite($sockets[1], json_encode(['run_locked' => true]).PHP_EOL); + if (fgets($sockets[1]) !== "continue\n") { + throw new RuntimeException('Repair was not released.'); + } + } + }); + $result = ['accepted' => WorkflowStub::loadRun($runId)->repair()->accepted()]; + } else { + $body = ['lease_owner' => 'repair-race-worker', + 'workflow_task_attempt' => $claim['workflow_task_attempt']]; + if ($operation === 'complete') { + $body['commands'] = [['type' => 'complete_workflow', + 'result' => Serializer::serializeWithCodec('avro', 'done')]]; + } elseif (in_array($operation, ['fail', 'waiting_history'], true)) { + $body['failure'] = ['message' => 'Synthetic worker failure', + 'type' => $operation === 'waiting_history' ? 'WorkflowTaskWaitingForHistory' : 'TestFailure']; + } + $endpoint = $operation === 'waiting_history' ? 'fail' : $operation; + $response = $this->postJson('/api/worker/workflow-tasks/'.$claim['task_id'].'/'.$endpoint, + $body, $this->workerHeaders()); + $result = ['status' => $response->status(), 'body' => array_intersect_key($response->json(), + array_flip(['renewed', 'recorded', 'reason', 'outcome', 'run_status', 'errors']))]; + } + fwrite($sockets[1], json_encode(['result' => $result], JSON_THROW_ON_ERROR).PHP_EOL); + $ok = true; + } catch (Throwable $error) { + fwrite($sockets[1], json_encode(['error' => $error::class.': '.$error->getMessage()]).PHP_EOL); + } finally { + DB::disconnect(); + fclose($sockets[1]); + } + exit($ok ? 0 : 1); + } + fclose($sockets[1]); + stream_set_timeout($sockets[0], 15); + + return ['pid' => $pid, 'socket' => $sockets[0]]; + } + + private function awaitRunWait(int $connectionId): void + { + $query = DB::connection()->getDriverName() === 'mysql' + ? 'SELECT COUNT(*) AS waiting FROM performance_schema.data_lock_waits w JOIN performance_schema.threads t ON t.THREAD_ID = w.REQUESTING_THREAD_ID WHERE t.PROCESSLIST_ID = ?' + : 'SELECT COUNT(*) AS waiting FROM pg_locks WHERE pid = ? AND NOT granted'; + $until = microtime(true) + 5; + do { + if ((int) DB::selectOne($query, [$connectionId])->waiting > 0) { + $this->addToAssertionCount(1); + + return; + } + usleep(10_000); + } while (microtime(true) < $until); + $this->fail('Worker mutation did not wait on the run lock, connection '.$connectionId.'.'); + } + + private function readMessage(array $actor): array + { + $line = fgets($actor['socket']); + $this->assertIsString($line, 'Actor did not respond.'); + $message = json_decode($line, true, flags: JSON_THROW_ON_ERROR); + $this->assertArrayNotHasKey('error', $message, $message['error'] ?? ''); + + return $message; + } + + private function finishActor(array $actor): array + { + $message = $this->readMessage($actor); + pcntl_waitpid($actor['pid'], $status); + $this->assertTrue(pcntl_wifexited($status)); + $this->assertSame(0, pcntl_wexitstatus($status)); + fclose($actor['socket']); + + return $message['result']; + } +} diff --git a/tests/Feature/WorkflowWorkerProtocolTest.php b/tests/Feature/WorkflowWorkerProtocolTest.php index af566687..42512931 100644 --- a/tests/Feature/WorkflowWorkerProtocolTest.php +++ b/tests/Feature/WorkflowWorkerProtocolTest.php @@ -53,6 +53,42 @@ protected function setUp(): void ]); } + public function test_worker_mutations_refuse_a_task_whose_run_is_outside_its_namespace(): void + { + Queue::fake(); + $this->configureWorkflowTypes(); + $this->createNamespace('default', 'Default'); + $this->createNamespace('other', 'Other'); + $runId = $this->postJson('/api/workflows', [ + 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'namespace-lock', 'input' => ['Ada'], + ], $this->apiHeaders())->assertCreated()->json('run_id'); + $this->registerWorker('namespace-lock-worker', 'namespace-lock'); + $claim = $this->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'namespace-lock-worker', 'task_queue' => 'namespace-lock', + ], $this->workerHeaders())->assertOk()->json('task'); + $this->assertIsArray($claim); + WorkflowRun::query()->whereKey($runId)->update(['namespace' => 'other']); + $taskBefore = WorkflowTask::query()->findOrFail($claim['task_id'])->getRawOriginal(); + $historyBefore = WorkflowHistoryEvent::query()->where('workflow_run_id', $runId)->count(); + + foreach (['heartbeat', 'complete', 'fail'] as $operation) { + $body = ['lease_owner' => $claim['lease_owner'], 'workflow_task_attempt' => $claim['workflow_task_attempt']]; + if ($operation === 'complete') { + $body['commands'] = [['type' => 'complete_workflow', 'result' => Serializer::serializeWithCodec('avro', 'refuse')]]; + } elseif ($operation === 'fail') { + $body['failure'] = ['message' => 'Refuse cross-namespace mutation']; + } + $this->postJson('/api/worker/workflow-tasks/'.$claim['task_id'].'/'.$operation, $body, $this->workerHeaders()) + ->assertNotFound()->assertJsonPath('reason', 'task_not_found') + ->assertJsonPath('error', 'Workflow task not found.') + ->assertJsonPath('task_id', $claim['task_id']) + ->assertJsonPath('workflow_task_attempt', $claim['workflow_task_attempt']); + } + $this->assertSame($taskBefore, WorkflowTask::query()->findOrFail($claim['task_id'])->getRawOriginal()); + $this->assertSame($historyBefore, WorkflowHistoryEvent::query()->where('workflow_run_id', $runId)->count()); + $this->assertSame('pending', WorkflowRun::query()->findOrFail($runId)->status->value); + } + public function test_it_starts_workflows_and_completes_workflow_tasks_through_the_external_worker_protocol(): void { Queue::fake(); @@ -2473,6 +2509,16 @@ public function test_it_drops_claimed_workflow_tasks_when_the_bridge_cannot_buil 'run_count' => 0, ]); + WorkflowRun::query()->create([ + 'id' => 'run-bridge-missing-task', + 'workflow_instance_id' => 'wf-bridge-missing-task', + 'workflow_class' => ExternalGreetingWorkflow::class, + 'workflow_type' => 'tests.external-greeting-workflow', + 'namespace' => 'default', + 'run_number' => 1, + 'status' => 'pending', + ]); + $recordedAt = now()->toJSON(); $this->mock(WorkflowTaskBridge::class, function (MockInterface $mock) use ($recordedAt): void { @@ -5872,6 +5918,16 @@ public function test_poll_response_compresses_history_above_threshold_via_mock() 'run_count' => 0, ]); + WorkflowRun::query()->create([ + 'id' => 'run-compress', + 'workflow_instance_id' => 'wf-compress-mock', + 'workflow_class' => ExternalGreetingWorkflow::class, + 'workflow_type' => 'tests.external-greeting-workflow', + 'namespace' => 'default', + 'run_number' => 1, + 'status' => 'pending', + ]); + $recordedAt = now()->toJSON(); $events = []; diff --git a/tests/Fixtures/CooperativeCancellationRaceRequest.php b/tests/Fixtures/CooperativeCancellationRaceRequest.php new file mode 100644 index 00000000..4acb7f69 --- /dev/null +++ b/tests/Fixtures/CooperativeCancellationRaceRequest.php @@ -0,0 +1,117 @@ +make(Kernel::class); +$app->make(Illuminate\Contracts\Console\Kernel::class)->bootstrap(); + +config([ + 'app.env' => 'testing', + 'app.key' => 'base64:dGVzdGluZy10ZXN0aW5nLXRlc3RpbmctdGVzdGluZzEyMzQ1Ng==', + 'database.default' => 'sqlite', + 'database.connections.sqlite.database' => $argv[1], + 'database.connections.sqlite.busy_timeout' => 100, + 'database.connections.sqlite.journal_mode' => 'WAL', + 'database.connections.sqlite.transaction_mode' => 'IMMEDIATE', + 'cache.default' => 'array', + 'server.auth.driver' => 'none', + 'server.worker_protocol.version' => '1.20', + 'server.polling.timeout' => 0, + 'server.polling.interval_ms' => 1, + 'server.polling.cache_path' => sys_get_temp_dir().'/dw-cooperative-race-'.getmypid(), + 'workflows.v2.types.workflows' => [ + 'tests.external-greeting-workflow' => ExternalGreetingWorkflow::class, + ], +]); +DB::purge('sqlite'); +// Initialize the connection before releasing the request barrier. The race +// exercises the durable mutations rather than connection setup PRAGMAs. +DB::connection()->getPdo(); +Queue::fake(); +if (isset($argv[7])) { + Carbon::setTestNow(Carbon::parse($argv[7])); + config(['workflows.v2.workflow_task_lease_seconds' => 1]); + if ($argv[2] === 'poll') { + Carbon::setTestNow(Carbon::parse($argv[7])->addSeconds(2)); + } elseif ($argv[2] === 'heartbeat') { + $native = $app->make(WorkflowTaskBridge::class); + // Only add an IPC barrier. Both ownership status and renewal still + // execute the published Native implementation against the real DB. + $bridge = Mockery::mock(WorkflowTaskBridge::class); + $bridge->shouldReceive('status')->andReturnUsing($native->status(...)); + $paused = false; + $bridge->shouldReceive('heartbeat')->andReturnUsing(static function (string $taskId) use ($native, &$paused, $argv): array { + if (! $paused) { + $paused = true; + Carbon::setTestNow(Carbon::parse($argv[7])->addSeconds(2)); + fwrite(STDOUT, "checked\n"); + if (trim((string) fgets(STDIN)) !== 'go') { + exit(2); + } + } + + return $native->heartbeat($taskId); + }); + $app->instance(WorkflowTaskBridge::class, $bridge); + $app->instance(WorkflowTaskOwnership::class, new WorkflowTaskOwnership($bridge)); + } +} +register_shutdown_function(static function (): void { + (new Filesystem)->deleteDirectory( + sys_get_temp_dir().'/dw-cooperative-race-'.getmypid(), + ); +}); + +fwrite(STDOUT, "ready\n"); +if (trim((string) fgets(STDIN)) !== 'go') { + exit(2); +} + +$operation = $argv[2]; +$path = match ($operation) { + 'request' => '/api/workflows/'.$argv[3].'/request-cancellation', + 'heartbeat' => '/api/worker/workflow-tasks/'.$argv[3].'/heartbeat', + default => '/api/worker/workflow-tasks/poll', +}; +$body = match ($operation) { + 'request' => [], + 'heartbeat' => ['lease_owner' => $argv[4], 'workflow_task_attempt' => (int) $argv[8]], + default => ['worker_id' => $argv[4], 'task_queue' => $argv[5], 'poll_request_id' => 'race-poll'], +}; +$attempts = []; +for ($attempt = 1; $attempt <= 3; $attempt++) { + $request = Request::create($path, 'POST', server: [ + 'CONTENT_TYPE' => 'application/json', + 'HTTP_ACCEPT' => 'application/json', + 'HTTP_X_NAMESPACE' => 'default', + 'HTTP_X_DURABLE_WORKFLOW_PROTOCOL_VERSION' => $argv[6], + 'HTTP_X_DURABLE_WORKFLOW_CONTROL_PLANE_VERSION' => '2', + ], content: json_encode($body, JSON_THROW_ON_ERROR)); + $response = $kernel->handle($request); + $responseBody = json_decode($response->getContent(), true, flags: JSON_THROW_ON_ERROR); + $attempts[] = ['status' => $response->getStatusCode(), 'reason' => $responseBody['reason'] ?? null]; + $kernel->terminate($request, $response); + if ($response->getStatusCode() !== 503 || ($responseBody['reason'] ?? null) !== 'backend_lock_pressure' || $attempt === 3) { + break; + } + // Follow the published pressure response without changing request or poll + // identity. Every other failure remains visible to the parent assertion. + usleep(max(1, (int) ($responseBody['retry_after_seconds'] ?? 1)) * 1000000); +} +fwrite(STDOUT, json_encode([ + 'status' => $response->getStatusCode(), + 'body' => $responseBody, + 'attempts' => $attempts, +], JSON_THROW_ON_ERROR)); diff --git a/tests/Source/CancellationCascadeDiagnosticsTest.php b/tests/Source/CancellationCascadeDiagnosticsTest.php new file mode 100644 index 00000000..bb2de119 --- /dev/null +++ b/tests/Source/CancellationCascadeDiagnosticsTest.php @@ -0,0 +1,245 @@ +assertTrue(class_exists(CancellationCascadeView::class), 'This qualification requires the exact Native cancellation inspection source.'); + Queue::fake(); + config([ + 'server.worker_protocol.version' => '1.20', + 'workflows.v2.types.workflows' => ['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class], + ]); + WorkflowNamespace::query()->create([ + 'name' => 'default', 'description' => 'Test', 'retention_days' => 30, 'status' => 'active', + ]); + $this->travelTo(now()->startOfSecond()); + } + + public function test_requested_delivered_and_completed_cleanup_share_the_original_budget(): void + { + [$workflowId, $runId] = $this->start(); + $task = $this->poll()->json('task'); + $original = $this->requestCancellation($workflowId)->assertStatus(202)->json('cancellation_request'); + $before = WorkflowHistoryEvent::query()->count(); + $requested = $this->debug($workflowId)->assertOk() + ->assertJsonPath('cancellation_cascade_supported', true) + ->assertJsonPath('cancellation_cascade.schema', 'durable-workflow.cancellation-cascade/v1') + ->assertJsonPath('cancellation_cascade.selected_run_id', $runId) + ->assertJsonPath('cancellation_cascade.root.root_request_id', $original['request_id']) + ->assertJsonPath('cancellation_cascade.root.cleanup_deadline_at', $original['cleanup_deadline_at']) + ->assertJsonPath('cancellation_cascade.runs.0.lifecycle', 'requested') + ->assertJsonPath('cancellation_cascade.inspection_complete', true) + ->assertJsonPath('cancellation_cascade.truncated', false); + $this->assertSame($before, WorkflowHistoryEvent::query()->count(), 'Diagnostics must not mutate history.'); + $this->assertSame([], $requested->json('cancellation_cascade.runs.0.activity_stops')); + + $this->deliver($task, $original['request_id'])->assertOk()->assertJsonPath('delivered', true); + $this->debug($workflowId)->assertOk() + ->assertJsonPath('cancellation_cascade.runs.0.lifecycle', 'delivered') + ->assertJsonPath('cancellation_cascade.runs.0.delivery.sequence', 1); + $this->complete($task, [['type' => 'complete_workflow']])->assertOk()->assertJsonPath('run_status', 'cancelled'); + $finished = $this->debug($workflowId, $runId)->assertOk() + ->assertJsonPath('cancellation_cascade.runs.0.lifecycle', 'cancelled') + ->assertJsonPath('cancellation_cascade.runs.0.cleanup.outcome', 'completed') + ->assertJsonPath('cancellation_cascade.runs.0.cleanup.cleanup_deadline_at', $original['cleanup_deadline_at']) + ->assertJsonPath('cancellation_cascade.inspection_complete', true) + ->json('cancellation_cascade'); + $this->travel(10)->seconds(); + $this->requestCancellation($workflowId, 300)->assertOk() + ->assertJsonPath('duplicate', true)->assertJsonPath('cancellation_request.request_id', $original['request_id']) + ->assertJsonPath('cancellation_request.cleanup_deadline_at', $original['cleanup_deadline_at']); + $this->assertSame($finished, $this->debug($workflowId, $runId)->assertOk()->json('cancellation_cascade')); + } + + public function test_child_propagation_is_visible_from_either_run_without_inventing_a_second_root(): void + { + [$workflowId, $runId] = $this->start(); + $task = $this->poll()->json('task'); + $this->complete($task, [[ + 'type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', + 'cancellation_policy' => 'wait_cancellation_completed', + ]])->assertOk(); + $link = WorkflowLink::query()->where('parent_workflow_run_id', $runId)->where('link_type', 'child_workflow')->sole(); + $original = $this->requestCancellation($workflowId)->assertStatus(202)->json('cancellation_request'); + // The child was scheduled first. Hold its claim so the following poll obtains the parent cancellation task. + $childTask = $this->poll('child')->assertOk()->json('task'); + $this->assertSame($link->child_workflow_run_id, $childTask['run_id']); + $parentTask = $this->poll('parent')->assertOk()->json('task'); + $this->assertSame($runId, $parentTask['run_id']); + $this->deliver($parentTask, $original['request_id'], 'child')->assertOk() + ->assertJsonPath('delivered', false)->assertJsonPath('reason', 'cancellation_waiting_for_child'); + + foreach ([[$workflowId, $runId], [$link->child_workflow_instance_id, $link->child_workflow_run_id]] as [$selectedWorkflow, $selectedRun]) { + $view = $this->debug($selectedWorkflow, $selectedRun)->assertOk() + ->assertJsonPath('cancellation_cascade.selected_run_id', $selectedRun) + ->assertJsonPath('cancellation_cascade.root.root_request_id', $original['request_id']) + ->assertJsonPath('cancellation_cascade.root.cleanup_deadline_at', $original['cleanup_deadline_at']) + ->assertJsonPath('cancellation_cascade.inspection_complete', true)->json('cancellation_cascade'); + $this->assertCount(2, $view['runs']); + $this->assertCount(1, $view['edges']); + $this->assertSame($runId, $view['edges'][0]['parent_run_id']); + $this->assertSame($link->child_workflow_run_id, $view['edges'][0]['child_run_id']); + $this->assertSame('requested', $view['runs'][1]['lifecycle']); + $this->assertTrue($view['runs'][1]['same_root_budget']); + $this->assertNotSame($original['request_id'], $view['runs'][1]['request']['request_id']); + $this->assertSame($original['request_id'], $view['runs'][1]['request']['parent_request_id']); + $this->assertSame('wait_cancellation_completed', $view['runs'][0]['child_propagation'][0]['policy']); + } + } + + public function test_a_deadline_expiry_is_a_terminal_outcome_separate_from_completed_cleanup(): void + { + [$workflowId] = $this->start(); + $task = $this->poll()->json('task'); + $original = $this->requestCancellation($workflowId, 1)->assertStatus(202)->json('cancellation_request'); + $this->travel(2)->seconds(); + $this->deliver($task, $original['request_id'])->assertStatus(409)->assertJsonPath('reason', 'run_cancelled'); + $this->debug($workflowId)->assertOk() + ->assertJsonPath('cancellation_cascade.runs.0.projected_status', 'cancelled') + ->assertJsonPath('cancellation_cascade.runs.0.lifecycle', 'deadline_expired') + ->assertJsonPath('cancellation_cascade.runs.0.cleanup.outcome', 'deadline_expired') + ->assertJsonPath('cancellation_cascade.runs.0.delivery', null) + ->assertJsonPath('cancellation_cascade.inspection_complete', true); + } + + public function test_missing_canonical_history_cannot_be_presented_as_a_successful_cascade(): void + { + [$workflowId, $runId] = $this->start(); + $this->requestCancellation($workflowId)->assertStatus(202); + WorkflowHistoryEvent::query()->where('workflow_run_id', $runId) + ->where('event_type', HistoryEventType::CooperativeCancellationRequested->value)->delete(); + $view = $this->debug($workflowId)->assertOk() + ->assertJsonPath('cancellation_cascade.root', null) + ->assertJsonPath('cancellation_cascade.inspection_complete', false)->json('cancellation_cascade'); + $this->assertContains('request_history_unavailable', array_column($view['findings'], 'code')); + } + + public function test_related_foreign_runs_and_their_cancellation_metadata_are_not_exposed(): void + { + [$workflowId, $runId] = $this->start(); + $this->requestCancellation($workflowId)->assertStatus(202); + [$foreignWorkflow, $foreignRunId] = $this->start(); + $this->requestCancellation($foreignWorkflow)->assertStatus(202); + $foreign = WorkflowRun::query()->findOrFail($foreignRunId); + $foreign->instance->forceFill(['namespace' => 'foreign'])->save(); + $foreign->forceFill(['namespace' => 'foreign'])->save(); + WorkflowLink::query()->create([ + 'parent_workflow_instance_id' => $workflowId, 'parent_workflow_run_id' => $runId, + 'child_workflow_instance_id' => $foreignWorkflow, 'child_workflow_run_id' => $foreignRunId, + 'link_type' => 'child_workflow', 'sequence' => 99, 'is_primary_parent' => false, + ]); + $view = $this->debug($workflowId)->assertOk() + ->assertJsonPath('cancellation_cascade.inspection_complete', false) + ->assertJsonPath('cancellation_cascade.edges.0.child_run_id', null)->json('cancellation_cascade'); + $encoded = json_encode($view, JSON_THROW_ON_ERROR); + $this->assertStringNotContainsString($foreignWorkflow, $encoded); + $this->assertStringNotContainsString($foreignRunId, $encoded); + $this->debug($foreignWorkflow)->assertNotFound(); + $this->debug($workflowId, $foreignRunId)->assertNotFound(); + } + + public function test_run_scoped_debug_keeps_the_historical_cancellation_selection(): void + { + [$workflowId, $runId] = $this->start(); + $original = $this->requestCancellation($workflowId)->assertStatus(202)->json('cancellation_request'); + $selected = WorkflowRun::query()->findOrFail($runId); + $replacement = $selected->replicate(['id']); + $replacement->run_number = $selected->run_number + 1; + $replacement->cancellation_request_command_id = null; + $replacement->cancellation_requested_at = null; + $replacement->cancellation_deadline_at = null; + $replacement->save(); + $selected->instance->forceFill(['current_run_id' => $replacement->id])->save(); + $this->debug($workflowId)->assertOk()->assertJsonPath('run_id', $replacement->id) + ->assertJsonPath('cancellation_cascade', null); + $historical = $this->debug($workflowId, $runId)->assertOk() + ->assertJsonPath('run_id', $runId)->assertJsonPath('cancellation_cascade.selected_run_id', $runId) + ->assertJsonPath('cancellation_cascade.root.root_request_id', $original['request_id']); + $this->assertStringNotContainsString($replacement->id, json_encode($historical->json('cancellation_cascade'), JSON_THROW_ON_ERROR)); + } + + private function start(): array + { + $started = $this->postJson('/api/workflows', [ + 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'cooperative', 'input' => ['Ada'], + ], $this->controlHeaders())->assertCreated(); + + return [$started->json('workflow_id'), $started->json('run_id')]; + } + + private function poll(string $workerId = 'inspector'): TestResponse + { + $this->postJson('/api/worker/register', [ + 'worker_id' => $workerId, 'task_queue' => 'cooperative', 'runtime' => 'php', + 'supported_workflow_types' => ['tests.external-greeting-workflow'], + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY], + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + 'max_concurrent_workflow_tasks' => 2, + ], $this->workerHeaders())->assertCreated(); + + return $this->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => $workerId, 'task_queue' => 'cooperative', + ], $this->workerHeaders())->assertOk(); + } + + private function requestCancellation(string $workflowId, int $timeout = 30): TestResponse + { + return $this->postJson("/api/workflows/{$workflowId}/request-cancellation", [ + 'reason' => 'Maintenance', 'cleanup_timeout_seconds' => $timeout, + ], $this->controlHeaders()); + } + + private function deliver(array $task, string $requestId, string $kind = 'activity'): TestResponse + { + return $this->postJson("/api/worker/workflow-tasks/{$task['task_id']}/deliver-cancellation", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'request_id' => $requestId, 'sequence' => 1, 'call_kind' => $kind, + ], $this->workerHeaders()); + } + + private function complete(array $task, array $commands): TestResponse + { + return $this->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => $commands, + ], $this->workerHeaders()); + } + + private function debug(string $workflowId, ?string $runId = null): TestResponse + { + $target = $runId === null ? "/api/workflows/{$workflowId}" : "/api/workflows/{$workflowId}/runs/{$runId}"; + + return $this->getJson($target.'/debug', $this->controlHeaders()); + } + + private function controlHeaders(): array + { + return ['X-Namespace' => 'default', 'X-Durable-Workflow-Control-Plane-Version' => '2']; + } + + private function workerHeaders(): array + { + return ['X-Namespace' => 'default', WorkerProtocol::HEADER => '1.20']; + } +} diff --git a/tests/Source/CancellationScopeAdmissionTest.php b/tests/Source/CancellationScopeAdmissionTest.php new file mode 100644 index 00000000..59c3d344 --- /dev/null +++ b/tests/Source/CancellationScopeAdmissionTest.php @@ -0,0 +1,754 @@ + '1.20', + 'workflows.v2.workflow_task_lease_seconds' => 10, + 'workflows.v2.types.workflows' => ['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class], + ]); + WorkflowNamespace::query()->create(['name' => 'default', 'retention_days' => 30, 'status' => 'active']); + $this->assertTrue(class_exists(CancellationScopeHistory::class)); + } + + public function test_scope_prefix_requires_no_local_activity_capability_and_replays_before_opening(): void + { + $task = $this->claim(capabilities: [CooperativeCancellationPolicy::CAPABILITY]); + $claim = WorkflowTask::query()->findOrFail($task['task_id']); + $originalLease = $claim->lease_expires_at->toISOString(); + $first = $this->scopePrefix($task)->assertOk()->assertJsonPath('checkpointed', true) + ->assertJsonPath('duplicate', false)->assertJsonPath('start_sequence', 1)->assertJsonPath('next_sequence', 2)->json(); + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/CancellationScopeCheckpointResponse/allOf/1', json_decode(json_encode($first, JSON_THROW_ON_ERROR), flags: JSON_THROW_ON_ERROR)); + $this->assertIsString($first['history_refresh_page_token']); + $before = WorkflowHistoryEvent::query()->count(); + $this->scopePrefix($task)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('fingerprint', $first['fingerprint']); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->openScope($task, ['sequence' => 2])->assertOk()->assertJsonPath('opened', true); + $this->assertSame($originalLease, $claim->refresh()->lease_expires_at->toISOString()); + $this->assertSame('leased', $claim->status->value); + $this->assertArrayHasKey('portable_scope_checkpoint', $claim->payload); + $this->assertArrayNotHasKey('portable_local_checkpoint', $claim->payload); + $this->assertSame(0, ActivityExecution::query()->count()); + } + + public static function refusedScopePrefixes(): array + { + return [ + 'wrong owner' => [['lease_owner' => 'other-worker'], 'lease_owner_mismatch'], + 'wrong attempt' => [['workflow_task_attempt' => 99], 'workflow_task_attempt_mismatch'], + 'future sequence' => [['start_sequence' => 2], 'cancellation_scope_checkpoint_sequence_mismatch'], + 'terminal command' => [['commands' => [['type' => 'complete_workflow', 'result' => Serializer::serializeWithCodec('avro', 'done')]]], 'invalid_cancellation_scope_checkpoint_commands'], + ]; + } + + #[DataProvider('refusedScopePrefixes')] + public function test_scope_prefix_refuses_before_committing_history(array $overrides, string $reason): void + { + $task = $this->claim(capabilities: [CooperativeCancellationPolicy::CAPABILITY]); + $before = WorkflowHistoryEvent::query()->count(); + $this->scopePrefix($task, $overrides)->assertStatus(409)->assertJsonPath('reason', $reason); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_scope_prefix_changed_retry_and_replaced_claim_are_fenced(): void + { + $task = $this->claim(capabilities: [CooperativeCancellationPolicy::CAPABILITY]); + $this->scopePrefix($task)->assertOk(); + $before = WorkflowHistoryEvent::query()->count(); + $this->scopePrefix($task, ['commands' => [['type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 'changed')]]]) + ->assertStatus(409)->assertJsonPath('reason', 'cancellation_scope_checkpoint_mismatch'); + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $replacement = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => $task['lease_owner'], 'task_queue' => 'scope-admission', + ])->assertOk()->json('task'); + $this->scopePrefix($task)->assertStatus(409)->assertJsonPath('reason', 'workflow_task_attempt_mismatch'); + $this->openScope($replacement, ['sequence' => 2])->assertOk()->assertJsonPath('opened', true); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'SideEffectRecorded')->count()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeOpened')->count()); + } + + public function test_scope_prefix_rejects_legacy_missing_backend_and_incompatible_issued_claim(): void + { + $task = $this->claim(capabilities: [CooperativeCancellationPolicy::CAPABILITY]); + $this->scopePrefix($task, version: '1.19')->assertStatus(409)->assertJsonPath('reason', 'cancellation_scope_checkpoint_requires_protocol_1_20'); + $bridge = app(WorkflowTaskBridge::class); + $this->app->instance(WorkflowTaskBridge::class, $this->createMock(WorkflowTaskBridge::class)); + $this->scopePrefix($task)->assertStatus(409)->assertJsonPath('reason', 'cancellation_scope_checkpoint_unavailable'); + $this->app->instance(WorkflowTaskBridge::class, $bridge); + $claim = WorkflowTask::query()->findOrFail($task['task_id']); + $payload = $claim->payload; + $payload['_server_workflow_claim']['capabilities'] = []; + $claim->forceFill(['payload' => $payload])->save(); + $this->scopePrefix($task)->assertStatus(409)->assertJsonPath('reason', 'active_claim_cancellation_not_supported'); + } + + public function test_scope_prefix_hides_other_namespaces_and_refuses_stale_workers(): void + { + $task = $this->claim(capabilities: [CooperativeCancellationPolicy::CAPABILITY]); + WorkflowNamespace::query()->create(['name' => 'other', 'retention_days' => 30, 'status' => 'active']); + $before = WorkflowHistoryEvent::query()->count(); + $this->scopePrefix($task, namespace: 'other')->assertNotFound(); + WorkerRegistration::query()->where('worker_id', $task['lease_owner'])->update(['last_heartbeat_at' => now()->subMinutes(10)]); + $this->scopePrefix($task)->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + private function scopePrefix(array $task, array $overrides = [], string $version = '1.20', string $namespace = 'default') + { + return $this->withHeaders(['X-Namespace' => $namespace, WorkerProtocol::HEADER => $version]) + ->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/checkpoint", array_replace([ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'checkpoint_id' => 'scope-prefix', 'start_sequence' => 1, + 'commands' => [['type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 'before-scope')]], + ], $overrides)); + } + + public function test_scope_prefix_uploads_during_draining_share_one_fenced_claim_allowance(): void + { + $task = $this->claim(capabilities: [CooperativeCancellationPolicy::CAPABILITY]); + $directory = storage_path('framework/testing/scope-prefix-uploads'); + WorkflowNamespace::query()->where('name', 'default')->update(['external_payload_storage' => [ + 'driver' => 'local', 'enabled' => true, 'threshold_bytes' => 32, + 'config' => ['uri' => 'file://'.$directory], + ]]); + $this->configureStoragePressure('draining'); + $payload = Serializer::serializeWithCodec('avro', str_repeat('before-scope', 200)); + $context = [ + 'schema' => RuntimePayloadCompletionContext::PREPARED_SCHEMA, 'kind' => 'workflow', + 'task_id' => $task['task_id'], 'attempt' => $task['workflow_task_attempt'], 'lease_owner' => $task['lease_owner'], + 'operation' => 'cancellation_scope_checkpoint', 'checkpoint_id' => 'scope-prefix', 'slot' => ['commands', 0, 'result'], + ]; + $upload = fn (array $value) => $this->call('POST', '/api/external-payloads/v1', [], [], [], [ + 'CONTENT_TYPE' => 'application/octet-stream', 'HTTP_X_NAMESPACE' => 'default', + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_CODEC' => 'avro', 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_SIZE' => (string) strlen($payload), + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_SHA256' => hash('sha256', $payload), + 'HTTP_X_DURABLE_WORKFLOW_PAYLOAD_COMPLETION' => json_encode($value, JSON_THROW_ON_ERROR), + ], $payload); + try { + $reference = $upload($context)->assertCreated()->json('reference'); + $this->scopePrefix($task, ['commands' => [['type' => 'record_side_effect', 'result' => ['codec' => 'avro', 'external_payload' => $reference]]]]) + ->assertOk()->assertJsonPath('checkpointed', true); + $stored = WorkflowHistoryEvent::query()->where('event_type', 'SideEffectRecorded')->sole()->payload['result']; + $this->assertSame('avro', $stored['codec']); + $this->assertSame(hash('sha256', $payload), $stored['external_storage']['sha256']); + $fetch = $this->withHeaders([ + 'X-Namespace' => 'default', 'X-Durable-Workflow-Payload-Codec' => $reference['codec'], + 'X-Durable-Workflow-Payload-Size' => (string) $reference['size_bytes'], + 'X-Durable-Workflow-Payload-SHA256' => $reference['sha256'], + ])->get('/api/external-payloads/v1/'.$reference['reference_id'])->assertOk(); + $this->assertSame($payload, $fetch->streamedContent()); + $legacy = array_diff_key($context, ['checkpoint_id' => true]); + $legacy['schema'] = RuntimePayloadCompletionContext::SCHEMA; + $legacy['operation'] = 'complete'; + $upload($legacy)->assertCreated(); + $this->assertSame(1, RuntimePayloadCompletionBudget::query()->count()); + $this->openScope($task, ['sequence' => 2])->assertOk()->assertJsonPath('opened', true); + $claim = WorkflowTask::query()->findOrFail($task['task_id']); + $claim->forceFill(['lease_owner' => 'replacement', 'attempt_count' => $task['workflow_task_attempt'] + 1])->save(); + $context['checkpoint_id'] = 'new-prefix'; + $upload($context)->assertStatus(409); + } finally { + $this->removeStoragePressure(); + File::deleteDirectory($directory); + } + } + + public function test_scope_opening_records_once_and_preserves_the_live_claim(): void + { + $task = $this->claim(); + $claim = WorkflowTask::query()->findOrFail($task['task_id']); + $originalLease = $claim->lease_expires_at->toISOString(); + $before = WorkflowHistoryEvent::query()->count(); + $response = $this->openScope($task)->assertOk()->assertJsonPath('opened', true) + ->assertJsonPath('duplicate', false)->assertJsonPath('claim_released', false) + ->assertJsonPath('parent_scope_id', 'root')->assertJsonPath('shield_parent', false) + ->assertJsonPath('created_task_ids', []); + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/CancellationScopeOpeningResponse/allOf/1', json_decode($response->getContent(), flags: JSON_THROW_ON_ERROR)); + $opened = $response->json(); + $this->assertSame($before + 1, WorkflowHistoryEvent::query()->count()); + $this->openScope($task)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('history_event_id', $opened['history_event_id'])->assertJsonPath('scope_id', $opened['scope_id']); + $this->assertSame($before + 1, WorkflowHistoryEvent::query()->count()); + $this->assertSame('leased', $claim->refresh()->status->value); + $this->assertSame($originalLease, $claim->lease_expires_at->toISOString()); + $this->complete($task, $opened['scope_id'], 'start_timer')->assertOk()->assertJsonPath('recorded', true); + $this->assertSame($opened['scope_id'], WorkflowHistoryEvent::query()->where('event_type', 'TimerScheduled')->sole()->payload['cancellation_scope_id']); + } + + public function test_scope_opening_refresh_cursor_replays_canonical_history_on_only_the_live_claim(): void + { + $task = $this->claim(capabilities: [CooperativeCancellationPolicy::CAPABILITY]); + $claim = WorkflowTask::query()->findOrFail($task['task_id']); + $originalLease = $claim->lease_expires_at->toISOString(); + $opened = $this->openScope($task)->assertOk() + ->assertJsonPath('lease_owner', $task['lease_owner']) + ->assertJsonPath('workflow_task_attempt', $task['workflow_task_attempt'])->json(); + $this->assertIsString($opened['history_refresh_page_token']); + $read = function (array $authority, string $token, string $namespace = 'default') use ($task) { + return $this->withHeaders(['X-Namespace' => $namespace, WorkerProtocol::HEADER => '1.20']) + ->postJson("/api/worker/workflow-tasks/{$task['task_id']}/history", [ + 'lease_owner' => $authority['lease_owner'], 'workflow_task_attempt' => $authority['workflow_task_attempt'], + 'next_history_page_token' => $token, 'history_page_size' => 1, + ]); + }; + $token = $opened['history_refresh_page_token']; + $events = []; + $seen = []; + do { + $this->assertArrayNotHasKey($token, $seen); + $seen[$token] = true; + $page = $read($task, $token)->assertOk()->json(); + $events = [...$events, ...$page['history_events']]; + $token = $page['next_history_page_token']; + } while ($token !== null); + $this->assertSame('StartAccepted', $events[0]['event_type']); + $this->assertContains('WorkflowStarted', array_column($events, 'event_type')); + $scope = array_values(array_filter($events, fn (array $event): bool => $event['event_type'] === 'CancellationScopeOpened')); + $this->assertCount(1, $scope); + $this->assertSame($opened['history_event_id'], $scope[0]['id']); + $this->assertSame($opened['scope_id'], $scope[0]['payload']['scope_id']); + $this->assertSame($originalLease, $claim->refresh()->lease_expires_at->toISOString()); + $this->assertSame('leased', $claim->status->value); + + WorkflowNamespace::query()->create(['name' => 'other', 'retention_days' => 30, 'status' => 'active']); + $read($task, $opened['history_refresh_page_token'], 'other')->assertNotFound(); + $read(array_replace($task, ['lease_owner' => 'other-worker']), $opened['history_refresh_page_token']) + ->assertStatus(409)->assertJsonPath('reason', 'lease_owner_mismatch'); + $claim->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $replacement = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => $task['lease_owner'], 'task_queue' => 'scope-admission', + ])->assertOk()->json('task'); + $read($task, $opened['history_refresh_page_token'])->assertStatus(409) + ->assertJsonPath('reason', 'workflow_task_attempt_mismatch'); + $replayed = $this->openScope($replacement)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('scope_id', $opened['scope_id'])->assertJsonPath('history_event_id', $opened['history_event_id']) + ->assertJsonPath('workflow_task_attempt', $replacement['workflow_task_attempt'])->json(); + $read($replacement, $replayed['history_refresh_page_token'])->assertOk(); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeOpened')->count()); + } + + public function test_nested_shield_is_recorded_and_changed_replay_is_refused(): void + { + $task = $this->claim(); + $parent = $this->openScope($task)->assertOk()->json('scope_id'); + $nested = $this->openScope($task, ['sequence' => 2, 'parent_scope_id' => $parent, 'shield_parent' => true]) + ->assertOk()->assertJsonPath('parent_scope_id', $parent)->assertJsonPath('shield_parent', true)->json('scope_id'); + $this->assertNotSame($parent, $nested); + $before = WorkflowHistoryEvent::query()->count(); + $this->openScope($task, ['sequence' => 2, 'parent_scope_id' => $parent, 'shield_parent' => false]) + ->assertStatus(409)->assertJsonPath('reason', 'cancellation_scope_replay_mismatch'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_replacement_claim_reuses_the_scope_boundary_and_fences_the_old_attempt(): void + { + $task = $this->claim(); + $opened = $this->openScope($task)->assertOk()->json(); + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $replacement = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => $task['lease_owner'], 'task_queue' => 'scope-admission', + ])->assertOk()->json('task'); + $this->assertSame($task['task_id'], $replacement['task_id']); + $this->assertSame($task['workflow_task_attempt'] + 1, $replacement['workflow_task_attempt']); + $before = WorkflowHistoryEvent::query()->count(); + $this->openScope($replacement)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('history_event_id', $opened['history_event_id'])->assertJsonPath('scope_id', $opened['scope_id']); + $this->openScope($task)->assertStatus(409)->assertJsonPath('reason', 'workflow_task_attempt_mismatch'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public static function refusedScopeOpenings(): array + { + return [ + 'wrong owner' => [['lease_owner' => 'other-worker'], 'lease_owner_mismatch'], + 'wrong attempt' => [['workflow_task_attempt' => 99], 'workflow_task_attempt_mismatch'], + 'unknown parent' => [['parent_scope_id' => 'foreign-scope'], 'cancellation_scope_parent_not_recorded'], + 'future sequence' => [['sequence' => 2], 'cancellation_scope_sequence_mismatch'], + ]; + } + + #[DataProvider('refusedScopeOpenings')] + public function test_invalid_scope_opening_cannot_mutate_history(array $overrides, string $reason): void + { + $task = $this->claim(); + $before = WorkflowHistoryEvent::query()->count(); + $this->openScope($task, $overrides)->assertStatus(409)->assertJsonPath('opened', false)->assertJsonPath('reason', $reason); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_foreign_namespace_scope_opening_does_not_reveal_or_mutate_the_task(): void + { + $task = $this->claim(); + WorkflowNamespace::query()->create(['name' => 'other', 'retention_days' => 30, 'status' => 'active']); + $before = WorkflowHistoryEvent::query()->count(); + $this->openScope($task, namespace: 'other')->assertNotFound()->assertJsonPath('reason', 'task_not_found'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_stale_worker_cannot_open_a_scope_with_a_live_task_lease(): void + { + $task = $this->claim(); + WorkerRegistration::query()->where('worker_id', $task['lease_owner'])->update(['status' => 'inactive']); + $before = WorkflowHistoryEvent::query()->count(); + $this->openScope($task)->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_expired_claim_cannot_open_or_replay_a_scope(): void + { + $task = $this->claim(); + $this->openScope($task)->assertOk(); + $before = WorkflowHistoryEvent::query()->count(); + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $this->openScope($task)->assertStatus(409)->assertJsonPath('reason', 'cancellation_scope_workflow_claim_mismatch'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_incompatible_issued_claim_cannot_be_upgraded_by_current_registration(): void + { + $task = $this->claim(); + $claim = WorkflowTask::query()->findOrFail($task['task_id']); + $payload = $claim->payload; + $payload['_server_workflow_claim']['capabilities'] = []; + $claim->forceFill(['payload' => $payload])->save(); + $before = WorkflowHistoryEvent::query()->count(); + $this->openScope($task)->assertStatus(409)->assertJsonPath('reason', 'active_claim_cancellation_not_supported'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_scope_opening_rejects_legacy_protocol_and_missing_optional_backend(): void + { + $task = $this->claim(); + $this->openScope($task, version: '1.19')->assertStatus(409)->assertJsonPath('reason', 'cancellation_scope_requires_protocol_1_20'); + $this->assertInstanceOf(CancellationScopeTaskBridge::class, app(WorkflowTaskBridge::class)); + $this->app->instance(WorkflowTaskBridge::class, \Mockery::mock(WorkflowTaskBridge::class)); + $this->openScope($task)->assertStatus(409)->assertJsonPath('reason', 'cancellation_scope_opening_unavailable') + ->assertJsonPath('unavailable', ['installed_runtime_scope_opening']); + } + + public function test_scope_opening_requires_authenticated_worker_authority(): void + { + $task = $this->claim(); + config(['server.auth.driver' => 'token', 'server.auth.token' => 'scope-test-token']); + $before = WorkflowHistoryEvent::query()->count(); + $this->openScope($task)->assertUnauthorized(); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->withHeaders(['Authorization' => 'Bearer scope-test-token']); + $this->openScope($task)->assertOk()->assertJsonPath('opened', true); + } + + public static function malformedScopeOpenings(): array + { + return [ + [['sequence' => 0]], [['workflow_task_attempt' => null]], [['lease_owner' => '']], + [['parent_scope_id' => '']], [['parent_scope_id' => str_repeat('x', 256)]], [['shield_parent' => 'invalid']], + ]; + } + + #[DataProvider('malformedScopeOpenings')] + public function test_malformed_scope_opening_is_rejected_before_history(array $overrides): void + { + $task = $this->claim(); + $before = WorkflowHistoryEvent::query()->count(); + $this->openScope($task, $overrides)->assertStatus(422); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + private function openScope(array $task, array $overrides = [], string $version = '1.20', string $namespace = 'default') + { + return $this->withHeaders(['X-Namespace' => $namespace, WorkerProtocol::HEADER => $version]) + ->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/open", array_replace([ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], 'sequence' => 1, + ], $overrides)); + } + + public static function operations(): array + { + return [['schedule_activity'], ['start_timer'], ['start_child_workflow'], ['open_signal_wait'], ['open_condition_wait']]; + } + + #[DataProvider('operations')] + public function test_unknown_scope_is_refused_without_publishing_a_stream_item(string $type): void + { + $task = $this->claim(); + $before = WorkflowHistoryEvent::query()->count(); + $this->complete($task, 'unknown-scope', $type)->assertStatus(409) + ->assertJsonPath('recorded', false)->assertJsonPath('reason', 'operation_scope_not_recorded'); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + $this->assertDatabaseCount('workflow_durable_streams', 0); + $this->assertDatabaseCount('activity_executions', 0); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + #[DataProvider('operations')] + public function test_recorded_scope_reaches_durable_history_and_commits_the_stream(string $type): void + { + $task = $this->claim(); + $run = WorkflowRun::query()->findOrFail($task['run_id']); + $scope = CancellationScopeHistory::open($run, WorkflowTask::query()->findOrFail($task['task_id']), 1, '1.20'); + $scopeId = $scope->payload['scope_id']; + $this->complete($task, $scopeId, $type)->assertOk()->assertJsonPath('recorded', true); + $this->assertDatabaseCount('workflow_durable_stream_items', 1); + $eventType = match ($type) { + 'schedule_activity' => 'ActivityScheduled', 'start_timer' => 'TimerScheduled', 'start_child_workflow' => 'ChildWorkflowScheduled', + 'open_signal_wait' => 'SignalWaitOpened', 'open_condition_wait' => 'ConditionWaitOpened', + }; + $event = WorkflowHistoryEvent::query()->where('workflow_run_id', $run->id)->where('event_type', $eventType)->sole(); + $this->assertSame($scopeId, $type === 'schedule_activity' ? $event->payload['activity']['cancellation_scope_id'] : $event->payload['cancellation_scope_id']); + } + + public function test_invalid_scope_is_refused_before_resolving_a_payload(): void + { + $task = $this->claim(); + $this->complete($task, 'unknown-scope', arguments: ['codec' => 'avro', 'invalid_reference' => true]) + ->assertStatus(409)->assertJsonPath('reason', 'operation_scope_not_recorded'); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + $this->assertDatabaseCount('activity_executions', 0); + } + + public function test_a_runtime_without_the_optional_admission_role_cannot_silently_drop_membership(): void + { + $task = $this->claim(); + $current = app(WorkflowTaskBridge::class); + $older = \Mockery::mock(WorkflowTaskBridge::class); + $older->shouldReceive('status')->andReturnUsing(fn (string $id) => $current->status($id)); + $older->shouldNotReceive('complete'); + $this->app->instance(WorkflowTaskBridge::class, $older); + $this->complete($task, 'root')->assertStatus(409)->assertJsonPath('reason', 'cancellation_scope_membership_unavailable') + ->assertJsonPath('unavailable', ['installed_runtime_scope_admission']); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + } + + public function test_a_legacy_request_cannot_silently_drop_membership(): void + { + $task = $this->claim(); + $this->complete($task, 'root', version: '1.19')->assertStatus(409) + ->assertJsonPath('reason', 'cancellation_scope_membership_unavailable')->assertJsonPath('unavailable', ['request_protocol']); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + } + + public static function malformedScopes(): array + { + return [[null], [''], [true], [str_repeat('x', 256)]]; + } + + #[DataProvider('malformedScopes')] + public function test_malformed_membership_is_refused_before_effects(mixed $scope): void + { + $task = $this->claim(); + $this->complete($task, $scope)->assertStatus(422); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + $this->assertDatabaseCount('activity_executions', 0); + } + + public function test_scope_on_an_unrelated_command_is_refused_before_effects(): void + { + $task = $this->claim(); + $this->complete($task, 'root', 'record_side_effect')->assertStatus(409)->assertJsonPath('reason', 'invalid_cancellation_scope_command'); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + } + + public function test_admission_is_rechecked_under_the_run_lock_before_stream_effects(): void + { + $task = $this->claim(); + $current = app(WorkflowTaskBridge::class); + $bridge = \Mockery::mock(WorkflowTaskBridge::class.', '.CancellationScopeAdmission::class); + $bridge->shouldReceive('status')->andReturnUsing(fn (string $id) => $current->status($id)); + $bridge->shouldReceive('validateCancellationScopeMembership')->twice()->andReturn(null, 'operation_scope_not_recorded'); + $bridge->shouldNotReceive('complete'); + $this->app->instance(WorkflowTaskBridge::class, $bridge); + $this->complete($task, 'root')->assertStatus(409)->assertJsonPath('reason', 'operation_scope_not_recorded'); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + $this->assertDatabaseCount('activity_executions', 0); + } + + public static function groupScopes(): array + { + return [[true], [false]]; + } + + #[DataProvider('groupScopes')] + public function test_retained_prefix_rejects_unknown_scope_or_commits_stream_once(bool $valid): void + { + $task = $this->claim(); + $run = WorkflowRun::query()->findOrFail($task['run_id']); + $scope = CancellationScopeHistory::open($run, WorkflowTask::query()->findOrFail($task['task_id']), 1, '1.20')->payload['scope_id']; + $body = ['lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'checkpoint_id' => 'scope-prefix', 'start_sequence' => 2, + 'commands' => $this->commandBatch($task, $valid ? $scope : 'unknown-scope', 'start_timer')]; + $route = "/api/worker/workflow-tasks/{$task['task_id']}/local-activities/checkpoint"; + $response = $this->withHeaders($this->headers())->postJson($route, $body); + if ($valid) { + $response->assertOk()->assertJsonPath('checkpointed', true); + $before = WorkflowHistoryEvent::query()->count(); + $this->withHeaders($this->headers())->postJson($route, $body)->assertOk()->assertJsonPath('duplicate', true); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseCount('workflow_durable_stream_items', 1); + } else { + $response->assertStatus(409)->assertJsonPath('reason', 'operation_scope_not_recorded'); + $this->assertDatabaseCount('workflow_durable_stream_items', 0); + $this->assertDatabaseCount('workflow_run_timers', 0); + } + } + + #[DataProvider('groupScopes')] + public function test_atomic_group_preflights_local_membership_before_any_sibling(bool $valid): void + { + $task = $this->claim(); + $run = WorkflowRun::query()->findOrFail($task['run_id']); + $scope = CancellationScopeHistory::open($run, WorkflowTask::query()->findOrFail($task['task_id']), 1, '1.20')->payload['scope_id']; + $before = WorkflowHistoryEvent::query()->count(); + $response = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/checkpoint-group", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'checkpoint_id' => 'scope-group', 'start_sequence' => 2, 'commands' => [[ + 'type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', + 'arguments' => Serializer::serializeWithCodec('avro', ['child']), 'payload_codec' => 'avro', 'cancellation_scope_id' => $scope, + ...ParallelChildGroup::itemMetadata(2, 2, 0, 'mixed'), + ], [ + 'type' => 'prepare_local_activity', 'activity_type' => 'opaque-local', 'payload_codec' => 'avro', + 'arguments' => $valid ? Serializer::serializeWithCodec('avro', ['Ada']) : ['codec' => 'avro', 'invalid_reference' => true], + 'cancellation_scope_id' => $valid ? $scope : 'unknown-scope', + ...ParallelChildGroup::itemMetadata(2, 2, 1, 'mixed'), + ]], + ]); + if ($valid) { + $response->assertOk()->assertJsonPath('checkpointed', true); + $this->assertSame($scope, ActivityExecution::query()->sole()->activity_options['cancellation_scope_id']); + } else { + $response->assertStatus(409)->assertJsonPath('reason', 'local_activity_scope_not_recorded'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseCount('activity_executions', 0); + $this->assertDatabaseCount('workflow_child_calls', 0); + } + } + + public static function singleLocalOperations(): array + { + return [['prepare'], ['recover']]; + } + + #[DataProvider('singleLocalOperations')] + public function test_single_local_rejects_unknown_scope_before_payload_resolution(string $operation): void + { + $task = $this->claim(); + $storage = \Mockery::mock(NamespaceExternalPayloadStorage::class); + $storage->shouldNotReceive('driverFor'); + $this->app->instance(NamespaceExternalPayloadStorage::class, $storage); + $before = WorkflowHistoryEvent::query()->count(); + $this->singleLocal($task, $operation, 'unknown-scope', arguments: ['codec' => 'avro', 'invalid_reference' => true]) + ->assertConflict()->assertJsonPath('recorded', false)->assertJsonPath('reason', 'local_activity_scope_not_recorded'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseCount('activity_executions', 0); + $this->assertDatabaseCount('activity_attempts', 0); + } + + public static function malformedSingleLocalScopes(): array + { + $cases = []; + foreach (self::singleLocalOperations() as [$operation]) { + foreach (self::malformedScopes() as [$scope]) { + $cases[] = [$operation, $scope]; + } + } + + return $cases; + } + + #[DataProvider('malformedSingleLocalScopes')] + public function test_single_local_validates_scope_syntax_before_payload_resolution(string $operation, mixed $scope): void + { + $task = $this->claim(); + $this->singleLocal($task, $operation, $scope, arguments: ['codec' => 'avro', 'invalid_reference' => true]) + ->assertUnprocessable()->assertJsonValidationErrors('descriptor.cancellation_scope_id'); + $this->assertDatabaseCount('activity_executions', 0); + $this->assertDatabaseCount('activity_attempts', 0); + } + + #[DataProvider('singleLocalOperations')] + public function test_single_local_rejects_a_legacy_request_before_payload_resolution(string $operation): void + { + $task = $this->claim(); + $this->singleLocal($task, $operation, 'root', arguments: ['codec' => 'avro', 'invalid_reference' => true], version: '1.19') + ->assertConflict()->assertJsonPath('reason', 'prepared_local_activity_not_supported') + ->assertJsonPath('unavailable', ['request_protocol']); + $this->assertDatabaseCount('activity_executions', 0); + } + + #[DataProvider('singleLocalOperations')] + public function test_single_local_rejects_a_backend_without_scope_admission_before_payload_resolution(string $operation): void + { + $task = $this->claim(); + $older = \Mockery::mock(PreparedLocalActivityTaskBridge::class); + $older->shouldNotReceive('prepareLocalActivity'); + $older->shouldNotReceive('recoverLocalActivity'); + $this->app->instance(WorkflowTaskBridge::class, $older); + $this->singleLocal($task, $operation, 'root', arguments: ['codec' => 'avro', 'invalid_reference' => true]) + ->assertConflict()->assertJsonPath('reason', 'cancellation_scope_membership_unavailable') + ->assertJsonPath('unavailable', ['installed_runtime_scope_admission']); + $this->assertDatabaseCount('activity_executions', 0); + } + + #[DataProvider('singleLocalOperations')] + public function test_single_local_cannot_use_a_scope_before_its_recorded_sequence(string $operation): void + { + $task = $this->claim(); + $scope = CancellationScopeHistory::open(WorkflowRun::query()->findOrFail($task['run_id']), + WorkflowTask::query()->findOrFail($task['task_id']), 1, '1.20')->payload['scope_id']; + $this->singleLocal($task, $operation, $scope, sequence: 1, arguments: ['codec' => 'avro', 'invalid_reference' => true]) + ->assertConflict()->assertJsonPath('reason', 'local_activity_scope_not_recorded'); + $this->assertDatabaseCount('activity_executions', 0); + } + + #[DataProvider('singleLocalOperations')] + public function test_single_local_cannot_use_a_scope_from_another_run(string $operation): void + { + $other = $this->claim(); + $scope = CancellationScopeHistory::open(WorkflowRun::query()->findOrFail($other['run_id']), + WorkflowTask::query()->findOrFail($other['task_id']), 1, '1.20')->payload['scope_id']; + $task = $this->claim('other-scope-worker'); + $this->singleLocal($task, $operation, $scope, arguments: ['codec' => 'avro', 'invalid_reference' => true]) + ->assertConflict()->assertJsonPath('reason', 'local_activity_scope_not_recorded'); + $this->assertDatabaseCount('activity_executions', 0); + } + + public function test_single_local_records_scope_and_recovers_once_after_claim_replacement(): void + { + $task = $this->claim(); + $scope = CancellationScopeHistory::open(WorkflowRun::query()->findOrFail($task['run_id']), + WorkflowTask::query()->findOrFail($task['task_id']), 1, '1.20')->payload['scope_id']; + $first = $this->singleLocal($task, 'prepare', $scope)->assertOk()->assertJsonPath('prepared', true)->json(); + $before = WorkflowHistoryEvent::query()->count(); + $this->singleLocal($task, 'prepare', $scope)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('activity_attempt_id', $first['activity_attempt_id']); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertSame($scope, ActivityExecution::query()->sole()->activity_options['cancellation_scope_id']); + ActivityAttempt::query()->findOrFail($first['activity_attempt_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $replacement = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'scope-worker', 'task_queue' => 'scope-admission', + ])->assertOk()->json('task'); + $this->assertSame($task['task_id'], $replacement['task_id']); + $this->assertSame($task['workflow_task_attempt'] + 1, $replacement['workflow_task_attempt']); + $recovered = $this->singleLocal($replacement, 'recover', $scope)->assertOk()->assertJsonPath('recovered', true) + ->assertJsonPath('claim_released', true)->assertJsonPath('callback_stop_state', 'unknown')->json(); + $before = WorkflowHistoryEvent::query()->count(); + $this->singleLocal($replacement, 'recover', $scope)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('event_id', $recovered['event_id']); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertSame($scope, ActivityExecution::query()->sole()->activity_options['cancellation_scope_id']); + } + + public function test_single_local_omitted_scope_keeps_preparation_compatible(): void + { + $task = $this->claim(); + $this->singleLocal($task, 'prepare', sequence: 1, includeScope: false)->assertOk()->assertJsonPath('prepared', true); + $this->assertArrayNotHasKey('cancellation_scope_id', ActivityExecution::query()->sole()->activity_options); + } + + private function singleLocal(array $task, string $operation, mixed $scope = null, int $sequence = 2, mixed $arguments = null, string $version = '1.20', bool $includeScope = true) + { + return $this->withHeaders(['X-Namespace' => 'default', WorkerProtocol::HEADER => $version]) + ->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/{$operation}", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'worker_attempt_id' => 'scope-local-attempt', 'sequence' => $sequence, + 'descriptor' => ['type' => 'record_local_activity', 'activity_type' => 'opaque-local', 'payload_codec' => 'avro', + 'arguments' => $arguments ?? Serializer::serializeWithCodec('avro', ['Ada']), + 'retry_policy' => ['max_attempts' => 2, 'backoff_seconds' => [2]], + ...($includeScope ? ['cancellation_scope_id' => $scope] : [])], + ]); + } + + private function claim(string $worker = 'scope-worker', ?array $capabilities = null): array + { + $this->withHeaders(['X-Namespace' => 'default', 'X-Durable-Workflow-Control-Plane-Version' => '2']) + ->postJson('/api/workflows', ['workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'scope-admission', 'input' => ['Ada']])->assertCreated(); + $this->withHeaders($this->headers())->postJson('/api/worker/register', [ + 'worker_id' => $worker, 'task_queue' => 'scope-admission', 'runtime' => 'php', + 'supported_workflow_types' => ['tests.external-greeting-workflow'], + 'capabilities' => $capabilities ?? [CooperativeCancellationPolicy::CAPABILITY, PreparedLocalActivityPolicy::CAPABILITY, PreparedLocalActivityPolicy::GROUP_CAPABILITY], + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), 'max_concurrent_workflow_tasks' => 1, + 'process_metrics' => ['host' => 'test-worker', 'process_started_at' => now()->toISOString(), 'process_id' => 10], + ])->assertCreated(); + + return $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => $worker, 'task_queue' => 'scope-admission', + ])->assertOk()->json('task'); + } + + private function complete(array $task, mixed $scopeId, string $type = 'schedule_activity', mixed $arguments = null, string $version = '1.20') + { + return $this->withHeaders(['X-Namespace' => 'default', WorkerProtocol::HEADER => $version])->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => $this->commandBatch($task, $scopeId, $type, $arguments), + ]); + } + + private function commandBatch(array $task, mixed $scopeId, string $type, mixed $arguments = null): array + { + $identity = $task['workflow_command_id'] ?: $task['task_id']; + $operation = match ($type) { + 'schedule_activity' => ['activity_type' => 'opaque-activity', 'arguments' => $arguments ?? Serializer::serializeWithCodec('avro', ['Ada']), 'payload_codec' => 'avro'], + 'start_timer' => ['delay_seconds' => 60], + 'start_child_workflow' => ['workflow_type' => 'tests.external-greeting-workflow', 'arguments' => Serializer::serializeWithCodec('avro', ['child']), 'payload_codec' => 'avro'], + 'open_signal_wait' => ['signal_name' => 'ready', 'timeout_seconds' => 5], + 'open_condition_wait' => ['condition_key' => 'ready', 'timeout_seconds' => 5], + 'record_side_effect' => ['result' => Serializer::serializeWithCodec('avro', null)], + }; + + return [[ + 'type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', null), + 'workflow_stream' => ['operation' => 'append', 'stream_name' => 'tokens', 'command_identity' => $identity, + 'command_ordinal' => 0, 'items' => [['payload' => Serializer::serializeWithCodec('avro', ['token']), + 'payload_codec' => 'avro', 'idempotency_key' => "dw-stream:{$identity}:0:0"]]], + ], [ + 'type' => $type, ...$operation, 'cancellation_scope_id' => $scopeId, + ]]; + } + + private function headers(): array + { + return ['X-Namespace' => 'default', WorkerProtocol::HEADER => '1.20']; + } +} diff --git a/tests/Source/CancellationScopeDeliveryTest.php b/tests/Source/CancellationScopeDeliveryTest.php new file mode 100644 index 00000000..4241c85c --- /dev/null +++ b/tests/Source/CancellationScopeDeliveryTest.php @@ -0,0 +1,788 @@ + '1.20', + 'server.auth.driver' => 'token', + 'server.auth.role_tokens' => ['operator' => 'fixture-operator', 'worker' => 'fixture-worker'], + 'workflows.v2.types.workflows' => ['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class]]); + WorkflowNamespace::query()->create(['name' => 'default', 'retention_days' => 30, 'status' => 'active']); + } + + public function test_preparation_and_delivery_reuse_first_boundary_with_recoverable_original_claim(): void + { + [$task, $scope, $request] = $this->claim(); + $before = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + $prepared = $this->mutate('prepare', $task, $scope, $request)->assertOk() + ->assertJsonPath('prepared', true)->assertJsonPath('delivered', false) + ->assertJsonPath('claim_released', false)->assertJsonPath('activity_members', [])->json(); + $this->assertContract($prepared); + $this->assertRecoverableHostingClaim($before, $task['task_id']); + $this->mutate('prepare', $task, $scope, $request)->assertOk() + ->assertJsonPath('preparation_history_event_id', $prepared['preparation_history_event_id']) + ->assertJsonPath('cancellation', $prepared['cancellation']) + ->assertJsonPath('authority_deadline_at', $prepared['authority_deadline_at']); + $this->assertRecoverableHostingClaim($before, $task['task_id']); + $delivery = $this->mutate('deliver', $task, $scope, $request)->assertOk()->assertJsonPath('delivered', true)->json(); + $this->assertContract($delivery); + $this->mutate('deliver', $task, $scope, $request)->assertOk()->assertJsonPath('history_event_id', $delivery['history_event_id']); + $this->assertRecoverableHostingClaim($before, $task['task_id']); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDeliveryPrepared')->count()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDelivered')->count()); + $this->assertSame($prepared['preparation_history_event_id'], $delivery['preparation_history_event_id']); + } + + public static function phases(): array + { + return [['prepare'], ['deliver']]; + } + + public function test_accepted_unprepared_request_recovers_through_watchdog_and_http_poll(): void + { + [$task, $scope, $request] = $this->claim(); + $accepted = WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeRequested')->sole(); + $before = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + $historyBefore = WorkflowHistoryEvent::query()->count(); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDeliveryPrepared')->count()); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDelivered')->count()); + $this->travel(11)->seconds(); + $this->mutate('prepare', $task, $scope, $request)->assertConflict() + ->assertJsonPath('reason', 'cancellation_scope_workflow_claim_mismatch'); + $this->assertSame($historyBefore, WorkflowHistoryEvent::query()->count()); + $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $repair = TaskWatchdog::runPass(respectThrottle: false, runIds: [$task['run_id']]); + $this->assertSame([], $repair['existing_task_failures']); + $this->assertSame(1, $repair['repaired_existing_tasks']); + $replacement = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'delivery-owner', 'task_queue' => 'scoped-delivery', + ])->assertOk()->json('task'); + $this->assertSame($task['task_id'], $replacement['task_id']); + $this->assertSame($task['workflow_task_attempt'] + 1, $replacement['workflow_task_attempt']); + $run = WorkflowRun::query()->findOrFail($task['run_id']); + $replacementBefore = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + $duplicate = CancellationScopeRequests::request($run, $scope, '1.20', 300); + $this->assertSame($accepted->id, $duplicate->id); + $this->assertEquals($accepted->payload, $duplicate->payload); + $this->assertSame($replacementBefore, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $this->mutate('prepare', $task, $scope, $request)->assertConflict() + ->assertJsonPath('reason', 'workflow_task_attempt_mismatch'); + $prepared = $this->mutate('prepare', $replacement, $scope, $request)->assertOk()->json(); + $this->assertContract($prepared); + $this->assertSame($request, $prepared['request_id']); + $this->assertSame($accepted->payload['cancellation']['root_context']['cleanup_deadline_at'], $prepared['authority_deadline_at']); + $this->assertNull($run->fresh()->cancellation_request_command_id); + $this->assertRecoverableHostingClaim($replacementBefore, $task['task_id']); + $this->mutate('deliver', $replacement, $scope, $request)->assertOk() + ->assertJsonPath('delivered', true)->assertJsonPath('preparation_history_event_id', $prepared['preparation_history_event_id']); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeRequested')->count()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDeliveryPrepared')->count()); + } + + #[DataProvider('phases')] + public function test_invalid_authority_and_namespace_are_refused_without_effects(string $phase): void + { + [$task, $scope, $request] = $this->claim(); + WorkflowNamespace::query()->create(['name' => 'other', 'retention_days' => 30, 'status' => 'active']); + $before = WorkflowHistoryEvent::query()->count(); + $original = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + $this->mutate($phase, $task, $scope, $request, ['lease_owner' => 'other'])->assertConflict()->assertJsonPath('reason', 'lease_owner_mismatch'); + $this->mutate($phase, $task, $scope, $request, ['workflow_task_attempt' => 99])->assertConflict()->assertJsonPath('reason', 'workflow_task_attempt_mismatch'); + $this->mutate($phase, $task, $scope, $request, namespace: 'other')->assertNotFound()->assertJsonPath('reason', 'task_not_found'); + $this->mutate($phase, $task, $scope, $request, role: 'operator')->assertForbidden(); + $this->mutate($phase, $task, $scope, $request, version: '1.19')->assertConflict()->assertJsonPath('reason', 'cancellation_scope_requires_protocol_1_20'); + $this->mutate($phase, $task, $scope, $request, version: '1.21')->assertBadRequest(); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertSame($original, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + } + + #[DataProvider('phases')] + public function test_claim_capability_and_fresh_registration_are_required(string $phase): void + { + [$task, $scope, $request] = $this->claim(); + $before = WorkflowHistoryEvent::query()->count(); + $claim = WorkflowTask::query()->findOrFail($task['task_id']); + $payload = $claim->payload; + $claim->forceFill(['payload' => []])->save(); + $this->mutate($phase, $task, $scope, $request)->assertConflict()->assertJsonPath('reason', 'active_claim_cancellation_not_supported'); + $claim->forceFill(['payload' => $payload])->save(); + WorkerRegistration::query()->where('worker_id', 'delivery-owner')->update(['last_heartbeat_at' => now()->subHour()]); + $this->mutate($phase, $task, $scope, $request)->assertConflict()->assertJsonPath('reason', 'stale_worker_registration'); + WorkerRegistration::query()->where('worker_id', 'delivery-owner')->update(['last_heartbeat_at' => now()]); + $claim->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $this->mutate($phase, $task, $scope, $request)->assertConflict()->assertJsonPath('reason', 'cancellation_scope_workflow_claim_mismatch'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + #[DataProvider('phases')] + public function test_missing_optional_backend_is_explicit(string $phase): void + { + [$task, $scope, $request] = $this->claim(); + $before = WorkflowHistoryEvent::query()->count(); + $this->app->instance(WorkflowTaskBridge::class, \Mockery::mock(WorkflowTaskBridge::class)); + $this->mutate($phase, $task, $scope, $request)->assertConflict()->assertJsonPath('reason', 'cancellation_scope_delivery_unavailable') + ->assertJsonPath('unavailable', ['installed_runtime_scope_preparation_delivery']); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_delivery_without_preparation_and_changed_retry_are_refused(): void + { + [$task, $scope, $request] = $this->claim(); + $this->mutate('deliver', $task, $scope, $request)->assertConflict()->assertJsonPath('reason', 'cancellation_scope_delivery_not_prepared'); + $prepared = $this->mutate('prepare', $task, $scope, $request)->assertOk()->json(); + $before = WorkflowHistoryEvent::query()->count(); + $this->mutate('prepare', $task, $scope, $request, ['sequence' => 3])->assertConflict(); + $this->mutate('deliver', $task, $scope, $request, ['sequence' => 3])->assertConflict(); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->mutate('prepare', $task, $scope, $request)->assertOk()->assertJsonPath('preparation_history_event_id', $prepared['preparation_history_event_id']); + } + + public function test_preparation_and_delivery_remain_available_during_storage_drain(): void + { + [$task, $scope, $request] = $this->claim(); + $this->configureStoragePressure('draining'); + try { + $this->mutate('prepare', $task, $scope, $request)->assertOk()->assertJsonPath('prepared', true); + $this->mutate('deliver', $task, $scope, $request)->assertOk()->assertJsonPath('delivered', true); + } finally { + $this->removeStoragePressure(); + } + } + + public static function malformed(): array + { + return [[['scope_id' => '']], [['request_id' => null]], [['sequence' => 0]], [['call_kind' => 'unknown']], + [['sequence_span' => 0]], [['operation_sequence' => 0]], [['workflow_task_attempt' => 0]]]; + } + + #[DataProvider('malformed')] + public function test_malformed_boundary_is_rejected_before_history(array $overrides): void + { + [$task, $scope, $request] = $this->claim(); + $before = WorkflowHistoryEvent::query()->count(); + foreach (['prepare', 'deliver'] as $phase) { + $this->mutate($phase, $task, $scope, $request, $overrides)->assertUnprocessable(); + } + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + #[DataProvider('phases')] + public function test_claim_replaced_after_server_checks_is_refused_by_native_before_commit(string $phase): void + { + [$task, $scope, $request] = $this->claim(); + if ($phase === 'deliver') { + $this->mutate('prepare', $task, $scope, $request)->assertOk(); + } + $before = WorkflowHistoryEvent::query()->count(); + $native = app(WorkflowTaskBridge::class); + $method = $phase === 'prepare' ? 'prepareCancellationScopeDelivery' : 'deliverCancellationScope'; + $bridge = \Mockery::mock(WorkflowTaskBridge::class.', '.PreparedCancellationScopeTaskBridge::class); + $bridge->shouldReceive($method)->once()->andReturnUsing(function (...$arguments) use ($task, $native, $method): array { + $this->assertSame(0, DB::connection()->transactionLevel()); + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_owner' => 'replacement'])->save(); + + return $native->$method(...$arguments); + }); + $this->app->instance(WorkflowTaskBridge::class, $bridge); + $this->mutate($phase, $task, $scope, $request)->assertConflict()->assertJsonPath('reason', 'cancellation_scope_workflow_claim_mismatch'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_replacement_claim_reuses_preparation_and_original_deadline(): void + { + [$task, $scope, $request] = $this->claim(); + $prepared = $this->mutate('prepare', $task, $scope, $request)->assertOk()->json(); + $this->travel(11)->seconds(); + $repair = TaskWatchdog::runPass(respectThrottle: false, runIds: [$task['run_id']]); + $this->assertSame([], $repair['existing_task_failures']); + $this->assertSame(1, $repair['repaired_existing_tasks']); + $replacement = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'delivery-owner', 'task_queue' => 'scoped-delivery', + ])->assertOk()->json('task'); + $this->assertSame($task['task_id'], $replacement['task_id']); + $this->assertSame($task['workflow_task_attempt'] + 1, $replacement['workflow_task_attempt']); + $this->mutate('prepare', $task, $scope, $request)->assertConflict()->assertJsonPath('reason', 'workflow_task_attempt_mismatch'); + $this->mutate('prepare', $replacement, $scope, $request)->assertOk() + ->assertJsonPath('preparation_history_event_id', $prepared['preparation_history_event_id']) + ->assertJsonPath('cancellation', $prepared['cancellation']) + ->assertJsonPath('authority_deadline_at', $prepared['authority_deadline_at']); + $this->mutate('deliver', $replacement, $scope, $request)->assertOk()->assertJsonPath('delivered', true); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDeliveryPrepared')->count()); + } + + public function test_timer_dispatch_reuses_original_preparation_and_fences_late_jobs(): void + { + [$task, $scope, $request] = $this->claim(timer: true); + $prepared = $this->mutate('prepare', $task, $scope, $request, ['call_kind' => 'timer'])->assertOk()->json(); + $before = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + $timer = WorkflowTimer::query()->sole(); + $this->assertSame($timer->id, $prepared['timer_members'][0]['timer_id']); + $response = $this->mutate('deliver', $task, $scope, $request, ['call_kind' => 'timer'])->assertOk() + ->assertJsonPath('delivered', true)->assertJsonPath('prepared', true) + ->assertJsonPath('timer_cancellations.0.fenced', true) + ->assertJsonPath('timer_cancellations.0.timer_id', $timer->id) + ->assertJsonPath('preparation_history_event_id', $prepared['preparation_history_event_id'])->json(); + $this->assertContract($response); + $this->assertContract($prepared); + $this->assertRecoverableHostingClaim($before, $task['task_id']); + $duplicate = $this->mutate('deliver', $task, $scope, $request, ['call_kind' => 'timer'])->assertOk()->json(); + $this->assertSame($response['timer_cancellations'], $duplicate['timer_cancellations']); + $this->assertSame($response['history_event_id'], $duplicate['history_event_id']); + $this->assertSame($prepared['authority_deadline_at'], $duplicate['authority_deadline_at']); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDelivered')->count()); + $this->assertSame(TimerStatus::Cancelled, $timer->fresh()->status); + $timerTask = WorkflowTask::query()->where('task_type', TaskType::Timer)->sole(); + $timerTask->forceFill(['status' => TaskStatus::Ready, 'available_at' => now()->subSecond()])->save(); + $timer->refresh()->forceFill(['status' => TimerStatus::Pending])->save(); + $historyBefore = WorkflowHistoryEvent::query()->count(); + $this->app->call([new RunTimerTask($timerTask->id), 'handle']); + $this->assertSame($historyBefore, WorkflowHistoryEvent::query()->count()); + $this->assertSame(TaskStatus::Cancelled, $timerTask->fresh()->status); + $this->assertSame(TimerStatus::Cancelled, $timer->fresh()->status); + $this->assertSame(1, WorkflowTask::query()->where('task_type', TaskType::Workflow)->count()); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'TimerFired')->count()); + } + + #[DataProvider('waits')] + public function test_scoped_wait_delivery_retains_original_membership_and_claim(string $kind, ?int $timeout): void + { + [$task, $scope, $request] = $this->claim(); + $run = WorkflowRun::query()->findOrFail($task['run_id']); + $command = array_filter(['type' => 'open_'.$kind.'_wait', 'cancellation_scope_id' => $scope, + 'timeout_seconds' => $timeout, ...($kind === 'signal' ? ['signal_name' => 'ready'] : ['condition_key' => 'ready'])], + static fn (mixed $value): bool => $value !== null); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [$command], + ])->assertOk(); + $this->assertSame(TaskStatus::Completed, WorkflowTask::query()->findOrFail($task['task_id'])->status); + $run->tasks()->create(['namespace' => $run->namespace, 'task_type' => TaskType::Workflow, 'status' => TaskStatus::Ready, + 'available_at' => now(), 'connection' => $run->connection, 'queue' => $run->queue, 'compatibility' => $run->compatibility]); + $task = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'delivery-owner', 'task_queue' => 'scoped-delivery', + ])->assertOk()->json('task'); + $before = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + $prepared = $this->mutate('prepare', $task, $scope, $request, ['call_kind' => $kind])->assertOk()->json(); + $this->assertContract($prepared); + $this->assertCount(1, $prepared['wait_members']); + $this->assertSame($kind, $prepared['wait_members'][0]['kind']); + $this->assertSame($timeout !== null, $prepared['wait_members'][0]['timer_id'] !== null); + $response = $this->mutate('deliver', $task, $scope, $request, ['call_kind' => $kind])->assertOk() + ->assertJsonPath('delivered', true)->assertJsonPath('wait_cancellations.0.cancelled', true)->json(); + $this->assertContract($response); + $duplicate = $this->mutate('deliver', $task, $scope, $request, ['call_kind' => $kind])->assertOk()->json(); + $this->assertSame($response['wait_cancellations'], $duplicate['wait_cancellations']); + $this->assertSame($response['history_event_id'], $duplicate['history_event_id']); + $this->assertSame($prepared['wait_members'], $duplicate['wait_members']); + $this->assertSame($prepared['authority_deadline_at'], $duplicate['authority_deadline_at']); + $this->assertRecoverableHostingClaim($before, $task['task_id']); + $marker = WorkflowHistoryEvent::query()->where('event_type', $kind === 'signal' ? 'SignalWaitCancelled' : 'ConditionWaitCancelled')->sole(); + $this->assertSame($prepared['preparation_history_event_id'], $marker->payload['cancellation_scope']['preparation_history_event_id']); + if ($timeout !== null) { + $this->assertSame(TimerStatus::Cancelled, $run->timers()->sole()->status); + $this->assertSame(TaskStatus::Cancelled, $run->tasks()->where('task_type', TaskType::Timer)->sole()->status); + } + } + + public static function waits(): iterable + { + foreach (['signal', 'condition'] as $kind) { + yield $kind.':untimed' => [$kind, null]; + yield $kind.':timed' => [$kind, 5]; + } + } + + #[DataProvider('childPolicies')] + public function test_child_delivery_reconciles_original_policy_receipts_without_releasing_the_host_claim(string $policy): void + { + [$task, $scope, $request] = $this->claim(childPolicy: $policy); + $before = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + $prepared = $this->mutate('prepare', $task, $scope, $request, ['call_kind' => 'child'])->assertOk()->json(); + $this->assertContract($prepared); + $this->assertCount(1, $prepared['child_members']); + $scheduled = WorkflowHistoryEvent::query()->where('event_type', 'ChildWorkflowScheduled')->sole(); + $member = $prepared['child_members'][0]; + $this->assertSame($scheduled->payload['child_call_id'], $member['child_call_id']); + $this->assertSame($scheduled->payload['child_workflow_instance_id'], $member['child_workflow_instance_id']); + $this->assertSame($scheduled->payload['child_workflow_run_id'], $member['child_workflow_run_id']); + $this->assertSame($policy, $member['cancellation_policy']); + $this->assertSame($prepared['child_members'], $this->mutate('prepare', $task, $scope, $request, + ['call_kind' => 'child'])->assertOk()->json('child_members')); + $this->assertNull(WorkflowRun::query()->findOrFail($member['child_workflow_run_id'])->cancellation_request_command_id); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDelivered')->count()); + $response = $this->mutate('deliver', $task, $scope, $request, ['call_kind' => 'child']); + if ($policy === 'wait_cancellation_completed') { + $response->assertConflict()->assertJsonPath('delivered', false) + ->assertJsonPath('reason', 'cancellation_scope_child_completion_not_established'); + } else { + $response->assertOk()->assertJsonPath('delivered', true); + } + $delivery = $response->json(); + $this->assertContract($delivery); + $this->assertCount(1, $delivery['child_cancellations']); + $receipt = $delivery['child_cancellations'][0]; + $this->assertSame($member['child_call_id'], $receipt['child_call_id']); + $this->assertSame($member['child_workflow_run_id'], $receipt['child_workflow_run_id']); + $this->assertSame($policy, $receipt['policy']); + $target = WorkflowStub::loadRun($member['child_workflow_run_id']); + $context = CooperativeCancellationDelivery::context($target->run()->fresh()); + $this->assertSame($policy === 'abandon', $context === null); + $this->assertFalse($target->run()->fresh()->status->isTerminal()); + if ($context !== null) { + $this->assertSame($prepared['cancellation'], $context->scopeOrigin->toArray()); + $this->assertSame($prepared['authority_deadline_at'], $context->deadline()->toISOString()); + $this->assertSame($context->requestId, $receipt['request_id']); + } + $historyBefore = WorkflowHistoryEvent::query()->count(); + $duplicate = $this->mutate('deliver', $task, $scope, $request, ['call_kind' => 'child'])->json(); + $this->assertContract($duplicate); + $this->assertSame($receipt, $duplicate['child_cancellations'][0]); + $this->assertSame($delivery['authority_deadline_at'], $duplicate['authority_deadline_at']); + $this->assertSame($historyBefore, WorkflowHistoryEvent::query()->count()); + if ($policy === 'wait_cancellation_completed') { + $this->assertFalse($receipt['ready']); + $this->assertNull($receipt['resolution_history_event_id']); + $target->attemptCancel('canonical terminal fixture'); + $resolved = $this->mutate('deliver', $task, $scope, $request, ['call_kind' => 'child']) + ->assertOk()->assertJsonPath('delivered', true)->json(); + $this->assertContract($resolved); + $this->assertSame($receipt['history_event_id'], $resolved['child_cancellations'][0]['history_event_id']); + $this->assertSame($receipt['request_id'], $resolved['child_cancellations'][0]['request_id']); + $this->assertTrue($resolved['child_cancellations'][0]['ready']); + $this->assertNotNull($resolved['child_cancellations'][0]['resolution_history_event_id']); + $this->assertSame($prepared['authority_deadline_at'], $resolved['authority_deadline_at']); + } else { + $this->assertTrue($receipt['ready']); + $this->assertSame($delivery['history_event_id'], $duplicate['history_event_id']); + } + $this->assertRecoverableHostingClaim($before, $task['task_id']); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'ChildCancellationRequested')->count()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDelivered')->count()); + } + + public static function childPolicies(): iterable + { + foreach (['try_cancel', 'wait_cancellation_completed', 'abandon'] as $policy) { + yield $policy => [$policy]; + } + } + + public function test_scoped_cleanup_prepares_and_reuses_original_runtime_snapshot(): void + { + [$task, $scope, $request, $proof] = $this->cleanupClaim(); + $response = $this->cleanupLocal($task, 'prepare', $scope, $proof)->assertOk() + ->assertJsonPath('prepared', true)->assertJsonPath('duplicate', false); + $first = $response->json(); + $snapshot = $first['cancellation_cleanup']; + $this->assertCleanupContract($snapshot); + $this->assertSame($scope, $snapshot['scope_id']); + $this->assertSame($scope, $snapshot['operation_scope_id']); + $this->assertSame($request, $snapshot['request_id']); + $this->assertSame($proof['delivery_history_event_id'], $snapshot['delivery_history_event_id']); + $this->assertSame($snapshot['authority_deadline_at'], $first['schedule_to_close_deadline_at']); + $before = WorkflowHistoryEvent::query()->count(); + $duplicate = $this->cleanupLocal($task, 'prepare', $scope, $proof)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('activity_attempt_id', $first['activity_attempt_id'])->json(); + $duplicateSnapshot = $duplicate['cancellation_cleanup']; + ksort($snapshot); + ksort($duplicateSnapshot); + $this->assertSame($snapshot, $duplicateSnapshot); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertNull(WorkflowRun::query()->findOrFail($task['run_id'])->cancellation_request_command_id); + } + + #[DataProvider('localControlRequests')] + public function test_running_local_control_commits_original_scope_preparation_before_fencing(string $operation, bool $runRequest, bool $staleRegistration): void + { + [$task, $scope] = $this->claim(prepared: true, requestCancellation: false); + $path = "/api/worker/workflow-tasks/{$task['task_id']}/local-activities"; + $owner = ['lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt']]; + $local = $this->withHeaders($this->headers())->postJson($path.'/prepare', [ + ...$owner, 'sequence' => 2, 'worker_attempt_id' => 'running-scoped-local', + 'descriptor' => ['type' => 'record_local_activity', 'activity_type' => 'opaque-local', + 'arguments' => Serializer::serializeWithCodec('avro', []), 'payload_codec' => 'avro', + 'cancellation_scope_id' => $scope], + ])->assertOk()->json(); + $run = WorkflowRun::query()->findOrFail($task['run_id']); + if ($runRequest) { + WorkflowStub::loadRun($run->id)->requestCancellation('finish the run', 30); + } else { + CancellationScopeRequests::request($run, $scope, '1.20', 30); + } + $request = CancellationScopeRequests::context($run->fresh(), $scope); + if ($staleRegistration) { + WorkerRegistration::query()->where('worker_id', 'delivery-owner') + ->update(['last_heartbeat_at' => now()->subHour()]); + } + $before = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + $response = $this->withHeaders($this->headers())->postJson($path.'/'.$local['activity_attempt_id'].'/'.$operation, + [...$owner, 'renew_lease' => true, 'progress' => ['message' => 'must not record']]); + $response->assertOk()->assertJsonPath('active', false)->assertJsonPath('renewed', false) + ->assertJsonPath('heartbeat_recorded', false)->assertJsonPath('fenced', true) + ->assertJsonPath('reason', 'cancellation_scope_requested'); + $this->assertSame( + $request->toArray(), + $request::fromArray($response->json('cancellation_scope.cancellation'))->toArray(), + ); + $this->assertIsString($response->json('history_refresh_page_token')); + $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $preparation = WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDeliveryPrepared')->sole(); + $fence = WorkflowHistoryEvent::query()->where('event_type', 'ActivityCancelled')->sole(); + $this->assertLessThan($fence->sequence, $preparation->sequence); + $this->assertSame($request->requestId, $fence->workflow_command_id); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'ActivityHeartbeatRecorded')->count()); + $this->withHeaders($this->headers())->postJson($path.'/'.$local['activity_attempt_id'].'/acknowledge-cancellation', + [...$owner, 'request_id' => $request->requestId])->assertOk()->assertJsonPath('acknowledged', true); + $this->withHeaders($this->headers())->postJson($path.'/'.$local['activity_attempt_id'].'/outcome', + [...$owner, 'report' => ['outcome' => 'completed', 'result' => Serializer::serializeWithCodec('avro', 'stale'), + 'payload_codec' => 'avro']])->assertConflict()->assertJsonPath('recorded', false); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'ActivityCompleted')->count()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDeliveryPrepared')->count()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'ActivityCancelled')->count()); + } + + public static function localControlRequests(): iterable + { + foreach (['control', 'heartbeat'] as $operation) { + foreach ([false, true] as $runRequest) { + foreach ([false, true] as $staleRegistration) { + yield $operation.':'.($runRequest ? 'run' : 'scope').':'.($staleRegistration ? 'stale' : 'fresh') => [$operation, $runRequest, $staleRegistration]; + } + } + } + } + + public function test_scoped_cleanup_recovery_keeps_original_proof_and_refuses_stale_publication(): void + { + [$task, $scope, , $proof] = $this->cleanupClaim(); + $first = $this->cleanupLocal($task, 'prepare', $scope, $proof)->assertOk()->json(); + ActivityAttempt::query()->findOrFail($first['activity_attempt_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $replacement = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'delivery-owner', 'task_queue' => 'scoped-delivery', + ])->assertOk()->json('task'); + $this->assertSame($task['task_id'], $replacement['task_id']); + $this->assertSame($task['workflow_task_attempt'] + 1, $replacement['workflow_task_attempt']); + $recovery = $this->cleanupLocal($replacement, 'recover', $scope, $proof)->assertOk() + ->assertJsonPath('recovered', true)->assertJsonPath('claim_released', true) + ->assertJsonPath('callback_stop_state', 'unknown')->json(); + $snapshot = ActivityExecution::query()->findOrFail($first['activity_execution_id'])->activity_options['cancellation_cleanup']; + $this->assertCleanupContract($snapshot); + $expected = $first['cancellation_cleanup']; + ksort($expected); + ksort($snapshot); + $this->assertSame($expected, $snapshot); + $before = WorkflowHistoryEvent::query()->count(); + $this->cleanupLocal($replacement, 'recover', $scope, $proof)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('event_id', $recovery['event_id']); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/{$first['activity_attempt_id']}/outcome", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'report' => ['outcome' => 'completed', 'result' => Serializer::serializeWithCodec('avro', 'stale'), 'payload_codec' => 'avro'], + ])->assertConflict()->assertJsonPath('recorded', false); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'ActivityCompleted')->count()); + $retryTask = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'delivery-owner', 'task_queue' => 'scoped-delivery', + ])->assertOk()->json('task'); + $this->assertSame($recovery['created_task_ids'][0], $retryTask['task_id']); + $retryResponse = $this->cleanupLocal($retryTask, 'prepare', $scope, $proof, workerAttempt: 'replacement-cleanup-attempt'); + $this->assertSame(200, $retryResponse->status(), json_encode($retryResponse->json('reason'), JSON_THROW_ON_ERROR)); + $retry = $retryResponse->assertJsonPath('prepared', true)->assertJsonPath('attempt_number', 2)->json(); + $this->assertNotSame($first['activity_attempt_id'], $retry['activity_attempt_id']); + $retrySnapshot = $retry['cancellation_cleanup']; + ksort($retrySnapshot); + $this->assertSame($expected, $retrySnapshot); + $this->assertSame($first['schedule_to_close_deadline_at'], $retry['schedule_to_close_deadline_at']); + $body = [ + 'lease_owner' => $retryTask['lease_owner'], 'workflow_task_attempt' => $retryTask['workflow_task_attempt'], + 'report' => ['outcome' => 'completed', 'result' => Serializer::serializeWithCodec('avro', 'resumed'), 'payload_codec' => 'avro'], + ]; + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/PreparedLocalOutcomeRequest', + json_decode(json_encode($body, JSON_THROW_ON_ERROR), flags: JSON_THROW_ON_ERROR)); + $outcome = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$retryTask['task_id']}/local-activities/{$retry['activity_attempt_id']}/outcome", $body); + $this->assertSame(200, $outcome->status(), json_encode($outcome->json('reason'), JSON_THROW_ON_ERROR)); + $outcome->assertJsonPath('recorded', true)->assertJsonPath('claim_released', false); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'CancellationScopeDelivered')->count()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'ActivityCompleted')->count()); + $this->assertNull(WorkflowRun::query()->findOrFail($task['run_id'])->cancellation_request_command_id); + } + + public static function forgedCleanupProofs(): iterable + { + foreach (['prepare', 'recover'] as $operation) { + foreach (['scope_id', 'request_id', 'delivery_history_event_id', 'cleanup_deadline_at'] as $field) { + yield $operation.':'.$field => [$operation, $field]; + } + } + } + + #[DataProvider('forgedCleanupProofs')] + public function test_scoped_cleanup_rejects_forged_authority_before_payload_resolution(string $operation, string $field): void + { + [$task, $scope, , $proof] = $this->cleanupClaim(); + $proof[$field] = 'invented-authority'; + $storage = \Mockery::mock(NamespaceExternalPayloadStorage::class); + $storage->shouldNotReceive('driverFor'); + $this->app->instance(NamespaceExternalPayloadStorage::class, $storage); + $before = WorkflowHistoryEvent::query()->count(); + $this->cleanupLocal($task, $operation, $scope, $proof, ['codec' => 'avro', 'invalid_reference' => true]) + ->assertConflict()->assertJsonPath('recorded', false)->assertJsonPath('reason', 'operation_scope_cancellation_prepared'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertSame(0, ActivityExecution::query()->count()); + $this->assertSame(0, ActivityAttempt::query()->count()); + } + + public static function cleanupGroupValidity(): iterable + { + yield 'original proofs' => [false]; + yield 'second proof invalid' => [true]; + } + + #[DataProvider('cleanupGroupValidity')] + public function test_scoped_cleanup_group_validates_all_proofs_before_any_sibling(bool $invalidSecond): void + { + [$task, $scope, , $proof] = $this->cleanupClaim(); + $commands = []; + foreach ([0, 1] as $index) { + $commands[] = [...$this->cleanupDescriptor($scope, $proof), 'type' => 'prepare_local_activity', + ...ParallelChildGroup::itemMetadata(3, 2, $index, 'activity')]; + } + if ($invalidSecond) { + $commands[1]['cancellation_cleanup']['request_id'] = 'invented-authority'; + $commands[1]['arguments'] = ['codec' => 'avro', 'invalid_reference' => true]; + } + $before = WorkflowHistoryEvent::query()->count(); + $response = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/checkpoint-group", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'checkpoint_id' => 'scoped-cleanup-group', 'start_sequence' => 3, 'commands' => $commands, + ]); + if ($invalidSecond) { + $response->assertConflict()->assertJsonPath('recorded', false)->assertJsonPath('reason', 'operation_scope_cancellation_prepared'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertSame(0, ActivityExecution::query()->count()); + $this->assertSame(0, ActivityAttempt::query()->count()); + } else { + $response->assertOk()->assertJsonPath('checkpointed', true); + $this->assertCount(2, $response->json('local_activities')); + $this->assertSame(2, ActivityExecution::query()->count()); + foreach (ActivityExecution::query()->get() as $execution) { + $this->assertCleanupContract($execution->activity_options['cancellation_cleanup']); + $this->assertSame($proof['delivery_history_event_id'], $execution->activity_options['cancellation_cleanup']['delivery_history_event_id']); + } + } + } + + public function test_completion_preserves_original_scoped_timer_cleanup_proof_after_metadata_prefix(): void + { + [$task, $scope, , $proof] = $this->cleanupClaim(); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [ + ['type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 'cleanup entry')], + ['type' => 'start_timer', 'delay_seconds' => 1, 'cancellation_scope_id' => $scope, + 'cancellation_cleanup' => $proof], + ], + ])->assertOk(); + $timer = WorkflowHistoryEvent::query()->where('event_type', 'TimerScheduled')->sole(); + $this->assertSame(4, $timer->payload['sequence']); + $this->assertSame($scope, $timer->payload['cancellation_scope_id']); + $snapshot = $timer->payload['cancellation_cleanup']; + $this->assertSame($proof['request_id'], $snapshot['request_id']); + $this->assertSame($proof['delivery_history_event_id'], $snapshot['delivery_history_event_id']); + $delivery = WorkflowHistoryEvent::query()->findOrFail($proof['delivery_history_event_id']); + $this->assertSame($delivery->payload['authority_deadline_at'], $snapshot['authority_deadline_at']); + $this->assertSame($delivery->payload['cancellation']['root_context']['cleanup_deadline_at'], $snapshot['cleanup_deadline_at']); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('event_type', 'SideEffectRecorded')->count()); + $this->assertSame(TimerStatus::Pending, WorkflowTimer::query()->sole()->status); + } + + public static function invalidTimerCleanupProofs(): iterable + { + foreach (['null', 'empty', 'missing-scope', 'new-deadline', 'different-request', 'too-long'] as $mutation) { + yield $mutation => [$mutation]; + } + } + + #[DataProvider('invalidTimerCleanupProofs')] + public function test_completion_refuses_invalid_timer_cleanup_before_any_prefix_commit(string $mutation): void + { + [$task, $scope, , $proof] = $this->cleanupClaim(); + $command = ['type' => 'start_timer', 'delay_seconds' => 1, 'cancellation_scope_id' => $scope, + 'cancellation_cleanup' => $proof]; + if ($mutation === 'null') { $command['cancellation_cleanup'] = null; } + elseif ($mutation === 'empty') { $command['cancellation_cleanup'] = []; } + elseif ($mutation === 'missing-scope') { unset($command['cancellation_cleanup']['scope_id']); } + elseif ($mutation === 'new-deadline') { $command['cancellation_cleanup']['cleanup_deadline_at'] = now()->addHour()->toIso8601String(); } + elseif ($mutation === 'different-request') { $command['cancellation_cleanup']['request_id'] = 'invented-request'; } + else { $command['delay_seconds'] = 31; } + $before = WorkflowHistoryEvent::query()->count(); + $response = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/complete", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [ + ['type' => 'record_side_effect', 'result' => Serializer::serializeWithCodec('avro', 'must not commit')], + $command, + ], + ]); + if (in_array($mutation, ['missing-scope', 'different-request', 'too-long'], true)) { + $response->assertConflict()->assertJsonPath('recorded', false)->assertJsonPath('reason', + $mutation === 'too-long' ? 'cancellation_scope_cleanup_timer_exceeds_deadline' + : 'cancellation_scope_cleanup_authority_mismatch'); + } else { + $response->assertUnprocessable(); + } + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertSame(0, WorkflowTimer::query()->count()); + $this->assertSame(TaskStatus::Leased, WorkflowTask::query()->findOrFail($task['task_id'])->status); + } + + private function cleanupClaim(): array + { + [$task, $scope, $request] = $this->claim(prepared: true); + $this->mutate('prepare', $task, $scope, $request, ['call_kind' => 'local_activity'])->assertOk(); + $delivery = $this->mutate('deliver', $task, $scope, $request, ['call_kind' => 'local_activity'])->assertOk()->json(); + $proof = ['scope_id' => $scope, 'request_id' => $request, 'delivery_history_event_id' => $delivery['history_event_id']]; + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/PreparedLocalCleanupProof', (object) $proof); + + return [$task, $scope, $request, $proof]; + } + + private function cleanupDescriptor(string $scope, array $proof, mixed $arguments = null): array + { + return ['type' => 'record_local_activity', 'activity_type' => 'opaque-local', 'payload_codec' => 'avro', + 'arguments' => $arguments ?? Serializer::serializeWithCodec('avro', []), + 'retry_policy' => ['max_attempts' => 2, 'backoff_seconds' => [0]], + 'cancellation_scope_id' => $scope, 'cancellation_cleanup' => $proof]; + } + + private function cleanupLocal(array $task, string $operation, string $scope, array $proof, mixed $arguments = null, + string $workerAttempt = 'scoped-cleanup-attempt') + { + $body = [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'worker_attempt_id' => $workerAttempt, 'sequence' => 3, + 'descriptor' => $this->cleanupDescriptor($scope, $proof, $arguments), + ]; + + return $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/{$operation}", $body); + } + + private function assertCleanupContract(array $snapshot): void + { + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/PreparedLocalCleanupSnapshot', + json_decode(json_encode($snapshot, JSON_THROW_ON_ERROR), flags: JSON_THROW_ON_ERROR)); + } + + private function claim(bool $timer = false, ?string $childPolicy = null, bool $prepared = false, bool $requestCancellation = true): array + { + $this->withHeaders($this->headers('operator'))->postJson('/api/workflows', [ + 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'scoped-delivery', 'input' => ['Ada'], + ])->assertCreated(); + $this->withHeaders($this->headers())->postJson('/api/worker/register', [ + 'worker_id' => 'delivery-owner', 'task_queue' => 'scoped-delivery', 'runtime' => 'php', + 'supported_workflow_types' => ['tests.external-greeting-workflow'], + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY, + ...($prepared ? [PreparedLocalActivityPolicy::CAPABILITY, PreparedLocalActivityPolicy::GROUP_CAPABILITY] : [])], + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), 'max_concurrent_workflow_tasks' => 1, + ])->assertCreated(); + $task = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'delivery-owner', 'task_queue' => 'scoped-delivery', + ])->assertOk()->json('task'); + $scope = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/open", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], 'sequence' => 1, + ])->assertOk()->json('scope_id'); + if ($timer) { + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/checkpoint", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'checkpoint_id' => 'timer-prefix', 'start_sequence' => 2, + 'commands' => [['type' => 'start_timer', 'delay_seconds' => 60, 'cancellation_scope_id' => $scope]], + ])->assertOk()->assertJsonPath('checkpointed', true); + } + if ($childPolicy !== null) { + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/checkpoint", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'checkpoint_id' => 'child-prefix', 'start_sequence' => 2, + 'commands' => [['type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', + 'arguments' => Serializer::serializeWithCodec('avro', ['child']), + 'payload_codec' => 'avro', 'cancellation_scope_id' => $scope, 'cancellation_policy' => $childPolicy]], + ])->assertOk()->assertJsonPath('checkpointed', true); + } + $beforeRequest = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + $request = $requestCancellation + ? CancellationScopeRequests::request(WorkflowRun::query()->findOrFail($task['run_id']), $scope, '1.20', 30) : null; + if ($requestCancellation) { + $this->assertRecoverableHostingClaim($beforeRequest, $task['task_id'], shortened: true); + } else { + $this->assertSame($beforeRequest, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + } + + return [$task, $scope, $request?->payload['request_id']]; + } + + private function mutate(string $phase, array $task, string $scope, string $request, array $overrides = [], + string $namespace = 'default', string $role = 'worker', string $version = '1.20') + { + return $this->withHeaders($this->headers($role, $namespace, $version)) + ->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/{$phase}", array_replace([ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'scope_id' => $scope, 'request_id' => $request, 'sequence' => 2, 'call_kind' => 'activity', + ], $overrides)); + } + + private function headers(string $role = 'worker', string $namespace = 'default', string $version = '1.20'): array + { + return ['Authorization' => 'Bearer fixture-'.$role, 'X-Namespace' => $namespace, + 'X-Durable-Workflow-Control-Plane-Version' => '2', WorkerProtocol::HEADER => $version]; + } + + private function assertContract(array $response): void + { + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/CancellationScopeDeliveryResponse/allOf/1', + json_decode(json_encode($response, JSON_THROW_ON_ERROR), flags: JSON_THROW_ON_ERROR)); + } +} diff --git a/tests/Source/PreparedLocalCancellationPolicyTest.php b/tests/Source/PreparedLocalCancellationPolicyTest.php new file mode 100644 index 00000000..2077c687 --- /dev/null +++ b/tests/Source/PreparedLocalCancellationPolicyTest.php @@ -0,0 +1,282 @@ + '1.20', + 'workflows.v2.workflow_task_lease_seconds' => 10, + 'workflows.v2.types.workflows' => ['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class], + ]); + WorkflowNamespace::query()->create(['name' => 'default', 'description' => 'Test', 'retention_days' => 30, 'status' => 'active']); + $this->assertSame(['try_cancel', 'wait_cancellation_completed'], PreparedLocalActivityPolicy::cancellationPolicies()); + } + + public static function policies(): array + { + return [['try_cancel'], ['wait_cancellation_completed']]; + } + + #[DataProvider('policies')] + public function test_explicit_policy_is_admitted_once_and_canonical_history_preserves_it(string $policy): void + { + $task = $this->claim(); + $descriptor = $this->descriptor($policy); + $first = $this->local($task, 'prepare', $descriptor)->assertOk()->assertJsonPath('prepared', true)->json(); + $before = WorkflowHistoryEvent::query()->count(); + $this->local($task, 'prepare', $descriptor)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('activity_attempt_id', $first['activity_attempt_id']); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertSame($policy, ActivityExecution::query()->sole()->activity_options['cancellation_policy']); + foreach ([HistoryEventType::ActivityScheduled, HistoryEventType::ActivityStarted] as $eventType) { + $event = WorkflowHistoryEvent::query()->where('event_type', $eventType)->sole(); + $this->assertSame('local', $event->payload['execution_mode']); + $this->assertSame($policy, $event->payload['activity']['cancellation_policy']); + } + $changed = $policy === 'try_cancel' ? 'wait_cancellation_completed' : 'try_cancel'; + $this->local($task, 'prepare', $this->descriptor($changed))->assertStatus(409)->assertJsonPath('reason', 'local_activity_preparation_mismatch'); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + } + + public function test_default_protocol_and_an_older_optional_role_do_not_advertise_policies(): void + { + config(['server.worker_protocol.version' => '1.19']); + $this->assertSame([], WorkerProtocol::serverCapabilities()['prepared_local_activity_cancellation_policies']); + config(['server.worker_protocol.version' => '1.20']); + $task = $this->claim(); + $older = \Mockery::mock(PreparedLocalActivityTaskBridge::class); + $this->app->instance(WorkflowTaskBridge::class, $older); + $this->assertTrue(PreparedLocalActivityPolicy::backendSupported()); + $this->assertSame([], WorkerProtocol::serverCapabilities()['prepared_local_activity_cancellation_policies']); + $this->local($task, 'prepare', $this->descriptor('try_cancel'))->assertStatus(409) + ->assertJsonPath('unavailable', ['installed_runtime_local_activity_cancellation_policy']); + $this->assertDatabaseCount('activity_executions', 0); + } + + #[DataProvider('policies')] + public function test_replay_requires_the_specific_policy_supported_by_the_installed_backend(string $policy): void + { + $task = $this->claim(); + $this->local($task, 'prepare', $this->descriptor($policy))->assertOk(); + $restricted = \Mockery::mock(PreparedLocalActivityTaskBridge::class.', '.LocalCancellationPolicyBridge::class); + $restricted->shouldReceive('supportedLocalActivityCancellationPolicies')->andReturn(['try_cancel']); + $this->app->instance(WorkflowTaskBridge::class, $restricted); + $this->assertSame(['try_cancel'], WorkerProtocol::serverCapabilities()['prepared_local_activity_cancellation_policies']); + $this->assertSame($policy === 'try_cancel', PreparedLocalActivityPolicy::canReplayRun($task['run_id'], $this->capabilities(), '1.20')); + $this->assertFalse(PreparedLocalActivityPolicy::canReplayRun($task['run_id'], $this->capabilities(false), '1.20')); + $this->local($task, 'recover', $this->descriptor('wait_cancellation_completed'))->assertStatus(409) + ->assertJsonPath('unavailable', ['installed_runtime_local_activity_cancellation_policy']); + } + + public function test_registration_requires_preparation_cooperation_and_protocol_1_20(): void + { + foreach ([['1.20', []], ['1.20', [CooperativeCancellationPolicy::CAPABILITY]], + ['1.20', [PreparedLocalActivityPolicy::CAPABILITY]], + ['1.19', [CooperativeCancellationPolicy::CAPABILITY, PreparedLocalActivityPolicy::CAPABILITY]]] as [$version, $capabilities]) { + $this->register('unsupported', [...$capabilities, PreparedLocalActivityPolicy::CANCELLATION_POLICIES_CAPABILITY], $version) + ->assertStatus(409)->assertJsonPath('registered', false) + ->assertJsonPath('reason', 'prepared_local_activity_cancellation_policy_capability_mismatch'); + } + $this->assertDatabaseCount('workflow_worker_registrations', 0); + } + + public function test_registration_changes_cannot_upgrade_an_existing_claim_or_resolve_its_payload(): void + { + $task = $this->claim(false); + $before = WorkflowHistoryEvent::query()->count(); + $this->register('original', $this->capabilities())->assertCreated(); + $descriptor = [...$this->descriptor('try_cancel'), 'arguments' => ['codec' => 'avro', 'invalid_reference' => true]]; + foreach (['prepare', 'recover'] as $operation) { + $this->local($task, $operation, $descriptor)->assertStatus(409) + ->assertJsonPath('reason', 'prepared_local_activity_cancellation_policy_not_supported') + ->assertJsonPath('worker_id', 'original')->assertJsonPath('requested_policy', 'try_cancel') + ->assertJsonPath('required_capability', PreparedLocalActivityPolicy::CANCELLATION_POLICIES_CAPABILITY) + ->assertJsonPath('unavailable', ['worker_claim_cancellation_policy_capability']); + } + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseCount('activity_executions', 0); + $this->assertDatabaseCount('activity_attempts', 0); + } + + public function test_omission_preserves_admission_for_an_older_prepared_claim(): void + { + $task = $this->claim(false); + $descriptor = $this->descriptor('try_cancel'); + unset($descriptor['cancellation_policy']); + $this->local($task, 'prepare', $descriptor)->assertOk()->assertJsonPath('prepared', true); + $this->assertArrayNotHasKey('cancellation_policy', ActivityExecution::query()->sole()->activity_options); + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $this->register('legacy', $this->capabilities(false))->assertCreated(); + $this->assertSame($task['task_id'], $this->poll('legacy')->assertOk()->json('task.task_id')); + } + + public static function invalidPolicies(): array + { + return [['abandon'], [null], ['unknown'], [true]]; + } + + #[DataProvider('invalidPolicies')] + public function test_invalid_or_abandoned_local_policy_is_refused_before_payload_and_admission(mixed $policy): void + { + $task = $this->claim(); + $before = WorkflowHistoryEvent::query()->count(); + $this->local($task, 'prepare', [...$this->descriptor($policy), 'schedule_to_close_timeout' => 30, + 'arguments' => ['codec' => 'avro', 'invalid_reference' => true]])->assertStatus(409) + ->assertJsonPath('recorded', false)->assertJsonPath('unavailable', ['local_activity_cancellation_policy']); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseCount('activity_executions', 0); + } + + #[DataProvider('policies')] + public function test_replacement_requires_the_recorded_policy_and_recovers_without_changing_its_lifetime(string $policy): void + { + $task = $this->claim(); + $descriptor = [...$this->descriptor($policy), 'schedule_to_close_timeout' => 60, + 'retry_policy' => ['max_attempts' => 2, 'backoff_seconds' => [2]]]; + $first = $this->local($task, 'prepare', $descriptor)->assertOk()->json(); + ActivityAttempt::query()->findOrFail($first['activity_attempt_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + WorkflowTask::query()->findOrFail($task['task_id'])->forceFill(['lease_expires_at' => now()->subSecond()])->save(); + $this->register('legacy', $this->capabilities(false))->assertCreated(); + $history = WorkflowHistoryEvent::query()->count(); + $this->poll('legacy')->assertOk()->assertJsonPath('task', null); + // Poll maintenance records the expired lease's repair request even + // when this poller cannot receive or replay the resulting claim. + $this->assertSame($history + 1, WorkflowHistoryEvent::query()->count()); + $this->assertSame(HistoryEventType::RepairRequested, WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id'])->orderByDesc('sequence')->firstOrFail()->event_type); + $this->assertSame($task['workflow_task_attempt'], WorkflowTask::query()->findOrFail($task['task_id'])->attempt_count); + $this->register('replacement', $this->capabilities())->assertCreated(); + $replacement = $this->poll('replacement')->assertOk()->json('task'); + $this->assertSame($task['task_id'], $replacement['task_id']); + $this->assertSame($task['workflow_task_attempt'] + 1, $replacement['workflow_task_attempt']); + $recovered = $this->local($replacement, 'recover', $descriptor)->assertOk() + ->assertJsonPath('recovered', true)->assertJsonPath('claim_released', true)->assertJsonPath('callback_stop_state', 'unknown')->json(); + $this->local($replacement, 'recover', $descriptor)->assertOk()->assertJsonPath('duplicate', true) + ->assertJsonPath('event_id', $recovered['event_id']); + $this->travel(2)->seconds(); + $retry = $this->poll('replacement')->assertOk()->json('task'); + $second = $this->local($retry, 'prepare', $descriptor, 'sdk-recovered')->assertOk()->assertJsonPath('attempt_number', 2)->json(); + $this->assertSame($first['schedule_to_close_deadline_at'], $second['schedule_to_close_deadline_at']); + $this->assertSame($policy, ActivityExecution::query()->sole()->activity_options['cancellation_policy']); + } + + #[DataProvider('policies')] + public function test_group_policy_is_checked_before_any_child_or_local_member_is_committed(string $policy): void + { + $task = $this->claim(false, true); + $before = WorkflowHistoryEvent::query()->count(); + $this->group($task, $policy)->assertStatus(409)->assertJsonPath('unavailable', ['worker_claim_cancellation_policy_capability']); + $this->assertSame($before, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseCount('workflow_runs', 1); + $this->assertDatabaseCount('activity_executions', 0); + $this->assertArrayNotHasKey('portable_local_group_checkpoint', WorkflowTask::query()->findOrFail($task['task_id'])->payload); + } + + #[DataProvider('policies')] + public function test_capable_group_records_the_explicit_local_policy(string $policy): void + { + $task = $this->claim(true, true); + $this->group($task, $policy)->assertOk()->assertJsonPath('checkpointed', true); + $event = WorkflowHistoryEvent::query()->where('event_type', HistoryEventType::ActivityScheduled)->sole(); + $this->assertSame('local', $event->payload['execution_mode']); + $this->assertSame($policy, $event->payload['activity']['cancellation_policy']); + $this->assertDatabaseCount('workflow_runs', 2); + $this->assertDatabaseCount('activity_executions', 1); + } + + private function capabilities(bool $policy = true, bool $group = false): array + { + return [CooperativeCancellationPolicy::CAPABILITY, PreparedLocalActivityPolicy::CAPABILITY, + ...($policy ? [PreparedLocalActivityPolicy::CANCELLATION_POLICIES_CAPABILITY] : []), + ...($group ? [PreparedLocalActivityPolicy::GROUP_CAPABILITY] : [])]; + } + + private function claim(bool $policy = true, bool $group = false): array + { + $this->withHeaders(['X-Namespace' => 'default', 'X-Durable-Workflow-Control-Plane-Version' => '2'])->postJson('/api/workflows', [ + 'workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'local-policy', 'input' => ['Ada'], + ])->assertCreated(); + $this->register('original', $this->capabilities($policy, $group))->assertCreated(); + + return $this->poll('original')->assertOk()->json('task'); + } + + private function register(string $id, array $capabilities, string $version = '1.20'): TestResponse + { + return $this->withHeaders($this->headers($version))->postJson('/api/worker/register', [ + 'worker_id' => $id, 'task_queue' => 'local-policy', 'runtime' => 'php', + 'supported_workflow_types' => ['tests.external-greeting-workflow'], 'capabilities' => $capabilities, + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), 'max_concurrent_workflow_tasks' => 1, + 'process_metrics' => ['host' => 'test-worker', 'process_started_at' => now()->toISOString(), 'process_id' => $id === 'original' ? 10 : 20], + ]); + } + + private function poll(string $id): TestResponse + { + return $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', ['worker_id' => $id, 'task_queue' => 'local-policy']); + } + + private function descriptor(mixed $policy): array + { + return ['type' => 'record_local_activity', 'activity_type' => 'opaque-local', + 'arguments' => Serializer::serializeWithCodec('avro', ['Ada']), 'payload_codec' => 'avro', 'cancellation_policy' => $policy]; + } + + private function local(array $task, string $operation, array $descriptor, string $workerAttemptId = 'sdk-local-1'): TestResponse + { + return $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/{$operation}", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'sequence' => 1, 'worker_attempt_id' => $workerAttemptId, 'descriptor' => $descriptor, + ]); + } + + private function group(array $task, string $policy): TestResponse + { + return $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/local-activities/checkpoint-group", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'checkpoint_id' => 'policy-group', 'start_sequence' => 1, 'commands' => [[ + 'type' => 'start_child_workflow', 'workflow_type' => 'tests.external-greeting-workflow', + 'arguments' => Serializer::serializeWithCodec('avro', ['child']), 'payload_codec' => 'avro', + ...ParallelChildGroup::itemMetadata(1, 2, 0, 'mixed'), + ], [...$this->descriptor($policy), 'type' => 'prepare_local_activity', + ...ParallelChildGroup::itemMetadata(1, 2, 1, 'mixed')]], + ]); + } + + private function headers(string $version = '1.20'): array + { + return ['X-Namespace' => 'default', WorkerProtocol::HEADER => $version]; + } +} + +interface LocalCancellationPolicyBridge +{ + public function supportedLocalActivityCancellationPolicies(): array; +} diff --git a/tests/Source/ScopedActivityCancellationReceiptTest.php b/tests/Source/ScopedActivityCancellationReceiptTest.php new file mode 100644 index 00000000..bf6a411d --- /dev/null +++ b/tests/Source/ScopedActivityCancellationReceiptTest.php @@ -0,0 +1,333 @@ + '1.20', + 'server.auth.driver' => 'token', + 'server.auth.role_tokens' => ['operator' => 'fixture-operator', 'worker' => 'fixture-worker'], + 'workflows.v2.types.workflows' => ['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class], + ]); + WorkflowNamespace::query()->create(['name' => 'default', 'retention_days' => 30, 'status' => 'active']); + $this->assertTrue(class_exists(ScopedActivityCancellation::class)); + $this->assertTrue(class_exists(ActivityCancellationContext::class)); + } + + public function test_scoped_receipt_and_original_owner_acknowledgement_leave_sibling_and_workflow_running(): void + { + [$workflowTask, $target, $sibling, $request, $root, $fence] = $this->scopedPair(); + $before = $this->snapshot($workflowTask, $target, $sibling); + $observed = $this->observe($target)->assertOk()->assertJsonPath('can_continue', false) + ->assertJsonPath('cancel_requested', true)->assertJsonPath('heartbeat_recorded', false) + ->assertJsonPath('cancellation_acknowledgement.callback_state', 'unknown') + ->assertJsonPath('cancellation_acknowledgement.request_id', $request->payload['request_id']) + ->assertJsonPath('cancellation_acknowledgement.root_request_id', $root->payload['request_id']) + ->assertJsonPath('cancellation_acknowledgement.cleanup_deadline_at', $fence['cleanup_deadline_at']) + ->assertJsonPath('cancellation_acknowledgement.cancellation_history_event_id', $fence['history_event_id']); + $this->assertSame($before, $this->snapshot($workflowTask, $target, $sibling)); + $this->assertNull(WorkflowRun::query()->findOrFail($target['run_id'])->cancellation_request_command_id); + $this->assertNotSame($root->payload['request_id'], $request->payload['request_id']); + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/ActivityCancellationReceipt', json_decode($observed->getContent(), flags: JSON_THROW_ON_ERROR)->cancellation_acknowledgement); + $this->observe($sibling)->assertOk()->assertJsonPath('can_continue', true) + ->assertJsonPath('cancel_requested', false)->assertJsonMissingPath('cancellation_acknowledgement'); + $this->withHeaders($this->headers())->postJson($this->path($target, 'complete'), $this->attempt($target) + ['result' => null]) + ->assertConflict()->assertJsonPath('recorded', false); + $this->withHeaders($this->headers())->postJson($this->path($target, 'fail'), $this->attempt($target) + ['failure' => ['message' => 'stale', 'non_retryable' => true]]) + ->assertConflict()->assertJsonPath('recorded', false); + $this->assertSame($before, $this->snapshot($workflowTask, $target, $sibling)); + + $ack = $this->acknowledge($target, $request->payload['request_id'])->assertOk() + ->assertJsonPath('acknowledged', true)->assertJsonPath('duplicate', false) + ->assertJsonPath('heartbeat_recorded', false)->json('history_event_id'); + $this->observe($target)->assertOk()->assertJsonPath('cancellation_acknowledgement.callback_state', 'stopped') + ->assertJsonPath('cancellation_acknowledgement.history_event_id', $ack) + ->assertJsonPath('cancellation_acknowledgement.received_after_deadline', false); + $this->acknowledge($target, $request->payload['request_id'])->assertOk() + ->assertJsonPath('duplicate', true)->assertJsonPath('history_event_id', $ack); + $after = $this->snapshot($workflowTask, $target, $sibling); + $this->assertSame($before[3] + 1, $after[3]); + unset($before[3], $after[3]); + $this->assertSame($before[2]['last_history_sequence'] + 1, $after[2]['last_history_sequence']); + foreach (['last_history_sequence', 'last_progress_at', 'updated_at'] as $field) { + unset($before[2][$field], $after[2][$field]); + } + $this->assertSame($before, $after); + $run = WorkflowRun::query()->findOrFail($target['run_id']); + $this->assertTrue(ActivityCancellationCompletion::resolved($run, $target['activity_execution_id'], CancellationScopeRequests::context($run, $fence['scope_id']))); + } + + public function test_delivery_waits_for_original_owner_receipt_and_preserves_the_prepared_frame(): void + { + [$task, $target, $sibling, $request, , $fence] = $this->scopedPair(); + $path = "/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/deliver"; + $body = ['lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'scope_id' => $fence['scope_id'], 'request_id' => $request->payload['request_id'], 'sequence' => 4, 'call_kind' => 'activity']; + $before = $this->snapshot($task, $target, $sibling); + $pending = $this->withHeaders($this->headers())->postJson($path, $body)->assertOk() + ->assertJsonPath('delivered', false)->assertJsonPath('prepared', true) + ->assertJsonPath('reason', 'cancellation_scope_activity_stop_not_acknowledged') + ->assertJsonPath('claim_released', false)->json(); + $this->assertRecoverableHostingClaim($before[0], $task['task_id']); + $afterPending = $this->snapshot($task, $target, $sibling); + $beforePending = $before; + foreach (['lease_expires_at', 'updated_at'] as $field) { + unset($beforePending[0][$field], $afterPending[0][$field]); + } + $this->assertSame($beforePending, $afterPending); + $ack = $this->acknowledge($target, $request->payload['request_id'])->assertOk()->json('history_event_id'); + $delivered = $this->withHeaders($this->headers())->postJson($path, $body)->assertOk() + ->assertJsonPath('delivered', true) + ->assertJsonPath('preparation_history_event_id', $pending['preparation_history_event_id']) + ->assertJsonPath('cancellation', $pending['cancellation']) + ->assertJsonPath('authority_deadline_at', $pending['authority_deadline_at'])->json(); + $this->withHeaders($this->headers())->postJson($path, $body)->assertOk()->assertJsonPath('history_event_id', $delivered['history_event_id']); + $this->assertRecoverableHostingClaim($before[0], $task['task_id']); + $this->assertSame($before[1][1], $this->snapshot($task, $target, $sibling)[1][1]); + $this->assertLessThan(WorkflowHistoryEvent::query()->findOrFail($delivered['history_event_id'])->sequence, + WorkflowHistoryEvent::query()->findOrFail($ack)->sequence); + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/CancellationScopeDeliveryResponse/allOf/1', + json_decode(json_encode($pending, JSON_THROW_ON_ERROR), flags: JSON_THROW_ON_ERROR)); + } + + public function test_later_whole_run_request_and_late_receipt_preserve_original_scoped_identity_and_deadline(): void + { + [, $target, , $request, $root, $fence] = $this->scopedPair(); + $this->travel(7)->seconds(); + $result = WorkflowStub::load($target['workflow_id'])->requestCancellation('later whole-run cleanup', 90); + $this->assertTrue($result->accepted()); + $run = WorkflowRun::query()->findOrFail($target['run_id']); + $this->assertNotSame($request->payload['request_id'], $run->cancellation_request_command_id); + $this->assertNotSame($root->payload['request_id'], $run->cancellation_request_command_id); + $this->observe($target)->assertOk()->assertJsonPath('cancellation_acknowledgement.request_id', $request->payload['request_id']) + ->assertJsonPath('cancellation_acknowledgement.root_request_id', $root->payload['request_id']) + ->assertJsonPath('cancellation_acknowledgement.cleanup_deadline_at', $fence['cleanup_deadline_at']); + $this->acknowledge($target, $run->cancellation_request_command_id)->assertConflict()->assertJsonPath('acknowledged', false); + $this->travel(24)->seconds(); + $ack = $this->acknowledge($target, $request->payload['request_id'])->assertOk()->json('history_event_id'); + $this->observe($target)->assertOk()->assertJsonPath('cancellation_acknowledgement.callback_state', 'stopped') + ->assertJsonPath('cancellation_acknowledgement.request_id', $request->payload['request_id']) + ->assertJsonPath('cancellation_acknowledgement.root_request_id', $root->payload['request_id']) + ->assertJsonPath('cancellation_acknowledgement.cleanup_deadline_at', $fence['cleanup_deadline_at']) + ->assertJsonPath('cancellation_acknowledgement.received_after_deadline', true); + $this->acknowledge($target, $request->payload['request_id'])->assertOk() + ->assertJsonPath('duplicate', true)->assertJsonPath('history_event_id', $ack); + $this->assertSame($request->id, CancellationScopeRequests::request($run->fresh(), $fence['scope_id'], '1.20', 300)->id); + $this->assertSame($run->cancellation_deadline_at->toISOString(), $run->fresh()->cancellation_deadline_at->toISOString()); + } + + public static function invalidAcknowledgements(): array + { + return [ + 'wrong original owner' => ['lease_owner', 'other-owner'], + 'wrong root' => ['root_request_id', 'other-root'], + 'wrong original request' => ['request_id', 'other-request'], + 'changed budget' => ['cleanup_deadline_at', '2099-01-01T00:00:00.000000Z'], + 'still running' => ['callback_state', 'running'], + 'wrong evidence source' => ['evidence_source', 'workflow_worker'], + 'wrong authored operation' => ['sequence', 99], + ]; + } + + #[DataProvider('invalidAcknowledgements')] + public function test_mismatched_stop_report_cannot_make_scoped_callback_look_stopped(string $field, mixed $value): void + { + [$workflowTask, $target, $sibling, $request] = $this->scopedPair(); + $ack = $this->acknowledge($target, $request->payload['request_id'])->assertOk()->json('history_event_id'); + $event = WorkflowHistoryEvent::query()->findOrFail($ack); + $payload = $event->payload; + $payload[$field] = $value; + $event->forceFill(['payload' => $payload])->save(); + $before = $this->snapshot($workflowTask, $target, $sibling); + $this->observe($target)->assertOk()->assertJsonPath('cancellation_acknowledgement.callback_state', 'unknown') + ->assertJsonPath('cancellation_acknowledgement.history_event_id', null); + $this->assertSame($before, $this->snapshot($workflowTask, $target, $sibling)); + } + + public static function invalidScopeFacts(): array + { + return [ + 'missing context' => ['cancellation_scope', null], + 'wrong run' => ['workflow_run_id', 'other-run'], + 'wrong accepted request' => ['request_id', 'other-request'], + 'wrong scope membership' => ['scope_id', 'other-scope'], + 'malformed request event ID' => ['request_history_event_id', []], + 'inflated authority budget' => ['authority_deadline_at', '2099-01-01T00:00:00.000000Z'], + ]; + } + + #[DataProvider('invalidScopeFacts')] + public function test_malformed_scoped_fact_never_falls_back_to_later_whole_run_request(string $field, mixed $value): void + { + [$workflowTask, $target, $sibling, $request, , $fence] = $this->scopedPair(); + $this->assertTrue(WorkflowStub::load($target['workflow_id'])->requestCancellation('later', 90)->accepted()); + $event = WorkflowHistoryEvent::query()->findOrFail($fence['history_event_id']); + $payload = $event->payload; + if ($field === 'cancellation_scope') { + $payload[$field] = $value; + } else { + $payload['cancellation_scope'][$field] = $value; + } + $event->forceFill(['payload' => $payload])->save(); + $before = $this->snapshot($workflowTask, $target, $sibling); + $this->observe($target)->assertOk()->assertJsonPath('can_continue', false) + ->assertJsonPath('cancel_requested', true)->assertJsonMissingPath('cancellation_acknowledgement'); + $this->acknowledge($target, $request->payload['request_id'])->assertConflict()->assertJsonPath('acknowledged', false); + $this->assertSame($before, $this->snapshot($workflowTask, $target, $sibling)); + } + + public function test_scoped_receipt_remains_bound_to_original_worker_role_owner_attempt_and_namespace(): void + { + [$workflowTask, $target, $sibling, $request] = $this->scopedPair(); + WorkflowNamespace::query()->create(['name' => 'other', 'retention_days' => 30, 'status' => 'active']); + $before = $this->snapshot($workflowTask, $target, $sibling); + $this->withHeaders($this->headers(role: 'operator'))->postJson($this->path($target), $this->attempt($target))->assertForbidden(); + $this->withHeaders($this->headers(role: 'operator'))->postJson($this->path($target, 'acknowledge-cancellation'), $this->attempt($target) + ['request_id' => $request->payload['request_id']])->assertForbidden(); + $this->withHeaders($this->headers(namespace: 'other'))->postJson($this->path($target), $this->attempt($target))->assertNotFound(); + $this->withHeaders($this->headers(namespace: 'other'))->postJson($this->path($target, 'acknowledge-cancellation'), $this->attempt($target) + ['request_id' => $request->payload['request_id']])->assertNotFound(); + $this->observe($target, ['lease_owner' => 'other-owner'])->assertConflict()->assertJsonPath('reason', 'lease_owner_mismatch'); + $this->acknowledge($target, $request->payload['request_id'], ['lease_owner' => 'other-owner'])->assertConflict(); + $this->observe($target, ['activity_attempt_id' => $sibling['activity_attempt_id']])->assertConflict()->assertJsonPath('reason', 'task_mismatch'); + $this->acknowledge($target, $request->payload['request_id'], ['activity_attempt_id' => $sibling['activity_attempt_id']])->assertConflict(); + $this->assertSame($before, $this->snapshot($workflowTask, $target, $sibling)); + } + + /** Scope request remains internal; admission, preparation, dispatch and receipts use HTTP. */ + private function scopedPair(): array + { + $this->withHeaders($this->headers(role: 'operator')) + ->postJson('/api/workflows', ['workflow_type' => 'tests.external-greeting-workflow', 'task_queue' => 'scoped-receipt', 'input' => ['Ada']])->assertCreated(); + $this->withHeaders($this->headers())->postJson('/api/worker/register', [ + 'worker_id' => 'scope-owner', 'task_queue' => 'scoped-receipt', 'runtime' => 'php', + 'supported_workflow_types' => ['tests.external-greeting-workflow'], + 'supported_activity_types' => ['opaque-scoped-activity'], + 'capabilities' => [CooperativeCancellationPolicy::CAPABILITY], + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + 'max_concurrent_workflow_tasks' => 1, + 'process_metrics' => ['host' => 'fixture', 'process_started_at' => now()->toISOString(), 'process_id' => 10], + ])->assertCreated(); + $task = $this->withHeaders($this->headers())->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'scope-owner', 'task_queue' => 'scoped-receipt', + ])->assertOk()->json('task'); + $scopes = []; + foreach (range(1, 3) as $sequence) { + $scopes[] = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/open", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'sequence' => $sequence, 'parent_scope_id' => $sequence === 2 ? $scopes[0] : 'root', + ])->assertOk()->assertJsonPath('opened', true)->json('scope_id'); + } + $commands = []; + foreach ([$scopes[1], $scopes[2]] as $scope) { + $commands[] = ['type' => 'schedule_activity', 'activity_type' => 'opaque-scoped-activity', + 'arguments' => Serializer::serializeWithCodec('avro', ['Ada']), 'payload_codec' => 'avro', + 'queue' => 'scoped-receipt', 'cancellation_scope_id' => $scope, + 'cancellation_policy' => 'wait_cancellation_completed', 'schedule_to_close_timeout' => 120]; + } + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/checkpoint", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'checkpoint_id' => 'scoped-activities', 'start_sequence' => 4, 'commands' => $commands, + ])->assertOk()->assertJsonPath('checkpointed', true); + $activities = []; + foreach (range(1, 2) as $_) { + $activities[] = $this->withHeaders($this->headers())->postJson('/api/worker/activity-tasks/poll', [ + 'worker_id' => 'scope-owner', 'task_queue' => 'scoped-receipt', + ])->assertOk()->json('task'); + } + usort($activities, fn (array $a, array $b): int => ActivityExecution::query()->findOrFail($a['activity_execution_id'])->sequence + <=> ActivityExecution::query()->findOrFail($b['activity_execution_id'])->sequence); + $run = WorkflowRun::query()->findOrFail($task['run_id']); + $root = CancellationScopeRequests::request($run, $scopes[0], '1.20', 30, 'original scoped cleanup'); + $request = CancellationScopeRequests::request($run, $scopes[1], '1.20', 300, parentScopeId: $scopes[0]); + $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/prepare", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'scope_id' => $scopes[1], 'request_id' => $request->payload['request_id'], 'sequence' => 4, 'call_kind' => 'activity', + ])->assertOk()->assertJsonPath('prepared', true)->assertJsonPath('delivered', false); + $this->assertSame('running', ActivityExecution::query()->findOrFail($activities[0]['activity_execution_id'])->status->value); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('event_type', 'ActivityCancelled')->count()); + $response = $this->withHeaders($this->headers())->postJson("/api/worker/workflow-tasks/{$task['task_id']}/cancellation-scopes/deliver", [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'scope_id' => $scopes[1], 'request_id' => $request->payload['request_id'], 'sequence' => 4, 'call_kind' => 'activity', + ])->assertOk()->assertJsonPath('prepared', true)->assertJsonPath('delivered', false) + ->assertJsonPath('reason', 'cancellation_scope_activity_stop_not_acknowledged'); + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/CancellationScopeDeliveryResponse/allOf/1', + json_decode($response->getContent(), flags: JSON_THROW_ON_ERROR)); + $fence = $response->json('activity_cancellations.0'); + $this->assertSame($activities[0]['activity_execution_id'], $fence['activity_execution_id']); + $this->assertTrue($fence['fenced']); + $this->assertTrue($fence['waiting_for_stop']); + + return [$task, $activities[0], $activities[1], $request, $root, $fence]; + } + + private function headers(string $role = 'worker', string $namespace = 'default'): array + { + return ['Authorization' => 'Bearer fixture-'.$role, 'X-Namespace' => $namespace, + 'X-Durable-Workflow-Control-Plane-Version' => '2', WorkerProtocol::HEADER => '1.20']; + } + + private function path(array $task, string $action = 'status'): string + { + return "/api/worker/activity-tasks/{$task['task_id']}/{$action}"; + } + + private function attempt(array $task): array + { + return ['activity_attempt_id' => $task['activity_attempt_id'], 'lease_owner' => $task['lease_owner']]; + } + + private function observe(array $task, array $overrides = []): TestResponse + { + return $this->withHeaders($this->headers())->postJson($this->path($task), array_replace($this->attempt($task), $overrides)); + } + + private function acknowledge(array $task, string $requestId, array $overrides = []): TestResponse + { + return $this->withHeaders($this->headers())->postJson($this->path($task, 'acknowledge-cancellation'), + array_replace($this->attempt($task) + ['request_id' => $requestId], $overrides)); + } + + private function snapshot(array $workflowTask, array $target, array $sibling): array + { + return [WorkflowTask::query()->findOrFail($workflowTask['task_id'])->getRawOriginal(), + array_map(fn (array $task): array => [ + ActivityExecution::query()->findOrFail($task['activity_execution_id'])->getRawOriginal(), + ActivityAttempt::query()->findOrFail($task['activity_attempt_id'])->getRawOriginal(), + WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(), + ], [$target, $sibling]), + WorkflowRun::query()->findOrFail($target['run_id'])->getRawOriginal(), + WorkflowHistoryEvent::query()->where('workflow_run_id', $target['run_id'])->count()]; + } +} diff --git a/tests/TestCase.php b/tests/TestCase.php index 5fcfb67b..d4597676 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -4,6 +4,7 @@ use App\Support\WorkerProtocol; use Illuminate\Foundation\Testing\TestCase as BaseTestCase; +use Illuminate\Support\Carbon; use Workflow\V2\Models\WorkflowInstance; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowRunSummary; @@ -71,6 +72,19 @@ protected function portableWorkerAffinityRefusalManifest(): array ]); } + protected function assertRecoverableHostingClaim(array $before, string $taskId, bool $shortened = false): void + { + $claim = WorkflowTask::query()->findOrFail($taskId); + $after = $claim->getRawOriginal(); + $this->assertTrue($claim->lease_expires_at->gt(now())); + $this->assertTrue($claim->lease_expires_at->lte(now()->addSeconds(10))); + if ($shortened) { + $this->assertTrue($claim->lease_expires_at->lt(Carbon::parse($before['lease_expires_at'], 'UTC'))); + } + unset($before['lease_expires_at'], $before['updated_at'], $after['lease_expires_at'], $after['updated_at']); + $this->assertSame($before, $after); + } + private function pollingCachePath(): string { return sys_get_temp_dir().'/dw-server-test-polling-'.getmypid(); diff --git a/tests/Unit/RuntimePayloadCompletionContextTest.php b/tests/Unit/RuntimePayloadCompletionContextTest.php index 868eec12..308f76f2 100644 --- a/tests/Unit/RuntimePayloadCompletionContextTest.php +++ b/tests/Unit/RuntimePayloadCompletionContextTest.php @@ -79,6 +79,56 @@ public function test_oversized_header_is_rejected_before_json_decode(): void RuntimePayloadCompletionContext::parse(str_repeat(' ', 4097)); } + public function test_prepared_operations_and_legacy_completion_share_one_workflow_allowance(): void + { + $legacy = RuntimePayloadCompletionContext::parse(json_encode($this->value('workflow', 'complete', ['commands', 0, 'result']), JSON_THROW_ON_ERROR)); + $identities = []; + foreach ([ + ['local_activity_checkpoint', ['commands', 0, 'result'], ['checkpoint_id' => 'prefix-1']], + ['local_activity_checkpoint', ['commands', 0, 'result'], ['checkpoint_id' => 'prefix-2']], + ['cancellation_scope_checkpoint', ['commands', 0, 'result'], ['checkpoint_id' => 'prefix-1']], + ['cancellation_scope_checkpoint', ['commands', 0, 'result'], ['checkpoint_id' => 'prefix-2']], + ['local_activity_group_checkpoint', ['commands', 0, 'arguments'], ['checkpoint_id' => 'group-1']], + ['local_activity_group_checkpoint', ['commands', 1, 'arguments'], ['checkpoint_id' => 'group-1']], + ['local_activity_group_checkpoint', ['commands', 0, 'arguments'], ['checkpoint_id' => 'group-2']], + ['local_activity_prepare', ['descriptor', 'arguments'], ['sequence' => 2]], + ['local_activity_prepare', ['descriptor', 'arguments'], ['sequence' => 3]], + ['local_activity_recover', ['descriptor', 'arguments'], ['sequence' => 2]], + ['local_activity_outcome', ['report', 'result'], ['activity_attempt_id' => 'attempt-1']], + ['local_activity_outcome', ['report', 'result'], ['activity_attempt_id' => 'attempt-2']], + ] as [$operation, $slot, $identity]) { + $value = array_replace($this->value('workflow', $operation, $slot), ['schema' => RuntimePayloadCompletionContext::PREPARED_SCHEMA], $identity); + $context = RuntimePayloadCompletionContext::parse(json_encode($value, JSON_THROW_ON_ERROR)); + $this->assertSame($value, $context->toArray()); + $this->assertSame($legacy->scope('default'), $context->scope('default')); + $this->assertNotSame($context->scope('default'), $context->scope('other')); + $this->assertSame($context->slotIdentity(), RuntimePayloadCompletionContext::parse(json_encode(array_reverse($value, true), JSON_THROW_ON_ERROR))->slotIdentity()); + $identities[] = $context->slotIdentity(); + } + $this->assertCount(count($identities), array_unique($identities)); + } + + #[DataProvider('invalidPreparedContexts')] + public function test_prepared_context_requires_exact_claim_identity_and_operation_slot(array $changes): void + { + $this->expectException(RuntimeExternalPayloadException::class); + $value = array_replace($this->value('workflow', 'local_activity_prepare', ['descriptor', 'arguments']), + ['schema' => RuntimePayloadCompletionContext::PREPARED_SCHEMA, 'sequence' => 2], $changes); + RuntimePayloadCompletionContext::parse(json_encode($value, JSON_THROW_ON_ERROR)); + } + + public static function invalidPreparedContexts(): array + { + return array_map(static fn (array $changes): array => [$changes], [ + ['sequence' => 0], ['sequence' => '2'], ['sequence' => null], ['sequence' => []], + ['kind' => 'activity'], ['kind' => 'query'], ['operation' => 'complete'], + ['slot' => ['report', 'result']], ['attempt' => '1'], ['attempt' => 0], + ['checkpoint_id' => 'extra'], ['activity_attempt_id' => 'extra'], + ['operation' => 'local_activity_checkpoint'], ['operation' => 'local_activity_outcome'], + ['schema' => RuntimePayloadCompletionContext::SCHEMA], + ]); + } + private function value(string $kind = 'activity', string $operation = 'complete', array $slot = ['result']): array { return ['schema' => RuntimePayloadCompletionContext::SCHEMA, 'kind' => $kind, diff --git a/tests/Unit/WorkerPollFenceTest.php b/tests/Unit/WorkerPollFenceTest.php index d80ec470..bc6ec00e 100644 --- a/tests/Unit/WorkerPollFenceTest.php +++ b/tests/Unit/WorkerPollFenceTest.php @@ -73,6 +73,27 @@ public function test_worker_fence_rejects_superseded_registration_even_with_fres $this->assertFalse(WorkerPollFence::isCurrent($snapshot)); } + public function test_persisted_process_identity_ignores_key_order_and_preserves_strict_fencing(): void + { + $worker = $this->createWorker('php-worker-json-fence'); + $worker->forceFill(['process_metrics' => [ + 'host' => 'worker-host', 'process_started_at' => now()->toISOString(), 'process_id' => 123, + ]])->save(); + $snapshot = WorkerPollFence::snapshot($worker); + $snapshot['process_identity'] = array_reverse($snapshot['process_identity'], true); + + $this->assertTrue(WorkerPollFence::isCurrent($snapshot)); + $this->assertTrue(WorkerPollFence::isCurrentForUpdate($snapshot)); + foreach ([ + [...$snapshot['process_identity'], 'process_id' => 124], + [...$snapshot['process_identity'], 'process_id' => '123'], + [...$snapshot['process_identity'], 'extra' => 'unissued'], + array_diff_key($snapshot['process_identity'], ['host' => true]), + ] as $changedIdentity) { + $this->assertFalse(WorkerPollFence::isCurrent([...$snapshot, 'process_identity' => $changedIdentity])); + } + } + private function createWorker(string $workerId): WorkerRegistration { return WorkerRegistration::query()->create([ diff --git a/tests/Unit/WorkerProtocolOpenApiContractTest.php b/tests/Unit/WorkerProtocolOpenApiContractTest.php index 3298f1ef..1a878c08 100644 --- a/tests/Unit/WorkerProtocolOpenApiContractTest.php +++ b/tests/Unit/WorkerProtocolOpenApiContractTest.php @@ -3,9 +3,11 @@ namespace Tests\Unit; use App\Support\WorkerProtocol; +use PHPUnit\Framework\AssertionFailedError; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Symfony\Component\Yaml\Yaml; +use Tests\Support\OpenApiSchema; class WorkerProtocolOpenApiContractTest extends TestCase { @@ -41,6 +43,40 @@ public function test_http_spec_publishes_the_runtime_negotiation_contract(): voi ); } + public function test_prepared_cleanup_contract_preserves_root_shapes_and_requires_scoped_authority(): void + { + $schema = OpenApiSchema::fromFile(dirname(__DIR__, 2).'/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml'); + $rootProof = ['request_id' => 'request', 'delivery_history_event_id' => 'delivery']; + $rootSnapshot = [...$rootProof, 'root_request_id' => 'root-request', 'cleanup_deadline_at' => '2026-10-04T00:00:30.000000Z']; + $scopedSnapshot = [...$rootSnapshot, 'scope_id' => 'scope', 'operation_scope_id' => 'child-scope', + 'preparation_history_event_id' => 'preparation', 'authority_deadline_at' => '2026-10-04T00:00:15.000000Z']; + $schema->assertReferenceMatches('#/components/schemas/PreparedLocalCleanupProof', (object) $rootProof); + $schema->assertReferenceMatches('#/components/schemas/PreparedLocalCleanupProof', (object) [...$rootProof, 'scope_id' => 'scope']); + $schema->assertReferenceMatches('#/components/schemas/PreparedLocalCleanupSnapshot', null); + $schema->assertReferenceMatches('#/components/schemas/PreparedLocalCleanupSnapshot', (object) $rootSnapshot); + $schema->assertReferenceMatches('#/components/schemas/PreparedLocalCleanupSnapshot', (object) $scopedSnapshot); + } + + public static function invalidPreparedCleanupAuthority(): iterable + { + $proof = ['scope_id' => 'scope', 'request_id' => 'request', 'delivery_history_event_id' => 'delivery']; + $snapshot = [...$proof, 'root_request_id' => 'root-request', 'operation_scope_id' => 'child-scope', + 'preparation_history_event_id' => 'preparation', 'cleanup_deadline_at' => '2026-10-04T00:00:30.000000Z', + 'authority_deadline_at' => '2026-10-04T00:00:15.000000Z']; + yield 'invented proof deadline' => ['PreparedLocalCleanupProof', [...$proof, 'cleanup_deadline_at' => 'later']]; + yield 'null scope' => ['PreparedLocalCleanupProof', [...$proof, 'scope_id' => null]]; + yield 'missing preparation' => ['PreparedLocalCleanupSnapshot', array_diff_key($snapshot, ['preparation_history_event_id' => true])]; + yield 'missing ceiling' => ['PreparedLocalCleanupSnapshot', array_diff_key($snapshot, ['authority_deadline_at' => true])]; + } + + #[DataProvider('invalidPreparedCleanupAuthority')] + public function test_prepared_cleanup_contract_refuses_incomplete_or_worker_invented_authority(string $name, array $value): void + { + $this->expectException(AssertionFailedError::class); + OpenApiSchema::fromFile(dirname(__DIR__, 2).'/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml') + ->assertReferenceMatches('#/components/schemas/'.$name, (object) $value); + } + public function test_http_and_stream_specs_publish_the_runtime_negotiation_floor(): void { $streamSpecPath = dirname(__DIR__, 2).'/resources/platform-protocol-specs/worker-protocol-stream.asyncapi.yaml';