diff --git a/.github/pr-test-selection.json b/.github/pr-test-selection.json index 306535de5..a1135e7a2 100644 --- a/.github/pr-test-selection.json +++ b/.github/pr-test-selection.json @@ -20,7 +20,11 @@ "tests/Unit/V2/LaravelDependencySecurityPolicyTest.php", "tests/Unit/V2/PlatformProtocolSpecsTest.php", "tests/Unit/V2/PreparedLocalActivityReportTest.php", + "tests/Unit/V2/PreparedLocalActivityRetryTaskBridgeTest.php", + "tests/Unit/V2/PreparedLocalRetryDecisionTest.php", + "tests/Unit/V2/PreparedLocalRetryDescriptorTest.php", "tests/Unit/V2/PreparedLocalRetryPolicyTest.php", + "tests/Unit/V2/PreparedLocalRetryTimestampTest.php", "tests/Unit/V2/ReadmeBranchDestinationsTest.php", "tests/Unit/V2/ScopedCancellationContextTest.php", "tests/Unit/V2/SqliteTransactionTest.php", diff --git a/src/V2/Contracts/PreparedLocalActivityRetryTaskBridge.php b/src/V2/Contracts/PreparedLocalActivityRetryTaskBridge.php new file mode 100644 index 000000000..d6aefd216 --- /dev/null +++ b/src/V2/Contracts/PreparedLocalActivityRetryTaskBridge.php @@ -0,0 +1,74 @@ + $descriptor + * @return array + */ + public function prepareLocalActivityWithRetryPolicy( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + string $workerAttemptId, + array $descriptor, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; + + /** @param array $report + * @return array + */ + public function recordLocalActivityRetryOutcome( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + array $report, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; + + /** + * @return array + */ + public function controlLocalActivityWithRetryPolicy( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + bool $renewLease = false, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; + + /** @param array $progress + * @return array + */ + public function heartbeatLocalActivityWithRetryPolicy( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + array $progress = [], + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; + + /** @param array $descriptor + * @return array + */ + public function recoverLocalActivityWithRetryPolicy( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + array $descriptor, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; +} diff --git a/src/V2/Exceptions/PreparedLocalActivityContractException.php b/src/V2/Exceptions/PreparedLocalActivityContractException.php new file mode 100644 index 000000000..234ea52bb --- /dev/null +++ b/src/V2/Exceptions/PreparedLocalActivityContractException.php @@ -0,0 +1,12 @@ + [ 'local_outcome', + 'retry_disposition', 'local_recovery', 'activity_execution_id', 'activity_attempt_id', @@ -266,6 +267,7 @@ final class HistoryEventPayloadContract ], 'ActivityFailed' => [ 'local_outcome', + 'retry_disposition', 'local_recovery', 'activity_execution_id', 'activity_attempt_id', @@ -345,6 +347,7 @@ final class HistoryEventPayloadContract ], 'ActivityTimedOut' => [ 'local_outcome', + 'retry_disposition', 'local_recovery', 'activity_execution_id', 'activity_attempt_id', diff --git a/src/V2/Support/LocalActivityExecutor.php b/src/V2/Support/LocalActivityExecutor.php index 4398d6e70..0940f5422 100644 --- a/src/V2/Support/LocalActivityExecutor.php +++ b/src/V2/Support/LocalActivityExecutor.php @@ -4,6 +4,7 @@ namespace Workflow\V2\Support; +use Illuminate\Support\Carbon; use Illuminate\Support\Str; use Illuminate\Validation\ValidationException; use JsonException; @@ -18,6 +19,7 @@ use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Enums\TaskType; use Workflow\V2\Exceptions\ActivityTimeoutException; +use Workflow\V2\Exceptions\PreparedLocalActivityContractException; use Workflow\V2\Exceptions\RestoredWorkflowException; use Workflow\V2\Exceptions\StructuralLimitExceededException; use Workflow\V2\Models\ActivityAttempt; @@ -45,6 +47,320 @@ public function recordPortableOutcome( int $workflowTaskAttempt, array $report, string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return $this->recordPortableOutcomeForContract( + 1, + $attemptId, + $leaseOwner, + $workflowTaskAttempt, + $report, + $protocolVersion + ); + } + + /** @internal Unadvertised until the installed bridge and retained claim bind contract 2. + * @param array $report + * @return array + */ + public function recordPortableRetryOutcome( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + array $report, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return $this->recordPortableOutcomeForContract( + 2, + $attemptId, + $leaseOwner, + $workflowTaskAttempt, + $report, + $protocolVersion + ); + } + + /** + * @return array{status: 'completed'|'failed'|'waiting', event: WorkflowHistoryEvent|null, next_task: WorkflowTask|null} + */ + public function execute( + WorkflowRun $run, + WorkflowTask $task, + int $sequence, + LocalActivityCall $activityCall, + ): array { + $started = $this->prepareAttempt($run, $task, $sequence, $activityCall); + + if ($started['event'] instanceof WorkflowHistoryEvent) { + return [ + 'status' => $started['event']->event_type === HistoryEventType::ActivityCompleted + ? 'completed' + : 'failed', + 'event' => $started['event'], + 'next_task' => null, + ]; + } + + if ($started['next_task'] instanceof WorkflowTask) { + return [ + 'status' => 'waiting', + 'event' => null, + 'next_task' => $started['next_task'], + ]; + } + + /** @var ActivityExecution $execution */ + $execution = $started['execution']; + /** @var ActivityAttempt $attempt */ + $attempt = $started['attempt']; + + $operation = fn (): array => $this->invokeCallback($run, $task, $execution); + if (ExecutionTracing::enabled() && ! WorkflowStub::hasMock($execution->activity_class)) { + $storedContext = TraceCarrier::fromRun($run, $diagnostic); + [$result, $throwable] = ExecutionTracing::localActivity( + [ + 'workflow_instance_id' => $run->workflow_instance_id, + 'workflow_run_id' => $run->id, + 'workflow_type' => $run->workflow_type, + 'namespace' => $run->namespace, + 'task_id' => $task->id, + 'activity_id' => $execution->id, + 'activity_type' => $execution->activity_type, + 'activity_attempt_id' => $attempt->id, + 'activity_attempt_number' => $attempt->attempt_number, + 'durable.trace_context.disposition' => $diagnostic, + ], + $storedContext, + static function () use ($operation): array { + [$result, $throwable] = $operation(); + if ($throwable !== null) { + ExecutionTracing::recordFailure($throwable); + } + + return [$result, $throwable]; + }, + ExecutionTracing::baggageAllowlist(), + true, + ); + } else { + [$result, $throwable] = $operation(); + } + + return $this->recordAttemptOutcome($task, $execution, $attempt, $result, $throwable); + } + + /** + * @internal Portable preparation validates and locks the execution, run and + * hosting claim before calling this shared attempt recorder. + * @param array $preparation + */ + public function startPortableAttempt( + WorkflowRun $run, + WorkflowTask $task, + ActivityExecution $execution, + string $workerAttemptId, + array $preparation, + ?Carbon $preparedAt = null, + ): ActivityAttempt { + if (($execution->activity_options['local_activity_contract_version'] ?? null) === 2 + && (($preparation['version'] ?? null) !== 2 + || ($preparation['local_activity_contract_version'] ?? null) !== 2 || $preparedAt === null)) { + throw new PreparedLocalActivityContractException( + 'Prepared retry attempts require their versioned preparation and locked sample.' + ); + } + $preparation['schedule_to_close_deadline_at'] = $execution->schedule_to_close_deadline_at?->toISOString(); + return $this->startAttempt($run, $task, $execution, $workerAttemptId, $preparation, $preparedAt); + } + + /** + * @internal The caller holds canonical locks and has validated the retry + * authority and the expired original total deadline. No callback is admitted. + */ + public function expirePortableRetry( + WorkflowRun $run, + WorkflowTask $task, + ActivityExecution $execution, + int $localContractVersion = 1, + ?Carbon $sample = null, + ): WorkflowHistoryEvent { + if (! in_array($localContractVersion, [1, 2], true) + || ($execution->activity_options['local_activity_contract_version'] ?? 1) !== $localContractVersion) { + throw new PreparedLocalActivityContractException('Prepared retry expiry requires its original contract.'); + } + $decision = null; + $metadata = []; + if ($localContractVersion === 2) { + $attempt = self::currentAttempt($execution); + if (! $attempt instanceof ActivityAttempt || $sample === null) { + throw new PreparedLocalActivityContractException( + 'Prepared retry expiry requires its original attempt and locked sample.' + ); + } + $decision = self::selectPortableRetryDecision( + $execution, + $attempt, + [ + 'outcome' => 'timed_out', + ], + $sample, + 'schedule_to_close' + ); + $metadata['retry_disposition'] = $decision; + } + $outcome = $this->recordTimeoutOutcome( + $run, + $task, + $execution, + 'schedule_to_close', + $metadata, + $decision, + $sample + ); + + return $outcome['event']; + } + + /** + * @internal The caller holds canonical locks, has validated original start + * history and has proved loss of the previous claim. Lease expiry fences + * publication. It does not establish physical callback stop. + * @param array $recovery + * @return array{event: WorkflowHistoryEvent, next_task: WorkflowTask|null} + */ + public function recoverPortableAttempt( + WorkflowRun $run, + WorkflowTask $task, + ActivityExecution $execution, + ActivityAttempt $attempt, + array $recovery, + ): array { + if (($execution->activity_options['local_activity_contract_version'] ?? null) === 2) { + throw new PreparedLocalActivityContractException('Prepared retry recovery requires its versioned entry.'); + } + $metadata = [ + 'worker_attempt_id' => $attempt->worker_attempt_id, + 'local_recovery' => $recovery, + ]; + self::closeAttempt($attempt, ActivityAttemptStatus::Expired); + $timeoutKind = self::timeoutKind($execution); + $outcome = $timeoutKind === null + ? $this->recordInterruptedAttempt($run, $task, $execution, $metadata) + : $this->recordTimeoutOutcome($run, $task, $execution, $timeoutKind, $metadata); + return self::finishPortableRecovery($run, $task, $attempt, $outcome); + } + + /** + * @internal The caller has validated original snapshot/Started authority, + * locked the canonical graph and proved loss of the original claim using + * this one sample. Recovery does not admit or join a callback. + * @param array $recovery + * @return array{event: WorkflowHistoryEvent, next_task: WorkflowTask|null} + */ + public function recoverPortableRetryAttempt( + WorkflowRun $run, + WorkflowTask $task, + ActivityExecution $execution, + ActivityAttempt $attempt, + array $recovery, + Carbon $sample, + ): array { + if (($execution->activity_options['local_activity_contract_version'] ?? null) !== 2) { + throw new PreparedLocalActivityContractException('Prepared retry recovery requires its original contract.'); + } + $connection = ConfiguredV2Models::connection(); + PreparedLocalRetryTimestamp::addForConnection($sample, 0, $connection); + $timeout = self::timeoutKind($execution, $sample); + $decision = PreparedLocalRetryDecision::selectInterruption( + $execution->retry_policy, + $attempt->attempt_number, + $sample, + new PreparedLocalRetryWindow( + $execution->schedule_to_close_deadline_at, + PreparedLocalRetryTimestamp::latestForConnection($connection) + ), + $timeout + ); + $metadata = [ + 'worker_attempt_id' => $attempt->worker_attempt_id, + 'local_recovery' => $recovery, + 'retry_disposition' => $decision, + ]; + self::closeAttempt($attempt, ActivityAttemptStatus::Expired, $sample); + if ($timeout !== null) { + $outcome = $this->recordTimeoutOutcome($run, $task, $execution, $timeout, $metadata, $decision, $sample); + } else { + $throwable = new RuntimeException(PreparedLocalRetryDecision::INTERRUPTION_MESSAGE); + $outcome = $decision['decision'] === 'scheduled' ? [ + 'event' => null, + 'next_task' => $this->scheduleRetry( + $run, + $task, + $execution, + $attempt, + $throwable, + LocalActivityRuntime::RETRY_REASON_COLD_REPLAY, + historyMetadata: $metadata, + retryDecision: $decision, + failureTime: $sample, + ), + ] : [ + 'event' => $this->recordTerminalFailure( + $run, + $task, + $execution, + $attempt, + $throwable, + $metadata, + $sample + ), + 'next_task' => null, + ]; + } + + return self::finishPortableRecovery($run, $task, $attempt, $outcome); + } + + /** @param array{event: WorkflowHistoryEvent|null, next_task: WorkflowTask|null} $outcome + * @return array{event: WorkflowHistoryEvent, next_task: WorkflowTask|null} + */ + private static function finishPortableRecovery( + WorkflowRun $run, + WorkflowTask $task, + ActivityAttempt $attempt, + array $outcome, + ): array { + if ($outcome['next_task'] instanceof WorkflowTask) { + $task->forceFill([ + 'status' => TaskStatus::Completed, + 'lease_expires_at' => null, + ])->save(); + self::projectRun($run); + } + $event = $outcome['event'] ?? $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityRetryScheduled) + ->where('payload->activity_attempt_id', $attempt->id) + ->orderByDesc('sequence') + ->first(); + if (! $event instanceof WorkflowHistoryEvent) { + throw new RuntimeException('Portable local recovery did not record its canonical receipt.'); + } + + return [ + 'event' => $event, + 'next_task' => $outcome['next_task'], + ]; + } + + /** @param array $report + * @return array + */ + private function recordPortableOutcomeForContract( + int $localContractVersion, + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + array $report, + string $protocolVersion, ): array { $refused = static fn (string $reason): array => [ 'recorded' => false, @@ -58,11 +374,21 @@ public function recordPortableOutcome( || version_compare($protocolVersion, PortableLocalActivityPreparation::MINIMUM_PROTOCOL_VERSION, '<')) { return $refused('local_activity_outcome_requires_protocol_1_20'); } - try { - $normalized = self::normalizePortableReport($report); - $fingerprint = hash('sha256', json_encode($normalized, JSON_THROW_ON_ERROR)); - } catch (ValidationException|JsonException) { - return $refused('invalid_local_activity_outcome'); + $normalized = null; + $fingerprint = null; + if ($localContractVersion === 2 && array_key_exists('next_retry_delay_seconds', $report)) { + $reason = PreparedLocalActivityReport::refusal($report); + if ($reason !== null) { + return $refused($reason); + } + } + if ($localContractVersion === 1) { + try { + $normalized = self::normalizePortableReport($report); + $fingerprint = hash('sha256', json_encode($normalized, JSON_THROW_ON_ERROR)); + } catch (ValidationException|JsonException) { + return $refused('invalid_local_activity_outcome'); + } } return ConfiguredV2Models::connection()->transaction(function () use ( @@ -72,6 +398,8 @@ public function recordPortableOutcome( $normalized, $fingerprint, $refused, + $report, + $localContractVersion, ): array { $rows = ActivityRowLockOrder::lockForAttempt($attemptId); $attempt = $rows['attempt']; @@ -96,10 +424,23 @@ public function recordPortableOutcome( $attempt, $leaseOwner, $workflowTaskAttempt, + $localContractVersion, ); if ($started === null) { return $refused('local_activity_preparation_mismatch'); } + if ($localContractVersion === 2) { + $reason = PreparedLocalActivityReport::refusal($report); + if ($reason !== null) { + return $refused($reason); + } + try { + $normalized = self::normalizePortableReport($report, 2); + $fingerprint = hash('sha256', json_encode($normalized, JSON_THROW_ON_ERROR)); + } catch (ValidationException|JsonException) { + return $refused('invalid_local_activity_outcome'); + } + } $receipt = $run->historyEvents() ->whereIn('event_type', [HistoryEventType::ActivityCompleted, HistoryEventType::ActivityFailed, HistoryEventType::ActivityTimedOut, HistoryEventType::ActivityRetryScheduled]) @@ -110,15 +451,24 @@ public function recordPortableOutcome( if ($receipt instanceof WorkflowHistoryEvent) { if ($receipt->sequence <= $started->sequence || $receipt->workflow_task_id !== $task->id || ($receipt->payload['activity_execution_id'] ?? null) !== $execution->id - || ($receipt->payload['local_outcome']['version'] ?? null) !== 1 + || ($receipt->payload['local_outcome']['version'] ?? null) !== $localContractVersion || ($receipt->payload['local_outcome']['report_fingerprint'] ?? null) !== $fingerprint || ($receipt->payload['local_outcome']['workflow_task_attempt'] ?? null) !== $workflowTaskAttempt || ($receipt->payload['task']['lease_owner'] ?? null) !== $leaseOwner || ($receipt->payload['task']['attempt_count'] ?? null) !== $workflowTaskAttempt) { return $refused('local_activity_outcome_mismatch'); } + if ($localContractVersion === 2 && ! self::matchesStoredRetryReceipt( + $receipt, + $execution, + $attempt, + $report + )) { + return $refused('local_activity_outcome_mismatch'); + } return self::portableReceipt($receipt, true); } + $failureTime = $localContractVersion === 2 ? now() : null; if ($execution->current_attempt_id !== $attemptId || $execution->attempt_count !== $attempt->attempt_number) { return $refused('stale_activity_attempt'); } @@ -131,7 +481,7 @@ public function recordPortableOutcome( $execution->activity_options['cancellation_cleanup'], ); if ($run->cancellation_request_command_id !== null - && (! $runCleanup || now()->gte($run->cancellation_deadline_at))) { + && (! $runCleanup || ($failureTime ?? now())->gte($run->cancellation_deadline_at))) { $cancelled = $run->historyEvents() ->where('event_type', HistoryEventType::ActivityCancelled) ->where('payload->activity_execution_id', $execution->id) @@ -166,45 +516,65 @@ public function recordPortableOutcome( if ($run->status->isTerminal()) { return $refused('run_closed'); } - if (($run->execution_deadline_at !== null && now()->gte($run->execution_deadline_at)) - || ($run->run_deadline_at !== null && now()->gte($run->run_deadline_at))) { + if (($run->execution_deadline_at !== null && ($failureTime ?? now())->gte($run->execution_deadline_at)) + || ($run->run_deadline_at !== null && ($failureTime ?? now())->gte($run->run_deadline_at))) { return $refused('run_deadline_expired'); } - if (PortableLocalActivityCleanup::currentAuthorityExpired($run, $execution)) { + if (PortableLocalActivityCleanup::currentAuthorityExpired($run, $execution, $failureTime)) { return $refused('local_activity_cleanup_authority_expired'); } } - if ($task->lease_expires_at === null || now()->gte($task->lease_expires_at) - || $attempt->lease_expires_at === null || now() + if ($task->lease_expires_at === null || ($failureTime ?? now())->gte($task->lease_expires_at) + || $attempt->lease_expires_at === null || ($failureTime ?? now()) ->gte($attempt->lease_expires_at)) { return $refused('workflow_claim_expired'); } if ($run->status->isTerminal()) { return $refused('run_closed'); } - if (($run->execution_deadline_at !== null && now()->gte($run->execution_deadline_at)) - || ($run->run_deadline_at !== null && now()->gte($run->run_deadline_at))) { + if (($run->execution_deadline_at !== null && ($failureTime ?? now())->gte($run->execution_deadline_at)) + || ($run->run_deadline_at !== null && ($failureTime ?? now())->gte($run->run_deadline_at))) { return $refused('run_deadline_expired'); } $metadata = [ 'worker_attempt_id' => $attempt->worker_attempt_id, 'local_outcome' => [ - 'version' => 1, + 'version' => $localContractVersion, 'report_fingerprint' => $fingerprint, 'workflow_task_attempt' => $workflowTaskAttempt, 'submitted_outcome' => $normalized['outcome'], ], ]; - $timeoutKind = self::timeoutKind($execution); + $timeoutKind = self::timeoutKind($execution, $failureTime); + if ($failureTime !== null && ($timeoutKind !== null || $normalized['outcome'] === 'failed')) { + $decision = self::selectPortableRetryDecision( + $execution, + $attempt, + $report, + $failureTime, + $timeoutKind + ); + $metadata['retry_disposition'] = $decision; + } else { + $decision = null; + } if ($timeoutKind !== null) { - $outcome = $this->recordTimeoutOutcome($run, $task, $execution, $timeoutKind, $metadata); + $outcome = $this->recordTimeoutOutcome( + $run, + $task, + $execution, + $timeoutKind, + $metadata, + $decision, + $failureTime + ); } elseif ($normalized['outcome'] === 'timed_out') { return $refused('local_activity_timeout_not_due'); } elseif ($normalized['outcome'] === 'completed') { $outcome = $this->recordSuccess($run, $task, $execution, $attempt, null, [ 'blob' => $normalized['result'], 'codec' => $normalized['payload_codec'], - ], $metadata); + ], $metadata, $failureTime); } else { $failure = new RestoredWorkflowException([ 'class' => $normalized['exception_type'], @@ -213,7 +583,16 @@ public function recordPortableOutcome( 'non_retryable' => $normalized['non_retryable'], 'code' => 0, ]); - $outcome = $this->recordFailureOrRetry($run, $task, $execution, $attempt, $failure, $metadata); + $outcome = $this->recordFailureOrRetry( + $run, + $task, + $execution, + $attempt, + $failure, + $metadata, + $decision, + $failureTime + ); } if ($outcome['next_task'] instanceof WorkflowTask) { $task->forceFill([ @@ -235,150 +614,6 @@ public function recordPortableOutcome( }, 5); } - /** - * @return array{status: 'completed'|'failed'|'waiting', event: WorkflowHistoryEvent|null, next_task: WorkflowTask|null} - */ - public function execute( - WorkflowRun $run, - WorkflowTask $task, - int $sequence, - LocalActivityCall $activityCall, - ): array { - $started = $this->prepareAttempt($run, $task, $sequence, $activityCall); - - if ($started['event'] instanceof WorkflowHistoryEvent) { - return [ - 'status' => $started['event']->event_type === HistoryEventType::ActivityCompleted - ? 'completed' - : 'failed', - 'event' => $started['event'], - 'next_task' => null, - ]; - } - - if ($started['next_task'] instanceof WorkflowTask) { - return [ - 'status' => 'waiting', - 'event' => null, - 'next_task' => $started['next_task'], - ]; - } - - /** @var ActivityExecution $execution */ - $execution = $started['execution']; - /** @var ActivityAttempt $attempt */ - $attempt = $started['attempt']; - - $operation = fn (): array => $this->invokeCallback($run, $task, $execution); - if (ExecutionTracing::enabled() && ! WorkflowStub::hasMock($execution->activity_class)) { - $storedContext = TraceCarrier::fromRun($run, $diagnostic); - [$result, $throwable] = ExecutionTracing::localActivity( - [ - 'workflow_instance_id' => $run->workflow_instance_id, - 'workflow_run_id' => $run->id, - 'workflow_type' => $run->workflow_type, - 'namespace' => $run->namespace, - 'task_id' => $task->id, - 'activity_id' => $execution->id, - 'activity_type' => $execution->activity_type, - 'activity_attempt_id' => $attempt->id, - 'activity_attempt_number' => $attempt->attempt_number, - 'durable.trace_context.disposition' => $diagnostic, - ], - $storedContext, - static function () use ($operation): array { - [$result, $throwable] = $operation(); - if ($throwable !== null) { - ExecutionTracing::recordFailure($throwable); - } - - return [$result, $throwable]; - }, - ExecutionTracing::baggageAllowlist(), - true, - ); - } else { - [$result, $throwable] = $operation(); - } - - return $this->recordAttemptOutcome($task, $execution, $attempt, $result, $throwable); - } - - /** - * @internal Portable preparation validates and locks the execution, run and - * hosting claim before calling this shared attempt recorder. - * @param array $preparation - */ - public function startPortableAttempt( - WorkflowRun $run, - WorkflowTask $task, - ActivityExecution $execution, - string $workerAttemptId, - array $preparation, - ): ActivityAttempt { - $preparation['schedule_to_close_deadline_at'] = $execution->schedule_to_close_deadline_at?->toISOString(); - return $this->startAttempt($run, $task, $execution, $workerAttemptId, $preparation); - } - - /** - * @internal The caller holds canonical locks and has validated the retry - * authority and the expired original total deadline. No callback is admitted. - */ - public function expirePortableRetry( - WorkflowRun $run, - WorkflowTask $task, - ActivityExecution $execution, - ): WorkflowHistoryEvent { - $outcome = $this->recordTimeoutOutcome($run, $task, $execution, 'schedule_to_close'); - - return $outcome['event']; - } - - /** - * @internal The caller holds canonical locks, has validated original start - * history and has proved loss of the previous claim. Lease expiry fences - * publication. It does not establish physical callback stop. - * @param array $recovery - * @return array{event: WorkflowHistoryEvent, next_task: WorkflowTask|null} - */ - public function recoverPortableAttempt( - WorkflowRun $run, - WorkflowTask $task, - ActivityExecution $execution, - ActivityAttempt $attempt, - array $recovery, - ): array { - $metadata = [ - 'worker_attempt_id' => $attempt->worker_attempt_id, - 'local_recovery' => $recovery, - ]; - self::closeAttempt($attempt, ActivityAttemptStatus::Expired); - $timeoutKind = self::timeoutKind($execution); - $outcome = $timeoutKind === null - ? $this->recordInterruptedAttempt($run, $task, $execution, $metadata) - : $this->recordTimeoutOutcome($run, $task, $execution, $timeoutKind, $metadata); - if ($outcome['next_task'] instanceof WorkflowTask) { - $task->forceFill([ - 'status' => TaskStatus::Completed, - 'lease_expires_at' => null, - ])->save(); - self::projectRun($run); - } - $event = $outcome['event'] ?? $run->historyEvents() - ->where('event_type', HistoryEventType::ActivityRetryScheduled) - ->where('payload->activity_attempt_id', $attempt->id) - ->orderByDesc('sequence') - ->first(); - if (! $event instanceof WorkflowHistoryEvent) { - throw new RuntimeException('Portable local recovery did not record its canonical receipt.'); - } - - return [ - 'event' => $event, - 'next_task' => $outcome['next_task'], - ]; - } - /** * @return array{mixed, Throwable|null} */ @@ -430,12 +665,13 @@ private function invokeCallback(WorkflowRun $run, WorkflowTask $task, ActivityEx /** @param array $report * @return array */ - private static function normalizePortableReport(array $report): array + private static function normalizePortableReport(array $report, int $localContractVersion = 1): array { $outcome = $report['outcome'] ?? null; $fields = match ($outcome) { 'completed' => ['outcome', 'result', 'payload_codec'], - 'failed' => ['outcome', 'message', 'exception_type', 'non_retryable'], + 'failed' => ['outcome', 'message', 'exception_type', 'non_retryable', + ...($localContractVersion === 2 ? ['next_retry_delay_seconds'] : [])], 'timed_out' => ['outcome'], default => [], }; @@ -473,6 +709,10 @@ private static function normalizePortableReport(array $report): array 'message' => $message, 'exception_type' => $type, 'non_retryable' => $report['non_retryable'] ?? false, + ...(array_key_exists('next_retry_delay_seconds', $report) + ? [ + 'next_retry_delay_seconds' => $report['next_retry_delay_seconds'], + ] : []), ]; } @@ -504,9 +744,91 @@ private static function portableReceipt(WorkflowHistoryEvent $event, bool $dupli 'recorded_at' => $event->recorded_at->toISOString(), 'claim_released' => $event->event_type === HistoryEventType::ActivityRetryScheduled, 'created_task_ids' => is_string($retryTaskId) ? [$retryTaskId] : [], + ...(isset($payload['retry_disposition']) ? [ + 'retry_disposition' => $payload['retry_disposition'], + ] : []), ]; } + /** @param array $report + * @return array + */ + private static function selectPortableRetryDecision( + ActivityExecution $execution, + ActivityAttempt $attempt, + array $report, + Carbon $failureTime, + ?string $timeoutKind = null, + bool $nonRetryable = false, + ): array { + $connection = ConfiguredV2Models::connection(); + PreparedLocalRetryTimestamp::addForConnection($failureTime, 0, $connection); + + return PreparedLocalRetryDecision::select( + $execution->retry_policy, + $attempt->attempt_number, + $report, + $failureTime, + new PreparedLocalRetryWindow( + $execution->schedule_to_close_deadline_at, + PreparedLocalRetryTimestamp::latestForConnection($connection) + ), + $timeoutKind, + $nonRetryable, + ); + } + + /** + * @param array $report + */ + private static function matchesStoredRetryReceipt( + WorkflowHistoryEvent $event, + ActivityExecution $execution, + ActivityAttempt $attempt, + array $report, + ): bool { + $payload = $event->payload; + $outcome = $payload['local_outcome']; + if (count($outcome) !== 4 || array_diff(array_keys($outcome), [ + 'version', 'report_fingerprint', 'workflow_task_attempt', 'submitted_outcome', + ]) !== [] || $outcome['submitted_outcome'] !== $report['outcome']) { + return false; + } + if ($event->event_type === HistoryEventType::ActivityCompleted) { + return ! array_key_exists('retry_disposition', $payload); + } + $stored = $payload['retry_disposition'] ?? null; + if (! is_array($stored)) { + return false; + } + $nonRetryable = false; + if ($report['outcome'] === 'completed' && $event->event_type === HistoryEventType::ActivityFailed) { + if (($payload['failure_category'] ?? null) !== FailureCategory::StructuralLimit->value + || ! is_string($payload['message'] ?? null) || ! is_bool($payload['non_retryable'] ?? null)) { + return false; + } + $report = [ + 'outcome' => 'failed', + 'message' => $payload['message'], + ]; + $nonRetryable = $payload['non_retryable']; + } + $connection = ConfiguredV2Models::connection(); + + return PreparedLocalRetryDecision::matchesStored( + $execution->retry_policy, + $attempt->attempt_number, + $report, + new PreparedLocalRetryWindow( + $execution->schedule_to_close_deadline_at, + PreparedLocalRetryTimestamp::latestForConnection($connection), + PreparedLocalRetryTimestamp::earliestForDriver($connection->getDriverName()) + ), + $stored, + $nonRetryable, + ); + } + /** * @return array{execution: ActivityExecution|null, attempt: ActivityAttempt|null, event: WorkflowHistoryEvent|null, next_task: WorkflowTask|null} */ @@ -542,6 +864,11 @@ private function prepareAttempt( if (! $execution instanceof ActivityExecution) { $execution = $this->createExecution($lockedRun, $lockedTask, $sequence, $activityCall); } + if (($execution->activity_options['local_activity_contract_version'] ?? null) === 2) { + throw new PreparedLocalActivityContractException( + 'Prepared retry execution requires its versioned portable entry.' + ); + } if (! LocalActivityRuntime::isExecution($execution)) { throw new RuntimeException('Workflow history at this sequence belongs to an ordinary queued activity.'); @@ -662,8 +989,19 @@ private function startAttempt( ActivityExecution $execution, ?string $workerAttemptId = null, array $preparation = [], + ?Carbon $preparedAt = null, ): ActivityAttempt { - $now = now(); + $now = $preparedAt?->copy() ?? now(); + if (($execution->activity_options['local_activity_contract_version'] ?? null) === 2) { + PreparedLocalRetryPolicy::validateSnapshot($execution->retry_policy); + foreach (['start_to_close_timeout', 'heartbeat_timeout'] as $field) { + PreparedLocalRetryTimestamp::addForConnection( + $now, + $execution->retry_policy[$field] ?? 0, + ConfiguredV2Models::connection() + ); + } + } $attemptId = (string) Str::ulid(); $attemptNumber = ((int) $execution->attempt_count) + 1; $retryPolicy = is_array($execution->retry_policy) ? $execution->retry_policy : []; @@ -833,6 +1171,7 @@ private function recordSuccess( mixed $result, ?array $encodedResult = null, array $historyMetadata = [], + ?Carbon $failureTime = null, ): array { $encoded = $encodedResult ?? self::serializeWithCodec($result, self::preferredPayloadCodec($execution, $run)); $encoded['blob'] = ExternalPayloads::externalizeForNamespace( @@ -844,8 +1183,30 @@ private function recordSuccess( try { StructuralLimits::guardPayloadSize($encoded['blob']); } catch (StructuralLimitExceededException $limitExceeded) { + $decision = $failureTime === null ? null : self::selectPortableRetryDecision( + $execution, + $attempt, + [ + 'outcome' => 'failed', + 'message' => $limitExceeded->getMessage(), + ], + $failureTime, + nonRetryable: ActivityRetryPolicy::isNonRetryableFailure($execution, $limitExceeded), + ); + if ($decision !== null) { + $historyMetadata['retry_disposition'] = $decision; + } /** @var array{status: 'failed'|'waiting', event: WorkflowHistoryEvent|null, next_task: WorkflowTask|null} $failure */ - $failure = $this->recordFailureOrRetry($run, $task, $execution, $attempt, $limitExceeded, $historyMetadata); + $failure = $this->recordFailureOrRetry( + $run, + $task, + $execution, + $attempt, + $limitExceeded, + $historyMetadata, + $decision, + $failureTime + ); return $failure; } @@ -855,12 +1216,12 @@ private function recordSuccess( 'result' => $encoded['blob'], 'payload_codec' => $encoded['codec'], 'exception' => null, - 'closed_at' => now(), + 'closed_at' => $failureTime ?? now(), 'close_deadline_at' => null, 'heartbeat_deadline_at' => null, ])->save(); - self::closeAttempt($attempt, ActivityAttemptStatus::Completed); + self::closeAttempt($attempt, ActivityAttemptStatus::Completed, $failureTime); $event = WorkflowHistoryEvent::record( $run, @@ -914,11 +1275,14 @@ private function recordFailureOrRetry( ActivityAttempt $attempt, Throwable $throwable, array $historyMetadata = [], + ?array $retryDecision = null, + ?Carbon $failureTime = null, ): array { $attemptNumber = max(1, (int) $attempt->attempt_number); $maxAttempts = ActivityRetryPolicy::maxAttemptsFromSnapshot($execution); - if (! ActivityRetryPolicy::isNonRetryableFailure($execution, $throwable) && $attemptNumber < $maxAttempts) { + if ($retryDecision !== null ? $retryDecision['decision'] === 'scheduled' + : ! ActivityRetryPolicy::isNonRetryableFailure($execution, $throwable) && $attemptNumber < $maxAttempts) { $retry = $this->scheduleRetry( $run, $task, @@ -927,6 +1291,8 @@ private function recordFailureOrRetry( $throwable, LocalActivityRuntime::RETRY_REASON_FAILURE, historyMetadata: $historyMetadata, + retryDecision: $retryDecision, + failureTime: $failureTime, ); return [ @@ -936,7 +1302,15 @@ private function recordFailureOrRetry( ]; } - $event = $this->recordTerminalFailure($run, $task, $execution, $attempt, $throwable, $historyMetadata); + $event = $this->recordTerminalFailure( + $run, + $task, + $execution, + $attempt, + $throwable, + $historyMetadata, + $failureTime + ); return [ 'status' => 'failed', @@ -954,17 +1328,21 @@ private function scheduleRetry( string $retryReason, ?string $timeoutKind = null, array $historyMetadata = [], + ?array $retryDecision = null, + ?Carbon $failureTime = null, ): WorkflowTask { $attemptNumber = max(1, (int) ($attempt?->attempt_number ?? $execution->attempt_count)); $maxAttempts = ActivityRetryPolicy::maxAttemptsFromSnapshot($execution); - $backoffSeconds = ActivityRetryPolicy::backoffSecondsFromSnapshot($execution, $attemptNumber); - $retryAvailableAt = now() - ->addSeconds($backoffSeconds); + $backoffSeconds = $retryDecision['candidate_delay_seconds'] + ?? ActivityRetryPolicy::backoffSecondsFromSnapshot($execution, $attemptNumber); + $retryAvailableAt = $retryDecision === null ? now() + ->addSeconds($backoffSeconds) + : Carbon::parse($retryDecision['retry_available_at']); $exceptionPayload = FailureFactory::payload($throwable); $runCodec = is_string($run->payload_codec) && $run->payload_codec !== '' ? $run->payload_codec : null; if ($attempt instanceof ActivityAttempt && ! isset($historyMetadata['local_recovery'])) { - self::closeAttempt($attempt, ActivityAttemptStatus::Failed); + self::closeAttempt($attempt, ActivityAttemptStatus::Failed, $failureTime); } $execution->forceFill([ @@ -1044,6 +1422,7 @@ private function recordTerminalFailure( ?ActivityAttempt $attempt, Throwable $throwable, array $historyMetadata = [], + ?Carbon $failureTime = null, ): WorkflowHistoryEvent { $exceptionPayload = FailureFactory::payload($throwable); $failureCategory = $throwable instanceof StructuralLimitExceededException @@ -1072,13 +1451,13 @@ private function recordTerminalFailure( $runCodec ?? CodecRegistry::defaultCodec(), $exceptionPayload ), - 'closed_at' => now(), + 'closed_at' => $failureTime ?? now(), 'close_deadline_at' => null, 'heartbeat_deadline_at' => null, ])->save(); if ($attempt instanceof ActivityAttempt && ! isset($historyMetadata['local_recovery'])) { - self::closeAttempt($attempt, ActivityAttemptStatus::Failed); + self::closeAttempt($attempt, ActivityAttemptStatus::Failed, $failureTime); } $event = WorkflowHistoryEvent::record( @@ -1142,11 +1521,14 @@ private function recordTimeoutOutcome( ActivityExecution $execution, string $timeoutKind, array $historyMetadata = [], + ?array $retryDecision = null, + ?Carbon $failureTime = null, ): array { $attempt = self::currentAttempt($execution); $attemptNumber = max(1, (int) ($attempt?->attempt_number ?? $execution->attempt_count)); $maxAttempts = ActivityRetryPolicy::maxAttemptsFromSnapshot($execution); - $canRetry = $attemptNumber < $maxAttempts && $timeoutKind !== 'schedule_to_close'; + $canRetry = $retryDecision !== null ? $retryDecision['decision'] === 'scheduled' + : $attemptNumber < $maxAttempts && $timeoutKind !== 'schedule_to_close'; $throwable = new RuntimeException(self::timeoutMessage($execution, $timeoutKind)); if ($canRetry) { @@ -1161,15 +1543,17 @@ private function recordTimeoutOutcome( LocalActivityRuntime::RETRY_REASON_TIMEOUT, $timeoutKind, $historyMetadata, + $retryDecision, + $failureTime, ), ]; } if ($attempt instanceof ActivityAttempt && ! isset($historyMetadata['local_recovery'])) { - self::closeAttempt($attempt, ActivityAttemptStatus::Failed); + self::closeAttempt($attempt, ActivityAttemptStatus::Failed, $failureTime); } - $now = now(); + $now = $failureTime ?? now(); $failureCategory = FailureCategory::Timeout; $exceptionClass = ActivityTimeoutException::class; @@ -1335,18 +1719,21 @@ private static function currentAttempt(ActivityExecution $execution): ?ActivityA return $attempt; } - private static function closeAttempt(ActivityAttempt $attempt, ActivityAttemptStatus $status): void - { + private static function closeAttempt( + ActivityAttempt $attempt, + ActivityAttemptStatus $status, + ?Carbon $sample = null, + ): void { $attempt->forceFill([ 'status' => $status, 'lease_expires_at' => null, - 'closed_at' => $attempt->closed_at ?? now(), + 'closed_at' => $attempt->closed_at ?? $sample ?? now(), ])->save(); } - private static function timeoutKind(ActivityExecution $execution): ?string + private static function timeoutKind(ActivityExecution $execution, ?Carbon $sample = null): ?string { - $now = now(); + $now = $sample ?? now(); if ($execution->schedule_to_close_deadline_at !== null && $now->gte( $execution->schedule_to_close_deadline_at diff --git a/src/V2/Support/LocalActivityRuntime.php b/src/V2/Support/LocalActivityRuntime.php index 49e7e3d75..52c4ce46b 100644 --- a/src/V2/Support/LocalActivityRuntime.php +++ b/src/V2/Support/LocalActivityRuntime.php @@ -74,6 +74,7 @@ public static function renewWorkflowTask( ?CarbonInterface $deadline = null, bool $cleanup = false, ?WorkflowRun $run = null, + bool $validatePreparedRetryTimestamp = false, ): ?CarbonInterface { if ($task->task_type !== TaskType::Workflow || $task->status !== TaskStatus::Leased) { return null; @@ -86,6 +87,9 @@ public static function renewWorkflowTask( $leaseExpiresAt = $deadline === null ? ($cleanup ? CancellationCleanupLease::renewableExpiry() : CancellationCleanupLease::forScopes($run)) : CancellationCleanupLease::forDeadline($deadline); + if ($validatePreparedRetryTimestamp) { + PreparedLocalRetryTimestamp::addForConnection($leaseExpiresAt, 0, ConfiguredV2Models::connection()); + } $task->forceFill([ 'lease_expires_at' => $leaseExpiresAt, diff --git a/src/V2/Support/PortableLocalActivityCleanup.php b/src/V2/Support/PortableLocalActivityCleanup.php index 8a5609f13..7cc780afa 100644 --- a/src/V2/Support/PortableLocalActivityCleanup.php +++ b/src/V2/Support/PortableLocalActivityCleanup.php @@ -251,14 +251,17 @@ public static function currentDeadline(WorkflowRun $run, ?array $snapshot): ?Car return $current === null ? $deadline : $deadline->min(\Illuminate\Support\Carbon::parse($current)); } - public static function currentAuthorityExpired(WorkflowRun $run, ActivityExecution $execution): bool - { + public static function currentAuthorityExpired( + WorkflowRun $run, + ActivityExecution $execution, + ?CarbonInterface $sample = null, + ): bool { if (! self::isScoped($execution)) { return false; } $deadline = self::currentDeadline($run, $execution->activity_options['cancellation_cleanup']); - return $deadline !== null && $deadline->lt(self::deadline($execution)) && now()->gte($deadline); + return $deadline !== null && $deadline->lt(self::deadline($execution)) && ($sample ?? now())->gte($deadline); } public static function bound( diff --git a/src/V2/Support/PortableLocalActivityControl.php b/src/V2/Support/PortableLocalActivityControl.php index 83b85611e..561d2eb8c 100644 --- a/src/V2/Support/PortableLocalActivityControl.php +++ b/src/V2/Support/PortableLocalActivityControl.php @@ -39,6 +39,28 @@ public static function poll( return self::observe($attemptId, $leaseOwner, $workflowTaskAttempt, $renewLease, false, null, $protocolVersion); } + /** @internal Unadvertised versioned supervisor entry. + * @return array + */ + public static function pollWithRetryPolicy( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + bool $renewLease = false, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return self::observe( + $attemptId, + $leaseOwner, + $workflowTaskAttempt, + $renewLease, + false, + null, + $protocolVersion, + 2 + ); + } + /** @param array $progress * @return array */ @@ -58,6 +80,35 @@ public static function heartbeat( return self::observe($attemptId, $leaseOwner, $workflowTaskAttempt, false, true, $normalized, $protocolVersion); } + /** @internal Unadvertised versioned application heartbeat entry. + * @param array $progress + * @return array + */ + public static function heartbeatWithRetryPolicy( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + array $progress = [], + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + try { + $normalized = HeartbeatProgress::normalizeForWrite($progress); + } catch (LogicException) { + return self::response('invalid_local_activity_heartbeat'); + } + + return self::observe( + $attemptId, + $leaseOwner, + $workflowTaskAttempt, + false, + true, + $normalized, + $protocolVersion, + 2 + ); + } + /** @param array|null $progress * @return array */ @@ -69,6 +120,7 @@ private static function observe( bool $applicationHeartbeat, ?array $progress, string $protocolVersion, + int $localContractVersion = 1, ): array { $reply = self::lockedObservation( $attemptId, @@ -77,7 +129,8 @@ private static function observe( $renewLease, $applicationHeartbeat, $progress, - $protocolVersion + $protocolVersion, + $localContractVersion ); $pending = $reply['_scope_control'] ?? null; if (! is_array($pending)) { @@ -141,7 +194,8 @@ private static function observe( $renewLease, $applicationHeartbeat, $progress, - $protocolVersion + $protocolVersion, + $localContractVersion ); } @@ -156,6 +210,7 @@ private static function lockedObservation( bool $applicationHeartbeat, ?array $progress, string $protocolVersion, + int $localContractVersion = 1, ): array { if (preg_match('/^[0-9]+\.[0-9]+$/D', $protocolVersion) !== 1 || version_compare($protocolVersion, PortableLocalActivityPreparation::MINIMUM_PROTOCOL_VERSION, '<')) { @@ -172,6 +227,7 @@ private static function lockedObservation( $renewLease, $applicationHeartbeat, $progress, + $localContractVersion, ): array { $rows = ActivityRowLockOrder::lockForAttempt($attemptId); $attempt = $rows['attempt']; @@ -196,6 +252,7 @@ private static function lockedObservation( $attempt, $leaseOwner, $workflowTaskAttempt, + $localContractVersion, ); if ($started === null) { return self::response('local_activity_preparation_mismatch'); @@ -372,6 +429,7 @@ private static function lockedObservation( ) ? null : PortableLocalActivityCleanup::deadline($execution), $cleanup, $run, + validatePreparedRetryTimestamp: $localContractVersion === 2, ); $deadline = PortableLocalActivityCleanup::currentDeadline( $run, @@ -388,6 +446,13 @@ private static function lockedObservation( if ($applicationHeartbeat) { $heartbeatAt = now(); $timeout = $execution->retry_policy['heartbeat_timeout'] ?? null; + if ($localContractVersion === 2) { + PreparedLocalRetryTimestamp::addForConnection( + $heartbeatAt, + $timeout ?? 0, + ConfiguredV2Models::connection() + ); + } $execution->forceFill([ 'last_heartbeat_at' => $heartbeatAt, 'heartbeat_deadline_at' => PortableLocalActivityCleanup::bound( diff --git a/src/V2/Support/PortableLocalActivityPreparation.php b/src/V2/Support/PortableLocalActivityPreparation.php index 07d3f3021..e37db1d68 100644 --- a/src/V2/Support/PortableLocalActivityPreparation.php +++ b/src/V2/Support/PortableLocalActivityPreparation.php @@ -4,8 +4,11 @@ namespace Workflow\V2\Support; +use DateTimeImmutable; +use DateTimeZone; use Illuminate\Support\Carbon; use Illuminate\Validation\ValidationException; +use InvalidArgumentException; use JsonException; use Workflow\V2\Enums\ActivityAttemptStatus; use Workflow\V2\Enums\ActivityStatus; @@ -42,6 +45,744 @@ public static function prepare( string $workerAttemptId, array $descriptor, string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return self::prepareForContract( + 1, + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $sequence, + $workerAttemptId, + $descriptor, + $protocolVersion + ); + } + + /** + * @internal Unadvertised v2 primitive. A service adapter must separately + * prove its installed prepared role and original issued claim capability + * inside the containing transaction before invoking this method. + * + * @param array $descriptor + * @return array + */ + public static function prepareWithRetryPolicy( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + string $workerAttemptId, + array $descriptor, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return self::prepareForContract( + 2, + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $sequence, + $workerAttemptId, + $descriptor, + $protocolVersion + ); + } + + /** + * @internal The complete group was validated and the caller holds run/task + * locks inside its atomic checkpoint transaction. This creates no attempt. + * @param array $descriptor + */ + public static function admitGroupMember( + WorkflowRun $run, + WorkflowTask $task, + int $sequence, + array $descriptor, + string $checkpointId, + string $batchFingerprint + ): ActivityExecution { + $normalized = self::normalizeDescriptor($descriptor); + if (ConfiguredV2Models::connection()->transactionLevel() < 1 || ($normalized['parallel_group_path'] ?? []) === []) { + throw ValidationException::withMessages([ + 'local_activity' => ['Expected an atomic group transaction.'], + ]); + } + if (! CancellationScopeHistory::isRecordedBefore( + $run, + $normalized['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, + $sequence, + )) { + throw ValidationException::withMessages([ + 'local_activity.cancellation_scope_id' => ['Scope must be recorded in this run before its operation.'], + ]); + } + $cleanup = PortableLocalActivityCleanup::snapshot( + $run, + $normalized['cancellation_cleanup'] ?? null, + $sequence, + $normalized['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, + ); + $admission = [ + 'version' => 1, + 'descriptor_fingerprint' => hash('sha256', json_encode($normalized, JSON_THROW_ON_ERROR)), + 'checkpoint_id' => $checkpointId, + 'batch_fingerprint' => $batchFingerprint, + 'workflow_task_attempt' => $task->attempt_count, + ...($cleanup === null ? [] : [ + 'cancellation_cleanup' => $cleanup, + ]), + ]; + return self::createExecution( + $run, + $task, + $sequence, + $normalized, + $cleanup, + [ + 'local_group_admission' => $admission, + ] + ); + } + + /** + * @internal Recover an interrupted prepared call. This never admits an + * application callback. A scheduled retry releases this claim, and the SDK + * returns to polling. A terminal outcome is replayed on the retained claim. + * @param array $descriptor + * @return array + */ + public static function recover( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + array $descriptor, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return self::recoverForContract( + 1, + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $sequence, + $descriptor, + $protocolVersion + ); + } + + /** + * @internal Unadvertised recovery entry. The adapter must prove its + * installed retry role and the retained original capability/contract. + * Recovery fences publication; it does not prove physical callback stop. + * @param array $descriptor + * @return array + */ + public static function recoverWithRetryPolicy( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + array $descriptor, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return self::recoverForContract( + 2, + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $sequence, + $descriptor, + $protocolVersion + ); + } + + /** @param array $descriptor + * @return array + */ + public static function normalizeDescriptor(array $descriptor, int $localContractVersion = 1): array + { + if (! in_array($localContractVersion, [1, 2], true)) { + throw new InvalidArgumentException('Unknown prepared local activity contract.'); + } + $allowed = ['type', 'activity_type', 'arguments', 'payload_codec', 'retry_policy', + 'start_to_close_timeout', 'schedule_to_close_timeout', 'heartbeat_timeout', 'execution_mode', + 'cancellation_cleanup', 'cancellation_policy', 'cancellation_scope_id', 'parallel_group_id', 'parallel_group_kind', 'parallel_group_mode', + 'parallel_group_base_sequence', 'parallel_group_size', 'parallel_group_index', 'parallel_group_path']; + if (($descriptor['type'] ?? null) !== 'record_local_activity' + || array_diff(array_keys($descriptor), $allowed) !== [] + || (isset($descriptor['execution_mode']) && $descriptor['execution_mode'] !== LocalActivityRuntime::EXECUTION_MODE)) { + throw ValidationException::withMessages([ + 'local_activity' => ['Expected a local activity preparation descriptor.'], + ]); + } + if (array_key_exists('cancellation_policy', $descriptor) + && ! in_array($descriptor['cancellation_policy'], ['try_cancel', 'wait_cancellation_completed'], true)) { + throw ValidationException::withMessages([ + 'local_activity.cancellation_policy' => [ + 'Prepared local activities support try_cancel and wait_cancellation_completed. Abandon requires an independent callback lifetime.', + ], + ]); + } + if (array_key_exists('cancellation_scope_id', $descriptor) + && (! is_string($descriptor['cancellation_scope_id']) + || trim($descriptor['cancellation_scope_id']) === '' + || strlen($descriptor['cancellation_scope_id']) > 255 + || preg_match('//u', $descriptor['cancellation_scope_id']) !== 1)) { + throw ValidationException::withMessages([ + 'local_activity.cancellation_scope_id' => ['Expected a nonempty canonical scope identity.'], + ]); + } + // Common activity input/retry/timeout validation does not need a + // fabricated outcome or a claim that an application attempt ran. + $input = $descriptor; + unset($input['execution_mode'], $input['cancellation_cleanup'], $input['cancellation_scope_id']); + $retrySnapshot = null; + if ($localContractVersion === 2) { + if (! is_array($descriptor['retry_policy'] ?? null)) { + throw new InvalidArgumentException('Prepared contract 2 requires an explicit version-2 retry policy.'); + } + $retrySnapshot = PreparedLocalRetryPolicy::normalize($descriptor['retry_policy'], array_intersect_key( + $descriptor, + array_flip(['start_to_close_timeout', 'schedule_to_close_timeout', 'heartbeat_timeout']) + )); + unset($input['retry_policy']); + } elseif (is_array($descriptor['retry_policy'] ?? null) + && array_intersect( + array_keys($descriptor['retry_policy']), + ['contract_version', 'snapshot_version', 'mode', 'maximum_interval_seconds'] + ) !== []) { + throw ValidationException::withMessages([ + 'local_activity.retry_policy' => ['Versioned retry policies require their explicit prepared contract.'], + ]); + } + $input['type'] = 'schedule_activity'; + $normalized = WorkflowCommandNormalizer::normalize([$input], self::MINIMUM_PROTOCOL_VERSION)[0]; + if (! is_string($normalized['arguments'] ?? null) || ! is_string($normalized['payload_codec'] ?? null)) { + throw ValidationException::withMessages([ + 'local_activity.arguments' => ['Preparation requires encoded arguments.'], + ]); + } + $normalized['type'] = 'record_local_activity'; + $normalized['execution_mode'] = LocalActivityRuntime::EXECUTION_MODE; + if ($retrySnapshot !== null) { + $normalized['retry_policy'] = $retrySnapshot; + $normalized['local_activity_contract_version'] = 2; + } + if (array_key_exists('cancellation_scope_id', $descriptor)) { + $normalized['cancellation_scope_id'] = $descriptor['cancellation_scope_id']; + } + if (array_key_exists('cancellation_cleanup', $descriptor)) { + $proof = $descriptor['cancellation_cleanup']; + if (! is_array($proof) || array_diff( + array_keys($proof), + ['request_id', 'delivery_history_event_id', 'scope_id'] + ) !== [] + || ! is_string($proof['request_id'] ?? null) || trim($proof['request_id']) === '' + || ! is_string($proof['delivery_history_event_id'] ?? null) + || trim($proof['delivery_history_event_id']) === '' + || (array_key_exists('scope_id', $proof) && (! is_string($proof['scope_id']) + || trim( + $proof['scope_id'] + ) === '' || $proof['scope_id'] === CancellationScopeHistory::ROOT_SCOPE_ID))) { + throw ValidationException::withMessages([ + 'local_activity.cancellation_cleanup' => ['Expected canonical request and delivery identities.'], + ]); + } + $normalized['cancellation_cleanup'] = [ + 'request_id' => $proof['request_id'], + 'delivery_history_event_id' => $proof['delivery_history_event_id'], + ...(array_key_exists('scope_id', $proof) ? [ + 'scope_id' => $proof['scope_id'], + ] : []), + ]; + } + + return $normalized; + } + + /** + * @internal Read the original immutable preparation authority. The caller + * holds attempt/execution/run locks. Current task ownership is separate. + */ + public static function originalStart( + WorkflowRun $run, + ActivityExecution $execution, + ActivityAttempt $attempt, + string $leaseOwner, + int $workflowTaskAttempt, + int $localContractVersion = 1, + ): ?WorkflowHistoryEvent { + if (! LocalActivityRuntime::isExecution($execution) || $execution->workflow_run_id !== $run->id + || $attempt->workflow_run_id !== $run->id || $attempt->activity_execution_id !== $execution->id + || $attempt->lease_owner !== $leaseOwner || $workflowTaskAttempt < 1) { + return null; + } + $started = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted) + ->where('payload->activity_attempt_id', $attempt->id) + ->first(); + if (! $started instanceof WorkflowHistoryEvent + || $started->workflow_task_id !== $attempt->workflow_task_id + || ($started->payload['activity_execution_id'] ?? null) !== $execution->id + || ($started->payload['activity_type'] ?? null) !== $execution->activity_type + || ($started->payload['activity_class'] ?? null) !== $execution->activity_class + || ($started->payload['sequence'] ?? null) !== $execution->sequence + || ($started->payload['workflow_task_id'] ?? null) !== $attempt->workflow_task_id + || ($started->payload['local_activity'] ?? null) !== true + || ($started->payload['execution_mode'] ?? null) !== LocalActivityRuntime::EXECUTION_MODE + || ($started->payload['local_preparation']['version'] ?? null) !== $localContractVersion + || ! is_string($started->payload['local_preparation']['descriptor_fingerprint'] ?? null) + || ($started->payload['local_preparation']['worker_attempt_id'] ?? null) !== $attempt->worker_attempt_id + || ($started->payload['local_preparation']['workflow_task_attempt'] ?? null) !== $workflowTaskAttempt + || ($started->payload['task']['id'] ?? null) !== $attempt->workflow_task_id + || ($started->payload['task']['type'] ?? null) !== TaskType::Workflow->value + || ($started->payload['task']['status'] ?? null) !== TaskStatus::Leased->value + || ($started->payload['task']['attempt_count'] ?? null) !== $workflowTaskAttempt + || ($started->payload['task']['lease_owner'] ?? null) !== $leaseOwner + || ($started->payload['activity_attempt']['id'] ?? null) !== $attempt->id + || ($started->payload['activity_attempt']['activity_execution_id'] ?? null) !== $execution->id + || ($started->payload['activity_attempt']['task_id'] ?? null) !== $attempt->workflow_task_id + || ($started->payload['activity_attempt']['attempt_number'] ?? null) !== $attempt->attempt_number + || ($started->payload['activity_attempt']['lease_owner'] ?? null) !== $leaseOwner + || ($started->payload['activity_attempt']['worker_attempt_id'] ?? null) !== $attempt->worker_attempt_id) { + return null; + } + if ($localContractVersion === 2) { + if (($started->payload['local_preparation']['local_activity_contract_version'] ?? null) !== 2 + || ($execution->activity_options['local_activity_contract_version'] ?? null) !== 2 + || ! is_array($started->payload['activity']['retry_policy'] ?? null) + || ! is_array($execution->retry_policy)) { + return null; + } + try { + if (PreparedLocalRetryPolicy::validateSnapshot($execution->retry_policy) + !== PreparedLocalRetryPolicy::validateSnapshot($started->payload['activity']['retry_policy'])) { + return null; + } + } catch (InvalidArgumentException) { + return null; + } + } elseif ($localContractVersion !== 1 + || array_key_exists('local_activity_contract_version', $started->payload['local_preparation']) + || ($execution->retry_policy['snapshot_version'] ?? null) === 2) { + return null; + } + $scopeId = $execution->activity_options['cancellation_scope_id'] ?? null; + if (($started->payload['activity']['cancellation_scope_id'] ?? null) !== $scopeId) { + return null; + } + if ($scopeId !== null) { + $scheduled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityScheduled) + ->where('payload->activity_execution_id', $execution->id) + ->first(); + if (! is_string($scopeId) + || ! CancellationScopeHistory::isRecordedBefore($run, $scopeId, $execution->sequence) + || ($scheduled?->payload['activity']['cancellation_scope_id'] ?? null) !== $scopeId) { + return null; + } + } + if (array_key_exists('schedule_to_close_deadline_at', $started->payload['local_preparation']) + && $started->payload['local_preparation']['schedule_to_close_deadline_at'] + !== $execution->schedule_to_close_deadline_at?->toISOString()) { + return null; + } + if ((array_key_exists('cancellation_cleanup', $execution->activity_options ?? []) + || array_key_exists('cancellation_cleanup', $started->payload['local_preparation'])) + && ! PortableLocalActivityCleanup::isExecution($run, $execution, $started)) { + return null; + } + + return $started; + } + + private static function findRecoveryEvent( + string $runId, + string $taskId, + int $epoch, + string $owner, + int $sequence, + ): ?WorkflowHistoryEvent { + return ConfiguredV2Models::query('history_event_model', WorkflowHistoryEvent::class) + ->where('workflow_run_id', $runId) + ->whereIn('event_type', [HistoryEventType::ActivityRetryScheduled, HistoryEventType::ActivityFailed, + HistoryEventType::ActivityTimedOut]) + ->where('payload->sequence', $sequence) + ->where('payload->local_recovery->workflow_task_id', $taskId) + ->where('payload->local_recovery->workflow_task_attempt', $epoch) + ->where('payload->local_recovery->lease_owner', $owner) + ->orderBy('sequence') + ->first(); + } + + private static function recoveryStart( + WorkflowRun $run, + ActivityExecution $execution, + ActivityAttempt $attempt, + string $fingerprint, + int $contract, + ): ?WorkflowHistoryEvent { + $started = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted) + ->where('payload->activity_attempt_id', $attempt->id) + ->first(); + $epoch = $started?->payload['local_preparation']['workflow_task_attempt'] ?? null; + if (! is_int( + $epoch + ) || ($started->payload['local_preparation']['descriptor_fingerprint'] ?? null) !== $fingerprint) { + return null; + } + + return self::originalStart($run, $execution, $attempt, $attempt->lease_owner, $epoch, $contract); + } + + private static function matchesRetryRecoveryReceipt( + WorkflowHistoryEvent $event, + ActivityExecution $execution, + ActivityAttempt $attempt, + ?WorkflowHistoryEvent $started, + ): bool { + $payload = $event->payload; + $decision = $payload['retry_disposition'] ?? null; + $recovery = $payload['local_recovery'] ?? null; + if ($started === null || ! is_array($decision) || ! is_array($recovery) + || array_key_exists('local_outcome', $payload)) { + return false; + } + $connection = ConfiguredV2Models::connection(); + if (! PreparedLocalRetryDecision::matchesStoredInterruption( + $execution->retry_policy, + $attempt->attempt_number, + new PreparedLocalRetryWindow( + $execution->schedule_to_close_deadline_at, + PreparedLocalRetryTimestamp::latestForConnection($connection), + PreparedLocalRetryTimestamp::earliestForDriver($connection->getDriverName()) + ), + $decision + ) || ! self::recoveryLeaseMatches($recovery['original_lease_expires_at'] ?? null, $decision['failure_time'])) { + return false; + } + $expected = [ + 'version' => 1, + 'descriptor_fingerprint' => $started->payload['local_preparation']['descriptor_fingerprint'], + 'workflow_task_id' => $event->workflow_task_id, + 'workflow_task_attempt' => $payload['task']['attempt_count'], + 'lease_owner' => $payload['task']['lease_owner'], + 'original_workflow_task_id' => $attempt->workflow_task_id, + 'original_workflow_task_attempt' => $started->payload['local_preparation']['workflow_task_attempt'], + 'original_lease_owner' => $attempt->lease_owner, + 'original_lease_expires_at' => $recovery['original_lease_expires_at'], + 'callback_stop_state' => 'unknown', + ]; + ksort($expected); + ksort($recovery); + + return $expected === $recovery + && ($payload['activity_attempt_id'] ?? null) === $attempt->id + && ($payload['worker_attempt_id'] ?? null) === $attempt->worker_attempt_id + && ($payload[$event->event_type === HistoryEventType::ActivityRetryScheduled ? 'retry_after_attempt' : 'attempt_number'] ?? null) === $attempt->attempt_number + && ($payload['activity_attempt']['status'] ?? null) === ActivityAttemptStatus::Expired->value + && ($payload['activity_attempt']['closed_at'] ?? null) === $decision['failure_time'] + && self::recoveryEventMatches($event, $decision); + } + + private static function recoveryLeaseMatches(mixed $expiry, string $failure): bool + { + if (! is_string($expiry) || preg_match('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/D', $expiry) !== 1) { + return false; + } + $lease = DateTimeImmutable::createFromFormat('!Y-m-d\TH:i:s.u\Z', $expiry, new DateTimeZone('UTC')); + + return $lease instanceof DateTimeImmutable && $lease->format('Y-m-d\TH:i:s.u\Z') === $expiry + && $expiry <= $failure; + } + + /** + * @param array $decision + */ + private static function recoveryEventMatches(WorkflowHistoryEvent $event, array $decision): bool + { + $payload = $event->payload; + $timeout = $decision['failure_source'] === 'engine_timeout'; + $scheduled = $decision['decision'] === 'scheduled'; + $type = $scheduled ? HistoryEventType::ActivityRetryScheduled + : ($timeout ? HistoryEventType::ActivityTimedOut : HistoryEventType::ActivityFailed); + + return $event->event_type === $type + && ($payload['timeout_kind'] ?? null) === ($decision['timeout_kind'] ?? null) + && (! $scheduled || (($payload['retry_reason'] ?? null) === ($timeout ? LocalActivityRuntime::RETRY_REASON_TIMEOUT : LocalActivityRuntime::RETRY_REASON_COLD_REPLAY) + && is_string($payload['retry_task_id'] ?? null) && $payload['retry_task_id'] !== '' + && ($payload['retry_backoff_seconds'] ?? null) === $decision['candidate_delay_seconds'] + && ($payload['retry_available_at'] ?? null) === $decision['retry_available_at'])) + && ($scheduled || ! array_key_exists('retry_task_id', $payload)) + && ($timeout || (($payload['exception_class'] ?? null) === \RuntimeException::class + && ($payload['message'] ?? null) === PreparedLocalRetryDecision::INTERRUPTION_MESSAGE)); + } + + /** @param array $descriptor + * @return array + */ + private static function recoverForContract( + int $localContractVersion, + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + array $descriptor, + string $protocolVersion, + ): array { + $refused = static fn (string $reason): array => [ + 'recovered' => false, + 'duplicate' => false, + 'reason' => $reason, + 'claim_released' => false, + 'created_task_ids' => [], + ]; + if (preg_match('/^[0-9]+\.[0-9]+$/D', $protocolVersion) !== 1 + || version_compare($protocolVersion, self::MINIMUM_PROTOCOL_VERSION, '<')) { + return $refused('local_activity_recovery_requires_protocol_1_20'); + } + if ($workflowTaskAttempt < 1 || $sequence < 1 || $leaseOwner === '') { + return $refused('invalid_local_activity_recovery'); + } + try { + $fingerprint = hash( + 'sha256', + json_encode(self::normalizeDescriptor($descriptor, $localContractVersion), JSON_THROW_ON_ERROR) + ); + } catch (ValidationException|JsonException|InvalidArgumentException) { + return $refused('invalid_local_activity_recovery'); + } + /** @var WorkflowTask|null $snapshotTask */ + $snapshotTask = ConfiguredV2Models::query('task_model', WorkflowTask::class)->find($taskId); + /** @var ActivityExecution|null $snapshotExecution */ + $snapshotExecution = $snapshotTask === null ? null : ActivityExecution::query() + ->where('workflow_run_id', $snapshotTask->workflow_run_id) + ->where('sequence', $sequence) + ->first(); + if ($snapshotTask === null || $snapshotExecution === null + || ! is_string($snapshotExecution->current_attempt_id)) { + return $refused('local_activity_preparation_not_found'); + } + $snapshotAttemptId = $snapshotExecution->current_attempt_id; + if ($localContractVersion === 2) { + $prior = self::findRecoveryEvent( + $snapshotTask->workflow_run_id, + $taskId, + $workflowTaskAttempt, + $leaseOwner, + $sequence + ); + if ($prior !== null) { + $snapshotAttemptId = $prior->payload['activity_attempt_id'] ?? null; + if (! is_string($snapshotAttemptId) || $snapshotAttemptId === '') { + return $refused('local_activity_recovery_mismatch'); + } + } + } + return ConfiguredV2Models::connection()->transaction(static function () use ( + $localContractVersion, + $snapshotAttemptId, + $snapshotTask, + $snapshotExecution, + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $sequence, + $fingerprint, + $refused, + ): array { + $rows = ActivityRowLockOrder::lockForAttempt($snapshotAttemptId); + $execution = $rows['execution']; + $attempt = $rows['attempt']; + /** @var WorkflowRun|null $run */ + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->lockForUpdate() + ->find($snapshotTask->workflow_run_id); + /** @var WorkflowTask|null $task */ + $task = ConfiguredV2Models::query('task_model', WorkflowTask::class) + ->lockForUpdate() + ->find($taskId); + if (! $execution instanceof ActivityExecution || ! $attempt instanceof ActivityAttempt + || $execution->id !== $snapshotExecution->id || $execution->sequence !== $sequence + || $run === null || $execution->workflow_run_id !== $run->id + || $task === null || $task->workflow_run_id !== $run->id) { + return $refused('local_activity_preparation_not_found'); + } + $started = null; + $originalTask = null; + if ($localContractVersion === 2) { + $started = self::recoveryStart($run, $execution, $attempt, $fingerprint, 2); + if ($started === null) { + return $refused('local_activity_preparation_mismatch'); + } + $originalTask = ConfiguredV2Models::query('task_model', WorkflowTask::class) + ->lockForUpdate() + ->find($attempt->workflow_task_id); + if (! $originalTask instanceof WorkflowTask || $originalTask->workflow_run_id !== $run->id + || $originalTask->task_type !== TaskType::Workflow) { + return $refused('original_workflow_claim_not_found'); + } + } + $receipt = self::findRecoveryEvent($run->id, $taskId, $workflowTaskAttempt, $leaseOwner, $sequence); + if ($receipt instanceof WorkflowHistoryEvent) { + if (($receipt->payload['local_recovery']['version'] ?? null) !== 1 + || ($receipt->payload['local_recovery']['descriptor_fingerprint'] ?? null) !== $fingerprint + || ($receipt->payload['activity_execution_id'] ?? null) !== $execution->id + || $receipt->workflow_task_id !== $taskId + || ($receipt->payload['task']['id'] ?? null) !== $taskId + || ($receipt->payload['task']['lease_owner'] ?? null) !== $leaseOwner + || ($receipt->payload['task']['attempt_count'] ?? null) !== $workflowTaskAttempt + || ($receipt->payload['local_recovery']['callback_stop_state'] ?? null) !== 'unknown' + || ($localContractVersion === 2 + && ! self::matchesRetryRecoveryReceipt($receipt, $execution, $attempt, $started))) { + return $refused('local_activity_recovery_mismatch'); + } + return self::recoveryReceipt($receipt, true); + } + $sample = $localContractVersion === 2 ? now() : null; + if ($sample !== null) { + PreparedLocalRetryTimestamp::addForConnection($sample, 0, ConfiguredV2Models::connection()); + } + if ($task->task_type !== TaskType::Workflow || $task->status !== TaskStatus::Leased + || $task->lease_owner !== $leaseOwner || $task->attempt_count !== $workflowTaskAttempt) { + return $refused('workflow_claim_mismatch'); + } + if ($task->lease_expires_at === null || ($sample ?? now())->gte($task->lease_expires_at)) { + return $refused('workflow_claim_expired'); + } + $started ??= $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted) + ->where('payload->activity_attempt_id', $attempt->id) + ->first(); + $originalTaskAttempt = $started?->payload['local_preparation']['workflow_task_attempt'] ?? null; + if (! is_int($originalTaskAttempt) + || self::originalStart( + $run, + $execution, + $attempt, + $attempt->lease_owner, + $originalTaskAttempt, + $localContractVersion + ) === null + || ($started->payload['local_preparation']['descriptor_fingerprint'] ?? null) !== $fingerprint + || $execution->current_attempt_id !== $attempt->id || $execution->attempt_count !== $attempt->attempt_number) { + return $refused('local_activity_preparation_mismatch'); + } + // Accepted cancellation may fence immediately, without waiting for + // lease expiry. The replacement claim and root budget stay intact. + $cleanup = PortableLocalActivityCleanup::isExecution($run, $execution, $started); + if ($cleanup && ($sample ?? now())->gte(PortableLocalActivityCleanup::deadline($execution))) { + return $refused(PortableLocalActivityCleanup::isScoped($execution) + ? 'local_activity_cleanup_deadline_expired' : 'cancellation_deadline_expired'); + } + if ($run->cancellation_request_command_id !== null && (! $cleanup + || ! PortableLocalActivityCleanup::belongsToRunRequest( + $run, + $execution->activity_options['cancellation_cleanup'] + ))) { + $cancelled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancelled) + ->where('payload->activity_attempt_id', $attempt->id) + ->where('payload->workflow_command_id', $run->cancellation_request_command_id) + ->first(); + if (! $cancelled instanceof WorkflowHistoryEvent) { + if ($execution->status !== ActivityStatus::Running || $attempt->status !== ActivityAttemptStatus::Running) { + return $refused('stale_activity_attempt'); + } + $cancelled = ActivityCancellation::record( + $run, + $execution, + command: $run->cancellation_request_command_id + ); + } + return [ + ...$refused('cancellation_requested'), + 'fenced' => true, + 'cancellation_history_event_id' => $cancelled?->id, + ]; + } + if ($run->status->isTerminal()) { + return $refused('run_closed'); + } + if (($run->execution_deadline_at !== null && ($sample ?? now())->gte($run->execution_deadline_at)) + || ($run->run_deadline_at !== null && ($sample ?? now())->gte($run->run_deadline_at))) { + return $refused('run_deadline_expired'); + } + if ($execution->status !== ActivityStatus::Running || $attempt->status !== ActivityAttemptStatus::Running) { + return $refused('local_activity_previous_attempt_unresolved'); + } + if (PortableLocalActivityCleanup::currentAuthorityExpired($run, $execution, $sample)) { + return $refused('local_activity_cleanup_authority_expired'); + } + if ($attempt->lease_expires_at === null || ($sample ?? now())->lt($attempt->lease_expires_at)) { + return $refused('local_activity_previous_attempt_live'); + } + /** @var WorkflowTask|null $originalTask */ + $originalTask ??= ConfiguredV2Models::query('task_model', WorkflowTask::class) + ->lockForUpdate() + ->find($attempt->workflow_task_id); + if ($originalTask === null || $originalTask->workflow_run_id !== $run->id + || $originalTask->task_type !== TaskType::Workflow) { + return $refused('original_workflow_claim_not_found'); + } + if ($originalTask->id === $task->id) { + if ($task->attempt_count <= $originalTaskAttempt) { + return $refused('local_activity_original_claim_not_reclaimed'); + } + } elseif ($originalTask->status === TaskStatus::Leased && $originalTask->lease_expires_at !== null + && ($sample ?? now()) + ->lt($originalTask->lease_expires_at)) { + return $refused('local_activity_original_claim_live'); + } + $recovery = [ + 'version' => 1, + 'descriptor_fingerprint' => $fingerprint, + 'workflow_task_id' => $task->id, + 'workflow_task_attempt' => $workflowTaskAttempt, + 'lease_owner' => $leaseOwner, + 'original_workflow_task_id' => $originalTask->id, + 'original_workflow_task_attempt' => $originalTaskAttempt, + 'original_lease_owner' => $attempt->lease_owner, + 'original_lease_expires_at' => $attempt->lease_expires_at->toISOString(), + 'callback_stop_state' => 'unknown', + ]; + if ($originalTask->id !== $task->id && in_array( + $originalTask->status, + [TaskStatus::Ready, TaskStatus::Leased], + true + )) { + $originalTask->forceFill([ + 'status' => TaskStatus::Completed, + 'lease_expires_at' => null, + ])->save(); + } + $executor = app(LocalActivityExecutor::class); + $outcome = $sample === null ? $executor->recoverPortableAttempt( + $run, + $task, + $execution, + $attempt, + $recovery + ) + : $executor->recoverPortableRetryAttempt($run, $task, $execution, $attempt, $recovery, $sample); + + return self::recoveryReceipt($outcome['event'], false); + }, 5); + } + + /** + * @param array $descriptor + * @return array + */ + private static function prepareForContract( + int $localContractVersion, + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + string $workerAttemptId, + array $descriptor, + string $protocolVersion, ): array { if (preg_match('/^[0-9]+\.[0-9]+$/D', $protocolVersion) !== 1 || version_compare($protocolVersion, self::MINIMUM_PROTOCOL_VERSION, '<')) { @@ -53,10 +794,15 @@ public static function prepare( return self::response('invalid_local_activity_preparation'); } try { - $normalized = self::normalizeDescriptor($descriptor); + $normalized = self::normalizeDescriptor($descriptor, $localContractVersion); $fingerprint = hash('sha256', json_encode($normalized, JSON_THROW_ON_ERROR)); } catch (ValidationException|JsonException) { return self::response('invalid_local_activity_preparation'); + } catch (InvalidArgumentException $error) { + if ($localContractVersion === 1) { + throw $error; + } + return self::response('invalid_local_activity_preparation'); } /** @var WorkflowTask|null $snapshotTask */ $snapshotTask = ConfiguredV2Models::query('task_model', WorkflowTask::class)->find($taskId); @@ -79,6 +825,7 @@ public static function prepare( $workerAttemptId, $normalized, $fingerprint, + $localContractVersion, ): array { // Follow the shared attempt/execution/run/task lock order whenever // an execution exists. An unseen concurrent creation is retried. @@ -98,11 +845,12 @@ public static function prepare( if ($run === null || $task === null || $task->workflow_run_id !== $run->id) { return self::response('workflow_claim_not_found'); } + $preparedAt = $localContractVersion === 2 ? now() : null; if ($task->task_type !== TaskType::Workflow || $task->status !== TaskStatus::Leased || $task->lease_owner !== $leaseOwner || $task->attempt_count !== $workflowTaskAttempt) { return self::response('workflow_claim_mismatch'); } - if ($task->lease_expires_at === null || now()->gte($task->lease_expires_at)) { + if ($task->lease_expires_at === null || ($preparedAt ?? now())->gte($task->lease_expires_at)) { return self::response('workflow_claim_expired'); } if ($run->status->isTerminal()) { @@ -142,20 +890,28 @@ public static function prepare( if ($run->cancellation_request_command_id === null && isset($normalized['cancellation_cleanup']) && ! $scopedCleanup) { return self::response('local_activity_cleanup_authority_mismatch'); } - if ($cleanup !== null && now()->gte(PortableLocalActivityCleanup::snapshotDeadline($cleanup))) { + if ($cleanup !== null && ($preparedAt ?? now())->gte( + PortableLocalActivityCleanup::snapshotDeadline($cleanup) + )) { return self::response( $scopedCleanup ? 'local_activity_cleanup_deadline_expired' : 'cancellation_deadline_expired' ); } - if (($run->execution_deadline_at !== null && now()->gte($run->execution_deadline_at)) - || ($run->run_deadline_at !== null && now()->gte($run->run_deadline_at))) { + if (($run->execution_deadline_at !== null && ($preparedAt ?? now())->gte($run->execution_deadline_at)) + || ($run->run_deadline_at !== null && ($preparedAt ?? now())->gte($run->run_deadline_at))) { return self::response('run_deadline_expired'); } - if ($scopedCleanup && now()->gte(PortableLocalActivityCleanup::currentDeadline($run, $cleanup))) { + if ($scopedCleanup && ($preparedAt ?? now())->gte( + PortableLocalActivityCleanup::currentDeadline($run, $cleanup) + )) { return self::response('local_activity_cleanup_authority_expired'); } $execution = $rows['execution'] ?? null; $attempt = $rows['attempt'] ?? null; + if ($execution instanceof ActivityExecution + && ($execution->activity_options['local_activity_contract_version'] ?? 1) !== $localContractVersion) { + return self::response('local_activity_preparation_mismatch'); + } if ($snapshotExecution !== null) { if (! $execution instanceof ActivityExecution || $execution->id !== $snapshotExecution->id || $execution->sequence !== $sequence @@ -176,474 +932,76 @@ public static function prepare( return self::response('local_activity_previous_attempt_unresolved'); } if ($execution->status === ActivityStatus::Pending) { - return self::prepareRetry($run, $task, $execution, $attempt, $workerAttemptId, $fingerprint); - } - return self::duplicate($run, $task, $execution, $attempt, $workerAttemptId, $fingerprint); - } - if (ActivityExecution::query()->where('workflow_run_id', $run->id)->where( - 'sequence', - $sequence - )->exists()) { - return self::response('local_activity_preparation_retry'); - } - if (WorkflowStepHistory::nextDurableCommandSequence($run) !== $sequence) { - return self::response('local_activity_command_prefix_not_recorded'); - } - if (($normalized['parallel_group_path'] ?? []) !== []) { - return self::response('local_activity_group_not_admitted'); - } - $preparation = [ - 'version' => 1, - 'descriptor_fingerprint' => $fingerprint, - 'worker_attempt_id' => $workerAttemptId, - 'workflow_task_attempt' => $workflowTaskAttempt, - ...($cleanup === null ? [] : [ - 'cancellation_cleanup' => $cleanup, - ]), - ]; - $execution = self::createExecution( - $run, - $task, - $sequence, - $normalized, - $cleanup, - [ - 'local_preparation' => $preparation, - ] - ); - $attempt = app(LocalActivityExecutor::class)->startPortableAttempt( - $run, - $task, - $execution, - $workerAttemptId, - $preparation - ); - - return self::response(null, $execution, $attempt, task: $task); - }, 5); - } - - /** - * @internal The complete group was validated and the caller holds run/task - * locks inside its atomic checkpoint transaction. This creates no attempt. - * @param array $descriptor - */ - public static function admitGroupMember( - WorkflowRun $run, - WorkflowTask $task, - int $sequence, - array $descriptor, - string $checkpointId, - string $batchFingerprint - ): ActivityExecution { - $normalized = self::normalizeDescriptor($descriptor); - if (ConfiguredV2Models::connection()->transactionLevel() < 1 || ($normalized['parallel_group_path'] ?? []) === []) { - throw ValidationException::withMessages([ - 'local_activity' => ['Expected an atomic group transaction.'], - ]); - } - if (! CancellationScopeHistory::isRecordedBefore( - $run, - $normalized['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, - $sequence, - )) { - throw ValidationException::withMessages([ - 'local_activity.cancellation_scope_id' => ['Scope must be recorded in this run before its operation.'], - ]); - } - $cleanup = PortableLocalActivityCleanup::snapshot( - $run, - $normalized['cancellation_cleanup'] ?? null, - $sequence, - $normalized['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, - ); - $admission = [ - 'version' => 1, - 'descriptor_fingerprint' => hash('sha256', json_encode($normalized, JSON_THROW_ON_ERROR)), - 'checkpoint_id' => $checkpointId, - 'batch_fingerprint' => $batchFingerprint, - 'workflow_task_attempt' => $task->attempt_count, - ...($cleanup === null ? [] : [ - 'cancellation_cleanup' => $cleanup, - ]), - ]; - return self::createExecution( - $run, - $task, - $sequence, - $normalized, - $cleanup, - [ - 'local_group_admission' => $admission, - ] - ); - } - - /** - * @internal Recover an interrupted prepared call. This never admits an - * application callback. A scheduled retry releases this claim, and the SDK - * returns to polling. A terminal outcome is replayed on the retained claim. - * @param array $descriptor - * @return array - */ - public static function recover( - string $taskId, - string $leaseOwner, - int $workflowTaskAttempt, - int $sequence, - array $descriptor, - string $protocolVersion = WorkerProtocolVersion::VERSION, - ): array { - $refused = static fn (string $reason): array => [ - 'recovered' => false, - 'duplicate' => false, - 'reason' => $reason, - 'claim_released' => false, - 'created_task_ids' => [], - ]; - if (preg_match('/^[0-9]+\.[0-9]+$/D', $protocolVersion) !== 1 - || version_compare($protocolVersion, self::MINIMUM_PROTOCOL_VERSION, '<')) { - return $refused('local_activity_recovery_requires_protocol_1_20'); - } - if ($workflowTaskAttempt < 1 || $sequence < 1 || $leaseOwner === '') { - return $refused('invalid_local_activity_recovery'); - } - try { - $fingerprint = hash('sha256', json_encode(self::normalizeDescriptor($descriptor), JSON_THROW_ON_ERROR)); - } catch (ValidationException|JsonException) { - return $refused('invalid_local_activity_recovery'); - } - /** @var WorkflowTask|null $snapshotTask */ - $snapshotTask = ConfiguredV2Models::query('task_model', WorkflowTask::class)->find($taskId); - /** @var ActivityExecution|null $snapshotExecution */ - $snapshotExecution = $snapshotTask === null ? null : ActivityExecution::query() - ->where('workflow_run_id', $snapshotTask->workflow_run_id) - ->where('sequence', $sequence) - ->first(); - if ($snapshotTask === null || $snapshotExecution === null - || ! is_string($snapshotExecution->current_attempt_id)) { - return $refused('local_activity_preparation_not_found'); - } - return ConfiguredV2Models::connection()->transaction(static function () use ( - $snapshotTask, - $snapshotExecution, - $taskId, - $leaseOwner, - $workflowTaskAttempt, - $sequence, - $fingerprint, - $refused, - ): array { - $rows = ActivityRowLockOrder::lockForAttempt($snapshotExecution->current_attempt_id); - $execution = $rows['execution']; - $attempt = $rows['attempt']; - /** @var WorkflowRun|null $run */ - $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) - ->lockForUpdate() - ->find($snapshotTask->workflow_run_id); - /** @var WorkflowTask|null $task */ - $task = ConfiguredV2Models::query('task_model', WorkflowTask::class) - ->lockForUpdate() - ->find($taskId); - if (! $execution instanceof ActivityExecution || ! $attempt instanceof ActivityAttempt - || $execution->id !== $snapshotExecution->id || $execution->sequence !== $sequence - || $run === null || $execution->workflow_run_id !== $run->id - || $task === null || $task->workflow_run_id !== $run->id) { - return $refused('local_activity_preparation_not_found'); - } - $receipt = $run->historyEvents() - ->whereIn('event_type', [HistoryEventType::ActivityRetryScheduled, HistoryEventType::ActivityFailed, - HistoryEventType::ActivityTimedOut]) - ->where('payload->sequence', $sequence) - ->where('payload->local_recovery->workflow_task_id', $taskId) - ->where('payload->local_recovery->workflow_task_attempt', $workflowTaskAttempt) - ->where('payload->local_recovery->lease_owner', $leaseOwner) - ->orderBy('sequence') - ->first(); - if ($receipt instanceof WorkflowHistoryEvent) { - if (($receipt->payload['local_recovery']['version'] ?? null) !== 1 - || ($receipt->payload['local_recovery']['descriptor_fingerprint'] ?? null) !== $fingerprint - || ($receipt->payload['activity_execution_id'] ?? null) !== $execution->id - || $receipt->workflow_task_id !== $taskId - || ($receipt->payload['task']['id'] ?? null) !== $taskId - || ($receipt->payload['task']['lease_owner'] ?? null) !== $leaseOwner - || ($receipt->payload['task']['attempt_count'] ?? null) !== $workflowTaskAttempt - || ($receipt->payload['local_recovery']['callback_stop_state'] ?? null) !== 'unknown') { - return $refused('local_activity_recovery_mismatch'); - } - return self::recoveryReceipt($receipt, true); - } - if ($task->task_type !== TaskType::Workflow || $task->status !== TaskStatus::Leased - || $task->lease_owner !== $leaseOwner || $task->attempt_count !== $workflowTaskAttempt) { - return $refused('workflow_claim_mismatch'); - } - if ($task->lease_expires_at === null || now()->gte($task->lease_expires_at)) { - return $refused('workflow_claim_expired'); - } - $started = $run->historyEvents() - ->where('event_type', HistoryEventType::ActivityStarted) - ->where('payload->activity_attempt_id', $attempt->id) - ->first(); - $originalTaskAttempt = $started?->payload['local_preparation']['workflow_task_attempt'] ?? null; - if (! is_int($originalTaskAttempt) - || self::originalStart($run, $execution, $attempt, $attempt->lease_owner, $originalTaskAttempt) === null - || ($started->payload['local_preparation']['descriptor_fingerprint'] ?? null) !== $fingerprint - || $execution->current_attempt_id !== $attempt->id || $execution->attempt_count !== $attempt->attempt_number) { - return $refused('local_activity_preparation_mismatch'); - } - // Accepted cancellation may fence immediately, without waiting for - // lease expiry. The replacement claim and root budget stay intact. - $cleanup = PortableLocalActivityCleanup::isExecution($run, $execution, $started); - if ($cleanup && now()->gte(PortableLocalActivityCleanup::deadline($execution))) { - return $refused(PortableLocalActivityCleanup::isScoped($execution) - ? 'local_activity_cleanup_deadline_expired' : 'cancellation_deadline_expired'); - } - if ($run->cancellation_request_command_id !== null && (! $cleanup - || ! PortableLocalActivityCleanup::belongsToRunRequest( - $run, - $execution->activity_options['cancellation_cleanup'] - ))) { - $cancelled = $run->historyEvents() - ->where('event_type', HistoryEventType::ActivityCancelled) - ->where('payload->activity_attempt_id', $attempt->id) - ->where('payload->workflow_command_id', $run->cancellation_request_command_id) - ->first(); - if (! $cancelled instanceof WorkflowHistoryEvent) { - if ($execution->status !== ActivityStatus::Running || $attempt->status !== ActivityAttemptStatus::Running) { - return $refused('stale_activity_attempt'); - } - $cancelled = ActivityCancellation::record( + return self::prepareRetry( $run, + $task, $execution, - command: $run->cancellation_request_command_id + $attempt, + $workerAttemptId, + $fingerprint, + $localContractVersion, + $preparedAt ); } - return [ - ...$refused('cancellation_requested'), - 'fenced' => true, - 'cancellation_history_event_id' => $cancelled?->id, - ]; - } - if ($run->status->isTerminal()) { - return $refused('run_closed'); - } - if (($run->execution_deadline_at !== null && now()->gte($run->execution_deadline_at)) - || ($run->run_deadline_at !== null && now()->gte($run->run_deadline_at))) { - return $refused('run_deadline_expired'); - } - if ($execution->status !== ActivityStatus::Running || $attempt->status !== ActivityAttemptStatus::Running) { - return $refused('local_activity_previous_attempt_unresolved'); - } - if (PortableLocalActivityCleanup::currentAuthorityExpired($run, $execution)) { - return $refused('local_activity_cleanup_authority_expired'); + return self::duplicate( + $run, + $task, + $execution, + $attempt, + $workerAttemptId, + $fingerprint, + $localContractVersion, + $preparedAt + ); } - if ($attempt->lease_expires_at === null || now()->lt($attempt->lease_expires_at)) { - return $refused('local_activity_previous_attempt_live'); + if (ActivityExecution::query()->where('workflow_run_id', $run->id)->where( + 'sequence', + $sequence + )->exists()) { + return self::response('local_activity_preparation_retry'); } - /** @var WorkflowTask|null $originalTask */ - $originalTask = ConfiguredV2Models::query('task_model', WorkflowTask::class) - ->lockForUpdate() - ->find($attempt->workflow_task_id); - if ($originalTask === null || $originalTask->workflow_run_id !== $run->id - || $originalTask->task_type !== TaskType::Workflow) { - return $refused('original_workflow_claim_not_found'); + if (WorkflowStepHistory::nextDurableCommandSequence($run) !== $sequence) { + return self::response('local_activity_command_prefix_not_recorded'); } - if ($originalTask->id === $task->id) { - if ($task->attempt_count <= $originalTaskAttempt) { - return $refused('local_activity_original_claim_not_reclaimed'); - } - } elseif ($originalTask->status === TaskStatus::Leased && $originalTask->lease_expires_at !== null - && now() - ->lt($originalTask->lease_expires_at)) { - return $refused('local_activity_original_claim_live'); + if (($normalized['parallel_group_path'] ?? []) !== []) { + return self::response('local_activity_group_not_admitted'); } - $recovery = [ - 'version' => 1, + $preparation = [ + 'version' => $localContractVersion, + ...($localContractVersion === 2 ? [ + 'local_activity_contract_version' => 2, + ] : []), 'descriptor_fingerprint' => $fingerprint, - 'workflow_task_id' => $task->id, + 'worker_attempt_id' => $workerAttemptId, 'workflow_task_attempt' => $workflowTaskAttempt, - 'lease_owner' => $leaseOwner, - 'original_workflow_task_id' => $originalTask->id, - 'original_workflow_task_attempt' => $originalTaskAttempt, - 'original_lease_owner' => $attempt->lease_owner, - 'original_lease_expires_at' => $attempt->lease_expires_at->toISOString(), - 'callback_stop_state' => 'unknown', + ...($cleanup === null ? [] : [ + 'cancellation_cleanup' => $cleanup, + ]), ]; - if ($originalTask->id !== $task->id && in_array( - $originalTask->status, - [TaskStatus::Ready, TaskStatus::Leased], - true - )) { - $originalTask->forceFill([ - 'status' => TaskStatus::Completed, - 'lease_expires_at' => null, - ])->save(); - } - $outcome = app(LocalActivityExecutor::class)->recoverPortableAttempt( + $execution = self::createExecution( + $run, + $task, + $sequence, + $normalized, + $cleanup, + [ + 'local_preparation' => $preparation, + ], + $preparedAt + ); + $attempt = app(LocalActivityExecutor::class)->startPortableAttempt( $run, $task, $execution, - $attempt, - $recovery + $workerAttemptId, + $preparation, + $preparedAt ); - return self::recoveryReceipt($outcome['event'], false); + return self::response(null, $execution, $attempt, task: $task); }, 5); } - /** @param array $descriptor - * @return array - */ - public static function normalizeDescriptor(array $descriptor): array - { - $allowed = ['type', 'activity_type', 'arguments', 'payload_codec', 'retry_policy', - 'start_to_close_timeout', 'schedule_to_close_timeout', 'heartbeat_timeout', 'execution_mode', - 'cancellation_cleanup', 'cancellation_policy', 'cancellation_scope_id', 'parallel_group_id', 'parallel_group_kind', 'parallel_group_mode', - 'parallel_group_base_sequence', 'parallel_group_size', 'parallel_group_index', 'parallel_group_path']; - if (($descriptor['type'] ?? null) !== 'record_local_activity' - || array_diff(array_keys($descriptor), $allowed) !== [] - || (isset($descriptor['execution_mode']) && $descriptor['execution_mode'] !== LocalActivityRuntime::EXECUTION_MODE)) { - throw ValidationException::withMessages([ - 'local_activity' => ['Expected a local activity preparation descriptor.'], - ]); - } - if (array_key_exists('cancellation_policy', $descriptor) - && ! in_array($descriptor['cancellation_policy'], ['try_cancel', 'wait_cancellation_completed'], true)) { - throw ValidationException::withMessages([ - 'local_activity.cancellation_policy' => [ - 'Prepared local activities support try_cancel and wait_cancellation_completed. Abandon requires an independent callback lifetime.', - ], - ]); - } - if (array_key_exists('cancellation_scope_id', $descriptor) - && (! is_string($descriptor['cancellation_scope_id']) - || trim($descriptor['cancellation_scope_id']) === '' - || strlen($descriptor['cancellation_scope_id']) > 255 - || preg_match('//u', $descriptor['cancellation_scope_id']) !== 1)) { - throw ValidationException::withMessages([ - 'local_activity.cancellation_scope_id' => ['Expected a nonempty canonical scope identity.'], - ]); - } - // Common activity input/retry/timeout validation does not need a - // fabricated outcome or a claim that an application attempt ran. - $input = $descriptor; - unset($input['execution_mode'], $input['cancellation_cleanup'], $input['cancellation_scope_id']); - $input['type'] = 'schedule_activity'; - $normalized = WorkflowCommandNormalizer::normalize([$input], self::MINIMUM_PROTOCOL_VERSION)[0]; - if (! is_string($normalized['arguments'] ?? null) || ! is_string($normalized['payload_codec'] ?? null)) { - throw ValidationException::withMessages([ - 'local_activity.arguments' => ['Preparation requires encoded arguments.'], - ]); - } - $normalized['type'] = 'record_local_activity'; - $normalized['execution_mode'] = LocalActivityRuntime::EXECUTION_MODE; - if (array_key_exists('cancellation_scope_id', $descriptor)) { - $normalized['cancellation_scope_id'] = $descriptor['cancellation_scope_id']; - } - if (array_key_exists('cancellation_cleanup', $descriptor)) { - $proof = $descriptor['cancellation_cleanup']; - if (! is_array($proof) || array_diff( - array_keys($proof), - ['request_id', 'delivery_history_event_id', 'scope_id'] - ) !== [] - || ! is_string($proof['request_id'] ?? null) || trim($proof['request_id']) === '' - || ! is_string($proof['delivery_history_event_id'] ?? null) - || trim($proof['delivery_history_event_id']) === '' - || (array_key_exists('scope_id', $proof) && (! is_string($proof['scope_id']) - || trim( - $proof['scope_id'] - ) === '' || $proof['scope_id'] === CancellationScopeHistory::ROOT_SCOPE_ID))) { - throw ValidationException::withMessages([ - 'local_activity.cancellation_cleanup' => ['Expected canonical request and delivery identities.'], - ]); - } - $normalized['cancellation_cleanup'] = [ - 'request_id' => $proof['request_id'], - 'delivery_history_event_id' => $proof['delivery_history_event_id'], - ...(array_key_exists('scope_id', $proof) ? [ - 'scope_id' => $proof['scope_id'], - ] : []), - ]; - } - - return $normalized; - } - - /** - * @internal Read the original immutable preparation authority. The caller - * holds attempt/execution/run locks. Current task ownership is separate. - */ - public static function originalStart( - WorkflowRun $run, - ActivityExecution $execution, - ActivityAttempt $attempt, - string $leaseOwner, - int $workflowTaskAttempt, - ): ?WorkflowHistoryEvent { - if (! LocalActivityRuntime::isExecution($execution) || $execution->workflow_run_id !== $run->id - || $attempt->workflow_run_id !== $run->id || $attempt->activity_execution_id !== $execution->id - || $attempt->lease_owner !== $leaseOwner || $workflowTaskAttempt < 1) { - return null; - } - $started = $run->historyEvents() - ->where('event_type', HistoryEventType::ActivityStarted) - ->where('payload->activity_attempt_id', $attempt->id) - ->first(); - if (! $started instanceof WorkflowHistoryEvent - || $started->workflow_task_id !== $attempt->workflow_task_id - || ($started->payload['activity_execution_id'] ?? null) !== $execution->id - || ($started->payload['activity_type'] ?? null) !== $execution->activity_type - || ($started->payload['activity_class'] ?? null) !== $execution->activity_class - || ($started->payload['sequence'] ?? null) !== $execution->sequence - || ($started->payload['workflow_task_id'] ?? null) !== $attempt->workflow_task_id - || ($started->payload['local_activity'] ?? null) !== true - || ($started->payload['execution_mode'] ?? null) !== LocalActivityRuntime::EXECUTION_MODE - || ($started->payload['local_preparation']['version'] ?? null) !== 1 - || ! is_string($started->payload['local_preparation']['descriptor_fingerprint'] ?? null) - || ($started->payload['local_preparation']['worker_attempt_id'] ?? null) !== $attempt->worker_attempt_id - || ($started->payload['local_preparation']['workflow_task_attempt'] ?? null) !== $workflowTaskAttempt - || ($started->payload['task']['id'] ?? null) !== $attempt->workflow_task_id - || ($started->payload['task']['type'] ?? null) !== TaskType::Workflow->value - || ($started->payload['task']['status'] ?? null) !== TaskStatus::Leased->value - || ($started->payload['task']['attempt_count'] ?? null) !== $workflowTaskAttempt - || ($started->payload['task']['lease_owner'] ?? null) !== $leaseOwner - || ($started->payload['activity_attempt']['id'] ?? null) !== $attempt->id - || ($started->payload['activity_attempt']['activity_execution_id'] ?? null) !== $execution->id - || ($started->payload['activity_attempt']['task_id'] ?? null) !== $attempt->workflow_task_id - || ($started->payload['activity_attempt']['attempt_number'] ?? null) !== $attempt->attempt_number - || ($started->payload['activity_attempt']['lease_owner'] ?? null) !== $leaseOwner - || ($started->payload['activity_attempt']['worker_attempt_id'] ?? null) !== $attempt->worker_attempt_id) { - return null; - } - $scopeId = $execution->activity_options['cancellation_scope_id'] ?? null; - if (($started->payload['activity']['cancellation_scope_id'] ?? null) !== $scopeId) { - return null; - } - if ($scopeId !== null) { - $scheduled = $run->historyEvents() - ->where('event_type', HistoryEventType::ActivityScheduled) - ->where('payload->activity_execution_id', $execution->id) - ->first(); - if (! is_string($scopeId) - || ! CancellationScopeHistory::isRecordedBefore($run, $scopeId, $execution->sequence) - || ($scheduled?->payload['activity']['cancellation_scope_id'] ?? null) !== $scopeId) { - return null; - } - } - if (array_key_exists('schedule_to_close_deadline_at', $started->payload['local_preparation']) - && $started->payload['local_preparation']['schedule_to_close_deadline_at'] - !== $execution->schedule_to_close_deadline_at?->toISOString()) { - return null; - } - if ((array_key_exists('cancellation_cleanup', $execution->activity_options ?? []) - || array_key_exists('cancellation_cleanup', $started->payload['local_preparation'])) - && ! PortableLocalActivityCleanup::isExecution($run, $execution, $started)) { - return null; - } - - return $started; - } - /** @param array $normalized * @param array|null $cleanup * @param array $opening @@ -654,9 +1012,21 @@ private static function createExecution( int $sequence, array $normalized, ?array $cleanup, - array $opening + array $opening, + ?Carbon $preparedAt = null, ): ActivityExecution { StructuralLimits::guardPendingActivities($run); + $now = $preparedAt ?? now(); + if (($normalized['local_activity_contract_version'] ?? null) === 2) { + $connection = ConfiguredV2Models::connection(); + foreach (['start_to_close_timeout', 'schedule_to_close_timeout', 'heartbeat_timeout'] as $field) { + PreparedLocalRetryTimestamp::addForConnection( + $now, + $normalized['retry_policy'][$field] ?? 0, + $connection + ); + } + } $codec = $normalized['payload_codec']; $arguments = ExternalPayloads::externalizeForNamespace( $normalized['arguments'], @@ -664,7 +1034,6 @@ private static function createExecution( is_string($run->namespace) ? $run->namespace : null ); StructuralLimits::guardPayloadSize($arguments); - $now = now(); $options = new ActivityOptions( startToCloseTimeout: $normalized['start_to_close_timeout'] ?? null, scheduleToCloseTimeout: $normalized['schedule_to_close_timeout'] ?? null, @@ -683,11 +1052,17 @@ private static function createExecution( 'connection' => $run->connection, 'queue' => $run->queue, 'parallel_group_path' => $normalized['parallel_group_path'] ?? null, - 'retry_policy' => ActivityRetryPolicy::snapshotExternal($normalized['retry_policy'] ?? null, $options), + 'retry_policy' => ($normalized['local_activity_contract_version'] ?? null) === 2 + ? PreparedLocalRetryPolicy::validateSnapshot($normalized['retry_policy']) + : ActivityRetryPolicy::snapshotExternal($normalized['retry_policy'] ?? null, $options), 'activity_options' => [ 'execution_mode' => LocalActivityRuntime::EXECUTION_MODE, 'queue_bypassed' => true, 'routing' => 'workflow_worker_process', + ...(($normalized['local_activity_contract_version'] ?? null) === 2 + ? [ + 'local_activity_contract_version' => 2, + ] : []), ...(isset($normalized['cancellation_policy']) ? [ 'cancellation_policy' => $normalized['cancellation_policy'], ] : []), @@ -839,6 +1214,8 @@ private static function prepareRetry( ActivityAttempt $previous, string $workerAttemptId, string $fingerprint, + int $localContractVersion = 1, + ?Carbon $preparedAt = null, ): array { $started = $run->historyEvents() ->where('event_type', HistoryEventType::ActivityStarted) @@ -850,7 +1227,14 @@ private static function prepareRetry( || $previous->closed_at === null || $execution->attempt_count !== $previous->attempt_number || ! is_int($originalTaskAttempt) - || self::originalStart($run, $execution, $previous, $previous->lease_owner, $originalTaskAttempt) === null + || self::originalStart( + $run, + $execution, + $previous, + $previous->lease_owner, + $originalTaskAttempt, + $localContractVersion + ) === null || ($started->payload['local_preparation']['descriptor_fingerprint'] ?? null) !== $fingerprint) { return self::response('local_activity_retry_preparation_mismatch'); } @@ -865,7 +1249,15 @@ private static function prepareRetry( || ($retry->payload['retry_after_attempt_id'] ?? null) !== $previous->id || ($retry->payload['retry_after_attempt'] ?? null) !== $previous->attempt_number || ($retry->payload['retry_policy'] ?? null) !== $execution->retry_policy - || ! self::retryAuthorityMatches($retry, $previous, $originalTaskAttempt, $fingerprint) + || ! self::retryAuthorityMatches( + $retry, + $previous, + $originalTaskAttempt, + $fingerprint, + $localContractVersion, + $execution, + $started + ) || ! is_string($retry->payload['retry_available_at'] ?? null) || $execution->attempt_count >= ActivityRetryPolicy::maxAttemptsFromSnapshot($execution)) { return self::response('local_activity_retry_preparation_mismatch'); @@ -875,21 +1267,30 @@ private static function prepareRetry( } // The durable receipt owns backoff. Mutable task availability cannot // admit a callback early. Total timeout owns the entire retry chain. - if ($execution->schedule_to_close_deadline_at !== null && now()->gte( + if ($execution->schedule_to_close_deadline_at !== null && ($preparedAt ?? now())->gte( $execution->schedule_to_close_deadline_at )) { - $event = app(LocalActivityExecutor::class)->expirePortableRetry($run, $task, $execution); + $event = app(LocalActivityExecutor::class)->expirePortableRetry( + $run, + $task, + $execution, + $localContractVersion, + $preparedAt + ); return [ ...self::response('local_activity_deadline_expired'), 'event_id' => $event->id, 'event_type' => $event->event_type->value, ]; } - if (now()->lt(Carbon::parse($retry->payload['retry_available_at']))) { + if (($preparedAt ?? now())->lt(Carbon::parse($retry->payload['retry_available_at']))) { return self::response('local_activity_retry_not_due'); } $attempt = app(LocalActivityExecutor::class)->startPortableAttempt($run, $task, $execution, $workerAttemptId, [ - 'version' => 1, + 'version' => $localContractVersion, + ...($localContractVersion === 2 ? [ + 'local_activity_contract_version' => 2, + ] : []), 'descriptor_fingerprint' => $fingerprint, 'worker_attempt_id' => $workerAttemptId, 'workflow_task_attempt' => $task->attempt_count, @@ -897,7 +1298,7 @@ private static function prepareRetry( ? [ 'cancellation_cleanup' => $execution->activity_options['cancellation_cleanup'], ] : []), - ]); + ], $preparedAt); return self::response(null, $execution, $attempt, task: $task); } @@ -1001,21 +1402,40 @@ private static function groupAdmission(WorkflowRun $run, ActivityExecution $exec } /** - * @return array + * Validate original outcome or recovery authority before preparing a retry. */ private static function retryAuthorityMatches( WorkflowHistoryEvent $retry, ActivityAttempt $previous, int $originalTaskAttempt, string $fingerprint, + int $localContractVersion = 1, + ?ActivityExecution $execution = null, + ?WorkflowHistoryEvent $started = null, ): bool { $payload = $retry->payload; - if (($payload['local_outcome']['version'] ?? null) === 1) { + if ($localContractVersion === 2 && array_key_exists('local_recovery', $payload)) { + return $previous->status === ActivityAttemptStatus::Expired && $execution !== null + && self::matchesRetryRecoveryReceipt($retry, $execution, $previous, $started); + } + if (($payload['local_outcome']['version'] ?? null) === $localContractVersion) { return $previous->status === ActivityAttemptStatus::Failed && $retry->workflow_task_id === $previous->workflow_task_id && ($payload['local_outcome']['workflow_task_attempt'] ?? null) === $originalTaskAttempt && ($payload['task']['lease_owner'] ?? null) === $previous->lease_owner - && ($payload['task']['attempt_count'] ?? null) === $originalTaskAttempt; + && ($payload['task']['attempt_count'] ?? null) === $originalTaskAttempt + && ($localContractVersion === 1 || ( + ($payload['retry_disposition']['contract_version'] ?? null) === 2 + && ($payload['retry_disposition']['snapshot_version'] ?? null) === 2 + && ($payload['retry_disposition']['decision'] ?? null) === 'scheduled' + && ($payload['retry_disposition']['candidate_delay_seconds'] ?? null) + === ($payload['retry_backoff_seconds'] ?? null) + && ($payload['retry_disposition']['retry_available_at'] ?? null) + === ($payload['retry_available_at'] ?? null) + )); + } + if ($localContractVersion !== 1) { + return false; } $recovery = $payload['local_recovery'] ?? []; return $previous->status === ActivityAttemptStatus::Expired @@ -1043,6 +1463,8 @@ private static function duplicate( ActivityAttempt $attempt, string $workerAttemptId, string $fingerprint, + int $localContractVersion = 1, + ?Carbon $preparedAt = null, ): array { if (! LocalActivityRuntime::isExecution($execution) || $execution->workflow_run_id !== $run->id || $execution->status !== ActivityStatus::Running @@ -1050,12 +1472,19 @@ private static function duplicate( || $attempt->worker_attempt_id !== $workerAttemptId || $attempt->status !== ActivityAttemptStatus::Running) { return self::response('local_activity_preparation_mismatch'); } - $started = self::originalStart($run, $execution, $attempt, $task->lease_owner, $task->attempt_count); + $started = self::originalStart( + $run, + $execution, + $attempt, + $task->lease_owner, + $task->attempt_count, + $localContractVersion + ); if (! $started instanceof WorkflowHistoryEvent || ($started->payload['local_preparation']['descriptor_fingerprint'] ?? null) !== $fingerprint) { return self::response('local_activity_preparation_mismatch'); } - if ($attempt->lease_expires_at === null || now()->gte($attempt->lease_expires_at)) { + if ($attempt->lease_expires_at === null || ($preparedAt ?? now())->gte($attempt->lease_expires_at)) { return self::response('local_activity_preparation_lease_expired'); } foreach ([ @@ -1063,7 +1492,7 @@ private static function duplicate( $execution->schedule_to_close_deadline_at, $execution->heartbeat_deadline_at, ] as $deadline) { - if ($deadline !== null && now()->gte($deadline)) { + if ($deadline !== null && ($preparedAt ?? now())->gte($deadline)) { return self::response('local_activity_deadline_expired'); } } @@ -1091,6 +1520,9 @@ private static function recoveryReceipt(WorkflowHistoryEvent $event, bool $dupli 'callback_stop_state' => 'unknown', 'claim_released' => $event->event_type === HistoryEventType::ActivityRetryScheduled, 'created_task_ids' => is_string($retryTaskId) ? [$retryTaskId] : [], + ...(isset($event->payload['retry_disposition']) ? [ + 'retry_disposition' => $event->payload['retry_disposition'], + ] : []), ]; } diff --git a/src/V2/Support/PreparedLocalRetryDecision.php b/src/V2/Support/PreparedLocalRetryDecision.php new file mode 100644 index 000000000..588afd22c --- /dev/null +++ b/src/V2/Support/PreparedLocalRetryDecision.php @@ -0,0 +1,351 @@ + $snapshot + * @param array $report + * @param array $stored + */ + public static function matchesStored( + array $snapshot, + int $attemptNumber, + array $report, + PreparedLocalRetryWindow $window, + array $stored, + bool $nonRetryable = false, + ): bool { + return self::matchesStoredSelection( + $window, + $stored, + static fn (DateTimeImmutable $sample, ?string $timeout): array => self::select( + $snapshot, + $attemptNumber, + $report, + $sample, + $window, + $timeout, + $nonRetryable + ) + ); + } + + /** + * @internal The caller has proved original claim loss under canonical + * locks. No worker report can select this engine-owned provenance. + * @param array $snapshot + * @return array + */ + public static function selectInterruption( + array $snapshot, + int $attemptNumber, + DateTimeInterface $sample, + PreparedLocalRetryWindow $window, + ?string $dueTimeoutKind = null, + ): array { + $decision = self::select( + $snapshot, + $attemptNumber, + [ + 'outcome' => 'failed', + 'message' => self::INTERRUPTION_MESSAGE, + ], + $sample, + $window, + $dueTimeoutKind + ); + if ($decision['failure_source'] === 'application') { + $decision['failure_source'] = 'engine_interruption'; + } + + return $decision; + } + + /** @param array $snapshot + * @param array $stored + */ + public static function matchesStoredInterruption( + array $snapshot, + int $attemptNumber, + PreparedLocalRetryWindow $window, + array $stored, + ): bool { + return self::matchesStoredSelection( + $window, + $stored, + static fn (DateTimeImmutable $sample, ?string $timeout): array => self::selectInterruption( + $snapshot, + $attemptNumber, + $sample, + $window, + $timeout + ) + ); + } + + /** + * The caller owns original authority, fresh fences and actual timeout + * detection. Only a scheduled decision is a duration to execute. Terminal + * candidates remain portable diagnostics and need no timestamp addition. + * + * @param array $snapshot + * @param array $report + * @return array + */ + public static function select( + array $snapshot, + int $attemptNumber, + array $report, + DateTimeInterface $sample, + PreparedLocalRetryWindow $window, + ?string $dueTimeoutKind = null, + bool $nonRetryable = false, + ): array { + $snapshot = PreparedLocalRetryPolicy::validateSnapshot($snapshot); + self::validateReport($report, $attemptNumber, $dueTimeoutKind); + $sample = self::utc($sample); + $deadline = $window->deadline === null ? null : self::utc($window->deadline); + self::utc($window->latest); + if ($deadline !== null && $sample >= $deadline) { + $dueTimeoutKind = 'schedule_to_close'; + } + self::validateClock($sample, $window, $dueTimeoutKind, $report); + $decision = self::commonDecision($report, $sample, $window, $dueTimeoutKind); + $terminal = self::earlyTerminal($snapshot, $attemptNumber, $report, $dueTimeoutKind, $nonRetryable); + if ($terminal !== null) { + return [ + ...$decision, + 'terminal_classification' => $terminal, + ]; + } + + return self::selectedDelay($snapshot, $attemptNumber, $report, $sample, $window, $dueTimeoutKind, $decision); + } + + /** @param array $stored + * @param Closure(DateTimeImmutable, ?string): array $select + */ + private static function matchesStoredSelection( + PreparedLocalRetryWindow $window, + array $stored, + Closure $select, + ): bool { + $sample = self::storedSample($stored['failure_time'] ?? null); + $timeout = $stored['timeout_kind'] ?? null; + if ($sample === null || ($timeout !== null && ! is_string($timeout))) { + return false; + } + try { + PreparedLocalRetryTimestamp::add($sample, 0, $window->latest, $window->earliest); + $expected = $select($sample, $timeout); + } catch (InvalidArgumentException) { + return false; + } + return self::sameFields($expected, $stored); + } + + /** + * @param array $report + */ + private static function validateReport(array $report, int $attemptNumber, ?string $dueTimeoutKind): void + { + $refusal = PreparedLocalActivityReport::refusal($report); + if ($refusal !== null || $attemptNumber < 1 || $attemptNumber > PreparedLocalRetryPolicy::MAX_ATTEMPTS + || ($dueTimeoutKind !== null && ! in_array( + $dueTimeoutKind, + ['schedule_to_close', 'start_to_close', 'heartbeat'], + true + ))) { + throw new InvalidArgumentException($refusal ?? 'Invalid prepared retry decision input.'); + } + } + + /** + * @param array $report + */ + private static function validateClock( + DateTimeImmutable $sample, + PreparedLocalRetryWindow $window, + ?string $dueTimeoutKind, + array $report, + ): void { + if ($sample > $window->latest) { + throw new InvalidArgumentException('Prepared failure sample exceeds the backend timestamp range.'); + } + if ($dueTimeoutKind === 'schedule_to_close' && ($window->deadline === null || $sample < $window->deadline)) { + throw new InvalidArgumentException('Overall timeout requires the original due deadline.'); + } + if ($dueTimeoutKind === null && $report['outcome'] !== 'failed') { + throw new InvalidArgumentException( + 'A retry decision needs an application failure or an actually due timeout.' + ); + } + } + + /** @param array $report + * @return array + */ + private static function commonDecision( + array $report, + DateTimeImmutable $sample, + PreparedLocalRetryWindow $window, + ?string $dueTimeoutKind, + ): array { + $engineTimeout = $dueTimeoutKind !== null; + + return [ + 'contract_version' => 2, + 'snapshot_version' => 2, + 'decision' => 'terminal', + 'failure_time' => self::format($sample), + 'original_schedule_to_close_deadline_at' => $window->deadline === null ? null : self::format( + $window->deadline + ), + 'failure_source' => $engineTimeout ? 'engine_timeout' : 'application', + ...($engineTimeout ? [ + 'timeout_kind' => $dueTimeoutKind, + ] : []), + ...(array_key_exists('next_retry_delay_seconds', $report) + ? [ + 'requested_delay_seconds' => $report['next_retry_delay_seconds'], + ] : []), + ]; + } + + /** @param array $snapshot + * @param array $report + */ + private static function earlyTerminal( + array $snapshot, + int $attemptNumber, + array $report, + ?string $dueTimeoutKind, + bool $nonRetryable, + ): ?string { + $type = $report['exception_type'] ?? RuntimeException::class; + + return match (true) { + $dueTimeoutKind === 'schedule_to_close' => 'engine_timeout', + $dueTimeoutKind === null && ($nonRetryable || ($report['non_retryable'] ?? false) + || in_array($type, $snapshot['non_retryable_error_types'], true)) => 'non_retryable', + $attemptNumber >= $snapshot['max_attempts'] => 'attempts_exhausted', + default => null, + }; + } + + /** @param array $snapshot + * @param array $report + * @param array $decision + * @return array + */ + private static function selectedDelay( + array $snapshot, + int $attemptNumber, + array $report, + DateTimeImmutable $sample, + PreparedLocalRetryWindow $window, + ?string $dueTimeoutKind, + array $decision, + ): array { + $override = $dueTimeoutKind === null && array_key_exists('next_retry_delay_seconds', $report); + $candidate = $override ? $report['next_retry_delay_seconds'] + : PreparedLocalRetryPolicy::delayForRetry($snapshot, $attemptNumber); + $decision = [ + ...$decision, + 'candidate_delay_seconds' => $candidate, + 'maximum_interval_seconds' => $snapshot['maximum_interval_seconds'], + 'delay_source' => $override ? 'failure_override' : 'policy', + ]; + if ($candidate > $snapshot['maximum_interval_seconds']) { + return [ + ...$decision, + 'terminal_classification' => 'retry_delay_exceeds_policy_cap', + ]; + } + if ($window->deadline !== null && $candidate >= intdiv( + self::microseconds($window->deadline) - self::microseconds($sample) + 999999, + 1000000 + )) { + return [ + ...$decision, + 'terminal_classification' => 'retry_delay_exceeds_schedule_to_close', + ]; + } + // Check by division before multiplication or DateInterval construction. + // Even the largest portable integer cannot wrap or become a float. + if ($candidate > intdiv(self::microseconds($window->latest) - self::microseconds($sample), 1000000)) { + throw new InvalidArgumentException('Prepared retry availability exceeds the backend timestamp range.'); + } + $available = $sample->add(new DateInterval('PT' . $candidate . 'S')); + + return [ + ...$decision, + 'decision' => 'scheduled', + 'retry_available_at' => self::format($available), + ]; + } + + private static function storedSample(mixed $time): ?DateTimeImmutable + { + if (! is_string($time) || preg_match('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/D', $time) !== 1) { + return null; + } + $sample = DateTimeImmutable::createFromFormat('!Y-m-d\TH:i:s.u\Z', $time, new DateTimeZone('UTC')); + + return $sample instanceof DateTimeImmutable && self::format($sample) === $time ? $sample : null; + } + + /** @param array $expected + * @param array $stored + */ + private static function sameFields(array $expected, array $stored): bool + { + if (count($stored) !== count($expected) || array_diff(array_keys($stored), array_keys($expected)) !== []) { + return false; + } + foreach ($expected as $key => $value) { + if ($stored[$key] !== $value) { + return false; + } + } + + return true; + } + + private static function utc(DateTimeInterface $value): DateTimeImmutable + { + $utc = DateTimeImmutable::createFromInterface($value)->setTimezone(new DateTimeZone('UTC')); + $year = $utc->format('Y'); + if (strlen($year) !== 4 || (int) $year < 1) { + throw new InvalidArgumentException('Prepared retry timestamps require a four-digit UTC year.'); + } + + return $utc; + } + + private static function microseconds(DateTimeImmutable $value): int + { + return $value->getTimestamp() * 1000000 + (int) $value->format('u'); + } + + private static function format(DateTimeImmutable $value): string + { + return $value->format('Y-m-d\TH:i:s.u\Z'); + } +} diff --git a/src/V2/Support/PreparedLocalRetryTimestamp.php b/src/V2/Support/PreparedLocalRetryTimestamp.php new file mode 100644 index 000000000..9c4bd22ae --- /dev/null +++ b/src/V2/Support/PreparedLocalRetryTimestamp.php @@ -0,0 +1,104 @@ +getDriverName()) + ); + } + + public static function earliestForDriver(string $driver): DateTimeImmutable + { + if (in_array($driver, ['mysql', 'mariadb'], true)) { + return new DateTimeImmutable('1970-01-01T00:00:01.000000Z'); + } + if (! in_array($driver, ['sqlite', 'pgsql', 'sqlsrv'], true)) { + throw new InvalidArgumentException('Unsupported prepared retry timestamp backend.'); + } + + return new DateTimeImmutable('0001-01-01T00:00:00.000000Z'); + } + + public static function latestForConnection(Connection $connection): DateTimeImmutable + { + $driver = $connection->getDriverName(); + if ($connection instanceof MySqlConnection && $connection->isMaria()) { + $driver = 'mariadb'; + } + + return self::latestForDriver($driver, $driver === 'mariadb' ? $connection->getServerVersion() : null); + } + + public static function latestForDriver(string $driver, ?string $serverVersion = null): DateTimeImmutable + { + // MySQL TIMESTAMP(6), rather than DATETIME's broader year range: + // https://dev.mysql.com/doc/refman/8.4/en/datetime.html + if ($driver === 'mysql') { + return new DateTimeImmutable('2038-01-19T03:14:07.499999Z'); + } + if ($driver === 'mariadb') { + if (! is_string($serverVersion) + || preg_match('/^(?:5\.5\.5-)?([0-9]+\.[0-9]+(?:\.[0-9]+)?)/D', $serverVersion, $match) !== 1) { + throw new InvalidArgumentException( + 'The MariaDB timestamp range requires an identified server version.' + ); + } + // The unsigned seconds range starts at MariaDB 11.5. + // https://mariadb.com/docs/server/reference/data-types/date-and-time-data-types/timestamp + return new DateTimeImmutable(version_compare($match[1], '11.5', '>=') + ? '2106-02-07T06:28:15.999999Z' : '2038-01-19T03:14:07.999999Z'); + } + if (! in_array($driver, ['sqlite', 'pgsql', 'sqlsrv'], true)) { + throw new InvalidArgumentException('Unsupported prepared retry timestamp backend.'); + } + + // These stores admit the four-digit UTC domain used by this contract. + // Laravel's timestamp(..., 6) maps to datetime2(6) on SQL Server. + return new DateTimeImmutable('9999-12-31T23:59:59.999999Z'); + } + + public static function add( + DateTimeInterface $sample, + int $seconds, + DateTimeInterface $latest, + ?DateTimeInterface $earliest = null, + ): DateTimeImmutable { + $utc = DateTimeImmutable::createFromInterface($sample)->setTimezone(new DateTimeZone('UTC')); + $latest = DateTimeImmutable::createFromInterface($latest)->setTimezone(new DateTimeZone('UTC')); + $earliest ??= new DateTimeImmutable('0001-01-01T00:00:00.000000Z'); + if ($seconds < 0 || $seconds > PreparedLocalRetryPolicy::MAX_DELAY_SECONDS + || strlen($utc->format('Y')) !== 4 || (int) $utc->format('Y') < 1 + || strlen($latest->format('Y')) !== 4 || (int) $latest->format('Y') < 1 + || $utc < $earliest || $utc > $latest) { + throw new InvalidArgumentException('Invalid prepared retry timestamp input.'); + } + $remaining = ($latest->getTimestamp() - $utc->getTimestamp()) * 1000000 + + (int) $latest->format('u') - (int) $utc->format('u'); + if ($seconds > intdiv($remaining, 1000000)) { + throw new InvalidArgumentException('Prepared retry timestamp exceeds the backend range.'); + } + + return $utc->add(new DateInterval('PT' . $seconds . 'S')); + } +} diff --git a/src/V2/Support/PreparedLocalRetryWindow.php b/src/V2/Support/PreparedLocalRetryWindow.php new file mode 100644 index 000000000..c324cad2a --- /dev/null +++ b/src/V2/Support/PreparedLocalRetryWindow.php @@ -0,0 +1,34 @@ +deadline = $deadline === null ? null : DateTimeImmutable::createFromInterface($deadline)->setTimezone( + $utc + ); + $this->latest = DateTimeImmutable::createFromInterface($latest)->setTimezone($utc); + $this->earliest = $earliest === null ? null : DateTimeImmutable::createFromInterface($earliest)->setTimezone( + $utc + ); + } +} diff --git a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php index 60674e3f2..5587b76e7 100644 --- a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php +++ b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php @@ -1570,6 +1570,60 @@ public function testFixturesExecuteThroughDeclaredReplayConsumers(): void continue; } + if ($fixture['id'] === 'prepared-v1-policy-refusal-legacy-zero-tail') { + $this->clearWorkflowState(); + $run = $this->createRunFromFixture($fixture); + $scheduled = collect($fixture['history'])->firstWhere('event_type', 'ActivityScheduled')['payload']; + $completed = collect($fixture['history'])->firstWhere('event_type', 'ActivityCompleted')['payload']; + // Embedded execution reloads both retained history and its + // activity projection, unlike the history-only fiber runner. + ActivityExecution::query()->create([ + 'id' => $scheduled['activity_execution_id'], + 'workflow_run_id' => $run->id, + 'sequence' => $scheduled['sequence'], + 'activity_class' => $scheduled['activity_class'], + 'activity_type' => $scheduled['activity_type'], + 'status' => \Workflow\V2\Enums\ActivityStatus::Completed, + 'attempt_count' => $completed['attempt_number'], + 'arguments' => Serializer::serializeWithCodec('avro', $fixture['workflow']['arguments']), + 'result' => $completed['result'], + 'payload_codec' => $completed['payload_codec'], + 'retry_policy' => $scheduled['activity']['retry_policy'], + 'activity_options' => [ + 'execution_mode' => 'local', + 'queue_bypassed' => true, + 'routing' => 'workflow_worker_process', + ], + 'connection' => $run->connection, + 'queue' => $run->queue, + ]); + $before = $run->historyEvents() + ->count(); + $this->runReadyWorkflowTask($run); + $this->assertSame(RunStatus::Completed, $run->refresh()->status); + $this->assertSame([ + ...$fixture['expected']['result'], + 'workflow_id' => $run->workflow_instance_id, + 'run_id' => $run->id, + ], Serializer::unserializeWithCodec($run->outputPayloadCodec(), $run->output)); + $this->assertSame([120, 0], $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityRetryScheduled) + ->orderBy('sequence') + ->get() + ->map(static fn (WorkflowHistoryEvent $event): int => $event->payload['retry_backoff_seconds']) + ->all()); + $this->assertSame($before + 1, $run->historyEvents()->count()); + $this->assertSame(1, $run->activityExecutions()->count()); + $this->assertSame(0, ActivityAttempt::query()->where('workflow_run_id', $run->id)->count()); + $this->assertEquals( + $scheduled['activity']['retry_policy'], + $run->activityExecutions() + ->sole() +->retry_policy + ); + continue; + } + if ($fixture['id'] === 'nexus-sdk-caller-attribution') { $this->clearWorkflowState(); $this->assertNexusCallerSdkAttribution($fixture); diff --git a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php index d20d3a1c4..e559ce20f 100644 --- a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php +++ b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php @@ -30,10 +30,15 @@ use Workflow\V2\Support\ActivityCancellationAcknowledgement; use Workflow\V2\Support\ActivitySnapshot; use Workflow\V2\Support\CancellationScopeHistory; +use Workflow\V2\Support\ConfiguredV2Models; use Workflow\V2\Support\DefaultWorkflowTaskBridge; use Workflow\V2\Support\HistoryTimeline; +use Workflow\V2\Support\LocalActivityCall; +use Workflow\V2\Support\LocalActivityExecutor; use Workflow\V2\Support\ParallelChildGroup; +use Workflow\V2\Support\PortableLocalActivityControl; use Workflow\V2\Support\PortableLocalActivityPreparation; +use Workflow\V2\Support\PreparedLocalRetryTimestamp; use Workflow\V2\WorkflowStub; final class V2PortableLocalActivityPreparationTest extends TestCase @@ -89,6 +94,669 @@ public function testPreparationPersistsAWorkerAliasAndOriginalClaimBeforeAnyAppl } } + public function testRetryContractPreparationRetainsItsOriginalModeAndCannotUseTheLegacyOutcomeEntry(): void + { + [$run, $task] = $this->newClaim(); + $descriptor = [ + ...$this->descriptor(), + 'retry_policy' => [ + 'contract_version' => 2, + 'mode' => 'explicit', + 'max_attempts' => 3, + 'backoff_seconds' => [2, 5], + ], + ]; + $prepare = static fn (): array => PortableLocalActivityPreparation::prepareWithRetryPolicy( + $task->id, + 'portable-worker', + 1, + 1, + 'sdk-local-attempt', + $descriptor, + '1.20' + ); + $reply = $prepare(); + $this->assertTrue($reply['prepared']); + $execution = ActivityExecution::query()->findOrFail($reply['activity_execution_id']); + $attempt = ActivityAttempt::query()->findOrFail($reply['activity_attempt_id']); + $this->assertSame(2, $execution->activity_options['local_activity_contract_version']); + $this->assertSame(2, $execution->retry_policy['snapshot_version']); + $this->assertSame(5, $execution->retry_policy['maximum_interval_seconds']); + $started = PortableLocalActivityPreparation::originalStart($run, $execution, $attempt, 'portable-worker', 1, 2); + $this->assertInstanceOf(WorkflowHistoryEvent::class, $started); + $this->assertSame(2, $started->payload['local_preparation']['version']); + $this->assertSame(2, $started->payload['local_preparation']['local_activity_contract_version']); + $this->assertNull( + PortableLocalActivityPreparation::originalStart($run, $execution, $attempt, 'portable-worker', 1) + ); + $refused = app(LocalActivityExecutor::class)->recordPortableOutcome( + $attempt->id, + 'portable-worker', + 1, + [ + 'outcome' => 'failed', + 'message' => 'failure', + ], + '1.20' + ); + $this->assertFalse($refused['recorded']); + $this->assertSame('local_activity_preparation_mismatch', $refused['reason']); + $this->assertSame(ActivityAttemptStatus::Running, $attempt->refresh()->status); + $this->assertSame(2, $run->historyEvents()->count()); + $duplicate = $prepare(); + $this->assertTrue($duplicate['prepared']); + $this->assertTrue($duplicate['duplicate']); + $this->assertSame($attempt->id, $duplicate['activity_attempt_id']); + $this->assertSame(2, $run->historyEvents()->count()); + + $policy = $execution->retry_policy; + $execution->forceFill([ + 'retry_policy' => [ + ...$policy, + 'maximum_interval_seconds' => 6, + ], + ])->save(); + $this->assertNull(PortableLocalActivityPreparation::originalStart( + $run, + $execution->refresh(), + $attempt, + 'portable-worker', + 1, + 2 + )); + } + + public function testRetryContractPreparationRollsBackModeSnapshotAndAttemptWithItsOuterTransaction(): void + { + [$run, $task] = $this->newClaim(); + $connection = ConfiguredV2Models::connection(); + $connection->beginTransaction(); + try { + $reply = PortableLocalActivityPreparation::prepareWithRetryPolicy( + $task->id, + 'portable-worker', + 1, + 1, + 'sdk-local-attempt', + [ + ...$this->descriptor(), + 'retry_policy' => [ + 'contract_version' => 2, + 'mode' => 'explicit', + 'max_attempts' => 3, + 'backoff_seconds' => [2, 5], + ], + ], + '1.20' + ); + $this->assertTrue($reply['prepared']); + $this->assertSame(2, $run->historyEvents()->count()); + } finally { + $connection->rollBack(); + } + $this->assertSame(0, $run->activityExecutions()->count()); + $this->assertSame(0, ActivityAttempt::query()->where('workflow_run_id', $run->id)->count()); + $this->assertSame(0, $run->historyEvents()->count()); + $this->assertSame(TaskStatus::Leased, $task->refresh()->status); + } + + public function testRetryContractCannotDowngradeThroughLegacyInternalExecution(): void + { + [$run, $task] = $this->newClaim(); + $prepared = $this->prepareRetryContract($task); + $execution = ActivityExecution::query()->findOrFail($prepared['activity_execution_id']); + $executor = app(LocalActivityExecutor::class); + $operations = [ + static fn () => $executor->execute( + $run, + $task, + 1, + new LocalActivityCall('python-local-greeting', ['Taylor']) + ), + static fn () => $executor->expirePortableRetry($run, $task, $execution), + static fn () => $executor->startPortableAttempt($run, $task, $execution, 'replacement', [ + 'version' => 1, + ]), + static fn () => $executor->recoverPortableAttempt( + $run, + $task, + $execution, + ActivityAttempt::query()->findOrFail($prepared['activity_attempt_id']), + [] + ), + ]; + foreach ($operations as $operation) { + try { + $operation(); + $this->fail('A legacy internal entry accepted prepared retry contract 2.'); + } catch (\RuntimeException $error) { + $this->assertStringStartsWith('Prepared retry ', $error->getMessage()); + } + } + $this->assertSame(2, $run->historyEvents()->count()); + $this->assertSame(1, ActivityAttempt::query()->where('workflow_run_id', $run->id)->count()); + $this->assertSame(ActivityStatus::Running, $execution->refresh()->status); + $this->assertSame(TaskStatus::Leased, $task->refresh()->status); + } + + public function testRetryContractRejectsAnUnbindableDeadlineBeforePublishingPreparation(): void + { + [$run, $task] = $this->newClaim(); + $latest = PreparedLocalRetryTimestamp::latestForConnection(ConfiguredV2Models::connection()); + $task->forceFill([ + 'lease_expires_at' => $latest, + ])->save(); + $before = $task->refresh() + ->getAttributes(); + Carbon::setTestNow(Carbon::instance($latest)->subMicroseconds(500000)); + try { + PortableLocalActivityPreparation::prepareWithRetryPolicy( + $task->id, + 'portable-worker', + 1, + 1, + 'sdk-local-attempt', + [ + ...$this->descriptor(), + 'retry_policy' => [ + 'contract_version' => 2, + 'mode' => 'explicit', + 'max_attempts' => 3, + 'backoff_seconds' => [2, 5], + ], + 'start_to_close_timeout' => 1, + ], + '1.20' + ); + $this->fail('Prepared an attempt with an unbindable deadline.'); + } catch (\InvalidArgumentException) { + $this->addToAssertionCount(1); + } finally { + Carbon::setTestNow(); + } + $this->assertSame(0, $run->activityExecutions()->count()); + $this->assertSame(0, ActivityAttempt::query()->where('workflow_run_id', $run->id)->count()); + $this->assertSame(0, $run->historyEvents()->count()); + $this->assertSame($before, $task->refresh()->getAttributes()); + } + + #[DataProvider('retryContractOutcomes')] + public function testRetryContractCommitsAndReplaysItsOriginalDecision( + int $elapsed, + array $report, + array $timeouts, + string $eventType, + string $classification, + ?int $delay, + ): void { + $origin = Carbon::parse('2026-11-01T00:00:00.123456Z'); + Carbon::setTestNow($origin); + try { + [$run, $task] = $this->newClaim(); + $prepared = $this->prepareRetryContract($task, $timeouts); + $this->assertTrue($prepared['prepared']); + Carbon::setTestNow($origin->copy()->addSeconds($elapsed)); + $executor = app(LocalActivityExecutor::class); + $reply = $executor->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertTrue($reply['recorded'], json_encode($reply)); + $this->assertFalse($reply['duplicate']); + $this->assertSame($eventType, $reply['event_type']); + $decision = $reply['retry_disposition']; + $this->assertSame($classification, $decision['terminal_classification'] ?? $decision['decision']); + $this->assertSame($origin->copy()->addSeconds($elapsed)->toISOString(), $decision['failure_time']); + $this->assertSame( + $origin->copy() + ->addSeconds(100) + ->toISOString(), + $decision['original_schedule_to_close_deadline_at'] + ); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $this->assertSame(2, $event->payload['local_outcome']['version']); + $this->assertEquals($decision, $event->payload['retry_disposition']); + $this->assertSame($delay === null ? 'terminal' : 'scheduled', $decision['decision']); + $this->assertSame($delay !== null, $reply['claim_released']); + $this->assertCount($delay === null ? 0 : 1, $reply['created_task_ids']); + if ($delay !== null) { + $retry = WorkflowTask::query()->findOrFail($reply['created_task_ids'][0]); + $this->assertSame( + $origin->copy() + ->addSeconds($elapsed + $delay) + ->toISOString(), + $retry->available_at->toISOString() + ); + $this->assertSame($delay, $event->payload['retry_backoff_seconds']); + $this->assertSame(TaskStatus::Completed, $task->refresh()->status); + } else { + $this->assertSame(TaskStatus::Leased, $task->refresh()->status); + $this->assertArrayNotHasKey('retry_available_at', $decision); + } + if ($decision['failure_source'] === 'application') { + $this->assertSame($report['message'], $event->payload['message']); + $this->assertSame('SyntheticFailure', $event->payload['exception_type']); + } + $this->assertSame(1, $run->activityExecutions()->sole()->attempt_count); + $this->assertSame(1, ActivityAttempt::query()->where('workflow_run_id', $run->id)->count()); + $this->assertSame(3, $run->historyEvents()->count()); + $taskCount = $run->tasks() + ->count(); + Carbon::setTestNow($origin->copy()->addSeconds(400)); + $duplicate = $executor->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertTrue($duplicate['recorded']); + $this->assertTrue($duplicate['duplicate']); + $this->assertSame($reply['event_id'], $duplicate['event_id']); + $this->assertEquals($decision, $duplicate['retry_disposition']); + $this->assertSame(3, $run->historyEvents()->count()); + $this->assertSame($taskCount, $run->tasks()->count()); + } finally { + Carbon::setTestNow(); + } + } + + public static function retryContractOutcomes(): array + { + $failure = [ + 'outcome' => 'failed', + 'message' => 'original failure', + 'exception_type' => 'SyntheticFailure', + ]; + return [ + 'zero override before original deadline' => [99, [ + ...$failure, + 'next_retry_delay_seconds' => 0, + ], [], + 'ActivityRetryScheduled', 'scheduled', 0], + 'candidate reaches original deadline' => [99, [ + ...$failure, + 'next_retry_delay_seconds' => 1, + ], [], + 'ActivityFailed', 'retry_delay_exceeds_schedule_to_close', null], + 'fresh due deadline preempts zero override' => [100, [ + ...$failure, + 'next_retry_delay_seconds' => 0, + ], [], + 'ActivityTimedOut', 'engine_timeout', null], + 'override exceeds cap' => [1, [ + ...$failure, + 'next_retry_delay_seconds' => 6, + ], [], + 'ActivityFailed', 'retry_delay_exceeds_policy_cap', null], + 'absent override selects policy' => [1, $failure, [], 'ActivityRetryScheduled', 'scheduled', 2], + 'start timeout ignores application override' => [2, [ + ...$failure, + 'next_retry_delay_seconds' => 0, + ], + [ + 'start_to_close_timeout' => 2, + ], 'ActivityRetryScheduled', 'scheduled', 2], + 'heartbeat timeout uses policy' => [2, [ + 'outcome' => 'timed_out', + ], [ + 'heartbeat_timeout' => 2, + ], + 'ActivityRetryScheduled', 'scheduled', 2], + 'nonretryable precedes selection' => [1, [ + ...$failure, + 'non_retryable' => true, + 'next_retry_delay_seconds' => 6, + ], [], 'ActivityFailed', 'non_retryable', null], + ]; + } + + public function testRetryContractOutcomeAndItsTaskRollBackWithTheOuterTransaction(): void + { + [$run, $task] = $this->newClaim(); + $prepared = $this->prepareRetryContract($task); + $connection = ConfiguredV2Models::connection(); + $connection->beginTransaction(); + try { + $reply = app(LocalActivityExecutor::class)->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + [ + 'outcome' => 'failed', + 'message' => 'failure', + 'next_retry_delay_seconds' => 0, + ], + '1.20' + ); + $this->assertTrue($reply['recorded']); + $this->assertSame(3, $run->historyEvents()->count()); + $this->assertSame(2, $run->tasks()->count()); + } finally { + $connection->rollBack(); + } + $this->assertSame(2, $run->historyEvents()->count()); + $this->assertSame(1, $run->tasks()->count()); + $this->assertSame(TaskStatus::Leased, $task->refresh()->status); + $this->assertSame( + ActivityAttemptStatus::Running, + ActivityAttempt::query()->findOrFail($prepared['activity_attempt_id'])->status + ); + } + + public function testRetryContractSupervisorAndHeartbeatKeepTheirSeparateAuthority(): void + { + $origin = Carbon::parse('2026-11-01T00:00:00.123456Z'); + Carbon::setTestNow($origin); + try { + [$run, $task] = $this->newClaim(); + $prepared = $this->prepareRetryContract( + $task, + [ + 'start_to_close_timeout' => 10, + 'heartbeat_timeout' => 4, + ] + ); + $attemptId = $prepared['activity_attempt_id']; + $this->assertSame( + 'invalid_local_activity_heartbeat', + PortableLocalActivityControl::heartbeatWithRetryPolicy( + $attemptId, + 'portable-worker', + 1, + [ + 'step' => 1, + ], + '1.20' + )['reason'] + ); + $this->assertSame(2, $run->historyEvents()->count()); + $this->assertSame('local_activity_preparation_mismatch', PortableLocalActivityControl::poll( + $attemptId, + 'portable-worker', + 1, + true, + '1.20' + )['reason']); + Carbon::setTestNow($origin->copy()->addSecond()); + $renewed = PortableLocalActivityControl::pollWithRetryPolicy( + $attemptId, + 'portable-worker', + 1, + true, + '1.20' + ); + $this->assertTrue($renewed['active']); + $this->assertTrue($renewed['renewed']); + $this->assertFalse($renewed['heartbeat_recorded']); + $this->assertSame($renewed['lease_expires_at'], $renewed['workflow_lease_expires_at']); + $this->assertSame($prepared['start_to_close_deadline_at'], $renewed['start_to_close_deadline_at']); + $this->assertSame($prepared['schedule_to_close_deadline_at'], $renewed['schedule_to_close_deadline_at']); + $this->assertSame($prepared['heartbeat_deadline_at'], $renewed['heartbeat_deadline_at']); + $this->assertSame(2, $run->historyEvents()->count()); + Carbon::setTestNow($origin->copy()->addSeconds(2)); + $heartbeat = PortableLocalActivityControl::heartbeatWithRetryPolicy( + $attemptId, + 'portable-worker', + 1, + [ + 'current' => 1, + ], + '1.20' + ); + $this->assertTrue($heartbeat['active']); + $this->assertTrue($heartbeat['heartbeat_recorded']); + $this->assertFalse($heartbeat['renewed']); + $this->assertSame($renewed['lease_expires_at'], $heartbeat['lease_expires_at']); + $this->assertSame($origin->copy()->addSeconds(6)->toISOString(), $heartbeat['heartbeat_deadline_at']); + $this->assertSame($prepared['start_to_close_deadline_at'], $heartbeat['start_to_close_deadline_at']); + $this->assertSame($prepared['schedule_to_close_deadline_at'], $heartbeat['schedule_to_close_deadline_at']); + $this->assertSame(3, $run->historyEvents()->count()); + Carbon::setTestNow($origin->copy()->addSeconds(6)); + $expired = PortableLocalActivityControl::pollWithRetryPolicy( + $attemptId, + 'portable-worker', + 1, + true, + '1.20' + ); + $this->assertTrue($expired['stop_required']); + $this->assertSame('local_activity_deadline_expired', $expired['reason']); + $this->assertFalse($expired['renewed']); + $this->assertSame($renewed['lease_expires_at'], $expired['lease_expires_at']); + $this->assertSame(3, $run->historyEvents()->count()); + } finally { + Carbon::setTestNow(); + } + } + + public function testRetryContractSupervisorDoesNotBorrowLegacyOrForeignPreparation(): void + { + [$run, $task] = $this->newClaim(); + $prepared = $this->prepare($task); + $refused = PortableLocalActivityControl::pollWithRetryPolicy( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + true, + '1.20' + ); + $this->assertSame('local_activity_preparation_mismatch', $refused['reason']); + $refused = PortableLocalActivityControl::heartbeatWithRetryPolicy( + $prepared['activity_attempt_id'], + 'foreign-worker', + 1, + [ + 'current' => 1, + ], + '1.20' + ); + $this->assertSame('local_activity_preparation_mismatch', $refused['reason']); + $this->assertSame(2, $run->historyEvents()->count()); + $this->assertSame(TaskStatus::Leased, $task->refresh()->status); + } + + public function testRetryContractRenewalAndHeartbeatRefuseUnbindableTimestampsBeforeWrites(): void + { + config()->set('workflows.v2.workflow_task_lease_seconds', 300); + [$run, $task] = $this->newClaim(); + $latest = Carbon::instance(PreparedLocalRetryTimestamp::latestForConnection(ConfiguredV2Models::connection())); + $task->forceFill([ + 'lease_expires_at' => $latest, + ])->save(); + Carbon::setTestNow($latest->copy()->subMicroseconds(2500000)); + try { + $prepared = $this->prepareRetryContract( + $task, + [ + 'schedule_to_close_timeout' => null, + 'heartbeat_timeout' => 2, + ] + ); + $this->assertTrue($prepared['prepared']); + $execution = $run->activityExecutions() + ->sole(); + $attempt = ActivityAttempt::query()->findOrFail($prepared['activity_attempt_id']); + $executionBefore = $execution->getAttributes(); + $attemptBefore = $attempt->getAttributes(); + $taskBefore = $task->refresh() + ->getAttributes(); + Carbon::setTestNow($latest->copy()->subMicroseconds(1500000)); + foreach ([ + static fn () => PortableLocalActivityControl::heartbeatWithRetryPolicy( + $attempt->id, + 'portable-worker', + 1, + [ + 'current' => 1, + ], + '1.20' + ), + static fn () => PortableLocalActivityControl::pollWithRetryPolicy( + $attempt->id, + 'portable-worker', + 1, + true, + '1.20' + ), + ] as $operation) { + try { + $operation(); + $this->fail('Published an unbindable heartbeat or renewal timestamp.'); + } catch (\InvalidArgumentException) { + $this->addToAssertionCount(1); + } + $this->assertSame($executionBefore, $execution->refresh()->getAttributes()); + $this->assertSame($attemptBefore, $attempt->refresh()->getAttributes()); + $this->assertSame($taskBefore, $task->refresh()->getAttributes()); + $this->assertSame(2, $run->historyEvents()->count()); + } + } finally { + Carbon::setTestNow(); + } + } + + public function testRetryContractCompletionKeepsItsClosedReceiptWithoutARetryDecision(): void + { + [$run, $task] = $this->newClaim(); + $prepared = $this->prepareRetryContract($task); + $executor = app(LocalActivityExecutor::class); + $report = [ + 'outcome' => 'completed', + 'result' => Serializer::serializeWithCodec('avro', 'done'), + 'payload_codec' => 'avro', + ]; + $reply = $executor->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertTrue($reply['recorded']); + $this->assertSame('ActivityCompleted', $reply['event_type']); + $this->assertArrayNotHasKey('retry_disposition', $reply); + $this->assertTrue($executor->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + )['duplicate']); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $event->forceFill([ + 'payload' => [ + ...$event->payload, + 'retry_disposition' => [], + ], + ])->save(); + $changed = $executor->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertFalse($changed['recorded']); + $this->assertSame('local_activity_outcome_mismatch', $changed['reason']); + $this->assertSame(3, $run->historyEvents()->count()); + $this->assertSame(1, $run->tasks()->count()); + } + + public function testRetainedTerminalDecisionCannotBeChangedIntoRetryPermission(): void + { + [$run, $task] = $this->newClaim(); + $prepared = $this->prepareRetryContract($task); + $report = [ + 'outcome' => 'failed', + 'message' => 'original failure', + 'next_retry_delay_seconds' => 6, + ]; + $executor = app(LocalActivityExecutor::class); + $reply = $executor->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertTrue($reply['recorded']); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $payload = $event->payload; + $payload['retry_disposition']['decision'] = 'scheduled'; + $payload['retry_disposition']['retry_available_at'] = $payload['retry_disposition']['failure_time']; + $event->forceFill([ + 'payload' => $payload, + ])->save(); + $changed = $executor->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertFalse($changed['recorded']); + $this->assertSame('local_activity_outcome_mismatch', $changed['reason']); + $this->assertSame(3, $run->historyEvents()->count()); + $this->assertSame(1, $run->tasks()->count()); + $this->assertSame(1, $run->activityExecutions()->sole()->attempt_count); + } + + public function testRetryContractReportPreflightDoesNotGrantModeAuthority(): void + { + [$run, $task] = $this->newClaim(); + $legacy = $this->prepare($task); + $executor = app(LocalActivityExecutor::class); + $connection = ConfiguredV2Models::connection(); + $connection->enableQueryLog(); + $connection->flushQueryLog(); + try { + $refused = $executor->recordPortableRetryOutcome( + 'missing', + 'foreign', + 1, + [ + 'outcome' => 'failed', + 'message' => false, + 'next_retry_delay_seconds' => null, + ], + '1.20' + ); + $this->assertSame('invalid_activity_retry_delay', $refused['reason']); + $this->assertSame([], $connection->getQueryLog()); + } finally { + $connection->disableQueryLog(); + $connection->flushQueryLog(); + } + $refused = $executor->recordPortableRetryOutcome( + $legacy['activity_attempt_id'], + 'portable-worker', + 1, + [ + 'outcome' => 'failed', + 'message' => 'failure', + 'next_retry_delay_seconds' => 0, + ], + '1.20' + ); + $this->assertSame('local_activity_preparation_mismatch', $refused['reason']); + $this->assertSame(2, $run->historyEvents()->count()); + $refused = $executor->recordPortableRetryOutcome( + 'missing', + 'foreign', + 1, + [ + 'outcome' => 'failed', + 'message' => false, + ], + '1.20' + ); + $this->assertSame('activity_attempt_not_found', $refused['reason']); + } + public function testLostPreparationResponseReturnsTheOriginalAttemptWithoutRenewingAnyDeadline(): void { [, $task] = $this->newClaim(); @@ -1341,6 +2009,171 @@ public static function changedCanonicalScopeReceipt(): iterable yield 'started snapshot reparented' => ['started']; } + public function testRetryContractPreparesOnlyTheStoredDueRetryAndKeepsTheOriginalBudget(): void + { + $origin = Carbon::parse('2026-11-01T00:00:00.123456Z'); + Carbon::setTestNow($origin); + try { + [$run, $task] = $this->newClaim(); + $prepared = $this->prepareRetryContract($task); + $executor = app(LocalActivityExecutor::class); + $report = [ + 'outcome' => 'failed', + 'message' => 'failure', + ]; + $first = $executor->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertTrue($first['recorded']); + $retry = WorkflowTask::query()->findOrFail($first['created_task_ids'][0]); + $retry->forceFill([ + 'status' => TaskStatus::Leased, + 'attempt_count' => 1, + 'lease_owner' => 'portable-worker', + 'lease_expires_at' => $origin->copy() + ->addMinutes(5), + 'available_at' => $origin, + ])->save(); + Carbon::setTestNow($origin->copy()->addSecond()); + $early = $this->prepareRetryContract($retry, workerAttemptId: 'sdk-local-attempt-2'); + $this->assertSame('local_activity_retry_not_due', $early['reason']); + $this->assertSame(1, $run->activityExecutions()->sole()->attempt_count); + $this->assertSame(3, $run->historyEvents()->count()); + Carbon::setTestNow($origin->copy()->addSeconds(2)); + $second = $this->prepareRetryContract($retry, workerAttemptId: 'sdk-local-attempt-2'); + $this->assertTrue($second['prepared'], json_encode($second)); + $this->assertNotSame($prepared['activity_attempt_id'], $second['activity_attempt_id']); + $this->assertSame( + $origin->copy() + ->addSeconds(100) + ->toISOString(), + $run->activityExecutions() + ->sole() + ->schedule_to_close_deadline_at->toISOString() + ); + $lost = $executor->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertTrue($lost['duplicate']); + $this->assertSame($first['event_id'], $lost['event_id']); + Carbon::setTestNow($origin->copy()->addSeconds(3)); + $secondFailure = $executor->recordPortableRetryOutcome( + $second['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertSame(5, $secondFailure['retry_disposition']['candidate_delay_seconds']); + $thirdTask = WorkflowTask::query()->findOrFail($secondFailure['created_task_ids'][0]); + $thirdTask->forceFill([ + 'status' => TaskStatus::Leased, + 'attempt_count' => 1, + 'lease_owner' => 'portable-worker', + 'lease_expires_at' => $origin->copy() + ->addMinutes(5), + ])->save(); + Carbon::setTestNow($origin->copy()->addSeconds(8)); + $third = $this->prepareRetryContract($thirdTask, workerAttemptId: 'sdk-local-attempt-3'); + $this->assertTrue($third['prepared'], json_encode($third)); + $terminal = $executor->recordPortableRetryOutcome( + $third['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertSame('attempts_exhausted', $terminal['retry_disposition']['terminal_classification']); + $this->assertSame([], $terminal['created_task_ids']); + $this->assertSame(3, $run->activityExecutions()->sole()->attempt_count); + $this->assertSame(3, ActivityAttempt::query()->where('workflow_run_id', $run->id)->count()); + $this->assertSame(7, $run->historyEvents()->count()); + } finally { + Carbon::setTestNow(); + } + } + + public function testRetryContractExpiresAPendingRetryWithoutAnAdditionalAttempt(): void + { + $origin = Carbon::parse('2026-11-01T00:00:00.123456Z'); + Carbon::setTestNow($origin); + try { + [$run, $task] = $this->newClaim(); + $prepared = $this->prepareRetryContract($task); + $reply = app(LocalActivityExecutor::class)->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + [ + 'outcome' => 'failed', + 'message' => 'failure', + ], + '1.20' + ); + $retry = WorkflowTask::query()->findOrFail($reply['created_task_ids'][0]); + $retry->forceFill([ + 'status' => TaskStatus::Leased, + 'attempt_count' => 1, + 'lease_owner' => 'portable-worker', + 'lease_expires_at' => $origin->copy() + ->addMinutes(5), + ])->save(); + Carbon::setTestNow($origin->copy()->addSeconds(100)); + $expired = $this->prepareRetryContract($retry, workerAttemptId: 'sdk-local-attempt-2'); + $this->assertFalse($expired['prepared']); + $this->assertSame('local_activity_deadline_expired', $expired['reason']); + $this->assertSame('ActivityTimedOut', $expired['event_type']); + $event = WorkflowHistoryEvent::query()->findOrFail($expired['event_id']); + $this->assertSame('engine_timeout', $event->payload['retry_disposition']['terminal_classification']); + $this->assertSame( + $origin->copy() + ->addSeconds(100) + ->toISOString(), + $event->payload['retry_disposition']['failure_time'] + ); + $this->assertSame(1, $run->activityExecutions()->sole()->attempt_count); + $this->assertSame(1, ActivityAttempt::query()->where('workflow_run_id', $run->id)->count()); + $this->assertSame(2, $run->tasks()->count()); + $this->assertSame(4, $run->historyEvents()->count()); + } finally { + Carbon::setTestNow(); + } + } + + private function prepareRetryContract( + WorkflowTask $task, + array $timeouts = [], + string $workerAttemptId = 'sdk-local-attempt', + ): array { + return PortableLocalActivityPreparation::prepareWithRetryPolicy( + $task->id, + 'portable-worker', + 1, + 1, + $workerAttemptId, + [ + ...$this->descriptor(), + 'retry_policy' => [ + 'contract_version' => 2, + 'mode' => 'explicit', + 'max_attempts' => 3, + 'backoff_seconds' => [2, 5], + ], + 'schedule_to_close_timeout' => 100, + ...$timeouts, + ], + '1.20' + ); + } + /** @param list> $expected * @param list> $actual */ diff --git a/tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php b/tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php index 9bde846ec..59c036e72 100644 --- a/tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php +++ b/tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php @@ -5,11 +5,15 @@ namespace Tests\Feature\V2; use Illuminate\Support\Carbon; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Queue; +use PHPUnit\Framework\Attributes\DataProvider; use Tests\Fixtures\V2\TestGreetingWorkflow; use Tests\TestCase; +use Throwable; use Workflow\Serializers\Serializer; use Workflow\V2\Contracts\CooperativeWorkflowTaskBridge; +use Workflow\V2\Contracts\PreparedLocalActivityRetryTaskBridge; use Workflow\V2\Contracts\PreparedLocalActivityTaskBridge; use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Enums\ActivityAttemptStatus; @@ -24,6 +28,7 @@ use Workflow\V2\Models\WorkflowInstance; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; +use Workflow\V2\Support\ConfiguredV2Models; use Workflow\V2\Support\HistoryTimeline; use Workflow\V2\Support\LocalActivityCall; use Workflow\V2\Support\LocalActivityExecutor; @@ -49,6 +54,472 @@ protected function tearDown(): void parent::tearDown(); } + #[DataProvider('interruptionDelays')] + public function testRetryContractRecoveryUsesOnePolicyDecisionWithoutAdmittingAnotherAttempt(int $delay): void + { + [$run, $task, $first, $descriptor] = $this->reclaimedRetryCall([ + 'backoff_seconds' => [$delay], + ]); + $sample = now() + ->toISOString(); + $reply = $this->recoverRetryContract($task, $descriptor); + $this->assertTrue($reply['recovered'], $reply['reason'] ?? ''); + $this->assertCount(13, $reply); + $this->assertArrayNotHasKey('recorded', $reply); + $this->assertSame('unknown', $reply['callback_stop_state']); + $this->assertSame('engine_interruption', $reply['retry_disposition']['failure_source']); + $this->assertSame($sample, $reply['retry_disposition']['failure_time']); + $this->assertSame('policy', $reply['retry_disposition']['delay_source']); + $this->assertSame($delay, $reply['retry_disposition']['candidate_delay_seconds']); + $this->assertSame(now()->addSeconds($delay)->toISOString(), $reply['retry_disposition']['retry_available_at']); + $this->assertArrayNotHasKey('timeout_kind', $reply['retry_disposition']); + $this->assertArrayNotHasKey('requested_delay_seconds', $reply['retry_disposition']); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $this->assertSame('cold_replay', $event->payload['retry_reason']); + $this->assertCount(10, $event->payload['local_recovery']); + $this->assertSame(1, $event->payload['local_recovery']['version']); + $this->assertSame(7, $event->payload['local_recovery']['original_workflow_task_attempt']); + $this->assertSame(8, $event->payload['local_recovery']['workflow_task_attempt']); + $attempt = ActivityAttempt::query()->findOrFail($first['activity_attempt_id']); + $this->assertSame(ActivityAttemptStatus::Expired, $attempt->status); + $this->assertSame($sample, $attempt->closed_at->toISOString()); + $this->assertSame(TaskStatus::Completed, $task->refresh()->status); + $this->assertCount(1, $reply['created_task_ids']); + $this->assertSame(1, ActivityAttempt::query()->where('workflow_run_id', $run->id)->count()); + $this->assertSame(2, $run->tasks()->count()); + $this->assertRecoveryReply([ + ...$reply, + 'duplicate' => true, + ], $this->recoverRetryContract($task, $descriptor)); + $this->assertSame(3, $run->historyEvents()->count()); + } + + public static function interruptionDelays(): iterable + { + yield 'zero remains selected' => [0]; + yield 'positive policy delay' => [2]; + } + + #[DataProvider('interruptionTerminals')] + public function testRetryContractRecoveryTerminatesWithoutAnExtraTaskOrAttempt(array $policy, string $reason): void + { + [$run, $task, $first, $descriptor] = $this->reclaimedRetryCall($policy); + $before = $task->getAttributes(); + $reply = $this->recoverRetryContract($task, $descriptor); + $this->assertTrue($reply['recovered'], $reply['reason'] ?? ''); + $this->assertSame('ActivityFailed', $reply['event_type']); + $this->assertSame('engine_interruption', $reply['retry_disposition']['failure_source']); + $this->assertSame($reason, $reply['retry_disposition']['terminal_classification']); + $this->assertArrayNotHasKey('retry_available_at', $reply['retry_disposition']); + $this->assertSame([], $reply['created_task_ids']); + $this->assertFalse($reply['claim_released']); + $this->assertSame($before, $task->refresh()->getAttributes()); + $this->assertSame(1, $run->tasks()->count()); + $this->assertSame(1, ActivityExecution::query()->findOrFail($first['activity_execution_id'])->attempt_count); + $this->assertRecoveryReply([ + ...$reply, + 'duplicate' => true, + ], $this->recoverRetryContract($task, $descriptor)); + } + + public static function interruptionTerminals(): iterable + { + yield 'policy nonretryable precedes budget' => [[ + 'max_attempts' => 1, + 'non_retryable_error_types' => [\RuntimeException::class], + ], 'non_retryable']; + yield 'attempt budget' => [[ + 'max_attempts' => 1, + ], 'attempts_exhausted']; + yield 'original total deadline' => [[ + 'backoff_seconds' => [30], + ], 'retry_delay_exceeds_schedule_to_close']; + } + + #[DataProvider('recoveryTimeouts')] + public function testRetryContractRecoveryRetainsActualTimeoutAuthority( + int $total, + int $start, + array $timeouts, + string $kind, + bool $scheduled + ): void { + [$run, $task, , $descriptor] = $this->reclaimedRetryCall( + totalTimeout: $total, + startTimeout: $start, + timeouts: $timeouts + ); + $reply = $this->recoverRetryContract($task, $descriptor); + $this->assertTrue($reply['recovered'], $reply['reason'] ?? ''); + $decision = $reply['retry_disposition']; + $this->assertSame('engine_timeout', $decision['failure_source']); + $this->assertSame($kind, $decision['timeout_kind']); + $this->assertSame($scheduled ? 'ActivityRetryScheduled' : 'ActivityTimedOut', $reply['event_type']); + $this->assertSame($scheduled, $reply['claim_released']); + $this->assertCount($scheduled ? 1 : 0, $reply['created_task_ids']); + $this->assertSame($scheduled ? 2 : 1, $run->tasks()->count()); + $this->assertRecoveryReply([ + ...$reply, + 'duplicate' => true, + ], $this->recoverRetryContract($task, $descriptor)); + } + + public static function recoveryTimeouts(): iterable + { + yield 'total deadline precedes interruption' => [4, 4, [], 'schedule_to_close', false]; + yield 'start deadline still retries policy' => [30, 4, [], 'start_to_close', true]; + yield 'heartbeat precedes start timeout' => [30, 4, [ + 'heartbeat_timeout' => 3, + ], 'heartbeat', true]; + } + + public function testRetryRecoveryDuplicateAfterDeadlineKeepsItsOriginalAttemptAndDecision(): void + { + [$run, $task, $first, $descriptor] = $this->reclaimedRetryCall(); + $reply = $this->recoverRetryContract($task, $descriptor); + $this->assertTrue($reply['recovered'], $reply['reason'] ?? ''); + $retry = WorkflowTask::query()->findOrFail($reply['created_task_ids'][0]); + $retry->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'next-worker', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addMinute(), + ])->save(); + Carbon::setTestNow(now()->addSeconds(2)); + $next = PortableLocalActivityPreparation::prepareWithRetryPolicy( + $retry->id, + 'next-worker', + 1, + 1, + 'next-local-attempt', + $descriptor, + '1.20' + ); + $this->assertTrue($next['prepared'], $next['reason'] ?? ''); + $this->assertNotSame($first['activity_attempt_id'], $next['activity_attempt_id']); + Carbon::setTestNow(now()->addMinutes(2)); + config() + ->set('workflows.v2.default_retry_policy', [ + 'max_attempts' => 1, + ]); + $before = $run->historyEvents() + ->count(); + $this->assertRecoveryReply([ + ...$reply, + 'duplicate' => true, + ], $this->recoverRetryContract($task, $descriptor)); + $this->assertSame($before, $run->historyEvents()->count()); + $this->assertSame('unknown', $reply['callback_stop_state']); + $this->assertSame(2, ActivityExecution::query()->findOrFail($first['activity_execution_id'])->attempt_count); + } + + #[DataProvider('recoverySnapshotMutations')] + public function testCorruptRecoveryAuthorityCannotPublishOrReturnAReceipt(bool $committed, bool $invalidCap): void + { + [$run, $task, $first, $descriptor] = $this->reclaimedRetryCall([ + 'backoff_seconds' => [5], + 'maximum_interval_seconds' => 20, + ]); + $reply = $committed ? $this->recoverRetryContract($task, $descriptor) : null; + $execution = ActivityExecution::query()->findOrFail($first['activity_execution_id']); + $execution->forceFill([ + 'retry_policy' => [ + ...$execution->retry_policy, + 'backoff_seconds' => [$invalidCap ? 21 : 3], + ], + ])->save(); + $before = [$run->historyEvents()->count(), $run->tasks()->count(), $execution->refresh()->getAttributes(), + ActivityAttempt::query()->findOrFail($first['activity_attempt_id'])->getAttributes()]; + $rejected = $this->recoverRetryContract($task, $descriptor); + $this->assertFalse($rejected['recovered']); + $this->assertSame('local_activity_preparation_mismatch', $rejected['reason']); + $this->assertArrayNotHasKey('retry_disposition', $rejected); + $this->assertArrayNotHasKey('event_id', $rejected); + $this->assertSame( + $before, + [$run->historyEvents()->count(), $run->tasks()->count(), $execution->refresh()->getAttributes(), + ActivityAttempt::query()->findOrFail($first['activity_attempt_id'])->getAttributes()] + ); + if ($reply !== null) { + $this->assertSame( + $reply['retry_disposition'], + WorkflowHistoryEvent::query()->findOrFail($reply['event_id'])->payload['retry_disposition'] + ); + } + } + + public static function recoverySnapshotMutations(): iterable + { + yield 'invalid retained cap before selection' => [false, true]; + yield 'valid but replaced original snapshot before selection' => [false, false]; + yield 'invalid retained cap after committed receipt' => [true, true]; + yield 'valid but replaced original snapshot after committed receipt' => [true, false]; + } + + #[DataProvider('recoveryReceiptMutations')] + public function testRecoveryReceiptCannotChangeItsClosedIdentityOrDecision(string $location, mixed $value): void + { + [$run, $task, , $descriptor] = $this->reclaimedRetryCall(); + $reply = $this->recoverRetryContract($task, $descriptor); + $this->assertTrue($reply['recovered'], $reply['reason'] ?? ''); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $payload = $event->payload; + data_set($payload, $location, $value); + $event->forceFill([ + 'payload' => $payload, + ])->save(); + $before = $run->historyEvents() + ->count(); + $refused = $this->recoverRetryContract($task, $descriptor); + $this->assertSame('local_activity_recovery_mismatch', $refused['reason']); + $this->assertArrayNotHasKey('event_id', $refused); + $this->assertSame($before, $run->historyEvents()->count()); + $expectedRecovery = $payload['local_recovery']; + $actualRecovery = $event->refresh() +->payload['local_recovery']; + ksort($expectedRecovery); + ksort($actualRecovery); + $this->assertSame($expectedRecovery, $actualRecovery); + } + + public static function recoveryReceiptMutations(): iterable + { + yield 'worker source substitution' => ['retry_disposition.failure_source', 'application']; + yield 'worker override injection' => ['retry_disposition.requested_delay_seconds', 1]; + yield 'timeout injection' => ['retry_disposition.timeout_kind', 'heartbeat']; + yield 'unknown metadata member' => ['local_recovery.extra', true]; + yield 'stop state cannot be rewritten' => ['local_recovery.callback_stop_state', 'stopped']; + yield 'original owner substituted' => ['local_recovery.original_lease_owner', 'foreign']; + yield 'retry delay substituted' => ['retry_backoff_seconds', 0]; + yield 'outcome and recovery authority mixed' => ['local_outcome.version', 2]; + } + + public function testRetryPreparationRefusesMixedOutcomeAndRecoveryAuthority(): void + { + [$run, $task, $first, $descriptor] = $this->reclaimedRetryCall(); + $reply = $this->recoverRetryContract($task, $descriptor); + $this->assertTrue($reply['recovered'], $reply['reason'] ?? ''); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $event->forceFill([ + 'payload' => [ + ...$event->payload, + 'local_outcome' => [ + 'version' => 2, + ], + ], + ])->save(); + $retry = WorkflowTask::query()->findOrFail($reply['created_task_ids'][0]); + $retry->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'next-worker', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addMinute(), + ])->save(); + Carbon::setTestNow(now()->addSeconds(2)); + $refused = PortableLocalActivityPreparation::prepareWithRetryPolicy( + $retry->id, + 'next-worker', + 1, + 1, + 'next-local-attempt', + $descriptor, + '1.20' + ); + $this->assertSame('local_activity_retry_preparation_mismatch', $refused['reason']); + $this->assertSame(3, $run->historyEvents()->count()); + $this->assertSame(1, ActivityExecution::query()->findOrFail($first['activity_execution_id'])->attempt_count); + } + + public function testRetryRecoveryRollbackPublishesNoFailureSampleReceiptOrTask(): void + { + [$run, $task, $first, $descriptor] = $this->reclaimedRetryCall(); + $connection = ConfiguredV2Models::connection(); + $before = $task->getAttributes(); + $connection->beginTransaction(); + try { + $reply = $this->recoverRetryContract($task, $descriptor); + $this->assertTrue($reply['recovered'], $reply['reason'] ?? ''); + $this->assertSame('engine_interruption', $reply['retry_disposition']['failure_source']); + $this->assertSame(3, $run->historyEvents()->count()); + } finally { + $connection->rollBack(); + } + $this->assertSame(2, $run->historyEvents()->count()); + $this->assertSame(1, $run->tasks()->count()); + $this->assertSame($before, $task->refresh()->getAttributes()); + $this->assertSame( + ActivityAttemptStatus::Running, + ActivityAttempt::query()->findOrFail($first['activity_attempt_id'])->status + ); + } + + public function testRetryRecoveryCannotUseAStillLiveIndependentOriginalClaim(): void + { + [$run, $original, , $descriptor] = $this->reclaimedRetryCall(reclaim: false); + $task = $original->replicate(); + $task->forceFill([ + 'lease_owner' => 'replacement-worker', + 'attempt_count' => 8, + 'lease_expires_at' => now() + ->addMinute(), + ])->save(); + $original->forceFill([ + 'lease_expires_at' => now() + ->addMinute(), + ])->save(); + $reply = $this->recoverRetryContract($task, $descriptor); + $this->assertSame('local_activity_original_claim_live', $reply['reason']); + $this->assertSame(2, $run->historyEvents()->count()); + } + + public function testRetryRecoveryKeepsIndependentRecoveringAndOriginalTaskIdentities(): void + { + [$run, $original, $first, $descriptor] = $this->reclaimedRetryCall(reclaim: false); + $task = $original->replicate(); + $task->forceFill([ + 'lease_owner' => 'replacement-worker', + 'attempt_count' => 8, + 'lease_expires_at' => now() + ->addMinute(), + ])->save(); + $reply = $this->recoverRetryContract($task, $descriptor); + $this->assertTrue($reply['recovered'], $reply['reason'] ?? ''); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $this->assertSame($task->id, $reply['workflow_task_id']); + $this->assertSame($task->id, $event->payload['local_recovery']['workflow_task_id']); + $this->assertSame($original->id, $event->payload['local_recovery']['original_workflow_task_id']); + $this->assertSame($original->id, $event->payload['activity_attempt']['task_id']); + $this->assertSame(TaskStatus::Completed, $original->refresh()->status); + $this->assertSame(TaskStatus::Completed, $task->refresh()->status); + $this->assertSame(1, ActivityExecution::query()->findOrFail($first['activity_execution_id'])->attempt_count); + $this->assertSame(3, $run->tasks()->count()); + } + + public function testRetryRecoveryCannotUpgradeAnOriginalVersionOnePreparation(): void + { + [$run, $task, , $descriptor] = $this->reclaimedCall(); + $versioned = [ + ...$descriptor, + 'retry_policy' => [ + 'contract_version' => 2, + 'mode' => 'explicit', + 'max_attempts' => 3, + 'backoff_seconds' => [2], + ], + ]; + $refused = $this->recoverRetryContract($task, $versioned); + $this->assertSame('local_activity_preparation_mismatch', $refused['reason']); + $this->assertSame(2, $run->historyEvents()->count()); + $legacy = $this->recover($task, $descriptor); + $this->assertTrue($legacy['recovered']); + $this->assertArrayNotHasKey('retry_disposition', $legacy); + } + + public function testConcurrentRetryRecoveriesSerializeOneDecisionAndReturnTheOriginalAfterDeadline(): void + { + if (ConfiguredV2Models::connection()->getDriverName() !== 'mysql' + || ! function_exists('pcntl_fork') || ! function_exists('posix_kill')) { + $this->markTestSkipped('The two-process lock control requires MySQL and pcntl.'); + } + self::stopWorkers(); + [$run, $task, $first, $descriptor] = $this->reclaimedRetryCall(); + $connectionName = ConfiguredV2Models::connection()->getName(); + $table = (new ActivityAttempt())->getTable(); + $sockets = stream_socket_pair(STREAM_PF_UNIX, STREAM_SOCK_STREAM, STREAM_IPPROTO_IP); + $this->assertIsArray($sockets); + stream_set_timeout($sockets[0], 10); + stream_set_timeout($sockets[1], 10); + DB::purge($connectionName); + $pid = pcntl_fork(); + $this->assertNotSame(-1, $pid); + if ($pid === 0) { + fclose($sockets[0]); + try { + $connection = DB::reconnect($connectionName); + $announced = false; + $connection->beforeExecuting(static function (string $query) use ($sockets, $table, &$announced): void { + if (! $announced && str_contains($query, $table) && str_contains( + strtolower($query), + 'for update' + )) { + $announced = true; + fwrite($sockets[1], "locking\n"); + } + }); + fwrite($sockets[1], "ready\n"); + if (fgets($sockets[1]) !== "recover\n") { + throw new \LogicException('Recovery race was not released.'); + } + Carbon::setTestNow(now()->addMinutes(2)); + $reply = PortableLocalActivityPreparation::recoverWithRetryPolicy( + $task->id, + 'replacement-worker', + 8, + 1, + $descriptor, + '1.20' + ); + fwrite($sockets[1], json_encode($reply, JSON_THROW_ON_ERROR) . "\n"); + fclose($sockets[1]); + exit(0); + } catch (Throwable $error) { + fwrite($sockets[1], json_encode([ + 'error' => substr($error->getMessage(), 0, 800), + ]) . "\n"); + fclose($sockets[1]); + exit(1); + } + } + fclose($sockets[1]); + $connection = DB::reconnect($connectionName); + try { + $this->assertSame("ready\n", fgets($sockets[0])); + $connection->beginTransaction(); + ActivityAttempt::query()->lockForUpdate()->findOrFail($first['activity_attempt_id']); + fwrite($sockets[0], "recover\n"); + $this->assertSame("locking\n", fgets($sockets[0])); + $winner = $this->recoverRetryContract($task, $descriptor); + $this->assertTrue($winner['recovered'], $winner['reason'] ?? ''); + $connection->commit(); + $line = fgets($sockets[0]); + $this->assertIsString($line); + $loser = json_decode($line, true, flags: JSON_THROW_ON_ERROR); + $this->assertRecoveryReply([ + ...$winner, + 'duplicate' => true, + ], $loser); + $until = microtime(true) + 5; + do { + $exited = pcntl_waitpid($pid, $status, WNOHANG); + if ($exited === 0) { + usleep(10000); + } + } while ($exited === 0 && microtime(true) < $until); + $this->assertSame($pid, $exited); + $pid = 0; + $this->assertSame(0, pcntl_wexitstatus($status)); + $this->assertSame(3, $run->historyEvents()->count()); + $this->assertSame(2, $run->tasks()->count()); + $this->assertSame( + 1, + ActivityExecution::query()->findOrFail($first['activity_execution_id'])->attempt_count + ); + } finally { + if ($connection->transactionLevel() > 0) { + $connection->rollBack(); + } + fclose($sockets[0]); + if ($pid > 0) { + posix_kill($pid, SIGKILL); + pcntl_waitpid($pid, $status); + } + DB::purge($connectionName); + DB::reconnect($connectionName); + } + } + public function testColdRecoveryFencesTheOldAttemptAndPreparesARealRetryUnderTheOriginalBudget(): void { [$run, $task, $first, $descriptor] = $this->reclaimedCall(); @@ -373,12 +844,15 @@ public function testProtocol119CannotEnterThePreparedRecoveryPath(): void public function testPreparedLocalRoleUsesTheExistingWorkflowBindingWithoutExpandingCustomBridges(): void { $this->assertSame(app(WorkflowTaskBridge::class), $this->bridge()); + $this->assertFalse($this->app->bound(PreparedLocalActivityRetryTaskBridge::class)); + $this->assertNotInstanceOf(PreparedLocalActivityRetryTaskBridge::class, $this->bridge()); foreach ([WorkflowTaskBridge::class, CooperativeWorkflowTaskBridge::class] as $contract) { $custom = \Mockery::mock($contract); $this->app->instance(WorkflowTaskBridge::class, $custom); $resolved = $this->app->make(PreparedLocalActivityTaskBridge::class); $this->assertSame($custom, $resolved); $this->assertNotInstanceOf(PreparedLocalActivityTaskBridge::class, $resolved); + $this->assertNotInstanceOf(PreparedLocalActivityRetryTaskBridge::class, $resolved); } } @@ -400,6 +874,52 @@ public function testLocalStopAdmissionRejectsProtocol119ThroughTheOptionalRole() $this->assertSame($before, $run->historyEvents()->count()); } + private function assertRecoveryReply(array $expected, array $actual): void + { + // Native JSON storage may reorder object keys. Preserve exact scalar + // types and every field while comparing the closed disposition. + if (isset($expected['retry_disposition'], $actual['retry_disposition'])) { + ksort($expected['retry_disposition']); + ksort($actual['retry_disposition']); + } + $this->assertSame($expected, $actual); + } + + private function recoverRetryContract(WorkflowTask $task, array $descriptor): array + { + return PortableLocalActivityPreparation::recoverWithRetryPolicy( + $task->id, + 'replacement-worker', + 8, + 1, + $descriptor, + '1.20' + ); + } + + private function reclaimedRetryCall( + array $policy = [], + int $totalTimeout = 30, + int $startTimeout = 20, + bool $reclaim = true, + array $timeouts = [], + ): array { + return $this->reclaimedCall( + maxAttempts: $policy['max_attempts'] ?? 3, + totalTimeout: $totalTimeout, + startTimeout: $startTimeout, + reclaim: $reclaim, + versionedPolicy: [ + 'contract_version' => 2, + 'mode' => 'explicit', + 'max_attempts' => 3, + 'backoff_seconds' => [2], + ...$policy, + ], + timeouts: $timeouts + ); + } + private function bridge(): PreparedLocalActivityTaskBridge { return $this->app->make(PreparedLocalActivityTaskBridge::class); @@ -453,6 +973,8 @@ private function reclaimedCall( int $totalTimeout = 30, int $startTimeout = 20, bool $reclaim = true, + ?array $versionedPolicy = null, + array $timeouts = [], ): array { $instance = WorkflowInstance::query()->create([ 'workflow_class' => TestGreetingWorkflow::class, @@ -502,9 +1024,22 @@ private function reclaimedCall( 'max_attempts' => $maxAttempts, 'backoff_seconds' => [2], ], + ...$timeouts, ]; - $first = $this->bridge() - ->prepareLocalActivity($task->id, 'original-worker', 7, 1, 'original-local-attempt', $descriptor, '1.20'); + if ($versionedPolicy !== null) { + $descriptor['retry_policy'] = $versionedPolicy; + } + $first = $versionedPolicy === null ? $this->bridge() + ->prepareLocalActivity($task->id, 'original-worker', 7, 1, 'original-local-attempt', $descriptor, '1.20') + : PortableLocalActivityPreparation::prepareWithRetryPolicy( + $task->id, + 'original-worker', + 7, + 1, + 'original-local-attempt', + $descriptor, + '1.20' + ); $this->assertTrue($first['prepared']); Carbon::setTestNow(now()->addSeconds($elapsed)); if ($reclaim) { diff --git a/tests/Fixtures/V2/ReplayRegression/prepared-v1-policy-refusal-legacy-zero-tail.json b/tests/Fixtures/V2/ReplayRegression/prepared-v1-policy-refusal-legacy-zero-tail.json new file mode 100644 index 000000000..2de7b7de7 --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/prepared-v1-policy-refusal-legacy-zero-tail.json @@ -0,0 +1,211 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "prepared-v1-policy-refusal-legacy-zero-tail", + "protocol_version": "1.20", + "bindings": [ + "php" + ], + "consumers": [ + "workflow-fiber-runner", + "workflow-executor" + ], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestLocalActivityWorkflow", + "arguments": [ + "Ada" + ], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "recorded_at": "2026-11-05T00:02:04.000001Z", + "payload": {} + }, + { + "sequence": 2, + "event_type": "ActivityScheduled", + "recorded_at": "2026-11-05T00:02:04.000002Z", + "payload": { + "sequence": 1, + "activity_execution_id": "01j00000000000000000000001", + "activity_type": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "execution_mode": "local", + "local_activity": true, + "activity": { + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 3, + "backoff_seconds": [ + 120 + ], + "start_to_close_timeout": null, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null, + "non_retryable_error_types": [] + } + } + } + }, + { + "sequence": 3, + "event_type": "ActivityStarted", + "recorded_at": "2026-11-05T00:02:04.000003Z", + "payload": { + "sequence": 1, + "activity_execution_id": "01j00000000000000000000001", + "activity_type": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "execution_mode": "local", + "local_activity": true, + "activity_attempt_id": "01j0000000000000000000000a", + "attempt_number": 1, + "worker_attempt_id": "legacy-local-1" + } + }, + { + "sequence": 4, + "event_type": "ActivityRetryScheduled", + "recorded_at": "2026-11-05T00:02:04.000004Z", + "payload": { + "sequence": 1, + "activity_execution_id": "01j00000000000000000000001", + "activity_type": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "execution_mode": "local", + "local_activity": true, + "activity_attempt_id": "01j0000000000000000000000a", + "worker_attempt_id": "legacy-local-1", + "retry_reason": "failure", + "retry_backoff_seconds": 120, + "activity": { + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 3, + "backoff_seconds": [ + 120 + ], + "start_to_close_timeout": null, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null, + "non_retryable_error_types": [] + } + }, + "retry_after_attempt_id": "01j0000000000000000000000a", + "retry_after_attempt": 1, + "message": "Legacy SDK local attempt failed." + } + }, + { + "sequence": 5, + "event_type": "ActivityStarted", + "recorded_at": "2026-11-05T00:02:04.000005Z", + "payload": { + "sequence": 1, + "activity_execution_id": "01j00000000000000000000001", + "activity_type": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "execution_mode": "local", + "local_activity": true, + "activity_attempt_id": "01j0000000000000000000000b", + "attempt_number": 2, + "worker_attempt_id": "legacy-local-2" + } + }, + { + "sequence": 6, + "event_type": "ActivityRetryScheduled", + "recorded_at": "2026-11-05T00:02:04.000006Z", + "payload": { + "sequence": 1, + "activity_execution_id": "01j00000000000000000000001", + "activity_type": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "execution_mode": "local", + "local_activity": true, + "activity_attempt_id": "01j0000000000000000000000b", + "worker_attempt_id": "legacy-local-2", + "retry_reason": "failure", + "retry_backoff_seconds": 0, + "activity": { + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 3, + "backoff_seconds": [ + 120 + ], + "start_to_close_timeout": null, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null, + "non_retryable_error_types": [] + } + }, + "retry_after_attempt_id": "01j0000000000000000000000b", + "retry_after_attempt": 2, + "message": "Legacy SDK local attempt failed." + } + }, + { + "sequence": 7, + "event_type": "ActivityStarted", + "recorded_at": "2026-11-05T00:02:04.000007Z", + "payload": { + "sequence": 1, + "activity_execution_id": "01j00000000000000000000001", + "activity_type": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "execution_mode": "local", + "local_activity": true, + "activity_attempt_id": "01j0000000000000000000000c", + "attempt_number": 3, + "worker_attempt_id": "legacy-local-3" + } + }, + { + "sequence": 8, + "event_type": "ActivityCompleted", + "recorded_at": "2026-11-05T00:02:04.000008Z", + "payload": { + "sequence": 1, + "activity_execution_id": "01j00000000000000000000001", + "activity_type": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "execution_mode": "local", + "local_activity": true, + "activity_attempt_id": "01j0000000000000000000000c", + "attempt_number": 3, + "worker_attempt_id": "legacy-local-3", + "result": "wwHioz3/VYAiNwoWSGVsbG8sIEFkYSE=", + "payload_codec": "avro" + } + } + ], + "expected": { + "completed": true, + "result": { + "greeting": "Hello, Ada!", + "workflow_id": "regression-corpus-prepared-v1-policy-refusal-legacy-zero-tail", + "run_id": "regression-corpus-run-prepared-v1-policy-refusal-legacy-zero-tail" + }, + "commands": [ + { + "type": "complete_workflow" + } + ] + }, + "unqualified_preparation_policy": { + "contract_version": 2, + "mode": "explicit", + "max_attempts": 3, + "backoff_seconds": [ + 120 + ], + "maximum_interval_seconds": 120 + } +} diff --git a/tests/Unit/V2/PreparedLocalActivityReportTest.php b/tests/Unit/V2/PreparedLocalActivityReportTest.php index 5672276ad..e73cf400d 100644 --- a/tests/Unit/V2/PreparedLocalActivityReportTest.php +++ b/tests/Unit/V2/PreparedLocalActivityReportTest.php @@ -131,7 +131,8 @@ public function testNoOverrideReportStillHasAClosedVersionTwoGrammar(): void ]; } foreach (['retry_disposition', 'backoff_seconds', 'retry_backoff_seconds', 'retry_available_at', - 'failure_time', 'original_schedule_to_close_deadline_at', 'snapshot_version', 'extra'] as $field) { + 'failure_time', 'original_schedule_to_close_deadline_at', 'snapshot_version', 'failure_source', + 'requested_delay_seconds', 'local_recovery', 'callback_stop_state', 'extra'] as $field) { $invalid[] = [ 'outcome' => 'failed', 'message' => '', @@ -142,6 +143,9 @@ public function testNoOverrideReportStillHasAClosedVersionTwoGrammar(): void $field => 0, ]; } + $invalid[] = [ + 'outcome' => 'engine_interruption', + ]; $invalid[] = [ 'outcome' => 'timed_out', 'timeout_kind' => 'heartbeat', diff --git a/tests/Unit/V2/PreparedLocalActivityRetryTaskBridgeTest.php b/tests/Unit/V2/PreparedLocalActivityRetryTaskBridgeTest.php new file mode 100644 index 000000000..30c6d003d --- /dev/null +++ b/tests/Unit/V2/PreparedLocalActivityRetryTaskBridgeTest.php @@ -0,0 +1,115 @@ +assertTrue($role->isInterface()); + $this->assertTrue($role->isSubclassOf(PreparedLocalActivityTaskBridge::class)); + $newMethods = []; + foreach ($role->getMethods() as $method) { + if ($method->getDeclaringClass()->getName() === $role->getName()) { + $newMethods[] = $method->getName(); + } + } + sort($newMethods); + $this->assertSame([ + 'controlLocalActivityWithRetryPolicy', + 'heartbeatLocalActivityWithRetryPolicy', + 'prepareLocalActivityWithRetryPolicy', + 'recordLocalActivityRetryOutcome', + 'recoverLocalActivityWithRetryPolicy', + ], $newMethods); + } + + public function testVersionedOperationSignaturesKeepIdentityOrderingAndExistingProtocolDefault(): void + { + $role = new ReflectionClass(PreparedLocalActivityRetryTaskBridge::class); + $identity = [ + 'taskId' => 'string', + 'leaseOwner' => 'string', + 'workflowTaskAttempt' => 'int', + ]; + $attempt = [ + 'attemptId' => 'string', + 'leaseOwner' => 'string', + 'workflowTaskAttempt' => 'int', + ]; + $protocol = [ + 'protocolVersion' => 'string', + ]; + foreach ([ + 'prepareLocalActivityWithRetryPolicy' => [ + ...$identity, + 'sequence' => 'int', + 'workerAttemptId' => 'string', + 'descriptor' => 'array', + ...$protocol, + ], + 'recordLocalActivityRetryOutcome' => [ + ...$attempt, + 'report' => 'array', + ...$protocol, + ], + 'controlLocalActivityWithRetryPolicy' => [ + ...$attempt, + 'renewLease' => 'bool', + ...$protocol, + ], + 'heartbeatLocalActivityWithRetryPolicy' => [ + ...$attempt, + 'progress' => 'array', + ...$protocol, + ], + 'recoverLocalActivityWithRetryPolicy' => [ + ...$identity, + 'sequence' => 'int', + 'descriptor' => 'array', + ...$protocol, + ], + ] as $name => $expected) { + $method = $role->getMethod($name); + $actual = []; + foreach ($method->getParameters() as $parameter) { + $this->assertFalse($parameter->isPassedByReference()); + $this->assertFalse($parameter->isVariadic()); + $this->assertFalse($parameter->allowsNull()); + $actual[$parameter->getName()] = (string) $parameter->getType(); + if ($parameter->getName() === 'protocolVersion') { + $this->assertTrue($parameter->isDefaultValueAvailable()); + $this->assertSame(WorkerProtocolVersion::VERSION, $parameter->getDefaultValue()); + } elseif ($parameter->getName() === 'renewLease') { + $this->assertSame(false, $parameter->getDefaultValue()); + } elseif ($parameter->getName() === 'progress') { + $this->assertSame([], $parameter->getDefaultValue()); + } else { + $this->assertFalse($parameter->isOptional()); + } + } + $this->assertSame($expected, $actual, $name); + $this->assertSame('array', (string) $method->getReturnType()); + } + } + + public function testExistingDefaultAndCustomVersionOneBridgesDoNotGainTheOptionalRole(): void + { + $this->assertFalse(is_a(DefaultWorkflowTaskBridge::class, PreparedLocalActivityRetryTaskBridge::class, true)); + foreach ([WorkflowTaskBridge::class, PreparedLocalActivityTaskBridge::class] as $contract) { + $bridge = \Mockery::mock($contract); + $this->assertNotInstanceOf(PreparedLocalActivityRetryTaskBridge::class, $bridge); + } + } +} diff --git a/tests/Unit/V2/PreparedLocalRetryDecisionTest.php b/tests/Unit/V2/PreparedLocalRetryDecisionTest.php new file mode 100644 index 000000000..8749bc890 --- /dev/null +++ b/tests/Unit/V2/PreparedLocalRetryDecisionTest.php @@ -0,0 +1,732 @@ +modify('+10 seconds'), $sample->modify('+100 seconds')); + foreach ([0, 2] as $delay) { + $snapshot = [ + ...$this->snapshot(), + 'backoff_seconds' => [$delay], + 'maximum_interval_seconds' => $delay, + ]; + $decision = PreparedLocalRetryDecision::selectInterruption($snapshot, 1, $sample, $window); + self::assertSame('engine_interruption', $decision['failure_source']); + self::assertSame('scheduled', $decision['decision']); + self::assertSame('policy', $decision['delay_source']); + self::assertSame($delay, $decision['candidate_delay_seconds']); + self::assertSame( + $sample->modify('+' . $delay . ' seconds')->format('Y-m-d\TH:i:s.u\Z'), + $decision['retry_available_at'] + ); + self::assertArrayNotHasKey('requested_delay_seconds', $decision); + self::assertArrayNotHasKey('timeout_kind', $decision); + self::assertTrue( + PreparedLocalRetryDecision::matchesStoredInterruption($snapshot, 1, $window, array_reverse( + $decision, + true + )) + ); + } + } + + public function testInterruptionKeepsNonRetryableBudgetAndOriginalDeadlinePrecedence(): void + { + $sample = new DateTimeImmutable('2026-11-05T00:00:00.000000Z'); + $window = new PreparedLocalRetryWindow($sample->modify('+2 seconds'), $sample->modify('+100 seconds')); + foreach ([ + [ + 'snapshot' => [ + ...$this->snapshot(), + 'non_retryable_error_types' => [\RuntimeException::class], + ], + 'attempt' => 3, + 'reason' => 'non_retryable', + ], + [ + 'snapshot' => $this->snapshot(), + 'attempt' => 3, + 'reason' => 'attempts_exhausted', + ], + [ + 'snapshot' => $this->snapshot(), + 'attempt' => 1, + 'reason' => 'retry_delay_exceeds_schedule_to_close', + ], + ] as $case) { + $decision = PreparedLocalRetryDecision::selectInterruption( + $case['snapshot'], + $case['attempt'], + $sample, + $window + ); + self::assertSame('terminal', $decision['decision']); + self::assertSame('engine_interruption', $decision['failure_source']); + self::assertSame($case['reason'], $decision['terminal_classification']); + self::assertArrayNotHasKey('retry_available_at', $decision); + self::assertTrue( + PreparedLocalRetryDecision::matchesStoredInterruption( + $case['snapshot'], + $case['attempt'], + $window, + $decision + ) + ); + if ($case['reason'] !== 'retry_delay_exceeds_schedule_to_close') { + self::assertArrayNotHasKey('candidate_delay_seconds', $decision); + self::assertArrayNotHasKey('delay_source', $decision); + } + } + } + + public function testActualDueTimeoutKeepsAuthorityAheadOfInterruption(): void + { + $sample = new DateTimeImmutable('2026-11-05T00:00:00.000000Z'); + $latest = $sample->modify('+100 seconds'); + $snapshot = [ + ...$this->snapshot(), + 'non_retryable_error_types' => [\RuntimeException::class], + ]; + foreach (['start_to_close', 'heartbeat'] as $kind) { + $window = new PreparedLocalRetryWindow($sample->modify('+10 seconds'), $latest); + $decision = PreparedLocalRetryDecision::selectInterruption($snapshot, 1, $sample, $window, $kind); + self::assertSame('engine_timeout', $decision['failure_source']); + self::assertSame($kind, $decision['timeout_kind']); + self::assertSame('scheduled', $decision['decision']); + self::assertSame('policy', $decision['delay_source']); + self::assertTrue(PreparedLocalRetryDecision::matchesStoredInterruption($snapshot, 1, $window, $decision)); + } + $window = new PreparedLocalRetryWindow($sample, $latest); + $decision = PreparedLocalRetryDecision::selectInterruption($snapshot, 3, $sample, $window); + self::assertSame('engine_timeout', $decision['failure_source']); + self::assertSame('schedule_to_close', $decision['timeout_kind']); + self::assertSame('engine_timeout', $decision['terminal_classification']); + self::assertArrayNotHasKey('candidate_delay_seconds', $decision); + } + + public function testRetainedInterruptionCannotGainWorkerOverrideOrTimeoutAuthority(): void + { + $sample = new DateTimeImmutable('2026-11-05T00:00:00.000000Z'); + $window = new PreparedLocalRetryWindow(null, $sample->modify('+100 seconds')); + $stored = PreparedLocalRetryDecision::selectInterruption($this->snapshot(), 1, $sample, $window); + foreach ([ + [ + 'failure_source' => 'application', + ], [ + 'requested_delay_seconds' => 1, + ], + [ + 'timeout_kind' => 'heartbeat', + ], [ + 'delay_source' => 'failure_override', + ], + [ + 'callback_stop_state' => 'stopped', + ], [ + 'candidate_delay_seconds' => 0, + ], + ] as $mutation) { + self::assertFalse( + PreparedLocalRetryDecision::matchesStoredInterruption($this->snapshot(), 1, $window, [ + ...$stored, + ...$mutation, + ]) + ); + } + $this->expectException(InvalidArgumentException::class); + PreparedLocalRetryDecision::selectInterruption([ + ...$this->snapshot(), + 'maximum_interval_seconds' => 1, + ], 1, $sample, $window); + } + + public function testOriginalWindowCopiesMutableTimestampsWithoutReadingAClock(): void + { + $deadline = new DateTime('2026-11-05T01:01:00.123456+01:00'); + $latest = new DateTime('9999-12-31T23:59:59.999999Z'); + $earliest = new DateTime('1970-01-01T00:00:01.000000Z'); + $window = new PreparedLocalRetryWindow($deadline, $latest, $earliest); + $deadline->modify('+1 hour'); + $latest->modify('-1 day'); + $earliest->modify('+1 second'); + self::assertSame('2026-11-05T00:01:00.123456+00:00', $window->deadline->format('Y-m-d\TH:i:s.uP')); + self::assertSame('9999-12-31T23:59:59.999999+00:00', $window->latest->format('Y-m-d\TH:i:s.uP')); + self::assertSame('1970-01-01T00:00:01.000000+00:00', $window->earliest->format('Y-m-d\TH:i:s.uP')); + $unbounded = new PreparedLocalRetryWindow(null, $window->latest); + self::assertNull($unbounded->deadline); + self::assertNull($unbounded->earliest); + } + + public function testCanonicalApplicationSelectionProjections(): void + { + $vectors = json_decode(file_get_contents(dirname(__DIR__, 3) + . '/resources/protocol/activity-retry-policy-v2-vectors.json'), true, flags: JSON_THROW_ON_ERROR); + $count = 0; + foreach ($vectors['decision_cases'] as $case) { + if (! in_array($case['expected']['decision'], ['scheduled', 'terminal'], true) + || $case['id'] === 'zero_at_original_deadline_is_terminal') { + continue; + } + // Authority/fence/retained-receipt rows are not selector tests. + // The abstract at-deadline failure is preempted in prepared mode. + $input = $case['input']; + $snapshot = [ + ...$input['snapshot'], + 'start_to_close_timeout' => null, + 'schedule_to_start_timeout' => null, + 'schedule_to_close_timeout' => null, + 'heartbeat_timeout' => null, + 'non_retryable_error_types' => [], + ]; + $failure = $input['failure']; + $report = [ + 'outcome' => 'failed', + 'message' => 'canonical failure', + 'exception_type' => $failure['type'], + 'non_retryable' => $failure['non_retryable'], + ...(array_key_exists('next_retry_delay_seconds', $failure) + ? [ + 'next_retry_delay_seconds' => $failure['next_retry_delay_seconds'], + ] : []), + ]; + $sample = new DateTimeImmutable('@' . $input['failure_time_seconds']); + $deadline = $input['original_schedule_to_close_seconds'] === null ? null + : new DateTimeImmutable('@' . $input['original_schedule_to_close_seconds']); + $decision = PreparedLocalRetryDecision::select( + $snapshot, + $input['completed_attempts'], + $report, + $sample, + new PreparedLocalRetryWindow($deadline, new DateTimeImmutable('9999-12-31T23:59:59.999999Z')), + ); + self::assertSame($case['expected']['decision'], $decision['decision'], $case['id']); + if ($decision['decision'] === 'terminal') { + self::assertSame( + $case['expected']['reason'] ?? $case['expected']['terminal_classification'], + $decision['terminal_classification'], + $case['id'] + ); + self::assertArrayNotHasKey('retry_available_at', $decision); + } else { + self::assertSame( + $case['expected']['retry_available_at_seconds'], + (new DateTimeImmutable($decision['retry_available_at']))->getTimestamp(), + $case['id'] + ); + } + if (array_key_exists('selected_delay_seconds', $case['expected'])) { + self::assertSame( + $case['expected']['selected_delay_seconds'], + $decision['candidate_delay_seconds'], + $case['id'] + ); + self::assertSame( + $case['expected']['effective_cap_seconds'], + $decision['maximum_interval_seconds'], + $case['id'] + ); + } + ++$count; + } + self::assertSame(18, $count); + } + + public function testSelectionUsesTheSuppliedEngineSampleAndPreservesMicroseconds(): void + { + $sample = new DateTimeImmutable('2026-11-05T01:00:00.123456+01:00'); + $decision = $this->select(sample: $sample); + self::assertSame([ + 'contract_version' => 2, + 'snapshot_version' => 2, + 'decision' => 'scheduled', + 'failure_time' => '2026-11-05T00:00:00.123456Z', + 'original_schedule_to_close_deadline_at' => null, + 'failure_source' => 'application', + 'candidate_delay_seconds' => 2, + 'maximum_interval_seconds' => 5, + 'delay_source' => 'policy', + 'retry_available_at' => '2026-11-05T00:00:02.123456Z', + ], $decision); + self::assertSame('2026-11-05T01:00:00.123456+01:00', $sample->format('Y-m-d\TH:i:s.uP')); + self::assertSame($decision, $this->select(sample: $sample)); + } + + public function testZeroOverrideIsSelectedAndAnOverCapOverrideRemainsTerminalDiagnosticData(): void + { + $zero = $this->select(report: [ + 'outcome' => 'failed', + 'message' => 'failure', + 'next_retry_delay_seconds' => 0, + ]); + self::assertSame(0, $zero['requested_delay_seconds']); + self::assertSame(0, $zero['candidate_delay_seconds']); + self::assertSame($zero['failure_time'], $zero['retry_available_at']); + self::assertSame('failure_override', $zero['delay_source']); + + $large = $this->select(report: [ + 'outcome' => 'failed', + 'message' => 'failure', + 'next_retry_delay_seconds' => 9007199254740991, + ]); + self::assertSame('retry_delay_exceeds_policy_cap', $large['terminal_classification']); + self::assertSame(9007199254740991, $large['candidate_delay_seconds']); + self::assertArrayNotHasKey('retry_available_at', $large); + } + + public function testDeadlineComparisonDoesNotFloorTheOriginalMicroseconds(): void + { + $sample = new DateTimeImmutable('2026-11-05T00:00:00.000000Z'); + $report = [ + 'outcome' => 'failed', + 'message' => 'original failure', + 'next_retry_delay_seconds' => 1, + ]; + foreach (['00:00:00.999999', '00:00:01.000000'] as $time) { + $decision = $this->select( + report: $report, + sample: $sample, + deadline: new DateTimeImmutable('2026-11-05T' . $time . 'Z') + ); + self::assertSame('retry_delay_exceeds_schedule_to_close', $decision['terminal_classification']); + self::assertSame('application', $decision['failure_source']); + self::assertArrayNotHasKey('retry_available_at', $decision); + } + $scheduled = $this->select( + report: $report, + sample: $sample, + deadline: new DateTimeImmutable('2026-11-05T00:00:01.000001Z') + ); + self::assertSame('scheduled', $scheduled['decision']); + self::assertSame('2026-11-05T00:00:01.000000Z', $scheduled['retry_available_at']); + } + + public function testActuallyDueOverallTimeoutPreemptsApplicationInputAndCompletedOutcome(): void + { + $sample = new DateTimeImmutable('2026-11-05T00:00:01.000000Z'); + $deadline = new DateTimeImmutable('2026-11-05T00:00:01.000000Z'); + foreach ([[ + 'outcome' => 'failed', + 'message' => 'failure', + 'non_retryable' => true, + 'next_retry_delay_seconds' => 0, + ], + [ + 'outcome' => 'completed', + 'result' => 'opaque-avro', + 'payload_codec' => 'avro', + ], + [ + 'outcome' => 'timed_out', + ]] as $report) { + $decision = $this->select(report: $report, sample: $sample, deadline: $deadline); + self::assertSame('engine_timeout', $decision['failure_source']); + self::assertSame('engine_timeout', $decision['terminal_classification']); + self::assertSame('schedule_to_close', $decision['timeout_kind']); + foreach ([ + 'candidate_delay_seconds', + 'maximum_interval_seconds', + 'delay_source', + 'retry_available_at', + ] as $field) { + self::assertArrayNotHasKey($field, $decision); + } + } + self::assertSame(0, $this->select( + report: [ + 'outcome' => 'failed', + 'message' => '', + 'next_retry_delay_seconds' => 0, + ], + sample: $sample, + deadline: $deadline + )['requested_delay_seconds']); + } + + public function testNonRetryableAndExhaustedBranchesDoNotSelectADelay(): void + { + foreach ([ + $this->select(report: [ + 'outcome' => 'failed', + 'message' => '', + 'non_retryable' => true, + ]), + $this->select(report: [ + 'outcome' => 'failed', + 'message' => '', + 'exception_type' => 'PermanentFailure', + ]), + $this->select(nonRetryable: true), + $this->select(attempt: 3), + ] as $index => $decision) { + self::assertSame( + $index === 3 ? 'attempts_exhausted' : 'non_retryable', + $decision['terminal_classification'] + ); + foreach ([ + 'candidate_delay_seconds', + 'maximum_interval_seconds', + 'delay_source', + 'retry_available_at', + ] as $field) { + self::assertArrayNotHasKey($field, $decision); + } + } + } + + public function testHeartbeatAndStartTimeoutsUsePolicyUntilTheTotalDeadline(): void + { + foreach (['heartbeat', 'start_to_close'] as $kind) { + $decision = $this->select(report: [ + 'outcome' => 'failed', + 'message' => '', + 'non_retryable' => true, + 'next_retry_delay_seconds' => 4, + ], timeout: $kind); + self::assertSame('scheduled', $decision['decision']); + self::assertSame('engine_timeout', $decision['failure_source']); + self::assertSame($kind, $decision['timeout_kind']); + self::assertSame('policy', $decision['delay_source']); + self::assertSame(2, $decision['candidate_delay_seconds']); + self::assertSame(4, $decision['requested_delay_seconds']); + self::assertSame( + 'attempts_exhausted', + $this->select(attempt: 3, timeout: $kind)['terminal_classification'] + ); + self::assertSame('scheduled', $this->select(report: [ + 'outcome' => 'timed_out', + ], timeout: $kind)['decision']); + } + } + + public function testOnlyExecutableAvailabilityMustFitTheProvidedStorageBoundary(): void + { + $sample = new DateTimeImmutable('9999-12-31T23:59:58.999999Z'); + $latest = new DateTimeImmutable('9999-12-31T23:59:59.999999Z'); + $report = [ + 'outcome' => 'failed', + 'message' => '', + 'next_retry_delay_seconds' => 1, + ]; + self::assertSame( + '9999-12-31T23:59:59.999999Z', + $this->select(report: $report, sample: $sample, latest: $latest)['retry_available_at'] + ); + $huge = $this->select( + report: array_replace($report, [ + 'next_retry_delay_seconds' => 9007199254740991, + ]), + sample: $sample, + latest: $latest + ); + self::assertSame('retry_delay_exceeds_policy_cap', $huge['terminal_classification']); + $this->expectException(InvalidArgumentException::class); + $this->select(sample: $sample, latest: $latest); + } + + public function testInvalidInputOrUnprovenTimeoutCannotCreateADecision(): void + { + foreach ([[ + 'attempt' => 0, + ], [ + 'attempt' => 1025, + ], [ + 'report' => [ + 'outcome' => 'timed_out', + ], + ], + [ + 'report' => [ + 'outcome' => 'completed', + 'result' => 'opaque-avro', + 'payload_codec' => 'avro', + ], + ], + [ + 'report' => [ + 'outcome' => 'failed', + 'message' => '', + 'next_retry_delay_seconds' => null, + ], + ], + [ + 'timeout' => 'worker_selected', + ], [ + 'timeout' => 'schedule_to_close', + ], + [ + 'timeout' => 'schedule_to_close', + 'deadline' => new DateTimeImmutable('2030-01-01T00:00:00Z'), + ], + [ + 'sample' => new DateTimeImmutable('2026-11-06T00:00:00Z'), + 'latest' => new DateTimeImmutable('2026-11-05T00:00:00Z'), + ], + [ + 'sample' => new DateTimeImmutable('0000-01-01T00:00:00Z'), + ], + ] as $index => $changes) { + try { + $this->select(...$changes); + self::fail('Admitted invalid decision input ' . $index); + } catch (InvalidArgumentException) { + self::addToAssertionCount(1); + } + } + } + + public function testRetainedDecisionRequiresTheExactTypedClosedSelection(): void + { + $report = [ + 'outcome' => 'failed', + 'message' => 'failure', + 'next_retry_delay_seconds' => 0, + ]; + $deadline = new DateTimeImmutable('2026-11-05T00:01:00.000000Z'); + $latest = new DateTimeImmutable('9999-12-31T23:59:59.999999Z'); + $stored = $this->select(report: $report, deadline: $deadline); + $matches = fn (array $value): bool => PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + new PreparedLocalRetryWindow($deadline, $latest), + $value, + ); + self::assertTrue($matches(array_reverse($stored, true))); + $missing = $stored; + unset($missing['candidate_delay_seconds']); + foreach ([ + $missing, + [ + ...$stored, + 'unexpected' => true, + ], + [ + ...$stored, + 'contract_version' => '2', + ], + [ + ...$stored, + 'snapshot_version' => 1, + ], + [ + ...$stored, + 'candidate_delay_seconds' => '0', + ], + [ + ...$stored, + 'requested_delay_seconds' => null, + ], + [ + ...$stored, + 'maximum_interval_seconds' => 6, + ], + [ + ...$stored, + 'retry_available_at' => '2026-11-05T00:00:01.000000Z', + ], + [ + ...$stored, + 'original_schedule_to_close_deadline_at' => null, + ], + [ + ...$stored, + 'failure_time' => [], + ], + [ + ...$stored, + 'failure_time' => '2026-02-30T00:00:00.000000Z', + ], + [ + ...$stored, + 'failure_time' => '2026-11-05T24:00:00.000000Z', + ], + [ + ...$stored, + 'failure_time' => '2026-11-05T00:00:00Z', + ], + [ + ...$stored, + 'failure_time' => '٢٠٢٦-11-05T00:00:00.000000Z', + ], + [ + ...$stored, + 'timeout_kind' => [], + ], + [ + ...$stored, + 'timeout_kind' => 'worker_selected', + ], + ] as $index => $changed) { + self::assertFalse($matches($changed), 'Retained selection substitution ' . $index); + } + self::assertFalse(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + [ + ...$report, + 'next_retry_delay_seconds' => 1, + ], + new PreparedLocalRetryWindow($deadline, $latest), + $stored, + )); + } + + public function testRetainedTerminalDecisionsCannotGainAnExecutableTimestampOrUnselectedDelay(): void + { + $latest = new DateTimeImmutable('9999-12-31T23:59:59.999999Z'); + $report = [ + 'outcome' => 'failed', + 'message' => 'failure', + 'next_retry_delay_seconds' => 6, + ]; + $stored = $this->select(report: $report); + self::assertTrue(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + new PreparedLocalRetryWindow(null, $latest), + $stored, + )); + self::assertFalse(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + new PreparedLocalRetryWindow(null, $latest), + [ + ...$stored, + 'retry_available_at' => '2026-11-05T00:00:06.000000Z', + ], + )); + $report = [ + 'outcome' => 'timed_out', + ]; + $deadline = new DateTimeImmutable('2026-11-05T00:00:00.000000Z'); + $stored = $this->select(report: $report, deadline: $deadline); + self::assertTrue(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + new PreparedLocalRetryWindow($deadline, $latest), + $stored, + )); + self::assertFalse(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + new PreparedLocalRetryWindow($deadline, $latest), + [ + ...$stored, + 'delay_source' => 'policy', + ], + )); + self::assertFalse(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + new PreparedLocalRetryWindow($deadline->modify('+1 second'), $latest), + $stored, + )); + } + + public function testRetainedDecisionKeepsBackendBoundsAndEngineNonRetryableClassification(): void + { + $report = [ + 'outcome' => 'failed', + 'message' => 'failure', + ]; + $sample = new DateTimeImmutable('2026-11-05T00:00:00.000000Z'); + $latest = new DateTimeImmutable('9999-12-31T23:59:59.999999Z'); + $stored = $this->select(report: $report, nonRetryable: true); + self::assertTrue(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + new PreparedLocalRetryWindow(null, $latest), + $stored, + true, + )); + self::assertFalse(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + new PreparedLocalRetryWindow(null, $latest), + $stored, + )); + self::assertFalse(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + new PreparedLocalRetryWindow(null, $sample->modify('-1 microsecond')), + $stored, + true, + )); + self::assertFalse(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + new PreparedLocalRetryWindow(null, $latest, $sample->modify('+1 microsecond')), + $stored, + true, + )); + $snapshot = $this->snapshot(); + $snapshot['snapshot_version'] = 1; + self::assertFalse(PreparedLocalRetryDecision::matchesStored( + $snapshot, + 1, + $report, + new PreparedLocalRetryWindow(null, $latest), + $stored, + true, + )); + } + + private function snapshot(): array + { + return PreparedLocalRetryPolicy::normalize([ + 'contract_version' => 2, + 'mode' => 'explicit', + 'max_attempts' => 3, + 'backoff_seconds' => [2, 5], + 'non_retryable_error_types' => ['PermanentFailure'], + ]); + } + + private function select( + array $report = [ + 'outcome' => 'failed', + 'message' => 'failure', + ], + int $attempt = 1, + ?DateTimeImmutable $sample = null, + ?DateTimeImmutable $deadline = null, + ?DateTimeImmutable $latest = null, + ?string $timeout = null, + bool $nonRetryable = false, + ): array { + $snapshot = $this->snapshot(); + + return PreparedLocalRetryDecision::select( + $snapshot, + $attempt, + $report, + $sample ?? new DateTimeImmutable('2026-11-05T00:00:00.000000Z'), + new PreparedLocalRetryWindow($deadline, $latest ?? new DateTimeImmutable('9999-12-31T23:59:59.999999Z')), + $timeout, + $nonRetryable, + ); + } +} diff --git a/tests/Unit/V2/PreparedLocalRetryDescriptorTest.php b/tests/Unit/V2/PreparedLocalRetryDescriptorTest.php new file mode 100644 index 000000000..7e19cba01 --- /dev/null +++ b/tests/Unit/V2/PreparedLocalRetryDescriptorTest.php @@ -0,0 +1,91 @@ +instance('translator', $translator); + $container->instance('validator', $factory); + $container->instance('app', $container); + + // Save and replace the facade application so downstream tests in the + // same process are not affected by our minimal container. + $this->priorFacadeApp = \Illuminate\Support\Facades\Facade::getFacadeApplication(); + \Illuminate\Support\Facades\Facade::clearResolvedInstances(); + \Illuminate\Support\Facades\Facade::setFacadeApplication($container); + } + + protected function tearDown(): void + { + \Illuminate\Support\Facades\Facade::clearResolvedInstances(); + \Illuminate\Support\Facades\Facade::setFacadeApplication($this->priorFacadeApp); + $this->priorFacadeApp = null; + + parent::tearDown(); + } + + public function testPreparedRetryNormalizationRequiresTheExplicitVersionTwoEntry(): void + { + $descriptor = [ + 'type' => 'record_local_activity', + 'activity_type' => 'local-worker-alias', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'retry_policy' => [ + 'contract_version' => 2, + 'mode' => 'explicit', + 'max_attempts' => 3, + 'backoff_seconds' => [2, 5], + ], + 'start_to_close_timeout' => 30, + ]; + $normalized = PortableLocalActivityPreparation::normalizeDescriptor($descriptor, 2); + $this->assertSame(2, $normalized['local_activity_contract_version']); + $this->assertSame(2, $normalized['retry_policy']['snapshot_version']); + $this->assertSame([2, 5], $normalized['retry_policy']['backoff_seconds']); + $this->assertSame(5, $normalized['retry_policy']['maximum_interval_seconds']); + $this->assertSame(30, $normalized['retry_policy']['start_to_close_timeout']); + try { + PortableLocalActivityPreparation::normalizeDescriptor($descriptor); + $this->fail('The v1 entry silently accepted a v2 policy.'); + } catch (ValidationException) { + $this->addToAssertionCount(1); + } + $legacy = $descriptor; + $legacy['retry_policy'] = [ + 'max_attempts' => 3, + 'backoff_seconds' => [2, 5], + ]; + $this->assertArrayNotHasKey( + 'local_activity_contract_version', + PortableLocalActivityPreparation::normalizeDescriptor($legacy) + ); + $this->expectException(\InvalidArgumentException::class); + PortableLocalActivityPreparation::normalizeDescriptor($legacy, 2); + } +} diff --git a/tests/Unit/V2/PreparedLocalRetryTimestampTest.php b/tests/Unit/V2/PreparedLocalRetryTimestampTest.php new file mode 100644 index 000000000..99e7b9605 --- /dev/null +++ b/tests/Unit/V2/PreparedLocalRetryTimestampTest.php @@ -0,0 +1,147 @@ +format('Y-m-d\TH:i:s.u\Z') + ); + self::assertSame( + '9999-12-31T23:59:59.999999Z', + PreparedLocalRetryTimestamp::latestForDriver($driver)->format('Y-m-d\TH:i:s.u\Z') + ); + } + self::assertSame( + '2038-01-19T03:14:07.499999Z', + PreparedLocalRetryTimestamp::latestForDriver('mysql')->format('Y-m-d\TH:i:s.u\Z') + ); + foreach (['10.11.14', '11.4.0-MariaDB', '5.5.5-11.4.8-MariaDB'] as $version) { + self::assertSame( + '2038-01-19T03:14:07.999999Z', + PreparedLocalRetryTimestamp::latestForDriver('mariadb', $version)->format('Y-m-d\TH:i:s.u\Z') + ); + } + foreach (['11.5.1', '11.8.2-MariaDB', '12.0.0'] as $version) { + self::assertSame( + '2106-02-07T06:28:15.999999Z', + PreparedLocalRetryTimestamp::latestForDriver('mariadb', $version)->format('Y-m-d\TH:i:s.u\Z') + ); + } + } + + public function testMariaDbThroughTheMysqlConnectionUsesItsActualVendorVersion(): void + { + $connection = $this->createMock(MySqlConnection::class); + $connection->method('getDriverName') + ->willReturn('mysql'); + $connection->method('isMaria') + ->willReturn(true); + $connection->method('getServerVersion') + ->willReturn('11.8.2'); + $connection->expects(self::never())->method('getPdo'); + self::assertSame('2106', PreparedLocalRetryTimestamp::latestForConnection($connection)->format('Y')); + + $mysql = $this->createMock(MySqlConnection::class); + $mysql->method('getDriverName') + ->willReturn('mysql'); + $mysql->method('isMaria') + ->willReturn(false); + $mysql->expects(self::never())->method('getServerVersion'); + self::assertSame('2038', PreparedLocalRetryTimestamp::latestForConnection($mysql)->format('Y')); + } + + public function testAdditionPreservesTheSampleAndChecksBeforeAnyLargeIntervalConstruction(): void + { + $sample = new DateTimeImmutable('2026-11-05T01:00:00.123456+01:00'); + $latest = new DateTimeImmutable('9999-12-31T23:59:59.999999Z'); + self::assertSame( + '2026-11-05T00:00:02.123456Z', + PreparedLocalRetryTimestamp::add($sample, 2, $latest)->format('Y-m-d\TH:i:s.u\Z') + ); + self::assertSame('2026-11-05T01:00:00.123456+01:00', $sample->format('Y-m-d\TH:i:s.uP')); + $this->expectException(InvalidArgumentException::class); + PreparedLocalRetryTimestamp::add($sample, 9007199254740991, $latest); + } + + public function testUpperBoundaryRetainsMicrosecondsAndRefusesTheOverflowingSuccessor(): void + { + $latest = new DateTimeImmutable('2038-01-19T03:14:07.499999Z'); + self::assertEquals( + $latest, + PreparedLocalRetryTimestamp::add(new DateTimeImmutable('2038-01-19T03:14:06.499999Z'), 1, $latest) + ); + self::assertEquals($latest, PreparedLocalRetryTimestamp::add($latest, 0, $latest)); + $this->expectException(InvalidArgumentException::class); + PreparedLocalRetryTimestamp::add(new DateTimeImmutable('2038-01-19T03:14:06.500000Z'), 1, $latest); + } + + public function testConnectionWrapperAlsoEnforcesTheTimestampLowerBound(): void + { + $connection = $this->createMock(Connection::class); + $connection->method('getDriverName') + ->willReturn('mysql'); + $connection->expects(self::never())->method('getPdo'); + $earliest = new DateTimeImmutable('1970-01-01T00:00:01.000000Z'); + self::assertEquals($earliest, PreparedLocalRetryTimestamp::addForConnection($earliest, 0, $connection)); + $this->expectException(InvalidArgumentException::class); + PreparedLocalRetryTimestamp::addForConnection( + new DateTimeImmutable('1970-01-01T00:00:00.999999Z'), + 1, + $connection + ); + } + + public function testUnknownBackendAndUnidentifiedMariaDbVersionAreRefused(): void + { + foreach ([['unknown', null], + ['mariadb', null], + ['mariadb', ''], + ['mariadb', 'unidentified'], + ] as [$driver, $version]) { + try { + PreparedLocalRetryTimestamp::latestForDriver($driver, $version); + self::fail('Invented a timestamp bound for an unidentified backend.'); + } catch (InvalidArgumentException) { + self::addToAssertionCount(1); + } + } + $this->expectException(InvalidArgumentException::class); + PreparedLocalRetryTimestamp::earliestForDriver('unknown'); + } + + public function testInvalidDurationAndOutOfDomainInstantsAreRefused(): void + { + foreach ([ + ['2026-11-05T00:00:00Z', -1, '9999-12-31T23:59:59.999999Z'], + ['2026-11-05T00:00:00Z', 9007199254740992, '9999-12-31T23:59:59.999999Z'], + ['2026-11-06T00:00:00Z', 0, '2026-11-05T00:00:00Z'], + ['0000-01-01T00:00:00Z', 0, '9999-12-31T23:59:59.999999Z'], + ['2026-11-05T00:00:00Z', 0, '0000-01-01T00:00:00Z'], + ] as [$sample, $seconds, $latest]) { + try { + PreparedLocalRetryTimestamp::add( + new DateTimeImmutable($sample), + $seconds, + new DateTimeImmutable($latest) + ); + self::fail('Invented a representable timestamp.'); + } catch (InvalidArgumentException) { + self::addToAssertionCount(1); + } + } + } +} diff --git a/tests/Unit/V2/ReplayRegressionCorpusTest.php b/tests/Unit/V2/ReplayRegressionCorpusTest.php index 27b15eca6..8f7d514fa 100644 --- a/tests/Unit/V2/ReplayRegressionCorpusTest.php +++ b/tests/Unit/V2/ReplayRegressionCorpusTest.php @@ -7,6 +7,11 @@ use Illuminate\Config\Repository; use Illuminate\Container\Container; use Illuminate\Foundation\Application; +use Illuminate\Support\Facades\Facade; +use Illuminate\Translation\ArrayLoader; +use Illuminate\Translation\Translator; +use Illuminate\Validation\Factory as ValidationFactory; +use Illuminate\Validation\ValidationException; use OpenTelemetry\API\Instrumentation\Configurator; use OpenTelemetry\Context\Context; use OpenTelemetry\SDK\Trace\SpanExporter\InMemoryExporter; @@ -26,6 +31,8 @@ use Workflow\V2\Support\CooperativeCancellationDelivery; use Workflow\V2\Support\ExecutionTracing; use Workflow\V2\Support\HistoryEventPayloadContract; +use Workflow\V2\Support\PortableLocalActivityPreparation; +use Workflow\V2\Support\PreparedLocalRetryPolicy; use Workflow\V2\Support\ServiceCallCancellationOrigin; use Workflow\V2\Support\WorkflowFiberRunner; use Workflow\V2\Support\WorkflowStep; @@ -149,7 +156,14 @@ public function testFixtureExecutesThroughColdReplayRunner(array $fixture): void } $profiledWorkflow = null; - if ($fixture['id'] === 'replay-unrelated-payload-read-budget') { + if (in_array( + $fixture['id'], + ['replay-unrelated-payload-read-budget', 'prepared-v1-policy-refusal-legacy-zero-tail'], + true + )) { + if ($fixture['id'] === 'prepared-v1-policy-refusal-legacy-zero-tail') { + $this->assertPreparedV1RefusesTheVersionedPolicy($fixture); + } Container::getInstance()->resolving( $workflowClass, static function (Workflow $instance) use (&$profiledWorkflow): void { @@ -188,6 +202,20 @@ static function (WorkflowHistoryEvent $event): ReplayBudgetHistoryEvent { $step = $runner->step(); + if ($fixture['id'] === 'prepared-v1-policy-refusal-legacy-zero-tail') { + $this->assertInstanceOf(Workflow::class, $profiledWorkflow); + $retries = $profiledWorkflow->run->historyEvents + ->where('event_type', HistoryEventType::ActivityRetryScheduled); + $this->assertSame([120, 0], $retries->map( + static fn (WorkflowHistoryEvent $event): int => $event->payload['retry_backoff_seconds'] + )->values() + ->all()); + $scheduled = $profiledWorkflow->run->historyEvents + ->firstWhere('event_type', HistoryEventType::ActivityScheduled); + $this->assertSame(1, ActivitySnapshot::fromEvent($scheduled)['retry_policy']['snapshot_version']); + $this->assertSame([120], ActivitySnapshot::fromEvent($scheduled)['retry_policy']['backoff_seconds']); + } + if ($fixture['id'] === 'replay-unrelated-payload-read-budget') { $this->assertSame( 0, @@ -381,6 +409,33 @@ public static function replayRegressionFixtures(): array return $fixtures; } + private function assertPreparedV1RefusesTheVersionedPolicy(array $fixture): void + { + $policy = $fixture['unqualified_preparation_policy']; + $this->assertSame(2, PreparedLocalRetryPolicy::normalize($policy)['snapshot_version']); + $container = new Container(); + $translator = new Translator(new ArrayLoader(), 'en'); + $container->instance('validator', new ValidationFactory($translator, $container)); + $prior = Facade::getFacadeApplication(); + Facade::clearResolvedInstance('validator'); + Facade::setFacadeApplication($container); + try { + PortableLocalActivityPreparation::normalizeDescriptor([ + 'type' => 'record_local_activity', + 'activity_type' => $fixture['history'][1]['payload']['activity_type'], + 'arguments' => Serializer::serializeWithCodec('avro', $fixture['workflow']['arguments']), + 'payload_codec' => 'avro', + 'retry_policy' => $policy, + ]); + $this->fail('Prepared v1 discarded the versioned policy instead of refusing it.'); + } catch (ValidationException $error) { + $this->assertArrayHasKey('local_activity.retry_policy', $error->errors()); + } finally { + Facade::clearResolvedInstance('validator'); + Facade::setFacadeApplication($prior); + } + } + private function assertWorkflowFiberObservabilityContext(array $fixture): void { $exporter = new InMemoryExporter();