From eda7b3bae3aea34858028c7a79b567841179d14a Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:11:28 +0000 Subject: [PATCH 01/15] Select prepared retry dispositions from one engine time sample --- .github/pr-test-selection.json | 1 + src/V2/Support/PreparedLocalRetryDecision.php | 157 ++++++++ .../V2/PreparedLocalRetryDecisionTest.php | 364 ++++++++++++++++++ 3 files changed, 522 insertions(+) create mode 100644 src/V2/Support/PreparedLocalRetryDecision.php create mode 100644 tests/Unit/V2/PreparedLocalRetryDecisionTest.php diff --git a/.github/pr-test-selection.json b/.github/pr-test-selection.json index 306535de..0a19c80e 100644 --- a/.github/pr-test-selection.json +++ b/.github/pr-test-selection.json @@ -20,6 +20,7 @@ "tests/Unit/V2/LaravelDependencySecurityPolicyTest.php", "tests/Unit/V2/PlatformProtocolSpecsTest.php", "tests/Unit/V2/PreparedLocalActivityReportTest.php", + "tests/Unit/V2/PreparedLocalRetryDecisionTest.php", "tests/Unit/V2/PreparedLocalRetryPolicyTest.php", "tests/Unit/V2/ReadmeBranchDestinationsTest.php", "tests/Unit/V2/ScopedCancellationContextTest.php", diff --git a/src/V2/Support/PreparedLocalRetryDecision.php b/src/V2/Support/PreparedLocalRetryDecision.php new file mode 100644 index 00000000..696242b6 --- /dev/null +++ b/src/V2/Support/PreparedLocalRetryDecision.php @@ -0,0 +1,157 @@ + $snapshot + * @param array $report + * @return array + */ + public static function select( + array $snapshot, + int $attemptNumber, + array $report, + DateTimeInterface $sample, + ?DateTimeInterface $originalDeadline, + DateTimeInterface $latestBindableTimestamp, + ?string $dueTimeoutKind = null, + bool $nonRetryable = false, + ): array { + $snapshot = PreparedLocalRetryPolicy::validateSnapshot($snapshot); + $refusal = PreparedLocalActivityReport::refusal($report); + if ($refusal !== null || $attemptNumber < 1 || $attemptNumber > PreparedLocalRetryPolicy::MAX_ATTEMPTS + || ($dueTimeoutKind !== null && ! in_array( + $dueTimeoutKind, + ['schedule_to_close', 'start_to_close', 'heartbeat'], + true + ))) { + throw new InvalidArgumentException($refusal ?? 'Invalid prepared retry decision input.'); + } + $sample = self::utc($sample); + $deadline = $originalDeadline === null ? null : self::utc($originalDeadline); + $latest = self::utc($latestBindableTimestamp); + if ($sample > $latest) { + throw new InvalidArgumentException('Prepared failure sample exceeds the backend timestamp range.'); + } + if ($deadline !== null && $sample >= $deadline) { + $dueTimeoutKind = 'schedule_to_close'; + } + if ($dueTimeoutKind === 'schedule_to_close' && ($deadline === null || $sample < $deadline)) { + throw new InvalidArgumentException('Overall timeout requires the original due deadline.'); + } + if ($dueTimeoutKind === null && $report['outcome'] !== 'failed') { + throw new InvalidArgumentException( + 'A retry decision needs an application failure or an actually due timeout.' + ); + } + $engineTimeout = $dueTimeoutKind !== null; + $decision = [ + 'contract_version' => 2, + 'snapshot_version' => 2, + 'decision' => 'terminal', + 'failure_time' => self::format($sample), + 'original_schedule_to_close_deadline_at' => $deadline === null ? null : self::format($deadline), + 'failure_source' => $engineTimeout ? 'engine_timeout' : 'application', + ...($engineTimeout ? [ + 'timeout_kind' => $dueTimeoutKind, + ] : []), + ...(array_key_exists('next_retry_delay_seconds', $report) + ? [ + 'requested_delay_seconds' => $report['next_retry_delay_seconds'], + ] : []), + ]; + if ($dueTimeoutKind === 'schedule_to_close') { + return [ + ...$decision, + 'terminal_classification' => 'engine_timeout', + ]; + } + $type = $report['exception_type'] ?? RuntimeException::class; + if (! $engineTimeout && ($nonRetryable || ($report['non_retryable'] ?? false) + || in_array($type, $snapshot['non_retryable_error_types'], true))) { + return [ + ...$decision, + 'terminal_classification' => 'non_retryable', + ]; + } + if ($attemptNumber >= $snapshot['max_attempts']) { + return [ + ...$decision, + 'terminal_classification' => 'attempts_exhausted', + ]; + } + $override = ! $engineTimeout && array_key_exists('next_retry_delay_seconds', $report); + $candidate = $override ? $report['next_retry_delay_seconds'] + : PreparedLocalRetryPolicy::delayForRetry($snapshot, $attemptNumber); + $decision = [ + ...$decision, + 'candidate_delay_seconds' => $candidate, + 'maximum_interval_seconds' => $snapshot['maximum_interval_seconds'], + 'delay_source' => $override ? 'failure_override' : 'policy', + ]; + if ($candidate > $snapshot['maximum_interval_seconds']) { + return [ + ...$decision, + 'terminal_classification' => 'retry_delay_exceeds_policy_cap', + ]; + } + if ($deadline !== null && $candidate >= intdiv( + self::microseconds($deadline) - self::microseconds($sample) + 999999, + 1000000 + )) { + return [ + ...$decision, + 'terminal_classification' => 'retry_delay_exceeds_schedule_to_close', + ]; + } + // Check by division before multiplication or DateInterval construction. + // Even the largest portable integer cannot wrap or become a float. + if ($candidate > intdiv(self::microseconds($latest) - self::microseconds($sample), 1000000)) { + throw new InvalidArgumentException('Prepared retry availability exceeds the backend timestamp range.'); + } + $available = $sample->add(new DateInterval('PT' . $candidate . 'S')); + + return [ + ...$decision, + 'decision' => 'scheduled', + 'retry_available_at' => self::format($available), + ]; + } + + private static function utc(DateTimeInterface $value): DateTimeImmutable + { + $utc = DateTimeImmutable::createFromInterface($value)->setTimezone(new DateTimeZone('UTC')); + $year = $utc->format('Y'); + if (strlen($year) !== 4 || (int) $year < 1) { + throw new InvalidArgumentException('Prepared retry timestamps require a four-digit UTC year.'); + } + + return $utc; + } + + private static function microseconds(DateTimeImmutable $value): int + { + return $value->getTimestamp() * 1000000 + (int) $value->format('u'); + } + + private static function format(DateTimeImmutable $value): string + { + return $value->format('Y-m-d\TH:i:s.u\Z'); + } +} diff --git a/tests/Unit/V2/PreparedLocalRetryDecisionTest.php b/tests/Unit/V2/PreparedLocalRetryDecisionTest.php new file mode 100644 index 00000000..45721ca7 --- /dev/null +++ b/tests/Unit/V2/PreparedLocalRetryDecisionTest.php @@ -0,0 +1,364 @@ + null, + 'schedule_to_start_timeout' => null, + 'schedule_to_close_timeout' => null, + 'heartbeat_timeout' => null, + 'non_retryable_error_types' => [], + ]; + $failure = $input['failure']; + $report = [ + 'outcome' => 'failed', + 'message' => 'canonical failure', + 'exception_type' => $failure['type'], + 'non_retryable' => $failure['non_retryable'], + ...(array_key_exists('next_retry_delay_seconds', $failure) + ? [ + 'next_retry_delay_seconds' => $failure['next_retry_delay_seconds'], + ] : []), + ]; + $sample = new DateTimeImmutable('@' . $input['failure_time_seconds']); + $deadline = $input['original_schedule_to_close_seconds'] === null ? null + : new DateTimeImmutable('@' . $input['original_schedule_to_close_seconds']); + $decision = PreparedLocalRetryDecision::select( + $snapshot, + $input['completed_attempts'], + $report, + $sample, + $deadline, + new DateTimeImmutable('9999-12-31T23:59:59.999999Z') + ); + self::assertSame($case['expected']['decision'], $decision['decision'], $case['id']); + if ($decision['decision'] === 'terminal') { + self::assertSame( + $case['expected']['reason'] ?? $case['expected']['terminal_classification'], + $decision['terminal_classification'], + $case['id'] + ); + self::assertArrayNotHasKey('retry_available_at', $decision); + } else { + self::assertSame( + $case['expected']['retry_available_at_seconds'], + (new DateTimeImmutable($decision['retry_available_at']))->getTimestamp(), + $case['id'] + ); + } + if (array_key_exists('selected_delay_seconds', $case['expected'])) { + self::assertSame( + $case['expected']['selected_delay_seconds'], + $decision['candidate_delay_seconds'], + $case['id'] + ); + self::assertSame( + $case['expected']['effective_cap_seconds'], + $decision['maximum_interval_seconds'], + $case['id'] + ); + } + ++$count; + } + self::assertSame(18, $count); + } + + public function testSelectionUsesTheSuppliedEngineSampleAndPreservesMicroseconds(): void + { + $sample = new DateTimeImmutable('2026-11-05T01:00:00.123456+01:00'); + $decision = $this->select(sample: $sample); + self::assertSame([ + 'contract_version' => 2, + 'snapshot_version' => 2, + 'decision' => 'scheduled', + 'failure_time' => '2026-11-05T00:00:00.123456Z', + 'original_schedule_to_close_deadline_at' => null, + 'failure_source' => 'application', + 'candidate_delay_seconds' => 2, + 'maximum_interval_seconds' => 5, + 'delay_source' => 'policy', + 'retry_available_at' => '2026-11-05T00:00:02.123456Z', + ], $decision); + self::assertSame('2026-11-05T01:00:00.123456+01:00', $sample->format('Y-m-d\TH:i:s.uP')); + self::assertSame($decision, $this->select(sample: $sample)); + } + + public function testZeroOverrideIsSelectedAndAnOverCapOverrideRemainsTerminalDiagnosticData(): void + { + $zero = $this->select(report: [ + 'outcome' => 'failed', + 'message' => 'failure', + 'next_retry_delay_seconds' => 0, + ]); + self::assertSame(0, $zero['requested_delay_seconds']); + self::assertSame(0, $zero['candidate_delay_seconds']); + self::assertSame($zero['failure_time'], $zero['retry_available_at']); + self::assertSame('failure_override', $zero['delay_source']); + + $large = $this->select(report: [ + 'outcome' => 'failed', + 'message' => 'failure', + 'next_retry_delay_seconds' => 9007199254740991, + ]); + self::assertSame('retry_delay_exceeds_policy_cap', $large['terminal_classification']); + self::assertSame(9007199254740991, $large['candidate_delay_seconds']); + self::assertArrayNotHasKey('retry_available_at', $large); + } + + public function testDeadlineComparisonDoesNotFloorTheOriginalMicroseconds(): void + { + $sample = new DateTimeImmutable('2026-11-05T00:00:00.000000Z'); + $report = [ + 'outcome' => 'failed', + 'message' => 'original failure', + 'next_retry_delay_seconds' => 1, + ]; + foreach (['00:00:00.999999', '00:00:01.000000'] as $time) { + $decision = $this->select( + report: $report, + sample: $sample, + deadline: new DateTimeImmutable('2026-11-05T' . $time . 'Z') + ); + self::assertSame('retry_delay_exceeds_schedule_to_close', $decision['terminal_classification']); + self::assertSame('application', $decision['failure_source']); + self::assertArrayNotHasKey('retry_available_at', $decision); + } + $scheduled = $this->select( + report: $report, + sample: $sample, + deadline: new DateTimeImmutable('2026-11-05T00:00:01.000001Z') + ); + self::assertSame('scheduled', $scheduled['decision']); + self::assertSame('2026-11-05T00:00:01.000000Z', $scheduled['retry_available_at']); + } + + public function testActuallyDueOverallTimeoutPreemptsApplicationInputAndCompletedOutcome(): void + { + $sample = new DateTimeImmutable('2026-11-05T00:00:01.000000Z'); + $deadline = new DateTimeImmutable('2026-11-05T00:00:01.000000Z'); + foreach ([[ + 'outcome' => 'failed', + 'message' => 'failure', + 'non_retryable' => true, + 'next_retry_delay_seconds' => 0, + ], + [ + 'outcome' => 'completed', + 'result' => 'opaque-avro', + 'payload_codec' => 'avro', + ], + [ + 'outcome' => 'timed_out', + ]] as $report) { + $decision = $this->select(report: $report, sample: $sample, deadline: $deadline); + self::assertSame('engine_timeout', $decision['failure_source']); + self::assertSame('engine_timeout', $decision['terminal_classification']); + self::assertSame('schedule_to_close', $decision['timeout_kind']); + foreach ([ + 'candidate_delay_seconds', + 'maximum_interval_seconds', + 'delay_source', + 'retry_available_at', + ] as $field) { + self::assertArrayNotHasKey($field, $decision); + } + } + self::assertSame(0, $this->select( + report: [ + 'outcome' => 'failed', + 'message' => '', + 'next_retry_delay_seconds' => 0, + ], + sample: $sample, + deadline: $deadline + )['requested_delay_seconds']); + } + + public function testNonRetryableAndExhaustedBranchesDoNotSelectADelay(): void + { + foreach ([ + $this->select(report: [ + 'outcome' => 'failed', + 'message' => '', + 'non_retryable' => true, + ]), + $this->select(report: [ + 'outcome' => 'failed', + 'message' => '', + 'exception_type' => 'PermanentFailure', + ]), + $this->select(nonRetryable: true), + $this->select(attempt: 3), + ] as $index => $decision) { + self::assertSame( + $index === 3 ? 'attempts_exhausted' : 'non_retryable', + $decision['terminal_classification'] + ); + foreach ([ + 'candidate_delay_seconds', + 'maximum_interval_seconds', + 'delay_source', + 'retry_available_at', + ] as $field) { + self::assertArrayNotHasKey($field, $decision); + } + } + } + + public function testHeartbeatAndStartTimeoutsUsePolicyUntilTheTotalDeadline(): void + { + foreach (['heartbeat', 'start_to_close'] as $kind) { + $decision = $this->select(report: [ + 'outcome' => 'failed', + 'message' => '', + 'non_retryable' => true, + 'next_retry_delay_seconds' => 4, + ], timeout: $kind); + self::assertSame('scheduled', $decision['decision']); + self::assertSame('engine_timeout', $decision['failure_source']); + self::assertSame($kind, $decision['timeout_kind']); + self::assertSame('policy', $decision['delay_source']); + self::assertSame(2, $decision['candidate_delay_seconds']); + self::assertSame(4, $decision['requested_delay_seconds']); + self::assertSame( + 'attempts_exhausted', + $this->select(attempt: 3, timeout: $kind)['terminal_classification'] + ); + self::assertSame('scheduled', $this->select(report: [ + 'outcome' => 'timed_out', + ], timeout: $kind)['decision']); + } + } + + public function testOnlyExecutableAvailabilityMustFitTheProvidedStorageBoundary(): void + { + $sample = new DateTimeImmutable('9999-12-31T23:59:58.999999Z'); + $latest = new DateTimeImmutable('9999-12-31T23:59:59.999999Z'); + $report = [ + 'outcome' => 'failed', + 'message' => '', + 'next_retry_delay_seconds' => 1, + ]; + self::assertSame( + '9999-12-31T23:59:59.999999Z', + $this->select(report: $report, sample: $sample, latest: $latest)['retry_available_at'] + ); + $huge = $this->select( + report: array_replace($report, [ + 'next_retry_delay_seconds' => 9007199254740991, + ]), + sample: $sample, + latest: $latest + ); + self::assertSame('retry_delay_exceeds_policy_cap', $huge['terminal_classification']); + $this->expectException(InvalidArgumentException::class); + $this->select(sample: $sample, latest: $latest); + } + + public function testInvalidInputOrUnprovenTimeoutCannotCreateADecision(): void + { + foreach ([[ + 'attempt' => 0, + ], [ + 'attempt' => 1025, + ], [ + 'report' => [ + 'outcome' => 'timed_out', + ], + ], + [ + 'report' => [ + 'outcome' => 'completed', + 'result' => 'opaque-avro', + 'payload_codec' => 'avro', + ], + ], + [ + 'report' => [ + 'outcome' => 'failed', + 'message' => '', + 'next_retry_delay_seconds' => null, + ], + ], + [ + 'timeout' => 'worker_selected', + ], [ + 'timeout' => 'schedule_to_close', + ], + [ + 'timeout' => 'schedule_to_close', + 'deadline' => new DateTimeImmutable('2030-01-01T00:00:00Z'), + ], + [ + 'sample' => new DateTimeImmutable('2026-11-06T00:00:00Z'), + 'latest' => new DateTimeImmutable('2026-11-05T00:00:00Z'), + ], + [ + 'sample' => new DateTimeImmutable('0000-01-01T00:00:00Z'), + ], + ] as $index => $changes) { + try { + $this->select(...$changes); + self::fail('Admitted invalid decision input ' . $index); + } catch (InvalidArgumentException) { + self::addToAssertionCount(1); + } + } + } + + private function select( + array $report = [ + 'outcome' => 'failed', + 'message' => 'failure', + ], + int $attempt = 1, + ?DateTimeImmutable $sample = null, + ?DateTimeImmutable $deadline = null, + ?DateTimeImmutable $latest = null, + ?string $timeout = null, + bool $nonRetryable = false, + ): array { + $snapshot = PreparedLocalRetryPolicy::normalize([ + 'contract_version' => 2, + 'mode' => 'explicit', + 'max_attempts' => 3, + 'backoff_seconds' => [2, 5], + 'non_retryable_error_types' => ['PermanentFailure'], + ]); + + return PreparedLocalRetryDecision::select( + $snapshot, + $attempt, + $report, + $sample ?? new DateTimeImmutable('2026-11-05T00:00:00.000000Z'), + $deadline, + $latest ?? new DateTimeImmutable('9999-12-31T23:59:59.999999Z'), + $timeout, + $nonRetryable + ); + } +} From d18d92a17bf6c1dd86418dd5301db64fdd8bf000 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:31:46 +0000 Subject: [PATCH 02/15] Retain explicit prepared retry authority and refuse legacy downgrade --- .github/pr-test-selection.json | 1 + .../PortableLocalActivityPreparation.php | 486 +++++++++++------- ...V2PortableLocalActivityPreparationTest.php | 108 ++++ .../V2/PreparedLocalRetryDescriptorTest.php | 91 ++++ 4 files changed, 510 insertions(+), 176 deletions(-) create mode 100644 tests/Unit/V2/PreparedLocalRetryDescriptorTest.php diff --git a/.github/pr-test-selection.json b/.github/pr-test-selection.json index 0a19c80e..6619ba6c 100644 --- a/.github/pr-test-selection.json +++ b/.github/pr-test-selection.json @@ -21,6 +21,7 @@ "tests/Unit/V2/PlatformProtocolSpecsTest.php", "tests/Unit/V2/PreparedLocalActivityReportTest.php", "tests/Unit/V2/PreparedLocalRetryDecisionTest.php", + "tests/Unit/V2/PreparedLocalRetryDescriptorTest.php", "tests/Unit/V2/PreparedLocalRetryPolicyTest.php", "tests/Unit/V2/ReadmeBranchDestinationsTest.php", "tests/Unit/V2/ScopedCancellationContextTest.php", diff --git a/src/V2/Support/PortableLocalActivityPreparation.php b/src/V2/Support/PortableLocalActivityPreparation.php index 07d3f302..e2722974 100644 --- a/src/V2/Support/PortableLocalActivityPreparation.php +++ b/src/V2/Support/PortableLocalActivityPreparation.php @@ -6,6 +6,7 @@ use Illuminate\Support\Carbon; use Illuminate\Validation\ValidationException; +use InvalidArgumentException; use JsonException; use Workflow\V2\Enums\ActivityAttemptStatus; use Workflow\V2\Enums\ActivityStatus; @@ -43,184 +44,45 @@ public static function prepare( array $descriptor, string $protocolVersion = WorkerProtocolVersion::VERSION, ): array { - if (preg_match('/^[0-9]+\.[0-9]+$/D', $protocolVersion) !== 1 - || version_compare($protocolVersion, self::MINIMUM_PROTOCOL_VERSION, '<')) { - return self::response('local_activity_preparation_requires_protocol_1_20'); - } - if ($sequence < 1 || $workflowTaskAttempt < 1 || $leaseOwner === '' - || trim($workerAttemptId) === '' || strlen($workerAttemptId) > 255 - || preg_match('//u', $workerAttemptId) !== 1) { - return self::response('invalid_local_activity_preparation'); - } - try { - $normalized = self::normalizeDescriptor($descriptor); - $fingerprint = hash('sha256', json_encode($normalized, JSON_THROW_ON_ERROR)); - } catch (ValidationException|JsonException) { - return self::response('invalid_local_activity_preparation'); - } - /** @var WorkflowTask|null $snapshotTask */ - $snapshotTask = ConfiguredV2Models::query('task_model', WorkflowTask::class)->find($taskId); - if ($snapshotTask === null) { - return self::response('task_not_found'); - } - /** @var ActivityExecution|null $snapshotExecution */ - $snapshotExecution = ActivityExecution::query() - ->where('workflow_run_id', $snapshotTask->workflow_run_id) - ->where('sequence', $sequence) - ->first(); - - return ConfiguredV2Models::connection()->transaction(static function () use ( - $snapshotTask, - $snapshotExecution, + return self::prepareForContract( + 1, $taskId, $leaseOwner, $workflowTaskAttempt, $sequence, $workerAttemptId, - $normalized, - $fingerprint, - ): array { - // Follow the shared attempt/execution/run/task lock order whenever - // an execution exists. An unseen concurrent creation is retried. - $rows = $snapshotExecution === null ? null : ( - is_string($snapshotExecution->current_attempt_id) - ? ActivityRowLockOrder::lockForAttempt($snapshotExecution->current_attempt_id) - : ActivityRowLockOrder::lockForExecution($snapshotExecution->id) - ); - /** @var WorkflowRun|null $run */ - $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) - ->lockForUpdate() - ->find($snapshotTask->workflow_run_id); - /** @var WorkflowTask|null $task */ - $task = ConfiguredV2Models::query('task_model', WorkflowTask::class) - ->lockForUpdate() - ->find($taskId); - if ($run === null || $task === null || $task->workflow_run_id !== $run->id) { - return self::response('workflow_claim_not_found'); - } - if ($task->task_type !== TaskType::Workflow || $task->status !== TaskStatus::Leased - || $task->lease_owner !== $leaseOwner || $task->attempt_count !== $workflowTaskAttempt) { - return self::response('workflow_claim_mismatch'); - } - if ($task->lease_expires_at === null || now()->gte($task->lease_expires_at)) { - return self::response('workflow_claim_expired'); - } - if ($run->status->isTerminal()) { - return self::response('run_closed'); - } - if (! CancellationScopeHistory::isRecordedBefore( - $run, - $normalized['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, - $sequence, - )) { - return self::response('local_activity_scope_not_recorded'); - } - $cleanup = PortableLocalActivityCleanup::snapshot( - $run, - $normalized['cancellation_cleanup'] ?? null, - $sequence, - $normalized['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, - ); - if (isset($normalized['cancellation_cleanup']) && $cleanup === null) { - return self::response('local_activity_cleanup_authority_mismatch'); - } - $scopeRefusal = CancellationScopeDelivery::admissionRefusal( - $run, - $normalized['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, - $sequence, - $normalized['cancellation_cleanup'] ?? null, - ); - if ($scopeRefusal !== null) { - return self::response($scopeRefusal); - } - $scopedCleanup = isset($cleanup['scope_id']); - if ($run->cancellation_request_command_id !== null - && ! PortableLocalActivityCleanup::belongsToRunRequest($run, $cleanup)) { - return self::response(isset($normalized['cancellation_cleanup']) - && ! $scopedCleanup ? 'local_activity_cleanup_authority_mismatch' : 'cancellation_requested'); - } - if ($run->cancellation_request_command_id === null && isset($normalized['cancellation_cleanup']) && ! $scopedCleanup) { - return self::response('local_activity_cleanup_authority_mismatch'); - } - if ($cleanup !== null && now()->gte(PortableLocalActivityCleanup::snapshotDeadline($cleanup))) { - return self::response( - $scopedCleanup ? 'local_activity_cleanup_deadline_expired' : 'cancellation_deadline_expired' - ); - } - if (($run->execution_deadline_at !== null && now()->gte($run->execution_deadline_at)) - || ($run->run_deadline_at !== null && now()->gte($run->run_deadline_at))) { - return self::response('run_deadline_expired'); - } - if ($scopedCleanup && now()->gte(PortableLocalActivityCleanup::currentDeadline($run, $cleanup))) { - return self::response('local_activity_cleanup_authority_expired'); - } - $execution = $rows['execution'] ?? null; - $attempt = $rows['attempt'] ?? null; - if ($snapshotExecution !== null) { - if (! $execution instanceof ActivityExecution - || $execution->id !== $snapshotExecution->id || $execution->sequence !== $sequence - || $execution->current_attempt_id !== ($rows['snapshot_attempt_id'] ?? null)) { - return self::response('local_activity_previous_attempt_unresolved'); - } - if ($attempt === null) { - return self::prepareAdmittedMember( - $run, - $task, - $execution, - $normalized, - $workerAttemptId, - $fingerprint - ); - } - if (! $attempt instanceof ActivityAttempt) { - return self::response('local_activity_previous_attempt_unresolved'); - } - if ($execution->status === ActivityStatus::Pending) { - return self::prepareRetry($run, $task, $execution, $attempt, $workerAttemptId, $fingerprint); - } - return self::duplicate($run, $task, $execution, $attempt, $workerAttemptId, $fingerprint); - } - if (ActivityExecution::query()->where('workflow_run_id', $run->id)->where( - 'sequence', - $sequence - )->exists()) { - return self::response('local_activity_preparation_retry'); - } - if (WorkflowStepHistory::nextDurableCommandSequence($run) !== $sequence) { - return self::response('local_activity_command_prefix_not_recorded'); - } - if (($normalized['parallel_group_path'] ?? []) !== []) { - return self::response('local_activity_group_not_admitted'); - } - $preparation = [ - 'version' => 1, - 'descriptor_fingerprint' => $fingerprint, - 'worker_attempt_id' => $workerAttemptId, - 'workflow_task_attempt' => $workflowTaskAttempt, - ...($cleanup === null ? [] : [ - 'cancellation_cleanup' => $cleanup, - ]), - ]; - $execution = self::createExecution( - $run, - $task, - $sequence, - $normalized, - $cleanup, - [ - 'local_preparation' => $preparation, - ] - ); - $attempt = app(LocalActivityExecutor::class)->startPortableAttempt( - $run, - $task, - $execution, - $workerAttemptId, - $preparation - ); + $descriptor, + $protocolVersion + ); + } - return self::response(null, $execution, $attempt, task: $task); - }, 5); + /** + * @internal Unadvertised v2 primitive. A service adapter must separately + * prove its installed prepared role and original issued claim capability + * inside the containing transaction before invoking this method. + * + * @param array $descriptor + * @return array + */ + public static function prepareWithRetryPolicy( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + string $workerAttemptId, + array $descriptor, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return self::prepareForContract( + 2, + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $sequence, + $workerAttemptId, + $descriptor, + $protocolVersion + ); } /** @@ -494,8 +356,11 @@ public static function recover( /** @param array $descriptor * @return array */ - public static function normalizeDescriptor(array $descriptor): array + public static function normalizeDescriptor(array $descriptor, int $localContractVersion = 1): array { + if (! in_array($localContractVersion, [1, 2], true)) { + throw new InvalidArgumentException('Unknown prepared local activity contract.'); + } $allowed = ['type', 'activity_type', 'arguments', 'payload_codec', 'retry_policy', 'start_to_close_timeout', 'schedule_to_close_timeout', 'heartbeat_timeout', 'execution_mode', 'cancellation_cleanup', 'cancellation_policy', 'cancellation_scope_id', 'parallel_group_id', 'parallel_group_kind', 'parallel_group_mode', @@ -528,6 +393,25 @@ public static function normalizeDescriptor(array $descriptor): array // fabricated outcome or a claim that an application attempt ran. $input = $descriptor; unset($input['execution_mode'], $input['cancellation_cleanup'], $input['cancellation_scope_id']); + $retrySnapshot = null; + if ($localContractVersion === 2) { + if (! is_array($descriptor['retry_policy'] ?? null)) { + throw new InvalidArgumentException('Prepared contract 2 requires an explicit version-2 retry policy.'); + } + $retrySnapshot = PreparedLocalRetryPolicy::normalize($descriptor['retry_policy'], array_intersect_key( + $descriptor, + array_flip(['start_to_close_timeout', 'schedule_to_close_timeout', 'heartbeat_timeout']) + )); + unset($input['retry_policy']); + } elseif (is_array($descriptor['retry_policy'] ?? null) + && array_intersect( + array_keys($descriptor['retry_policy']), + ['contract_version', 'snapshot_version', 'mode', 'maximum_interval_seconds'] + ) !== []) { + throw ValidationException::withMessages([ + 'local_activity.retry_policy' => ['Versioned retry policies require their explicit prepared contract.'], + ]); + } $input['type'] = 'schedule_activity'; $normalized = WorkflowCommandNormalizer::normalize([$input], self::MINIMUM_PROTOCOL_VERSION)[0]; if (! is_string($normalized['arguments'] ?? null) || ! is_string($normalized['payload_codec'] ?? null)) { @@ -537,6 +421,10 @@ public static function normalizeDescriptor(array $descriptor): array } $normalized['type'] = 'record_local_activity'; $normalized['execution_mode'] = LocalActivityRuntime::EXECUTION_MODE; + if ($retrySnapshot !== null) { + $normalized['retry_policy'] = $retrySnapshot; + $normalized['local_activity_contract_version'] = 2; + } if (array_key_exists('cancellation_scope_id', $descriptor)) { $normalized['cancellation_scope_id'] = $descriptor['cancellation_scope_id']; } @@ -579,6 +467,7 @@ public static function originalStart( ActivityAttempt $attempt, string $leaseOwner, int $workflowTaskAttempt, + int $localContractVersion = 1, ): ?WorkflowHistoryEvent { if (! LocalActivityRuntime::isExecution($execution) || $execution->workflow_run_id !== $run->id || $attempt->workflow_run_id !== $run->id || $attempt->activity_execution_id !== $execution->id @@ -598,7 +487,7 @@ public static function originalStart( || ($started->payload['workflow_task_id'] ?? null) !== $attempt->workflow_task_id || ($started->payload['local_activity'] ?? null) !== true || ($started->payload['execution_mode'] ?? null) !== LocalActivityRuntime::EXECUTION_MODE - || ($started->payload['local_preparation']['version'] ?? null) !== 1 + || ($started->payload['local_preparation']['version'] ?? null) !== $localContractVersion || ! is_string($started->payload['local_preparation']['descriptor_fingerprint'] ?? null) || ($started->payload['local_preparation']['worker_attempt_id'] ?? null) !== $attempt->worker_attempt_id || ($started->payload['local_preparation']['workflow_task_attempt'] ?? null) !== $workflowTaskAttempt @@ -615,6 +504,26 @@ public static function originalStart( || ($started->payload['activity_attempt']['worker_attempt_id'] ?? null) !== $attempt->worker_attempt_id) { return null; } + if ($localContractVersion === 2) { + if (($started->payload['local_preparation']['local_activity_contract_version'] ?? null) !== 2 + || ($execution->activity_options['local_activity_contract_version'] ?? null) !== 2 + || ! is_array($started->payload['activity']['retry_policy'] ?? null) + || ! is_array($execution->retry_policy)) { + return null; + } + try { + if (PreparedLocalRetryPolicy::validateSnapshot($execution->retry_policy) + !== PreparedLocalRetryPolicy::validateSnapshot($started->payload['activity']['retry_policy'])) { + return null; + } + } catch (InvalidArgumentException) { + return null; + } + } elseif ($localContractVersion !== 1 + || array_key_exists('local_activity_contract_version', $started->payload['local_preparation']) + || ($execution->retry_policy['snapshot_version'] ?? null) === 2) { + return null; + } $scopeId = $execution->activity_options['cancellation_scope_id'] ?? null; if (($started->payload['activity']['cancellation_scope_id'] ?? null) !== $scopeId) { return null; @@ -644,6 +553,217 @@ public static function originalStart( return $started; } + /** + * @param array $descriptor + * @return array + */ + private static function prepareForContract( + int $localContractVersion, + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + string $workerAttemptId, + array $descriptor, + string $protocolVersion, + ): array { + if (preg_match('/^[0-9]+\.[0-9]+$/D', $protocolVersion) !== 1 + || version_compare($protocolVersion, self::MINIMUM_PROTOCOL_VERSION, '<')) { + return self::response('local_activity_preparation_requires_protocol_1_20'); + } + if ($sequence < 1 || $workflowTaskAttempt < 1 || $leaseOwner === '' + || trim($workerAttemptId) === '' || strlen($workerAttemptId) > 255 + || preg_match('//u', $workerAttemptId) !== 1) { + return self::response('invalid_local_activity_preparation'); + } + try { + $normalized = self::normalizeDescriptor($descriptor, $localContractVersion); + $fingerprint = hash('sha256', json_encode($normalized, JSON_THROW_ON_ERROR)); + } catch (ValidationException|JsonException) { + return self::response('invalid_local_activity_preparation'); + } catch (InvalidArgumentException $error) { + if ($localContractVersion === 1) { + throw $error; + } + return self::response('invalid_local_activity_preparation'); + } + /** @var WorkflowTask|null $snapshotTask */ + $snapshotTask = ConfiguredV2Models::query('task_model', WorkflowTask::class)->find($taskId); + if ($snapshotTask === null) { + return self::response('task_not_found'); + } + /** @var ActivityExecution|null $snapshotExecution */ + $snapshotExecution = ActivityExecution::query() + ->where('workflow_run_id', $snapshotTask->workflow_run_id) + ->where('sequence', $sequence) + ->first(); + + return ConfiguredV2Models::connection()->transaction(static function () use ( + $snapshotTask, + $snapshotExecution, + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $sequence, + $workerAttemptId, + $normalized, + $fingerprint, + $localContractVersion, + ): array { + // Follow the shared attempt/execution/run/task lock order whenever + // an execution exists. An unseen concurrent creation is retried. + $rows = $snapshotExecution === null ? null : ( + is_string($snapshotExecution->current_attempt_id) + ? ActivityRowLockOrder::lockForAttempt($snapshotExecution->current_attempt_id) + : ActivityRowLockOrder::lockForExecution($snapshotExecution->id) + ); + /** @var WorkflowRun|null $run */ + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->lockForUpdate() + ->find($snapshotTask->workflow_run_id); + /** @var WorkflowTask|null $task */ + $task = ConfiguredV2Models::query('task_model', WorkflowTask::class) + ->lockForUpdate() + ->find($taskId); + if ($run === null || $task === null || $task->workflow_run_id !== $run->id) { + return self::response('workflow_claim_not_found'); + } + if ($task->task_type !== TaskType::Workflow || $task->status !== TaskStatus::Leased + || $task->lease_owner !== $leaseOwner || $task->attempt_count !== $workflowTaskAttempt) { + return self::response('workflow_claim_mismatch'); + } + if ($task->lease_expires_at === null || now()->gte($task->lease_expires_at)) { + return self::response('workflow_claim_expired'); + } + if ($run->status->isTerminal()) { + return self::response('run_closed'); + } + if (! CancellationScopeHistory::isRecordedBefore( + $run, + $normalized['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, + $sequence, + )) { + return self::response('local_activity_scope_not_recorded'); + } + $cleanup = PortableLocalActivityCleanup::snapshot( + $run, + $normalized['cancellation_cleanup'] ?? null, + $sequence, + $normalized['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, + ); + if (isset($normalized['cancellation_cleanup']) && $cleanup === null) { + return self::response('local_activity_cleanup_authority_mismatch'); + } + $scopeRefusal = CancellationScopeDelivery::admissionRefusal( + $run, + $normalized['cancellation_scope_id'] ?? CancellationScopeHistory::ROOT_SCOPE_ID, + $sequence, + $normalized['cancellation_cleanup'] ?? null, + ); + if ($scopeRefusal !== null) { + return self::response($scopeRefusal); + } + $scopedCleanup = isset($cleanup['scope_id']); + if ($run->cancellation_request_command_id !== null + && ! PortableLocalActivityCleanup::belongsToRunRequest($run, $cleanup)) { + return self::response(isset($normalized['cancellation_cleanup']) + && ! $scopedCleanup ? 'local_activity_cleanup_authority_mismatch' : 'cancellation_requested'); + } + if ($run->cancellation_request_command_id === null && isset($normalized['cancellation_cleanup']) && ! $scopedCleanup) { + return self::response('local_activity_cleanup_authority_mismatch'); + } + if ($cleanup !== null && now()->gte(PortableLocalActivityCleanup::snapshotDeadline($cleanup))) { + return self::response( + $scopedCleanup ? 'local_activity_cleanup_deadline_expired' : 'cancellation_deadline_expired' + ); + } + if (($run->execution_deadline_at !== null && now()->gte($run->execution_deadline_at)) + || ($run->run_deadline_at !== null && now()->gte($run->run_deadline_at))) { + return self::response('run_deadline_expired'); + } + if ($scopedCleanup && now()->gte(PortableLocalActivityCleanup::currentDeadline($run, $cleanup))) { + return self::response('local_activity_cleanup_authority_expired'); + } + $execution = $rows['execution'] ?? null; + $attempt = $rows['attempt'] ?? null; + if ($snapshotExecution !== null) { + if (! $execution instanceof ActivityExecution + || $execution->id !== $snapshotExecution->id || $execution->sequence !== $sequence + || $execution->current_attempt_id !== ($rows['snapshot_attempt_id'] ?? null)) { + return self::response('local_activity_previous_attempt_unresolved'); + } + if ($attempt === null) { + return self::prepareAdmittedMember( + $run, + $task, + $execution, + $normalized, + $workerAttemptId, + $fingerprint + ); + } + if (! $attempt instanceof ActivityAttempt) { + return self::response('local_activity_previous_attempt_unresolved'); + } + if ($execution->status === ActivityStatus::Pending) { + return self::prepareRetry($run, $task, $execution, $attempt, $workerAttemptId, $fingerprint); + } + return self::duplicate( + $run, + $task, + $execution, + $attempt, + $workerAttemptId, + $fingerprint, + $localContractVersion + ); + } + if (ActivityExecution::query()->where('workflow_run_id', $run->id)->where( + 'sequence', + $sequence + )->exists()) { + return self::response('local_activity_preparation_retry'); + } + if (WorkflowStepHistory::nextDurableCommandSequence($run) !== $sequence) { + return self::response('local_activity_command_prefix_not_recorded'); + } + if (($normalized['parallel_group_path'] ?? []) !== []) { + return self::response('local_activity_group_not_admitted'); + } + $preparation = [ + 'version' => $localContractVersion, + ...($localContractVersion === 2 ? [ + 'local_activity_contract_version' => 2, + ] : []), + 'descriptor_fingerprint' => $fingerprint, + 'worker_attempt_id' => $workerAttemptId, + 'workflow_task_attempt' => $workflowTaskAttempt, + ...($cleanup === null ? [] : [ + 'cancellation_cleanup' => $cleanup, + ]), + ]; + $execution = self::createExecution( + $run, + $task, + $sequence, + $normalized, + $cleanup, + [ + 'local_preparation' => $preparation, + ] + ); + $attempt = app(LocalActivityExecutor::class)->startPortableAttempt( + $run, + $task, + $execution, + $workerAttemptId, + $preparation + ); + + return self::response(null, $execution, $attempt, task: $task); + }, 5); + } + /** @param array $normalized * @param array|null $cleanup * @param array $opening @@ -683,11 +803,17 @@ private static function createExecution( 'connection' => $run->connection, 'queue' => $run->queue, 'parallel_group_path' => $normalized['parallel_group_path'] ?? null, - 'retry_policy' => ActivityRetryPolicy::snapshotExternal($normalized['retry_policy'] ?? null, $options), + 'retry_policy' => ($normalized['local_activity_contract_version'] ?? null) === 2 + ? PreparedLocalRetryPolicy::validateSnapshot($normalized['retry_policy']) + : ActivityRetryPolicy::snapshotExternal($normalized['retry_policy'] ?? null, $options), 'activity_options' => [ 'execution_mode' => LocalActivityRuntime::EXECUTION_MODE, 'queue_bypassed' => true, 'routing' => 'workflow_worker_process', + ...(($normalized['local_activity_contract_version'] ?? null) === 2 + ? [ + 'local_activity_contract_version' => 2, + ] : []), ...(isset($normalized['cancellation_policy']) ? [ 'cancellation_policy' => $normalized['cancellation_policy'], ] : []), @@ -1043,6 +1169,7 @@ private static function duplicate( ActivityAttempt $attempt, string $workerAttemptId, string $fingerprint, + int $localContractVersion = 1, ): array { if (! LocalActivityRuntime::isExecution($execution) || $execution->workflow_run_id !== $run->id || $execution->status !== ActivityStatus::Running @@ -1050,7 +1177,14 @@ private static function duplicate( || $attempt->worker_attempt_id !== $workerAttemptId || $attempt->status !== ActivityAttemptStatus::Running) { return self::response('local_activity_preparation_mismatch'); } - $started = self::originalStart($run, $execution, $attempt, $task->lease_owner, $task->attempt_count); + $started = self::originalStart( + $run, + $execution, + $attempt, + $task->lease_owner, + $task->attempt_count, + $localContractVersion + ); if (! $started instanceof WorkflowHistoryEvent || ($started->payload['local_preparation']['descriptor_fingerprint'] ?? null) !== $fingerprint) { return self::response('local_activity_preparation_mismatch'); diff --git a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php index d20d3a1c..16aefa8b 100644 --- a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php +++ b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php @@ -30,8 +30,10 @@ use Workflow\V2\Support\ActivityCancellationAcknowledgement; use Workflow\V2\Support\ActivitySnapshot; use Workflow\V2\Support\CancellationScopeHistory; +use Workflow\V2\Support\ConfiguredV2Models; use Workflow\V2\Support\DefaultWorkflowTaskBridge; use Workflow\V2\Support\HistoryTimeline; +use Workflow\V2\Support\LocalActivityExecutor; use Workflow\V2\Support\ParallelChildGroup; use Workflow\V2\Support\PortableLocalActivityPreparation; use Workflow\V2\WorkflowStub; @@ -89,6 +91,112 @@ public function testPreparationPersistsAWorkerAliasAndOriginalClaimBeforeAnyAppl } } + public function testRetryContractPreparationRetainsItsOriginalModeAndCannotUseTheLegacyOutcomeEntry(): void + { + [$run, $task] = $this->newClaim(); + $descriptor = [ + ...$this->descriptor(), + 'retry_policy' => [ + 'contract_version' => 2, + 'mode' => 'explicit', + 'max_attempts' => 3, + 'backoff_seconds' => [2, 5], + ], + ]; + $prepare = static fn (): array => PortableLocalActivityPreparation::prepareWithRetryPolicy( + $task->id, + 'portable-worker', + 1, + 1, + 'sdk-local-attempt', + $descriptor, + '1.20' + ); + $reply = $prepare(); + $this->assertTrue($reply['prepared']); + $execution = ActivityExecution::query()->findOrFail($reply['activity_execution_id']); + $attempt = ActivityAttempt::query()->findOrFail($reply['activity_attempt_id']); + $this->assertSame(2, $execution->activity_options['local_activity_contract_version']); + $this->assertSame(2, $execution->retry_policy['snapshot_version']); + $this->assertSame(5, $execution->retry_policy['maximum_interval_seconds']); + $started = PortableLocalActivityPreparation::originalStart($run, $execution, $attempt, 'portable-worker', 1, 2); + $this->assertInstanceOf(WorkflowHistoryEvent::class, $started); + $this->assertSame(2, $started->payload['local_preparation']['version']); + $this->assertSame(2, $started->payload['local_preparation']['local_activity_contract_version']); + $this->assertNull( + PortableLocalActivityPreparation::originalStart($run, $execution, $attempt, 'portable-worker', 1) + ); + $refused = app(LocalActivityExecutor::class)->recordPortableOutcome( + $attempt->id, + 'portable-worker', + 1, + [ + 'outcome' => 'failed', + 'message' => 'failure', + ], + '1.20' + ); + $this->assertFalse($refused['recorded']); + $this->assertSame('local_activity_preparation_mismatch', $refused['reason']); + $this->assertSame(ActivityAttemptStatus::Running, $attempt->refresh()->status); + $this->assertSame(2, $run->historyEvents()->count()); + $duplicate = $prepare(); + $this->assertTrue($duplicate['prepared']); + $this->assertTrue($duplicate['duplicate']); + $this->assertSame($attempt->id, $duplicate['activity_attempt_id']); + $this->assertSame(2, $run->historyEvents()->count()); + + $policy = $execution->retry_policy; + $execution->forceFill([ + 'retry_policy' => [ + ...$policy, + 'maximum_interval_seconds' => 6, + ], + ])->save(); + $this->assertNull(PortableLocalActivityPreparation::originalStart( + $run, + $execution->refresh(), + $attempt, + 'portable-worker', + 1, + 2 + )); + } + + public function testRetryContractPreparationRollsBackModeSnapshotAndAttemptWithItsOuterTransaction(): void + { + [$run, $task] = $this->newClaim(); + $connection = ConfiguredV2Models::connection(); + $connection->beginTransaction(); + try { + $reply = PortableLocalActivityPreparation::prepareWithRetryPolicy( + $task->id, + 'portable-worker', + 1, + 1, + 'sdk-local-attempt', + [ + ...$this->descriptor(), + 'retry_policy' => [ + 'contract_version' => 2, + 'mode' => 'explicit', + 'max_attempts' => 3, + 'backoff_seconds' => [2, 5], + ], + ], + '1.20' + ); + $this->assertTrue($reply['prepared']); + $this->assertSame(2, $run->historyEvents()->count()); + } finally { + $connection->rollBack(); + } + $this->assertSame(0, $run->activityExecutions()->count()); + $this->assertSame(0, ActivityAttempt::query()->where('workflow_run_id', $run->id)->count()); + $this->assertSame(0, $run->historyEvents()->count()); + $this->assertSame(TaskStatus::Leased, $task->refresh()->status); + } + public function testLostPreparationResponseReturnsTheOriginalAttemptWithoutRenewingAnyDeadline(): void { [, $task] = $this->newClaim(); diff --git a/tests/Unit/V2/PreparedLocalRetryDescriptorTest.php b/tests/Unit/V2/PreparedLocalRetryDescriptorTest.php new file mode 100644 index 00000000..7e19cba0 --- /dev/null +++ b/tests/Unit/V2/PreparedLocalRetryDescriptorTest.php @@ -0,0 +1,91 @@ +instance('translator', $translator); + $container->instance('validator', $factory); + $container->instance('app', $container); + + // Save and replace the facade application so downstream tests in the + // same process are not affected by our minimal container. + $this->priorFacadeApp = \Illuminate\Support\Facades\Facade::getFacadeApplication(); + \Illuminate\Support\Facades\Facade::clearResolvedInstances(); + \Illuminate\Support\Facades\Facade::setFacadeApplication($container); + } + + protected function tearDown(): void + { + \Illuminate\Support\Facades\Facade::clearResolvedInstances(); + \Illuminate\Support\Facades\Facade::setFacadeApplication($this->priorFacadeApp); + $this->priorFacadeApp = null; + + parent::tearDown(); + } + + public function testPreparedRetryNormalizationRequiresTheExplicitVersionTwoEntry(): void + { + $descriptor = [ + 'type' => 'record_local_activity', + 'activity_type' => 'local-worker-alias', + 'arguments' => Serializer::serializeWithCodec('avro', []), + 'payload_codec' => 'avro', + 'retry_policy' => [ + 'contract_version' => 2, + 'mode' => 'explicit', + 'max_attempts' => 3, + 'backoff_seconds' => [2, 5], + ], + 'start_to_close_timeout' => 30, + ]; + $normalized = PortableLocalActivityPreparation::normalizeDescriptor($descriptor, 2); + $this->assertSame(2, $normalized['local_activity_contract_version']); + $this->assertSame(2, $normalized['retry_policy']['snapshot_version']); + $this->assertSame([2, 5], $normalized['retry_policy']['backoff_seconds']); + $this->assertSame(5, $normalized['retry_policy']['maximum_interval_seconds']); + $this->assertSame(30, $normalized['retry_policy']['start_to_close_timeout']); + try { + PortableLocalActivityPreparation::normalizeDescriptor($descriptor); + $this->fail('The v1 entry silently accepted a v2 policy.'); + } catch (ValidationException) { + $this->addToAssertionCount(1); + } + $legacy = $descriptor; + $legacy['retry_policy'] = [ + 'max_attempts' => 3, + 'backoff_seconds' => [2, 5], + ]; + $this->assertArrayNotHasKey( + 'local_activity_contract_version', + PortableLocalActivityPreparation::normalizeDescriptor($legacy) + ); + $this->expectException(\InvalidArgumentException::class); + PortableLocalActivityPreparation::normalizeDescriptor($legacy, 2); + } +} From 2730aba7603a8f86b2d0a3acba97f52113263e0b Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 6 Nov 2026 00:08:35 +0000 Subject: [PATCH 03/15] Bound prepared retry timestamps and preserve legacy replay delays --- .github/pr-test-selection.json | 1 + src/V2/Support/LocalActivityExecutor.php | 17 +- .../PortableLocalActivityPreparation.php | 36 ++- .../Support/PreparedLocalRetryTimestamp.php | 104 +++++++++ ...V2PortableLocalActivityPreparationTest.php | 42 ++++ ...ed-v1-policy-refusal-legacy-zero-tail.json | 211 ++++++++++++++++++ .../V2/PreparedLocalRetryTimestampTest.php | 147 ++++++++++++ tests/Unit/V2/ReplayRegressionCorpusTest.php | 57 ++++- 8 files changed, 603 insertions(+), 12 deletions(-) create mode 100644 src/V2/Support/PreparedLocalRetryTimestamp.php create mode 100644 tests/Fixtures/V2/ReplayRegression/prepared-v1-policy-refusal-legacy-zero-tail.json create mode 100644 tests/Unit/V2/PreparedLocalRetryTimestampTest.php diff --git a/.github/pr-test-selection.json b/.github/pr-test-selection.json index 6619ba6c..bd652a5e 100644 --- a/.github/pr-test-selection.json +++ b/.github/pr-test-selection.json @@ -23,6 +23,7 @@ "tests/Unit/V2/PreparedLocalRetryDecisionTest.php", "tests/Unit/V2/PreparedLocalRetryDescriptorTest.php", "tests/Unit/V2/PreparedLocalRetryPolicyTest.php", + "tests/Unit/V2/PreparedLocalRetryTimestampTest.php", "tests/Unit/V2/ReadmeBranchDestinationsTest.php", "tests/Unit/V2/ScopedCancellationContextTest.php", "tests/Unit/V2/SqliteTransactionTest.php", diff --git a/src/V2/Support/LocalActivityExecutor.php b/src/V2/Support/LocalActivityExecutor.php index 4398d6e7..6f6ddf54 100644 --- a/src/V2/Support/LocalActivityExecutor.php +++ b/src/V2/Support/LocalActivityExecutor.php @@ -4,6 +4,7 @@ namespace Workflow\V2\Support; +use Illuminate\Support\Carbon; use Illuminate\Support\Str; use Illuminate\Validation\ValidationException; use JsonException; @@ -315,9 +316,10 @@ public function startPortableAttempt( ActivityExecution $execution, string $workerAttemptId, array $preparation, + ?Carbon $preparedAt = null, ): ActivityAttempt { $preparation['schedule_to_close_deadline_at'] = $execution->schedule_to_close_deadline_at?->toISOString(); - return $this->startAttempt($run, $task, $execution, $workerAttemptId, $preparation); + return $this->startAttempt($run, $task, $execution, $workerAttemptId, $preparation, $preparedAt); } /** @@ -662,8 +664,19 @@ private function startAttempt( ActivityExecution $execution, ?string $workerAttemptId = null, array $preparation = [], + ?Carbon $preparedAt = null, ): ActivityAttempt { - $now = now(); + $now = $preparedAt?->copy() ?? now(); + if (($execution->activity_options['local_activity_contract_version'] ?? null) === 2) { + PreparedLocalRetryPolicy::validateSnapshot($execution->retry_policy); + foreach (['start_to_close_timeout', 'heartbeat_timeout'] as $field) { + PreparedLocalRetryTimestamp::addForConnection( + $now, + $execution->retry_policy[$field] ?? 0, + ConfiguredV2Models::connection() + ); + } + } $attemptId = (string) Str::ulid(); $attemptNumber = ((int) $execution->attempt_count) + 1; $retryPolicy = is_array($execution->retry_policy) ? $execution->retry_policy : []; diff --git a/src/V2/Support/PortableLocalActivityPreparation.php b/src/V2/Support/PortableLocalActivityPreparation.php index e2722974..37ada876 100644 --- a/src/V2/Support/PortableLocalActivityPreparation.php +++ b/src/V2/Support/PortableLocalActivityPreparation.php @@ -628,11 +628,12 @@ private static function prepareForContract( if ($run === null || $task === null || $task->workflow_run_id !== $run->id) { return self::response('workflow_claim_not_found'); } + $preparedAt = $localContractVersion === 2 ? now() : null; if ($task->task_type !== TaskType::Workflow || $task->status !== TaskStatus::Leased || $task->lease_owner !== $leaseOwner || $task->attempt_count !== $workflowTaskAttempt) { return self::response('workflow_claim_mismatch'); } - if ($task->lease_expires_at === null || now()->gte($task->lease_expires_at)) { + if ($task->lease_expires_at === null || ($preparedAt ?? now())->gte($task->lease_expires_at)) { return self::response('workflow_claim_expired'); } if ($run->status->isTerminal()) { @@ -672,16 +673,20 @@ private static function prepareForContract( if ($run->cancellation_request_command_id === null && isset($normalized['cancellation_cleanup']) && ! $scopedCleanup) { return self::response('local_activity_cleanup_authority_mismatch'); } - if ($cleanup !== null && now()->gte(PortableLocalActivityCleanup::snapshotDeadline($cleanup))) { + if ($cleanup !== null && ($preparedAt ?? now())->gte( + PortableLocalActivityCleanup::snapshotDeadline($cleanup) + )) { return self::response( $scopedCleanup ? 'local_activity_cleanup_deadline_expired' : 'cancellation_deadline_expired' ); } - if (($run->execution_deadline_at !== null && now()->gte($run->execution_deadline_at)) - || ($run->run_deadline_at !== null && now()->gte($run->run_deadline_at))) { + if (($run->execution_deadline_at !== null && ($preparedAt ?? now())->gte($run->execution_deadline_at)) + || ($run->run_deadline_at !== null && ($preparedAt ?? now())->gte($run->run_deadline_at))) { return self::response('run_deadline_expired'); } - if ($scopedCleanup && now()->gte(PortableLocalActivityCleanup::currentDeadline($run, $cleanup))) { + if ($scopedCleanup && ($preparedAt ?? now())->gte( + PortableLocalActivityCleanup::currentDeadline($run, $cleanup) + )) { return self::response('local_activity_cleanup_authority_expired'); } $execution = $rows['execution'] ?? null; @@ -750,14 +755,16 @@ private static function prepareForContract( $cleanup, [ 'local_preparation' => $preparation, - ] + ], + $preparedAt ); $attempt = app(LocalActivityExecutor::class)->startPortableAttempt( $run, $task, $execution, $workerAttemptId, - $preparation + $preparation, + $preparedAt ); return self::response(null, $execution, $attempt, task: $task); @@ -774,9 +781,21 @@ private static function createExecution( int $sequence, array $normalized, ?array $cleanup, - array $opening + array $opening, + ?Carbon $preparedAt = null, ): ActivityExecution { StructuralLimits::guardPendingActivities($run); + $now = $preparedAt ?? now(); + if (($normalized['local_activity_contract_version'] ?? null) === 2) { + $connection = ConfiguredV2Models::connection(); + foreach (['start_to_close_timeout', 'schedule_to_close_timeout', 'heartbeat_timeout'] as $field) { + PreparedLocalRetryTimestamp::addForConnection( + $now, + $normalized['retry_policy'][$field] ?? 0, + $connection + ); + } + } $codec = $normalized['payload_codec']; $arguments = ExternalPayloads::externalizeForNamespace( $normalized['arguments'], @@ -784,7 +803,6 @@ private static function createExecution( is_string($run->namespace) ? $run->namespace : null ); StructuralLimits::guardPayloadSize($arguments); - $now = now(); $options = new ActivityOptions( startToCloseTimeout: $normalized['start_to_close_timeout'] ?? null, scheduleToCloseTimeout: $normalized['schedule_to_close_timeout'] ?? null, diff --git a/src/V2/Support/PreparedLocalRetryTimestamp.php b/src/V2/Support/PreparedLocalRetryTimestamp.php new file mode 100644 index 00000000..9c4bd22a --- /dev/null +++ b/src/V2/Support/PreparedLocalRetryTimestamp.php @@ -0,0 +1,104 @@ +getDriverName()) + ); + } + + public static function earliestForDriver(string $driver): DateTimeImmutable + { + if (in_array($driver, ['mysql', 'mariadb'], true)) { + return new DateTimeImmutable('1970-01-01T00:00:01.000000Z'); + } + if (! in_array($driver, ['sqlite', 'pgsql', 'sqlsrv'], true)) { + throw new InvalidArgumentException('Unsupported prepared retry timestamp backend.'); + } + + return new DateTimeImmutable('0001-01-01T00:00:00.000000Z'); + } + + public static function latestForConnection(Connection $connection): DateTimeImmutable + { + $driver = $connection->getDriverName(); + if ($connection instanceof MySqlConnection && $connection->isMaria()) { + $driver = 'mariadb'; + } + + return self::latestForDriver($driver, $driver === 'mariadb' ? $connection->getServerVersion() : null); + } + + public static function latestForDriver(string $driver, ?string $serverVersion = null): DateTimeImmutable + { + // MySQL TIMESTAMP(6), rather than DATETIME's broader year range: + // https://dev.mysql.com/doc/refman/8.4/en/datetime.html + if ($driver === 'mysql') { + return new DateTimeImmutable('2038-01-19T03:14:07.499999Z'); + } + if ($driver === 'mariadb') { + if (! is_string($serverVersion) + || preg_match('/^(?:5\.5\.5-)?([0-9]+\.[0-9]+(?:\.[0-9]+)?)/D', $serverVersion, $match) !== 1) { + throw new InvalidArgumentException( + 'The MariaDB timestamp range requires an identified server version.' + ); + } + // The unsigned seconds range starts at MariaDB 11.5. + // https://mariadb.com/docs/server/reference/data-types/date-and-time-data-types/timestamp + return new DateTimeImmutable(version_compare($match[1], '11.5', '>=') + ? '2106-02-07T06:28:15.999999Z' : '2038-01-19T03:14:07.999999Z'); + } + if (! in_array($driver, ['sqlite', 'pgsql', 'sqlsrv'], true)) { + throw new InvalidArgumentException('Unsupported prepared retry timestamp backend.'); + } + + // These stores admit the four-digit UTC domain used by this contract. + // Laravel's timestamp(..., 6) maps to datetime2(6) on SQL Server. + return new DateTimeImmutable('9999-12-31T23:59:59.999999Z'); + } + + public static function add( + DateTimeInterface $sample, + int $seconds, + DateTimeInterface $latest, + ?DateTimeInterface $earliest = null, + ): DateTimeImmutable { + $utc = DateTimeImmutable::createFromInterface($sample)->setTimezone(new DateTimeZone('UTC')); + $latest = DateTimeImmutable::createFromInterface($latest)->setTimezone(new DateTimeZone('UTC')); + $earliest ??= new DateTimeImmutable('0001-01-01T00:00:00.000000Z'); + if ($seconds < 0 || $seconds > PreparedLocalRetryPolicy::MAX_DELAY_SECONDS + || strlen($utc->format('Y')) !== 4 || (int) $utc->format('Y') < 1 + || strlen($latest->format('Y')) !== 4 || (int) $latest->format('Y') < 1 + || $utc < $earliest || $utc > $latest) { + throw new InvalidArgumentException('Invalid prepared retry timestamp input.'); + } + $remaining = ($latest->getTimestamp() - $utc->getTimestamp()) * 1000000 + + (int) $latest->format('u') - (int) $utc->format('u'); + if ($seconds > intdiv($remaining, 1000000)) { + throw new InvalidArgumentException('Prepared retry timestamp exceeds the backend range.'); + } + + return $utc->add(new DateInterval('PT' . $seconds . 'S')); + } +} diff --git a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php index 16aefa8b..8946e1fe 100644 --- a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php +++ b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php @@ -36,6 +36,7 @@ use Workflow\V2\Support\LocalActivityExecutor; use Workflow\V2\Support\ParallelChildGroup; use Workflow\V2\Support\PortableLocalActivityPreparation; +use Workflow\V2\Support\PreparedLocalRetryTimestamp; use Workflow\V2\WorkflowStub; final class V2PortableLocalActivityPreparationTest extends TestCase @@ -197,6 +198,47 @@ public function testRetryContractPreparationRollsBackModeSnapshotAndAttemptWithI $this->assertSame(TaskStatus::Leased, $task->refresh()->status); } + public function testRetryContractRejectsAnUnbindableDeadlineBeforePublishingPreparation(): void + { + [$run, $task] = $this->newClaim(); + $latest = PreparedLocalRetryTimestamp::latestForConnection(ConfiguredV2Models::connection()); + $task->forceFill([ + 'lease_expires_at' => $latest, + ])->save(); + $before = $task->refresh() + ->getAttributes(); + Carbon::setTestNow(Carbon::instance($latest)->subMicroseconds(500000)); + try { + PortableLocalActivityPreparation::prepareWithRetryPolicy( + $task->id, + 'portable-worker', + 1, + 1, + 'sdk-local-attempt', + [ + ...$this->descriptor(), + 'retry_policy' => [ + 'contract_version' => 2, + 'mode' => 'explicit', + 'max_attempts' => 3, + 'backoff_seconds' => [2, 5], + ], + 'start_to_close_timeout' => 1, + ], + '1.20' + ); + $this->fail('Prepared an attempt with an unbindable deadline.'); + } catch (\InvalidArgumentException) { + $this->addToAssertionCount(1); + } finally { + Carbon::setTestNow(); + } + $this->assertSame(0, $run->activityExecutions()->count()); + $this->assertSame(0, ActivityAttempt::query()->where('workflow_run_id', $run->id)->count()); + $this->assertSame(0, $run->historyEvents()->count()); + $this->assertSame($before, $task->refresh()->getAttributes()); + } + public function testLostPreparationResponseReturnsTheOriginalAttemptWithoutRenewingAnyDeadline(): void { [, $task] = $this->newClaim(); diff --git a/tests/Fixtures/V2/ReplayRegression/prepared-v1-policy-refusal-legacy-zero-tail.json b/tests/Fixtures/V2/ReplayRegression/prepared-v1-policy-refusal-legacy-zero-tail.json new file mode 100644 index 00000000..2de7b7de --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/prepared-v1-policy-refusal-legacy-zero-tail.json @@ -0,0 +1,211 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "prepared-v1-policy-refusal-legacy-zero-tail", + "protocol_version": "1.20", + "bindings": [ + "php" + ], + "consumers": [ + "workflow-fiber-runner", + "workflow-executor" + ], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestLocalActivityWorkflow", + "arguments": [ + "Ada" + ], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "recorded_at": "2026-11-05T00:02:04.000001Z", + "payload": {} + }, + { + "sequence": 2, + "event_type": "ActivityScheduled", + "recorded_at": "2026-11-05T00:02:04.000002Z", + "payload": { + "sequence": 1, + "activity_execution_id": "01j00000000000000000000001", + "activity_type": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "execution_mode": "local", + "local_activity": true, + "activity": { + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 3, + "backoff_seconds": [ + 120 + ], + "start_to_close_timeout": null, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null, + "non_retryable_error_types": [] + } + } + } + }, + { + "sequence": 3, + "event_type": "ActivityStarted", + "recorded_at": "2026-11-05T00:02:04.000003Z", + "payload": { + "sequence": 1, + "activity_execution_id": "01j00000000000000000000001", + "activity_type": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "execution_mode": "local", + "local_activity": true, + "activity_attempt_id": "01j0000000000000000000000a", + "attempt_number": 1, + "worker_attempt_id": "legacy-local-1" + } + }, + { + "sequence": 4, + "event_type": "ActivityRetryScheduled", + "recorded_at": "2026-11-05T00:02:04.000004Z", + "payload": { + "sequence": 1, + "activity_execution_id": "01j00000000000000000000001", + "activity_type": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "execution_mode": "local", + "local_activity": true, + "activity_attempt_id": "01j0000000000000000000000a", + "worker_attempt_id": "legacy-local-1", + "retry_reason": "failure", + "retry_backoff_seconds": 120, + "activity": { + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 3, + "backoff_seconds": [ + 120 + ], + "start_to_close_timeout": null, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null, + "non_retryable_error_types": [] + } + }, + "retry_after_attempt_id": "01j0000000000000000000000a", + "retry_after_attempt": 1, + "message": "Legacy SDK local attempt failed." + } + }, + { + "sequence": 5, + "event_type": "ActivityStarted", + "recorded_at": "2026-11-05T00:02:04.000005Z", + "payload": { + "sequence": 1, + "activity_execution_id": "01j00000000000000000000001", + "activity_type": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "execution_mode": "local", + "local_activity": true, + "activity_attempt_id": "01j0000000000000000000000b", + "attempt_number": 2, + "worker_attempt_id": "legacy-local-2" + } + }, + { + "sequence": 6, + "event_type": "ActivityRetryScheduled", + "recorded_at": "2026-11-05T00:02:04.000006Z", + "payload": { + "sequence": 1, + "activity_execution_id": "01j00000000000000000000001", + "activity_type": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "execution_mode": "local", + "local_activity": true, + "activity_attempt_id": "01j0000000000000000000000b", + "worker_attempt_id": "legacy-local-2", + "retry_reason": "failure", + "retry_backoff_seconds": 0, + "activity": { + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 3, + "backoff_seconds": [ + 120 + ], + "start_to_close_timeout": null, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null, + "non_retryable_error_types": [] + } + }, + "retry_after_attempt_id": "01j0000000000000000000000b", + "retry_after_attempt": 2, + "message": "Legacy SDK local attempt failed." + } + }, + { + "sequence": 7, + "event_type": "ActivityStarted", + "recorded_at": "2026-11-05T00:02:04.000007Z", + "payload": { + "sequence": 1, + "activity_execution_id": "01j00000000000000000000001", + "activity_type": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "execution_mode": "local", + "local_activity": true, + "activity_attempt_id": "01j0000000000000000000000c", + "attempt_number": 3, + "worker_attempt_id": "legacy-local-3" + } + }, + { + "sequence": 8, + "event_type": "ActivityCompleted", + "recorded_at": "2026-11-05T00:02:04.000008Z", + "payload": { + "sequence": 1, + "activity_execution_id": "01j00000000000000000000001", + "activity_type": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "activity_class": "Tests\\Fixtures\\V2\\TestGreetingActivity", + "execution_mode": "local", + "local_activity": true, + "activity_attempt_id": "01j0000000000000000000000c", + "attempt_number": 3, + "worker_attempt_id": "legacy-local-3", + "result": "wwHioz3/VYAiNwoWSGVsbG8sIEFkYSE=", + "payload_codec": "avro" + } + } + ], + "expected": { + "completed": true, + "result": { + "greeting": "Hello, Ada!", + "workflow_id": "regression-corpus-prepared-v1-policy-refusal-legacy-zero-tail", + "run_id": "regression-corpus-run-prepared-v1-policy-refusal-legacy-zero-tail" + }, + "commands": [ + { + "type": "complete_workflow" + } + ] + }, + "unqualified_preparation_policy": { + "contract_version": 2, + "mode": "explicit", + "max_attempts": 3, + "backoff_seconds": [ + 120 + ], + "maximum_interval_seconds": 120 + } +} diff --git a/tests/Unit/V2/PreparedLocalRetryTimestampTest.php b/tests/Unit/V2/PreparedLocalRetryTimestampTest.php new file mode 100644 index 00000000..99e7b960 --- /dev/null +++ b/tests/Unit/V2/PreparedLocalRetryTimestampTest.php @@ -0,0 +1,147 @@ +format('Y-m-d\TH:i:s.u\Z') + ); + self::assertSame( + '9999-12-31T23:59:59.999999Z', + PreparedLocalRetryTimestamp::latestForDriver($driver)->format('Y-m-d\TH:i:s.u\Z') + ); + } + self::assertSame( + '2038-01-19T03:14:07.499999Z', + PreparedLocalRetryTimestamp::latestForDriver('mysql')->format('Y-m-d\TH:i:s.u\Z') + ); + foreach (['10.11.14', '11.4.0-MariaDB', '5.5.5-11.4.8-MariaDB'] as $version) { + self::assertSame( + '2038-01-19T03:14:07.999999Z', + PreparedLocalRetryTimestamp::latestForDriver('mariadb', $version)->format('Y-m-d\TH:i:s.u\Z') + ); + } + foreach (['11.5.1', '11.8.2-MariaDB', '12.0.0'] as $version) { + self::assertSame( + '2106-02-07T06:28:15.999999Z', + PreparedLocalRetryTimestamp::latestForDriver('mariadb', $version)->format('Y-m-d\TH:i:s.u\Z') + ); + } + } + + public function testMariaDbThroughTheMysqlConnectionUsesItsActualVendorVersion(): void + { + $connection = $this->createMock(MySqlConnection::class); + $connection->method('getDriverName') + ->willReturn('mysql'); + $connection->method('isMaria') + ->willReturn(true); + $connection->method('getServerVersion') + ->willReturn('11.8.2'); + $connection->expects(self::never())->method('getPdo'); + self::assertSame('2106', PreparedLocalRetryTimestamp::latestForConnection($connection)->format('Y')); + + $mysql = $this->createMock(MySqlConnection::class); + $mysql->method('getDriverName') + ->willReturn('mysql'); + $mysql->method('isMaria') + ->willReturn(false); + $mysql->expects(self::never())->method('getServerVersion'); + self::assertSame('2038', PreparedLocalRetryTimestamp::latestForConnection($mysql)->format('Y')); + } + + public function testAdditionPreservesTheSampleAndChecksBeforeAnyLargeIntervalConstruction(): void + { + $sample = new DateTimeImmutable('2026-11-05T01:00:00.123456+01:00'); + $latest = new DateTimeImmutable('9999-12-31T23:59:59.999999Z'); + self::assertSame( + '2026-11-05T00:00:02.123456Z', + PreparedLocalRetryTimestamp::add($sample, 2, $latest)->format('Y-m-d\TH:i:s.u\Z') + ); + self::assertSame('2026-11-05T01:00:00.123456+01:00', $sample->format('Y-m-d\TH:i:s.uP')); + $this->expectException(InvalidArgumentException::class); + PreparedLocalRetryTimestamp::add($sample, 9007199254740991, $latest); + } + + public function testUpperBoundaryRetainsMicrosecondsAndRefusesTheOverflowingSuccessor(): void + { + $latest = new DateTimeImmutable('2038-01-19T03:14:07.499999Z'); + self::assertEquals( + $latest, + PreparedLocalRetryTimestamp::add(new DateTimeImmutable('2038-01-19T03:14:06.499999Z'), 1, $latest) + ); + self::assertEquals($latest, PreparedLocalRetryTimestamp::add($latest, 0, $latest)); + $this->expectException(InvalidArgumentException::class); + PreparedLocalRetryTimestamp::add(new DateTimeImmutable('2038-01-19T03:14:06.500000Z'), 1, $latest); + } + + public function testConnectionWrapperAlsoEnforcesTheTimestampLowerBound(): void + { + $connection = $this->createMock(Connection::class); + $connection->method('getDriverName') + ->willReturn('mysql'); + $connection->expects(self::never())->method('getPdo'); + $earliest = new DateTimeImmutable('1970-01-01T00:00:01.000000Z'); + self::assertEquals($earliest, PreparedLocalRetryTimestamp::addForConnection($earliest, 0, $connection)); + $this->expectException(InvalidArgumentException::class); + PreparedLocalRetryTimestamp::addForConnection( + new DateTimeImmutable('1970-01-01T00:00:00.999999Z'), + 1, + $connection + ); + } + + public function testUnknownBackendAndUnidentifiedMariaDbVersionAreRefused(): void + { + foreach ([['unknown', null], + ['mariadb', null], + ['mariadb', ''], + ['mariadb', 'unidentified'], + ] as [$driver, $version]) { + try { + PreparedLocalRetryTimestamp::latestForDriver($driver, $version); + self::fail('Invented a timestamp bound for an unidentified backend.'); + } catch (InvalidArgumentException) { + self::addToAssertionCount(1); + } + } + $this->expectException(InvalidArgumentException::class); + PreparedLocalRetryTimestamp::earliestForDriver('unknown'); + } + + public function testInvalidDurationAndOutOfDomainInstantsAreRefused(): void + { + foreach ([ + ['2026-11-05T00:00:00Z', -1, '9999-12-31T23:59:59.999999Z'], + ['2026-11-05T00:00:00Z', 9007199254740992, '9999-12-31T23:59:59.999999Z'], + ['2026-11-06T00:00:00Z', 0, '2026-11-05T00:00:00Z'], + ['0000-01-01T00:00:00Z', 0, '9999-12-31T23:59:59.999999Z'], + ['2026-11-05T00:00:00Z', 0, '0000-01-01T00:00:00Z'], + ] as [$sample, $seconds, $latest]) { + try { + PreparedLocalRetryTimestamp::add( + new DateTimeImmutable($sample), + $seconds, + new DateTimeImmutable($latest) + ); + self::fail('Invented a representable timestamp.'); + } catch (InvalidArgumentException) { + self::addToAssertionCount(1); + } + } + } +} diff --git a/tests/Unit/V2/ReplayRegressionCorpusTest.php b/tests/Unit/V2/ReplayRegressionCorpusTest.php index 27b15eca..8f7d514f 100644 --- a/tests/Unit/V2/ReplayRegressionCorpusTest.php +++ b/tests/Unit/V2/ReplayRegressionCorpusTest.php @@ -7,6 +7,11 @@ use Illuminate\Config\Repository; use Illuminate\Container\Container; use Illuminate\Foundation\Application; +use Illuminate\Support\Facades\Facade; +use Illuminate\Translation\ArrayLoader; +use Illuminate\Translation\Translator; +use Illuminate\Validation\Factory as ValidationFactory; +use Illuminate\Validation\ValidationException; use OpenTelemetry\API\Instrumentation\Configurator; use OpenTelemetry\Context\Context; use OpenTelemetry\SDK\Trace\SpanExporter\InMemoryExporter; @@ -26,6 +31,8 @@ use Workflow\V2\Support\CooperativeCancellationDelivery; use Workflow\V2\Support\ExecutionTracing; use Workflow\V2\Support\HistoryEventPayloadContract; +use Workflow\V2\Support\PortableLocalActivityPreparation; +use Workflow\V2\Support\PreparedLocalRetryPolicy; use Workflow\V2\Support\ServiceCallCancellationOrigin; use Workflow\V2\Support\WorkflowFiberRunner; use Workflow\V2\Support\WorkflowStep; @@ -149,7 +156,14 @@ public function testFixtureExecutesThroughColdReplayRunner(array $fixture): void } $profiledWorkflow = null; - if ($fixture['id'] === 'replay-unrelated-payload-read-budget') { + if (in_array( + $fixture['id'], + ['replay-unrelated-payload-read-budget', 'prepared-v1-policy-refusal-legacy-zero-tail'], + true + )) { + if ($fixture['id'] === 'prepared-v1-policy-refusal-legacy-zero-tail') { + $this->assertPreparedV1RefusesTheVersionedPolicy($fixture); + } Container::getInstance()->resolving( $workflowClass, static function (Workflow $instance) use (&$profiledWorkflow): void { @@ -188,6 +202,20 @@ static function (WorkflowHistoryEvent $event): ReplayBudgetHistoryEvent { $step = $runner->step(); + if ($fixture['id'] === 'prepared-v1-policy-refusal-legacy-zero-tail') { + $this->assertInstanceOf(Workflow::class, $profiledWorkflow); + $retries = $profiledWorkflow->run->historyEvents + ->where('event_type', HistoryEventType::ActivityRetryScheduled); + $this->assertSame([120, 0], $retries->map( + static fn (WorkflowHistoryEvent $event): int => $event->payload['retry_backoff_seconds'] + )->values() + ->all()); + $scheduled = $profiledWorkflow->run->historyEvents + ->firstWhere('event_type', HistoryEventType::ActivityScheduled); + $this->assertSame(1, ActivitySnapshot::fromEvent($scheduled)['retry_policy']['snapshot_version']); + $this->assertSame([120], ActivitySnapshot::fromEvent($scheduled)['retry_policy']['backoff_seconds']); + } + if ($fixture['id'] === 'replay-unrelated-payload-read-budget') { $this->assertSame( 0, @@ -381,6 +409,33 @@ public static function replayRegressionFixtures(): array return $fixtures; } + private function assertPreparedV1RefusesTheVersionedPolicy(array $fixture): void + { + $policy = $fixture['unqualified_preparation_policy']; + $this->assertSame(2, PreparedLocalRetryPolicy::normalize($policy)['snapshot_version']); + $container = new Container(); + $translator = new Translator(new ArrayLoader(), 'en'); + $container->instance('validator', new ValidationFactory($translator, $container)); + $prior = Facade::getFacadeApplication(); + Facade::clearResolvedInstance('validator'); + Facade::setFacadeApplication($container); + try { + PortableLocalActivityPreparation::normalizeDescriptor([ + 'type' => 'record_local_activity', + 'activity_type' => $fixture['history'][1]['payload']['activity_type'], + 'arguments' => Serializer::serializeWithCodec('avro', $fixture['workflow']['arguments']), + 'payload_codec' => 'avro', + 'retry_policy' => $policy, + ]); + $this->fail('Prepared v1 discarded the versioned policy instead of refusing it.'); + } catch (ValidationException $error) { + $this->assertArrayHasKey('local_activity.retry_policy', $error->errors()); + } finally { + Facade::clearResolvedInstance('validator'); + Facade::setFacadeApplication($prior); + } + } + private function assertWorkflowFiberObservabilityContext(array $fixture): void { $exporter = new InMemoryExporter(); From 96dfda45b93294c04e74c7b9369a1aad2ef458fe Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 6 Nov 2026 00:14:02 +0000 Subject: [PATCH 04/15] Exercise legacy zero-tail fixture through the embedded executor --- .../V2EmbeddedReplayRegressionCorpusTest.php | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php index 60674e3f..aa200827 100644 --- a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php +++ b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php @@ -1570,6 +1570,27 @@ public function testFixturesExecuteThroughDeclaredReplayConsumers(): void continue; } + if ($fixture['id'] === 'prepared-v1-policy-refusal-legacy-zero-tail') { + $run = $this->createRunFromFixture($fixture); + $before = $run->historyEvents() + ->count(); + $this->runReadyWorkflowTask($run); + $this->assertSame(RunStatus::Completed, $run->refresh()->status); + $this->assertSame($fixture['expected_step']['result'], Serializer::unserializeWithCodec( + $run->outputPayloadCodec(), + $run->output + )); + $this->assertSame([120, 0], $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityRetryScheduled) + ->orderBy('sequence') + ->get() + ->map(static fn (WorkflowHistoryEvent $event): int => $event->payload['retry_backoff_seconds']) + ->all()); + $this->assertSame($before + 1, $run->historyEvents()->count()); + $this->assertSame(0, $run->activityExecutions()->count()); + continue; + } + if ($fixture['id'] === 'nexus-sdk-caller-attribution') { $this->clearWorkflowState(); $this->assertNexusCallerSdkAttribution($fixture); From 564f792032837ea16098a329b4416d03111ab742 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 6 Nov 2026 00:18:05 +0000 Subject: [PATCH 05/15] Record guarded prepared retry decisions under the original authority --- .../Support/HistoryEventPayloadContract.php | 3 + src/V2/Support/LocalActivityExecutor.php | 550 ++++++++++++------ .../Support/PortableLocalActivityCleanup.php | 9 +- .../V2EmbeddedReplayRegressionCorpusTest.php | 6 +- ...V2PortableLocalActivityPreparationTest.php | 238 ++++++++ 5 files changed, 617 insertions(+), 189 deletions(-) diff --git a/src/V2/Support/HistoryEventPayloadContract.php b/src/V2/Support/HistoryEventPayloadContract.php index 563b1857..8861d58c 100644 --- a/src/V2/Support/HistoryEventPayloadContract.php +++ b/src/V2/Support/HistoryEventPayloadContract.php @@ -200,6 +200,7 @@ final class HistoryEventPayloadContract ], 'ActivityRetryScheduled' => [ 'local_outcome', + 'retry_disposition', 'local_recovery', 'activity_execution_id', 'activity_attempt_id', @@ -266,6 +267,7 @@ final class HistoryEventPayloadContract ], 'ActivityFailed' => [ 'local_outcome', + 'retry_disposition', 'local_recovery', 'activity_execution_id', 'activity_attempt_id', @@ -345,6 +347,7 @@ final class HistoryEventPayloadContract ], 'ActivityTimedOut' => [ 'local_outcome', + 'retry_disposition', 'local_recovery', 'activity_execution_id', 'activity_attempt_id', diff --git a/src/V2/Support/LocalActivityExecutor.php b/src/V2/Support/LocalActivityExecutor.php index 6f6ddf54..61558427 100644 --- a/src/V2/Support/LocalActivityExecutor.php +++ b/src/V2/Support/LocalActivityExecutor.php @@ -46,6 +46,193 @@ public function recordPortableOutcome( int $workflowTaskAttempt, array $report, string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return $this->recordPortableOutcomeForContract( + 1, + $attemptId, + $leaseOwner, + $workflowTaskAttempt, + $report, + $protocolVersion + ); + } + + /** @internal Unadvertised until the installed bridge and retained claim bind contract 2. + * @param array $report + * @return array + */ + public function recordPortableRetryOutcome( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + array $report, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return $this->recordPortableOutcomeForContract( + 2, + $attemptId, + $leaseOwner, + $workflowTaskAttempt, + $report, + $protocolVersion + ); + } + + /** + * @return array{status: 'completed'|'failed'|'waiting', event: WorkflowHistoryEvent|null, next_task: WorkflowTask|null} + */ + public function execute( + WorkflowRun $run, + WorkflowTask $task, + int $sequence, + LocalActivityCall $activityCall, + ): array { + $started = $this->prepareAttempt($run, $task, $sequence, $activityCall); + + if ($started['event'] instanceof WorkflowHistoryEvent) { + return [ + 'status' => $started['event']->event_type === HistoryEventType::ActivityCompleted + ? 'completed' + : 'failed', + 'event' => $started['event'], + 'next_task' => null, + ]; + } + + if ($started['next_task'] instanceof WorkflowTask) { + return [ + 'status' => 'waiting', + 'event' => null, + 'next_task' => $started['next_task'], + ]; + } + + /** @var ActivityExecution $execution */ + $execution = $started['execution']; + /** @var ActivityAttempt $attempt */ + $attempt = $started['attempt']; + + $operation = fn (): array => $this->invokeCallback($run, $task, $execution); + if (ExecutionTracing::enabled() && ! WorkflowStub::hasMock($execution->activity_class)) { + $storedContext = TraceCarrier::fromRun($run, $diagnostic); + [$result, $throwable] = ExecutionTracing::localActivity( + [ + 'workflow_instance_id' => $run->workflow_instance_id, + 'workflow_run_id' => $run->id, + 'workflow_type' => $run->workflow_type, + 'namespace' => $run->namespace, + 'task_id' => $task->id, + 'activity_id' => $execution->id, + 'activity_type' => $execution->activity_type, + 'activity_attempt_id' => $attempt->id, + 'activity_attempt_number' => $attempt->attempt_number, + 'durable.trace_context.disposition' => $diagnostic, + ], + $storedContext, + static function () use ($operation): array { + [$result, $throwable] = $operation(); + if ($throwable !== null) { + ExecutionTracing::recordFailure($throwable); + } + + return [$result, $throwable]; + }, + ExecutionTracing::baggageAllowlist(), + true, + ); + } else { + [$result, $throwable] = $operation(); + } + + return $this->recordAttemptOutcome($task, $execution, $attempt, $result, $throwable); + } + + /** + * @internal Portable preparation validates and locks the execution, run and + * hosting claim before calling this shared attempt recorder. + * @param array $preparation + */ + public function startPortableAttempt( + WorkflowRun $run, + WorkflowTask $task, + ActivityExecution $execution, + string $workerAttemptId, + array $preparation, + ?Carbon $preparedAt = null, + ): ActivityAttempt { + $preparation['schedule_to_close_deadline_at'] = $execution->schedule_to_close_deadline_at?->toISOString(); + return $this->startAttempt($run, $task, $execution, $workerAttemptId, $preparation, $preparedAt); + } + + /** + * @internal The caller holds canonical locks and has validated the retry + * authority and the expired original total deadline. No callback is admitted. + */ + public function expirePortableRetry( + WorkflowRun $run, + WorkflowTask $task, + ActivityExecution $execution, + ): WorkflowHistoryEvent { + $outcome = $this->recordTimeoutOutcome($run, $task, $execution, 'schedule_to_close'); + + return $outcome['event']; + } + + /** + * @internal The caller holds canonical locks, has validated original start + * history and has proved loss of the previous claim. Lease expiry fences + * publication. It does not establish physical callback stop. + * @param array $recovery + * @return array{event: WorkflowHistoryEvent, next_task: WorkflowTask|null} + */ + public function recoverPortableAttempt( + WorkflowRun $run, + WorkflowTask $task, + ActivityExecution $execution, + ActivityAttempt $attempt, + array $recovery, + ): array { + $metadata = [ + 'worker_attempt_id' => $attempt->worker_attempt_id, + 'local_recovery' => $recovery, + ]; + self::closeAttempt($attempt, ActivityAttemptStatus::Expired); + $timeoutKind = self::timeoutKind($execution); + $outcome = $timeoutKind === null + ? $this->recordInterruptedAttempt($run, $task, $execution, $metadata) + : $this->recordTimeoutOutcome($run, $task, $execution, $timeoutKind, $metadata); + if ($outcome['next_task'] instanceof WorkflowTask) { + $task->forceFill([ + 'status' => TaskStatus::Completed, + 'lease_expires_at' => null, + ])->save(); + self::projectRun($run); + } + $event = $outcome['event'] ?? $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityRetryScheduled) + ->where('payload->activity_attempt_id', $attempt->id) + ->orderByDesc('sequence') + ->first(); + if (! $event instanceof WorkflowHistoryEvent) { + throw new RuntimeException('Portable local recovery did not record its canonical receipt.'); + } + + return [ + 'event' => $event, + 'next_task' => $outcome['next_task'], + ]; + } + + /** @param array $report + * @return array + */ + private function recordPortableOutcomeForContract( + int $localContractVersion, + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + array $report, + string $protocolVersion, ): array { $refused = static fn (string $reason): array => [ 'recorded' => false, @@ -59,11 +246,21 @@ public function recordPortableOutcome( || version_compare($protocolVersion, PortableLocalActivityPreparation::MINIMUM_PROTOCOL_VERSION, '<')) { return $refused('local_activity_outcome_requires_protocol_1_20'); } - try { - $normalized = self::normalizePortableReport($report); - $fingerprint = hash('sha256', json_encode($normalized, JSON_THROW_ON_ERROR)); - } catch (ValidationException|JsonException) { - return $refused('invalid_local_activity_outcome'); + $normalized = null; + $fingerprint = null; + if ($localContractVersion === 2 && array_key_exists('next_retry_delay_seconds', $report)) { + $reason = PreparedLocalActivityReport::refusal($report); + if ($reason !== null) { + return $refused($reason); + } + } + if ($localContractVersion === 1) { + try { + $normalized = self::normalizePortableReport($report); + $fingerprint = hash('sha256', json_encode($normalized, JSON_THROW_ON_ERROR)); + } catch (ValidationException|JsonException) { + return $refused('invalid_local_activity_outcome'); + } } return ConfiguredV2Models::connection()->transaction(function () use ( @@ -73,6 +270,8 @@ public function recordPortableOutcome( $normalized, $fingerprint, $refused, + $report, + $localContractVersion, ): array { $rows = ActivityRowLockOrder::lockForAttempt($attemptId); $attempt = $rows['attempt']; @@ -97,10 +296,23 @@ public function recordPortableOutcome( $attempt, $leaseOwner, $workflowTaskAttempt, + $localContractVersion, ); if ($started === null) { return $refused('local_activity_preparation_mismatch'); } + if ($localContractVersion === 2) { + $reason = PreparedLocalActivityReport::refusal($report); + if ($reason !== null) { + return $refused($reason); + } + try { + $normalized = self::normalizePortableReport($report, 2); + $fingerprint = hash('sha256', json_encode($normalized, JSON_THROW_ON_ERROR)); + } catch (ValidationException|JsonException) { + return $refused('invalid_local_activity_outcome'); + } + } $receipt = $run->historyEvents() ->whereIn('event_type', [HistoryEventType::ActivityCompleted, HistoryEventType::ActivityFailed, HistoryEventType::ActivityTimedOut, HistoryEventType::ActivityRetryScheduled]) @@ -111,7 +323,7 @@ public function recordPortableOutcome( if ($receipt instanceof WorkflowHistoryEvent) { if ($receipt->sequence <= $started->sequence || $receipt->workflow_task_id !== $task->id || ($receipt->payload['activity_execution_id'] ?? null) !== $execution->id - || ($receipt->payload['local_outcome']['version'] ?? null) !== 1 + || ($receipt->payload['local_outcome']['version'] ?? null) !== $localContractVersion || ($receipt->payload['local_outcome']['report_fingerprint'] ?? null) !== $fingerprint || ($receipt->payload['local_outcome']['workflow_task_attempt'] ?? null) !== $workflowTaskAttempt || ($receipt->payload['task']['lease_owner'] ?? null) !== $leaseOwner @@ -120,6 +332,7 @@ public function recordPortableOutcome( } return self::portableReceipt($receipt, true); } + $failureTime = $localContractVersion === 2 ? now() : null; if ($execution->current_attempt_id !== $attemptId || $execution->attempt_count !== $attempt->attempt_number) { return $refused('stale_activity_attempt'); } @@ -132,7 +345,7 @@ public function recordPortableOutcome( $execution->activity_options['cancellation_cleanup'], ); if ($run->cancellation_request_command_id !== null - && (! $runCleanup || now()->gte($run->cancellation_deadline_at))) { + && (! $runCleanup || ($failureTime ?? now())->gte($run->cancellation_deadline_at))) { $cancelled = $run->historyEvents() ->where('event_type', HistoryEventType::ActivityCancelled) ->where('payload->activity_execution_id', $execution->id) @@ -167,45 +380,65 @@ public function recordPortableOutcome( if ($run->status->isTerminal()) { return $refused('run_closed'); } - if (($run->execution_deadline_at !== null && now()->gte($run->execution_deadline_at)) - || ($run->run_deadline_at !== null && now()->gte($run->run_deadline_at))) { + if (($run->execution_deadline_at !== null && ($failureTime ?? now())->gte($run->execution_deadline_at)) + || ($run->run_deadline_at !== null && ($failureTime ?? now())->gte($run->run_deadline_at))) { return $refused('run_deadline_expired'); } - if (PortableLocalActivityCleanup::currentAuthorityExpired($run, $execution)) { + if (PortableLocalActivityCleanup::currentAuthorityExpired($run, $execution, $failureTime)) { return $refused('local_activity_cleanup_authority_expired'); } } - if ($task->lease_expires_at === null || now()->gte($task->lease_expires_at) - || $attempt->lease_expires_at === null || now() + if ($task->lease_expires_at === null || ($failureTime ?? now())->gte($task->lease_expires_at) + || $attempt->lease_expires_at === null || ($failureTime ?? now()) ->gte($attempt->lease_expires_at)) { return $refused('workflow_claim_expired'); } if ($run->status->isTerminal()) { return $refused('run_closed'); } - if (($run->execution_deadline_at !== null && now()->gte($run->execution_deadline_at)) - || ($run->run_deadline_at !== null && now()->gte($run->run_deadline_at))) { + if (($run->execution_deadline_at !== null && ($failureTime ?? now())->gte($run->execution_deadline_at)) + || ($run->run_deadline_at !== null && ($failureTime ?? now())->gte($run->run_deadline_at))) { return $refused('run_deadline_expired'); } $metadata = [ 'worker_attempt_id' => $attempt->worker_attempt_id, 'local_outcome' => [ - 'version' => 1, + 'version' => $localContractVersion, 'report_fingerprint' => $fingerprint, 'workflow_task_attempt' => $workflowTaskAttempt, 'submitted_outcome' => $normalized['outcome'], ], ]; - $timeoutKind = self::timeoutKind($execution); + $timeoutKind = self::timeoutKind($execution, $failureTime); + if ($failureTime !== null && ($timeoutKind !== null || $normalized['outcome'] === 'failed')) { + $decision = self::selectPortableRetryDecision( + $execution, + $attempt, + $report, + $failureTime, + $timeoutKind + ); + $metadata['retry_disposition'] = $decision; + } else { + $decision = null; + } if ($timeoutKind !== null) { - $outcome = $this->recordTimeoutOutcome($run, $task, $execution, $timeoutKind, $metadata); + $outcome = $this->recordTimeoutOutcome( + $run, + $task, + $execution, + $timeoutKind, + $metadata, + $decision, + $failureTime + ); } elseif ($normalized['outcome'] === 'timed_out') { return $refused('local_activity_timeout_not_due'); } elseif ($normalized['outcome'] === 'completed') { $outcome = $this->recordSuccess($run, $task, $execution, $attempt, null, [ 'blob' => $normalized['result'], 'codec' => $normalized['payload_codec'], - ], $metadata); + ], $metadata, $failureTime); } else { $failure = new RestoredWorkflowException([ 'class' => $normalized['exception_type'], @@ -214,7 +447,16 @@ public function recordPortableOutcome( 'non_retryable' => $normalized['non_retryable'], 'code' => 0, ]); - $outcome = $this->recordFailureOrRetry($run, $task, $execution, $attempt, $failure, $metadata); + $outcome = $this->recordFailureOrRetry( + $run, + $task, + $execution, + $attempt, + $failure, + $metadata, + $decision, + $failureTime + ); } if ($outcome['next_task'] instanceof WorkflowTask) { $task->forceFill([ @@ -236,151 +478,6 @@ public function recordPortableOutcome( }, 5); } - /** - * @return array{status: 'completed'|'failed'|'waiting', event: WorkflowHistoryEvent|null, next_task: WorkflowTask|null} - */ - public function execute( - WorkflowRun $run, - WorkflowTask $task, - int $sequence, - LocalActivityCall $activityCall, - ): array { - $started = $this->prepareAttempt($run, $task, $sequence, $activityCall); - - if ($started['event'] instanceof WorkflowHistoryEvent) { - return [ - 'status' => $started['event']->event_type === HistoryEventType::ActivityCompleted - ? 'completed' - : 'failed', - 'event' => $started['event'], - 'next_task' => null, - ]; - } - - if ($started['next_task'] instanceof WorkflowTask) { - return [ - 'status' => 'waiting', - 'event' => null, - 'next_task' => $started['next_task'], - ]; - } - - /** @var ActivityExecution $execution */ - $execution = $started['execution']; - /** @var ActivityAttempt $attempt */ - $attempt = $started['attempt']; - - $operation = fn (): array => $this->invokeCallback($run, $task, $execution); - if (ExecutionTracing::enabled() && ! WorkflowStub::hasMock($execution->activity_class)) { - $storedContext = TraceCarrier::fromRun($run, $diagnostic); - [$result, $throwable] = ExecutionTracing::localActivity( - [ - 'workflow_instance_id' => $run->workflow_instance_id, - 'workflow_run_id' => $run->id, - 'workflow_type' => $run->workflow_type, - 'namespace' => $run->namespace, - 'task_id' => $task->id, - 'activity_id' => $execution->id, - 'activity_type' => $execution->activity_type, - 'activity_attempt_id' => $attempt->id, - 'activity_attempt_number' => $attempt->attempt_number, - 'durable.trace_context.disposition' => $diagnostic, - ], - $storedContext, - static function () use ($operation): array { - [$result, $throwable] = $operation(); - if ($throwable !== null) { - ExecutionTracing::recordFailure($throwable); - } - - return [$result, $throwable]; - }, - ExecutionTracing::baggageAllowlist(), - true, - ); - } else { - [$result, $throwable] = $operation(); - } - - return $this->recordAttemptOutcome($task, $execution, $attempt, $result, $throwable); - } - - /** - * @internal Portable preparation validates and locks the execution, run and - * hosting claim before calling this shared attempt recorder. - * @param array $preparation - */ - public function startPortableAttempt( - WorkflowRun $run, - WorkflowTask $task, - ActivityExecution $execution, - string $workerAttemptId, - array $preparation, - ?Carbon $preparedAt = null, - ): ActivityAttempt { - $preparation['schedule_to_close_deadline_at'] = $execution->schedule_to_close_deadline_at?->toISOString(); - return $this->startAttempt($run, $task, $execution, $workerAttemptId, $preparation, $preparedAt); - } - - /** - * @internal The caller holds canonical locks and has validated the retry - * authority and the expired original total deadline. No callback is admitted. - */ - public function expirePortableRetry( - WorkflowRun $run, - WorkflowTask $task, - ActivityExecution $execution, - ): WorkflowHistoryEvent { - $outcome = $this->recordTimeoutOutcome($run, $task, $execution, 'schedule_to_close'); - - return $outcome['event']; - } - - /** - * @internal The caller holds canonical locks, has validated original start - * history and has proved loss of the previous claim. Lease expiry fences - * publication. It does not establish physical callback stop. - * @param array $recovery - * @return array{event: WorkflowHistoryEvent, next_task: WorkflowTask|null} - */ - public function recoverPortableAttempt( - WorkflowRun $run, - WorkflowTask $task, - ActivityExecution $execution, - ActivityAttempt $attempt, - array $recovery, - ): array { - $metadata = [ - 'worker_attempt_id' => $attempt->worker_attempt_id, - 'local_recovery' => $recovery, - ]; - self::closeAttempt($attempt, ActivityAttemptStatus::Expired); - $timeoutKind = self::timeoutKind($execution); - $outcome = $timeoutKind === null - ? $this->recordInterruptedAttempt($run, $task, $execution, $metadata) - : $this->recordTimeoutOutcome($run, $task, $execution, $timeoutKind, $metadata); - if ($outcome['next_task'] instanceof WorkflowTask) { - $task->forceFill([ - 'status' => TaskStatus::Completed, - 'lease_expires_at' => null, - ])->save(); - self::projectRun($run); - } - $event = $outcome['event'] ?? $run->historyEvents() - ->where('event_type', HistoryEventType::ActivityRetryScheduled) - ->where('payload->activity_attempt_id', $attempt->id) - ->orderByDesc('sequence') - ->first(); - if (! $event instanceof WorkflowHistoryEvent) { - throw new RuntimeException('Portable local recovery did not record its canonical receipt.'); - } - - return [ - 'event' => $event, - 'next_task' => $outcome['next_task'], - ]; - } - /** * @return array{mixed, Throwable|null} */ @@ -432,12 +529,13 @@ private function invokeCallback(WorkflowRun $run, WorkflowTask $task, ActivityEx /** @param array $report * @return array */ - private static function normalizePortableReport(array $report): array + private static function normalizePortableReport(array $report, int $localContractVersion = 1): array { $outcome = $report['outcome'] ?? null; $fields = match ($outcome) { 'completed' => ['outcome', 'result', 'payload_codec'], - 'failed' => ['outcome', 'message', 'exception_type', 'non_retryable'], + 'failed' => ['outcome', 'message', 'exception_type', 'non_retryable', + ...($localContractVersion === 2 ? ['next_retry_delay_seconds'] : [])], 'timed_out' => ['outcome'], default => [], }; @@ -475,6 +573,10 @@ private static function normalizePortableReport(array $report): array 'message' => $message, 'exception_type' => $type, 'non_retryable' => $report['non_retryable'] ?? false, + ...(array_key_exists('next_retry_delay_seconds', $report) + ? [ + 'next_retry_delay_seconds' => $report['next_retry_delay_seconds'], + ] : []), ]; } @@ -506,9 +608,38 @@ private static function portableReceipt(WorkflowHistoryEvent $event, bool $dupli 'recorded_at' => $event->recorded_at->toISOString(), 'claim_released' => $event->event_type === HistoryEventType::ActivityRetryScheduled, 'created_task_ids' => is_string($retryTaskId) ? [$retryTaskId] : [], + ...(isset($payload['retry_disposition']) ? [ + 'retry_disposition' => $payload['retry_disposition'], + ] : []), ]; } + /** @param array $report + * @return array + */ + private static function selectPortableRetryDecision( + ActivityExecution $execution, + ActivityAttempt $attempt, + array $report, + Carbon $failureTime, + ?string $timeoutKind = null, + bool $nonRetryable = false, + ): array { + $connection = ConfiguredV2Models::connection(); + PreparedLocalRetryTimestamp::addForConnection($failureTime, 0, $connection); + + return PreparedLocalRetryDecision::select( + $execution->retry_policy, + $attempt->attempt_number, + $report, + $failureTime, + $execution->schedule_to_close_deadline_at, + PreparedLocalRetryTimestamp::latestForConnection($connection), + $timeoutKind, + $nonRetryable, + ); + } + /** * @return array{execution: ActivityExecution|null, attempt: ActivityAttempt|null, event: WorkflowHistoryEvent|null, next_task: WorkflowTask|null} */ @@ -846,6 +977,7 @@ private function recordSuccess( mixed $result, ?array $encodedResult = null, array $historyMetadata = [], + ?Carbon $failureTime = null, ): array { $encoded = $encodedResult ?? self::serializeWithCodec($result, self::preferredPayloadCodec($execution, $run)); $encoded['blob'] = ExternalPayloads::externalizeForNamespace( @@ -857,8 +989,30 @@ private function recordSuccess( try { StructuralLimits::guardPayloadSize($encoded['blob']); } catch (StructuralLimitExceededException $limitExceeded) { + $decision = $failureTime === null ? null : self::selectPortableRetryDecision( + $execution, + $attempt, + [ + 'outcome' => 'failed', + 'message' => $limitExceeded->getMessage(), + ], + $failureTime, + nonRetryable: ActivityRetryPolicy::isNonRetryableFailure($execution, $limitExceeded), + ); + if ($decision !== null) { + $historyMetadata['retry_disposition'] = $decision; + } /** @var array{status: 'failed'|'waiting', event: WorkflowHistoryEvent|null, next_task: WorkflowTask|null} $failure */ - $failure = $this->recordFailureOrRetry($run, $task, $execution, $attempt, $limitExceeded, $historyMetadata); + $failure = $this->recordFailureOrRetry( + $run, + $task, + $execution, + $attempt, + $limitExceeded, + $historyMetadata, + $decision, + $failureTime + ); return $failure; } @@ -868,12 +1022,12 @@ private function recordSuccess( 'result' => $encoded['blob'], 'payload_codec' => $encoded['codec'], 'exception' => null, - 'closed_at' => now(), + 'closed_at' => $failureTime ?? now(), 'close_deadline_at' => null, 'heartbeat_deadline_at' => null, ])->save(); - self::closeAttempt($attempt, ActivityAttemptStatus::Completed); + self::closeAttempt($attempt, ActivityAttemptStatus::Completed, $failureTime); $event = WorkflowHistoryEvent::record( $run, @@ -927,11 +1081,14 @@ private function recordFailureOrRetry( ActivityAttempt $attempt, Throwable $throwable, array $historyMetadata = [], + ?array $retryDecision = null, + ?Carbon $failureTime = null, ): array { $attemptNumber = max(1, (int) $attempt->attempt_number); $maxAttempts = ActivityRetryPolicy::maxAttemptsFromSnapshot($execution); - if (! ActivityRetryPolicy::isNonRetryableFailure($execution, $throwable) && $attemptNumber < $maxAttempts) { + if ($retryDecision !== null ? $retryDecision['decision'] === 'scheduled' + : ! ActivityRetryPolicy::isNonRetryableFailure($execution, $throwable) && $attemptNumber < $maxAttempts) { $retry = $this->scheduleRetry( $run, $task, @@ -940,6 +1097,8 @@ private function recordFailureOrRetry( $throwable, LocalActivityRuntime::RETRY_REASON_FAILURE, historyMetadata: $historyMetadata, + retryDecision: $retryDecision, + failureTime: $failureTime, ); return [ @@ -949,7 +1108,15 @@ private function recordFailureOrRetry( ]; } - $event = $this->recordTerminalFailure($run, $task, $execution, $attempt, $throwable, $historyMetadata); + $event = $this->recordTerminalFailure( + $run, + $task, + $execution, + $attempt, + $throwable, + $historyMetadata, + $failureTime + ); return [ 'status' => 'failed', @@ -967,17 +1134,21 @@ private function scheduleRetry( string $retryReason, ?string $timeoutKind = null, array $historyMetadata = [], + ?array $retryDecision = null, + ?Carbon $failureTime = null, ): WorkflowTask { $attemptNumber = max(1, (int) ($attempt?->attempt_number ?? $execution->attempt_count)); $maxAttempts = ActivityRetryPolicy::maxAttemptsFromSnapshot($execution); - $backoffSeconds = ActivityRetryPolicy::backoffSecondsFromSnapshot($execution, $attemptNumber); - $retryAvailableAt = now() - ->addSeconds($backoffSeconds); + $backoffSeconds = $retryDecision['candidate_delay_seconds'] + ?? ActivityRetryPolicy::backoffSecondsFromSnapshot($execution, $attemptNumber); + $retryAvailableAt = $retryDecision === null ? now() + ->addSeconds($backoffSeconds) + : Carbon::parse($retryDecision['retry_available_at']); $exceptionPayload = FailureFactory::payload($throwable); $runCodec = is_string($run->payload_codec) && $run->payload_codec !== '' ? $run->payload_codec : null; if ($attempt instanceof ActivityAttempt && ! isset($historyMetadata['local_recovery'])) { - self::closeAttempt($attempt, ActivityAttemptStatus::Failed); + self::closeAttempt($attempt, ActivityAttemptStatus::Failed, $failureTime); } $execution->forceFill([ @@ -1057,6 +1228,7 @@ private function recordTerminalFailure( ?ActivityAttempt $attempt, Throwable $throwable, array $historyMetadata = [], + ?Carbon $failureTime = null, ): WorkflowHistoryEvent { $exceptionPayload = FailureFactory::payload($throwable); $failureCategory = $throwable instanceof StructuralLimitExceededException @@ -1085,13 +1257,13 @@ private function recordTerminalFailure( $runCodec ?? CodecRegistry::defaultCodec(), $exceptionPayload ), - 'closed_at' => now(), + 'closed_at' => $failureTime ?? now(), 'close_deadline_at' => null, 'heartbeat_deadline_at' => null, ])->save(); if ($attempt instanceof ActivityAttempt && ! isset($historyMetadata['local_recovery'])) { - self::closeAttempt($attempt, ActivityAttemptStatus::Failed); + self::closeAttempt($attempt, ActivityAttemptStatus::Failed, $failureTime); } $event = WorkflowHistoryEvent::record( @@ -1155,11 +1327,14 @@ private function recordTimeoutOutcome( ActivityExecution $execution, string $timeoutKind, array $historyMetadata = [], + ?array $retryDecision = null, + ?Carbon $failureTime = null, ): array { $attempt = self::currentAttempt($execution); $attemptNumber = max(1, (int) ($attempt?->attempt_number ?? $execution->attempt_count)); $maxAttempts = ActivityRetryPolicy::maxAttemptsFromSnapshot($execution); - $canRetry = $attemptNumber < $maxAttempts && $timeoutKind !== 'schedule_to_close'; + $canRetry = $retryDecision !== null ? $retryDecision['decision'] === 'scheduled' + : $attemptNumber < $maxAttempts && $timeoutKind !== 'schedule_to_close'; $throwable = new RuntimeException(self::timeoutMessage($execution, $timeoutKind)); if ($canRetry) { @@ -1174,15 +1349,17 @@ private function recordTimeoutOutcome( LocalActivityRuntime::RETRY_REASON_TIMEOUT, $timeoutKind, $historyMetadata, + $retryDecision, + $failureTime, ), ]; } if ($attempt instanceof ActivityAttempt && ! isset($historyMetadata['local_recovery'])) { - self::closeAttempt($attempt, ActivityAttemptStatus::Failed); + self::closeAttempt($attempt, ActivityAttemptStatus::Failed, $failureTime); } - $now = now(); + $now = $failureTime ?? now(); $failureCategory = FailureCategory::Timeout; $exceptionClass = ActivityTimeoutException::class; @@ -1348,18 +1525,21 @@ private static function currentAttempt(ActivityExecution $execution): ?ActivityA return $attempt; } - private static function closeAttempt(ActivityAttempt $attempt, ActivityAttemptStatus $status): void - { + private static function closeAttempt( + ActivityAttempt $attempt, + ActivityAttemptStatus $status, + ?Carbon $sample = null, + ): void { $attempt->forceFill([ 'status' => $status, 'lease_expires_at' => null, - 'closed_at' => $attempt->closed_at ?? now(), + 'closed_at' => $attempt->closed_at ?? $sample ?? now(), ])->save(); } - private static function timeoutKind(ActivityExecution $execution): ?string + private static function timeoutKind(ActivityExecution $execution, ?Carbon $sample = null): ?string { - $now = now(); + $now = $sample ?? now(); if ($execution->schedule_to_close_deadline_at !== null && $now->gte( $execution->schedule_to_close_deadline_at diff --git a/src/V2/Support/PortableLocalActivityCleanup.php b/src/V2/Support/PortableLocalActivityCleanup.php index 8a5609f1..7cc780af 100644 --- a/src/V2/Support/PortableLocalActivityCleanup.php +++ b/src/V2/Support/PortableLocalActivityCleanup.php @@ -251,14 +251,17 @@ public static function currentDeadline(WorkflowRun $run, ?array $snapshot): ?Car return $current === null ? $deadline : $deadline->min(\Illuminate\Support\Carbon::parse($current)); } - public static function currentAuthorityExpired(WorkflowRun $run, ActivityExecution $execution): bool - { + public static function currentAuthorityExpired( + WorkflowRun $run, + ActivityExecution $execution, + ?CarbonInterface $sample = null, + ): bool { if (! self::isScoped($execution)) { return false; } $deadline = self::currentDeadline($run, $execution->activity_options['cancellation_cleanup']); - return $deadline !== null && $deadline->lt(self::deadline($execution)) && now()->gte($deadline); + return $deadline !== null && $deadline->lt(self::deadline($execution)) && ($sample ?? now())->gte($deadline); } public static function bound( diff --git a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php index aa200827..fa1a7cff 100644 --- a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php +++ b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php @@ -1576,7 +1576,11 @@ public function testFixturesExecuteThroughDeclaredReplayConsumers(): void ->count(); $this->runReadyWorkflowTask($run); $this->assertSame(RunStatus::Completed, $run->refresh()->status); - $this->assertSame($fixture['expected_step']['result'], Serializer::unserializeWithCodec( + $this->assertSame([ + ...$fixture['expected']['result'], + 'workflow_id' => $run->workflow_instance_id, + 'run_id' => $run->id, + ], Serializer::unserializeWithCodec( $run->outputPayloadCodec(), $run->output )); diff --git a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php index 8946e1fe..d241207a 100644 --- a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php +++ b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php @@ -239,6 +239,221 @@ public function testRetryContractRejectsAnUnbindableDeadlineBeforePublishingPrep $this->assertSame($before, $task->refresh()->getAttributes()); } + #[DataProvider('retryContractOutcomes')] + public function testRetryContractCommitsAndReplaysItsOriginalDecision( + int $elapsed, + array $report, + array $timeouts, + string $eventType, + string $classification, + ?int $delay, + ): void { + $origin = Carbon::parse('2026-11-01T00:00:00.123456Z'); + Carbon::setTestNow($origin); + try { + [$run, $task] = $this->newClaim(); + $prepared = $this->prepareRetryContract($task, $timeouts); + $this->assertTrue($prepared['prepared']); + Carbon::setTestNow($origin->copy()->addSeconds($elapsed)); + $executor = app(LocalActivityExecutor::class); + $reply = $executor->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertTrue($reply['recorded'], json_encode($reply)); + $this->assertFalse($reply['duplicate']); + $this->assertSame($eventType, $reply['event_type']); + $decision = $reply['retry_disposition']; + $this->assertSame($classification, $decision['terminal_classification'] ?? $decision['decision']); + $this->assertSame($origin->copy()->addSeconds($elapsed)->toISOString(), $decision['failure_time']); + $this->assertSame( + $origin->copy()->addSeconds(100)->toISOString(), + $decision['original_schedule_to_close_deadline_at'] + ); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $this->assertSame(2, $event->payload['local_outcome']['version']); + $this->assertEquals($decision, $event->payload['retry_disposition']); + $this->assertSame($delay === null ? 'terminal' : 'scheduled', $decision['decision']); + $this->assertSame($delay !== null, $reply['claim_released']); + $this->assertCount($delay === null ? 0 : 1, $reply['created_task_ids']); + if ($delay !== null) { + $retry = WorkflowTask::query()->findOrFail($reply['created_task_ids'][0]); + $this->assertSame( + $origin->copy()->addSeconds($elapsed + $delay)->toISOString(), + $retry->available_at->toISOString() + ); + $this->assertSame($delay, $event->payload['retry_backoff_seconds']); + $this->assertSame(TaskStatus::Completed, $task->refresh()->status); + } else { + $this->assertSame(TaskStatus::Leased, $task->refresh()->status); + $this->assertArrayNotHasKey('retry_available_at', $decision); + } + if ($decision['failure_source'] === 'application') { + $this->assertSame($report['message'], $event->payload['message']); + $this->assertSame('SyntheticFailure', $event->payload['exception_type']); + } + $this->assertSame(1, $run->activityExecutions()->sole()->attempt_count); + $this->assertSame(1, ActivityAttempt::query()->where('workflow_run_id', $run->id)->count()); + $this->assertSame(3, $run->historyEvents()->count()); + $taskCount = $run->tasks() + ->count(); + Carbon::setTestNow($origin->copy()->addSeconds(400)); + $duplicate = $executor->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertTrue($duplicate['recorded']); + $this->assertTrue($duplicate['duplicate']); + $this->assertSame($reply['event_id'], $duplicate['event_id']); + $this->assertEquals($decision, $duplicate['retry_disposition']); + $this->assertSame(3, $run->historyEvents()->count()); + $this->assertSame($taskCount, $run->tasks()->count()); + } finally { + Carbon::setTestNow(); + } + } + + public static function retryContractOutcomes(): array + { + $failure = [ + 'outcome' => 'failed', + 'message' => 'original failure', + 'exception_type' => 'SyntheticFailure', + ]; + return [ + 'zero override before original deadline' => [99, [ + ...$failure, + 'next_retry_delay_seconds' => 0, + ], [], + 'ActivityRetryScheduled', 'scheduled', 0], + 'candidate reaches original deadline' => [99, [ + ...$failure, + 'next_retry_delay_seconds' => 1, + ], [], + 'ActivityFailed', 'retry_delay_exceeds_schedule_to_close', null], + 'fresh due deadline preempts zero override' => [100, [ + ...$failure, + 'next_retry_delay_seconds' => 0, + ], [], + 'ActivityTimedOut', 'engine_timeout', null], + 'override exceeds cap' => [1, [ + ...$failure, + 'next_retry_delay_seconds' => 6, + ], [], + 'ActivityFailed', 'retry_delay_exceeds_policy_cap', null], + 'absent override selects policy' => [1, $failure, [], 'ActivityRetryScheduled', 'scheduled', 2], + 'start timeout ignores application override' => [2, [ + ...$failure, + 'next_retry_delay_seconds' => 0, + ], + [ + 'start_to_close_timeout' => 2, + ], 'ActivityRetryScheduled', 'scheduled', 2], + 'heartbeat timeout uses policy' => [2, [ + 'outcome' => 'timed_out', + ], [ + 'heartbeat_timeout' => 2, + ], + 'ActivityRetryScheduled', 'scheduled', 2], + 'nonretryable precedes selection' => [1, [ + ...$failure, + 'non_retryable' => true, + 'next_retry_delay_seconds' => 6, + ], [], 'ActivityFailed', 'non_retryable', null], + ]; + } + + public function testRetryContractOutcomeAndItsTaskRollBackWithTheOuterTransaction(): void + { + [$run, $task] = $this->newClaim(); + $prepared = $this->prepareRetryContract($task); + $connection = ConfiguredV2Models::connection(); + $connection->beginTransaction(); + try { + $reply = app(LocalActivityExecutor::class)->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + [ + 'outcome' => 'failed', + 'message' => 'failure', + 'next_retry_delay_seconds' => 0, + ], + '1.20' + ); + $this->assertTrue($reply['recorded']); + $this->assertSame(3, $run->historyEvents()->count()); + $this->assertSame(2, $run->tasks()->count()); + } finally { + $connection->rollBack(); + } + $this->assertSame(2, $run->historyEvents()->count()); + $this->assertSame(1, $run->tasks()->count()); + $this->assertSame(TaskStatus::Leased, $task->refresh()->status); + $this->assertSame( + ActivityAttemptStatus::Running, + ActivityAttempt::query()->findOrFail($prepared['activity_attempt_id'])->status + ); + } + + public function testRetryContractReportPreflightDoesNotGrantModeAuthority(): void + { + [$run, $task] = $this->newClaim(); + $legacy = $this->prepare($task); + $executor = app(LocalActivityExecutor::class); + $connection = ConfiguredV2Models::connection(); + $connection->enableQueryLog(); + $connection->flushQueryLog(); + try { + $refused = $executor->recordPortableRetryOutcome( + 'missing', + 'foreign', + 1, + [ + 'outcome' => 'failed', + 'message' => false, + 'next_retry_delay_seconds' => null, + ], + '1.20' + ); + $this->assertSame('invalid_activity_retry_delay', $refused['reason']); + $this->assertSame([], $connection->getQueryLog()); + } finally { + $connection->disableQueryLog(); + $connection->flushQueryLog(); + } + $refused = $executor->recordPortableRetryOutcome( + $legacy['activity_attempt_id'], + 'portable-worker', + 1, + [ + 'outcome' => 'failed', + 'message' => 'failure', + 'next_retry_delay_seconds' => 0, + ], + '1.20' + ); + $this->assertSame('local_activity_preparation_mismatch', $refused['reason']); + $this->assertSame(2, $run->historyEvents()->count()); + $refused = $executor->recordPortableRetryOutcome( + 'missing', + 'foreign', + 1, + [ + 'outcome' => 'failed', + 'message' => false, + ], + '1.20' + ); + $this->assertSame('activity_attempt_not_found', $refused['reason']); + } + public function testLostPreparationResponseReturnsTheOriginalAttemptWithoutRenewingAnyDeadline(): void { [, $task] = $this->newClaim(); @@ -1491,6 +1706,29 @@ public static function changedCanonicalScopeReceipt(): iterable yield 'started snapshot reparented' => ['started']; } + private function prepareRetryContract(WorkflowTask $task, array $timeouts = []): array + { + return PortableLocalActivityPreparation::prepareWithRetryPolicy( + $task->id, + 'portable-worker', + 1, + 1, + 'sdk-local-attempt', + [ + ...$this->descriptor(), + 'retry_policy' => [ + 'contract_version' => 2, + 'mode' => 'explicit', + 'max_attempts' => 3, + 'backoff_seconds' => [2, 5], + ], + 'schedule_to_close_timeout' => 100, + ...$timeouts, + ], + '1.20' + ); + } + /** @param list> $expected * @param list> $actual */ From 618a5c6291f8da50c16bb7bdce42dc7a210598ba Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 6 Nov 2026 00:24:00 +0000 Subject: [PATCH 06/15] Bind retry preparation to stored v2 decisions and original deadlines --- src/V2/Support/LocalActivityExecutor.php | 30 +++- .../PortableLocalActivityPreparation.php | 78 +++++++-- .../V2EmbeddedReplayRegressionCorpusTest.php | 38 ++++- ...V2PortableLocalActivityPreparationTest.php | 154 +++++++++++++++++- 4 files changed, 277 insertions(+), 23 deletions(-) diff --git a/src/V2/Support/LocalActivityExecutor.php b/src/V2/Support/LocalActivityExecutor.php index 61558427..9d302a9a 100644 --- a/src/V2/Support/LocalActivityExecutor.php +++ b/src/V2/Support/LocalActivityExecutor.php @@ -172,8 +172,36 @@ public function expirePortableRetry( WorkflowRun $run, WorkflowTask $task, ActivityExecution $execution, + int $localContractVersion = 1, + ?Carbon $sample = null, ): WorkflowHistoryEvent { - $outcome = $this->recordTimeoutOutcome($run, $task, $execution, 'schedule_to_close'); + $decision = null; + $metadata = []; + if ($localContractVersion === 2) { + $attempt = self::currentAttempt($execution); + if (! $attempt instanceof ActivityAttempt || $sample === null) { + throw new RuntimeException('Prepared retry expiry requires its original attempt and locked sample.'); + } + $decision = self::selectPortableRetryDecision( + $execution, + $attempt, + [ + 'outcome' => 'timed_out', + ], + $sample, + 'schedule_to_close' + ); + $metadata['retry_disposition'] = $decision; + } + $outcome = $this->recordTimeoutOutcome( + $run, + $task, + $execution, + 'schedule_to_close', + $metadata, + $decision, + $sample + ); return $outcome['event']; } diff --git a/src/V2/Support/PortableLocalActivityPreparation.php b/src/V2/Support/PortableLocalActivityPreparation.php index 37ada876..a924f57d 100644 --- a/src/V2/Support/PortableLocalActivityPreparation.php +++ b/src/V2/Support/PortableLocalActivityPreparation.php @@ -691,6 +691,10 @@ private static function prepareForContract( } $execution = $rows['execution'] ?? null; $attempt = $rows['attempt'] ?? null; + if ($execution instanceof ActivityExecution + && ($execution->activity_options['local_activity_contract_version'] ?? 1) !== $localContractVersion) { + return self::response('local_activity_preparation_mismatch'); + } if ($snapshotExecution !== null) { if (! $execution instanceof ActivityExecution || $execution->id !== $snapshotExecution->id || $execution->sequence !== $sequence @@ -711,7 +715,16 @@ private static function prepareForContract( return self::response('local_activity_previous_attempt_unresolved'); } if ($execution->status === ActivityStatus::Pending) { - return self::prepareRetry($run, $task, $execution, $attempt, $workerAttemptId, $fingerprint); + return self::prepareRetry( + $run, + $task, + $execution, + $attempt, + $workerAttemptId, + $fingerprint, + $localContractVersion, + $preparedAt + ); } return self::duplicate( $run, @@ -720,7 +733,8 @@ private static function prepareForContract( $attempt, $workerAttemptId, $fingerprint, - $localContractVersion + $localContractVersion, + $preparedAt ); } if (ActivityExecution::query()->where('workflow_run_id', $run->id)->where( @@ -983,6 +997,8 @@ private static function prepareRetry( ActivityAttempt $previous, string $workerAttemptId, string $fingerprint, + int $localContractVersion = 1, + ?Carbon $preparedAt = null, ): array { $started = $run->historyEvents() ->where('event_type', HistoryEventType::ActivityStarted) @@ -994,7 +1010,14 @@ private static function prepareRetry( || $previous->closed_at === null || $execution->attempt_count !== $previous->attempt_number || ! is_int($originalTaskAttempt) - || self::originalStart($run, $execution, $previous, $previous->lease_owner, $originalTaskAttempt) === null + || self::originalStart( + $run, + $execution, + $previous, + $previous->lease_owner, + $originalTaskAttempt, + $localContractVersion + ) === null || ($started->payload['local_preparation']['descriptor_fingerprint'] ?? null) !== $fingerprint) { return self::response('local_activity_retry_preparation_mismatch'); } @@ -1009,7 +1032,13 @@ private static function prepareRetry( || ($retry->payload['retry_after_attempt_id'] ?? null) !== $previous->id || ($retry->payload['retry_after_attempt'] ?? null) !== $previous->attempt_number || ($retry->payload['retry_policy'] ?? null) !== $execution->retry_policy - || ! self::retryAuthorityMatches($retry, $previous, $originalTaskAttempt, $fingerprint) + || ! self::retryAuthorityMatches( + $retry, + $previous, + $originalTaskAttempt, + $fingerprint, + $localContractVersion + ) || ! is_string($retry->payload['retry_available_at'] ?? null) || $execution->attempt_count >= ActivityRetryPolicy::maxAttemptsFromSnapshot($execution)) { return self::response('local_activity_retry_preparation_mismatch'); @@ -1019,21 +1048,30 @@ private static function prepareRetry( } // The durable receipt owns backoff. Mutable task availability cannot // admit a callback early. Total timeout owns the entire retry chain. - if ($execution->schedule_to_close_deadline_at !== null && now()->gte( + if ($execution->schedule_to_close_deadline_at !== null && ($preparedAt ?? now())->gte( $execution->schedule_to_close_deadline_at )) { - $event = app(LocalActivityExecutor::class)->expirePortableRetry($run, $task, $execution); + $event = app(LocalActivityExecutor::class)->expirePortableRetry( + $run, + $task, + $execution, + $localContractVersion, + $preparedAt + ); return [ ...self::response('local_activity_deadline_expired'), 'event_id' => $event->id, 'event_type' => $event->event_type->value, ]; } - if (now()->lt(Carbon::parse($retry->payload['retry_available_at']))) { + if (($preparedAt ?? now())->lt(Carbon::parse($retry->payload['retry_available_at']))) { return self::response('local_activity_retry_not_due'); } $attempt = app(LocalActivityExecutor::class)->startPortableAttempt($run, $task, $execution, $workerAttemptId, [ - 'version' => 1, + 'version' => $localContractVersion, + ...($localContractVersion === 2 ? [ + 'local_activity_contract_version' => 2, + ] : []), 'descriptor_fingerprint' => $fingerprint, 'worker_attempt_id' => $workerAttemptId, 'workflow_task_attempt' => $task->attempt_count, @@ -1041,7 +1079,7 @@ private static function prepareRetry( ? [ 'cancellation_cleanup' => $execution->activity_options['cancellation_cleanup'], ] : []), - ]); + ], $preparedAt); return self::response(null, $execution, $attempt, task: $task); } @@ -1152,14 +1190,27 @@ private static function retryAuthorityMatches( ActivityAttempt $previous, int $originalTaskAttempt, string $fingerprint, + int $localContractVersion = 1, ): bool { $payload = $retry->payload; - if (($payload['local_outcome']['version'] ?? null) === 1) { + if (($payload['local_outcome']['version'] ?? null) === $localContractVersion) { return $previous->status === ActivityAttemptStatus::Failed && $retry->workflow_task_id === $previous->workflow_task_id && ($payload['local_outcome']['workflow_task_attempt'] ?? null) === $originalTaskAttempt && ($payload['task']['lease_owner'] ?? null) === $previous->lease_owner - && ($payload['task']['attempt_count'] ?? null) === $originalTaskAttempt; + && ($payload['task']['attempt_count'] ?? null) === $originalTaskAttempt + && ($localContractVersion === 1 || ( + ($payload['retry_disposition']['contract_version'] ?? null) === 2 + && ($payload['retry_disposition']['snapshot_version'] ?? null) === 2 + && ($payload['retry_disposition']['decision'] ?? null) === 'scheduled' + && ($payload['retry_disposition']['candidate_delay_seconds'] ?? null) + === ($payload['retry_backoff_seconds'] ?? null) + && ($payload['retry_disposition']['retry_available_at'] ?? null) + === ($payload['retry_available_at'] ?? null) + )); + } + if ($localContractVersion !== 1) { + return false; } $recovery = $payload['local_recovery'] ?? []; return $previous->status === ActivityAttemptStatus::Expired @@ -1188,6 +1239,7 @@ private static function duplicate( string $workerAttemptId, string $fingerprint, int $localContractVersion = 1, + ?Carbon $preparedAt = null, ): array { if (! LocalActivityRuntime::isExecution($execution) || $execution->workflow_run_id !== $run->id || $execution->status !== ActivityStatus::Running @@ -1207,7 +1259,7 @@ private static function duplicate( || ($started->payload['local_preparation']['descriptor_fingerprint'] ?? null) !== $fingerprint) { return self::response('local_activity_preparation_mismatch'); } - if ($attempt->lease_expires_at === null || now()->gte($attempt->lease_expires_at)) { + if ($attempt->lease_expires_at === null || ($preparedAt ?? now())->gte($attempt->lease_expires_at)) { return self::response('local_activity_preparation_lease_expired'); } foreach ([ @@ -1215,7 +1267,7 @@ private static function duplicate( $execution->schedule_to_close_deadline_at, $execution->heartbeat_deadline_at, ] as $deadline) { - if ($deadline !== null && now()->gte($deadline)) { + if ($deadline !== null && ($preparedAt ?? now())->gte($deadline)) { return self::response('local_activity_deadline_expired'); } } diff --git a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php index fa1a7cff..ed74ec35 100644 --- a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php +++ b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php @@ -1572,6 +1572,30 @@ public function testFixturesExecuteThroughDeclaredReplayConsumers(): void if ($fixture['id'] === 'prepared-v1-policy-refusal-legacy-zero-tail') { $run = $this->createRunFromFixture($fixture); + $scheduled = collect($fixture['history'])->firstWhere('event_type', 'ActivityScheduled')['payload']; + $completed = collect($fixture['history'])->firstWhere('event_type', 'ActivityCompleted')['payload']; + // Embedded execution reloads both retained history and its + // activity projection, unlike the history-only fiber runner. + ActivityExecution::query()->create([ + 'id' => $scheduled['activity_execution_id'], + 'workflow_run_id' => $run->id, + 'sequence' => $scheduled['sequence'], + 'activity_class' => $scheduled['activity_class'], + 'activity_type' => $scheduled['activity_type'], + 'status' => \Workflow\V2\Enums\ActivityStatus::Completed, + 'attempt_count' => $completed['attempt_number'], + 'arguments' => Serializer::serializeWithCodec('avro', $fixture['workflow']['arguments']), + 'result' => $completed['result'], + 'payload_codec' => $completed['payload_codec'], + 'retry_policy' => $scheduled['activity']['retry_policy'], + 'activity_options' => [ + 'execution_mode' => 'local', + 'queue_bypassed' => true, + 'routing' => 'workflow_worker_process', + ], + 'connection' => $run->connection, + 'queue' => $run->queue, + ]); $before = $run->historyEvents() ->count(); $this->runReadyWorkflowTask($run); @@ -1580,10 +1604,7 @@ public function testFixturesExecuteThroughDeclaredReplayConsumers(): void ...$fixture['expected']['result'], 'workflow_id' => $run->workflow_instance_id, 'run_id' => $run->id, - ], Serializer::unserializeWithCodec( - $run->outputPayloadCodec(), - $run->output - )); + ], Serializer::unserializeWithCodec($run->outputPayloadCodec(), $run->output)); $this->assertSame([120, 0], $run->historyEvents() ->where('event_type', HistoryEventType::ActivityRetryScheduled) ->orderBy('sequence') @@ -1591,7 +1612,14 @@ public function testFixturesExecuteThroughDeclaredReplayConsumers(): void ->map(static fn (WorkflowHistoryEvent $event): int => $event->payload['retry_backoff_seconds']) ->all()); $this->assertSame($before + 1, $run->historyEvents()->count()); - $this->assertSame(0, $run->activityExecutions()->count()); + $this->assertSame(1, $run->activityExecutions()->count()); + $this->assertSame(0, ActivityAttempt::query()->where('workflow_run_id', $run->id)->count()); + $this->assertEquals( + $scheduled['activity']['retry_policy'], + $run->activityExecutions() + ->sole() +->retry_policy + ); continue; } diff --git a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php index d241207a..4239c565 100644 --- a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php +++ b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php @@ -270,7 +270,9 @@ public function testRetryContractCommitsAndReplaysItsOriginalDecision( $this->assertSame($classification, $decision['terminal_classification'] ?? $decision['decision']); $this->assertSame($origin->copy()->addSeconds($elapsed)->toISOString(), $decision['failure_time']); $this->assertSame( - $origin->copy()->addSeconds(100)->toISOString(), + $origin->copy() + ->addSeconds(100) + ->toISOString(), $decision['original_schedule_to_close_deadline_at'] ); $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); @@ -282,7 +284,9 @@ public function testRetryContractCommitsAndReplaysItsOriginalDecision( if ($delay !== null) { $retry = WorkflowTask::query()->findOrFail($reply['created_task_ids'][0]); $this->assertSame( - $origin->copy()->addSeconds($elapsed + $delay)->toISOString(), + $origin->copy() + ->addSeconds($elapsed + $delay) + ->toISOString(), $retry->available_at->toISOString() ); $this->assertSame($delay, $event->payload['retry_backoff_seconds']); @@ -1706,14 +1710,156 @@ public static function changedCanonicalScopeReceipt(): iterable yield 'started snapshot reparented' => ['started']; } - private function prepareRetryContract(WorkflowTask $task, array $timeouts = []): array + public function testRetryContractPreparesOnlyTheStoredDueRetryAndKeepsTheOriginalBudget(): void { + $origin = Carbon::parse('2026-11-01T00:00:00.123456Z'); + Carbon::setTestNow($origin); + try { + [$run, $task] = $this->newClaim(); + $prepared = $this->prepareRetryContract($task); + $executor = app(LocalActivityExecutor::class); + $report = [ + 'outcome' => 'failed', + 'message' => 'failure', + ]; + $first = $executor->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertTrue($first['recorded']); + $retry = WorkflowTask::query()->findOrFail($first['created_task_ids'][0]); + $retry->forceFill([ + 'status' => TaskStatus::Leased, + 'attempt_count' => 1, + 'lease_owner' => 'portable-worker', + 'lease_expires_at' => $origin->copy() + ->addMinutes(5), + 'available_at' => $origin, + ])->save(); + Carbon::setTestNow($origin->copy()->addSecond()); + $early = $this->prepareRetryContract($retry, workerAttemptId: 'sdk-local-attempt-2'); + $this->assertSame('local_activity_retry_not_due', $early['reason']); + $this->assertSame(1, $run->activityExecutions()->sole()->attempt_count); + $this->assertSame(3, $run->historyEvents()->count()); + Carbon::setTestNow($origin->copy()->addSeconds(2)); + $second = $this->prepareRetryContract($retry, workerAttemptId: 'sdk-local-attempt-2'); + $this->assertTrue($second['prepared'], json_encode($second)); + $this->assertNotSame($prepared['activity_attempt_id'], $second['activity_attempt_id']); + $this->assertSame( + $origin->copy() + ->addSeconds(100) + ->toISOString(), + $run->activityExecutions() + ->sole() + ->schedule_to_close_deadline_at->toISOString() + ); + $lost = $executor->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertTrue($lost['duplicate']); + $this->assertSame($first['event_id'], $lost['event_id']); + Carbon::setTestNow($origin->copy()->addSeconds(3)); + $secondFailure = $executor->recordPortableRetryOutcome( + $second['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertSame(5, $secondFailure['retry_disposition']['candidate_delay_seconds']); + $thirdTask = WorkflowTask::query()->findOrFail($secondFailure['created_task_ids'][0]); + $thirdTask->forceFill([ + 'status' => TaskStatus::Leased, + 'attempt_count' => 1, + 'lease_owner' => 'portable-worker', + 'lease_expires_at' => $origin->copy() + ->addMinutes(5), + ])->save(); + Carbon::setTestNow($origin->copy()->addSeconds(8)); + $third = $this->prepareRetryContract($thirdTask, workerAttemptId: 'sdk-local-attempt-3'); + $this->assertTrue($third['prepared'], json_encode($third)); + $terminal = $executor->recordPortableRetryOutcome( + $third['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertSame('attempts_exhausted', $terminal['retry_disposition']['terminal_classification']); + $this->assertSame([], $terminal['created_task_ids']); + $this->assertSame(3, $run->activityExecutions()->sole()->attempt_count); + $this->assertSame(3, ActivityAttempt::query()->where('workflow_run_id', $run->id)->count()); + $this->assertSame(7, $run->historyEvents()->count()); + } finally { + Carbon::setTestNow(); + } + } + + public function testRetryContractExpiresAPendingRetryWithoutAnAdditionalAttempt(): void + { + $origin = Carbon::parse('2026-11-01T00:00:00.123456Z'); + Carbon::setTestNow($origin); + try { + [$run, $task] = $this->newClaim(); + $prepared = $this->prepareRetryContract($task); + $reply = app(LocalActivityExecutor::class)->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + [ + 'outcome' => 'failed', + 'message' => 'failure', + ], + '1.20' + ); + $retry = WorkflowTask::query()->findOrFail($reply['created_task_ids'][0]); + $retry->forceFill([ + 'status' => TaskStatus::Leased, + 'attempt_count' => 1, + 'lease_owner' => 'portable-worker', + 'lease_expires_at' => $origin->copy() + ->addMinutes(5), + ])->save(); + Carbon::setTestNow($origin->copy()->addSeconds(100)); + $expired = $this->prepareRetryContract($retry, workerAttemptId: 'sdk-local-attempt-2'); + $this->assertFalse($expired['prepared']); + $this->assertSame('local_activity_deadline_expired', $expired['reason']); + $this->assertSame('ActivityTimedOut', $expired['event_type']); + $event = WorkflowHistoryEvent::query()->findOrFail($expired['event_id']); + $this->assertSame('engine_timeout', $event->payload['retry_disposition']['terminal_classification']); + $this->assertSame( + $origin->copy() + ->addSeconds(100) + ->toISOString(), + $event->payload['retry_disposition']['failure_time'] + ); + $this->assertSame(1, $run->activityExecutions()->sole()->attempt_count); + $this->assertSame(1, ActivityAttempt::query()->where('workflow_run_id', $run->id)->count()); + $this->assertSame(2, $run->tasks()->count()); + $this->assertSame(4, $run->historyEvents()->count()); + } finally { + Carbon::setTestNow(); + } + } + + private function prepareRetryContract( + WorkflowTask $task, + array $timeouts = [], + string $workerAttemptId = 'sdk-local-attempt', + ): array { return PortableLocalActivityPreparation::prepareWithRetryPolicy( $task->id, 'portable-worker', 1, 1, - 'sdk-local-attempt', + $workerAttemptId, [ ...$this->descriptor(), 'retry_policy' => [ From 97fc03749d53305db17a8205336e136d34990af1 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 6 Nov 2026 00:26:06 +0000 Subject: [PATCH 07/15] Refuse legacy internal execution of prepared retry v2 rows --- src/V2/Support/LocalActivityExecutor.php | 14 ++++++++ ...V2PortableLocalActivityPreparationTest.php | 33 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/src/V2/Support/LocalActivityExecutor.php b/src/V2/Support/LocalActivityExecutor.php index 9d302a9a..79c189fa 100644 --- a/src/V2/Support/LocalActivityExecutor.php +++ b/src/V2/Support/LocalActivityExecutor.php @@ -160,6 +160,13 @@ public function startPortableAttempt( array $preparation, ?Carbon $preparedAt = null, ): ActivityAttempt { + if (($execution->activity_options['local_activity_contract_version'] ?? null) === 2 + && (($preparation['version'] ?? null) !== 2 + || ($preparation['local_activity_contract_version'] ?? null) !== 2 || $preparedAt === null)) { + throw new RuntimeException( + 'Prepared retry attempts require their versioned preparation and locked sample.' + ); + } $preparation['schedule_to_close_deadline_at'] = $execution->schedule_to_close_deadline_at?->toISOString(); return $this->startAttempt($run, $task, $execution, $workerAttemptId, $preparation, $preparedAt); } @@ -175,6 +182,10 @@ public function expirePortableRetry( int $localContractVersion = 1, ?Carbon $sample = null, ): WorkflowHistoryEvent { + if (! in_array($localContractVersion, [1, 2], true) + || ($execution->activity_options['local_activity_contract_version'] ?? 1) !== $localContractVersion) { + throw new RuntimeException('Prepared retry expiry requires its original contract.'); + } $decision = null; $metadata = []; if ($localContractVersion === 2) { @@ -703,6 +714,9 @@ private function prepareAttempt( if (! $execution instanceof ActivityExecution) { $execution = $this->createExecution($lockedRun, $lockedTask, $sequence, $activityCall); } + if (($execution->activity_options['local_activity_contract_version'] ?? null) === 2) { + throw new RuntimeException('Prepared retry execution requires its versioned portable entry.'); + } if (! LocalActivityRuntime::isExecution($execution)) { throw new RuntimeException('Workflow history at this sequence belongs to an ordinary queued activity.'); diff --git a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php index 4239c565..bec12155 100644 --- a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php +++ b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php @@ -33,6 +33,7 @@ use Workflow\V2\Support\ConfiguredV2Models; use Workflow\V2\Support\DefaultWorkflowTaskBridge; use Workflow\V2\Support\HistoryTimeline; +use Workflow\V2\Support\LocalActivityCall; use Workflow\V2\Support\LocalActivityExecutor; use Workflow\V2\Support\ParallelChildGroup; use Workflow\V2\Support\PortableLocalActivityPreparation; @@ -198,6 +199,38 @@ public function testRetryContractPreparationRollsBackModeSnapshotAndAttemptWithI $this->assertSame(TaskStatus::Leased, $task->refresh()->status); } + public function testRetryContractCannotDowngradeThroughLegacyInternalExecution(): void + { + [$run, $task] = $this->newClaim(); + $prepared = $this->prepareRetryContract($task); + $execution = ActivityExecution::query()->findOrFail($prepared['activity_execution_id']); + $executor = app(LocalActivityExecutor::class); + $operations = [ + static fn () => $executor->execute( + $run, + $task, + 1, + new LocalActivityCall('python-local-greeting', ['Taylor']) + ), + static fn () => $executor->expirePortableRetry($run, $task, $execution), + static fn () => $executor->startPortableAttempt($run, $task, $execution, 'replacement', [ + 'version' => 1, + ]), + ]; + foreach ($operations as $operation) { + try { + $operation(); + $this->fail('A legacy internal entry accepted prepared retry contract 2.'); + } catch (\RuntimeException $error) { + $this->assertStringStartsWith('Prepared retry ', $error->getMessage()); + } + } + $this->assertSame(2, $run->historyEvents()->count()); + $this->assertSame(1, ActivityAttempt::query()->where('workflow_run_id', $run->id)->count()); + $this->assertSame(ActivityStatus::Running, $execution->refresh()->status); + $this->assertSame(TaskStatus::Leased, $task->refresh()->status); + } + public function testRetryContractRejectsAnUnbindableDeadlineBeforePublishingPreparation(): void { [$run, $task] = $this->newClaim(); From 30f99eb0dd8738373af6221f647db5219d50d797 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 6 Nov 2026 00:29:48 +0000 Subject: [PATCH 08/15] Isolate retained projection state for the new replay corpus case --- tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php index ed74ec35..5587b76e 100644 --- a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php +++ b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php @@ -1571,6 +1571,7 @@ public function testFixturesExecuteThroughDeclaredReplayConsumers(): void } if ($fixture['id'] === 'prepared-v1-policy-refusal-legacy-zero-tail') { + $this->clearWorkflowState(); $run = $this->createRunFromFixture($fixture); $scheduled = collect($fixture['history'])->firstWhere('event_type', 'ActivityScheduled')['payload']; $completed = collect($fixture['history'])->firstWhere('event_type', 'ActivityCompleted')['payload']; From f9ac83a3f25c423a0189d42e24c95d5e3312f350 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 6 Nov 2026 00:39:43 +0000 Subject: [PATCH 09/15] Validate retained retry receipt selections before replay --- src/V2/Support/LocalActivityExecutor.php | 57 +++++ src/V2/Support/PreparedLocalRetryDecision.php | 56 +++++ ...V2PortableLocalActivityPreparationTest.php | 86 +++++++ .../V2/PreparedLocalRetryDecisionTest.php | 234 +++++++++++++++++- 4 files changed, 426 insertions(+), 7 deletions(-) diff --git a/src/V2/Support/LocalActivityExecutor.php b/src/V2/Support/LocalActivityExecutor.php index 79c189fa..b95f0d5e 100644 --- a/src/V2/Support/LocalActivityExecutor.php +++ b/src/V2/Support/LocalActivityExecutor.php @@ -369,6 +369,14 @@ private function recordPortableOutcomeForContract( || ($receipt->payload['task']['attempt_count'] ?? null) !== $workflowTaskAttempt) { return $refused('local_activity_outcome_mismatch'); } + if ($localContractVersion === 2 && ! self::matchesStoredRetryReceipt( + $receipt, + $execution, + $attempt, + $report + )) { + return $refused('local_activity_outcome_mismatch'); + } return self::portableReceipt($receipt, true); } $failureTime = $localContractVersion === 2 ? now() : null; @@ -679,6 +687,55 @@ private static function selectPortableRetryDecision( ); } + /** + * @param array $report + */ + private static function matchesStoredRetryReceipt( + WorkflowHistoryEvent $event, + ActivityExecution $execution, + ActivityAttempt $attempt, + array $report, + ): bool { + $payload = $event->payload; + $outcome = $payload['local_outcome']; + if (count($outcome) !== 4 || array_diff(array_keys($outcome), [ + 'version', 'report_fingerprint', 'workflow_task_attempt', 'submitted_outcome', + ]) !== [] || $outcome['submitted_outcome'] !== $report['outcome']) { + return false; + } + if ($event->event_type === HistoryEventType::ActivityCompleted) { + return ! array_key_exists('retry_disposition', $payload); + } + $stored = $payload['retry_disposition'] ?? null; + if (! is_array($stored)) { + return false; + } + $nonRetryable = false; + if ($report['outcome'] === 'completed' && $event->event_type === HistoryEventType::ActivityFailed) { + if (($payload['failure_category'] ?? null) !== FailureCategory::StructuralLimit->value + || ! is_string($payload['message'] ?? null) || ! is_bool($payload['non_retryable'] ?? null)) { + return false; + } + $report = [ + 'outcome' => 'failed', + 'message' => $payload['message'], + ]; + $nonRetryable = $payload['non_retryable']; + } + $connection = ConfiguredV2Models::connection(); + + return PreparedLocalRetryDecision::matchesStored( + $execution->retry_policy, + $attempt->attempt_number, + $report, + $execution->schedule_to_close_deadline_at, + PreparedLocalRetryTimestamp::latestForConnection($connection), + $stored, + $nonRetryable, + PreparedLocalRetryTimestamp::earliestForDriver($connection->getDriverName()), + ); + } + /** * @return array{execution: ActivityExecution|null, attempt: ActivityAttempt|null, event: WorkflowHistoryEvent|null, next_task: WorkflowTask|null} */ diff --git a/src/V2/Support/PreparedLocalRetryDecision.php b/src/V2/Support/PreparedLocalRetryDecision.php index 696242b6..4fb093b9 100644 --- a/src/V2/Support/PreparedLocalRetryDecision.php +++ b/src/V2/Support/PreparedLocalRetryDecision.php @@ -14,6 +14,62 @@ /** @internal Select from the one locked engine sample, never a worker clock. */ final class PreparedLocalRetryDecision { + /** Validate caller-bound retained bytes without a new clock or effects. + * Timeout kind is retained engine authority, never a worker-selected input. + * @param array $snapshot + * @param array $report + * @param array $stored + */ + public static function matchesStored( + array $snapshot, + int $attemptNumber, + array $report, + ?DateTimeInterface $originalDeadline, + DateTimeInterface $latestBindableTimestamp, + array $stored, + bool $nonRetryable = false, + ?DateTimeInterface $earliestBindableTimestamp = null, + ): bool { + $time = $stored['failure_time'] ?? null; + $timeout = $stored['timeout_kind'] ?? null; + if (! is_string($time) || preg_match( + '/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{6}Z$/D', + $time + ) !== 1 + || ($timeout !== null && ! is_string($timeout))) { + return false; + } + $sample = DateTimeImmutable::createFromFormat('!Y-m-d\TH:i:s.u\Z', $time, new DateTimeZone('UTC')); + if (! $sample instanceof DateTimeImmutable || self::format($sample) !== $time) { + return false; + } + try { + PreparedLocalRetryTimestamp::add($sample, 0, $latestBindableTimestamp, $earliestBindableTimestamp); + $expected = self::select( + $snapshot, + $attemptNumber, + $report, + $sample, + $originalDeadline, + $latestBindableTimestamp, + $timeout, + $nonRetryable + ); + } catch (InvalidArgumentException) { + return false; + } + if (count($stored) !== count($expected) || array_diff(array_keys($stored), array_keys($expected)) !== []) { + return false; + } + foreach ($expected as $key => $value) { + if ($stored[$key] !== $value) { + return false; + } + } + + return true; + } + /** * The caller owns original authority, fresh fences and actual timeout * detection. Only a scheduled decision is a duration to execute. Terminal diff --git a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php index bec12155..fa6abedf 100644 --- a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php +++ b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php @@ -439,6 +439,92 @@ public function testRetryContractOutcomeAndItsTaskRollBackWithTheOuterTransactio ); } + public function testRetryContractCompletionKeepsItsClosedReceiptWithoutARetryDecision(): void + { + [$run, $task] = $this->newClaim(); + $prepared = $this->prepareRetryContract($task); + $executor = app(LocalActivityExecutor::class); + $report = [ + 'outcome' => 'completed', + 'result' => Serializer::serializeWithCodec('avro', 'done'), + 'payload_codec' => 'avro', + ]; + $reply = $executor->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertTrue($reply['recorded']); + $this->assertSame('ActivityCompleted', $reply['event_type']); + $this->assertArrayNotHasKey('retry_disposition', $reply); + $this->assertTrue($executor->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + )['duplicate']); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $event->forceFill([ + 'payload' => [ + ...$event->payload, + 'retry_disposition' => [], + ], + ])->save(); + $changed = $executor->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertFalse($changed['recorded']); + $this->assertSame('local_activity_outcome_mismatch', $changed['reason']); + $this->assertSame(3, $run->historyEvents()->count()); + $this->assertSame(1, $run->tasks()->count()); + } + + public function testRetainedTerminalDecisionCannotBeChangedIntoRetryPermission(): void + { + [$run, $task] = $this->newClaim(); + $prepared = $this->prepareRetryContract($task); + $report = [ + 'outcome' => 'failed', + 'message' => 'original failure', + 'next_retry_delay_seconds' => 6, + ]; + $executor = app(LocalActivityExecutor::class); + $reply = $executor->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertTrue($reply['recorded']); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $payload = $event->payload; + $payload['retry_disposition']['decision'] = 'scheduled'; + $payload['retry_disposition']['retry_available_at'] = $payload['retry_disposition']['failure_time']; + $event->forceFill([ + 'payload' => $payload, + ])->save(); + $changed = $executor->recordPortableRetryOutcome( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + $report, + '1.20' + ); + $this->assertFalse($changed['recorded']); + $this->assertSame('local_activity_outcome_mismatch', $changed['reason']); + $this->assertSame(3, $run->historyEvents()->count()); + $this->assertSame(1, $run->tasks()->count()); + $this->assertSame(1, $run->activityExecutions()->sole()->attempt_count); + } + public function testRetryContractReportPreflightDoesNotGrantModeAuthority(): void { [$run, $task] = $this->newClaim(); diff --git a/tests/Unit/V2/PreparedLocalRetryDecisionTest.php b/tests/Unit/V2/PreparedLocalRetryDecisionTest.php index 45721ca7..9f6acdc6 100644 --- a/tests/Unit/V2/PreparedLocalRetryDecisionTest.php +++ b/tests/Unit/V2/PreparedLocalRetryDecisionTest.php @@ -330,6 +330,232 @@ public function testInvalidInputOrUnprovenTimeoutCannotCreateADecision(): void } } + public function testRetainedDecisionRequiresTheExactTypedClosedSelection(): void + { + $report = [ + 'outcome' => 'failed', + 'message' => 'failure', + 'next_retry_delay_seconds' => 0, + ]; + $deadline = new DateTimeImmutable('2026-11-05T00:01:00.000000Z'); + $latest = new DateTimeImmutable('9999-12-31T23:59:59.999999Z'); + $stored = $this->select(report: $report, deadline: $deadline); + $matches = fn (array $value): bool => PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + $deadline, + $latest, + $value + ); + self::assertTrue($matches(array_reverse($stored, true))); + $missing = $stored; + unset($missing['candidate_delay_seconds']); + foreach ([ + $missing, + [ + ...$stored, + 'unexpected' => true, + ], + [ + ...$stored, + 'contract_version' => '2', + ], + [ + ...$stored, + 'snapshot_version' => 1, + ], + [ + ...$stored, + 'candidate_delay_seconds' => '0', + ], + [ + ...$stored, + 'requested_delay_seconds' => null, + ], + [ + ...$stored, + 'maximum_interval_seconds' => 6, + ], + [ + ...$stored, + 'retry_available_at' => '2026-11-05T00:00:01.000000Z', + ], + [ + ...$stored, + 'original_schedule_to_close_deadline_at' => null, + ], + [ + ...$stored, + 'failure_time' => [], + ], + [ + ...$stored, + 'failure_time' => '2026-02-30T00:00:00.000000Z', + ], + [ + ...$stored, + 'failure_time' => '2026-11-05T24:00:00.000000Z', + ], + [ + ...$stored, + 'failure_time' => '2026-11-05T00:00:00Z', + ], + [ + ...$stored, + 'timeout_kind' => [], + ], + [ + ...$stored, + 'timeout_kind' => 'worker_selected', + ], + ] as $index => $changed) { + self::assertFalse($matches($changed), 'Retained selection substitution ' . $index); + } + self::assertFalse(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + [ + ...$report, + 'next_retry_delay_seconds' => 1, + ], + $deadline, + $latest, + $stored + )); + } + + public function testRetainedTerminalDecisionsCannotGainAnExecutableTimestampOrUnselectedDelay(): void + { + $latest = new DateTimeImmutable('9999-12-31T23:59:59.999999Z'); + $report = [ + 'outcome' => 'failed', + 'message' => 'failure', + 'next_retry_delay_seconds' => 6, + ]; + $stored = $this->select(report: $report); + self::assertTrue(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + null, + $latest, + $stored + )); + self::assertFalse(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + null, + $latest, + [ + ...$stored, + 'retry_available_at' => '2026-11-05T00:00:06.000000Z', + ] + )); + $report = [ + 'outcome' => 'timed_out', + ]; + $deadline = new DateTimeImmutable('2026-11-05T00:00:00.000000Z'); + $stored = $this->select(report: $report, deadline: $deadline); + self::assertTrue(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + $deadline, + $latest, + $stored + )); + self::assertFalse(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + $deadline, + $latest, + [ + ...$stored, + 'delay_source' => 'policy', + ] + )); + self::assertFalse(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + $deadline->modify('+1 second'), + $latest, + $stored + )); + } + + public function testRetainedDecisionKeepsBackendBoundsAndEngineNonRetryableClassification(): void + { + $report = [ + 'outcome' => 'failed', + 'message' => 'failure', + ]; + $sample = new DateTimeImmutable('2026-11-05T00:00:00.000000Z'); + $latest = new DateTimeImmutable('9999-12-31T23:59:59.999999Z'); + $stored = $this->select(report: $report, nonRetryable: true); + self::assertTrue(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + null, + $latest, + $stored, + true + )); + self::assertFalse(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + null, + $latest, + $stored + )); + self::assertFalse(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + null, + $sample->modify('-1 microsecond'), + $stored, + true + )); + self::assertFalse(PreparedLocalRetryDecision::matchesStored( + $this->snapshot(), + 1, + $report, + null, + $latest, + $stored, + true, + $sample->modify('+1 microsecond') + )); + $snapshot = $this->snapshot(); + $snapshot['snapshot_version'] = 1; + self::assertFalse(PreparedLocalRetryDecision::matchesStored( + $snapshot, + 1, + $report, + null, + $latest, + $stored, + true + )); + } + + private function snapshot(): array + { + return PreparedLocalRetryPolicy::normalize([ + 'contract_version' => 2, + 'mode' => 'explicit', + 'max_attempts' => 3, + 'backoff_seconds' => [2, 5], + 'non_retryable_error_types' => ['PermanentFailure'], + ]); + } + private function select( array $report = [ 'outcome' => 'failed', @@ -342,13 +568,7 @@ private function select( ?string $timeout = null, bool $nonRetryable = false, ): array { - $snapshot = PreparedLocalRetryPolicy::normalize([ - 'contract_version' => 2, - 'mode' => 'explicit', - 'max_attempts' => 3, - 'backoff_seconds' => [2, 5], - 'non_retryable_error_types' => ['PermanentFailure'], - ]); + $snapshot = $this->snapshot(); return PreparedLocalRetryDecision::select( $snapshot, From ac531e03fd6e01522a87cb7d3068dad543802d00 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 6 Nov 2026 00:47:50 +0000 Subject: [PATCH 10/15] Add guarded retry-v2 supervisor and heartbeat source entries --- src/V2/Support/LocalActivityRuntime.php | 4 + .../Support/PortableLocalActivityControl.php | 69 +++++++- ...V2PortableLocalActivityPreparationTest.php | 160 ++++++++++++++++++ 3 files changed, 231 insertions(+), 2 deletions(-) diff --git a/src/V2/Support/LocalActivityRuntime.php b/src/V2/Support/LocalActivityRuntime.php index 49e7e3d7..52c4ce46 100644 --- a/src/V2/Support/LocalActivityRuntime.php +++ b/src/V2/Support/LocalActivityRuntime.php @@ -74,6 +74,7 @@ public static function renewWorkflowTask( ?CarbonInterface $deadline = null, bool $cleanup = false, ?WorkflowRun $run = null, + bool $validatePreparedRetryTimestamp = false, ): ?CarbonInterface { if ($task->task_type !== TaskType::Workflow || $task->status !== TaskStatus::Leased) { return null; @@ -86,6 +87,9 @@ public static function renewWorkflowTask( $leaseExpiresAt = $deadline === null ? ($cleanup ? CancellationCleanupLease::renewableExpiry() : CancellationCleanupLease::forScopes($run)) : CancellationCleanupLease::forDeadline($deadline); + if ($validatePreparedRetryTimestamp) { + PreparedLocalRetryTimestamp::addForConnection($leaseExpiresAt, 0, ConfiguredV2Models::connection()); + } $task->forceFill([ 'lease_expires_at' => $leaseExpiresAt, diff --git a/src/V2/Support/PortableLocalActivityControl.php b/src/V2/Support/PortableLocalActivityControl.php index 83b85611..561d2eb8 100644 --- a/src/V2/Support/PortableLocalActivityControl.php +++ b/src/V2/Support/PortableLocalActivityControl.php @@ -39,6 +39,28 @@ public static function poll( return self::observe($attemptId, $leaseOwner, $workflowTaskAttempt, $renewLease, false, null, $protocolVersion); } + /** @internal Unadvertised versioned supervisor entry. + * @return array + */ + public static function pollWithRetryPolicy( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + bool $renewLease = false, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return self::observe( + $attemptId, + $leaseOwner, + $workflowTaskAttempt, + $renewLease, + false, + null, + $protocolVersion, + 2 + ); + } + /** @param array $progress * @return array */ @@ -58,6 +80,35 @@ public static function heartbeat( return self::observe($attemptId, $leaseOwner, $workflowTaskAttempt, false, true, $normalized, $protocolVersion); } + /** @internal Unadvertised versioned application heartbeat entry. + * @param array $progress + * @return array + */ + public static function heartbeatWithRetryPolicy( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + array $progress = [], + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + try { + $normalized = HeartbeatProgress::normalizeForWrite($progress); + } catch (LogicException) { + return self::response('invalid_local_activity_heartbeat'); + } + + return self::observe( + $attemptId, + $leaseOwner, + $workflowTaskAttempt, + false, + true, + $normalized, + $protocolVersion, + 2 + ); + } + /** @param array|null $progress * @return array */ @@ -69,6 +120,7 @@ private static function observe( bool $applicationHeartbeat, ?array $progress, string $protocolVersion, + int $localContractVersion = 1, ): array { $reply = self::lockedObservation( $attemptId, @@ -77,7 +129,8 @@ private static function observe( $renewLease, $applicationHeartbeat, $progress, - $protocolVersion + $protocolVersion, + $localContractVersion ); $pending = $reply['_scope_control'] ?? null; if (! is_array($pending)) { @@ -141,7 +194,8 @@ private static function observe( $renewLease, $applicationHeartbeat, $progress, - $protocolVersion + $protocolVersion, + $localContractVersion ); } @@ -156,6 +210,7 @@ private static function lockedObservation( bool $applicationHeartbeat, ?array $progress, string $protocolVersion, + int $localContractVersion = 1, ): array { if (preg_match('/^[0-9]+\.[0-9]+$/D', $protocolVersion) !== 1 || version_compare($protocolVersion, PortableLocalActivityPreparation::MINIMUM_PROTOCOL_VERSION, '<')) { @@ -172,6 +227,7 @@ private static function lockedObservation( $renewLease, $applicationHeartbeat, $progress, + $localContractVersion, ): array { $rows = ActivityRowLockOrder::lockForAttempt($attemptId); $attempt = $rows['attempt']; @@ -196,6 +252,7 @@ private static function lockedObservation( $attempt, $leaseOwner, $workflowTaskAttempt, + $localContractVersion, ); if ($started === null) { return self::response('local_activity_preparation_mismatch'); @@ -372,6 +429,7 @@ private static function lockedObservation( ) ? null : PortableLocalActivityCleanup::deadline($execution), $cleanup, $run, + validatePreparedRetryTimestamp: $localContractVersion === 2, ); $deadline = PortableLocalActivityCleanup::currentDeadline( $run, @@ -388,6 +446,13 @@ private static function lockedObservation( if ($applicationHeartbeat) { $heartbeatAt = now(); $timeout = $execution->retry_policy['heartbeat_timeout'] ?? null; + if ($localContractVersion === 2) { + PreparedLocalRetryTimestamp::addForConnection( + $heartbeatAt, + $timeout ?? 0, + ConfiguredV2Models::connection() + ); + } $execution->forceFill([ 'last_heartbeat_at' => $heartbeatAt, 'heartbeat_deadline_at' => PortableLocalActivityCleanup::bound( diff --git a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php index fa6abedf..6d8bd5d3 100644 --- a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php +++ b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php @@ -36,6 +36,7 @@ use Workflow\V2\Support\LocalActivityCall; use Workflow\V2\Support\LocalActivityExecutor; use Workflow\V2\Support\ParallelChildGroup; +use Workflow\V2\Support\PortableLocalActivityControl; use Workflow\V2\Support\PortableLocalActivityPreparation; use Workflow\V2\Support\PreparedLocalRetryTimestamp; use Workflow\V2\WorkflowStub; @@ -439,6 +440,165 @@ public function testRetryContractOutcomeAndItsTaskRollBackWithTheOuterTransactio ); } + public function testRetryContractSupervisorAndHeartbeatKeepTheirSeparateAuthority(): void + { + $origin = Carbon::parse('2026-11-01T00:00:00.123456Z'); + Carbon::setTestNow($origin); + try { + [$run, $task] = $this->newClaim(); + $prepared = $this->prepareRetryContract( + $task, + [ + 'start_to_close_timeout' => 10, + 'heartbeat_timeout' => 4, + ] + ); + $attemptId = $prepared['activity_attempt_id']; + $this->assertSame('local_activity_preparation_mismatch', PortableLocalActivityControl::poll( + $attemptId, + 'portable-worker', + 1, + true, + '1.20' + )['reason']); + Carbon::setTestNow($origin->copy()->addSecond()); + $renewed = PortableLocalActivityControl::pollWithRetryPolicy( + $attemptId, + 'portable-worker', + 1, + true, + '1.20' + ); + $this->assertTrue($renewed['active']); + $this->assertTrue($renewed['renewed']); + $this->assertFalse($renewed['heartbeat_recorded']); + $this->assertSame($renewed['lease_expires_at'], $renewed['workflow_lease_expires_at']); + $this->assertSame($prepared['start_to_close_deadline_at'], $renewed['start_to_close_deadline_at']); + $this->assertSame($prepared['schedule_to_close_deadline_at'], $renewed['schedule_to_close_deadline_at']); + $this->assertSame($prepared['heartbeat_deadline_at'], $renewed['heartbeat_deadline_at']); + $this->assertSame(2, $run->historyEvents()->count()); + Carbon::setTestNow($origin->copy()->addSeconds(2)); + $heartbeat = PortableLocalActivityControl::heartbeatWithRetryPolicy( + $attemptId, + 'portable-worker', + 1, + [ + 'step' => 1, + ], + '1.20' + ); + $this->assertTrue($heartbeat['active']); + $this->assertTrue($heartbeat['heartbeat_recorded']); + $this->assertFalse($heartbeat['renewed']); + $this->assertSame($renewed['lease_expires_at'], $heartbeat['lease_expires_at']); + $this->assertSame($origin->copy()->addSeconds(6)->toISOString(), $heartbeat['heartbeat_deadline_at']); + $this->assertSame($prepared['start_to_close_deadline_at'], $heartbeat['start_to_close_deadline_at']); + $this->assertSame($prepared['schedule_to_close_deadline_at'], $heartbeat['schedule_to_close_deadline_at']); + $this->assertSame(3, $run->historyEvents()->count()); + Carbon::setTestNow($origin->copy()->addSeconds(6)); + $expired = PortableLocalActivityControl::pollWithRetryPolicy( + $attemptId, + 'portable-worker', + 1, + true, + '1.20' + ); + $this->assertTrue($expired['stop_required']); + $this->assertSame('local_activity_deadline_expired', $expired['reason']); + $this->assertFalse($expired['renewed']); + $this->assertSame($renewed['lease_expires_at'], $expired['lease_expires_at']); + $this->assertSame(3, $run->historyEvents()->count()); + } finally { + Carbon::setTestNow(); + } + } + + public function testRetryContractSupervisorDoesNotBorrowLegacyOrForeignPreparation(): void + { + [$run, $task] = $this->newClaim(); + $prepared = $this->prepare($task); + $refused = PortableLocalActivityControl::pollWithRetryPolicy( + $prepared['activity_attempt_id'], + 'portable-worker', + 1, + true, + '1.20' + ); + $this->assertSame('local_activity_preparation_mismatch', $refused['reason']); + $refused = PortableLocalActivityControl::heartbeatWithRetryPolicy( + $prepared['activity_attempt_id'], + 'foreign-worker', + 1, + [ + 'step' => 1, + ], + '1.20' + ); + $this->assertSame('local_activity_preparation_mismatch', $refused['reason']); + $this->assertSame(2, $run->historyEvents()->count()); + $this->assertSame(TaskStatus::Leased, $task->refresh()->status); + } + + public function testRetryContractRenewalAndHeartbeatRefuseUnbindableTimestampsBeforeWrites(): void + { + config()->set('workflows.v2.workflow_task_lease_seconds', 300); + [$run, $task] = $this->newClaim(); + $latest = Carbon::instance(PreparedLocalRetryTimestamp::latestForConnection(ConfiguredV2Models::connection())); + $task->forceFill([ + 'lease_expires_at' => $latest, + ])->save(); + Carbon::setTestNow($latest->copy()->subMicroseconds(2500000)); + try { + $prepared = $this->prepareRetryContract( + $task, + [ + 'schedule_to_close_timeout' => null, + 'heartbeat_timeout' => 2, + ] + ); + $this->assertTrue($prepared['prepared']); + $execution = $run->activityExecutions() + ->sole(); + $attempt = ActivityAttempt::query()->findOrFail($prepared['activity_attempt_id']); + $executionBefore = $execution->getAttributes(); + $attemptBefore = $attempt->getAttributes(); + $taskBefore = $task->refresh() + ->getAttributes(); + Carbon::setTestNow($latest->copy()->subMicroseconds(1500000)); + foreach ([ + static fn () => PortableLocalActivityControl::heartbeatWithRetryPolicy( + $attempt->id, + 'portable-worker', + 1, + [ + 'step' => 1, + ], + '1.20' + ), + static fn () => PortableLocalActivityControl::pollWithRetryPolicy( + $attempt->id, + 'portable-worker', + 1, + true, + '1.20' + ), + ] as $operation) { + try { + $operation(); + $this->fail('Published an unbindable heartbeat or renewal timestamp.'); + } catch (\InvalidArgumentException) { + $this->addToAssertionCount(1); + } + $this->assertSame($executionBefore, $execution->refresh()->getAttributes()); + $this->assertSame($attemptBefore, $attempt->refresh()->getAttributes()); + $this->assertSame($taskBefore, $task->refresh()->getAttributes()); + $this->assertSame(2, $run->historyEvents()->count()); + } + } finally { + Carbon::setTestNow(); + } + } + public function testRetryContractCompletionKeepsItsClosedReceiptWithoutARetryDecision(): void { [$run, $task] = $this->newClaim(); From bc5dbf8d2c13511ad75a191c698617acc780f7e5 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 6 Nov 2026 00:52:19 +0000 Subject: [PATCH 11/15] Use the closed heartbeat progress shape in supervisor controls --- ...V2PortableLocalActivityPreparationTest.php | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php index 6d8bd5d3..90c8c066 100644 --- a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php +++ b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php @@ -454,6 +454,19 @@ public function testRetryContractSupervisorAndHeartbeatKeepTheirSeparateAuthorit ] ); $attemptId = $prepared['activity_attempt_id']; + $this->assertSame( + 'invalid_local_activity_heartbeat', + PortableLocalActivityControl::heartbeatWithRetryPolicy( + $attemptId, + 'portable-worker', + 1, + [ + 'step' => 1, + ], + '1.20' + )['reason'] + ); + $this->assertSame(2, $run->historyEvents()->count()); $this->assertSame('local_activity_preparation_mismatch', PortableLocalActivityControl::poll( $attemptId, 'portable-worker', @@ -483,7 +496,7 @@ public function testRetryContractSupervisorAndHeartbeatKeepTheirSeparateAuthorit 'portable-worker', 1, [ - 'step' => 1, + 'current' => 1, ], '1.20' ); @@ -530,7 +543,7 @@ public function testRetryContractSupervisorDoesNotBorrowLegacyOrForeignPreparati 'foreign-worker', 1, [ - 'step' => 1, + 'current' => 1, ], '1.20' ); @@ -571,7 +584,7 @@ public function testRetryContractRenewalAndHeartbeatRefuseUnbindableTimestampsBe 'portable-worker', 1, [ - 'step' => 1, + 'current' => 1, ], '1.20' ), From f03d5fcf2f83bd9c597f07ace9186559131fd464 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 6 Nov 2026 02:26:59 +0000 Subject: [PATCH 12/15] Refactor prepared retry selection around immutable timestamp bounds --- ...PreparedLocalActivityContractException.php | 12 ++ src/V2/Support/LocalActivityExecutor.php | 27 ++- src/V2/Support/PreparedLocalRetryDecision.php | 197 ++++++++++++------ src/V2/Support/PreparedLocalRetryWindow.php | 34 +++ .../V2/PreparedLocalRetryDecisionTest.php | 90 ++++---- 5 files changed, 241 insertions(+), 119 deletions(-) create mode 100644 src/V2/Exceptions/PreparedLocalActivityContractException.php create mode 100644 src/V2/Support/PreparedLocalRetryWindow.php diff --git a/src/V2/Exceptions/PreparedLocalActivityContractException.php b/src/V2/Exceptions/PreparedLocalActivityContractException.php new file mode 100644 index 00000000..234ea52b --- /dev/null +++ b/src/V2/Exceptions/PreparedLocalActivityContractException.php @@ -0,0 +1,12 @@ +activity_options['local_activity_contract_version'] ?? null) === 2 && (($preparation['version'] ?? null) !== 2 || ($preparation['local_activity_contract_version'] ?? null) !== 2 || $preparedAt === null)) { - throw new RuntimeException( + throw new PreparedLocalActivityContractException( 'Prepared retry attempts require their versioned preparation and locked sample.' ); } @@ -184,14 +185,16 @@ public function expirePortableRetry( ): WorkflowHistoryEvent { if (! in_array($localContractVersion, [1, 2], true) || ($execution->activity_options['local_activity_contract_version'] ?? 1) !== $localContractVersion) { - throw new RuntimeException('Prepared retry expiry requires its original contract.'); + throw new PreparedLocalActivityContractException('Prepared retry expiry requires its original contract.'); } $decision = null; $metadata = []; if ($localContractVersion === 2) { $attempt = self::currentAttempt($execution); if (! $attempt instanceof ActivityAttempt || $sample === null) { - throw new RuntimeException('Prepared retry expiry requires its original attempt and locked sample.'); + throw new PreparedLocalActivityContractException( + 'Prepared retry expiry requires its original attempt and locked sample.' + ); } $decision = self::selectPortableRetryDecision( $execution, @@ -680,8 +683,10 @@ private static function selectPortableRetryDecision( $attempt->attempt_number, $report, $failureTime, - $execution->schedule_to_close_deadline_at, - PreparedLocalRetryTimestamp::latestForConnection($connection), + new PreparedLocalRetryWindow( + $execution->schedule_to_close_deadline_at, + PreparedLocalRetryTimestamp::latestForConnection($connection) + ), $timeoutKind, $nonRetryable, ); @@ -728,11 +733,13 @@ private static function matchesStoredRetryReceipt( $execution->retry_policy, $attempt->attempt_number, $report, - $execution->schedule_to_close_deadline_at, - PreparedLocalRetryTimestamp::latestForConnection($connection), + new PreparedLocalRetryWindow( + $execution->schedule_to_close_deadline_at, + PreparedLocalRetryTimestamp::latestForConnection($connection), + PreparedLocalRetryTimestamp::earliestForDriver($connection->getDriverName()) + ), $stored, $nonRetryable, - PreparedLocalRetryTimestamp::earliestForDriver($connection->getDriverName()), ); } @@ -772,7 +779,9 @@ private function prepareAttempt( $execution = $this->createExecution($lockedRun, $lockedTask, $sequence, $activityCall); } if (($execution->activity_options['local_activity_contract_version'] ?? null) === 2) { - throw new RuntimeException('Prepared retry execution requires its versioned portable entry.'); + throw new PreparedLocalActivityContractException( + 'Prepared retry execution requires its versioned portable entry.' + ); } if (! LocalActivityRuntime::isExecution($execution)) { diff --git a/src/V2/Support/PreparedLocalRetryDecision.php b/src/V2/Support/PreparedLocalRetryDecision.php index 4fb093b9..23deafec 100644 --- a/src/V2/Support/PreparedLocalRetryDecision.php +++ b/src/V2/Support/PreparedLocalRetryDecision.php @@ -24,50 +24,22 @@ public static function matchesStored( array $snapshot, int $attemptNumber, array $report, - ?DateTimeInterface $originalDeadline, - DateTimeInterface $latestBindableTimestamp, + PreparedLocalRetryWindow $window, array $stored, bool $nonRetryable = false, - ?DateTimeInterface $earliestBindableTimestamp = null, ): bool { - $time = $stored['failure_time'] ?? null; + $sample = self::storedSample($stored['failure_time'] ?? null); $timeout = $stored['timeout_kind'] ?? null; - if (! is_string($time) || preg_match( - '/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{6}Z$/D', - $time - ) !== 1 - || ($timeout !== null && ! is_string($timeout))) { - return false; - } - $sample = DateTimeImmutable::createFromFormat('!Y-m-d\TH:i:s.u\Z', $time, new DateTimeZone('UTC')); - if (! $sample instanceof DateTimeImmutable || self::format($sample) !== $time) { + if ($sample === null || ($timeout !== null && ! is_string($timeout))) { return false; } try { - PreparedLocalRetryTimestamp::add($sample, 0, $latestBindableTimestamp, $earliestBindableTimestamp); - $expected = self::select( - $snapshot, - $attemptNumber, - $report, - $sample, - $originalDeadline, - $latestBindableTimestamp, - $timeout, - $nonRetryable - ); + PreparedLocalRetryTimestamp::add($sample, 0, $window->latest, $window->earliest); + $expected = self::select($snapshot, $attemptNumber, $report, $sample, $window, $timeout, $nonRetryable); } catch (InvalidArgumentException) { return false; } - if (count($stored) !== count($expected) || array_diff(array_keys($stored), array_keys($expected)) !== []) { - return false; - } - foreach ($expected as $key => $value) { - if ($stored[$key] !== $value) { - return false; - } - } - - return true; + return self::sameFields($expected, $stored); } /** @@ -84,12 +56,36 @@ public static function select( int $attemptNumber, array $report, DateTimeInterface $sample, - ?DateTimeInterface $originalDeadline, - DateTimeInterface $latestBindableTimestamp, + PreparedLocalRetryWindow $window, ?string $dueTimeoutKind = null, bool $nonRetryable = false, ): array { $snapshot = PreparedLocalRetryPolicy::validateSnapshot($snapshot); + self::validateReport($report, $attemptNumber, $dueTimeoutKind); + $sample = self::utc($sample); + $deadline = $window->deadline === null ? null : self::utc($window->deadline); + self::utc($window->latest); + if ($deadline !== null && $sample >= $deadline) { + $dueTimeoutKind = 'schedule_to_close'; + } + self::validateClock($sample, $window, $dueTimeoutKind, $report); + $decision = self::commonDecision($report, $sample, $window, $dueTimeoutKind); + $terminal = self::earlyTerminal($snapshot, $attemptNumber, $report, $dueTimeoutKind, $nonRetryable); + if ($terminal !== null) { + return [ + ...$decision, + 'terminal_classification' => $terminal, + ]; + } + + return self::selectedDelay($snapshot, $attemptNumber, $report, $sample, $window, $dueTimeoutKind, $decision); + } + + /** + * @param array $report + */ + private static function validateReport(array $report, int $attemptNumber, ?string $dueTimeoutKind): void + { $refusal = PreparedLocalActivityReport::refusal($report); if ($refusal !== null || $attemptNumber < 1 || $attemptNumber > PreparedLocalRetryPolicy::MAX_ATTEMPTS || ($dueTimeoutKind !== null && ! in_array( @@ -99,16 +95,21 @@ public static function select( ))) { throw new InvalidArgumentException($refusal ?? 'Invalid prepared retry decision input.'); } - $sample = self::utc($sample); - $deadline = $originalDeadline === null ? null : self::utc($originalDeadline); - $latest = self::utc($latestBindableTimestamp); - if ($sample > $latest) { + } + + /** + * @param array $report + */ + private static function validateClock( + DateTimeImmutable $sample, + PreparedLocalRetryWindow $window, + ?string $dueTimeoutKind, + array $report, + ): void { + if ($sample > $window->latest) { throw new InvalidArgumentException('Prepared failure sample exceeds the backend timestamp range.'); } - if ($deadline !== null && $sample >= $deadline) { - $dueTimeoutKind = 'schedule_to_close'; - } - if ($dueTimeoutKind === 'schedule_to_close' && ($deadline === null || $sample < $deadline)) { + if ($dueTimeoutKind === 'schedule_to_close' && ($window->deadline === null || $sample < $window->deadline)) { throw new InvalidArgumentException('Overall timeout requires the original due deadline.'); } if ($dueTimeoutKind === null && $report['outcome'] !== 'failed') { @@ -116,13 +117,27 @@ public static function select( 'A retry decision needs an application failure or an actually due timeout.' ); } + } + + /** @param array $report + * @return array + */ + private static function commonDecision( + array $report, + DateTimeImmutable $sample, + PreparedLocalRetryWindow $window, + ?string $dueTimeoutKind, + ): array { $engineTimeout = $dueTimeoutKind !== null; - $decision = [ + + return [ 'contract_version' => 2, 'snapshot_version' => 2, 'decision' => 'terminal', 'failure_time' => self::format($sample), - 'original_schedule_to_close_deadline_at' => $deadline === null ? null : self::format($deadline), + 'original_schedule_to_close_deadline_at' => $window->deadline === null ? null : self::format( + $window->deadline + ), 'failure_source' => $engineTimeout ? 'engine_timeout' : 'application', ...($engineTimeout ? [ 'timeout_kind' => $dueTimeoutKind, @@ -132,27 +147,44 @@ public static function select( 'requested_delay_seconds' => $report['next_retry_delay_seconds'], ] : []), ]; - if ($dueTimeoutKind === 'schedule_to_close') { - return [ - ...$decision, - 'terminal_classification' => 'engine_timeout', - ]; - } + } + + /** @param array $snapshot + * @param array $report + */ + private static function earlyTerminal( + array $snapshot, + int $attemptNumber, + array $report, + ?string $dueTimeoutKind, + bool $nonRetryable, + ): ?string { $type = $report['exception_type'] ?? RuntimeException::class; - if (! $engineTimeout && ($nonRetryable || ($report['non_retryable'] ?? false) - || in_array($type, $snapshot['non_retryable_error_types'], true))) { - return [ - ...$decision, - 'terminal_classification' => 'non_retryable', - ]; - } - if ($attemptNumber >= $snapshot['max_attempts']) { - return [ - ...$decision, - 'terminal_classification' => 'attempts_exhausted', - ]; - } - $override = ! $engineTimeout && array_key_exists('next_retry_delay_seconds', $report); + + return match (true) { + $dueTimeoutKind === 'schedule_to_close' => 'engine_timeout', + $dueTimeoutKind === null && ($nonRetryable || ($report['non_retryable'] ?? false) + || in_array($type, $snapshot['non_retryable_error_types'], true)) => 'non_retryable', + $attemptNumber >= $snapshot['max_attempts'] => 'attempts_exhausted', + default => null, + }; + } + + /** @param array $snapshot + * @param array $report + * @param array $decision + * @return array + */ + private static function selectedDelay( + array $snapshot, + int $attemptNumber, + array $report, + DateTimeImmutable $sample, + PreparedLocalRetryWindow $window, + ?string $dueTimeoutKind, + array $decision, + ): array { + $override = $dueTimeoutKind === null && array_key_exists('next_retry_delay_seconds', $report); $candidate = $override ? $report['next_retry_delay_seconds'] : PreparedLocalRetryPolicy::delayForRetry($snapshot, $attemptNumber); $decision = [ @@ -167,8 +199,8 @@ public static function select( 'terminal_classification' => 'retry_delay_exceeds_policy_cap', ]; } - if ($deadline !== null && $candidate >= intdiv( - self::microseconds($deadline) - self::microseconds($sample) + 999999, + if ($window->deadline !== null && $candidate >= intdiv( + self::microseconds($window->deadline) - self::microseconds($sample) + 999999, 1000000 )) { return [ @@ -178,7 +210,7 @@ public static function select( } // Check by division before multiplication or DateInterval construction. // Even the largest portable integer cannot wrap or become a float. - if ($candidate > intdiv(self::microseconds($latest) - self::microseconds($sample), 1000000)) { + if ($candidate > intdiv(self::microseconds($window->latest) - self::microseconds($sample), 1000000)) { throw new InvalidArgumentException('Prepared retry availability exceeds the backend timestamp range.'); } $available = $sample->add(new DateInterval('PT' . $candidate . 'S')); @@ -190,6 +222,33 @@ public static function select( ]; } + private static function storedSample(mixed $time): ?DateTimeImmutable + { + if (! is_string($time) || preg_match('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/D', $time) !== 1) { + return null; + } + $sample = DateTimeImmutable::createFromFormat('!Y-m-d\TH:i:s.u\Z', $time, new DateTimeZone('UTC')); + + return $sample instanceof DateTimeImmutable && self::format($sample) === $time ? $sample : null; + } + + /** @param array $expected + * @param array $stored + */ + private static function sameFields(array $expected, array $stored): bool + { + if (count($stored) !== count($expected) || array_diff(array_keys($stored), array_keys($expected)) !== []) { + return false; + } + foreach ($expected as $key => $value) { + if ($stored[$key] !== $value) { + return false; + } + } + + return true; + } + private static function utc(DateTimeInterface $value): DateTimeImmutable { $utc = DateTimeImmutable::createFromInterface($value)->setTimezone(new DateTimeZone('UTC')); diff --git a/src/V2/Support/PreparedLocalRetryWindow.php b/src/V2/Support/PreparedLocalRetryWindow.php new file mode 100644 index 00000000..c324cad2 --- /dev/null +++ b/src/V2/Support/PreparedLocalRetryWindow.php @@ -0,0 +1,34 @@ +deadline = $deadline === null ? null : DateTimeImmutable::createFromInterface($deadline)->setTimezone( + $utc + ); + $this->latest = DateTimeImmutable::createFromInterface($latest)->setTimezone($utc); + $this->earliest = $earliest === null ? null : DateTimeImmutable::createFromInterface($earliest)->setTimezone( + $utc + ); + } +} diff --git a/tests/Unit/V2/PreparedLocalRetryDecisionTest.php b/tests/Unit/V2/PreparedLocalRetryDecisionTest.php index 9f6acdc6..b3d450da 100644 --- a/tests/Unit/V2/PreparedLocalRetryDecisionTest.php +++ b/tests/Unit/V2/PreparedLocalRetryDecisionTest.php @@ -4,14 +4,33 @@ namespace Tests\Unit\V2; +use DateTime; use DateTimeImmutable; use InvalidArgumentException; use PHPUnit\Framework\TestCase; use Workflow\V2\Support\PreparedLocalRetryDecision; use Workflow\V2\Support\PreparedLocalRetryPolicy; +use Workflow\V2\Support\PreparedLocalRetryWindow; final class PreparedLocalRetryDecisionTest extends TestCase { + public function testOriginalWindowCopiesMutableTimestampsWithoutReadingAClock(): void + { + $deadline = new DateTime('2026-11-05T01:01:00.123456+01:00'); + $latest = new DateTime('9999-12-31T23:59:59.999999Z'); + $earliest = new DateTime('1970-01-01T00:00:01.000000Z'); + $window = new PreparedLocalRetryWindow($deadline, $latest, $earliest); + $deadline->modify('+1 hour'); + $latest->modify('-1 day'); + $earliest->modify('+1 second'); + self::assertSame('2026-11-05T00:01:00.123456+00:00', $window->deadline->format('Y-m-d\TH:i:s.uP')); + self::assertSame('9999-12-31T23:59:59.999999+00:00', $window->latest->format('Y-m-d\TH:i:s.uP')); + self::assertSame('1970-01-01T00:00:01.000000+00:00', $window->earliest->format('Y-m-d\TH:i:s.uP')); + $unbounded = new PreparedLocalRetryWindow(null, $window->latest); + self::assertNull($unbounded->deadline); + self::assertNull($unbounded->earliest); + } + public function testCanonicalApplicationSelectionProjections(): void { $vectors = json_decode(file_get_contents(dirname(__DIR__, 3) @@ -52,8 +71,7 @@ public function testCanonicalApplicationSelectionProjections(): void $input['completed_attempts'], $report, $sample, - $deadline, - new DateTimeImmutable('9999-12-31T23:59:59.999999Z') + new PreparedLocalRetryWindow($deadline, new DateTimeImmutable('9999-12-31T23:59:59.999999Z')), ); self::assertSame($case['expected']['decision'], $decision['decision'], $case['id']); if ($decision['decision'] === 'terminal') { @@ -344,9 +362,8 @@ public function testRetainedDecisionRequiresTheExactTypedClosedSelection(): void $this->snapshot(), 1, $report, - $deadline, - $latest, - $value + new PreparedLocalRetryWindow($deadline, $latest), + $value, ); self::assertTrue($matches(array_reverse($stored, true))); $missing = $stored; @@ -401,6 +418,10 @@ public function testRetainedDecisionRequiresTheExactTypedClosedSelection(): void ...$stored, 'failure_time' => '2026-11-05T00:00:00Z', ], + [ + ...$stored, + 'failure_time' => '٢٠٢٦-11-05T00:00:00.000000Z', + ], [ ...$stored, 'timeout_kind' => [], @@ -419,9 +440,8 @@ public function testRetainedDecisionRequiresTheExactTypedClosedSelection(): void ...$report, 'next_retry_delay_seconds' => 1, ], - $deadline, - $latest, - $stored + new PreparedLocalRetryWindow($deadline, $latest), + $stored, )); } @@ -438,20 +458,18 @@ public function testRetainedTerminalDecisionsCannotGainAnExecutableTimestampOrUn $this->snapshot(), 1, $report, - null, - $latest, - $stored + new PreparedLocalRetryWindow(null, $latest), + $stored, )); self::assertFalse(PreparedLocalRetryDecision::matchesStored( $this->snapshot(), 1, $report, - null, - $latest, + new PreparedLocalRetryWindow(null, $latest), [ ...$stored, 'retry_available_at' => '2026-11-05T00:00:06.000000Z', - ] + ], )); $report = [ 'outcome' => 'timed_out', @@ -462,28 +480,25 @@ public function testRetainedTerminalDecisionsCannotGainAnExecutableTimestampOrUn $this->snapshot(), 1, $report, - $deadline, - $latest, - $stored + new PreparedLocalRetryWindow($deadline, $latest), + $stored, )); self::assertFalse(PreparedLocalRetryDecision::matchesStored( $this->snapshot(), 1, $report, - $deadline, - $latest, + new PreparedLocalRetryWindow($deadline, $latest), [ ...$stored, 'delay_source' => 'policy', - ] + ], )); self::assertFalse(PreparedLocalRetryDecision::matchesStored( $this->snapshot(), 1, $report, - $deadline->modify('+1 second'), - $latest, - $stored + new PreparedLocalRetryWindow($deadline->modify('+1 second'), $latest), + $stored, )); } @@ -500,37 +515,32 @@ public function testRetainedDecisionKeepsBackendBoundsAndEngineNonRetryableClass $this->snapshot(), 1, $report, - null, - $latest, + new PreparedLocalRetryWindow(null, $latest), $stored, - true + true, )); self::assertFalse(PreparedLocalRetryDecision::matchesStored( $this->snapshot(), 1, $report, - null, - $latest, - $stored + new PreparedLocalRetryWindow(null, $latest), + $stored, )); self::assertFalse(PreparedLocalRetryDecision::matchesStored( $this->snapshot(), 1, $report, - null, - $sample->modify('-1 microsecond'), + new PreparedLocalRetryWindow(null, $sample->modify('-1 microsecond')), $stored, - true + true, )); self::assertFalse(PreparedLocalRetryDecision::matchesStored( $this->snapshot(), 1, $report, - null, - $latest, + new PreparedLocalRetryWindow(null, $latest, $sample->modify('+1 microsecond')), $stored, true, - $sample->modify('+1 microsecond') )); $snapshot = $this->snapshot(); $snapshot['snapshot_version'] = 1; @@ -538,10 +548,9 @@ public function testRetainedDecisionKeepsBackendBoundsAndEngineNonRetryableClass $snapshot, 1, $report, - null, - $latest, + new PreparedLocalRetryWindow(null, $latest), $stored, - true + true, )); } @@ -575,10 +584,9 @@ private function select( $attempt, $report, $sample ?? new DateTimeImmutable('2026-11-05T00:00:00.000000Z'), - $deadline, - $latest ?? new DateTimeImmutable('9999-12-31T23:59:59.999999Z'), + new PreparedLocalRetryWindow($deadline, $latest ?? new DateTimeImmutable('9999-12-31T23:59:59.999999Z')), $timeout, - $nonRetryable + $nonRetryable, ); } } From 11ef5a8f9ec866f11be5c626b57cf42d395e0bf8 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:11:45 +0000 Subject: [PATCH 13/15] Preserve engine interruption decisions through prepared retry recovery --- src/V2/Support/LocalActivityExecutor.php | 86 +++ .../PortableLocalActivityPreparation.php | 599 ++++++++++++------ src/V2/Support/PreparedLocalRetryDecision.php | 101 ++- ...V2PortableLocalActivityPreparationTest.php | 7 + .../V2PortableLocalActivityRecoveryTest.php | 486 +++++++++++++- .../V2/PreparedLocalActivityReportTest.php | 6 +- .../V2/PreparedLocalRetryDecisionTest.php | 140 ++++ 7 files changed, 1221 insertions(+), 204 deletions(-) diff --git a/src/V2/Support/LocalActivityExecutor.php b/src/V2/Support/LocalActivityExecutor.php index e9554c58..0940f542 100644 --- a/src/V2/Support/LocalActivityExecutor.php +++ b/src/V2/Support/LocalActivityExecutor.php @@ -234,6 +234,9 @@ public function recoverPortableAttempt( ActivityAttempt $attempt, array $recovery, ): array { + if (($execution->activity_options['local_activity_contract_version'] ?? null) === 2) { + throw new PreparedLocalActivityContractException('Prepared retry recovery requires its versioned entry.'); + } $metadata = [ 'worker_attempt_id' => $attempt->worker_attempt_id, 'local_recovery' => $recovery, @@ -243,6 +246,89 @@ public function recoverPortableAttempt( $outcome = $timeoutKind === null ? $this->recordInterruptedAttempt($run, $task, $execution, $metadata) : $this->recordTimeoutOutcome($run, $task, $execution, $timeoutKind, $metadata); + return self::finishPortableRecovery($run, $task, $attempt, $outcome); + } + + /** + * @internal The caller has validated original snapshot/Started authority, + * locked the canonical graph and proved loss of the original claim using + * this one sample. Recovery does not admit or join a callback. + * @param array $recovery + * @return array{event: WorkflowHistoryEvent, next_task: WorkflowTask|null} + */ + public function recoverPortableRetryAttempt( + WorkflowRun $run, + WorkflowTask $task, + ActivityExecution $execution, + ActivityAttempt $attempt, + array $recovery, + Carbon $sample, + ): array { + if (($execution->activity_options['local_activity_contract_version'] ?? null) !== 2) { + throw new PreparedLocalActivityContractException('Prepared retry recovery requires its original contract.'); + } + $connection = ConfiguredV2Models::connection(); + PreparedLocalRetryTimestamp::addForConnection($sample, 0, $connection); + $timeout = self::timeoutKind($execution, $sample); + $decision = PreparedLocalRetryDecision::selectInterruption( + $execution->retry_policy, + $attempt->attempt_number, + $sample, + new PreparedLocalRetryWindow( + $execution->schedule_to_close_deadline_at, + PreparedLocalRetryTimestamp::latestForConnection($connection) + ), + $timeout + ); + $metadata = [ + 'worker_attempt_id' => $attempt->worker_attempt_id, + 'local_recovery' => $recovery, + 'retry_disposition' => $decision, + ]; + self::closeAttempt($attempt, ActivityAttemptStatus::Expired, $sample); + if ($timeout !== null) { + $outcome = $this->recordTimeoutOutcome($run, $task, $execution, $timeout, $metadata, $decision, $sample); + } else { + $throwable = new RuntimeException(PreparedLocalRetryDecision::INTERRUPTION_MESSAGE); + $outcome = $decision['decision'] === 'scheduled' ? [ + 'event' => null, + 'next_task' => $this->scheduleRetry( + $run, + $task, + $execution, + $attempt, + $throwable, + LocalActivityRuntime::RETRY_REASON_COLD_REPLAY, + historyMetadata: $metadata, + retryDecision: $decision, + failureTime: $sample, + ), + ] : [ + 'event' => $this->recordTerminalFailure( + $run, + $task, + $execution, + $attempt, + $throwable, + $metadata, + $sample + ), + 'next_task' => null, + ]; + } + + return self::finishPortableRecovery($run, $task, $attempt, $outcome); + } + + /** @param array{event: WorkflowHistoryEvent|null, next_task: WorkflowTask|null} $outcome + * @return array{event: WorkflowHistoryEvent, next_task: WorkflowTask|null} + */ + private static function finishPortableRecovery( + WorkflowRun $run, + WorkflowTask $task, + ActivityAttempt $attempt, + array $outcome, + ): array { if ($outcome['next_task'] instanceof WorkflowTask) { $task->forceFill([ 'status' => TaskStatus::Completed, diff --git a/src/V2/Support/PortableLocalActivityPreparation.php b/src/V2/Support/PortableLocalActivityPreparation.php index a924f57d..50ac4559 100644 --- a/src/V2/Support/PortableLocalActivityPreparation.php +++ b/src/V2/Support/PortableLocalActivityPreparation.php @@ -4,6 +4,8 @@ namespace Workflow\V2\Support; +use DateTimeImmutable; +use DateTimeZone; use Illuminate\Support\Carbon; use Illuminate\Validation\ValidationException; use InvalidArgumentException; @@ -156,201 +158,41 @@ public static function recover( array $descriptor, string $protocolVersion = WorkerProtocolVersion::VERSION, ): array { - $refused = static fn (string $reason): array => [ - 'recovered' => false, - 'duplicate' => false, - 'reason' => $reason, - 'claim_released' => false, - 'created_task_ids' => [], - ]; - if (preg_match('/^[0-9]+\.[0-9]+$/D', $protocolVersion) !== 1 - || version_compare($protocolVersion, self::MINIMUM_PROTOCOL_VERSION, '<')) { - return $refused('local_activity_recovery_requires_protocol_1_20'); - } - if ($workflowTaskAttempt < 1 || $sequence < 1 || $leaseOwner === '') { - return $refused('invalid_local_activity_recovery'); - } - try { - $fingerprint = hash('sha256', json_encode(self::normalizeDescriptor($descriptor), JSON_THROW_ON_ERROR)); - } catch (ValidationException|JsonException) { - return $refused('invalid_local_activity_recovery'); - } - /** @var WorkflowTask|null $snapshotTask */ - $snapshotTask = ConfiguredV2Models::query('task_model', WorkflowTask::class)->find($taskId); - /** @var ActivityExecution|null $snapshotExecution */ - $snapshotExecution = $snapshotTask === null ? null : ActivityExecution::query() - ->where('workflow_run_id', $snapshotTask->workflow_run_id) - ->where('sequence', $sequence) - ->first(); - if ($snapshotTask === null || $snapshotExecution === null - || ! is_string($snapshotExecution->current_attempt_id)) { - return $refused('local_activity_preparation_not_found'); - } - return ConfiguredV2Models::connection()->transaction(static function () use ( - $snapshotTask, - $snapshotExecution, + return self::recoverForContract( + 1, $taskId, $leaseOwner, $workflowTaskAttempt, $sequence, - $fingerprint, - $refused, - ): array { - $rows = ActivityRowLockOrder::lockForAttempt($snapshotExecution->current_attempt_id); - $execution = $rows['execution']; - $attempt = $rows['attempt']; - /** @var WorkflowRun|null $run */ - $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) - ->lockForUpdate() - ->find($snapshotTask->workflow_run_id); - /** @var WorkflowTask|null $task */ - $task = ConfiguredV2Models::query('task_model', WorkflowTask::class) - ->lockForUpdate() - ->find($taskId); - if (! $execution instanceof ActivityExecution || ! $attempt instanceof ActivityAttempt - || $execution->id !== $snapshotExecution->id || $execution->sequence !== $sequence - || $run === null || $execution->workflow_run_id !== $run->id - || $task === null || $task->workflow_run_id !== $run->id) { - return $refused('local_activity_preparation_not_found'); - } - $receipt = $run->historyEvents() - ->whereIn('event_type', [HistoryEventType::ActivityRetryScheduled, HistoryEventType::ActivityFailed, - HistoryEventType::ActivityTimedOut]) - ->where('payload->sequence', $sequence) - ->where('payload->local_recovery->workflow_task_id', $taskId) - ->where('payload->local_recovery->workflow_task_attempt', $workflowTaskAttempt) - ->where('payload->local_recovery->lease_owner', $leaseOwner) - ->orderBy('sequence') - ->first(); - if ($receipt instanceof WorkflowHistoryEvent) { - if (($receipt->payload['local_recovery']['version'] ?? null) !== 1 - || ($receipt->payload['local_recovery']['descriptor_fingerprint'] ?? null) !== $fingerprint - || ($receipt->payload['activity_execution_id'] ?? null) !== $execution->id - || $receipt->workflow_task_id !== $taskId - || ($receipt->payload['task']['id'] ?? null) !== $taskId - || ($receipt->payload['task']['lease_owner'] ?? null) !== $leaseOwner - || ($receipt->payload['task']['attempt_count'] ?? null) !== $workflowTaskAttempt - || ($receipt->payload['local_recovery']['callback_stop_state'] ?? null) !== 'unknown') { - return $refused('local_activity_recovery_mismatch'); - } - return self::recoveryReceipt($receipt, true); - } - if ($task->task_type !== TaskType::Workflow || $task->status !== TaskStatus::Leased - || $task->lease_owner !== $leaseOwner || $task->attempt_count !== $workflowTaskAttempt) { - return $refused('workflow_claim_mismatch'); - } - if ($task->lease_expires_at === null || now()->gte($task->lease_expires_at)) { - return $refused('workflow_claim_expired'); - } - $started = $run->historyEvents() - ->where('event_type', HistoryEventType::ActivityStarted) - ->where('payload->activity_attempt_id', $attempt->id) - ->first(); - $originalTaskAttempt = $started?->payload['local_preparation']['workflow_task_attempt'] ?? null; - if (! is_int($originalTaskAttempt) - || self::originalStart($run, $execution, $attempt, $attempt->lease_owner, $originalTaskAttempt) === null - || ($started->payload['local_preparation']['descriptor_fingerprint'] ?? null) !== $fingerprint - || $execution->current_attempt_id !== $attempt->id || $execution->attempt_count !== $attempt->attempt_number) { - return $refused('local_activity_preparation_mismatch'); - } - // Accepted cancellation may fence immediately, without waiting for - // lease expiry. The replacement claim and root budget stay intact. - $cleanup = PortableLocalActivityCleanup::isExecution($run, $execution, $started); - if ($cleanup && now()->gte(PortableLocalActivityCleanup::deadline($execution))) { - return $refused(PortableLocalActivityCleanup::isScoped($execution) - ? 'local_activity_cleanup_deadline_expired' : 'cancellation_deadline_expired'); - } - if ($run->cancellation_request_command_id !== null && (! $cleanup - || ! PortableLocalActivityCleanup::belongsToRunRequest( - $run, - $execution->activity_options['cancellation_cleanup'] - ))) { - $cancelled = $run->historyEvents() - ->where('event_type', HistoryEventType::ActivityCancelled) - ->where('payload->activity_attempt_id', $attempt->id) - ->where('payload->workflow_command_id', $run->cancellation_request_command_id) - ->first(); - if (! $cancelled instanceof WorkflowHistoryEvent) { - if ($execution->status !== ActivityStatus::Running || $attempt->status !== ActivityAttemptStatus::Running) { - return $refused('stale_activity_attempt'); - } - $cancelled = ActivityCancellation::record( - $run, - $execution, - command: $run->cancellation_request_command_id - ); - } - return [ - ...$refused('cancellation_requested'), - 'fenced' => true, - 'cancellation_history_event_id' => $cancelled?->id, - ]; - } - if ($run->status->isTerminal()) { - return $refused('run_closed'); - } - if (($run->execution_deadline_at !== null && now()->gte($run->execution_deadline_at)) - || ($run->run_deadline_at !== null && now()->gte($run->run_deadline_at))) { - return $refused('run_deadline_expired'); - } - if ($execution->status !== ActivityStatus::Running || $attempt->status !== ActivityAttemptStatus::Running) { - return $refused('local_activity_previous_attempt_unresolved'); - } - if (PortableLocalActivityCleanup::currentAuthorityExpired($run, $execution)) { - return $refused('local_activity_cleanup_authority_expired'); - } - if ($attempt->lease_expires_at === null || now()->lt($attempt->lease_expires_at)) { - return $refused('local_activity_previous_attempt_live'); - } - /** @var WorkflowTask|null $originalTask */ - $originalTask = ConfiguredV2Models::query('task_model', WorkflowTask::class) - ->lockForUpdate() - ->find($attempt->workflow_task_id); - if ($originalTask === null || $originalTask->workflow_run_id !== $run->id - || $originalTask->task_type !== TaskType::Workflow) { - return $refused('original_workflow_claim_not_found'); - } - if ($originalTask->id === $task->id) { - if ($task->attempt_count <= $originalTaskAttempt) { - return $refused('local_activity_original_claim_not_reclaimed'); - } - } elseif ($originalTask->status === TaskStatus::Leased && $originalTask->lease_expires_at !== null - && now() - ->lt($originalTask->lease_expires_at)) { - return $refused('local_activity_original_claim_live'); - } - $recovery = [ - 'version' => 1, - 'descriptor_fingerprint' => $fingerprint, - 'workflow_task_id' => $task->id, - 'workflow_task_attempt' => $workflowTaskAttempt, - 'lease_owner' => $leaseOwner, - 'original_workflow_task_id' => $originalTask->id, - 'original_workflow_task_attempt' => $originalTaskAttempt, - 'original_lease_owner' => $attempt->lease_owner, - 'original_lease_expires_at' => $attempt->lease_expires_at->toISOString(), - 'callback_stop_state' => 'unknown', - ]; - if ($originalTask->id !== $task->id && in_array( - $originalTask->status, - [TaskStatus::Ready, TaskStatus::Leased], - true - )) { - $originalTask->forceFill([ - 'status' => TaskStatus::Completed, - 'lease_expires_at' => null, - ])->save(); - } - $outcome = app(LocalActivityExecutor::class)->recoverPortableAttempt( - $run, - $task, - $execution, - $attempt, - $recovery - ); + $descriptor, + $protocolVersion + ); + } - return self::recoveryReceipt($outcome['event'], false); - }, 5); + /** + * @internal Unadvertised recovery entry. The adapter must prove its + * installed retry role and the retained original capability/contract. + * Recovery fences publication; it does not prove physical callback stop. + * @param array $descriptor + * @return array + */ + public static function recoverWithRetryPolicy( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + array $descriptor, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array { + return self::recoverForContract( + 2, + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $sequence, + $descriptor, + $protocolVersion + ); } /** @param array $descriptor @@ -553,6 +395,380 @@ public static function originalStart( return $started; } + private static function findRecoveryEvent( + string $runId, + string $taskId, + int $epoch, + string $owner, + int $sequence, + ): ?WorkflowHistoryEvent { + return ConfiguredV2Models::query('history_event_model', WorkflowHistoryEvent::class) + ->where('workflow_run_id', $runId) + ->whereIn('event_type', [HistoryEventType::ActivityRetryScheduled, HistoryEventType::ActivityFailed, + HistoryEventType::ActivityTimedOut]) + ->where('payload->sequence', $sequence) + ->where('payload->local_recovery->workflow_task_id', $taskId) + ->where('payload->local_recovery->workflow_task_attempt', $epoch) + ->where('payload->local_recovery->lease_owner', $owner) + ->orderBy('sequence') + ->first(); + } + + private static function recoveryStart( + WorkflowRun $run, + ActivityExecution $execution, + ActivityAttempt $attempt, + string $fingerprint, + int $contract, + ): ?WorkflowHistoryEvent { + $started = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted) + ->where('payload->activity_attempt_id', $attempt->id) + ->first(); + $epoch = $started?->payload['local_preparation']['workflow_task_attempt'] ?? null; + if (! is_int( + $epoch + ) || ($started->payload['local_preparation']['descriptor_fingerprint'] ?? null) !== $fingerprint) { + return null; + } + + return self::originalStart($run, $execution, $attempt, $attempt->lease_owner, $epoch, $contract); + } + + private static function matchesRetryRecoveryReceipt( + WorkflowHistoryEvent $event, + ActivityExecution $execution, + ActivityAttempt $attempt, + ?WorkflowHistoryEvent $started, + ): bool { + $payload = $event->payload; + $decision = $payload['retry_disposition'] ?? null; + $recovery = $payload['local_recovery'] ?? null; + if ($started === null || ! is_array($decision) || ! is_array($recovery)) { + return false; + } + $connection = ConfiguredV2Models::connection(); + if (! PreparedLocalRetryDecision::matchesStoredInterruption( + $execution->retry_policy, + $attempt->attempt_number, + new PreparedLocalRetryWindow( + $execution->schedule_to_close_deadline_at, + PreparedLocalRetryTimestamp::latestForConnection($connection), + PreparedLocalRetryTimestamp::earliestForDriver($connection->getDriverName()) + ), + $decision + ) || ! self::recoveryLeaseMatches($recovery['original_lease_expires_at'] ?? null, $decision['failure_time'])) { + return false; + } + $expected = [ + 'version' => 1, + 'descriptor_fingerprint' => $started->payload['local_preparation']['descriptor_fingerprint'], + 'workflow_task_id' => $event->workflow_task_id, + 'workflow_task_attempt' => $payload['task']['attempt_count'], + 'lease_owner' => $payload['task']['lease_owner'], + 'original_workflow_task_id' => $attempt->workflow_task_id, + 'original_workflow_task_attempt' => $started->payload['local_preparation']['workflow_task_attempt'], + 'original_lease_owner' => $attempt->lease_owner, + 'original_lease_expires_at' => $recovery['original_lease_expires_at'], + 'callback_stop_state' => 'unknown', + ]; + ksort($expected); + ksort($recovery); + + return $expected === $recovery + && ($payload['activity_attempt_id'] ?? null) === $attempt->id + && ($payload['worker_attempt_id'] ?? null) === $attempt->worker_attempt_id + && ($payload[$event->event_type === HistoryEventType::ActivityRetryScheduled ? 'retry_after_attempt' : 'attempt_number'] ?? null) === $attempt->attempt_number + && ($payload['activity_attempt']['status'] ?? null) === ActivityAttemptStatus::Expired->value + && ($payload['activity_attempt']['closed_at'] ?? null) === $decision['failure_time'] + && self::recoveryEventMatches($event, $decision); + } + + private static function recoveryLeaseMatches(mixed $expiry, string $failure): bool + { + if (! is_string($expiry) || preg_match('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/D', $expiry) !== 1) { + return false; + } + $lease = DateTimeImmutable::createFromFormat('!Y-m-d\TH:i:s.u\Z', $expiry, new DateTimeZone('UTC')); + + return $lease instanceof DateTimeImmutable && $lease->format('Y-m-d\TH:i:s.u\Z') === $expiry + && $expiry <= $failure; + } + + /** + * @param array $decision + */ + private static function recoveryEventMatches(WorkflowHistoryEvent $event, array $decision): bool + { + $payload = $event->payload; + $timeout = $decision['failure_source'] === 'engine_timeout'; + $scheduled = $decision['decision'] === 'scheduled'; + $type = $scheduled ? HistoryEventType::ActivityRetryScheduled + : ($timeout ? HistoryEventType::ActivityTimedOut : HistoryEventType::ActivityFailed); + + return $event->event_type === $type + && ($payload['timeout_kind'] ?? null) === ($decision['timeout_kind'] ?? null) + && (! $scheduled || (($payload['retry_reason'] ?? null) === ($timeout ? LocalActivityRuntime::RETRY_REASON_TIMEOUT : LocalActivityRuntime::RETRY_REASON_COLD_REPLAY) + && is_string($payload['retry_task_id'] ?? null) && $payload['retry_task_id'] !== '' + && ($payload['retry_backoff_seconds'] ?? null) === $decision['candidate_delay_seconds'] + && ($payload['retry_available_at'] ?? null) === $decision['retry_available_at'])) + && ($scheduled || ! array_key_exists('retry_task_id', $payload)) + && ($timeout || (($payload['exception_class'] ?? null) === \RuntimeException::class + && ($payload['message'] ?? null) === PreparedLocalRetryDecision::INTERRUPTION_MESSAGE)); + } + + /** @param array $descriptor + * @return array + */ + private static function recoverForContract( + int $localContractVersion, + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + array $descriptor, + string $protocolVersion, + ): array { + $refused = static fn (string $reason): array => [ + 'recovered' => false, + 'duplicate' => false, + 'reason' => $reason, + 'claim_released' => false, + 'created_task_ids' => [], + ]; + if (preg_match('/^[0-9]+\.[0-9]+$/D', $protocolVersion) !== 1 + || version_compare($protocolVersion, self::MINIMUM_PROTOCOL_VERSION, '<')) { + return $refused('local_activity_recovery_requires_protocol_1_20'); + } + if ($workflowTaskAttempt < 1 || $sequence < 1 || $leaseOwner === '') { + return $refused('invalid_local_activity_recovery'); + } + try { + $fingerprint = hash( + 'sha256', + json_encode(self::normalizeDescriptor($descriptor, $localContractVersion), JSON_THROW_ON_ERROR) + ); + } catch (ValidationException|JsonException|InvalidArgumentException) { + return $refused('invalid_local_activity_recovery'); + } + /** @var WorkflowTask|null $snapshotTask */ + $snapshotTask = ConfiguredV2Models::query('task_model', WorkflowTask::class)->find($taskId); + /** @var ActivityExecution|null $snapshotExecution */ + $snapshotExecution = $snapshotTask === null ? null : ActivityExecution::query() + ->where('workflow_run_id', $snapshotTask->workflow_run_id) + ->where('sequence', $sequence) + ->first(); + if ($snapshotTask === null || $snapshotExecution === null + || ! is_string($snapshotExecution->current_attempt_id)) { + return $refused('local_activity_preparation_not_found'); + } + $snapshotAttemptId = $snapshotExecution->current_attempt_id; + if ($localContractVersion === 2) { + $prior = self::findRecoveryEvent( + $snapshotTask->workflow_run_id, + $taskId, + $workflowTaskAttempt, + $leaseOwner, + $sequence + ); + if ($prior !== null) { + $snapshotAttemptId = $prior->payload['activity_attempt_id'] ?? null; + if (! is_string($snapshotAttemptId) || $snapshotAttemptId === '') { + return $refused('local_activity_recovery_mismatch'); + } + } + } + return ConfiguredV2Models::connection()->transaction(static function () use ( + $localContractVersion, + $snapshotAttemptId, + $snapshotTask, + $snapshotExecution, + $taskId, + $leaseOwner, + $workflowTaskAttempt, + $sequence, + $fingerprint, + $refused, + ): array { + $rows = ActivityRowLockOrder::lockForAttempt($snapshotAttemptId); + $execution = $rows['execution']; + $attempt = $rows['attempt']; + /** @var WorkflowRun|null $run */ + $run = ConfiguredV2Models::query('run_model', WorkflowRun::class) + ->lockForUpdate() + ->find($snapshotTask->workflow_run_id); + /** @var WorkflowTask|null $task */ + $task = ConfiguredV2Models::query('task_model', WorkflowTask::class) + ->lockForUpdate() + ->find($taskId); + if (! $execution instanceof ActivityExecution || ! $attempt instanceof ActivityAttempt + || $execution->id !== $snapshotExecution->id || $execution->sequence !== $sequence + || $run === null || $execution->workflow_run_id !== $run->id + || $task === null || $task->workflow_run_id !== $run->id) { + return $refused('local_activity_preparation_not_found'); + } + $started = null; + $originalTask = null; + if ($localContractVersion === 2) { + $started = self::recoveryStart($run, $execution, $attempt, $fingerprint, 2); + if ($started === null) { + return $refused('local_activity_preparation_mismatch'); + } + $originalTask = ConfiguredV2Models::query('task_model', WorkflowTask::class) + ->lockForUpdate() + ->find($attempt->workflow_task_id); + if (! $originalTask instanceof WorkflowTask || $originalTask->workflow_run_id !== $run->id + || $originalTask->task_type !== TaskType::Workflow) { + return $refused('original_workflow_claim_not_found'); + } + } + $receipt = self::findRecoveryEvent($run->id, $taskId, $workflowTaskAttempt, $leaseOwner, $sequence); + if ($receipt instanceof WorkflowHistoryEvent) { + if (($receipt->payload['local_recovery']['version'] ?? null) !== 1 + || ($receipt->payload['local_recovery']['descriptor_fingerprint'] ?? null) !== $fingerprint + || ($receipt->payload['activity_execution_id'] ?? null) !== $execution->id + || $receipt->workflow_task_id !== $taskId + || ($receipt->payload['task']['id'] ?? null) !== $taskId + || ($receipt->payload['task']['lease_owner'] ?? null) !== $leaseOwner + || ($receipt->payload['task']['attempt_count'] ?? null) !== $workflowTaskAttempt + || ($receipt->payload['local_recovery']['callback_stop_state'] ?? null) !== 'unknown' + || ($localContractVersion === 2 + && ! self::matchesRetryRecoveryReceipt($receipt, $execution, $attempt, $started))) { + return $refused('local_activity_recovery_mismatch'); + } + return self::recoveryReceipt($receipt, true); + } + $sample = $localContractVersion === 2 ? now() : null; + if ($sample !== null) { + PreparedLocalRetryTimestamp::addForConnection($sample, 0, ConfiguredV2Models::connection()); + } + if ($task->task_type !== TaskType::Workflow || $task->status !== TaskStatus::Leased + || $task->lease_owner !== $leaseOwner || $task->attempt_count !== $workflowTaskAttempt) { + return $refused('workflow_claim_mismatch'); + } + if ($task->lease_expires_at === null || ($sample ?? now())->gte($task->lease_expires_at)) { + return $refused('workflow_claim_expired'); + } + $started ??= $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityStarted) + ->where('payload->activity_attempt_id', $attempt->id) + ->first(); + $originalTaskAttempt = $started?->payload['local_preparation']['workflow_task_attempt'] ?? null; + if (! is_int($originalTaskAttempt) + || self::originalStart( + $run, + $execution, + $attempt, + $attempt->lease_owner, + $originalTaskAttempt, + $localContractVersion + ) === null + || ($started->payload['local_preparation']['descriptor_fingerprint'] ?? null) !== $fingerprint + || $execution->current_attempt_id !== $attempt->id || $execution->attempt_count !== $attempt->attempt_number) { + return $refused('local_activity_preparation_mismatch'); + } + // Accepted cancellation may fence immediately, without waiting for + // lease expiry. The replacement claim and root budget stay intact. + $cleanup = PortableLocalActivityCleanup::isExecution($run, $execution, $started); + if ($cleanup && ($sample ?? now())->gte(PortableLocalActivityCleanup::deadline($execution))) { + return $refused(PortableLocalActivityCleanup::isScoped($execution) + ? 'local_activity_cleanup_deadline_expired' : 'cancellation_deadline_expired'); + } + if ($run->cancellation_request_command_id !== null && (! $cleanup + || ! PortableLocalActivityCleanup::belongsToRunRequest( + $run, + $execution->activity_options['cancellation_cleanup'] + ))) { + $cancelled = $run->historyEvents() + ->where('event_type', HistoryEventType::ActivityCancelled) + ->where('payload->activity_attempt_id', $attempt->id) + ->where('payload->workflow_command_id', $run->cancellation_request_command_id) + ->first(); + if (! $cancelled instanceof WorkflowHistoryEvent) { + if ($execution->status !== ActivityStatus::Running || $attempt->status !== ActivityAttemptStatus::Running) { + return $refused('stale_activity_attempt'); + } + $cancelled = ActivityCancellation::record( + $run, + $execution, + command: $run->cancellation_request_command_id + ); + } + return [ + ...$refused('cancellation_requested'), + 'fenced' => true, + 'cancellation_history_event_id' => $cancelled?->id, + ]; + } + if ($run->status->isTerminal()) { + return $refused('run_closed'); + } + if (($run->execution_deadline_at !== null && ($sample ?? now())->gte($run->execution_deadline_at)) + || ($run->run_deadline_at !== null && ($sample ?? now())->gte($run->run_deadline_at))) { + return $refused('run_deadline_expired'); + } + if ($execution->status !== ActivityStatus::Running || $attempt->status !== ActivityAttemptStatus::Running) { + return $refused('local_activity_previous_attempt_unresolved'); + } + if (PortableLocalActivityCleanup::currentAuthorityExpired($run, $execution, $sample)) { + return $refused('local_activity_cleanup_authority_expired'); + } + if ($attempt->lease_expires_at === null || ($sample ?? now())->lt($attempt->lease_expires_at)) { + return $refused('local_activity_previous_attempt_live'); + } + /** @var WorkflowTask|null $originalTask */ + $originalTask ??= ConfiguredV2Models::query('task_model', WorkflowTask::class) + ->lockForUpdate() + ->find($attempt->workflow_task_id); + if ($originalTask === null || $originalTask->workflow_run_id !== $run->id + || $originalTask->task_type !== TaskType::Workflow) { + return $refused('original_workflow_claim_not_found'); + } + if ($originalTask->id === $task->id) { + if ($task->attempt_count <= $originalTaskAttempt) { + return $refused('local_activity_original_claim_not_reclaimed'); + } + } elseif ($originalTask->status === TaskStatus::Leased && $originalTask->lease_expires_at !== null + && ($sample ?? now()) + ->lt($originalTask->lease_expires_at)) { + return $refused('local_activity_original_claim_live'); + } + $recovery = [ + 'version' => 1, + 'descriptor_fingerprint' => $fingerprint, + 'workflow_task_id' => $task->id, + 'workflow_task_attempt' => $workflowTaskAttempt, + 'lease_owner' => $leaseOwner, + 'original_workflow_task_id' => $originalTask->id, + 'original_workflow_task_attempt' => $originalTaskAttempt, + 'original_lease_owner' => $attempt->lease_owner, + 'original_lease_expires_at' => $attempt->lease_expires_at->toISOString(), + 'callback_stop_state' => 'unknown', + ]; + if ($originalTask->id !== $task->id && in_array( + $originalTask->status, + [TaskStatus::Ready, TaskStatus::Leased], + true + )) { + $originalTask->forceFill([ + 'status' => TaskStatus::Completed, + 'lease_expires_at' => null, + ])->save(); + } + $executor = app(LocalActivityExecutor::class); + $outcome = $sample === null ? $executor->recoverPortableAttempt( + $run, + $task, + $execution, + $attempt, + $recovery + ) + : $executor->recoverPortableRetryAttempt($run, $task, $execution, $attempt, $recovery, $sample); + + return self::recoveryReceipt($outcome['event'], false); + }, 5); + } + /** * @param array $descriptor * @return array @@ -1295,6 +1511,9 @@ private static function recoveryReceipt(WorkflowHistoryEvent $event, bool $dupli 'callback_stop_state' => 'unknown', 'claim_released' => $event->event_type === HistoryEventType::ActivityRetryScheduled, 'created_task_ids' => is_string($retryTaskId) ? [$retryTaskId] : [], + ...(isset($event->payload['retry_disposition']) ? [ + 'retry_disposition' => $event->payload['retry_disposition'], + ] : []), ]; } diff --git a/src/V2/Support/PreparedLocalRetryDecision.php b/src/V2/Support/PreparedLocalRetryDecision.php index 23deafec..588afd22 100644 --- a/src/V2/Support/PreparedLocalRetryDecision.php +++ b/src/V2/Support/PreparedLocalRetryDecision.php @@ -4,6 +4,7 @@ namespace Workflow\V2\Support; +use Closure; use DateInterval; use DateTimeImmutable; use DateTimeInterface; @@ -14,6 +15,8 @@ /** @internal Select from the one locked engine sample, never a worker clock. */ final class PreparedLocalRetryDecision { + public const INTERRUPTION_MESSAGE = 'Local activity attempt was interrupted before a terminal event.'; + /** Validate caller-bound retained bytes without a new clock or effects. * Timeout kind is retained engine authority, never a worker-selected input. * @param array $snapshot @@ -28,18 +31,72 @@ public static function matchesStored( array $stored, bool $nonRetryable = false, ): bool { - $sample = self::storedSample($stored['failure_time'] ?? null); - $timeout = $stored['timeout_kind'] ?? null; - if ($sample === null || ($timeout !== null && ! is_string($timeout))) { - return false; - } - try { - PreparedLocalRetryTimestamp::add($sample, 0, $window->latest, $window->earliest); - $expected = self::select($snapshot, $attemptNumber, $report, $sample, $window, $timeout, $nonRetryable); - } catch (InvalidArgumentException) { - return false; + return self::matchesStoredSelection( + $window, + $stored, + static fn (DateTimeImmutable $sample, ?string $timeout): array => self::select( + $snapshot, + $attemptNumber, + $report, + $sample, + $window, + $timeout, + $nonRetryable + ) + ); + } + + /** + * @internal The caller has proved original claim loss under canonical + * locks. No worker report can select this engine-owned provenance. + * @param array $snapshot + * @return array + */ + public static function selectInterruption( + array $snapshot, + int $attemptNumber, + DateTimeInterface $sample, + PreparedLocalRetryWindow $window, + ?string $dueTimeoutKind = null, + ): array { + $decision = self::select( + $snapshot, + $attemptNumber, + [ + 'outcome' => 'failed', + 'message' => self::INTERRUPTION_MESSAGE, + ], + $sample, + $window, + $dueTimeoutKind + ); + if ($decision['failure_source'] === 'application') { + $decision['failure_source'] = 'engine_interruption'; } - return self::sameFields($expected, $stored); + + return $decision; + } + + /** @param array $snapshot + * @param array $stored + */ + public static function matchesStoredInterruption( + array $snapshot, + int $attemptNumber, + PreparedLocalRetryWindow $window, + array $stored, + ): bool { + return self::matchesStoredSelection( + $window, + $stored, + static fn (DateTimeImmutable $sample, ?string $timeout): array => self::selectInterruption( + $snapshot, + $attemptNumber, + $sample, + $window, + $timeout + ) + ); } /** @@ -81,6 +138,28 @@ public static function select( return self::selectedDelay($snapshot, $attemptNumber, $report, $sample, $window, $dueTimeoutKind, $decision); } + /** @param array $stored + * @param Closure(DateTimeImmutable, ?string): array $select + */ + private static function matchesStoredSelection( + PreparedLocalRetryWindow $window, + array $stored, + Closure $select, + ): bool { + $sample = self::storedSample($stored['failure_time'] ?? null); + $timeout = $stored['timeout_kind'] ?? null; + if ($sample === null || ($timeout !== null && ! is_string($timeout))) { + return false; + } + try { + PreparedLocalRetryTimestamp::add($sample, 0, $window->latest, $window->earliest); + $expected = $select($sample, $timeout); + } catch (InvalidArgumentException) { + return false; + } + return self::sameFields($expected, $stored); + } + /** * @param array $report */ diff --git a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php index 90c8c066..e559ce20 100644 --- a/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php +++ b/tests/Feature/V2/V2PortableLocalActivityPreparationTest.php @@ -217,6 +217,13 @@ public function testRetryContractCannotDowngradeThroughLegacyInternalExecution() static fn () => $executor->startPortableAttempt($run, $task, $execution, 'replacement', [ 'version' => 1, ]), + static fn () => $executor->recoverPortableAttempt( + $run, + $task, + $execution, + ActivityAttempt::query()->findOrFail($prepared['activity_attempt_id']), + [] + ), ]; foreach ($operations as $operation) { try { diff --git a/tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php b/tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php index 9bde846e..f2c6f3ab 100644 --- a/tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php +++ b/tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php @@ -5,9 +5,12 @@ namespace Tests\Feature\V2; use Illuminate\Support\Carbon; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Queue; +use PHPUnit\Framework\Attributes\DataProvider; use Tests\Fixtures\V2\TestGreetingWorkflow; use Tests\TestCase; +use Throwable; use Workflow\Serializers\Serializer; use Workflow\V2\Contracts\CooperativeWorkflowTaskBridge; use Workflow\V2\Contracts\PreparedLocalActivityTaskBridge; @@ -24,6 +27,7 @@ use Workflow\V2\Models\WorkflowInstance; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; +use Workflow\V2\Support\ConfiguredV2Models; use Workflow\V2\Support\HistoryTimeline; use Workflow\V2\Support\LocalActivityCall; use Workflow\V2\Support\LocalActivityExecutor; @@ -49,6 +53,434 @@ protected function tearDown(): void parent::tearDown(); } + #[DataProvider('interruptionDelays')] + public function testRetryContractRecoveryUsesOnePolicyDecisionWithoutAdmittingAnotherAttempt(int $delay): void + { + [$run, $task, $first, $descriptor] = $this->reclaimedRetryCall([ + 'backoff_seconds' => [$delay], + ]); + $sample = now() + ->toISOString(); + $reply = $this->recoverRetryContract($task, $descriptor); + $this->assertTrue($reply['recovered'], $reply['reason'] ?? ''); + $this->assertCount(13, $reply); + $this->assertArrayNotHasKey('recorded', $reply); + $this->assertSame('unknown', $reply['callback_stop_state']); + $this->assertSame('engine_interruption', $reply['retry_disposition']['failure_source']); + $this->assertSame($sample, $reply['retry_disposition']['failure_time']); + $this->assertSame('policy', $reply['retry_disposition']['delay_source']); + $this->assertSame($delay, $reply['retry_disposition']['candidate_delay_seconds']); + $this->assertSame(now()->addSeconds($delay)->toISOString(), $reply['retry_disposition']['retry_available_at']); + $this->assertArrayNotHasKey('timeout_kind', $reply['retry_disposition']); + $this->assertArrayNotHasKey('requested_delay_seconds', $reply['retry_disposition']); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $this->assertSame('cold_replay', $event->payload['retry_reason']); + $this->assertCount(10, $event->payload['local_recovery']); + $this->assertSame(1, $event->payload['local_recovery']['version']); + $this->assertSame(7, $event->payload['local_recovery']['original_workflow_task_attempt']); + $this->assertSame(8, $event->payload['local_recovery']['workflow_task_attempt']); + $attempt = ActivityAttempt::query()->findOrFail($first['activity_attempt_id']); + $this->assertSame(ActivityAttemptStatus::Expired, $attempt->status); + $this->assertSame($sample, $attempt->closed_at->toISOString()); + $this->assertSame(TaskStatus::Completed, $task->refresh()->status); + $this->assertCount(1, $reply['created_task_ids']); + $this->assertSame(1, ActivityAttempt::query()->where('workflow_run_id', $run->id)->count()); + $this->assertSame(2, $run->tasks()->count()); + $this->assertSame([ + ...$reply, + 'duplicate' => true, + ], $this->recoverRetryContract($task, $descriptor)); + $this->assertSame(3, $run->historyEvents()->count()); + } + + public static function interruptionDelays(): iterable + { + yield 'zero remains selected' => [0]; + yield 'positive policy delay' => [2]; + } + + #[DataProvider('interruptionTerminals')] + public function testRetryContractRecoveryTerminatesWithoutAnExtraTaskOrAttempt(array $policy, string $reason): void + { + [$run, $task, $first, $descriptor] = $this->reclaimedRetryCall($policy); + $before = $task->getAttributes(); + $reply = $this->recoverRetryContract($task, $descriptor); + $this->assertTrue($reply['recovered'], $reply['reason'] ?? ''); + $this->assertSame('ActivityFailed', $reply['event_type']); + $this->assertSame('engine_interruption', $reply['retry_disposition']['failure_source']); + $this->assertSame($reason, $reply['retry_disposition']['terminal_classification']); + $this->assertArrayNotHasKey('retry_available_at', $reply['retry_disposition']); + $this->assertSame([], $reply['created_task_ids']); + $this->assertFalse($reply['claim_released']); + $this->assertSame($before, $task->refresh()->getAttributes()); + $this->assertSame(1, $run->tasks()->count()); + $this->assertSame(1, ActivityExecution::query()->findOrFail($first['activity_execution_id'])->attempt_count); + $this->assertSame([ + ...$reply, + 'duplicate' => true, + ], $this->recoverRetryContract($task, $descriptor)); + } + + public static function interruptionTerminals(): iterable + { + yield 'policy nonretryable precedes budget' => [[ + 'max_attempts' => 1, + 'non_retryable_error_types' => [\RuntimeException::class], + ], 'non_retryable']; + yield 'attempt budget' => [[ + 'max_attempts' => 1, + ], 'attempts_exhausted']; + yield 'original total deadline' => [[ + 'backoff_seconds' => [30], + ], 'retry_delay_exceeds_schedule_to_close']; + } + + #[DataProvider('recoveryTimeouts')] + public function testRetryContractRecoveryRetainsActualTimeoutAuthority( + int $total, + int $start, + array $timeouts, + string $kind, + bool $scheduled + ): void { + [$run, $task, , $descriptor] = $this->reclaimedRetryCall( + totalTimeout: $total, + startTimeout: $start, + timeouts: $timeouts + ); + $reply = $this->recoverRetryContract($task, $descriptor); + $this->assertTrue($reply['recovered'], $reply['reason'] ?? ''); + $decision = $reply['retry_disposition']; + $this->assertSame('engine_timeout', $decision['failure_source']); + $this->assertSame($kind, $decision['timeout_kind']); + $this->assertSame($scheduled ? 'ActivityRetryScheduled' : 'ActivityTimedOut', $reply['event_type']); + $this->assertSame($scheduled, $reply['claim_released']); + $this->assertCount($scheduled ? 1 : 0, $reply['created_task_ids']); + $this->assertSame($scheduled ? 2 : 1, $run->tasks()->count()); + $this->assertSame([ + ...$reply, + 'duplicate' => true, + ], $this->recoverRetryContract($task, $descriptor)); + } + + public static function recoveryTimeouts(): iterable + { + yield 'total deadline precedes interruption' => [4, 4, [], 'schedule_to_close', false]; + yield 'start deadline still retries policy' => [30, 4, [], 'start_to_close', true]; + yield 'heartbeat precedes start timeout' => [30, 4, [ + 'heartbeat_timeout' => 3, + ], 'heartbeat', true]; + } + + public function testRetryRecoveryDuplicateAfterDeadlineKeepsItsOriginalAttemptAndDecision(): void + { + [$run, $task, $first, $descriptor] = $this->reclaimedRetryCall(); + $reply = $this->recoverRetryContract($task, $descriptor); + $this->assertTrue($reply['recovered'], $reply['reason'] ?? ''); + $retry = WorkflowTask::query()->findOrFail($reply['created_task_ids'][0]); + $retry->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'next-worker', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addMinute(), + ])->save(); + Carbon::setTestNow(now()->addSeconds(2)); + $next = PortableLocalActivityPreparation::prepareWithRetryPolicy( + $retry->id, + 'next-worker', + 1, + 1, + 'next-local-attempt', + $descriptor, + '1.20' + ); + $this->assertTrue($next['prepared'], $next['reason'] ?? ''); + $this->assertNotSame($first['activity_attempt_id'], $next['activity_attempt_id']); + Carbon::setTestNow(now()->addMinutes(2)); + config() + ->set('workflows.v2.default_retry_policy', [ + 'max_attempts' => 1, + ]); + $before = $run->historyEvents() + ->count(); + $this->assertSame([ + ...$reply, + 'duplicate' => true, + ], $this->recoverRetryContract($task, $descriptor)); + $this->assertSame($before, $run->historyEvents()->count()); + $this->assertSame('unknown', $reply['callback_stop_state']); + $this->assertSame(2, ActivityExecution::query()->findOrFail($first['activity_execution_id'])->attempt_count); + } + + #[DataProvider('recoverySnapshotMutations')] + public function testCorruptRecoveryAuthorityCannotPublishOrReturnAReceipt(bool $committed, bool $invalidCap): void + { + [$run, $task, $first, $descriptor] = $this->reclaimedRetryCall([ + 'backoff_seconds' => [5], + 'maximum_interval_seconds' => 20, + ]); + $reply = $committed ? $this->recoverRetryContract($task, $descriptor) : null; + $execution = ActivityExecution::query()->findOrFail($first['activity_execution_id']); + $execution->forceFill([ + 'retry_policy' => [ + ...$execution->retry_policy, + 'backoff_seconds' => [$invalidCap ? 21 : 3], + ], + ])->save(); + $before = [$run->historyEvents()->count(), $run->tasks()->count(), $execution->getAttributes(), + ActivityAttempt::query()->findOrFail($first['activity_attempt_id'])->getAttributes()]; + $rejected = $this->recoverRetryContract($task, $descriptor); + $this->assertFalse($rejected['recovered']); + $this->assertSame('local_activity_preparation_mismatch', $rejected['reason']); + $this->assertArrayNotHasKey('retry_disposition', $rejected); + $this->assertArrayNotHasKey('event_id', $rejected); + $this->assertSame( + $before, + [$run->historyEvents()->count(), $run->tasks()->count(), $execution->refresh()->getAttributes(), + ActivityAttempt::query()->findOrFail($first['activity_attempt_id'])->getAttributes()] + ); + if ($reply !== null) { + $this->assertSame( + $reply['retry_disposition'], + WorkflowHistoryEvent::query()->findOrFail($reply['event_id'])->payload['retry_disposition'] + ); + } + } + + public static function recoverySnapshotMutations(): iterable + { + yield 'invalid retained cap before selection' => [false, true]; + yield 'valid but replaced original snapshot before selection' => [false, false]; + yield 'invalid retained cap after committed receipt' => [true, true]; + yield 'valid but replaced original snapshot after committed receipt' => [true, false]; + } + + #[DataProvider('recoveryReceiptMutations')] + public function testRecoveryReceiptCannotChangeItsClosedIdentityOrDecision(string $location, mixed $value): void + { + [$run, $task, , $descriptor] = $this->reclaimedRetryCall(); + $reply = $this->recoverRetryContract($task, $descriptor); + $this->assertTrue($reply['recovered'], $reply['reason'] ?? ''); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $payload = $event->payload; + data_set($payload, $location, $value); + $event->forceFill([ + 'payload' => $payload, + ])->save(); + $before = $run->historyEvents() + ->count(); + $refused = $this->recoverRetryContract($task, $descriptor); + $this->assertSame('local_activity_recovery_mismatch', $refused['reason']); + $this->assertArrayNotHasKey('event_id', $refused); + $this->assertSame($before, $run->historyEvents()->count()); + $expectedRecovery = $payload['local_recovery']; + $actualRecovery = $event->refresh() +->payload['local_recovery']; + ksort($expectedRecovery); + ksort($actualRecovery); + $this->assertSame($expectedRecovery, $actualRecovery); + } + + public static function recoveryReceiptMutations(): iterable + { + yield 'worker source substitution' => ['retry_disposition.failure_source', 'application']; + yield 'worker override injection' => ['retry_disposition.requested_delay_seconds', 1]; + yield 'timeout injection' => ['retry_disposition.timeout_kind', 'heartbeat']; + yield 'unknown metadata member' => ['local_recovery.extra', true]; + yield 'stop state cannot be rewritten' => ['local_recovery.callback_stop_state', 'stopped']; + yield 'original owner substituted' => ['local_recovery.original_lease_owner', 'foreign']; + yield 'retry delay substituted' => ['retry_backoff_seconds', 0]; + } + + public function testRetryRecoveryRollbackPublishesNoFailureSampleReceiptOrTask(): void + { + [$run, $task, $first, $descriptor] = $this->reclaimedRetryCall(); + $connection = ConfiguredV2Models::connection(); + $before = $task->getAttributes(); + $connection->beginTransaction(); + try { + $reply = $this->recoverRetryContract($task, $descriptor); + $this->assertTrue($reply['recovered'], $reply['reason'] ?? ''); + $this->assertSame('engine_interruption', $reply['retry_disposition']['failure_source']); + $this->assertSame(3, $run->historyEvents()->count()); + } finally { + $connection->rollBack(); + } + $this->assertSame(2, $run->historyEvents()->count()); + $this->assertSame(1, $run->tasks()->count()); + $this->assertSame($before, $task->refresh()->getAttributes()); + $this->assertSame( + ActivityAttemptStatus::Running, + ActivityAttempt::query()->findOrFail($first['activity_attempt_id'])->status + ); + } + + public function testRetryRecoveryCannotUseAStillLiveIndependentOriginalClaim(): void + { + [$run, $original, , $descriptor] = $this->reclaimedRetryCall(reclaim: false); + $task = $original->replicate(); + $task->forceFill([ + 'lease_owner' => 'replacement-worker', + 'attempt_count' => 8, + 'lease_expires_at' => now() + ->addMinute(), + ])->save(); + $original->forceFill([ + 'lease_expires_at' => now() + ->addMinute(), + ])->save(); + $reply = $this->recoverRetryContract($task, $descriptor); + $this->assertSame('local_activity_original_claim_live', $reply['reason']); + $this->assertSame(2, $run->historyEvents()->count()); + } + + public function testRetryRecoveryKeepsIndependentRecoveringAndOriginalTaskIdentities(): void + { + [$run, $original, $first, $descriptor] = $this->reclaimedRetryCall(reclaim: false); + $task = $original->replicate(); + $task->forceFill([ + 'lease_owner' => 'replacement-worker', + 'attempt_count' => 8, + 'lease_expires_at' => now() + ->addMinute(), + ])->save(); + $reply = $this->recoverRetryContract($task, $descriptor); + $this->assertTrue($reply['recovered'], $reply['reason'] ?? ''); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $this->assertSame($task->id, $reply['workflow_task_id']); + $this->assertSame($task->id, $event->payload['local_recovery']['workflow_task_id']); + $this->assertSame($original->id, $event->payload['local_recovery']['original_workflow_task_id']); + $this->assertSame($original->id, $event->payload['activity_attempt']['task_id']); + $this->assertSame(TaskStatus::Completed, $original->refresh()->status); + $this->assertSame(TaskStatus::Completed, $task->refresh()->status); + $this->assertSame(1, ActivityExecution::query()->findOrFail($first['activity_execution_id'])->attempt_count); + $this->assertSame(3, $run->tasks()->count()); + } + + public function testRetryRecoveryCannotUpgradeAnOriginalVersionOnePreparation(): void + { + [$run, $task, , $descriptor] = $this->reclaimedCall(); + $versioned = [ + ...$descriptor, + 'retry_policy' => [ + 'contract_version' => 2, + 'mode' => 'explicit', + 'max_attempts' => 3, + 'backoff_seconds' => [2], + ], + ]; + $refused = $this->recoverRetryContract($task, $versioned); + $this->assertSame('local_activity_preparation_mismatch', $refused['reason']); + $this->assertSame(2, $run->historyEvents()->count()); + $legacy = $this->recover($task, $descriptor); + $this->assertTrue($legacy['recovered']); + $this->assertArrayNotHasKey('retry_disposition', $legacy); + } + + public function testConcurrentRetryRecoveriesSerializeOneDecisionAndReturnTheOriginalAfterDeadline(): void + { + if (ConfiguredV2Models::connection()->getDriverName() !== 'mysql' + || ! function_exists('pcntl_fork') || ! function_exists('posix_kill')) { + $this->markTestSkipped('The two-process lock control requires MySQL and pcntl.'); + } + self::stopWorkers(); + [$run, $task, $first, $descriptor] = $this->reclaimedRetryCall(); + $connectionName = ConfiguredV2Models::connection()->getName(); + $table = (new ActivityAttempt())->getTable(); + $sockets = stream_socket_pair(STREAM_PF_UNIX, STREAM_SOCK_STREAM, STREAM_IPPROTO_IP); + $this->assertIsArray($sockets); + stream_set_timeout($sockets[0], 10); + stream_set_timeout($sockets[1], 10); + DB::purge($connectionName); + $pid = pcntl_fork(); + $this->assertNotSame(-1, $pid); + if ($pid === 0) { + fclose($sockets[0]); + try { + $connection = DB::reconnect($connectionName); + $announced = false; + $connection->beforeExecuting(static function (string $query) use ($sockets, $table, &$announced): void { + if (! $announced && str_contains($query, $table) && str_contains( + strtolower($query), + 'for update' + )) { + $announced = true; + fwrite($sockets[1], "locking\n"); + } + }); + fwrite($sockets[1], "ready\n"); + if (fgets($sockets[1]) !== "recover\n") { + throw new \LogicException('Recovery race was not released.'); + } + Carbon::setTestNow(now()->addMinutes(2)); + $reply = PortableLocalActivityPreparation::recoverWithRetryPolicy( + $task->id, + 'replacement-worker', + 8, + 1, + $descriptor, + '1.20' + ); + fwrite($sockets[1], json_encode($reply, JSON_THROW_ON_ERROR) . "\n"); + fclose($sockets[1]); + exit(0); + } catch (Throwable $error) { + fwrite($sockets[1], json_encode([ + 'error' => substr($error->getMessage(), 0, 800), + ]) . "\n"); + fclose($sockets[1]); + exit(1); + } + } + fclose($sockets[1]); + $connection = DB::reconnect($connectionName); + try { + $this->assertSame("ready\n", fgets($sockets[0])); + $connection->beginTransaction(); + ActivityAttempt::query()->lockForUpdate()->findOrFail($first['activity_attempt_id']); + fwrite($sockets[0], "recover\n"); + $this->assertSame("locking\n", fgets($sockets[0])); + $winner = $this->recoverRetryContract($task, $descriptor); + $this->assertTrue($winner['recovered'], $winner['reason'] ?? ''); + $connection->commit(); + $line = fgets($sockets[0]); + $this->assertIsString($line); + $loser = json_decode($line, true, flags: JSON_THROW_ON_ERROR); + $this->assertSame([ + ...$winner, + 'duplicate' => true, + ], $loser); + $until = microtime(true) + 5; + do { + $exited = pcntl_waitpid($pid, $status, WNOHANG); + if ($exited === 0) { + usleep(10000); + } + } while ($exited === 0 && microtime(true) < $until); + $this->assertSame($pid, $exited); + $pid = 0; + $this->assertSame(0, pcntl_wexitstatus($status)); + $this->assertSame(3, $run->historyEvents()->count()); + $this->assertSame(2, $run->tasks()->count()); + $this->assertSame( + 1, + ActivityExecution::query()->findOrFail($first['activity_execution_id'])->attempt_count + ); + } finally { + if ($connection->transactionLevel() > 0) { + $connection->rollBack(); + } + fclose($sockets[0]); + if ($pid > 0) { + posix_kill($pid, SIGKILL); + pcntl_waitpid($pid, $status); + } + DB::purge($connectionName); + DB::reconnect($connectionName); + } + } + public function testColdRecoveryFencesTheOldAttemptAndPreparesARealRetryUnderTheOriginalBudget(): void { [$run, $task, $first, $descriptor] = $this->reclaimedCall(); @@ -400,6 +832,41 @@ public function testLocalStopAdmissionRejectsProtocol119ThroughTheOptionalRole() $this->assertSame($before, $run->historyEvents()->count()); } + private function recoverRetryContract(WorkflowTask $task, array $descriptor): array + { + return PortableLocalActivityPreparation::recoverWithRetryPolicy( + $task->id, + 'replacement-worker', + 8, + 1, + $descriptor, + '1.20' + ); + } + + private function reclaimedRetryCall( + array $policy = [], + int $totalTimeout = 30, + int $startTimeout = 20, + bool $reclaim = true, + array $timeouts = [], + ): array { + return $this->reclaimedCall( + maxAttempts: $policy['max_attempts'] ?? 3, + totalTimeout: $totalTimeout, + startTimeout: $startTimeout, + reclaim: $reclaim, + versionedPolicy: [ + 'contract_version' => 2, + 'mode' => 'explicit', + 'max_attempts' => 3, + 'backoff_seconds' => [2], + ...$policy, + ], + timeouts: $timeouts + ); + } + private function bridge(): PreparedLocalActivityTaskBridge { return $this->app->make(PreparedLocalActivityTaskBridge::class); @@ -453,6 +920,8 @@ private function reclaimedCall( int $totalTimeout = 30, int $startTimeout = 20, bool $reclaim = true, + ?array $versionedPolicy = null, + array $timeouts = [], ): array { $instance = WorkflowInstance::query()->create([ 'workflow_class' => TestGreetingWorkflow::class, @@ -502,9 +971,22 @@ private function reclaimedCall( 'max_attempts' => $maxAttempts, 'backoff_seconds' => [2], ], + ...$timeouts, ]; - $first = $this->bridge() - ->prepareLocalActivity($task->id, 'original-worker', 7, 1, 'original-local-attempt', $descriptor, '1.20'); + if ($versionedPolicy !== null) { + $descriptor['retry_policy'] = $versionedPolicy; + } + $first = $versionedPolicy === null ? $this->bridge() + ->prepareLocalActivity($task->id, 'original-worker', 7, 1, 'original-local-attempt', $descriptor, '1.20') + : PortableLocalActivityPreparation::prepareWithRetryPolicy( + $task->id, + 'original-worker', + 7, + 1, + 'original-local-attempt', + $descriptor, + '1.20' + ); $this->assertTrue($first['prepared']); Carbon::setTestNow(now()->addSeconds($elapsed)); if ($reclaim) { diff --git a/tests/Unit/V2/PreparedLocalActivityReportTest.php b/tests/Unit/V2/PreparedLocalActivityReportTest.php index 5672276a..e73cf400 100644 --- a/tests/Unit/V2/PreparedLocalActivityReportTest.php +++ b/tests/Unit/V2/PreparedLocalActivityReportTest.php @@ -131,7 +131,8 @@ public function testNoOverrideReportStillHasAClosedVersionTwoGrammar(): void ]; } foreach (['retry_disposition', 'backoff_seconds', 'retry_backoff_seconds', 'retry_available_at', - 'failure_time', 'original_schedule_to_close_deadline_at', 'snapshot_version', 'extra'] as $field) { + 'failure_time', 'original_schedule_to_close_deadline_at', 'snapshot_version', 'failure_source', + 'requested_delay_seconds', 'local_recovery', 'callback_stop_state', 'extra'] as $field) { $invalid[] = [ 'outcome' => 'failed', 'message' => '', @@ -142,6 +143,9 @@ public function testNoOverrideReportStillHasAClosedVersionTwoGrammar(): void $field => 0, ]; } + $invalid[] = [ + 'outcome' => 'engine_interruption', + ]; $invalid[] = [ 'outcome' => 'timed_out', 'timeout_kind' => 'heartbeat', diff --git a/tests/Unit/V2/PreparedLocalRetryDecisionTest.php b/tests/Unit/V2/PreparedLocalRetryDecisionTest.php index b3d450da..8749bc89 100644 --- a/tests/Unit/V2/PreparedLocalRetryDecisionTest.php +++ b/tests/Unit/V2/PreparedLocalRetryDecisionTest.php @@ -14,6 +14,146 @@ final class PreparedLocalRetryDecisionTest extends TestCase { + public function testEngineInterruptionUsesOnlyTheRetainedPolicyAndItsCommittedSample(): void + { + $sample = new DateTimeImmutable('2026-11-05T00:00:00.123456Z'); + $window = new PreparedLocalRetryWindow($sample->modify('+10 seconds'), $sample->modify('+100 seconds')); + foreach ([0, 2] as $delay) { + $snapshot = [ + ...$this->snapshot(), + 'backoff_seconds' => [$delay], + 'maximum_interval_seconds' => $delay, + ]; + $decision = PreparedLocalRetryDecision::selectInterruption($snapshot, 1, $sample, $window); + self::assertSame('engine_interruption', $decision['failure_source']); + self::assertSame('scheduled', $decision['decision']); + self::assertSame('policy', $decision['delay_source']); + self::assertSame($delay, $decision['candidate_delay_seconds']); + self::assertSame( + $sample->modify('+' . $delay . ' seconds')->format('Y-m-d\TH:i:s.u\Z'), + $decision['retry_available_at'] + ); + self::assertArrayNotHasKey('requested_delay_seconds', $decision); + self::assertArrayNotHasKey('timeout_kind', $decision); + self::assertTrue( + PreparedLocalRetryDecision::matchesStoredInterruption($snapshot, 1, $window, array_reverse( + $decision, + true + )) + ); + } + } + + public function testInterruptionKeepsNonRetryableBudgetAndOriginalDeadlinePrecedence(): void + { + $sample = new DateTimeImmutable('2026-11-05T00:00:00.000000Z'); + $window = new PreparedLocalRetryWindow($sample->modify('+2 seconds'), $sample->modify('+100 seconds')); + foreach ([ + [ + 'snapshot' => [ + ...$this->snapshot(), + 'non_retryable_error_types' => [\RuntimeException::class], + ], + 'attempt' => 3, + 'reason' => 'non_retryable', + ], + [ + 'snapshot' => $this->snapshot(), + 'attempt' => 3, + 'reason' => 'attempts_exhausted', + ], + [ + 'snapshot' => $this->snapshot(), + 'attempt' => 1, + 'reason' => 'retry_delay_exceeds_schedule_to_close', + ], + ] as $case) { + $decision = PreparedLocalRetryDecision::selectInterruption( + $case['snapshot'], + $case['attempt'], + $sample, + $window + ); + self::assertSame('terminal', $decision['decision']); + self::assertSame('engine_interruption', $decision['failure_source']); + self::assertSame($case['reason'], $decision['terminal_classification']); + self::assertArrayNotHasKey('retry_available_at', $decision); + self::assertTrue( + PreparedLocalRetryDecision::matchesStoredInterruption( + $case['snapshot'], + $case['attempt'], + $window, + $decision + ) + ); + if ($case['reason'] !== 'retry_delay_exceeds_schedule_to_close') { + self::assertArrayNotHasKey('candidate_delay_seconds', $decision); + self::assertArrayNotHasKey('delay_source', $decision); + } + } + } + + public function testActualDueTimeoutKeepsAuthorityAheadOfInterruption(): void + { + $sample = new DateTimeImmutable('2026-11-05T00:00:00.000000Z'); + $latest = $sample->modify('+100 seconds'); + $snapshot = [ + ...$this->snapshot(), + 'non_retryable_error_types' => [\RuntimeException::class], + ]; + foreach (['start_to_close', 'heartbeat'] as $kind) { + $window = new PreparedLocalRetryWindow($sample->modify('+10 seconds'), $latest); + $decision = PreparedLocalRetryDecision::selectInterruption($snapshot, 1, $sample, $window, $kind); + self::assertSame('engine_timeout', $decision['failure_source']); + self::assertSame($kind, $decision['timeout_kind']); + self::assertSame('scheduled', $decision['decision']); + self::assertSame('policy', $decision['delay_source']); + self::assertTrue(PreparedLocalRetryDecision::matchesStoredInterruption($snapshot, 1, $window, $decision)); + } + $window = new PreparedLocalRetryWindow($sample, $latest); + $decision = PreparedLocalRetryDecision::selectInterruption($snapshot, 3, $sample, $window); + self::assertSame('engine_timeout', $decision['failure_source']); + self::assertSame('schedule_to_close', $decision['timeout_kind']); + self::assertSame('engine_timeout', $decision['terminal_classification']); + self::assertArrayNotHasKey('candidate_delay_seconds', $decision); + } + + public function testRetainedInterruptionCannotGainWorkerOverrideOrTimeoutAuthority(): void + { + $sample = new DateTimeImmutable('2026-11-05T00:00:00.000000Z'); + $window = new PreparedLocalRetryWindow(null, $sample->modify('+100 seconds')); + $stored = PreparedLocalRetryDecision::selectInterruption($this->snapshot(), 1, $sample, $window); + foreach ([ + [ + 'failure_source' => 'application', + ], [ + 'requested_delay_seconds' => 1, + ], + [ + 'timeout_kind' => 'heartbeat', + ], [ + 'delay_source' => 'failure_override', + ], + [ + 'callback_stop_state' => 'stopped', + ], [ + 'candidate_delay_seconds' => 0, + ], + ] as $mutation) { + self::assertFalse( + PreparedLocalRetryDecision::matchesStoredInterruption($this->snapshot(), 1, $window, [ + ...$stored, + ...$mutation, + ]) + ); + } + $this->expectException(InvalidArgumentException::class); + PreparedLocalRetryDecision::selectInterruption([ + ...$this->snapshot(), + 'maximum_interval_seconds' => 1, + ], 1, $sample, $window); + } + public function testOriginalWindowCopiesMutableTimestampsWithoutReadingAClock(): void { $deadline = new DateTime('2026-11-05T01:01:00.123456+01:00'); From e612b52d2cf96e4b40e17cf2fa36231ad0315a13 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:20:16 +0000 Subject: [PATCH 14/15] Bind retry preparation to retained versioned recovery authority --- .../PortableLocalActivityPreparation.php | 15 ++++- .../V2PortableLocalActivityRecoveryTest.php | 61 +++++++++++++++++-- 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/src/V2/Support/PortableLocalActivityPreparation.php b/src/V2/Support/PortableLocalActivityPreparation.php index 50ac4559..e37db1d6 100644 --- a/src/V2/Support/PortableLocalActivityPreparation.php +++ b/src/V2/Support/PortableLocalActivityPreparation.php @@ -444,7 +444,8 @@ private static function matchesRetryRecoveryReceipt( $payload = $event->payload; $decision = $payload['retry_disposition'] ?? null; $recovery = $payload['local_recovery'] ?? null; - if ($started === null || ! is_array($decision) || ! is_array($recovery)) { + if ($started === null || ! is_array($decision) || ! is_array($recovery) + || array_key_exists('local_outcome', $payload)) { return false; } $connection = ConfiguredV2Models::connection(); @@ -1253,7 +1254,9 @@ private static function prepareRetry( $previous, $originalTaskAttempt, $fingerprint, - $localContractVersion + $localContractVersion, + $execution, + $started ) || ! is_string($retry->payload['retry_available_at'] ?? null) || $execution->attempt_count >= ActivityRetryPolicy::maxAttemptsFromSnapshot($execution)) { @@ -1399,7 +1402,7 @@ private static function groupAdmission(WorkflowRun $run, ActivityExecution $exec } /** - * @return array + * Validate original outcome or recovery authority before preparing a retry. */ private static function retryAuthorityMatches( WorkflowHistoryEvent $retry, @@ -1407,8 +1410,14 @@ private static function retryAuthorityMatches( int $originalTaskAttempt, string $fingerprint, int $localContractVersion = 1, + ?ActivityExecution $execution = null, + ?WorkflowHistoryEvent $started = null, ): bool { $payload = $retry->payload; + if ($localContractVersion === 2 && array_key_exists('local_recovery', $payload)) { + return $previous->status === ActivityAttemptStatus::Expired && $execution !== null + && self::matchesRetryRecoveryReceipt($retry, $execution, $previous, $started); + } if (($payload['local_outcome']['version'] ?? null) === $localContractVersion) { return $previous->status === ActivityAttemptStatus::Failed && $retry->workflow_task_id === $previous->workflow_task_id diff --git a/tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php b/tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php index f2c6f3ab..41871e3f 100644 --- a/tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php +++ b/tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php @@ -86,7 +86,7 @@ public function testRetryContractRecoveryUsesOnePolicyDecisionWithoutAdmittingAn $this->assertCount(1, $reply['created_task_ids']); $this->assertSame(1, ActivityAttempt::query()->where('workflow_run_id', $run->id)->count()); $this->assertSame(2, $run->tasks()->count()); - $this->assertSame([ + $this->assertRecoveryReply([ ...$reply, 'duplicate' => true, ], $this->recoverRetryContract($task, $descriptor)); @@ -115,7 +115,7 @@ public function testRetryContractRecoveryTerminatesWithoutAnExtraTaskOrAttempt(a $this->assertSame($before, $task->refresh()->getAttributes()); $this->assertSame(1, $run->tasks()->count()); $this->assertSame(1, ActivityExecution::query()->findOrFail($first['activity_execution_id'])->attempt_count); - $this->assertSame([ + $this->assertRecoveryReply([ ...$reply, 'duplicate' => true, ], $this->recoverRetryContract($task, $descriptor)); @@ -157,7 +157,7 @@ public function testRetryContractRecoveryRetainsActualTimeoutAuthority( $this->assertSame($scheduled, $reply['claim_released']); $this->assertCount($scheduled ? 1 : 0, $reply['created_task_ids']); $this->assertSame($scheduled ? 2 : 1, $run->tasks()->count()); - $this->assertSame([ + $this->assertRecoveryReply([ ...$reply, 'duplicate' => true, ], $this->recoverRetryContract($task, $descriptor)); @@ -204,7 +204,7 @@ public function testRetryRecoveryDuplicateAfterDeadlineKeepsItsOriginalAttemptAn ]); $before = $run->historyEvents() ->count(); - $this->assertSame([ + $this->assertRecoveryReply([ ...$reply, 'duplicate' => true, ], $this->recoverRetryContract($task, $descriptor)); @@ -228,7 +228,7 @@ public function testCorruptRecoveryAuthorityCannotPublishOrReturnAReceipt(bool $ 'backoff_seconds' => [$invalidCap ? 21 : 3], ], ])->save(); - $before = [$run->historyEvents()->count(), $run->tasks()->count(), $execution->getAttributes(), + $before = [$run->historyEvents()->count(), $run->tasks()->count(), $execution->refresh()->getAttributes(), ActivityAttempt::query()->findOrFail($first['activity_attempt_id'])->getAttributes()]; $rejected = $this->recoverRetryContract($task, $descriptor); $this->assertFalse($rejected['recovered']); @@ -291,6 +291,44 @@ public static function recoveryReceiptMutations(): iterable yield 'stop state cannot be rewritten' => ['local_recovery.callback_stop_state', 'stopped']; yield 'original owner substituted' => ['local_recovery.original_lease_owner', 'foreign']; yield 'retry delay substituted' => ['retry_backoff_seconds', 0]; + yield 'outcome and recovery authority mixed' => ['local_outcome.version', 2]; + } + + public function testRetryPreparationRefusesMixedOutcomeAndRecoveryAuthority(): void + { + [$run, $task, $first, $descriptor] = $this->reclaimedRetryCall(); + $reply = $this->recoverRetryContract($task, $descriptor); + $this->assertTrue($reply['recovered'], $reply['reason'] ?? ''); + $event = WorkflowHistoryEvent::query()->findOrFail($reply['event_id']); + $event->forceFill([ + 'payload' => [ + ...$event->payload, + 'local_outcome' => [ + 'version' => 2, + ], + ], + ])->save(); + $retry = WorkflowTask::query()->findOrFail($reply['created_task_ids'][0]); + $retry->forceFill([ + 'status' => TaskStatus::Leased, + 'lease_owner' => 'next-worker', + 'attempt_count' => 1, + 'lease_expires_at' => now() + ->addMinute(), + ])->save(); + Carbon::setTestNow(now()->addSeconds(2)); + $refused = PortableLocalActivityPreparation::prepareWithRetryPolicy( + $retry->id, + 'next-worker', + 1, + 1, + 'next-local-attempt', + $descriptor, + '1.20' + ); + $this->assertSame('local_activity_retry_preparation_mismatch', $refused['reason']); + $this->assertSame(3, $run->historyEvents()->count()); + $this->assertSame(1, ActivityExecution::query()->findOrFail($first['activity_execution_id'])->attempt_count); } public function testRetryRecoveryRollbackPublishesNoFailureSampleReceiptOrTask(): void @@ -447,7 +485,7 @@ public function testConcurrentRetryRecoveriesSerializeOneDecisionAndReturnTheOri $line = fgets($sockets[0]); $this->assertIsString($line); $loser = json_decode($line, true, flags: JSON_THROW_ON_ERROR); - $this->assertSame([ + $this->assertRecoveryReply([ ...$winner, 'duplicate' => true, ], $loser); @@ -832,6 +870,17 @@ public function testLocalStopAdmissionRejectsProtocol119ThroughTheOptionalRole() $this->assertSame($before, $run->historyEvents()->count()); } + private function assertRecoveryReply(array $expected, array $actual): void + { + // Native JSON storage may reorder object keys. Preserve exact scalar + // types and every field while comparing the closed disposition. + if (isset($expected['retry_disposition'], $actual['retry_disposition'])) { + ksort($expected['retry_disposition']); + ksort($actual['retry_disposition']); + } + $this->assertSame($expected, $actual); + } + private function recoverRetryContract(WorkflowTask $task, array $descriptor): array { return PortableLocalActivityPreparation::recoverWithRetryPolicy( From eb3056ed9daebb1754281d6ff1c8a2507607d36e Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 6 Nov 2026 04:10:44 +0000 Subject: [PATCH 15/15] Declare the unbound prepared retry bridge contract --- .github/pr-test-selection.json | 1 + .../PreparedLocalActivityRetryTaskBridge.php | 74 +++++++++++ .../V2PortableLocalActivityRecoveryTest.php | 4 + ...eparedLocalActivityRetryTaskBridgeTest.php | 115 ++++++++++++++++++ 4 files changed, 194 insertions(+) create mode 100644 src/V2/Contracts/PreparedLocalActivityRetryTaskBridge.php create mode 100644 tests/Unit/V2/PreparedLocalActivityRetryTaskBridgeTest.php diff --git a/.github/pr-test-selection.json b/.github/pr-test-selection.json index bd652a5e..a1135e7a 100644 --- a/.github/pr-test-selection.json +++ b/.github/pr-test-selection.json @@ -20,6 +20,7 @@ "tests/Unit/V2/LaravelDependencySecurityPolicyTest.php", "tests/Unit/V2/PlatformProtocolSpecsTest.php", "tests/Unit/V2/PreparedLocalActivityReportTest.php", + "tests/Unit/V2/PreparedLocalActivityRetryTaskBridgeTest.php", "tests/Unit/V2/PreparedLocalRetryDecisionTest.php", "tests/Unit/V2/PreparedLocalRetryDescriptorTest.php", "tests/Unit/V2/PreparedLocalRetryPolicyTest.php", diff --git a/src/V2/Contracts/PreparedLocalActivityRetryTaskBridge.php b/src/V2/Contracts/PreparedLocalActivityRetryTaskBridge.php new file mode 100644 index 00000000..d6aefd21 --- /dev/null +++ b/src/V2/Contracts/PreparedLocalActivityRetryTaskBridge.php @@ -0,0 +1,74 @@ + $descriptor + * @return array + */ + public function prepareLocalActivityWithRetryPolicy( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + string $workerAttemptId, + array $descriptor, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; + + /** @param array $report + * @return array + */ + public function recordLocalActivityRetryOutcome( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + array $report, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; + + /** + * @return array + */ + public function controlLocalActivityWithRetryPolicy( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + bool $renewLease = false, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; + + /** @param array $progress + * @return array + */ + public function heartbeatLocalActivityWithRetryPolicy( + string $attemptId, + string $leaseOwner, + int $workflowTaskAttempt, + array $progress = [], + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; + + /** @param array $descriptor + * @return array + */ + public function recoverLocalActivityWithRetryPolicy( + string $taskId, + string $leaseOwner, + int $workflowTaskAttempt, + int $sequence, + array $descriptor, + string $protocolVersion = WorkerProtocolVersion::VERSION, + ): array; +} diff --git a/tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php b/tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php index 41871e3f..59c036e7 100644 --- a/tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php +++ b/tests/Feature/V2/V2PortableLocalActivityRecoveryTest.php @@ -13,6 +13,7 @@ use Throwable; use Workflow\Serializers\Serializer; use Workflow\V2\Contracts\CooperativeWorkflowTaskBridge; +use Workflow\V2\Contracts\PreparedLocalActivityRetryTaskBridge; use Workflow\V2\Contracts\PreparedLocalActivityTaskBridge; use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Enums\ActivityAttemptStatus; @@ -843,12 +844,15 @@ public function testProtocol119CannotEnterThePreparedRecoveryPath(): void public function testPreparedLocalRoleUsesTheExistingWorkflowBindingWithoutExpandingCustomBridges(): void { $this->assertSame(app(WorkflowTaskBridge::class), $this->bridge()); + $this->assertFalse($this->app->bound(PreparedLocalActivityRetryTaskBridge::class)); + $this->assertNotInstanceOf(PreparedLocalActivityRetryTaskBridge::class, $this->bridge()); foreach ([WorkflowTaskBridge::class, CooperativeWorkflowTaskBridge::class] as $contract) { $custom = \Mockery::mock($contract); $this->app->instance(WorkflowTaskBridge::class, $custom); $resolved = $this->app->make(PreparedLocalActivityTaskBridge::class); $this->assertSame($custom, $resolved); $this->assertNotInstanceOf(PreparedLocalActivityTaskBridge::class, $resolved); + $this->assertNotInstanceOf(PreparedLocalActivityRetryTaskBridge::class, $resolved); } } diff --git a/tests/Unit/V2/PreparedLocalActivityRetryTaskBridgeTest.php b/tests/Unit/V2/PreparedLocalActivityRetryTaskBridgeTest.php new file mode 100644 index 00000000..30c6d003 --- /dev/null +++ b/tests/Unit/V2/PreparedLocalActivityRetryTaskBridgeTest.php @@ -0,0 +1,115 @@ +assertTrue($role->isInterface()); + $this->assertTrue($role->isSubclassOf(PreparedLocalActivityTaskBridge::class)); + $newMethods = []; + foreach ($role->getMethods() as $method) { + if ($method->getDeclaringClass()->getName() === $role->getName()) { + $newMethods[] = $method->getName(); + } + } + sort($newMethods); + $this->assertSame([ + 'controlLocalActivityWithRetryPolicy', + 'heartbeatLocalActivityWithRetryPolicy', + 'prepareLocalActivityWithRetryPolicy', + 'recordLocalActivityRetryOutcome', + 'recoverLocalActivityWithRetryPolicy', + ], $newMethods); + } + + public function testVersionedOperationSignaturesKeepIdentityOrderingAndExistingProtocolDefault(): void + { + $role = new ReflectionClass(PreparedLocalActivityRetryTaskBridge::class); + $identity = [ + 'taskId' => 'string', + 'leaseOwner' => 'string', + 'workflowTaskAttempt' => 'int', + ]; + $attempt = [ + 'attemptId' => 'string', + 'leaseOwner' => 'string', + 'workflowTaskAttempt' => 'int', + ]; + $protocol = [ + 'protocolVersion' => 'string', + ]; + foreach ([ + 'prepareLocalActivityWithRetryPolicy' => [ + ...$identity, + 'sequence' => 'int', + 'workerAttemptId' => 'string', + 'descriptor' => 'array', + ...$protocol, + ], + 'recordLocalActivityRetryOutcome' => [ + ...$attempt, + 'report' => 'array', + ...$protocol, + ], + 'controlLocalActivityWithRetryPolicy' => [ + ...$attempt, + 'renewLease' => 'bool', + ...$protocol, + ], + 'heartbeatLocalActivityWithRetryPolicy' => [ + ...$attempt, + 'progress' => 'array', + ...$protocol, + ], + 'recoverLocalActivityWithRetryPolicy' => [ + ...$identity, + 'sequence' => 'int', + 'descriptor' => 'array', + ...$protocol, + ], + ] as $name => $expected) { + $method = $role->getMethod($name); + $actual = []; + foreach ($method->getParameters() as $parameter) { + $this->assertFalse($parameter->isPassedByReference()); + $this->assertFalse($parameter->isVariadic()); + $this->assertFalse($parameter->allowsNull()); + $actual[$parameter->getName()] = (string) $parameter->getType(); + if ($parameter->getName() === 'protocolVersion') { + $this->assertTrue($parameter->isDefaultValueAvailable()); + $this->assertSame(WorkerProtocolVersion::VERSION, $parameter->getDefaultValue()); + } elseif ($parameter->getName() === 'renewLease') { + $this->assertSame(false, $parameter->getDefaultValue()); + } elseif ($parameter->getName() === 'progress') { + $this->assertSame([], $parameter->getDefaultValue()); + } else { + $this->assertFalse($parameter->isOptional()); + } + } + $this->assertSame($expected, $actual, $name); + $this->assertSame('array', (string) $method->getReturnType()); + } + } + + public function testExistingDefaultAndCustomVersionOneBridgesDoNotGainTheOptionalRole(): void + { + $this->assertFalse(is_a(DefaultWorkflowTaskBridge::class, PreparedLocalActivityRetryTaskBridge::class, true)); + foreach ([WorkflowTaskBridge::class, PreparedLocalActivityTaskBridge::class] as $contract) { + $bridge = \Mockery::mock($contract); + $this->assertNotInstanceOf(PreparedLocalActivityRetryTaskBridge::class, $bridge); + } + } +}