From 98022f2efb63669075bc2fdaaf4fa45be39bcfc2 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 05:06:35 +0000 Subject: [PATCH 1/6] test: distinguish active and historical workflow task failures --- .../V2/WorkflowTaskFailureLivenessTest.php | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 tests/Unit/V2/WorkflowTaskFailureLivenessTest.php diff --git a/tests/Unit/V2/WorkflowTaskFailureLivenessTest.php b/tests/Unit/V2/WorkflowTaskFailureLivenessTest.php new file mode 100644 index 00000000..5291ff61 --- /dev/null +++ b/tests/Unit/V2/WorkflowTaskFailureLivenessTest.php @@ -0,0 +1,110 @@ +set('queue.default', 'database'); + Queue::fake(); + Carbon::setTestNow('2026-11-05T03:00:00Z'); + $workflow = WorkflowStub::make(TestGreetingWorkflow::class, 'workflow-task-failure-classification'); + $workflow->start('Taylor'); + $runId = $workflow->runId(); + $this->assertIsString($runId); + $bridge = $this->app->make(WorkflowTaskBridge::class); + $this->assertSame([ + 'open' => true, + 'liveness' => 'workflow_task_ready', + 'task_problem' => false, + 'badge' => null, + 'repair_needed' => 0, + 'resume_health' => 'ok', + ], $this->classificationFor($runId)); + + for ($failure = 1; $failure <= 2; ++$failure) { + $task = WorkflowTask::query()->where('workflow_run_id', $runId) + ->where('task_type', TaskType::Workflow->value) + ->where('status', TaskStatus::Ready->value)->sole(); + $this->assertNotNull($bridge->claim($task->id, 'source-classification-worker')); + Carbon::setTestNow(now()->addSecond()); + $result = $bridge->fail($task->id, 'Synthetic service replay failure'); + $this->assertTrue($result['recorded']); + $this->assertSame(TaskStatus::Failed, $task->fresh()->status); + $this->assertFalse((bool) ($task->fresh()->payload['replay_blocked'] ?? false)); + $this->assertSame([ + 'open' => true, + 'liveness' => 'repair_needed', + 'task_problem' => true, + 'badge' => 'active', + 'repair_needed' => 1, + 'resume_health' => 'warning', + ], $this->classificationFor($runId)); + + Carbon::setTestNow(now()->addSecond()); + $this->assertTrue($workflow->repair()->accepted()); + $this->assertSame([ + 'open' => true, + 'liveness' => 'workflow_task_ready', + 'task_problem' => true, + 'badge' => 'history', + 'repair_needed' => 0, + 'resume_health' => 'ok', + ], $this->classificationFor($runId)); + } + + $this->assertTrue($workflow->terminate('Source classification closure')->accepted()); + $this->assertSame([ + 'open' => false, + 'liveness' => 'closed', + 'task_problem' => true, + 'badge' => 'history', + 'repair_needed' => 0, + 'resume_health' => 'ok', + ], $this->classificationFor($runId)); + } + + /** + * @return array{open: bool, liveness: string|null, task_problem: bool, badge: string|null, repair_needed: int, resume_health: string} + */ + private function classificationFor(string $runId): array + { + $run = WorkflowRun::query()->findOrFail($runId); + $summary = WorkflowRunSummary::query()->findOrFail($runId); + $health = HealthCheck::snapshot(); + $checks = array_column($health['checks'], 'status', 'name'); + $problem = WorkflowTaskProblem::metadata($summary->task_problem, $summary->liveness_state, $summary->wait_kind); + + return [ + 'open' => ! $run->status->isTerminal(), + 'liveness' => $summary->liveness_state, + 'task_problem' => $summary->task_problem, + 'badge' => $problem['code'] ?? null, + 'repair_needed' => $health['operator_metrics']['runs']['repair_needed'], + 'resume_health' => $checks['durable_resume_paths'], + ]; + } +} From 585480461c8be18357dc58c48010e8e1585107a1 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 10:24:00 +0000 Subject: [PATCH 2/6] test: classify embedded replay blocking through repair and closure --- .../V2/WorkflowTaskFailureLivenessTest.php | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/tests/Unit/V2/WorkflowTaskFailureLivenessTest.php b/tests/Unit/V2/WorkflowTaskFailureLivenessTest.php index 5291ff61..a51a3f40 100644 --- a/tests/Unit/V2/WorkflowTaskFailureLivenessTest.php +++ b/tests/Unit/V2/WorkflowTaskFailureLivenessTest.php @@ -6,14 +6,18 @@ use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Queue; +use Tests\Fixtures\V2\TestAuthoredRetryReplayWorkflow; use Tests\Fixtures\V2\TestGreetingWorkflow; use Tests\TestCase; use Workflow\V2\Contracts\WorkflowTaskBridge; use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Enums\TaskType; +use Workflow\V2\Jobs\RunActivityTask; +use Workflow\V2\Jobs\RunWorkflowTask; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowRunSummary; use Workflow\V2\Models\WorkflowTask; +use Workflow\V2\Support\ActivityOptions; use Workflow\V2\Support\HealthCheck; use Workflow\V2\Support\WorkflowTaskProblem; use Workflow\V2\WorkflowStub; @@ -22,6 +26,7 @@ final class WorkflowTaskFailureLivenessTest extends TestCase { protected function tearDown(): void { + TestAuthoredRetryReplayWorkflow::$overrides = null; Carbon::setTestNow(); parent::tearDown(); } @@ -87,6 +92,69 @@ public function testServiceWorkerFailuresAndRepairKeepHistoricalAndActiveProblem ], $this->classificationFor($runId)); } + public function testEmbeddedReplayBlockClearsOnCompatibleRepairAndCompletion(): void + { + config()->set('queue.default', 'database'); + Queue::fake(); + Carbon::setTestNow('2026-11-05T03:00:00Z'); + $workflow = WorkflowStub::make(TestAuthoredRetryReplayWorkflow::class, 'embedded-replay-block-classification'); + $workflow->start(120, 'direct'); + $runId = $workflow->runId(); + $this->assertIsString($runId); + $this->runReadyTask($runId, TaskType::Workflow); + $this->runReadyTask($runId, TaskType::Activity); + + TestAuthoredRetryReplayWorkflow::$overrides = new ActivityOptions( + startToCloseTimeout: 120, + scheduleToCloseTimeout: 600, + maxAttempts: 1, + backoff: [1, 2, 5, 10, 15, 30, 60, 121], + ); + $blocked = $this->runReadyTask($runId, TaskType::Workflow); + $this->assertSame(TaskStatus::Failed, $blocked->fresh()->status); + $this->assertTrue($blocked->fresh()->payload['replay_blocked']); + $this->assertSame([ + 'open' => true, + 'liveness' => 'workflow_replay_blocked', + 'task_problem' => true, + 'badge' => 'replay_blocked', + 'repair_needed' => 0, + 'resume_health' => 'ok', + ], $this->classificationFor($runId)); + + TestAuthoredRetryReplayWorkflow::$overrides = null; + $this->assertTrue($workflow->attemptRepair()->accepted()); + $this->assertSame([ + 'open' => true, + 'liveness' => 'workflow_task_ready', + 'task_problem' => true, + 'badge' => 'history', + 'repair_needed' => 0, + 'resume_health' => 'ok', + ], $this->classificationFor($runId)); + $this->runReadyTask($runId, TaskType::Workflow); + $this->assertTrue($workflow->refresh()->completed()); + $this->assertSame([ + 'open' => false, + 'liveness' => 'closed', + 'task_problem' => true, + 'badge' => 'history', + 'repair_needed' => 0, + 'resume_health' => 'ok', + ], $this->classificationFor($runId)); + } + + private function runReadyTask(string $runId, TaskType $type): WorkflowTask + { + $task = WorkflowTask::query()->where('workflow_run_id', $runId) + ->where('task_type', $type->value) + ->where('status', TaskStatus::Ready->value)->sole(); + $job = $type === TaskType::Activity ? new RunActivityTask($task->id) : new RunWorkflowTask($task->id); + $this->app->call([$job, 'handle']); + + return $task; + } + /** * @return array{open: bool, liveness: string|null, task_problem: bool, badge: string|null, repair_needed: int, resume_health: string} */ From 10287ad852adf4c245aa67d5428f29826a0ef444 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 10:47:09 +0000 Subject: [PATCH 3/6] docs(protocol): propose workflow task failure episode contract vectors --- ...flow-task-failure-episodes-v1-vectors.json | 1134 +++++++++++++++++ 1 file changed, 1134 insertions(+) create mode 100644 resources/protocol/workflow-task-failure-episodes-v1-vectors.json diff --git a/resources/protocol/workflow-task-failure-episodes-v1-vectors.json b/resources/protocol/workflow-task-failure-episodes-v1-vectors.json new file mode 100644 index 00000000..5ff2f2f4 --- /dev/null +++ b/resources/protocol/workflow-task-failure-episodes-v1-vectors.json @@ -0,0 +1,1134 @@ +{ + "contract": "workflow-task-failure-episodes", + "format_version": 1, + "status": "source_proposal_not_enabled", + "normalized_transitions_are_wire_messages": false, + "proposed_operator_metric_path": "workflow_task_failure_episodes", + "proposed_fields": { + "evaluation": "complete | partial | unavailable", + "count": "Nonnegative count of authoritative open runs with an unresolved episode; null when membership is unavailable.", + "oldest_started_at": "UTC first accepted failure of the oldest active episode; null for no episodes or any unknown age.", + "max_age_ms": "Whole elapsed milliseconds at sample_at; zero only for a proven empty set or a known sub-millisecond age; null for unknown/unavailable age.", + "age_unknown_count": "Number of known active runs without a qualified first-failure clock; null if membership is unavailable." + }, + "authority": { + "clock": "Backend-authoritative acceptance/commit clock, persisted atomically with the first accepted failure.", + "membership": "Existing run/namespace/task/attempt/owner/deadline fences apply before any episode mutation. One open run counts once.", + "anchor": "Durable per-run first-failure/reset authority or equivalent bounded projection; task ancestry, generic created_at/updated_at and historical task_problem/attempt counts are insufficient.", + "repeat": "Preserve the first timestamp across accepted failures, retry, claim, heartbeat, replay blocking and automatic delivery/lease recovery.", + "reset": "Only applied failure-repair writer intent, new committed successful task completion/progress checkpoint, supported recorded waiting acknowledgement completing its task, or closure.", + "repair_origin": "Explicit writer intent distinguishes failure_episode_reset from delivery_recovery. Event names, repair_count and HTTP success do not establish intent.", + "noop": "Refused/stale/wrong-owner/wrong-attempt/duplicate/no-effect reports and repair_not_needed neither begin nor clear an episode.", + "legacy": "Do not invent an anchor for an existing run. Known active membership with unknown age is partial; unqualified membership or an unsupported backend is unavailable.", + "aggregate": "If any active age is unknown, global oldest_started_at/max_age_ms are null. A known subset is not the oldest age of the full set.", + "retention": "Closed runs are excluded. Missing active ancestry is incomplete evidence, not age zero. Future pruning must preserve any admitted anchor.", + "advisory": "Proposed dedicated health check is warning for a positive count, partial or unavailable evaluation; complete zero is OK. This is advisory, with existing HTTP failure rules preserved.", + "consumer": "Absent family means unavailable, never count zero. Dashboard label is Workflow task failures; an unknown age is displayed as unavailable." + }, + "adoption_gates": [ + "General/source contract review", + "Complete Core and Server writer/reset inventory", + "Existing-run and mixed-version compatibility policy", + "Backend qualification and downstream consumer review", + "Normal integration/release authorization" + ], + "vectors": [ + { + "id": "open_without_failure", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": false, + "first_failure_at": null + }, + "transitions": [], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": false, + "first_failure_at": null + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 0, + "oldest_started_at": null, + "max_age_ms": 0, + "age_unknown_count": 0 + }, + "expected_health": "ok" + }, + { + "id": "ordinary_first_failure_php", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": false, + "first_failure_at": null + }, + "transitions": [ + { + "kind": "ordinary_failure", + "at": "2026-11-05T12:00:01.000000Z", + "authority": "current", + "recorded": true + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z", + "liveness": "repair_needed" + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 1, + "oldest_started_at": "2026-11-05T12:00:01.000000Z", + "max_age_ms": 9000, + "age_unknown_count": 0 + }, + "expected_health": "warning" + }, + { + "id": "ordinary_first_failure_native_ready_retry", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": false, + "first_failure_at": null + }, + "transitions": [ + { + "kind": "ordinary_failure", + "at": "2026-11-05T12:00:01.000000Z", + "authority": "current", + "recorded": true + }, + { + "kind": "automatic_retry", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "current", + "recorded": true + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z", + "liveness": "workflow_task_ready" + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 1, + "oldest_started_at": "2026-11-05T12:00:01.000000Z", + "max_age_ms": 9000, + "age_unknown_count": 0 + }, + "expected_health": "warning" + }, + { + "id": "failure_without_retry_successor", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": false, + "first_failure_at": null + }, + "transitions": [ + { + "kind": "ordinary_failure", + "at": "2026-11-05T12:00:01.000000Z", + "authority": "current", + "recorded": true, + "retry_successor_created": false, + "another_workflow_task_open": true + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 1, + "oldest_started_at": "2026-11-05T12:00:01.000000Z", + "max_age_ms": 9000, + "age_unknown_count": 0 + }, + "expected_health": "warning" + }, + { + "id": "repeated_failure_keeps_first_timestamp", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + }, + "transitions": [ + { + "kind": "ordinary_failure", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "current", + "recorded": true + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 1, + "oldest_started_at": "2026-11-05T12:00:01.000000Z", + "max_age_ms": 9000, + "age_unknown_count": 0 + }, + "expected_health": "warning" + }, + { + "id": "first_replay_block", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": false, + "first_failure_at": null + }, + "transitions": [ + { + "kind": "replay_block", + "at": "2026-11-05T12:00:01.000000Z", + "authority": "current", + "recorded": true + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z", + "liveness": "workflow_replay_blocked" + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 1, + "oldest_started_at": "2026-11-05T12:00:01.000000Z", + "max_age_ms": 9000, + "age_unknown_count": 0 + }, + "expected_health": "warning" + }, + { + "id": "retry_then_replay_block_preserves_episode", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + }, + "transitions": [ + { + "kind": "replay_block", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "current", + "recorded": true + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z", + "liveness": "workflow_replay_blocked" + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 1, + "oldest_started_at": "2026-11-05T12:00:01.000000Z", + "max_age_ms": 9000, + "age_unknown_count": 0 + }, + "expected_health": "warning" + }, + { + "id": "accepted_repair_not_yet_applied", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + }, + "transitions": [ + { + "kind": "repair_command_accepted", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "current", + "recorded": true, + "applied": false + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 1, + "oldest_started_at": "2026-11-05T12:00:01.000000Z", + "max_age_ms": 9000, + "age_unknown_count": 0 + }, + "expected_health": "warning" + }, + { + "id": "effective_failure_repair_clears", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + }, + "transitions": [ + { + "kind": "failure_repair_applied", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "current", + "recorded": true, + "writer_intent": "failure_episode_reset" + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": false, + "first_failure_at": null, + "task_problem_history": true + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 0, + "oldest_started_at": null, + "max_age_ms": 0, + "age_unknown_count": 0 + }, + "expected_health": "ok" + }, + { + "id": "repair_not_needed_keeps_episode", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + }, + "transitions": [ + { + "kind": "repair_not_needed", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "current", + "recorded": true, + "state_changed": false + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 1, + "oldest_started_at": "2026-11-05T12:00:01.000000Z", + "max_age_ms": 9000, + "age_unknown_count": 0 + }, + "expected_health": "warning" + }, + { + "id": "automatic_lease_recovery_keeps_episode", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + }, + "transitions": [ + { + "kind": "automatic_lease_recovery", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "current", + "recorded": true, + "writer_intent": "delivery_recovery", + "emitted_event": "RepairRequested", + "repair_count_incremented": true + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 1, + "oldest_started_at": "2026-11-05T12:00:01.000000Z", + "max_age_ms": 9000, + "age_unknown_count": 0 + }, + "expected_health": "warning" + }, + { + "id": "abandoned_task_recovery_keeps_episode", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + }, + "transitions": [ + { + "kind": "abandoned_task_recovery", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "current", + "recorded": true, + "writer_intent": "delivery_recovery", + "emitted_event": "RepairRequested", + "repair_count_incremented": true + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 1, + "oldest_started_at": "2026-11-05T12:00:01.000000Z", + "max_age_ms": 9000, + "age_unknown_count": 0 + }, + "expected_health": "warning" + }, + { + "id": "delivery_redispatch_keeps_episode", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + }, + "transitions": [ + { + "kind": "delivery_redispatch", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "current", + "recorded": true, + "writer_intent": "delivery_recovery", + "emitted_event": "RepairRequested", + "repair_count_incremented": true + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 1, + "oldest_started_at": "2026-11-05T12:00:01.000000Z", + "max_age_ms": 9000, + "age_unknown_count": 0 + }, + "expected_health": "warning" + }, + { + "id": "waiting_history_completed_ack_clears", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + }, + "transitions": [ + { + "kind": "waiting_for_history_ack", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "current", + "recorded": true, + "task_completed": true, + "committed": true + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": false, + "first_failure_at": null, + "liveness": "workflow_waiting", + "task_problem_history": true + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 0, + "oldest_started_at": null, + "max_age_ms": 0, + "age_unknown_count": 0 + }, + "expected_health": "ok" + }, + { + "id": "refused_wait_ack_cannot_clear", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + }, + "transitions": [ + { + "kind": "waiting_for_history_ack", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "stale", + "recorded": false, + "task_completed": false + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 1, + "oldest_started_at": "2026-11-05T12:00:01.000000Z", + "max_age_ms": 9000, + "age_unknown_count": 0 + }, + "expected_health": "warning" + }, + { + "id": "new_completion_clears", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + }, + "transitions": [ + { + "kind": "task_completion", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "current", + "recorded": true, + "committed": true + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": false, + "first_failure_at": null, + "task_problem_history": true + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 0, + "oldest_started_at": null, + "max_age_ms": 0, + "age_unknown_count": 0 + }, + "expected_health": "ok" + }, + { + "id": "duplicate_completion_cannot_clear_later_episode", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + }, + "transitions": [ + { + "kind": "task_completion", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "current", + "recorded": false, + "receipt_replay": true, + "committed": false + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 1, + "oldest_started_at": "2026-11-05T12:00:01.000000Z", + "max_age_ms": 9000, + "age_unknown_count": 0 + }, + "expected_health": "warning" + }, + { + "id": "new_progress_checkpoint_clears", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + }, + "transitions": [ + { + "kind": "progress_checkpoint", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "current", + "recorded": true, + "committed": true, + "state_changed": true + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": false, + "first_failure_at": null, + "task_problem_history": true + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 0, + "oldest_started_at": null, + "max_age_ms": 0, + "age_unknown_count": 0 + }, + "expected_health": "ok" + }, + { + "id": "duplicate_checkpoint_keeps_episode", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + }, + "transitions": [ + { + "kind": "progress_checkpoint", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "current", + "recorded": false, + "committed": false, + "state_changed": false + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 1, + "oldest_started_at": "2026-11-05T12:00:01.000000Z", + "max_age_ms": 9000, + "age_unknown_count": 0 + }, + "expected_health": "warning" + }, + { + "id": "stale_failure_cannot_begin_episode", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": false, + "first_failure_at": null + }, + "transitions": [ + { + "kind": "ordinary_failure", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "wrong_attempt", + "recorded": false + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": false, + "first_failure_at": null + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 0, + "oldest_started_at": null, + "max_age_ms": 0, + "age_unknown_count": 0 + }, + "expected_health": "ok" + }, + { + "id": "run_closure_excludes_historical_failure", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + }, + "transitions": [ + { + "kind": "run_closure", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "current", + "recorded": true + } + ], + "after": [ + { + "id": "run-a", + "open": false, + "episode_active": false, + "first_failure_at": null, + "retained_failed_tasks": 9, + "task_problem_history": true + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 0, + "oldest_started_at": null, + "max_age_ms": 0, + "age_unknown_count": 0 + }, + "expected_health": "ok" + }, + { + "id": "known_active_legacy_episode_unknown_age", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": null, + "age_authority": "unknown" + }, + "transitions": [], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": null, + "age_authority": "unknown" + } + ], + "expected_metric": { + "evaluation": "partial", + "count": 1, + "oldest_started_at": null, + "max_age_ms": null, + "age_unknown_count": 1 + }, + "expected_health": "warning" + }, + { + "id": "future_clock_does_not_report_zero_age", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:20.000000Z" + }, + "transitions": [], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:20.000000Z" + } + ], + "expected_metric": { + "evaluation": "partial", + "count": 1, + "oldest_started_at": null, + "max_age_ms": null, + "age_unknown_count": 1 + }, + "expected_health": "warning" + }, + { + "id": "unsupported_backend_is_unavailable", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "unsupported", + "before": null, + "transitions": [], + "after": [], + "expected_metric": { + "evaluation": "unavailable", + "count": null, + "oldest_started_at": null, + "max_age_ms": null, + "age_unknown_count": null + }, + "expected_health": "warning" + }, + { + "id": "unknown_membership_is_unavailable", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "membership": "unknown" + }, + "transitions": [], + "after": [ + { + "id": "run-a", + "open": true, + "membership": "unknown" + } + ], + "expected_metric": { + "evaluation": "unavailable", + "count": null, + "oldest_started_at": null, + "max_age_ms": null, + "age_unknown_count": null + }, + "expected_health": "warning" + }, + { + "id": "aggregate_counts_open_runs_once", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": null, + "transitions": [], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z", + "retained_failed_tasks": 8 + }, + { + "id": "run-b", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:05.000000Z", + "liveness": "workflow_task_ready" + }, + { + "id": "run-closed", + "open": false, + "episode_active": false, + "first_failure_at": null, + "retained_failed_tasks": 99 + }, + { + "id": "run-history", + "open": true, + "episode_active": false, + "first_failure_at": null, + "task_problem_history": true + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 2, + "oldest_started_at": "2026-11-05T12:00:01.000000Z", + "max_age_ms": 9000, + "age_unknown_count": 0 + }, + "expected_health": "warning" + }, + { + "id": "any_unknown_age_makes_global_oldest_unknown", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": null, + "transitions": [], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + }, + { + "id": "run-b", + "open": true, + "episode_active": true, + "first_failure_at": null, + "age_authority": "unknown" + } + ], + "expected_metric": { + "evaluation": "partial", + "count": 2, + "oldest_started_at": null, + "max_age_ms": null, + "age_unknown_count": 1 + }, + "expected_health": "warning" + }, + { + "id": "effective_repair_then_new_failure_starts_new_clock", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + }, + "transitions": [ + { + "kind": "failure_repair_applied", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "current", + "recorded": true, + "writer_intent": "failure_episode_reset" + }, + { + "kind": "ordinary_failure", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "current", + "recorded": true + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:05.000000Z" + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 1, + "oldest_started_at": "2026-11-05T12:00:05.000000Z", + "max_age_ms": 5000, + "age_unknown_count": 0 + }, + "expected_health": "warning" + }, + { + "id": "wrong_namespace_failure_cannot_begin_episode", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": false, + "first_failure_at": null + }, + "transitions": [ + { + "kind": "ordinary_failure", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "wrong_namespace", + "recorded": false + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": false, + "first_failure_at": null + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 0, + "oldest_started_at": null, + "max_age_ms": 0, + "age_unknown_count": 0 + }, + "expected_health": "ok" + }, + { + "id": "wrong_owner_failure_cannot_begin_episode", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": false, + "first_failure_at": null + }, + "transitions": [ + { + "kind": "ordinary_failure", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "wrong_owner", + "recorded": false + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": false, + "first_failure_at": null + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 0, + "oldest_started_at": null, + "max_age_ms": 0, + "age_unknown_count": 0 + }, + "expected_health": "ok" + }, + { + "id": "deadline_closure_wins_before_failure", + "sample_at": "2026-11-05T12:00:10.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z" + }, + "transitions": [ + { + "kind": "deadline_closure", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "current", + "recorded": true + }, + { + "kind": "ordinary_failure", + "at": "2026-11-05T12:00:05.000000Z", + "authority": "run_closed", + "recorded": false + } + ], + "after": [ + { + "id": "run-a", + "open": false, + "episode_active": false, + "first_failure_at": null, + "retained_failed_tasks": 9, + "task_problem_history": true + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 0, + "oldest_started_at": null, + "max_age_ms": 0, + "age_unknown_count": 0 + }, + "expected_health": "ok" + } + ] +} From d8f01a34d47bf6fccbcb28b8573af6e1bf0ff501 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:38:59 +0000 Subject: [PATCH 4/6] Consume failure episode vectors with closed semantic controls --- .github/pr-test-selection.json | 4 +- ...flow-task-failure-episodes-v1-vectors.json | 108 ++++++ .../WorkflowTaskFailureEpisodeVectors.php | 339 ++++++++++++++++++ .../WorkflowTaskFailureEpisodeVectorTest.php | 140 ++++++++ 4 files changed, 590 insertions(+), 1 deletion(-) create mode 100644 tests/Support/WorkflowTaskFailureEpisodeVectors.php create mode 100644 tests/Unit/V2/WorkflowTaskFailureEpisodeVectorTest.php diff --git a/.github/pr-test-selection.json b/.github/pr-test-selection.json index f6dc75a8..f00b786b 100644 --- a/.github/pr-test-selection.json +++ b/.github/pr-test-selection.json @@ -31,7 +31,9 @@ "tests/Unit/V2/WorkflowHistoryPayloadCastTest.php", "tests/Unit/V2/WorkflowInstanceIdTest.php", "tests/Unit/V2/WorkflowScheduleDaylightSavingTest.php", - "tests/Unit/V2/WorkflowStepHistoryTest.php" + "tests/Unit/V2/WorkflowStepHistoryTest.php", + "tests/Unit/V2/WorkflowTaskFailureEpisodeVectorTest.php", + "tests/Unit/V2/WorkflowTaskFailureLivenessTest.php" ], "mysql_smoke": [ { diff --git a/resources/protocol/workflow-task-failure-episodes-v1-vectors.json b/resources/protocol/workflow-task-failure-episodes-v1-vectors.json index 5ff2f2f4..9a21a6cd 100644 --- a/resources/protocol/workflow-task-failure-episodes-v1-vectors.json +++ b/resources/protocol/workflow-task-failure-episodes-v1-vectors.json @@ -1129,6 +1129,114 @@ "age_unknown_count": 0 }, "expected_health": "ok" + }, + { + "id": "active_first_failure_at_sample", + "sample_at": "2026-11-05T12:00:01.000000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": false, + "first_failure_at": null + }, + "transitions": [ + { + "kind": "ordinary_failure", + "at": "2026-11-05T12:00:01.000000Z", + "authority": "current", + "recorded": true + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z", + "liveness": "repair_needed" + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 1, + "oldest_started_at": "2026-11-05T12:00:01.000000Z", + "max_age_ms": 0, + "age_unknown_count": 0 + }, + "expected_health": "warning" + }, + { + "id": "active_first_failure_plus_999us", + "sample_at": "2026-11-05T12:00:01.000999Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": false, + "first_failure_at": null + }, + "transitions": [ + { + "kind": "ordinary_failure", + "at": "2026-11-05T12:00:01.000000Z", + "authority": "current", + "recorded": true + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z", + "liveness": "repair_needed" + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 1, + "oldest_started_at": "2026-11-05T12:00:01.000000Z", + "max_age_ms": 0, + "age_unknown_count": 0 + }, + "expected_health": "warning" + }, + { + "id": "active_first_failure_plus_1000us", + "sample_at": "2026-11-05T12:00:01.001000Z", + "backend_support": "supported", + "before": { + "id": "run-a", + "open": true, + "episode_active": false, + "first_failure_at": null + }, + "transitions": [ + { + "kind": "ordinary_failure", + "at": "2026-11-05T12:00:01.000000Z", + "authority": "current", + "recorded": true + } + ], + "after": [ + { + "id": "run-a", + "open": true, + "episode_active": true, + "first_failure_at": "2026-11-05T12:00:01.000000Z", + "liveness": "repair_needed" + } + ], + "expected_metric": { + "evaluation": "complete", + "count": 1, + "oldest_started_at": "2026-11-05T12:00:01.000000Z", + "max_age_ms": 1, + "age_unknown_count": 0 + }, + "expected_health": "warning" } ] } diff --git a/tests/Support/WorkflowTaskFailureEpisodeVectors.php b/tests/Support/WorkflowTaskFailureEpisodeVectors.php new file mode 100644 index 00000000..c5ff503c --- /dev/null +++ b/tests/Support/WorkflowTaskFailureEpisodeVectors.php @@ -0,0 +1,339 @@ +contract === 'workflow-task-failure-episodes' + && $document->format_version === 1 && $document->status === 'source_proposal_not_enabled' + && $document->normalized_transitions_are_wire_messages === false + && $document->proposed_operator_metric_path === 'workflow_task_failure_episodes', 'Source boundary changed'); + self::fields( + $document->proposed_fields, + ['evaluation', 'count', 'oldest_started_at', 'max_age_ms', 'age_unknown_count'] + ); + self::fields($document->authority, ['clock', 'membership', 'anchor', 'repeat', 'reset', 'repair_origin', + 'noop', 'legacy', 'aggregate', 'retention', 'advisory', 'consumer']); + foreach ([$document->proposed_fields, $document->authority] as $descriptions) { + foreach (get_object_vars($descriptions) as $description) { + self::require(is_string($description) && trim($description) !== '', 'Empty contract description'); + } + } + self::require( + is_array($document->adoption_gates) && $document->adoption_gates !== [], + 'Missing adoption gates' + ); + foreach ($document->adoption_gates as $gate) { + self::require(is_string($gate) && trim($gate) !== '', 'Invalid adoption gate'); + } + self::require(is_array($document->vectors) && $document->vectors !== [] + && count($document->vectors) <= 128, 'Invalid vector list'); + $ids = []; + foreach ($document->vectors as $vector) { + self::fields($vector, ['id', 'sample_at', 'backend_support', 'before', 'transitions', 'after', + 'expected_metric', 'expected_health']); + self::require(is_string($vector->id) && preg_match('/\A[a-z][a-z0-9_]*\z/D', $vector->id) === 1 + && ! isset($ids[$vector->id]), 'Invalid or duplicate vector ID'); + $ids[$vector->id] = true; + self::vector($vector); + } + + return count($ids); + } + + private static function vector(stdClass $vector): void + { + $sample = self::micros($vector->sample_at); + self::require( + in_array($vector->backend_support, ['supported', 'unsupported'], true), + 'Invalid backend support' + ); + self::require(is_array($vector->transitions) && count($vector->transitions) <= 128 + && is_array($vector->after) && count($vector->after) <= 128, 'Invalid transitions or run list'); + if ($vector->before !== null) { + self::state($vector->before); + } + $projected = $vector->before; + $last = null; + foreach ($vector->transitions as $transition) { + $at = self::transition($transition); + self::require($at <= $sample && ($last === null || $at >= $last), 'Transition clock is out of order'); + self::require( + $projected instanceof stdClass && ! isset($projected->membership), + 'Transition lacks run authority' + ); + $last = $at; + $projected = self::apply($projected, $transition); + } + $runs = []; + foreach ($vector->after as $state) { + self::state($state); + self::require(is_string($state->id ?? null) && $state->id !== '', 'Run membership lacks an ID'); + self::require(! isset($runs[$state->id]), 'Duplicate run membership'); + $runs[$state->id] = $state; + } + if (isset($projected->membership)) { + self::require( + (bool) array_filter($vector->after, static fn (stdClass $run): bool => isset($run->membership)), + 'Unknown membership cannot disappear without authoritative evaluation' + ); + } + if ($projected !== null && ! isset($projected->membership)) { + self::require(isset($runs[$projected->id]), 'Projected run is missing'); + foreach (['open', 'episode_active', 'first_failure_at'] as $field) { + self::require( + $runs[$projected->id]->{$field} === $projected->{$field}, + 'Transition projection differs: ' . $field + ); + } + } + $metric = self::metric($vector->backend_support, $vector->after, $sample); + self::fields($vector->expected_metric, array_keys($metric)); + foreach ($metric as $field => $value) { + self::require($vector->expected_metric->{$field} === $value, $vector->id . ': metric differs: ' . $field); + } + $health = $metric['evaluation'] === 'complete' && $metric['count'] === 0 ? 'ok' : 'warning'; + self::require($vector->expected_health === $health, 'Health must preserve unknown and positive episodes'); + } + + private static function state(mixed $state): void + { + if ($state instanceof stdClass && isset($state->membership)) { + self::fields($state, ['membership'], ['id', 'open']); + self::require($state->membership === 'unknown', 'Invalid membership'); + if (property_exists($state, 'id')) { + self::require(is_string($state->id) && $state->id !== '', 'Invalid run ID'); + } + if (property_exists($state, 'open')) { + self::require(is_bool($state->open), 'Invalid open flag'); + } + + return; + } + self::fields( + $state, + ['id', 'open', 'episode_active', 'first_failure_at'], + ['age_authority', 'liveness', 'task_problem_history', 'retained_failed_tasks'] + ); + self::require(is_string($state->id) && $state->id !== '' && is_bool($state->open) + && is_bool($state->episode_active), 'Invalid run state'); + self::require($state->open || ! $state->episode_active, 'Closed run cannot have an active episode'); + self::require($state->episode_active || $state->first_failure_at === null, 'Inactive episode has an anchor'); + if ($state->first_failure_at !== null) { + self::micros($state->first_failure_at); + } + if (property_exists($state, 'age_authority')) { + self::require($state->age_authority === 'unknown' && $state->episode_active + && $state->first_failure_at === null, 'Invalid unknown-age authority'); + } + self::require(! $state->episode_active || $state->first_failure_at !== null + || ($state->age_authority ?? null) === 'unknown', 'Missing active age authority'); + if (property_exists($state, 'liveness')) { + self::require(in_array($state->liveness, ['repair_needed', 'workflow_task_ready', + 'workflow_replay_blocked', 'workflow_waiting'], true), 'Invalid liveness annotation'); + } + if (property_exists($state, 'task_problem_history')) { + self::require(is_bool($state->task_problem_history), 'Invalid historical badge'); + } + if (property_exists($state, 'retained_failed_tasks')) { + self::require( + is_int($state->retained_failed_tasks) && $state->retained_failed_tasks >= 0, + 'Invalid retained count' + ); + } + } + + private static function transition(mixed $transition): int + { + self::require($transition instanceof stdClass && is_string($transition->kind ?? null), 'Invalid transition'); + $extras = match ($transition->kind) { + 'ordinary_failure' => [[], ['retry_successor_created', 'another_workflow_task_open']], + 'automatic_retry', 'replay_block', 'run_closure', 'deadline_closure' => [[], []], + 'repair_command_accepted' => [['applied'], []], + 'failure_repair_applied' => [['writer_intent'], []], + 'repair_not_needed' => [['state_changed'], []], + 'automatic_lease_recovery', 'abandoned_task_recovery', 'delivery_redispatch' => + [['writer_intent', 'emitted_event', 'repair_count_incremented'], []], + 'waiting_for_history_ack' => [['task_completed'], ['committed']], + 'task_completion' => [['committed'], ['receipt_replay']], + 'progress_checkpoint' => [['committed', 'state_changed'], []], + default => throw new InvalidArgumentException('Unknown transition kind'), + }; + self::fields($transition, array_merge(['kind', 'at', 'authority', 'recorded'], $extras[0]), $extras[1]); + self::require(in_array($transition->authority, ['current', 'stale', 'wrong_attempt', 'wrong_namespace', + 'wrong_owner', 'run_closed'], true) && is_bool($transition->recorded), 'Invalid transition authority'); + self::require( + ! $transition->recorded || $transition->authority === 'current', + 'Refused authority cannot record' + ); + foreach (['applied', 'state_changed', 'retry_successor_created', 'another_workflow_task_open', + 'repair_count_incremented', 'task_completed', 'committed', 'receipt_replay'] as $flag) { + if (property_exists($transition, $flag)) { + self::require(is_bool($transition->{$flag}), 'Invalid transition Boolean: ' . $flag); + } + } + self::require( + ! ($transition->committed ?? false) || $transition->recorded, + 'Unrecorded result cannot commit progress' + ); + if ($transition->kind === 'failure_repair_applied') { + self::require($transition->writer_intent === 'failure_episode_reset', 'Repair lacks reset intent'); + } elseif (isset($transition->writer_intent)) { + self::require($transition->writer_intent === 'delivery_recovery' + && $transition->emitted_event === 'RepairRequested', 'Recovery intent differs'); + } + self::require( + $transition->kind !== 'repair_command_accepted' || ! $transition->applied, + 'Admission is not applied repair' + ); + self::require( + $transition->kind !== 'repair_not_needed' || ! $transition->state_changed, + 'No-op cannot change state' + ); + + return self::micros($transition->at); + } + + private static function apply(stdClass $before, stdClass $transition): stdClass + { + $after = clone $before; + if (! $transition->recorded) { + return $after; + } + self::require($after->open, 'Closed run cannot accept a transition'); + if (in_array($transition->kind, ['ordinary_failure', 'replay_block'], true) && ! $after->episode_active) { + $after->episode_active = true; + $after->first_failure_at = $transition->at; + } + $reset = match ($transition->kind) { + 'failure_repair_applied', 'run_closure', 'deadline_closure' => true, + 'task_completion' => $transition->committed && ! ($transition->receipt_replay ?? false), + 'progress_checkpoint' => $transition->committed && $transition->state_changed, + 'waiting_for_history_ack' => ($transition->committed ?? false) && $transition->task_completed, + default => false, + }; + if ($reset) { + $after->episode_active = false; + $after->first_failure_at = null; + unset($after->age_authority); + } + if (in_array($transition->kind, ['run_closure', 'deadline_closure'], true)) { + $after->open = false; + } + + return $after; + } + + /** @param list $runs + * @return array + */ + private static function metric(string $support, array $runs, int $sample): array + { + $metric = [ + 'evaluation' => 'unavailable', + 'count' => null, + 'oldest_started_at' => null, + 'max_age_ms' => null, + 'age_unknown_count' => null, + ]; + if ($support !== 'supported' || array_filter( + $runs, + static fn (stdClass $run): bool => isset($run->membership) + )) { + return $metric; + } + $active = array_filter($runs, static fn (stdClass $run): bool => $run->open && $run->episode_active); + $unknown = 0; + $oldest = null; + foreach ($active as $run) { + if ($run->first_failure_at === null || self::micros($run->first_failure_at) > $sample) { + ++$unknown; + } elseif ($oldest === null || $run->first_failure_at < $oldest) { + $oldest = $run->first_failure_at; + } + } + + return [ + 'evaluation' => $unknown > 0 ? 'partial' : 'complete', + 'count' => count($active), + 'oldest_started_at' => $unknown > 0 ? null : $oldest, + 'max_age_ms' => $unknown > 0 ? null : ($oldest === null ? 0 : intdiv( + $sample - self::micros($oldest), + 1000 + )), + 'age_unknown_count' => $unknown, + ]; + } + + private static function micros(mixed $value): int + { + self::require( + is_string($value) && preg_match('/\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z\z/D', $value) === 1, + 'Timestamp must be exact UTC microseconds' + ); + $date = DateTimeImmutable::createFromFormat('!Y-m-d\TH:i:s.u\Z', $value, new \DateTimeZone('UTC')); + self::require($date !== false && $date->format('Y-m-d\TH:i:s.u\Z') === $value, 'Invalid timestamp'); + + return (int) $date->format('U') * 1000000 + (int) $date->format('u'); + } + + /** @param list $required + * @param list $optional + */ + private static function fields(mixed $value, array $required, array $optional = []): void + { + self::require($value instanceof stdClass, 'Expected a JSON object'); + $keys = array_keys(get_object_vars($value)); + self::require(array_diff($required, $keys) === [] + && array_diff($keys, array_merge($required, $optional)) === [], 'Missing or unknown object member'); + } + + private static function uniqueObjectKeys(string $json): void + { + preg_match_all('/"(?:[^"\\\\]|\\\\.)*"|[{}\[\]:,]/s', $json, $matches); + $stack = []; + foreach ($matches[0] as $token) { + if ($token === '{' || $token === '[') { + $stack[] = [ + 'object' => $token === '{', + 'key' => true, + 'seen' => [], + ]; + } elseif ($token === '}' || $token === ']') { + array_pop($stack); + } elseif ($stack !== []) { + $index = array_key_last($stack); + if ($token === ',') { + $stack[$index]['key'] = true; + } elseif ($token === ':') { + $stack[$index]['key'] = false; + } elseif ($token[0] === '"' && $stack[$index]['object'] && $stack[$index]['key']) { + $key = json_decode($token, true, 2, JSON_THROW_ON_ERROR); + self::require(! isset($stack[$index]['seen'][$key]), 'Duplicate JSON object member'); + $stack[$index]['seen'][$key] = true; + } + } + } + } + + private static function require(bool $condition, string $message): void + { + if (! $condition) { + throw new InvalidArgumentException($message); + } + } +} diff --git a/tests/Unit/V2/WorkflowTaskFailureEpisodeVectorTest.php b/tests/Unit/V2/WorkflowTaskFailureEpisodeVectorTest.php new file mode 100644 index 00000000..86d6eedc --- /dev/null +++ b/tests/Unit/V2/WorkflowTaskFailureEpisodeVectorTest.php @@ -0,0 +1,140 @@ +assertSame(34, WorkflowTaskFailureEpisodeVectors::validate($this->source())); + $document = json_decode($this->source(), false, 64, JSON_THROW_ON_ERROR); + $vectors = array_column($document->vectors, null, 'id'); + foreach ([ + 'active_first_failure_at_sample' => 0, + 'active_first_failure_plus_999us' => 0, + 'active_first_failure_plus_1000us' => 1, + ] as $id => $age) { + $metric = $vectors[$id]->expected_metric; + $this->assertSame('complete', $metric->evaluation); + $this->assertSame(1, $metric->count); + $this->assertSame('2026-11-05T12:00:01.000000Z', $metric->oldest_started_at); + $this->assertSame($age, $metric->max_age_ms); + $this->assertSame('warning', $vectors[$id]->expected_health); + } + } + + public function testTheValidatorRefusesSchemaAndSemanticMutations(): void + { + $mutations = [ + 'unknown root member' => static function (stdClass $document): void { + $document->enabled = true; + }, + 'enabled source' => static function (stdClass $document): void { + $document->status = 'enabled'; + }, + 'empty vector list' => static function (stdClass $document): void { + $document->vectors = []; + }, + 'duplicate ID' => static function (stdClass $document): void { + $document->vectors[] = $document->vectors[0]; + }, + 'unknown vector member' => static function (stdClass $document): void { + $document->vectors[0]->extra = 1; + }, + 'missing metric field' => static function (stdClass $document): void { + unset($document->vectors[0]->expected_metric->count); + }, + 'Boolean metric count' => static function (stdClass $document): void { + $document->vectors[0]->expected_metric->count = false; + }, + 'invalid timestamp' => static function (stdClass $document): void { + $document->vectors[0]->sample_at = '2026-13-05T12:00:10.000000Z'; + }, + 'unknown state member' => static function (stdClass $document): void { + $document->vectors[0]->after[0]->new_clock = 0; + }, + 'duplicate run membership' => static function (stdClass $document): void { + $document->vectors[0]->after[] = $document->vectors[0]->after[0]; + }, + 'unknown transition member' => static function (stdClass $document): void { + $document->vectors[1]->transitions[0]->extra = true; + }, + 'unknown transition kind' => static function (stdClass $document): void { + $document->vectors[1]->transitions[0]->kind = 'guess'; + }, + 'refused failure recording' => static function (stdClass $document): void { + $document->vectors[1]->transitions[0]->authority = 'wrong_owner'; + }, + 'wrong failure projection' => static function (stdClass $document): void { + $document->vectors[1]->after[0]->first_failure_at = '2026-11-05T12:00:02.000000Z'; + }, + ]; + foreach ($mutations as $label => $mutation) { + $document = json_decode($this->source(), false, 64, JSON_THROW_ON_ERROR); + $mutation($document); + $this->assertRefused(json_encode($document, JSON_THROW_ON_ERROR), $label); + } + $document = json_decode($this->source(), false, 64, JSON_THROW_ON_ERROR); + foreach ($document->vectors as $vector) { + if ($vector->id === 'active_first_failure_plus_999us') { + $vector->expected_metric->max_age_ms = 1; + } + } + $this->assertRefused(json_encode($document, JSON_THROW_ON_ERROR), 'ceiling instead of flooring'); + foreach (['future_clock_does_not_report_zero_age', 'unknown_membership_is_unavailable', + 'automatic_lease_recovery_keeps_episode', 'waiting_history_completed_ack_clears', + 'duplicate_completion_cannot_clear_later_episode'] as $id) { + $document = json_decode($this->source(), false, 64, JSON_THROW_ON_ERROR); + foreach ($document->vectors as $vector) { + if ($vector->id !== $id) { + continue; + } + if ($id === 'future_clock_does_not_report_zero_age') { + $vector->expected_metric->max_age_ms = 0; + } elseif ($id === 'unknown_membership_is_unavailable') { + $vector->after = []; + } elseif ($id === 'automatic_lease_recovery_keeps_episode') { + $vector->after[0]->episode_active = false; + $vector->after[0]->first_failure_at = null; + } elseif ($id === 'waiting_history_completed_ack_clears') { + $vector->transitions[0]->committed = false; + } else { + $vector->transitions[0]->committed = true; + } + } + $this->assertRefused(json_encode($document, JSON_THROW_ON_ERROR), $id); + } + $this->assertRefused( + str_replace('"format_version": 1,', '"format_version": 9, "format_version": 1,', $this->source()), + 'duplicate JSON key cannot disappear during decode' + ); + $this->assertRefused( + str_replace('"format_version": 1,', '"format_versi\\u006fn": 9, "format_version": 1,', $this->source()), + 'escaped duplicate JSON key' + ); + } + + private function assertRefused(string $json, string $label): void + { + try { + WorkflowTaskFailureEpisodeVectors::validate($json); + $this->fail($label . ' was accepted'); + } catch (InvalidArgumentException $exception) { + $this->assertNotSame('', $exception->getMessage(), $label); + } + } + + private function source(): string + { + return (string) file_get_contents( + dirname(__DIR__, 3) . '/resources/protocol/workflow-task-failure-episodes-v1-vectors.json' + ); + } +} From 85a7a0f4c4e76a772b0174e9bee2742e464e3df0 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:43:15 +0000 Subject: [PATCH 5/6] Route episode database lifecycle tests through feature selection --- tests/{Unit => Feature}/V2/WorkflowTaskFailureLivenessTest.php | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{Unit => Feature}/V2/WorkflowTaskFailureLivenessTest.php (100%) diff --git a/tests/Unit/V2/WorkflowTaskFailureLivenessTest.php b/tests/Feature/V2/WorkflowTaskFailureLivenessTest.php similarity index 100% rename from tests/Unit/V2/WorkflowTaskFailureLivenessTest.php rename to tests/Feature/V2/WorkflowTaskFailureLivenessTest.php From 909b251c93e318f73b48a6ab57cc0a4578d35e6c Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:43:35 +0000 Subject: [PATCH 6/6] Complete lifecycle namespace and selector routing correction --- .github/pr-test-selection.json | 3 +-- tests/Feature/V2/WorkflowTaskFailureLivenessTest.php | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/pr-test-selection.json b/.github/pr-test-selection.json index f00b786b..7805ab5f 100644 --- a/.github/pr-test-selection.json +++ b/.github/pr-test-selection.json @@ -32,8 +32,7 @@ "tests/Unit/V2/WorkflowInstanceIdTest.php", "tests/Unit/V2/WorkflowScheduleDaylightSavingTest.php", "tests/Unit/V2/WorkflowStepHistoryTest.php", - "tests/Unit/V2/WorkflowTaskFailureEpisodeVectorTest.php", - "tests/Unit/V2/WorkflowTaskFailureLivenessTest.php" + "tests/Unit/V2/WorkflowTaskFailureEpisodeVectorTest.php" ], "mysql_smoke": [ { diff --git a/tests/Feature/V2/WorkflowTaskFailureLivenessTest.php b/tests/Feature/V2/WorkflowTaskFailureLivenessTest.php index a51a3f40..ea76a36d 100644 --- a/tests/Feature/V2/WorkflowTaskFailureLivenessTest.php +++ b/tests/Feature/V2/WorkflowTaskFailureLivenessTest.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Tests\Unit\V2; +namespace Tests\Feature\V2; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Queue;