From e2b9f025d8ed01574269a3f64714732cb5590363 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:55:12 +0000 Subject: [PATCH] Deliver a parent's cooperative cancellation to a retried child A retried child call has a link per attempt, and its call names the latest attempt's run. Native's cooperative cancellation expected one link per call, so it refused a parent's request once a child had been retried. child_link_policies now accepts a retried call's links, and the delivery boundary follows the call to its current run and link, accepting a retry's ChildRunStarted without the call's cancellation policy, as published PHP records it. Under try_cancel the retry run, not the failed first run, is asked to cancel. The retried-child-cancellation fixture records both policies against published 2.11.7. Closes #873. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/server-parity.yml | 2 + .../2026-10-29-retried-child-cancellation.md | 33 +++ docs/rust-server-port.md | 16 +- rust/src/runtime/children.rs | 11 +- rust/src/runtime/cooperative.rs | 83 ++++-- rust/tests/execution/child_retries.rs | 54 ++++ .../conformance/server-parity/contract.mjs | 3 + scripts/conformance/server-parity/probe.php | 6 + .../retried-child-cancellation-contract.mjs | 25 ++ .../retried-child-cancellation-probe.php | 150 +++++++++++ .../retried-child-cancellation.json | 247 ++++++++++++++++++ ...tyRetriedChildCancellationContractTest.mjs | 43 +++ 12 files changed, 642 insertions(+), 31 deletions(-) create mode 100644 docs/rust-server-port-checkpoints/2026-10-29-retried-child-cancellation.md create mode 100644 scripts/conformance/server-parity/retried-child-cancellation-contract.mjs create mode 100644 scripts/conformance/server-parity/retried-child-cancellation-probe.php create mode 100644 tests/Fixtures/ServerParity/retried-child-cancellation.json create mode 100644 tests/Unit/ServerParityRetriedChildCancellationContractTest.mjs diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 60e7067bd..c9b5183f4 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -68,6 +68,7 @@ on: - tests/Unit/ServerParityChildDescriptionsContractTest.mjs - tests/Unit/ServerParityChildRetriesContractTest.mjs - tests/Unit/ServerParityChildVisibilityContractTest.mjs + - tests/Unit/ServerParityRetriedChildCancellationContractTest.mjs - tests/Unit/ServerParityStartSearchAttributesContractTest.mjs - tests/Unit/ServerParityQueryArgumentsContractTest.mjs - tests/Unit/ServerParityQueryContractArgumentsContractTest.mjs @@ -2171,6 +2172,7 @@ jobs: tests/Unit/ServerParityChildDescriptionsContractTest.mjs \ tests/Unit/ServerParityChildRetriesContractTest.mjs \ tests/Unit/ServerParityChildVisibilityContractTest.mjs \ + tests/Unit/ServerParityRetriedChildCancellationContractTest.mjs \ tests/Unit/ServerParityWorkerErrorBodiesContractTest.mjs \ tests/Unit/ServerParityLargeCommandBatchesContractTest.mjs \ tests/Unit/ServerParityManyUpdatesContractTest.mjs \ diff --git a/docs/rust-server-port-checkpoints/2026-10-29-retried-child-cancellation.md b/docs/rust-server-port-checkpoints/2026-10-29-retried-child-cancellation.md new file mode 100644 index 000000000..7d2fe5d9f --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-29-retried-child-cancellation.md @@ -0,0 +1,33 @@ +# October 29, cooperative cancellation of a retried child (#873) + +- **Before.** After #869, a retried child call has a link per attempt, and + the call names the latest attempt's run. Native's cooperative cancellation + expected one link per call: + - `child_link_policies` rejected the call's links, so a parent's + cancellation request was refused (`422 + cooperative_cancellation_propagation_not_available`); + - the delivery boundary also required one link and one `ChildRunStarted` + naming the call's cancellation policy. + + Security's review of #869 found the gap. +- **Contract** (published Server 2.11.7). The request is admitted and + delivered at the child boundary, and the parent completes as cancelled. + - Under `abandon`, the retry keeps running. + - Under `try_cancel`, the parent records `ChildCancellationRequested` + naming the retry run, and that run, not the failed first run, records + `CooperativeCancellationRequested`. + - PHP's retry `ChildRunStarted` carries no `cancellation_policy`. +- **Fixture.** `retried-child-cancellation`, raw worker HTTP on protocol + 1.20 in a fresh namespace, with a cooperative parent worker and a separate + child worker. One case per policy (`abandon`, `try_cancel`). It records + the answers, the parent's child and cancellation events, the child's runs + and the retry run's events. +- **Native.** + - `child_link_policies` accepts a retried call's links: the first under + the call's id, each retry under its own, and earlier attempts. + - The delivery boundary reads every link of the call and picks the one + for the call's current run. It accepts the call's scheduled event, and + one started event per attempt; a retry's may omit the cancellation + policy. +- **Tests.** `a_parents_cooperative_cancellation_reaches_a_retried_child` + (`child_retries.rs`) covers both policies. diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index fc1c4b29c..f34587f7f 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -90,7 +90,6 @@ Updated October 27, 2026. `501 rust_capability_not_implemented` must match [`unserved-php-routes.v2.11.1.txt`](../rust/tests/fixtures/unserved-php-routes.v2.11.1.txt). - **Remaining for the release candidate:** - - landing the 2.11.5 reference move; - the scope question with the operator: prepared local activities, workflow-bound Nexus handlers and external payload takeover; - publishing the native image (General runs the release); @@ -614,9 +613,12 @@ contract fixture before its correction. ([checkpoint](rust-server-port-checkpoints/2026-10-29-child-retries.md)). A child timeout or backoff too far away for a timestamp is refused (`invalid_child_timeout`, `invalid_child_retry_policy`), as a timer delay - is. Remaining: native's cooperative cancellation delivery expects one - link per call, so it refuses one delivered to a retried child - (`cancellation_delivery_shape_mismatch`; #873, pending a PHP recording). + is. A parent's cooperative cancellation reaches a retried child (#873): + the request is admitted, and the delivery follows the call to its current + run and link, so under `try_cancel` the retry run, not the failed first + run, is asked to cancel. The `retried-child-cancellation` fixture records + both policies against published 2.11.7 + ([checkpoint](rust-server-port-checkpoints/2026-10-29-retried-child-cancellation.md)). - **Trace context (Core 2.10, published in Server 2.11.1).** Core adds nullable `observability_context` columns on runs, commands and tasks. Native has no carrier: it accepts and ignores a top-level carrier on @@ -686,9 +688,9 @@ contract fixture before its correction. and PHP's message, and records nothing. - **Cooperative cancellation boundaries.** Published PHP delivers a run's cancellation at an activity, local activity, timer, condition, signal, - child, parallel or selection-handle boundary. Native delivers it at a - timer, a condition or signal wait, a child and a parallel group whose - members are all timers. It refuses the other boundaries, including + child, parallel or selection-handle boundary. Native delivers it at an + activity, a timer, a condition or signal wait, a child (retried or not) + and a parallel group whose members are all timers. It refuses the other boundaries, including parallel groups with other member kinds, with `422 cancellation_delivery_boundary_not_available`. - **Activity results after a deadline, before its timeout runs.** Published diff --git a/rust/src/runtime/children.rs b/rust/src/runtime/children.rs index 1dccfaa1a..c28f239f9 100644 --- a/rust/src/runtime/children.rs +++ b/rust/src/runtime/children.rs @@ -197,14 +197,19 @@ where }; let metadata = DB::document_row(call, "metadata")?; let cancellation = metadata["cancellation_policy"].as_str().unwrap_or("abandon").to_owned(); + // A retried call (`markChildCallRetryStarted`) has a link per attempt: + // the first under the call's id, each retry under its own, and the + // call names the latest attempt's run. + let retried = metadata["attempt_count"].as_i64().is_some_and(|attempts| attempts > 1); let consistent = DB::string(call, "parent_workflow_instance_id")? == DB::string(link, "parent_workflow_instance_id")? && DB::optional_string(call, "resolved_child_instance_id")? == Some(DB::string(link, "child_workflow_instance_id")?) - && DB::optional_string(call, "resolved_child_run_id")? - == Some(DB::string(link, "child_workflow_run_id")?) + && (retried + || DB::optional_string(call, "resolved_child_run_id")? + == Some(DB::string(link, "child_workflow_run_id")?)) && DB::optional_string(call, "parent_close_policy")?.as_deref() == Some(parent_close.as_str()) - && metadata["child_call_id"] == DB::string(link, "id")? + && (retried || metadata["child_call_id"] == DB::string(link, "id")?) && matches!( DB::string(call, "status")?.as_str(), "started" | "completed" | "failed" | "cancelled" diff --git a/rust/src/runtime/cooperative.rs b/rust/src/runtime/cooperative.rs index c4b59dcb9..4c725be39 100644 --- a/rust/src/runtime/cooperative.rs +++ b/rust/src/runtime/cooperative.rs @@ -717,20 +717,40 @@ where request_sequence: i64, ) -> Result { let mismatch = || refuse(StatusCode::CONFLICT, "cancellation_delivery_shape_mismatch"); + // One link per attempt of the call: a retried child + // (`startChildRetryIfAvailable`) adds a link per retry, and the call + // names the current attempt's run. let links = Self::query( - "SELECT * FROM workflow_links WHERE parent_workflow_run_id=$1 AND sequence=$2 LIMIT 2", + "SELECT * FROM workflow_links WHERE parent_workflow_run_id=$1 AND sequence=$2 ORDER BY id LIMIT 101", ) .bind(run_id) .bind(sequence) .fetch_all(&mut **tx) .await?; - let link = links - .first() - .filter(|_| links.len() == 1) - .ok_or_else(mismatch)?; - let (parent_close, cancellation) = Self::child_link_policies(tx, link).await?.ok_or_else(mismatch)?; - let call = Self::query("SELECT child_workflow_type,child_workflow_class FROM workflow_child_calls WHERE parent_workflow_run_id=$1 AND sequence=$2") + if links.is_empty() || links.len() > 100 { + return Err(mismatch()); + } + let call = Self::query("SELECT child_workflow_type,child_workflow_class,resolved_child_run_id,metadata FROM workflow_child_calls WHERE parent_workflow_run_id=$1 AND sequence=$2") .bind(run_id).bind(sequence).fetch_one(&mut **tx).await?; + let metadata = DB::document_row(&call, "metadata")?; + let call_id = metadata["child_call_id"].as_str().map(str::to_owned).ok_or_else(mismatch)?; + let retried = metadata["attempt_count"].as_i64().is_some_and(|attempts| attempts > 1); + if links.len() > 1 && !retried { + return Err(mismatch()); + } + let current_run = DB::optional_string(&call, "resolved_child_run_id")?.ok_or_else(mismatch)?; + let mut current = None; + let mut first = None; + for link in &links { + if DB::string(link, "child_workflow_run_id")? == current_run { + current = Some(link); + } + if DB::string(link, "id")? == call_id { + first = Some(link); + } + } + let (current, first) = (current.ok_or_else(mismatch)?, first.ok_or_else(mismatch)?); + let (parent_close, cancellation) = Self::child_link_policies(tx, current).await?.ok_or_else(mismatch)?; let events = Self::query("SELECT sequence,event_type,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type IN ('ChildWorkflowScheduled','ChildRunStarted','ChildRunCompleted','ChildRunFailed','ChildRunCancelled','ChildRunTerminated') ORDER BY sequence LIMIT 1001") .bind(run_id).fetch_all(&mut **tx).await?; if events.len() > 1000 { @@ -740,7 +760,7 @@ where )); } let mut scheduled = false; - let mut started = false; + let mut started_links: Vec = Vec::new(); let mut resolved_before_request = false; for event in events { let payload = DB::document_row(&event, "payload")?; @@ -749,8 +769,26 @@ where } let kind = DB::string(&event, "event_type")?; if matches!(kind.as_str(), "ChildWorkflowScheduled" | "ChildRunStarted") { - if payload["workflow_link_id"] != DB::string(link, "id")? - || payload["child_call_id"] != DB::string(link, "id")? + // The call's first event names the first link; each started + // event names its own attempt's link and run. + let link = if kind == "ChildWorkflowScheduled" { + first + } else { + let named = payload["workflow_link_id"].as_str().unwrap_or_default(); + let mut found = None; + for link in &links { + if DB::string(link, "id")? == named { + found = Some(link); + } + } + found.ok_or_else(mismatch)? + }; + let link_id = DB::string(link, "id")?; + // PHP records a retry's `ChildRunStarted` without the call's + // cancellation policy. + let retry = kind == "ChildRunStarted" && payload["retry_attempt"].is_i64(); + if payload["workflow_link_id"] != link_id.as_str() + || payload["child_call_id"] != call_id.as_str() || payload["child_workflow_instance_id"] != DB::string(link, "child_workflow_instance_id")? || payload["child_workflow_run_id"] @@ -758,7 +796,8 @@ where || payload["child_workflow_type"] != DB::string(&call, "child_workflow_type")? || payload["child_workflow_class"] != DB::string(&call, "child_workflow_class")? || payload["parent_close_policy"] != parent_close.as_str() - || payload["cancellation_policy"] != cancellation.as_str() + || (payload["cancellation_policy"] != cancellation.as_str() + && !(retry && payload.get("cancellation_policy").is_none())) || payload.get("child_workflow").is_some() { return Err(mismatch()); @@ -772,20 +811,22 @@ where "cancellation_delivery_requires_scope", )); } - let seen = if kind == "ChildWorkflowScheduled" { - &mut scheduled + if kind == "ChildWorkflowScheduled" { + if scheduled { + return Err(mismatch()); + } + scheduled = true; } else { - &mut started - }; - if *seen { - return Err(mismatch()); + if started_links.contains(&link_id) { + return Err(mismatch()); + } + started_links.push(link_id); } - *seen = true; } else if DB::number(&event, "sequence")? < request_sequence { resolved_before_request = true; } } - if !scheduled || !started { + if !scheduled || !started_links.contains(&DB::string(current, "id")?) { return Err(mismatch()); } if resolved_before_request { @@ -795,8 +836,8 @@ where )); } Ok(super::child_cancellation::ChildBoundary { - child_instance: DB::string(link, "child_workflow_instance_id")?, - child_run: DB::string(link, "child_workflow_run_id")?, + child_instance: DB::string(current, "child_workflow_instance_id")?, + child_run: DB::string(current, "child_workflow_run_id")?, policy: cancellation, }) } diff --git a/rust/tests/execution/child_retries.rs b/rust/tests/execution/child_retries.rs index 75c4c45f0..5dfe2f4e3 100644 --- a/rust/tests/execution/child_retries.rs +++ b/rust/tests/execution/child_retries.rs @@ -207,3 +207,57 @@ async fn a_child_and_its_retry_inherit_the_parents_memos_and_search_attributes() runtime.close().await; database.remove().await; } + +// #873: published PHP admits and delivers a parent's cooperative cancellation +// to a child it retried. The delivery follows the call to its current run. +#[tokio::test] +async fn a_parents_cooperative_cancellation_reaches_a_retried_child() { + for policy in ["abandon", "try_cancel"] { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let registered = request_protocol(&app, "POST", "/api/worker/register", json!({"worker_id":"cleanup-worker", + "task_queue":"test","runtime":"php","supported_workflow_types":["echo"],"supported_activity_types":[], + "capabilities":["cooperative_cancellation"]}), "1.20").await; + assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); + register(&app, "child-worker", json!(["child"]), json!([])).await; + let parent = start(&app, "cooperative-retried").await; + let task = cooperative_poll(&app, "workflow").await; + // A child cancellation policy needs a cooperative claim on protocol 1.20. + let (status, body) = request_protocol(&app, "POST", &format!("/api/worker/workflow-tasks/{}/complete", + task["task_id"].as_str().unwrap()), completion(&task, json!([child(json!({"retry_policy":{"max_attempts":2}, + "cancellation_policy":policy}))])), "1.20").await; + assert_eq!(status, StatusCode::OK, "{policy}: {body}"); + let first = poll(&app, "child-worker", "workflow").await; + let (status, body) = finish_task(&app, &first, fail(false)).await; + assert_eq!(status, StatusCode::OK, "{policy}: {body}"); + let requested = request(&app, "POST", "/api/workflows/cooperative-retried/request-cancellation", + json!({"reason":"stop the retried child","cleanup_timeout_seconds":30})).await; + assert_eq!(requested.0, StatusCode::ACCEPTED, "{policy}: {}", requested.1); + let pending = requested.1["cancellation_request"].clone(); + let root = cooperative_poll(&app, "workflow").await; + let delivered = request_protocol(&app, "POST", &format!("/api/worker/workflow-tasks/{}/deliver-cancellation", + root["task_id"].as_str().unwrap()), json!({"lease_owner":root["lease_owner"], + "workflow_task_attempt":root["workflow_task_attempt"],"request_id":pending["request_id"],"sequence":1, + "call_kind":"child"}), "1.20").await; + assert_eq!(delivered.0, StatusCode::OK, "{policy}: {}", delivered.1); + let (status, body) = finish_task(&app, &root, json!([{"type":"complete_workflow","result":envelope(Payload::Null)}])).await; + assert_eq!((status, &body["run_status"]), (StatusCode::OK, &json!("cancelled")), "{policy}: {body}"); + let events = run_history(&app, &parent["workflow_id"], &parent["run_id"]).await; + let kinds: Vec<_> = events.as_array().unwrap().iter().map(|event| event["event_type"].as_str().unwrap().to_owned()) + .filter(|kind| kind.starts_with("Child") || kind.contains("Cancel")).collect(); + let mut expected = vec!["ChildWorkflowScheduled", "ChildRunStarted", "ChildRunStarted", "CooperativeCancellationRequested"]; + if policy == "try_cancel" { + expected.push("ChildCancellationRequested"); + } + expected.extend(["CooperativeCancellationDelivered", "WorkflowCancelled"]); + assert_eq!(kinds, expected, "{policy}"); + // The retry run, not the failed first run, takes a requested child cancellation. + let retry = child_events(&events)[2]["payload"].clone(); + let retry_history = run_history(&app, &retry["child_workflow_instance_id"], &retry["child_workflow_run_id"]).await; + let requested_child = retry_history.as_array().unwrap().iter().any(|event| event["event_type"] == "CooperativeCancellationRequested"); + assert_eq!(requested_child, policy == "try_cancel", "{policy}: {retry_history}"); + runtime.close().await; + database.remove().await; + } +} diff --git a/scripts/conformance/server-parity/contract.mjs b/scripts/conformance/server-parity/contract.mjs index a87f10df8..1762992b9 100644 --- a/scripts/conformance/server-parity/contract.mjs +++ b/scripts/conformance/server-parity/contract.mjs @@ -21,6 +21,7 @@ import {checkWorkerErrorBodies} from './worker-error-bodies-contract.mjs'; import {checkChildDescriptions} from './child-descriptions-contract.mjs'; import {checkChildRetries} from './child-retries-contract.mjs'; import {checkChildVisibility} from './child-visibility-contract.mjs'; +import {checkRetriedChildCancellation} from './retried-child-cancellation-contract.mjs'; import {checkStartSearchAttributes} from './start-search-attributes-contract.mjs'; import {checkQueryArgumentRouting} from './query-arguments-contract.mjs'; import {checkQueryContractArguments} from './query-contract-arguments-contract.mjs'; @@ -187,6 +188,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { const childDescriptions = fixture.child_descriptions ? checkChildDescriptions(fixture, observation) : null; const childRetries = fixture.child_retries ? checkChildRetries(fixture, observation) : null; const childVisibility = fixture.child_visibility ? checkChildVisibility(fixture, observation) : null; + const retriedChildCancellation = fixture.retried_child_cancellation ? checkRetriedChildCancellation(fixture, observation) : null; const startSearchAttributes = fixture.start_search_attributes ? checkStartSearchAttributes(fixture, observation) : null; const queryArguments = fixture.query_argument_routing ? checkQueryArgumentRouting(fixture, observation) : null; const queryContractArguments = fixture.query_contract_arguments ? checkQueryContractArguments(fixture, observation) : null; @@ -771,6 +773,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { ...(childDescriptions ? {child_descriptions: childDescriptions} : {}), ...(childRetries ? {child_retries: childRetries} : {}), ...(childVisibility ? {child_visibility: childVisibility} : {}), + ...(retriedChildCancellation ? {retried_child_cancellation: retriedChildCancellation} : {}), ...(startSearchAttributes ? {start_search_attributes: startSearchAttributes} : {}), ...(queryArguments ? {query_argument_routing: queryArguments} : {}), ...(queryContractArguments ? {query_contract_arguments: queryContractArguments} : {}), diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index bf0f1d719..24c5a2f1f 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -439,6 +439,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, $childDescriptions = isset($fixture['child_descriptions']) ? finishHttpChildDescriptions($client, $fixture, $workflowId, $url) : null; $childRetries = isset($fixture['child_retries']) ? finishHttpChildRetries($client, $fixture, $workflowId, $url) : null; $childVisibility = isset($fixture['child_visibility']) ? finishHttpChildVisibility($client, $fixture, $workflowId, $url) : null; + $retriedChildCancellation = isset($fixture['retried_child_cancellation']) ? finishHttpRetriedChildCancellation($client, $fixture, $workflowId, $url) : null; $startSearchAttributes = isset($fixture['start_search_attributes']) ? finishHttpStartSearchAttributes($client, $fixture, $workflowId, $url) : null; $queryArgumentRouting = isset($fixture['query_argument_routing']) ? finishHttpQueryArgumentRouting($client, $fixture, $workflowId, $url) : null; $queryContractArguments = isset($fixture['query_contract_arguments']) ? finishHttpQueryContractArguments($client, $fixture, $workflowId, $url) : null; @@ -576,6 +577,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, 'child_descriptions' => $childDescriptions, 'child_retries' => $childRetries, 'child_visibility' => $childVisibility, + 'retried_child_cancellation' => $retriedChildCancellation, 'start_search_attributes' => $startSearchAttributes, 'query_argument_routing' => $queryArgumentRouting, 'query_contract_arguments' => $queryContractArguments, @@ -916,6 +918,7 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa require __DIR__.'/child-descriptions-probe.php'; require __DIR__.'/child-retries-probe.php'; require __DIR__.'/child-visibility-probe.php'; + require __DIR__.'/retried-child-cancellation-probe.php'; require __DIR__.'/start-search-attributes-probe.php'; require __DIR__.'/query-arguments-probe.php'; require __DIR__.'/query-contract-arguments-probe.php'; @@ -1126,6 +1129,9 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa if ($mode === 'embedded' && isset($fixture['child_visibility'])) { $observation['child_visibility'] = ['applicable' => false, 'reason' => 'embedded_has_no_worker_protocol']; } + if ($mode === 'embedded' && isset($fixture['retried_child_cancellation'])) { + $observation['retried_child_cancellation'] = ['applicable' => false, 'reason' => 'embedded_has_no_worker_protocol']; + } if ($mode === 'embedded' && isset($fixture['start_search_attributes'])) { $observation['start_search_attributes'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_search_attribute_registry']; } diff --git a/scripts/conformance/server-parity/retried-child-cancellation-contract.mjs b/scripts/conformance/server-parity/retried-child-cancellation-contract.mjs new file mode 100644 index 000000000..5d5841de7 --- /dev/null +++ b/scripts/conformance/server-parity/retried-child-cancellation-contract.mjs @@ -0,0 +1,25 @@ +import assert from 'node:assert/strict'; + +// Reviewed cooperative cancellation of a retried child +// (retried-child-cancellation-probe.php), recorded against published Server +// 2.11.7. The fixture's `expected` holds each case's whole observation. +export function checkRetriedChildCancellation(fixture, observation) { + const definition = fixture.retried_child_cancellation; + const state = observation.retried_child_cancellation; + const projection = {scope: 'retried_child_cancellation', cases: Object.keys(definition.cases).length}; + assert.ok(state && typeof state === 'object', 'actual retried child cancellation observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_worker_protocol'}, + 'embedded retried child cancellation explicitly inapplicable'); + return projection; + } + assert.equal(state.applicable, true, 'HTTP targets record retried child cancellation'); + assert.deepStrictEqual(Object.keys(definition.expected).sort(), Object.keys(definition.cases).sort(), + 'every retried child cancellation case has an expectation'); + for (const [name, expected] of Object.entries(definition.expected)) { + assert.deepStrictEqual(state[name], expected, `retried child cancellation ${name}`); + } + assert.deepStrictEqual(Object.keys(state).sort(), ['applicable', ...Object.keys(definition.expected)].sort(), + 'retried child cancellation cases'); + return projection; +} diff --git a/scripts/conformance/server-parity/retried-child-cancellation-probe.php b/scripts/conformance/server-parity/retried-child-cancellation-probe.php new file mode 100644 index 000000000..751cc5494 --- /dev/null +++ b/scripts/conformance/server-parity/retried-child-cancellation-probe.php @@ -0,0 +1,150 @@ +createNamespace($namespace, 'retried child cancellation λ', 30); + $token = getenv('DW_PARITY_TOKEN') ?: null; + $stack = HandlerStack::create(); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $http = new RetriedChildCancellationHttp(['base_uri' => rtrim($url, '/').'/api/', 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); + $headers = array_filter([ + 'Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => $namespace, + 'X-Durable-Workflow-Control-Plane-Version' => '2', 'X-Durable-Workflow-Protocol-Version' => '1.20', + ]); + $raw = static function (string $method, string $path, ?array $body = null) use ($http, $headers): array { + $response = $http->request($method, $path, ['headers' => $headers, + ...($body === null ? [] : ['json' => $body === [] ? (object) [] : $body])]); + + return ['status' => $response->getStatusCode(), + 'response' => json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR)]; + }; + $answer = static fn (array $answer): array => ['status' => $answer['status'], + 'reason' => $answer['response']['reason'] ?? null, 'run_status' => $answer['response']['run_status'] ?? null, + 'outcome' => $answer['response']['outcome'] ?? null]; + $manifest = array_fill_keys(['local_activities', 'worker_sessions', 'sticky_execution'], + ['supported' => false, 'minimum_protocol_version' => '1.18', 'reason' => 'raw_http']); + $empty = ['codec' => 'avro', 'blob' => $definition['empty_blob']]; + $state = ['applicable' => true]; + foreach ($definition['cases'] as $case => $policy) { + $labels = []; + $label = static function (mixed $id) use (&$labels): mixed { + if (! is_string($id) || $id === '') { + return $id; + } + $labels[strtolower($id)] ??= '{id:'.(count($labels) + 1).'}'; + + return $labels[strtolower($id)]; + }; + $id = $workflowId.'-rcc-'.$case; + $queue = $id.'-queue'; + $parentWorker = $id.'-parent-worker'; + $childWorker = $id.'-child-worker'; + foreach ([[$parentWorker, $definition['workflow_type'], ['cooperative_cancellation']], [$childWorker, $definition['child_type'], []]] as [$worker, $type, $capabilities]) { + $registered = $raw('POST', 'worker/register', ['worker_id' => $worker, 'task_queue' => $queue, 'runtime' => 'php', + 'sdk_version' => 'raw-http', 'supported_workflow_types' => [$type], 'supported_activity_types' => [], + 'capabilities' => $capabilities, 'capability_manifest' => $manifest]); + if ($registered['status'] !== 201) { + throw new RuntimeException('Retried child cancellation registration: '.json_encode($registered)); + } + } + $started = $raw('POST', 'workflows', ['workflow_id' => $id, 'workflow_type' => $definition['workflow_type'], + 'task_queue' => $queue, 'input' => $empty]); + if ($started['status'] !== 201) { + throw new RuntimeException('Retried child cancellation start: '.json_encode($started)); + } + $parentRun = $started['response']['run_id']; + $poll = static function (string $worker, string $type) use ($raw, $queue, $case): array { + for ($attempt = 0; $attempt < 10; $attempt++) { + $task = $raw('POST', 'worker/workflow-tasks/poll', ['worker_id' => $worker, 'task_queue' => $queue, + 'timeout_seconds' => 1])['response']['task'] ?? null; + if (is_array($task)) { + if (($task['workflow_type'] ?? null) !== $type) { + throw new RuntimeException("Retried child cancellation {$case}: expected a {$type} task"); + } + + return $task; + } + } + throw new RuntimeException("Retried child cancellation {$case}: no {$type} task"); + }; + $complete = static fn (array $task, array $commands): array => $raw('POST', + 'worker/workflow-tasks/'.$task['task_id'].'/complete', ['lease_owner' => $task['lease_owner'], + 'workflow_task_attempt' => $task['workflow_task_attempt'], 'commands' => $commands]); + $observed = []; + $observed['child_start'] = $answer($complete($poll($parentWorker, $definition['workflow_type']), [[ + 'type' => 'start_child_workflow', 'workflow_type' => $definition['child_type'], 'arguments' => $empty, + 'retry_policy' => ['max_attempts' => 2], 'cancellation_policy' => $policy]])); + $observed['child_failure'] = $answer($complete($poll($childWorker, $definition['child_type']), [[ + 'type' => 'fail_workflow', 'message' => 'first attempt', 'exception_type' => 'Boom']])); + $requested = $raw('POST', 'workflows/'.rawurlencode($id).'/request-cancellation', + ['reason' => $definition['reason'], 'cleanup_timeout_seconds' => 30]); + $observed['request'] = ['status' => $requested['status'], 'reason' => $requested['response']['reason'] ?? null, + 'duplicate' => $requested['response']['duplicate'] ?? null]; + if ($requested['status'] === 202) { + $root = $poll($parentWorker, $definition['workflow_type']); + $delivered = $raw('POST', 'worker/workflow-tasks/'.$root['task_id'].'/deliver-cancellation', [ + 'lease_owner' => $root['lease_owner'], 'workflow_task_attempt' => $root['workflow_task_attempt'], + 'request_id' => $requested['response']['cancellation_request']['request_id'] ?? null, + 'sequence' => 1, 'call_kind' => 'child']); + $observed['delivery'] = $answer($delivered); + $observed['parent_completion'] = $answer($complete($root, [['type' => 'complete_workflow', 'result' => $empty]])); + } + $events = $raw('GET', 'workflows/'.rawurlencode($id).'/runs/'.$parentRun.'/history')['response']['events'] ?? []; + $observed['parent_events'] = []; + $child = null; + foreach ($events as $event) { + $type = $event['event_type']; + if (! str_starts_with($type, 'Child') && ! str_contains($type, 'Cancel')) { + continue; + } + $child ??= $event['payload']['child_workflow_instance_id'] ?? null; + $projected = ['event_type' => $type]; + foreach ($definition['event_fields'] as $field) { + if (array_key_exists($field, $event['payload'])) { + $value = $event['payload'][$field]; + $projected[$field] = str_ends_with($field, '_id') ? $label($value) : $value; + } + } + $observed['parent_events'][] = $projected; + } + if (! is_string($child)) { + throw new RuntimeException("Retried child cancellation {$case}: no child instance"); + } + $runs = $raw('GET', 'workflows/'.rawurlencode($child).'/runs')['response']['runs'] ?? []; + usort($runs, static fn (array $a, array $b): int => ($a['run_number'] ?? 0) <=> ($b['run_number'] ?? 0)); + $observed['child_runs'] = array_map(static fn (array $run): array => ['run_id' => $label($run['run_id'] ?? null), + 'run_number' => $run['run_number'] ?? null, 'status' => $run['status'] ?? null], $runs); + $retry = end($runs); + $history = $raw('GET', 'workflows/'.rawurlencode($child).'/runs/'.($retry['run_id'] ?? '').'/history')['response']['events'] ?? []; + $observed['retry_events'] = array_map(static fn (array $event): string => $event['event_type'], $history); + // End the retried child so nothing stays for later fixtures. + $terminated = $raw('POST', 'workflows/'.rawurlencode($child).'/terminate', ['reason' => 'retried child cancellation recorded']); + if (! in_array($terminated['status'], [200, 202], true)) { + throw new RuntimeException("Retried child cancellation {$case} cleanup: ".json_encode($terminated)); + } + if ($requested['status'] !== 202) { + $raw('POST', 'workflows/'.rawurlencode($id).'/terminate', ['reason' => 'retried child cancellation recorded']); + } + $state[$case] = $observed; + } + + return $state; +} diff --git a/tests/Fixtures/ServerParity/retried-child-cancellation.json b/tests/Fixtures/ServerParity/retried-child-cancellation.json new file mode 100644 index 000000000..eeef9f72c --- /dev/null +++ b/tests/Fixtures/ServerParity/retried-child-cancellation.json @@ -0,0 +1,247 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "retried-child-cancellation", + "review_state": "reviewed", + "contract": "A parent's cooperative cancellation delivered to a child it retried (published Server 2.11.7, Server #873), over raw worker HTTP on protocol 1.20 in a fresh namespace. A cooperative worker runs the parent and another worker the child. The child starts under a retry policy and the case's cancellation policy and its first run fails, so it is retried. The parent's cancellation request is admitted, delivered at the child boundary and completed as cancelled. Under abandon the retry keeps running; under try_cancel the parent records ChildCancellationRequested and the retry run, not the failed first run, records the cooperative cancellation request. IDs are labels. Embedded applications have no worker protocol.", + "workflow_type": "parity.v1.echo", + "input": { + "message": "retried child cancellation λ", + "count": 9007199254740993, + "optional": null + }, + "typed_value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "retried child cancellation λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + }, + "activity": false, + "expected_events": [ + "StartAccepted", + "WorkflowStarted", + "WorkflowCompleted" + ], + "retried_child_cancellation": { + "workflow_type": "parity.v1.retried_child_cancellation", + "child_type": "parity.v1.retried_child_cancellation_child", + "empty_blob": "wwHioz3/VYAiNwwA", + "reason": "stop the retried child λ", + "event_fields": [ + "sequence", + "workflow_link_id", + "child_call_id", + "child_workflow_run_id", + "child_run_number", + "retry_attempt", + "cancellation_policy", + "parent_close_policy", + "request_id", + "reason", + "call_kind", + "child_status" + ], + "cases": { + "abandoning": "abandon", + "try_cancelling": "try_cancel" + }, + "expected": { + "abandoning": { + "child_start": { + "status": 200, + "reason": null, + "run_status": "waiting", + "outcome": "completed" + }, + "child_failure": { + "status": 200, + "reason": null, + "run_status": "failed", + "outcome": "completed" + }, + "request": { + "status": 202, + "reason": null, + "duplicate": false + }, + "delivery": { + "status": 200, + "reason": null, + "run_status": null, + "outcome": null + }, + "parent_completion": { + "status": 200, + "reason": null, + "run_status": "cancelled", + "outcome": "completed" + }, + "parent_events": [ + { + "event_type": "ChildWorkflowScheduled", + "sequence": 1, + "workflow_link_id": "{id:1}", + "child_call_id": "{id:1}", + "child_workflow_run_id": "{id:2}", + "cancellation_policy": "abandon", + "parent_close_policy": "abandon" + }, + { + "event_type": "ChildRunStarted", + "sequence": 1, + "workflow_link_id": "{id:1}", + "child_call_id": "{id:1}", + "child_workflow_run_id": "{id:2}", + "child_run_number": 1, + "cancellation_policy": "abandon", + "parent_close_policy": "abandon" + }, + { + "event_type": "ChildRunStarted", + "sequence": 1, + "workflow_link_id": "{id:3}", + "child_call_id": "{id:1}", + "child_workflow_run_id": "{id:4}", + "child_run_number": 2, + "retry_attempt": 2, + "parent_close_policy": "abandon" + }, + { + "event_type": "CooperativeCancellationRequested", + "reason": "stop the retried child λ" + }, + { + "event_type": "CooperativeCancellationDelivered", + "sequence": 1, + "call_kind": "child" + }, + { + "event_type": "WorkflowCancelled", + "reason": "Cooperative cancellation completed." + } + ], + "child_runs": [ + { + "run_id": "{id:2}", + "run_number": 1, + "status": "failed" + }, + { + "run_id": "{id:4}", + "run_number": 2, + "status": "pending" + } + ], + "retry_events": [ + "WorkflowStarted" + ] + }, + "try_cancelling": { + "child_start": { + "status": 200, + "reason": null, + "run_status": "waiting", + "outcome": "completed" + }, + "child_failure": { + "status": 200, + "reason": null, + "run_status": "failed", + "outcome": "completed" + }, + "request": { + "status": 202, + "reason": null, + "duplicate": false + }, + "delivery": { + "status": 200, + "reason": null, + "run_status": null, + "outcome": null + }, + "parent_completion": { + "status": 200, + "reason": null, + "run_status": "cancelled", + "outcome": "completed" + }, + "parent_events": [ + { + "event_type": "ChildWorkflowScheduled", + "sequence": 1, + "workflow_link_id": "{id:1}", + "child_call_id": "{id:1}", + "child_workflow_run_id": "{id:2}", + "cancellation_policy": "try_cancel", + "parent_close_policy": "abandon" + }, + { + "event_type": "ChildRunStarted", + "sequence": 1, + "workflow_link_id": "{id:1}", + "child_call_id": "{id:1}", + "child_workflow_run_id": "{id:2}", + "child_run_number": 1, + "cancellation_policy": "try_cancel", + "parent_close_policy": "abandon" + }, + { + "event_type": "ChildRunStarted", + "sequence": 1, + "workflow_link_id": "{id:3}", + "child_call_id": "{id:1}", + "child_workflow_run_id": "{id:4}", + "child_run_number": 2, + "retry_attempt": 2, + "parent_close_policy": "abandon" + }, + { + "event_type": "CooperativeCancellationRequested", + "reason": "stop the retried child λ" + }, + { + "event_type": "ChildCancellationRequested", + "sequence": 1, + "child_workflow_run_id": "{id:4}" + }, + { + "event_type": "CooperativeCancellationDelivered", + "sequence": 1, + "call_kind": "child" + }, + { + "event_type": "WorkflowCancelled", + "reason": "Cooperative cancellation completed." + } + ], + "child_runs": [ + { + "run_id": "{id:2}", + "run_number": 1, + "status": "failed" + }, + { + "run_id": "{id:4}", + "run_number": 2, + "status": "pending" + } + ], + "retry_events": [ + "WorkflowStarted", + "CooperativeCancellationRequested" + ] + } + } + } +} diff --git a/tests/Unit/ServerParityRetriedChildCancellationContractTest.mjs b/tests/Unit/ServerParityRetriedChildCancellationContractTest.mjs new file mode 100644 index 000000000..23cd07410 --- /dev/null +++ b/tests/Unit/ServerParityRetriedChildCancellationContractTest.mjs @@ -0,0 +1,43 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {checkRetriedChildCancellation} from '../../scripts/conformance/server-parity/retried-child-cancellation-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/retried-child-cancellation.json', import.meta.url), 'utf8')); +const expected = fixture.retried_child_cancellation.expected; +const observation = () => structuredClone({mode: 'http', retried_child_cancellation: {applicable: true, ...expected}}); +const cases = Object.keys(fixture.retried_child_cancellation.cases).length; +const kinds = name => expected[name].parent_events.map(event => event.event_type); + +test('the published answers pass', () => { + assert.deepStrictEqual(checkRetriedChildCancellation(fixture, observation()), {scope: 'retried_child_cancellation', cases}); +}); + +test('embedded applications have no worker protocol', () => { + assert.deepStrictEqual(checkRetriedChildCancellation(fixture, + {mode: 'embedded', retried_child_cancellation: {applicable: false, reason: 'embedded_has_no_worker_protocol'}}), + {scope: 'retried_child_cancellation', cases}); +}); + +test('the cancellation is admitted, delivered at the retried child and closes the parent', () => { + for (const name of Object.keys(expected)) { + assert.deepStrictEqual([expected[name].request.status, expected[name].delivery.status, + expected[name].parent_completion.run_status], [202, 200, 'cancelled'], name); + } + assert.ok(!kinds('abandoning').includes('ChildCancellationRequested')); + assert.ok(kinds('try_cancelling').includes('ChildCancellationRequested')); + const requested = expected.try_cancelling.parent_events.find(event => event.event_type === 'ChildCancellationRequested'); + assert.equal(requested.child_workflow_run_id, expected.try_cancelling.child_runs[1].run_id, 'the retry run is asked'); + assert.deepStrictEqual(expected.try_cancelling.retry_events, ['WorkflowStarted', 'CooperativeCancellationRequested']); +}); + +for (const [name, mutate] of [ + ['a refused request', state => { state.abandoning.request = {status: 422, reason: 'cooperative_cancellation_propagation_not_available', duplicate: null}; }], + ['a cancellation sent to the failed first run', state => { + state.try_cancelling.parent_events.find(event => event.event_type === 'ChildCancellationRequested').child_workflow_run_id = + state.try_cancelling.child_runs[0].run_id; }], + ['an unexpected case', state => { state.extra = {}; }], +]) test(`the contract rejects ${name}`, () => { + const raw = observation(); mutate(raw.retried_child_cancellation); + assert.throws(() => checkRetriedChildCancellation(fixture, raw)); +});