From 8c0ca111be2529377f903b4d02fd7db9b2041ee0 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 08:09:52 +0000 Subject: [PATCH 01/58] Add the Rust cooperative cancellation request foundation --- src/cooperative_cancellation.rs | 218 +++++++++++++++ src/lib.rs | 10 + src/tests/cooperative_cancellation.rs | 377 ++++++++++++++++++++++++++ 3 files changed, 605 insertions(+) create mode 100644 src/cooperative_cancellation.rs create mode 100644 src/tests/cooperative_cancellation.rs diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs new file mode 100644 index 0000000..ff9b95d --- /dev/null +++ b/src/cooperative_cancellation.rs @@ -0,0 +1,218 @@ +use super::*; + +const CONTROL_BUDGET: Duration = Duration::from_secs(5); + +/// Optional reason and runtime-owned cleanup limit for a cooperative request. +#[derive(Clone, Debug, Default, Serialize)] +pub struct CooperativeCancellationOptions { + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cleanup_timeout_seconds: Option, +} + +/// The Server's original request identity and immutable cleanup deadline. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CancellationRequest { + pub request_id: String, + pub requested_at: String, + pub cleanup_deadline_at: String, + pub history_refresh_page_token: String, +} + +/// Acknowledgment of a request targeting the current durable run. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WorkflowCancellationRequest { + pub workflow_id: String, + pub run_id: String, + pub duplicate: bool, + pub cancellation_request: CancellationRequest, +} + +fn invalid(detail: impl Into) -> Error { + Error::InvalidCooperativeCancellation(detail.into()) +} + +fn text<'a>(value: &'a Value, field: &str) -> Result<&'a str> { + value + .get(field) + .and_then(Value::as_str) + .filter(|text| !text.trim().is_empty()) + .ok_or_else(|| invalid(format!("{field} must be a non-empty string"))) +} + +impl CancellationRequest { + pub(crate) fn from_observation(value: &Value) -> Result { + let requested_at = text(value, "requested_at")?; + let cleanup_deadline_at = text(value, "cleanup_deadline_at")?; + let requested = DateTime::parse_from_rfc3339(requested_at) + .map_err(|_| invalid("requested_at must be a timestamp with a timezone"))?; + let deadline = DateTime::parse_from_rfc3339(cleanup_deadline_at) + .map_err(|_| invalid("cleanup_deadline_at must be a timestamp with a timezone"))?; + if deadline <= requested { + return Err(invalid("cleanup deadline must follow the original request")); + } + Ok(Self { + request_id: text(value, "request_id")?.to_owned(), + requested_at: requested_at.to_owned(), + cleanup_deadline_at: cleanup_deadline_at.to_owned(), + history_refresh_page_token: text(value, "history_refresh_page_token")?.to_owned(), + }) + } +} + +fn supports_protocol(version: &str) -> bool { + let Some((major, minor)) = version.split_once('.') else { + return false; + }; + major == "1" + && !minor.is_empty() + && minor.bytes().all(|byte| byte.is_ascii_digit()) + && minor.parse::().is_ok_and(|minor| minor >= 20) +} + +fn require_discovery(info: &Value) -> Result<()> { + let protocol = &info["worker_protocol"]; + if protocol["server_capabilities"]["cooperative_cancellation"].as_bool() != Some(true) + || !protocol["version"].as_str().is_some_and(supports_protocol) + { + return Err(Error::CooperativeCancellationUnavailable( + "runtime discovery must explicitly advertise support and compatible protocol 1.20" + .to_string(), + )); + } + Ok(()) +} + +fn acknowledgment( + value: &Value, + workflow_id: &str, + selected_run: Option<&str>, +) -> Result { + if value["accepted"].as_bool() != Some(true) + || text(value, "workflow_id")? != workflow_id + || selected_run.is_some_and(|run| value["run_id"].as_str() != Some(run)) + { + return Err(invalid( + "acknowledgment does not match the accepted workflow/run request", + )); + } + Ok(WorkflowCancellationRequest { + workflow_id: workflow_id.to_owned(), + run_id: text(value, "run_id")?.to_owned(), + duplicate: value["duplicate"] + .as_bool() + .ok_or_else(|| invalid("duplicate must be a boolean"))?, + cancellation_request: CancellationRequest::from_observation( + &value["cancellation_request"], + )?, + }) +} + +impl Client { + /// Request bounded workflow-authored cleanup on a capable runtime. + /// + /// This is separate from terminal cancellation. Discovery must advertise + /// cooperative support and compatible protocol 1.20. The Server also + /// refuses an active workflow claim that cannot deliver cooperation. + /// Repeated requests retain its original request identity and deadline. + pub async fn request_workflow_cancellation( + &self, + workflow_id: &str, + options: CooperativeCancellationOptions, + ) -> Result { + self.request_workflow_cancellation_target(workflow_id, None, options) + .await + } + + /// Request cleanup only if the selected run remains current. + pub async fn request_workflow_run_cancellation( + &self, + workflow_id: &str, + run_id: &str, + options: CooperativeCancellationOptions, + ) -> Result { + self.request_workflow_cancellation_target(workflow_id, Some(run_id), options) + .await + } + + async fn request_workflow_cancellation_target( + &self, + workflow_id: &str, + run_id: Option<&str>, + options: CooperativeCancellationOptions, + ) -> Result { + if workflow_id.trim().is_empty() || run_id.is_some_and(|id| id.trim().is_empty()) { + return Err(invalid( + "workflow and selected run identities must be non-empty", + )); + } + if options + .cleanup_timeout_seconds + .is_some_and(|seconds| !(1..=3600).contains(&seconds)) + || options + .reason + .as_ref() + .is_some_and(|reason| reason.chars().count() > 1000) + { + return Err(invalid( + "cleanup timeout must be 1..3600 seconds and reason at most 1000 characters", + )); + } + + // Bound discovery, mutation and any storage retry by one total budget. + tokio::time::timeout(CONTROL_BUDGET, async { + let info: Value = self + .request_json( + reqwest::Method::GET, + "/cluster/info", + RequestProtocol::ControlPlane, + Option::<&Value>::None, + ) + .await?; + require_discovery(&info)?; + let mut path = format!("/workflows/{}", percent_encode_path_segment(workflow_id)); + if let Some(run_id) = run_id { + path.push_str(&format!("/runs/{}", percent_encode_path_segment(run_id))); + } + path.push_str("/request-cancellation"); + let response: Value = self + .request_json( + reqwest::Method::POST, + &path, + RequestProtocol::ControlPlane, + Some(&options), + ) + .await?; + acknowledgment(&response, workflow_id, run_id) + }) + .await + .map_err(|_| Error::Timeout)? + } +} + +impl WorkflowHandle { + /// Request bounded cleanup for whichever run is current. + pub async fn request_cancellation( + &self, + options: CooperativeCancellationOptions, + ) -> Result { + self.client + .request_workflow_cancellation(&self.workflow_id, options) + .await + } + + /// Request bounded cleanup only while this handle's selected run is current. + pub async fn request_selected_run_cancellation( + &self, + options: CooperativeCancellationOptions, + ) -> Result { + let run_id = self + .run_id + .as_deref() + .ok_or_else(|| invalid("selected run_id is required"))?; + self.client + .request_workflow_run_cancellation(&self.workflow_id, run_id, options) + .await + } +} diff --git a/src/lib.rs b/src/lib.rs index 93f50fe..6ba981f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,8 +1,13 @@ #![doc = include_str!("../README.md")] +mod cooperative_cancellation; mod runtime_payloads; mod runtime_uploads; +pub use cooperative_cancellation::{ + CancellationRequest, CooperativeCancellationOptions, WorkflowCancellationRequest, +}; + use std::{ any::{type_name, Any, TypeId}, collections::{BTreeMap, HashMap}, @@ -252,6 +257,10 @@ pub enum Error { "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support" )] WorkflowMemoUpdatesUnavailable, + #[error("cooperative cancellation is unavailable: {0}")] + CooperativeCancellationUnavailable(String), + #[error("invalid cooperative cancellation: {0}")] + InvalidCooperativeCancellation(String), #[error(transparent)] InvalidActivityOptions(ActivityOptionsError), #[error(transparent)] @@ -15085,6 +15094,7 @@ fn value_as_u64(value: &Value) -> Option { #[cfg(test)] mod tests { use super::*; + mod cooperative_cancellation; mod runtime_payloads; mod runtime_uploads; use std::{ diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs new file mode 100644 index 0000000..01c0814 --- /dev/null +++ b/src/tests/cooperative_cancellation.rs @@ -0,0 +1,377 @@ +use super::*; + +fn request_observation() -> Value { + json!({ + "request_id": "original-request", + "requested_at": "2026-10-01T08:00:00Z", + "cleanup_deadline_at": "2026-10-01T08:10:00Z", + "history_refresh_page_token": "opaque-server-token" + }) +} + +fn responses(path: &str, _body: &str, number: usize) -> Option<(&'static str, String)> { + let case = path.split('/').nth(1).unwrap_or_default(); + if path.ends_with("/api/cluster/info") { + if case == "discovery-http" { + return Some(("403 Forbidden", r#"{"reason":"forbidden"}"#.into())); + } + if case == "budget" { + thread::sleep(Duration::from_millis(2500)); + } + let mut discovery = json!({"worker_protocol": { + "version": "1.20", "server_capabilities": {"cooperative_cancellation": true} + }}); + match case { + "unsupported" => { + discovery["worker_protocol"]["server_capabilities"]["cooperative_cancellation"] = + json!(false) + } + "missing" => discovery = json!({}), + "string-flag" => { + discovery["worker_protocol"]["server_capabilities"]["cooperative_cancellation"] = + json!("true") + } + "old" => discovery["worker_protocol"]["version"] = json!("1.19"), + "future-major" => discovery["worker_protocol"]["version"] = json!("2.20"), + "malformed-version" => discovery["worker_protocol"]["version"] = json!("1.+20"), + "new-minor" => discovery["worker_protocol"]["version"] = json!("1.21"), + _ => {} + } + return Some(("200 OK", discovery.to_string())); + } + if !path.ends_with("/request-cancellation") { + return None; + } + if case == "refused" { + return Some(( + "409 Conflict", + r#"{"reason":"active_claim_cancellation_not_supported"}"#.into(), + )); + } + if case == "budget" { + thread::sleep(Duration::from_secs(3)); + } + let mut response = json!({ + "accepted": true, "duplicate": number > 1, + "workflow_id": "workflow", "run_id": "run", + "cancellation_request": request_observation() + }); + match case { + "escaped" => { + response["workflow_id"] = json!("workflow/with?path"); + response["run_id"] = json!("run/selected"); + } + "wrong-workflow" => response["workflow_id"] = json!("other"), + "wrong-run" => response["run_id"] = json!("other"), + "empty-run" => response["run_id"] = json!(" "), + "not-accepted" => response["accepted"] = json!(false), + "accepted-string" => response["accepted"] = json!("true"), + "duplicate-integer" => response["duplicate"] = json!(0), + "missing-request" => response["cancellation_request"] = Value::Null, + "blank-request" => response["cancellation_request"]["request_id"] = json!(" "), + "blank-token" => { + response["cancellation_request"]["history_refresh_page_token"] = json!(" ") + } + "no-timezone" => { + response["cancellation_request"]["requested_at"] = json!("2026-10-01T08:00:00") + } + "invalid-date" => { + response["cancellation_request"]["requested_at"] = json!("2026-02-30T08:00:00Z") + } + "equal-deadline" => { + response["cancellation_request"]["cleanup_deadline_at"] = json!("2026-10-01T08:00:00Z") + } + "earlier-deadline" => { + response["cancellation_request"]["cleanup_deadline_at"] = json!("2026-10-01T07:59:00Z") + } + _ => {} + } + Some(("202 Accepted", response.to_string())) +} + +fn server() -> MockWorkerServer { + MockWorkerServer::start_with_behavior(MockWorkerBehavior { + request_override: Some(responses), + ..MockWorkerBehavior::default() + }) +} + +fn client(server: &MockWorkerServer, case: &str) -> Client { + Client::builder(format!("{}/{case}", server.base_url())) + .control_token(Some("control-only".to_string())) + .worker_token(Some("worker-only".to_string())) + .namespace("caller-namespace") + .build() + .unwrap() +} + +#[tokio::test] +async fn cooperative_request_uses_control_role_namespace_and_preserves_server_identity() { + let server = server(); + let client = client(&server, "valid"); + let options = CooperativeCancellationOptions { + reason: Some("caller stopped".into()), + cleanup_timeout_seconds: Some(60), + }; + let first = client + .request_workflow_cancellation("workflow", options) + .await + .unwrap(); + let repeated = client + .request_workflow_cancellation( + "workflow", + CooperativeCancellationOptions { + reason: Some("changed reason".into()), + cleanup_timeout_seconds: Some(120), + }, + ) + .await + .unwrap(); + assert!(!first.duplicate); + assert!(repeated.duplicate); + assert_eq!(first.cancellation_request, repeated.cancellation_request); + assert_eq!(first.cancellation_request.request_id, "original-request"); + assert_eq!( + first.cancellation_request.cleanup_deadline_at, + "2026-10-01T08:10:00Z" + ); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 4); + for request in requests.iter() { + assert_eq!( + request.authorization.as_deref(), + Some("Bearer control-only") + ); + assert_eq!(request.namespace.as_deref(), Some("caller-namespace")); + assert_eq!(request.control_protocol.as_deref(), Some("2")); + assert_eq!(request.worker_protocol, None); + } + assert_eq!(requests[0].method, "GET"); + assert_eq!(requests[1].method, "POST"); + assert_eq!( + serde_json::from_str::(&requests[1].body).unwrap(), + json!({ + "reason":"caller stopped", "cleanup_timeout_seconds":60 + }) + ); +} + +#[tokio::test] +async fn cooperative_request_refuses_missing_unsupported_or_incompatible_discovery_before_mutation() +{ + for case in [ + "unsupported", + "missing", + "string-flag", + "old", + "future-major", + "malformed-version", + ] { + let server = server(); + let result = client(&server, case) + .request_workflow_cancellation("workflow", Default::default()) + .await; + assert!( + matches!(result, Err(Error::CooperativeCancellationUnavailable(_))), + "{case}: {result:?}" + ); + assert_eq!(server.requests.lock().unwrap().len(), 1, "{case}"); + } +} + +#[tokio::test] +async fn cooperative_request_accepts_additive_minor_discovery_without_changing_worker_default() { + let server = server(); + client(&server, "new-minor") + .request_workflow_cancellation("workflow", Default::default()) + .await + .unwrap(); + assert_eq!(WORKER_PROTOCOL_VERSION, "1.19"); + let ordinary = Client::builder(server.base_url()).build().unwrap(); + let mut worker = Worker::new(ordinary, "queue"); + worker.register_workflow("example", |_ctx, _input| async { Ok(Value::Null) }); + worker.register().await.unwrap(); + let requests = server.requests.lock().unwrap(); + let registration = requests + .iter() + .find(|request| request.path.ends_with("/worker/register")) + .unwrap(); + assert_eq!(registration.worker_protocol.as_deref(), Some("1.19")); + let body: Value = serde_json::from_str(®istration.body).unwrap(); + assert!(!body["capabilities"] + .as_array() + .unwrap() + .contains(&json!("cooperative_cancellation"))); +} + +#[tokio::test] +async fn cooperative_request_validates_selected_run_and_encodes_path_segments() { + let server = server(); + let result = client(&server, "escaped") + .request_workflow_run_cancellation("workflow/with?path", "run/selected", Default::default()) + .await + .unwrap(); + assert_eq!(result.run_id, "run/selected"); + assert_eq!(server.request_count("/escaped/api/workflows/workflow%2Fwith%3Fpath/runs/run%2Fselected/request-cancellation"), 1); +} + +#[tokio::test] +async fn cooperative_request_rejects_malformed_or_mismatched_acknowledgments() { + for case in [ + "wrong-workflow", + "wrong-run", + "empty-run", + "not-accepted", + "accepted-string", + "duplicate-integer", + "missing-request", + "blank-request", + "blank-token", + "no-timezone", + "invalid-date", + "equal-deadline", + "earlier-deadline", + ] { + let server = server(); + let result = client(&server, case) + .request_workflow_run_cancellation("workflow", "run", Default::default()) + .await; + assert!( + matches!(result, Err(Error::InvalidCooperativeCancellation(_))), + "{case}: {result:?}" + ); + } +} + +#[tokio::test] +async fn cooperative_request_rejects_invalid_inputs_without_network_activity() { + let server = server(); + let client = client(&server, "valid"); + for options in [ + CooperativeCancellationOptions { + cleanup_timeout_seconds: Some(0), + ..Default::default() + }, + CooperativeCancellationOptions { + cleanup_timeout_seconds: Some(3601), + ..Default::default() + }, + CooperativeCancellationOptions { + reason: Some("x".repeat(1001)), + ..Default::default() + }, + ] { + assert!(matches!( + client + .request_workflow_cancellation("workflow", options) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + } + assert!(matches!( + client + .request_workflow_cancellation(" ", Default::default()) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + assert!(matches!( + client + .request_workflow_run_cancellation("workflow", " ", Default::default()) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + assert!(server.requests.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn cooperative_request_does_not_substitute_worker_credentials() { + let server = server(); + let client = Client::builder(server.base_url()) + .worker_token(Some("worker-only".into())) + .build() + .unwrap(); + assert!(matches!( + client + .request_workflow_cancellation("workflow", Default::default()) + .await, + Err(Error::MissingRoleCredentials { + role: "control", + .. + }) + )); + assert!(server.requests.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn cooperative_request_preserves_discovery_and_active_claim_refusals() { + for (case, expected) in [ + ("discovery-http", reqwest::StatusCode::FORBIDDEN), + ("refused", reqwest::StatusCode::CONFLICT), + ] { + let server = server(); + let error = client(&server, case) + .request_workflow_cancellation("workflow", Default::default()) + .await + .unwrap_err(); + assert!(matches!(error, Error::Http {status, ..} if status==expected)); + assert_eq!( + server.requests.lock().unwrap().len(), + if case == "discovery-http" { 1 } else { 2 } + ); + } +} + +#[tokio::test] +async fn cooperative_request_handles_keep_current_and_selected_run_routing_separate() { + let server = server(); + let handle = WorkflowHandle { + client: client(&server, "valid"), + workflow_id: "workflow".into(), + run_id: Some("run".into()), + workflow_type: "example".into(), + }; + handle + .request_cancellation(Default::default()) + .await + .unwrap(); + handle + .request_selected_run_cancellation(Default::default()) + .await + .unwrap(); + assert_eq!( + server.request_count("/valid/api/workflows/workflow/request-cancellation"), + 1 + ); + assert_eq!( + server.request_count("/valid/api/workflows/workflow/runs/run/request-cancellation"), + 1 + ); + let missing = WorkflowHandle { + run_id: None, + ..handle + }; + assert!(matches!( + missing + .request_selected_run_cancellation(Default::default()) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + assert_eq!(server.requests.lock().unwrap().len(), 4); +} + +#[tokio::test] +async fn cooperative_request_bounds_discovery_and_mutation_with_one_total_budget() { + let server = server(); + let started = Instant::now(); + let result = client(&server, "budget") + .request_workflow_cancellation("workflow", Default::default()) + .await; + assert!(matches!(result, Err(Error::Timeout)), "{result:?}"); + assert!(started.elapsed() >= Duration::from_secs(4)); + assert!( + started.elapsed() < Duration::from_millis(5300), + "total budget exceeded: {:?}", + started.elapsed() + ); + assert_eq!(server.requests.lock().unwrap().len(), 2); +} From eeb81c6db6b46206ca20d1ac05463f8a971845cf Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 08:15:52 +0000 Subject: [PATCH 02/58] Validate canonical cooperative cancellation history and call ranges --- src/cooperative_cancellation.rs | 341 +++++++++++++++++++++++++- src/lib.rs | 3 +- src/tests/cooperative_cancellation.rs | 268 +++++++++++++++++++- 3 files changed, 608 insertions(+), 4 deletions(-) diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index ff9b95d..f3eb71d 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -1,4 +1,5 @@ use super::*; +use std::collections::BTreeSet; const CONTROL_BUDGET: Duration = Duration::from_secs(5); @@ -17,7 +18,7 @@ pub struct CancellationRequest { pub request_id: String, pub requested_at: String, pub cleanup_deadline_at: String, - pub history_refresh_page_token: String, + pub history_refresh_page_token: Option, } /// Acknowledgment of a request targeting the current durable run. @@ -56,11 +57,347 @@ impl CancellationRequest { request_id: text(value, "request_id")?.to_owned(), requested_at: requested_at.to_owned(), cleanup_deadline_at: cleanup_deadline_at.to_owned(), - history_refresh_page_token: text(value, "history_refresh_page_token")?.to_owned(), + history_refresh_page_token: Some(text(value, "history_refresh_page_token")?.to_owned()), }) } } +const REQUEST_EVENT: &str = "CooperativeCancellationRequested"; +const DELIVERY_EVENT: &str = "CooperativeCancellationDelivered"; +const MAX_SEQUENCE: u64 = i64::MAX as u64; + +/// The authored durable call where canonical cancellation is delivered. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum CancellationCallKind { + Activity, + LocalActivity, + Timer, + Condition, + Signal, + Child, + Parallel, + SelectionHandle, +} + +/// One committed delivery marker, bound to the original cancellation request. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CancellationDelivery { + pub request_id: String, + pub sequence: u64, + pub call_kind: CancellationCallKind, + pub sequence_span: u64, + pub operation_sequence: Option, + pub operation_sequence_span: u64, +} + +fn positive(value: &Value, field: &str, maximum: u64) -> Result { + value + .as_u64() + .filter(|number| (1..=maximum).contains(number)) + .ok_or_else(|| { + invalid(format!( + "{field} must be a positive integer within {maximum}" + )) + }) +} + +impl CancellationDelivery { + pub(crate) fn from_payload(value: &Value) -> Result { + let sequence = positive(&value["sequence"], "sequence", MAX_SEQUENCE)?; + let call_kind: CancellationCallKind = serde_json::from_value(value["call_kind"].clone()) + .map_err(|_| invalid("delivery must name a supported durable call kind"))?; + let span = positive( + value.get("sequence_span").unwrap_or(&json!(1)), + "sequence_span", + 1000, + )?; + let operation_span = positive( + value.get("operation_sequence_span").unwrap_or(&json!(1)), + "operation_sequence_span", + 1000, + )?; + let operation = value + .get("operation_sequence") + .filter(|value| !value.is_null()); + if (call_kind != CancellationCallKind::Parallel && span != 1) + || sequence > MAX_SEQUENCE - span + { + return Err(invalid( + "delivery call span is invalid or overflows the portable sequence range", + )); + } + let operation_sequence = if call_kind == CancellationCallKind::SelectionHandle { + let base = positive( + operation.unwrap_or(&Value::Null), + "operation_sequence", + MAX_SEQUENCE, + )?; + if base >= sequence || operation_span > sequence - base { + return Err(invalid( + "selection handle must name a complete earlier operation range", + )); + } + Some(base) + } else { + if operation.is_some() || operation_span != 1 { + return Err(invalid( + "only selection handles may name an operation range", + )); + } + None + }; + Ok(Self { + request_id: text(value, "workflow_command_id")?.to_owned(), + sequence, + call_kind, + sequence_span: span, + operation_sequence, + operation_sequence_span: operation_span, + }) + } + + fn range(&self) -> (u64, u64) { + self.operation_sequence + .map_or((self.sequence, self.sequence_span), |base| { + (base, self.operation_sequence_span) + }) + } + + /// Whether a sequence belongs to the canonical interrupted operation range. + pub fn interrupts(&self, sequence: u64) -> bool { + let (base, span) = self.range(); + sequence >= base && sequence - base < span + } +} + +/// Validated canonical cancellation state for one workflow run. +/// +/// An observation identifies a request. Only its committed delivery marker +/// authorizes interruption at an authored call. Earlier results retain priority. +#[derive(Clone, Debug)] +pub struct CancellationHistory { + pub request: Option, + pub delivery: Option, + pub request_index: usize, + pub delivery_index: Option, + resolved_before_request: BTreeSet, + failed_before_request: BTreeSet, + selected_before_request: BTreeSet<(u64, u64)>, +} + +impl CancellationHistory { + /// Read canonical markers without replacing original request identity. + pub fn from_events( + events: &[HistoryEvent], + run_id: &str, + observation: Option<&CancellationRequest>, + ) -> Result { + Self::read(events, run_id, observation).map_err(|error| { + invalid_recorded_history( + "cooperative_cancellation_history_invalid", + 0, + "one canonical request and matching authored delivery", + "invalid cancellation history", + &error.to_string(), + ) + }) + } + + fn read( + events: &[HistoryEvent], + run_id: &str, + observation: Option<&CancellationRequest>, + ) -> Result { + if let Some(observed) = observation { + let requested = DateTime::parse_from_rfc3339(&observed.requested_at) + .map_err(|_| invalid("observed request timestamp is invalid"))?; + let deadline = DateTime::parse_from_rfc3339(&observed.cleanup_deadline_at) + .map_err(|_| invalid("observed cleanup deadline is invalid"))?; + if observed.request_id.trim().is_empty() + || deadline <= requested + || observed + .history_refresh_page_token + .as_ref() + .is_some_and(|token| token.trim().is_empty()) + { + return Err(invalid( + "observed request identity, deadline or history token is invalid", + )); + } + } + let mut state = Self { + request: observation.cloned(), + delivery: None, + request_index: events.len(), + delivery_index: None, + resolved_before_request: BTreeSet::new(), + failed_before_request: BTreeSet::new(), + selected_before_request: BTreeSet::new(), + }; + let mut saw_request = false; + for (index, event) in events.iter().enumerate() { + if !matches!(event.event_type.as_str(), REQUEST_EVENT | DELIVERY_EVENT) { + continue; + } + let request_id = text(&event.payload, "workflow_command_id")?; + let event_run = text(&event.payload, "workflow_run_id")?; + if event + .raw + .get("workflow_command_id") + .is_some_and(|value| value.as_str() != Some(request_id)) + || (!run_id.is_empty() && event_run != run_id) + { + return Err(invalid( + "canonical event does not match its request or workflow run", + )); + } + if event.event_type == REQUEST_EVENT { + if saw_request || state.delivery.is_some() { + return Err(invalid("history must contain one request before delivery")); + } + let recorded_at = event + .raw + .get("recorded_at") + .or_else(|| event.raw.get("timestamp")) + .and_then(Value::as_str) + .ok_or_else(|| invalid("canonical request lacks its recorded timestamp"))?; + let requested = DateTime::parse_from_rfc3339(recorded_at) + .map_err(|_| invalid("canonical request timestamp is invalid"))?; + let deadline_text = text(&event.payload, "cleanup_deadline_at")?; + let deadline = DateTime::parse_from_rfc3339(deadline_text) + .map_err(|_| invalid("canonical cleanup deadline is invalid"))?; + if deadline <= requested { + return Err(invalid( + "canonical cleanup deadline must follow the request", + )); + } + if let Some(observed) = observation { + let observed_deadline = + DateTime::parse_from_rfc3339(&observed.cleanup_deadline_at) + .map_err(|_| invalid("observed cleanup deadline is invalid"))?; + if observed.request_id != request_id || observed_deadline != deadline { + return Err(invalid( + "observation changes the original request or cleanup deadline", + )); + } + } else { + state.request = Some(CancellationRequest { + request_id: request_id.to_owned(), + requested_at: recorded_at.to_owned(), + cleanup_deadline_at: deadline_text.to_owned(), + history_refresh_page_token: None, + }); + } + state.request_index = index; + saw_request = true; + } else { + if !saw_request || state.delivery.is_some() { + return Err(invalid( + "delivery requires one earlier request and one marker", + )); + } + let delivery = CancellationDelivery::from_payload(&event.payload)?; + if state + .request + .as_ref() + .map(|request| request.request_id.as_str()) + != Some(delivery.request_id.as_str()) + { + return Err(invalid("delivery names a different original request")); + } + state.delivery = Some(delivery); + state.delivery_index = Some(index); + } + } + if state.request.is_none() { + return Ok(state); + } + for event in &events[..state.request_index] { + if matches!( + event.event_type.as_str(), + "SelectionResolved" | "SelectionOperationCancelled" + ) { + let (base_field, span_field) = if event.event_type == "SelectionResolved" { + ("selection_group_base_sequence", "selection_group_size") + } else { + ("member_base_sequence", "member_size") + }; + if let (Ok(base), Ok(span)) = ( + positive(&event.payload[base_field], base_field, MAX_SEQUENCE), + positive(&event.payload[span_field], span_field, 1000), + ) { + if base <= MAX_SEQUENCE - span { + if event.event_type == "SelectionResolved" { + state.selected_before_request.insert((base, span)); + } else { + state.resolved_before_request.extend(base..base + span); + } + } + } + } + let Ok(sequence) = positive(&event.payload["sequence"], "sequence", MAX_SEQUENCE) + else { + continue; + }; + if matches!( + event.event_type.as_str(), + "ActivityCompleted" + | "ActivityFailed" + | "ActivityCancelled" + | "ActivityTimedOut" + | "TimerFired" + | "TimerCancelled" + | "ConditionWaitSatisfied" + | "ConditionWaitTimedOut" + | "SignalApplied" + | "ChildRunCompleted" + | "ChildRunFailed" + | "ChildRunCancelled" + | "ChildRunTerminated" + ) { + state.resolved_before_request.insert(sequence); + if matches!( + event.event_type.as_str(), + "ActivityFailed" + | "ActivityCancelled" + | "ActivityTimedOut" + | "ChildRunFailed" + | "ChildRunCancelled" + | "ChildRunTerminated" + ) { + state.failed_before_request.insert(sequence); + } + } + } + if let Some(delivery) = &state.delivery { + let (base, span) = delivery.range(); + if !state.range_eligible(base, span) { + return Err(invalid( + "delivery cannot replace an earlier committed result", + )); + } + } + Ok(state) + } + + fn range_eligible(&self, sequence: u64, span: u64) -> bool { + (1..=1000).contains(&span) + && sequence > 0 + && sequence <= MAX_SEQUENCE - span + && !(sequence..sequence + span) + .all(|sequence| self.resolved_before_request.contains(&sequence)) + && !(sequence..sequence + span) + .any(|sequence| self.failed_before_request.contains(&sequence)) + && !self.selected_before_request.contains(&(sequence, span)) + } + + /// Whether an unresolved authored call may deliver the pending request. + pub fn eligible(&self, sequence: u64, span: u64) -> bool { + self.request.is_some() && self.delivery.is_none() && self.range_eligible(sequence, span) + } +} + fn supports_protocol(version: &str) -> bool { let Some((major, minor)) = version.split_once('.') else { return false; diff --git a/src/lib.rs b/src/lib.rs index 6ba981f..6728c91 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,7 +5,8 @@ mod runtime_payloads; mod runtime_uploads; pub use cooperative_cancellation::{ - CancellationRequest, CooperativeCancellationOptions, WorkflowCancellationRequest, + CancellationCallKind, CancellationDelivery, CancellationHistory, CancellationRequest, + CooperativeCancellationOptions, WorkflowCancellationRequest, }; use std::{ diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs index 01c0814..29f0c55 100644 --- a/src/tests/cooperative_cancellation.rs +++ b/src/tests/cooperative_cancellation.rs @@ -9,6 +9,272 @@ fn request_observation() -> Value { }) } +fn event(kind: &str, payload: Value) -> HistoryEvent { + serde_json::from_value(json!({"event_type":kind, "payload":payload})).unwrap() +} + +fn canonical_request() -> HistoryEvent { + serde_json::from_value(json!({ + "event_type":"CooperativeCancellationRequested", "workflow_command_id":"original-request", + "recorded_at":"2026-10-01T08:00:00.000120Z", "payload":{ + "workflow_command_id":"original-request", "workflow_run_id":"run", + "cleanup_deadline_at":"2026-10-01T08:10:00Z" + } + })) + .unwrap() +} + +fn canonical_delivery(sequence: u64, kind: &str) -> HistoryEvent { + event( + "CooperativeCancellationDelivered", + json!({ + "workflow_command_id":"original-request", "workflow_run_id":"run", + "sequence":sequence, "call_kind":kind + }), + ) +} + +fn history(events: &[HistoryEvent]) -> Result { + CancellationHistory::from_events(events, "run", None) +} + +#[test] +fn cooperative_history_preserves_observation_without_authorizing_delivery() { + let observation = CancellationRequest::from_observation(&request_observation()).unwrap(); + let state = CancellationHistory::from_events(&[canonical_request()], "run", Some(&observation)) + .unwrap(); + assert_eq!(state.request.unwrap(), observation); + assert_eq!(state.delivery, None); + let pending = history(&[canonical_request()]).unwrap(); + assert!(pending.eligible(1, 1)); + assert_eq!(pending.request.unwrap().history_refresh_page_token, None); +} + +#[test] +fn cooperative_history_preserves_earlier_results_and_allows_later_unresolved_calls() { + let state = history(&[ + event("ActivityCompleted", json!({"sequence":1})), + canonical_request(), + event("TimerFired", json!({"sequence":2})), + ]) + .unwrap(); + assert!(!state.eligible(1, 1)); + assert!(state.eligible(2, 1)); + assert!(state.eligible(1, 2)); +} + +#[test] +fn cooperative_history_preserves_prior_parallel_failure_and_committed_selection() { + for prior in [ + event("ActivityFailed", json!({"sequence":1})), + event( + "SelectionResolved", + json!({ + "selection_group_base_sequence":1,"selection_group_size":2 + }), + ), + ] { + let state = history(&[prior.clone(), canonical_request()]).unwrap(); + assert!(!state.eligible(1, 2)); + let mut marker = canonical_delivery(1, "parallel"); + marker.payload["sequence_span"] = json!(2); + assert!(matches!( + history(&[prior, canonical_request(), marker]), + Err(Error::NonDeterministicReplay(_)) + )); + } +} + +#[test] +fn cooperative_history_preserves_cancelled_selection_member_ranges() { + let state = history(&[ + event( + "SelectionOperationCancelled", + json!({"member_base_sequence":3,"member_size":2}), + ), + canonical_request(), + ]) + .unwrap(); + assert!(!state.eligible(3, 2)); + assert!(state.eligible(4, 2)); +} + +#[test] +fn cooperative_history_cold_delivery_retains_parallel_and_selection_operation_ranges() { + let mut parallel = canonical_delivery(2, "parallel"); + parallel.payload["sequence_span"] = json!(3); + let state = history(&[canonical_request(), parallel]).unwrap(); + assert_eq!(state.request_index, 0); + assert_eq!(state.delivery_index, Some(1)); + assert!(!state.eligible(5, 1)); + let delivery = state.delivery.unwrap(); + assert_eq!( + (1..=5) + .map(|sequence| delivery.interrupts(sequence)) + .collect::>(), + vec![false, true, true, true, false] + ); + let mut selected = canonical_delivery(6, "selection_handle"); + selected.payload["operation_sequence"] = json!(2); + selected.payload["operation_sequence_span"] = json!(2); + let delivered = history(&[canonical_request(), selected]) + .unwrap() + .delivery + .unwrap(); + assert!(delivered.interrupts(2)); + assert!(delivered.interrupts(3)); + assert!(!delivered.interrupts(4)); + assert!(!delivered.interrupts(6)); +} + +#[test] +fn cooperative_history_rejects_reordered_duplicate_and_mismatched_markers() { + for events in [ + vec![canonical_delivery(2, "timer")], + vec![canonical_request(), canonical_request()], + vec![ + canonical_request(), + canonical_delivery(2, "timer"), + canonical_delivery(2, "timer"), + ], + vec![canonical_delivery(2, "timer"), canonical_request()], + ] { + assert!(matches!( + history(&events), + Err(Error::NonDeterministicReplay(_)) + )); + } + for (field, value) in [ + ("workflow_command_id", json!("different")), + ("workflow_run_id", json!("different")), + ("workflow_command_id", json!(" ")), + ("workflow_run_id", Value::Null), + ] { + let mut marker = canonical_delivery(2, "timer"); + marker.payload[field] = value; + assert!(matches!( + history(&[canonical_request(), marker]), + Err(Error::NonDeterministicReplay(_)) + )); + } + let mut request = canonical_request(); + request + .raw + .insert("workflow_command_id".into(), json!("different")); + assert!(matches!( + history(&[request]), + Err(Error::NonDeterministicReplay(_)) + )); +} + +#[test] +fn cooperative_history_rejects_changed_observation_identity_deadline_and_malformed_time() { + for (field, value) in [ + ("request_id", json!("different")), + ("cleanup_deadline_at", json!("2026-10-01T08:11:00Z")), + ("requested_at", json!("no timestamp")), + ("history_refresh_page_token", json!(" ")), + ] { + let mut observation = request_observation(); + observation[field] = value; + match CancellationRequest::from_observation(&observation) { + Ok(observed) => assert!(matches!( + CancellationHistory::from_events(&[canonical_request()], "run", Some(&observed)), + Err(Error::NonDeterministicReplay(_)) + )), + Err(error) => assert!(matches!(error, Error::InvalidCooperativeCancellation(_))), + } + } + let mut request = canonical_request(); + request.raw.insert("recorded_at".into(), json!("bad")); + assert!(matches!( + history(&[request]), + Err(Error::NonDeterministicReplay(_)) + )); +} + +#[test] +fn cooperative_history_accepts_equivalent_deadline_timezone_without_changing_original_text() { + let mut observation = request_observation(); + observation["cleanup_deadline_at"] = json!("2026-10-01T10:10:00+02:00"); + let observed = CancellationRequest::from_observation(&observation).unwrap(); + let state = + CancellationHistory::from_events(&[canonical_request()], "run", Some(&observed)).unwrap(); + assert_eq!(state.request.unwrap(), observed); +} + +#[test] +fn cooperative_history_rejects_invalid_call_ranges_and_portable_integer_overflow() { + for (field, value) in [ + ("sequence", json!(0)), + ("sequence", json!(true)), + ("sequence", json!("2")), + ("sequence", json!(i64::MAX)), + ("sequence", json!(u64::MAX)), + ("call_kind", json!("unknown")), + ("sequence_span", json!(2)), + ("sequence_span", Value::Null), + ("operation_sequence", json!(1)), + ("operation_sequence_span", json!(2)), + ] { + let mut marker = canonical_delivery(2, "timer"); + marker.payload[field] = value; + assert!(matches!( + history(&[canonical_request(), marker]), + Err(Error::NonDeterministicReplay(_)) + )); + } + for (base, span) in [(0, 1), (6, 1), (5, 2), (2, 1001)] { + let mut selected = canonical_delivery(6, "selection_handle"); + selected.payload["operation_sequence"] = json!(base); + selected.payload["operation_sequence_span"] = json!(span); + assert!(matches!( + history(&[canonical_request(), selected]), + Err(Error::NonDeterministicReplay(_)) + )); + } + let mut parallel = canonical_delivery(2, "parallel"); + parallel.payload["sequence_span"] = json!(1001); + assert!(matches!( + history(&[canonical_request(), parallel]), + Err(Error::NonDeterministicReplay(_)) + )); + assert!(!history(&[canonical_request()]) + .unwrap() + .eligible(u64::MAX, 1)); +} + +#[test] +fn cooperative_history_rejects_delivery_over_an_earlier_resolved_call() { + for kind in [ + "ActivityCompleted", + "ActivityFailed", + "ActivityCancelled", + "ActivityTimedOut", + "TimerFired", + "TimerCancelled", + "ConditionWaitSatisfied", + "ConditionWaitTimedOut", + "SignalApplied", + "ChildRunCompleted", + "ChildRunFailed", + "ChildRunCancelled", + "ChildRunTerminated", + ] { + assert!( + matches!( + history(&[ + event(kind, json!({"sequence":2})), + canonical_request(), + canonical_delivery(2, "timer") + ]), + Err(Error::NonDeterministicReplay(_)) + ), + "{kind}" + ); + } +} + fn responses(path: &str, _body: &str, number: usize) -> Option<(&'static str, String)> { let case = path.split('/').nth(1).unwrap_or_default(); if path.ends_with("/api/cluster/info") { @@ -369,7 +635,7 @@ async fn cooperative_request_bounds_discovery_and_mutation_with_one_total_budget assert!(matches!(result, Err(Error::Timeout)), "{result:?}"); assert!(started.elapsed() >= Duration::from_secs(4)); assert!( - started.elapsed() < Duration::from_millis(5300), + started.elapsed() < Duration::from_secs(7), "total budget exceeded: {:?}", started.elapsed() ); From 8126c865af56cd2d61cabdb62f2fd8d2781a9121 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 08:21:08 +0000 Subject: [PATCH 03/58] Fence malformed canonical cancellation before worker replay --- src/lib.rs | 5 +++ ...ooperative-duplicate-request-rejected.json | 37 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 tests/fixtures/replay-regressions/cooperative-duplicate-request-rejected.json diff --git a/src/lib.rs b/src/lib.rs index 6728c91..2f174ac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7603,6 +7603,11 @@ impl Worker { fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result { validate_workflow_task_payloads(&task)?; + CancellationHistory::from_events( + &task.history_events, + task.run_id.as_deref().unwrap_or_default(), + None, + )?; if let Some(update_id) = task .workflow_update_id diff --git a/tests/fixtures/replay-regressions/cooperative-duplicate-request-rejected.json b/tests/fixtures/replay-regressions/cooperative-duplicate-request-rejected.json new file mode 100644 index 0000000..c9cb5d2 --- /dev/null +++ b/tests/fixtures/replay-regressions/cooperative-duplicate-request-rejected.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-cooperative-duplicate-request-rejected", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": { + "type": "corpus.side-effect-version", + "input": [], + "payload_codec": "avro" + }, + "history": [ + { + "event_type": "CooperativeCancellationRequested", + "workflow_command_id": "original-request", + "recorded_at": "2026-10-01T08:00:00Z", + "payload": { + "workflow_command_id": "original-request", + "workflow_run_id": "regression-corpus-run", + "cleanup_deadline_at": "2026-10-01T08:10:00Z" + } + }, + { + "event_type": "CooperativeCancellationRequested", + "workflow_command_id": "original-request", + "recorded_at": "2026-10-01T08:00:00Z", + "payload": { + "workflow_command_id": "original-request", + "workflow_run_id": "regression-corpus-run", + "cleanup_deadline_at": "2026-10-01T08:10:00Z" + } + } + ], + "command_sequence": [{"type":"complete_workflow", "result":"unreachable"}], + "expected": {"type":"complete_workflow", "result":"unreachable"}, + "expected_failure": "cooperative_cancellation_history_invalid" +} From efcb24b08f463287dd2faa84ddff96b66c7903db Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 08:40:08 +0000 Subject: [PATCH 04/58] Replay committed scalar cancellation and shield saga cleanup --- src/cooperative_cancellation.rs | 172 ++++++++ src/lib.rs | 200 ++++++++-- src/tests/cooperative_cancellation.rs | 369 ++++++++++++++++++ .../cooperative-cold-activity-delivery.json | 33 ++ 4 files changed, 737 insertions(+), 37 deletions(-) create mode 100644 tests/fixtures/replay-regressions/cooperative-cold-activity-delivery.json diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index f3eb71d..5e1432b 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -30,6 +30,30 @@ pub struct WorkflowCancellationRequest { pub cancellation_request: CancellationRequest, } +/// Cancellation delivered at its committed authored call, with original identity. +#[derive(Clone, Debug, Error, PartialEq, Eq)] +#[error("workflow cancellation {request_id} was requested", request_id = .request.request_id)] +pub struct CooperativeCancellationRequested { + pub request: CancellationRequest, + pub delivery: CancellationDelivery, +} + +/// A workflow-local cleanup scope. Dropping it restores cancellation checks. +/// +/// This does not renew or extend the Server's original cleanup deadline. +#[derive(Debug)] +pub struct CancellationShield { + state: Arc>, +} + +impl Drop for CancellationShield { + fn drop(&mut self) { + if let Ok(mut state) = self.state.lock() { + state.cancellation_shield_depth = state.cancellation_shield_depth.saturating_sub(1); + } + } +} + fn invalid(detail: impl Into) -> Error { Error::InvalidCooperativeCancellation(detail.into()) } @@ -187,6 +211,55 @@ pub struct CancellationHistory { } impl CancellationHistory { + pub(super) fn bind_scalar_commands( + &self, + mut commands: Vec, + ) -> Result> { + let Some(delivery) = &self.delivery else { + return Ok(commands); + }; + if !matches!( + delivery.call_kind, + CancellationCallKind::Activity + | CancellationCallKind::Timer + | CancellationCallKind::Condition + | CancellationCallKind::Signal + | CancellationCallKind::Child + ) { + return Ok(commands); + } + let index = commands.partition_point(|command| command.sequence() < delivery.sequence); + let original = if commands + .get(index) + .is_some_and(|command| command.sequence() == delivery.sequence) + { + Some(Box::new(commands.remove(index))) + } else { + let previous = index + .checked_sub(1) + .map_or(0, |index| commands[index].sequence()); + if previous.checked_add(1) != Some(delivery.sequence) { + return Err(invalid_recorded_history( + "cooperative_cancellation_call_mismatch", + delivery.sequence, + "next authored durable call", + "missing earlier call", + "cancellation marker skips an unrecorded authored command", + )); + } + None + }; + commands.insert( + index, + RecordedCommand::CancellationBoundary { + sequence: delivery.sequence, + call_kind: delivery.call_kind, + original, + }, + ); + Ok(commands) + } + /// Read canonical markers without replacing original request identity. pub fn from_events( events: &[HistoryEvent], @@ -398,6 +471,105 @@ impl CancellationHistory { } } +impl WorkflowState { + pub(super) fn cancellation_error(&self) -> Error { + match ( + &self.cancellation_history.request, + &self.cancellation_history.delivery, + ) { + (Some(request), Some(delivery)) => { + Error::CooperativeCancellationRequested(CooperativeCancellationRequested { + request: request.clone(), + delivery: delivery.clone(), + }) + } + _ => Error::WorkflowCancellationRequested(WorkflowCancellationRequested), + } + } + + fn validate_cancellation_call( + &self, + sequence: u64, + kind: CancellationCallKind, + recorded_kind: CancellationCallKind, + ) -> Result<()> { + if kind != recorded_kind || self.cancellation_shield_depth > 0 { + return Err(invalid_recorded_history( + "cooperative_cancellation_call_mismatch", + sequence, + "matching unshielded authored call", + &format!("{kind:?}"), + "committed cancellation call kind or cleanup scope changed", + )); + } + Ok(()) + } + + pub(super) fn cancellation_replay_command( + &mut self, + index: usize, + kind: CancellationCallKind, + ) -> Result> { + match self.recorded_commands.get(index).cloned() { + Some(RecordedCommand::CancellationBoundary { + sequence, + call_kind, + original, + }) => { + if let Some(original) = original { + return Ok(Some(*original)); + } + self.validate_cancellation_call(sequence, kind, call_kind)?; + self.cancellation_consumed = true; + self.cancel_requested = true; + self.command_cursor = index + 1; + Err(self.cancellation_error()) + } + command => Ok(command), + } + } + + pub(super) fn replay_cancellation_at( + &mut self, + index: usize, + kind: CancellationCallKind, + ) -> Result<()> { + if let Some(RecordedCommand::CancellationBoundary { + sequence, + call_kind, + .. + }) = self.recorded_commands.get(index) + { + self.validate_cancellation_call(*sequence, kind, *call_kind)?; + self.cancellation_consumed = true; + self.cancel_requested = true; + self.command_cursor = index + 1; + return Err(self.cancellation_error()); + } + Ok(()) + } +} + +impl WorkflowContext { + /// Shield explicit cleanup from repeated cancellation checks. + /// + /// Keep the returned guard alive across cleanup awaits. Scopes can nest. + /// The Server retains and enforces the original immutable cleanup deadline. + pub fn cancellation_shield(&self) -> Result { + let mut state = self + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)?; + state.cancellation_shield_depth = state + .cancellation_shield_depth + .checked_add(1) + .ok_or_else(|| invalid("cancellation shield nesting overflowed"))?; + Ok(CancellationShield { + state: Arc::clone(&self.state), + }) + } +} + fn supports_protocol(version: &str) -> bool { let Some((major, minor)) = version.split_once('.') else { return false; diff --git a/src/lib.rs b/src/lib.rs index 2f174ac..43d5981 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,7 +6,8 @@ mod runtime_uploads; pub use cooperative_cancellation::{ CancellationCallKind, CancellationDelivery, CancellationHistory, CancellationRequest, - CooperativeCancellationOptions, WorkflowCancellationRequest, + CancellationShield, CooperativeCancellationOptions, CooperativeCancellationRequested, + WorkflowCancellationRequest, }; use std::{ @@ -191,6 +192,8 @@ pub enum Error { #[error(transparent)] WorkflowCancellationRequested(WorkflowCancellationRequested), #[error(transparent)] + CooperativeCancellationRequested(CooperativeCancellationRequested), + #[error(transparent)] WorkflowCommandRejected(WorkflowCommandRejection), #[error(transparent)] WorkflowFailed(WorkflowTerminalOutcome), @@ -7603,17 +7606,16 @@ impl Worker { fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result { validate_workflow_task_payloads(&task)?; - CancellationHistory::from_events( - &task.history_events, - task.run_id.as_deref().unwrap_or_default(), - None, - )?; - if let Some(update_id) = task .workflow_update_id .as_deref() .filter(|update_id| !update_id.is_empty()) { + CancellationHistory::from_events( + &task.history_events, + task.run_id.as_deref().unwrap_or_default(), + None, + )?; return self .execute_update_task(&task, update_id) .map(WorkflowTaskDecision::without_message_streams); @@ -7648,7 +7650,9 @@ impl Worker { )?; workflow_state.history_budget = history_budget; workflow_state.workflow_command_identity = workflow_command_identity; - workflow_state.cancel_requested = task.cancel_requested; + if workflow_state.cancellation_history.request.is_none() { + workflow_state.cancel_requested = task.cancel_requested; + } let state = Arc::new(Mutex::new(workflow_state)); let ctx = WorkflowContext { state }; let mut future = (workflow.execute)(ctx.clone(), input); @@ -8529,10 +8533,11 @@ impl WorkflowContext { Saga::new(self.clone()) } - /// Whether this task carries a cooperative cancellation request. + /// Whether cancellation has reached this replay's authored boundary. /// /// Server's current `/cancel` route is terminal and does not set this flag. - /// Service-mode cooperative cancellation is not yet available. + /// A canonical service request remains pending until its committed delivery + /// is consumed. Request observation alone does not authorize an exception. pub fn is_cancellation_requested(&self) -> Result { let state = self .state @@ -8546,10 +8551,12 @@ impl WorkflowContext { /// Passing this result to [`Saga::finish`] compensates already registered /// forward steps before the cancellation remains the initiating outcome. pub fn throw_if_cancellation_requested(&self) -> Result<()> { - if self.is_cancellation_requested()? { - return Err(Error::WorkflowCancellationRequested( - WorkflowCancellationRequested, - )); + let state = self + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)?; + if state.cancel_requested && state.cancellation_shield_depth == 0 { + return Err(state.cancellation_error()); } Ok(()) } @@ -9283,6 +9290,20 @@ impl WorkflowContext { "workflow completed before consuming all recorded durable commands", ))); } + if let Some(delivery) = state + .cancellation_history + .delivery + .as_ref() + .filter(|_| !state.cancellation_consumed) + { + return Err(invalid_recorded_history( + "cooperative_cancellation_unconsumed", + delivery.sequence, + "committed cancellation call", + "workflow completion", + "workflow completed without reaching its committed cancellation boundary", + )); + } if let Some(sequence) = state .recorded_continue_as_new_sequence .filter(|_| !state.continue_as_new_consumed) @@ -9342,6 +9363,9 @@ struct WorkflowState { history_events: Arc>, history_budget: WorkflowHistoryBudget, cancel_requested: bool, + cancellation_history: CancellationHistory, + cancellation_consumed: bool, + cancellation_shield_depth: u64, resume_signal: Option, recorded_commands: Vec, selection_markers: Vec, @@ -9387,14 +9411,19 @@ impl WorkflowState { payload_codec: String, resume_signal: Option, ) -> Result { - let recorded_commands = recorded_commands( + let cancellation_history = CancellationHistory::from_events( + &history, + run_id.as_deref().unwrap_or_default(), + None, + )?; + let recorded_commands = cancellation_history.bind_scalar_commands(recorded_commands( &history, &payload_codec, WorkflowIdentity { workflow_id: workflow_id.clone(), run_id: run_id.clone(), }, - )?; + )?)?; let selection_markers = recorded_selection_markers(&history)?; let cancelled_selection_members = recorded_selection_cancellations(&history)?; let recorded_continue_as_new = history @@ -9484,6 +9513,9 @@ impl WorkflowState { ..WorkflowHistoryBudget::default() }, cancel_requested, + cancellation_history, + cancellation_consumed: false, + cancellation_shield_depth: 0, resume_signal, recorded_commands, selection_markers, @@ -9569,6 +9601,11 @@ fn decode_message_stream_delivery(arguments: Vec) -> Result>, + }, Activity { sequence: u64, activity_type: Option, @@ -10082,7 +10119,8 @@ fn activity_options_description(options: &RecordedActivityOptions) -> String { impl RecordedCommand { fn sequence(&self) -> u64 { match self { - Self::Activity { sequence, .. } + Self::CancellationBoundary { sequence, .. } + | Self::Activity { sequence, .. } | Self::Timer { sequence, .. } | Self::ChildWorkflow { sequence, .. } | Self::SignalWait { sequence, .. } @@ -10096,6 +10134,7 @@ impl RecordedCommand { fn shape(&self) -> &'static str { match self { + Self::CancellationBoundary { .. } => "cooperative cancellation", Self::Activity { .. } => "activity", Self::Timer { .. } => "timer", Self::ChildWorkflow { .. } => "child workflow", @@ -11626,7 +11665,8 @@ fn recorded_selection_member_is_terminal( RecordedCommand::SearchAttributes { .. } | RecordedCommand::SideEffect { .. } | RecordedCommand::VersionMarker { .. } - | RecordedCommand::Memo { .. } => false, + | RecordedCommand::Memo { .. } + | RecordedCommand::CancellationBoundary { .. } => false, }; if !terminal { all_completed = false; @@ -11861,6 +11901,10 @@ impl Saga { /// Compensate `initiating_failure` and return the failure that must remain. pub async fn compensate(mut self, initiating_failure: Error) -> Error { + let _shield = match self.ctx.cancellation_shield() { + Ok(shield) => shield, + Err(error) => return error, + }; while let Some(compensation) = self.compensations.pop() { if let Err(compensation_failure) = self .ctx @@ -11936,7 +11980,13 @@ impl ActivityCall { retry_policy: current_activity_retry_snapshot(&options), }; - if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() { + let cursor = state.command_cursor; + let recorded = + match state.cancellation_replay_command(cursor, CancellationCallKind::Activity) { + Ok(recorded) => recorded, + Err(error) => return Poll::Ready(Err(error)), + }; + if let Some(recorded) = recorded { let sequence = recorded.sequence(); match recorded { RecordedCommand::Activity { @@ -12010,6 +12060,11 @@ impl ActivityCall { ))); } } + if let Err(error) = + state.replay_cancellation_at(cursor, CancellationCallKind::Activity) + { + return Poll::Ready(Err(error)); + } state.command_cursor += 1; if let Some(outcome) = outcome { return Poll::Ready(outcome.map_err(Error::ActivityFailed)); @@ -12113,7 +12168,13 @@ impl Future for TimerCall { Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)), }; - if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() { + let cursor = state.command_cursor; + let recorded = match state.cancellation_replay_command(cursor, CancellationCallKind::Timer) + { + Ok(recorded) => recorded, + Err(error) => return Poll::Ready(Err(error)), + }; + if let Some(recorded) = recorded { match recorded { RecordedCommand::Timer { sequence, @@ -12140,6 +12201,11 @@ impl Future for TimerCall { ), ))); } + if let Err(error) = + state.replay_cancellation_at(cursor, CancellationCallKind::Timer) + { + return Poll::Ready(Err(error)); + } state.command_cursor += 1; if fired { return Poll::Ready(Ok(())); @@ -12218,7 +12284,14 @@ impl Future for ConditionWaitCall { Ok(state) => state, Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)), }; - let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else { + let initial_cursor = state.command_cursor; + let recorded = match state + .cancellation_replay_command(initial_cursor, CancellationCallKind::Condition) + { + Ok(recorded) => recorded, + Err(error) => return Poll::Ready(Err(error)), + }; + let Some(recorded) = recorded else { drop(state); return self.poll_new_condition(options); }; @@ -12229,36 +12302,50 @@ impl Future for ConditionWaitCall { let mut cursor = state.command_cursor; let mut result = None; loop { + if cursor > initial_cursor + && matches!( + state.recorded_commands.get(cursor), + Some(RecordedCommand::CancellationBoundary { original: None, .. }) + ) + { + break; + } + let recorded = match state + .cancellation_replay_command(cursor, CancellationCallKind::Condition) + { + Ok(recorded) => recorded, + Err(error) => return Poll::Ready(Err(error)), + }; let Some(RecordedCommand::ConditionWait { sequence, - occurrence_id: recorded_occurrence_id, - condition_key, - predicate_identity, + occurrence_id: ref recorded_occurrence_id, + ref condition_key, + ref predicate_identity, timeout_seconds, result: recorded_result, - parallel_group_path, + ref parallel_group_path, .. - }) = state.recorded_commands.get(cursor) + }) = recorded else { break; }; - if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id { + if cursor > initial_cursor && recorded_occurrence_id != &occurrence_id { break; } if let Err(error) = ensure_parallel_path_matches( - *sequence, + sequence, parallel_group_path.as_deref(), &self.parallel_group_path, ) { return Poll::Ready(Err(error)); } if let Err(error) = validate_recorded_condition_wait( - *sequence, + sequence, recorded_occurrence_id, condition_key.as_deref(), predicate_identity, - *timeout_seconds, + timeout_seconds, &occurrence_id, &options, ) { @@ -12267,13 +12354,18 @@ impl Future for ConditionWaitCall { if result == Some(ConditionWaitResult::TimedOut) { return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new( "condition_wait_reopened_after_timeout", - Some(*sequence), + Some(sequence), Some("timed-out condition is terminal".to_string()), Some("another physical wait-open".to_string()), "condition history reopened one logical wait after its durable timeout", )))); } - result = *recorded_result; + if let Err(error) = + state.replay_cancellation_at(cursor, CancellationCallKind::Condition) + { + return Poll::Ready(Err(error)); + } + result = recorded_result; cursor += 1; } state.command_cursor = cursor; @@ -12417,7 +12509,13 @@ impl ChildWorkflowCall { Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)), }; - if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() { + let cursor = state.command_cursor; + let recorded = match state.cancellation_replay_command(cursor, CancellationCallKind::Child) + { + Ok(recorded) => recorded, + Err(error) => return Poll::Ready(Err(error)), + }; + if let Some(recorded) = recorded { let sequence = recorded.sequence(); match recorded { RecordedCommand::ChildWorkflow { @@ -12446,6 +12544,11 @@ impl ChildWorkflowCall { ))); } } + if let Err(error) = + state.replay_cancellation_at(cursor, CancellationCallKind::Child) + { + return Poll::Ready(Err(error)); + } state.command_cursor += 1; if let Some(outcome) = outcome { return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed)); @@ -12606,7 +12709,13 @@ impl SignalCall { Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)), }; - if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() { + let cursor = state.command_cursor; + let recorded = match state.cancellation_replay_command(cursor, CancellationCallKind::Signal) + { + Ok(recorded) => recorded, + Err(error) => return Poll::Ready(Err(error)), + }; + if let Some(recorded) = recorded { match recorded { RecordedCommand::SignalWait { sequence, @@ -12633,6 +12742,11 @@ impl SignalCall { ))); } + if let Err(error) = + state.replay_cancellation_at(cursor, CancellationCallKind::Signal) + { + return Poll::Ready(Err(error)); + } state.command_cursor += 1; if let Some(value) = value { return Poll::Ready(Ok(value)); @@ -14664,6 +14778,12 @@ fn workflow_failure_command( "durable_workflow::WorkflowCancellationRequested", json!({"reason": "cancelled"}), ), + Error::CooperativeCancellationRequested(cancellation) => ( + "WorkflowCancellationRequested", + "durable_workflow::CooperativeCancellationRequested", + json!({"reason": "cancelled", "request_id": cancellation.request.request_id, + "cleanup_deadline_at": cancellation.request.cleanup_deadline_at}), + ), Error::NonDeterministicReplay(_) => ( "NonDeterministicReplay", "durable_workflow::Error", @@ -14678,7 +14798,9 @@ fn workflow_failure_command( Error::SagaCompensationFailed(failure) => { workflow_error_non_retryable(&failure.compensation_failure) } - Error::WorkflowCancellationRequested(_) => true, + Error::WorkflowCancellationRequested(_) | Error::CooperativeCancellationRequested(_) => { + true + } Error::NonDeterministicReplay(_) => true, _ => false, }; @@ -14719,7 +14841,9 @@ fn workflow_error_type(error: &Error) -> &'static str { }, Error::ParallelFailed(_) => "ParallelFailed", Error::SagaCompensationFailed(_) => "SagaCompensationFailed", - Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested", + Error::WorkflowCancellationRequested(_) | Error::CooperativeCancellationRequested(_) => { + "WorkflowCancellationRequested" + } Error::NonDeterministicReplay(_) => "NonDeterministicReplay", _ => "RustWorkflowError", } @@ -14733,7 +14857,9 @@ fn workflow_error_non_retryable(error: &Error) -> bool { Error::SagaCompensationFailed(failure) => { workflow_error_non_retryable(&failure.compensation_failure) } - Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true, + Error::WorkflowCancellationRequested(_) + | Error::CooperativeCancellationRequested(_) + | Error::NonDeterministicReplay(_) => true, _ => false, } } diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs index 29f0c55..c25eff6 100644 --- a/src/tests/cooperative_cancellation.rs +++ b/src/tests/cooperative_cancellation.rs @@ -38,6 +38,375 @@ fn history(events: &[HistoryEvent]) -> Result { CancellationHistory::from_events(events, "run", None) } +fn cancellation_worker() -> Worker { + Worker::new( + Client::builder("http://127.0.0.1:1").build().unwrap(), + "queue", + ) +} + +fn cancellation_task(events: Vec) -> WorkflowTask { + let mut task = workflow_task("cancel", events, DEFAULT_CODEC); + task.run_id = Some("run".into()); + task +} + +fn scheduled_timer(sequence: u64) -> HistoryEvent { + event( + "TimerScheduled", + json!({"sequence":sequence, "timer_id":format!("timer-{sequence}"), "delay_seconds":5}), + ) +} + +fn completed_activity(sequence: u64, name: &str, value: Value) -> Vec { + vec![ + event( + "ActivityScheduled", + json!({"sequence":sequence,"activity_type":name,"task_queue":"queue"}), + ), + event( + "ActivityCompleted", + json!({"sequence":sequence,"activity_type":name,"result":fixture_envelope(value)}), + ), + ] +} + +#[test] +fn cooperative_replay_preserves_completed_forward_result_and_saga_cleanup_identity() { + for _cold_restart in 0..2 { + let mut events = completed_activity(1, "forward", json!(7)); + events.extend([ + scheduled_timer(2), + canonical_request(), + canonical_delivery(2, "timer"), + ]); + events.extend(completed_activity(3, "undo", Value::Null)); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + let result = ctx.activity("forward", json!([])).await?; + assert_eq!(result, json!(7)); + assert!(!ctx.is_cancellation_requested()?); + let mut saga = ctx.saga(); + saga.add_compensation("undo", json!([]))?; + saga.finish(ctx.sleep(Duration::from_secs(5)).await).await?; + Ok(Value::Null) + }); + let commands = worker + .execute_workflow_task(cancellation_task(events)) + .unwrap(); + assert_eq!(commands.len(), 1); + assert_eq!( + commands[0]["exception_type"], + "WorkflowCancellationRequested" + ); + assert_eq!( + commands[0]["exception"]["properties"]["request_id"], + "original-request" + ); + assert_eq!( + commands[0]["exception"]["properties"]["cleanup_deadline_at"], + "2026-10-01T08:10:00Z" + ); + } +} + +#[test] +fn cooperative_replay_saga_schedules_cleanup_once_after_committed_delivery() { + let mut events = completed_activity(1, "forward", json!(7)); + events.extend([ + scheduled_timer(2), + canonical_request(), + canonical_delivery(2, "timer"), + ]); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + ctx.activity("forward", json!([])).await?; + let mut saga = ctx.saga(); + saga.add_compensation("undo", json!([]))?; + saga.finish(ctx.sleep(Duration::from_secs(5)).await).await?; + Ok(Value::Null) + }); + let commands = worker + .execute_workflow_task(cancellation_task(events)) + .unwrap(); + assert_eq!(commands.len(), 1); + assert_eq!(commands[0]["type"], "schedule_activity"); + assert_eq!(commands[0]["activity_type"], "undo"); +} + +#[test] +fn cooperative_replay_consumes_reopened_condition_once_at_its_physical_delivery() { + let payload = |sequence| { + json!({"sequence":sequence,"condition_wait_id":format!("condition:{sequence}"), + "condition_wait_occurrence_id":"rust:condition-wait:0", "condition_key":"ready", + "condition_definition_fingerprint":"sha256:ready"}) + }; + let events = vec![ + event("ConditionWaitOpened", payload(1)), + event("ConditionWaitSatisfied", payload(1)), + event("ConditionWaitOpened", payload(2)), + canonical_request(), + canonical_delivery(2, "condition"), + ]; + for _cold_restart in 0..2 { + let ctx = workflow_context(events.clone()); + let mut wait = Box::pin( + ctx.wait_condition(ConditionWaitOptions::new("ready", "sha256:ready"), || { + Ok(false) + }), + ); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + let Poll::Ready(Err(Error::CooperativeCancellationRequested(cancellation))) = + wait.as_mut().poll(&mut cx) + else { + panic!("reopened condition must receive canonical cancellation") + }; + assert_eq!(cancellation.delivery.sequence, 2); + assert_eq!( + ctx.state.lock().unwrap().condition_wait_occurrence_counter, + 1 + ); + ctx.ensure_history_consumed().unwrap(); + assert!(ctx.take_commands().unwrap().is_empty()); + } +} + +#[test] +fn cooperative_replay_keeps_adjacent_condition_occurrences_and_cleanup_scopes_separate() { + let payload = |sequence, occurrence| { + json!({"sequence":sequence,"condition_wait_id":format!("condition:{sequence}"), + "condition_wait_occurrence_id":format!("rust:condition-wait:{occurrence}"), "condition_key":"ready", + "condition_definition_fingerprint":"sha256:ready"}) + }; + let ctx = workflow_context(vec![ + event("ConditionWaitOpened", payload(1, 0)), + event("ConditionWaitSatisfied", payload(1, 0)), + event("ConditionWaitOpened", payload(2, 1)), + canonical_request(), + canonical_delivery(2, "condition"), + ]); + let shield = ctx.cancellation_shield().unwrap(); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + let mut first = Box::pin( + ctx.wait_condition(ConditionWaitOptions::new("ready", "sha256:ready"), || { + Ok(false) + }), + ); + assert!(matches!( + first.as_mut().poll(&mut cx), + Poll::Ready(Ok(ConditionWaitResult::Satisfied)) + )); + drop(shield); + let mut second = Box::pin( + ctx.wait_condition(ConditionWaitOptions::new("ready", "sha256:ready"), || { + Ok(false) + }), + ); + assert!(matches!( + second.as_mut().poll(&mut cx), + Poll::Ready(Err(Error::CooperativeCancellationRequested(_))) + )); + assert_eq!( + ctx.state.lock().unwrap().condition_wait_occurrence_counter, + 2 + ); + ctx.ensure_history_consumed().unwrap(); +} + +#[test] +fn cooperative_replay_marker_keeps_priority_over_resolution_committed_after_request() { + let ctx = workflow_context(vec![ + scheduled_timer(1), + canonical_request(), + canonical_delivery(1, "timer"), + event( + "TimerFired", + json!({"sequence":1,"timer_id":"timer-1","delay_seconds":5}), + ), + ]); + let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5))); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + assert!(matches!( + timer.as_mut().poll(&mut cx), + Poll::Ready(Err(Error::CooperativeCancellationRequested(_))) + )); + ctx.ensure_history_consumed().unwrap(); +} + +#[test] +fn cooperative_replay_actual_worker_delivers_scalar_calls_with_original_identity() { + for kind in ["activity", "timer", "child", "signal", "condition"] { + for _cold_restart in 0..2 { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _input| async move { + match kind { + "activity" => { + ctx.activity("forward", json!([])).await?; + } + "timer" => { + ctx.sleep(Duration::from_secs(5)).await?; + } + "child" => { + ctx.start_child_workflow( + "child", + ChildWorkflowOptions::new("queue"), + json!([]), + ) + .await?; + } + "signal" => { + ctx.wait_signal("resume").await?; + } + "condition" => { + ctx.wait_condition( + ConditionWaitOptions::new("ready", "sha256:ready"), + || Ok(false), + ) + .await?; + } + _ => unreachable!(), + } + Ok(Value::Null) + }); + let commands = worker + .execute_workflow_task(cancellation_task(vec![ + canonical_request(), + canonical_delivery(1, kind), + ])) + .unwrap(); + assert_eq!(commands.len(), 1, "{kind}"); + assert_eq!(commands[0]["type"], "fail_workflow", "{kind}"); + assert_eq!( + commands[0]["exception_type"], "WorkflowCancellationRequested", + "{kind}" + ); + assert_eq!( + commands[0]["exception"]["properties"]["request_id"], "original-request", + "{kind}" + ); + assert_eq!( + commands[0]["exception"]["properties"]["cleanup_deadline_at"], + "2026-10-01T08:10:00Z", + "{kind}" + ); + assert_eq!(commands[0]["non_retryable"], true, "{kind}"); + } + } +} + +#[test] +fn cooperative_replay_observed_request_and_task_flag_do_not_inject_without_delivery() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + assert!(!ctx.is_cancellation_requested()?); + ctx.throw_if_cancellation_requested()?; + ctx.sleep(Duration::from_secs(5)).await?; + Ok(Value::Null) + }); + let mut task = cancellation_task(vec![scheduled_timer(1), canonical_request()]); + task.cancel_requested = true; + assert!(worker.execute_workflow_task(task).unwrap().is_empty()); +} + +#[test] +fn cooperative_replay_recorded_timer_validates_authored_details_before_delivery() { + for delay in [5, 500] { + let ctx = workflow_context(vec![ + scheduled_timer(1), + canonical_request(), + canonical_delivery(1, "timer"), + ]); + let mut timer = Box::pin(ctx.sleep(Duration::from_secs(delay))); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + match timer.as_mut().poll(&mut cx) { + Poll::Ready(Err(Error::CooperativeCancellationRequested(cancellation))) + if delay == 5 => + { + assert_eq!(cancellation.request.request_id, "original-request"); + assert_eq!(cancellation.delivery.sequence, 1); + ctx.ensure_history_consumed().unwrap(); + } + Poll::Ready(Err(Error::NonDeterministicReplay(failure))) if delay == 500 => { + assert_eq!(failure.reason, "timer_delay_mismatch"); + } + other => panic!("{delay}: {other:?}"), + } + assert!(ctx.take_commands().unwrap().is_empty()); + } +} + +#[test] +fn cooperative_replay_changed_call_kind_and_unconsumed_boundary_are_rejected() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + ctx.activity("changed", json!([])).await?; + Ok(Value::Null) + }); + assert!( + matches!(worker.execute_workflow_task(cancellation_task(vec![canonical_request(), canonical_delivery(1, "timer")])), + Err(Error::NonDeterministicReplay(ReplayFailure { reason, .. })) if reason == "cooperative_cancellation_call_mismatch") + ); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |_ctx, _input| async { Ok(Value::Null) }); + assert!( + matches!(worker.execute_workflow_task(cancellation_task(vec![canonical_request(), canonical_delivery(1, "timer")])), + Err(Error::NonDeterministicReplay(ReplayFailure { reason, .. })) if reason == "recorded_commands_unconsumed") + ); +} + +#[test] +fn cooperative_replay_rejects_a_marker_that_skips_unrecorded_authored_calls() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + ctx.sleep(Duration::from_secs(5)).await?; + Ok(Value::Null) + }); + assert!( + matches!(worker.execute_workflow_task(cancellation_task(vec![canonical_request(), canonical_delivery(999, "timer")])), + Err(Error::NonDeterministicReplay(ReplayFailure { reason, .. })) if reason == "cooperative_cancellation_call_mismatch") + ); +} + +#[test] +fn cooperative_replay_shields_nested_cleanup_and_preserves_original_request_after_drop() { + let ctx = workflow_context(vec![canonical_request(), canonical_delivery(1, "timer")]); + let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5))); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + assert!(matches!( + timer.as_mut().poll(&mut cx), + Poll::Ready(Err(Error::CooperativeCancellationRequested(_))) + )); + assert!(ctx.is_cancellation_requested().unwrap()); + let outer = ctx.cancellation_shield().unwrap(); + let inner = ctx.cancellation_shield().unwrap(); + ctx.throw_if_cancellation_requested().unwrap(); + drop(inner); + ctx.throw_if_cancellation_requested().unwrap(); + drop(outer); + let Error::CooperativeCancellationRequested(cancellation) = + ctx.throw_if_cancellation_requested().unwrap_err() + else { + panic!("original cancellation missing") + }; + assert_eq!(cancellation.request.request_id, "original-request"); + assert_eq!( + cancellation.request.cleanup_deadline_at, + "2026-10-01T08:10:00Z" + ); +} + +#[test] +fn cooperative_replay_cannot_shield_away_a_committed_authored_delivery() { + let ctx = workflow_context(vec![canonical_request(), canonical_delivery(1, "timer")]); + let _shield = ctx.cancellation_shield().unwrap(); + let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5))); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + assert!(matches!( + timer.as_mut().poll(&mut cx), + Poll::Ready(Err(Error::NonDeterministicReplay(_))) + )); +} + #[test] fn cooperative_history_preserves_observation_without_authorizing_delivery() { let observation = CancellationRequest::from_observation(&request_observation()).unwrap(); diff --git a/tests/fixtures/replay-regressions/cooperative-cold-activity-delivery.json b/tests/fixtures/replay-regressions/cooperative-cold-activity-delivery.json new file mode 100644 index 0000000..ab2c3d3 --- /dev/null +++ b/tests/fixtures/replay-regressions/cooperative-cold-activity-delivery.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-cooperative-cold-activity-delivery", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": {"type":"corpus.redrive-boundary", "input":[], "payload_codec":"avro"}, + "history": [ + { + "event_type":"CooperativeCancellationRequested", + "workflow_command_id":"original-request", + "recorded_at":"2026-10-01T08:00:00Z", + "payload": { + "workflow_command_id":"original-request", "workflow_run_id":"regression-corpus-run", + "cleanup_deadline_at":"2026-10-01T08:10:00Z" + } + }, + { + "event_type":"CooperativeCancellationDelivered", + "workflow_command_id":"original-request", + "payload": { + "workflow_command_id":"original-request", "workflow_run_id":"regression-corpus-run", + "sequence":1, "call_kind":"activity" + } + } + ], + "command_sequence": [{ + "type":"fail_workflow", "exception_type":"WorkflowCancellationRequested", "non_retryable":true, + "exception": {"properties": {"request_id":"original-request", "cleanup_deadline_at":"2026-10-01T08:10:00Z"}} + }], + "expected": {"type":"fail_workflow", "exception_type":"WorkflowCancellationRequested", "non_retryable":true, + "exception": {"properties": {"request_id":"original-request", "cleanup_deadline_at":"2026-10-01T08:10:00Z"}}} +} From 9ec19bcba837d1e96a7a34fc683a1808bf1d1197 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 09:11:33 +0000 Subject: [PATCH 05/58] Replay cancellation at the authored selection handle --- src/cooperative_cancellation.rs | 41 +++- src/lib.rs | 63 +++++- src/tests/cooperative_cancellation.rs | 206 ++++++++++++++++++ .../cooperative-cold-selection-handle.json | 69 ++++++ 4 files changed, 376 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/replay-regressions/cooperative-cold-selection-handle.json diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index 5e1432b..defcde8 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -211,7 +211,7 @@ pub struct CancellationHistory { } impl CancellationHistory { - pub(super) fn bind_scalar_commands( + pub(super) fn bind_commands( &self, mut commands: Vec, ) -> Result> { @@ -225,6 +225,7 @@ impl CancellationHistory { | CancellationCallKind::Condition | CancellationCallKind::Signal | CancellationCallKind::Child + | CancellationCallKind::SelectionHandle ) { return Ok(commands); } @@ -548,6 +549,44 @@ impl WorkflowState { } Ok(()) } + + pub(super) fn replay_selection_handle_cancellation( + &mut self, + handle: &DurableOperationHandle, + ) -> Result<()> { + let Some(RecordedCommand::CancellationBoundary { + sequence, + call_kind, + original, + }) = self.recorded_commands.get(self.command_cursor) + else { + return Ok(()); + }; + self.validate_cancellation_call( + *sequence, + CancellationCallKind::SelectionHandle, + *call_kind, + )?; + let delivery = self + .cancellation_history + .delivery + .as_ref() + .expect("bound canonical delivery"); + if original.is_some() + || delivery.operation_sequence != Some(handle.base_sequence) + || delivery.operation_sequence_span != handle.size as u64 + { + return Err(invalid_recorded_history( + "cooperative_cancellation_call_mismatch", + *sequence, + "selection handle for the committed operation range", + &format!("{}:{}", handle.base_sequence, handle.size), + "cancellation delivery targets a different authored selection member", + )); + } + validate_selection_delivery_handle(self, handle)?; + self.replay_cancellation_at(self.command_cursor, CancellationCallKind::SelectionHandle) + } } impl WorkflowContext { diff --git a/src/lib.rs b/src/lib.rs index 43d5981..63afd29 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9416,7 +9416,7 @@ impl WorkflowState { run_id.as_deref().unwrap_or_default(), None, )?; - let recorded_commands = cancellation_history.bind_scalar_commands(recorded_commands( + let recorded_commands = cancellation_history.bind_commands(recorded_commands( &history, &payload_codec, WorkflowIdentity { @@ -11755,10 +11755,13 @@ impl Future for DurableOperationAwaitCall { type Output = Result; fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll { - let state = match self.handle.ctx.state.lock() { + let mut state = match self.handle.ctx.state.lock() { Ok(state) => state, Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)), }; + if let Err(error) = state.replay_selection_handle_cancellation(&self.handle) { + return Poll::Ready(Err(error)); + } match selection_cancellation_for_handle(&state, &self.handle) { Err(error) => return Poll::Ready(Err(error)), Ok(false) => {} @@ -11782,6 +11785,62 @@ impl Future for DurableOperationAwaitCall { } } +fn validate_selection_delivery_handle( + state: &WorkflowState, + handle: &DurableOperationHandle, +) -> Result<()> { + let path = state + .recorded_commands + .iter() + .find(|command| command.sequence() == handle.base_sequence) + .and_then(|command| match command { + RecordedCommand::Activity { + parallel_group_path, + .. + } + | RecordedCommand::ChildWorkflow { + parallel_group_path, + .. + } + | RecordedCommand::Timer { + parallel_group_path, + .. + } + | RecordedCommand::SignalWait { + parallel_group_path, + .. + } + | RecordedCommand::ConditionWait { + parallel_group_path, + .. + } => parallel_group_path.as_ref(), + _ => None, + }) + .and_then(|path| path.first()); + let matches = path.is_some_and(|entry| { + entry.parallel_group_mode.as_deref() == Some("select") + && entry.parallel_group_id == handle.selection_group_id + && entry.selection_member_key.as_ref() == Some(&handle.key) + && entry.selection_member_index == Some(handle.index) + && entry.selection_member_base_sequence == Some(handle.base_sequence) + && entry.selection_member_size == Some(handle.size) + && entry.selection_member_kind.as_deref() == Some(handle.kind.as_str()) + }); + if !matches + || selection_operation_identity(state, &handle.kind, handle.base_sequence, handle.size) + != handle.identity + { + return Err(invalid_recorded_history( + "cooperative_cancellation_call_mismatch", + handle.base_sequence, + "selection handle matching its authored durable identity", + &format!("{handle:?}"), + "selection handle identity or member metadata changed before cancellation replay", + )); + } + Ok(()) +} + /// Future returned by [`DurableOperationHandle::cancel`]. pub struct CancelDurableOperationCall { handle: DurableOperationHandle, diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs index c25eff6..3b05e1a 100644 --- a/src/tests/cooperative_cancellation.rs +++ b/src/tests/cooperative_cancellation.rs @@ -71,6 +71,212 @@ fn completed_activity(sequence: u64, name: &str, value: Value) -> Vec Vec { + let mut delivery = canonical_delivery(3, "selection_handle"); + delivery.payload["operation_sequence"] = json!(1); + vec![ + selection_activity_event("ActivityScheduled", 0, "slow", None), + selection_activity_event("ActivityScheduled", 1, "fast", None), + selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner-value"))), + selection_winner_marker(), + canonical_request(), + delivery, + ] +} + +#[test] +fn cooperative_selection_handle_replays_original_request_after_committed_winner() { + for _cold_restart in 0..2 { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + let selected = keyed_activity_selection(&ctx).await?; + assert_eq!(selected.key, SelectionKey::Name("fast".into())); + assert_eq!( + selected.value, + Some(ParallelResult::Activity(json!("winner-value"))) + ); + assert!(!ctx.is_cancellation_requested()?); + selected + .handle(&SelectionKey::Name("slow".into())) + .unwrap() + .await_result() + .await?; + Ok(Value::Null) + }); + let commands = worker + .execute_workflow_task(cancellation_task(cancelled_selection_history())) + .unwrap(); + assert_eq!(commands.len(), 1); + assert_eq!( + commands[0]["exception_type"], + "WorkflowCancellationRequested" + ); + assert_eq!( + commands[0]["exception"]["properties"]["request_id"], + "original-request" + ); + assert_eq!( + commands[0]["exception"]["properties"]["cleanup_deadline_at"], + "2026-10-01T08:10:00Z" + ); + } +} + +#[test] +fn cooperative_selection_handle_delivery_owns_a_late_loser_completion() { + for before_delivery in [true, false] { + let mut events = cancelled_selection_history(); + let late = + selection_activity_event("ActivityCompleted", 0, "slow", Some(json!("too-late"))); + if before_delivery { + events.insert(events.len() - 1, late); + } else { + events.push(late); + } + let ctx = workflow_context(events); + let mut select = Box::pin(keyed_activity_selection(&ctx)); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + let Poll::Ready(Ok(selected)) = select.as_mut().poll(&mut cx) else { + panic!("committed winner must retain priority") + }; + assert_eq!( + selected.value, + Some(ParallelResult::Activity(json!("winner-value"))) + ); + let mut loser = Box::pin( + selected + .handle(&SelectionKey::Name("slow".into())) + .unwrap() + .await_result(), + ); + let Poll::Ready(Err(Error::CooperativeCancellationRequested(cancellation))) = + loser.as_mut().poll(&mut cx) + else { + panic!("late completion must not replace the committed cancellation") + }; + assert_eq!(cancellation.delivery.sequence, 3); + assert_eq!(cancellation.delivery.operation_sequence, Some(1)); + ctx.ensure_history_consumed().unwrap(); + assert!(ctx.take_commands().unwrap().is_empty()); + } +} + +#[test] +fn cooperative_selection_handle_rejects_changed_public_identity_and_member_range() { + for changed in 0..7 { + let ctx = workflow_context(cancelled_selection_history()); + let mut select = Box::pin(keyed_activity_selection(&ctx)); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + let Poll::Ready(Ok(selected)) = select.as_mut().poll(&mut cx) else { + panic!("winner") + }; + let mut handle = selected + .handle(&SelectionKey::Name("slow".into())) + .unwrap() + .clone(); + match changed { + 0 => handle.key = SelectionKey::Name("changed".into()), + 1 => handle.index = 9, + 2 => handle.kind = "timer".into(), + 3 => handle.identity = "changed".into(), + 4 => handle.selection_group_id = "changed".into(), + 5 => handle.base_sequence = 2, + 6 => handle.size = 2, + _ => unreachable!(), + } + let mut wait = Box::pin(handle.await_result()); + assert!( + matches!(wait.as_mut().poll(&mut cx), Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure { reason, .. }))) if reason == "cooperative_cancellation_call_mismatch") + ); + assert!(!ctx.is_cancellation_requested().unwrap()); + assert!(ctx.take_commands().unwrap().is_empty()); + } +} + +#[test] +fn cooperative_selection_handle_request_only_keeps_the_loser_pending() { + let mut events = cancelled_selection_history(); + events.pop(); + let ctx = workflow_context(events); + let mut select = Box::pin(keyed_activity_selection(&ctx)); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + let Poll::Ready(Ok(selected)) = select.as_mut().poll(&mut cx) else { + panic!("winner") + }; + let mut wait = Box::pin( + selected + .handle(&SelectionKey::Name("slow".into())) + .unwrap() + .await_result(), + ); + assert!(matches!(wait.as_mut().poll(&mut cx), Poll::Pending)); + assert!(!ctx.is_cancellation_requested().unwrap()); + assert!(ctx.take_commands().unwrap().is_empty()); +} + +#[test] +fn cooperative_selection_handle_cannot_be_skipped_or_replaced_by_a_scalar_call() { + for skip in [true, false] { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _input| async move { + let selected = keyed_activity_selection(&ctx).await?; + if !skip { + ctx.sleep(Duration::from_secs(5)).await?; + } + selected.into_result()?; + Ok(Value::Null) + }); + assert!(matches!( + worker.execute_workflow_task(cancellation_task(cancelled_selection_history())), + Err(Error::NonDeterministicReplay(_)) + )); + } +} + +#[test] +fn cooperative_selection_handle_cannot_hide_delivery_inside_a_cleanup_shield() { + let ctx = workflow_context(cancelled_selection_history()); + let mut select = Box::pin(keyed_activity_selection(&ctx)); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + let Poll::Ready(Ok(selected)) = select.as_mut().poll(&mut cx) else { + panic!("winner") + }; + let _shield = ctx.cancellation_shield().unwrap(); + let mut wait = Box::pin( + selected + .handle(&SelectionKey::Name("slow".into())) + .unwrap() + .await_result(), + ); + assert!(matches!( + wait.as_mut().poll(&mut cx), + Poll::Ready(Err(Error::NonDeterministicReplay(_))) + )); +} + +#[test] +fn cooperative_selection_handle_runs_saga_cleanup_as_the_next_durable_command() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + let selected = keyed_activity_selection(&ctx).await?; + let mut saga = ctx.saga(); + saga.add_compensation("undo", json!([]))?; + let result = selected + .handle(&SelectionKey::Name("slow".into())) + .unwrap() + .await_result() + .await; + saga.finish(result).await?; + Ok(Value::Null) + }); + let commands = worker + .execute_workflow_task(cancellation_task(cancelled_selection_history())) + .unwrap(); + assert_eq!(commands.len(), 1); + assert_eq!(commands[0]["type"], "schedule_activity"); + assert_eq!(commands[0]["activity_type"], "undo"); +} + #[test] fn cooperative_replay_preserves_completed_forward_result_and_saga_cleanup_identity() { for _cold_restart in 0..2 { diff --git a/tests/fixtures/replay-regressions/cooperative-cold-selection-handle.json b/tests/fixtures/replay-regressions/cooperative-cold-selection-handle.json new file mode 100644 index 0000000..0f970a7 --- /dev/null +++ b/tests/fixtures/replay-regressions/cooperative-cold-selection-handle.json @@ -0,0 +1,69 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-cooperative-cold-selection-handle", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": {"type":"corpus.durable-selection", "input":[], "payload_codec":"avro"}, + "history": [ + { + "id":"event-slow-open", "event_type":"ActivityScheduled", "payload": { + "sequence":1, "activity_execution_id":"activity-slow", "activity_type":"slow-activity", + "parallel_group_id":"select-calls:1:2", "parallel_group_kind":"activity", "parallel_group_mode":"select", + "parallel_group_base_sequence":1, "parallel_group_size":2, "parallel_group_index":0, + "selection_member_key":"slow", "selection_member_index":0, "selection_member_base_sequence":1, + "selection_member_size":1, "selection_member_kind":"activity" + } + }, + { + "id":"event-fast-open", "event_type":"ActivityScheduled", "payload": { + "sequence":2, "activity_execution_id":"activity-fast", "activity_type":"fast-activity", + "parallel_group_id":"select-calls:1:2", "parallel_group_kind":"activity", "parallel_group_mode":"select", + "parallel_group_base_sequence":1, "parallel_group_size":2, "parallel_group_index":1, + "selection_member_key":"fast", "selection_member_index":1, "selection_member_base_sequence":2, + "selection_member_size":1, "selection_member_kind":"activity" + } + }, + { + "id":"event-fast-completed", "event_type":"ActivityCompleted", "payload": { + "sequence":2, "activity_execution_id":"activity-fast", "activity_type":"fast-activity", + "result":"wwHioz3/VYAiNwoYd2lubmVyLXZhbHVl", "payload_codec":"avro" + } + }, + { + "id":"event-selection", "event_type":"SelectionResolved", "payload": { + "selection_group_id":"select-calls:1:2", "selection_group_base_sequence":1, "selection_group_size":2, + "member_key":"fast", "member_index":1, "member_base_sequence":2, "member_size":1, + "operation_kind":"activity", "operation_identity":"activity-fast", "outcome":"completed", + "resolution_event_id":"event-fast-completed", "resolution_event_type":"ActivityCompleted" + } + }, + { + "event_type":"CooperativeCancellationRequested", "workflow_command_id":"original-request", + "recorded_at":"2026-10-01T08:00:00Z", "payload": { + "workflow_command_id":"original-request", "workflow_run_id":"regression-corpus-run", + "cleanup_deadline_at":"2026-10-01T08:10:00Z" + } + }, + { + "event_type":"CooperativeCancellationDelivered", "workflow_command_id":"original-request", "payload": { + "workflow_command_id":"original-request", "workflow_run_id":"regression-corpus-run", + "sequence":3, "call_kind":"selection_handle", "operation_sequence":1 + } + }, + { + "id":"event-slow-late", "event_type":"ActivityCompleted", "payload": { + "sequence":1, "activity_execution_id":"activity-slow", "activity_type":"slow-activity", + "result":"wwHioz3/VYAiNwoWbG9zZXItdmFsdWU=", "payload_codec":"avro" + } + } + ], + "command_sequence": [{ + "type":"fail_workflow", "exception_type":"WorkflowCancellationRequested", "non_retryable":true, + "exception": {"properties": {"request_id":"original-request", "cleanup_deadline_at":"2026-10-01T08:10:00Z"}} + }], + "expected": { + "type":"fail_workflow", "exception_type":"WorkflowCancellationRequested", "non_retryable":true, + "exception": {"properties": {"request_id":"original-request", "cleanup_deadline_at":"2026-10-01T08:10:00Z"}} + } +} From 0c9e59579a1e5c3a00fcb428b5ca3cd07320634e Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 09:17:37 +0000 Subject: [PATCH 06/58] Replay cancellation across authored parallel and selection groups --- src/cooperative_cancellation.rs | 101 +++++++ src/lib.rs | 55 +++- src/tests/cooperative_cancellation.rs | 286 ++++++++++++++++++ .../cooperative-cold-parallel-group.json | 31 ++ 4 files changed, 468 insertions(+), 5 deletions(-) create mode 100644 tests/fixtures/replay-regressions/cooperative-cold-parallel-group.json diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index defcde8..3f2f950 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -218,6 +218,36 @@ impl CancellationHistory { let Some(delivery) = &self.delivery else { return Ok(commands); }; + if delivery.call_kind == CancellationCallKind::Parallel { + let index = commands.partition_point(|command| command.sequence() < delivery.sequence); + let end = commands.partition_point(|command| { + command.sequence() < delivery.sequence + delivery.sequence_span + }); + if commands.get(index).map(RecordedCommand::sequence) != Some(delivery.sequence) { + let previous = index + .checked_sub(1) + .map_or(0, |index| commands[index].sequence()); + if previous.checked_add(1) != Some(delivery.sequence) { + return Err(invalid_recorded_history( + "cooperative_cancellation_call_mismatch", + delivery.sequence, + "next authored parallel group", + "missing earlier call", + "parallel cancellation marker skips an unrecorded authored command", + )); + } + } + let original = commands.drain(index..end).collect(); + commands.insert( + index, + RecordedCommand::CancellationGroup { + sequence: delivery.sequence, + span: delivery.sequence_span, + original, + }, + ); + return Ok(commands); + } if !matches!( delivery.call_kind, CancellationCallKind::Activity @@ -473,6 +503,77 @@ impl CancellationHistory { } impl WorkflowState { + pub(super) fn expand_cancellation_group( + &mut self, + descriptors: &[ParallelDescriptor], + ) -> Result<()> { + let Some(RecordedCommand::CancellationGroup { + sequence, + span, + original, + }) = self.recorded_commands.get(self.command_cursor).cloned() + else { + return Ok(()); + }; + self.validate_cancellation_call( + sequence, + CancellationCallKind::Parallel, + CancellationCallKind::Parallel, + )?; + if descriptors.len() as u64 != span { + return Err(invalid_recorded_history( + "cooperative_cancellation_call_mismatch", + sequence, + &format!("parallel group with {span} durable leaves"), + &format!("{} durable leaves", descriptors.len()), + "authored parallel group span differs from its committed cancellation", + )); + } + let mut original = original.into_iter().peekable(); + let mut leaves = Vec::with_capacity(descriptors.len()); + for descriptor in descriptors { + let leaf_sequence = sequence + descriptor.offset as u64; + let command = if original.peek().map(RecordedCommand::sequence) == Some(leaf_sequence) { + original.next() + } else { + None + }; + if self + .cancellation_history + .resolved_before_request + .contains(&leaf_sequence) + { + let command = command.ok_or_else(|| { + invalid_recorded_history( + "cooperative_cancellation_call_mismatch", + leaf_sequence, + "recorded earlier result", + "missing durable call", + "parallel cancellation cannot discard an earlier committed result", + ) + })?; + leaves.push(command); + continue; + } + let kind = match &descriptor.operation { + ParallelOperation::Activity { .. } => CancellationCallKind::Activity, + ParallelOperation::ChildWorkflow { .. } => CancellationCallKind::Child, + ParallelOperation::Timer(_) => CancellationCallKind::Timer, + ParallelOperation::Signal(_) => CancellationCallKind::Signal, + ParallelOperation::Condition { .. } => CancellationCallKind::Condition, + ParallelOperation::Group(_) => unreachable!("descriptor is a durable leaf"), + }; + leaves.push(RecordedCommand::CancellationBoundary { + sequence: leaf_sequence, + call_kind: kind, + original: command.map(Box::new), + }); + } + self.recorded_commands + .splice(self.command_cursor..=self.command_cursor, leaves); + Ok(()) + } + pub(super) fn cancellation_error(&self) -> Error { match ( &self.cancellation_history.request, diff --git a/src/lib.rs b/src/lib.rs index 63afd29..c79f52d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9601,6 +9601,11 @@ fn decode_message_stream_delivery(arguments: Vec) -> Result, + }, CancellationBoundary { sequence: u64, call_kind: CancellationCallKind, @@ -10119,7 +10124,8 @@ fn activity_options_description(options: &RecordedActivityOptions) -> String { impl RecordedCommand { fn sequence(&self) -> u64 { match self { - Self::CancellationBoundary { sequence, .. } + Self::CancellationGroup { sequence, .. } + | Self::CancellationBoundary { sequence, .. } | Self::Activity { sequence, .. } | Self::Timer { sequence, .. } | Self::ChildWorkflow { sequence, .. } @@ -10134,7 +10140,9 @@ impl RecordedCommand { fn shape(&self) -> &'static str { match self { - Self::CancellationBoundary { .. } => "cooperative cancellation", + Self::CancellationBoundary { .. } | Self::CancellationGroup { .. } => { + "cooperative cancellation" + } Self::Activity { .. } => "activity", Self::Timer { .. } => "timer", Self::ChildWorkflow { .. } => "child workflow", @@ -10687,7 +10695,13 @@ impl ParallelCall { } }; - self.leaves = parallel_descriptors(operations, base_sequence)? + let descriptors = parallel_descriptors(operations, base_sequence)?; + self.ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)? + .expand_cancellation_group(&descriptors)?; + self.leaves = descriptors .into_iter() .map(|descriptor| { let call = parallel_leaf_call( @@ -10739,6 +10753,12 @@ impl ParallelCall { { return Poll::Ready(Err(failures.remove(position).1)); } + if let Some(position) = failures + .iter() + .position(|(_, error)| matches!(error, Error::CooperativeCancellationRequested(_))) + { + return Poll::Ready(Err(failures.remove(position).1)); + } failures.sort_by_key(|(index, _)| *index); let (failed_index, cause) = failures.remove(0); let failed = &self.leaves[failed_index]; @@ -11127,7 +11147,12 @@ impl SelectCall { .state .lock() .map_err(|_| Error::WorkflowStatePoisoned)?; - if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) { + if let Some(RecordedCommand::CancellationGroup { sequence, .. }) = + state.recorded_commands.get(state.command_cursor) + { + *sequence + } else if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) + { marker.selection_group_base_sequence } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) { recorded.sequence() @@ -11144,6 +11169,11 @@ impl SelectCall { }; let (descriptors, members) = selection_descriptors(operations, base_sequence)?; let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len()); + self.ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)? + .expand_cancellation_group(&descriptors)?; self.leaves = descriptors .into_iter() .map(|descriptor| SelectionLeaf { @@ -11183,6 +11213,20 @@ impl Future for SelectCall { } } + if let Some(leaf) = self.leaves.iter_mut().find(|leaf| { + matches!( + leaf.outcome, + Some(Err(Error::CooperativeCancellationRequested(_))) + ) + }) { + return Poll::Ready( + leaf.outcome + .take() + .expect("matched cancellation") + .map(|_| unreachable!()), + ); + } + let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some()); let selection_member_range = self .members @@ -11666,7 +11710,8 @@ fn recorded_selection_member_is_terminal( | RecordedCommand::SideEffect { .. } | RecordedCommand::VersionMarker { .. } | RecordedCommand::Memo { .. } - | RecordedCommand::CancellationBoundary { .. } => false, + | RecordedCommand::CancellationBoundary { .. } + | RecordedCommand::CancellationGroup { .. } => false, }; if !terminal { all_completed = false; diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs index 3b05e1a..d0870ba 100644 --- a/src/tests/cooperative_cancellation.rs +++ b/src/tests/cooperative_cancellation.rs @@ -84,6 +84,292 @@ fn cancelled_selection_history() -> Vec { ] } +fn cancelled_parallel_history() -> Vec { + let paths = nested_parallel_paths(); + let mut delivery = canonical_delivery(1, "parallel"); + delivery.payload["sequence_span"] = json!(3); + vec![ + parallel_history_event( + "ActivityScheduled", + 1, + "activity_type", + "first", + paths[0].clone(), + None, + ), + parallel_history_event( + "ChildWorkflowScheduled", + 2, + "workflow_type", + "second", + paths[1].clone(), + None, + ), + parallel_history_event( + "ActivityScheduled", + 3, + "activity_type", + "third", + paths[2].clone(), + None, + ), + canonical_request(), + delivery, + ] +} + +#[test] +fn cooperative_parallel_replays_one_original_cancellation_for_nested_mixed_leaves() { + for _cold_restart in 0..2 { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + ctx.parallel(nested_parallel_operations()).await?; + Ok(Value::Null) + }); + let commands = worker + .execute_workflow_task(cancellation_task(cancelled_parallel_history())) + .unwrap(); + assert_eq!(commands.len(), 1); + assert_eq!( + commands[0]["exception_type"], + "WorkflowCancellationRequested" + ); + assert_eq!( + commands[0]["exception"]["properties"]["request_id"], + "original-request" + ); + assert_eq!(commands[0]["non_retryable"], true); + } +} + +#[test] +fn cooperative_parallel_preserves_prior_result_and_overrides_a_late_leaf_failure() { + let mut events = cancelled_parallel_history(); + events.insert( + 3, + parallel_history_event( + "ActivityCompleted", + 1, + "activity_type", + "first", + nested_parallel_paths()[0].clone(), + Some(json!(7)), + ), + ); + events.push(event("ActivityFailed", json!({"sequence":3, "activity_type":"third", "exception_type":"LateFailure", "exception":{"class":"LateFailure", "message":"too late"}}))); + let ctx = workflow_context(events); + let mut call = Box::pin(ctx.parallel(nested_parallel_operations())); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + let Poll::Ready(Err(Error::CooperativeCancellationRequested(cancellation))) = + call.as_mut().poll(&mut cx) + else { + panic!("group cancellation must retain priority") + }; + assert_eq!(cancellation.delivery.sequence_span, 3); + assert_eq!(ctx.state.lock().unwrap().command_cursor, 3); + ctx.ensure_history_consumed().unwrap(); + assert!(ctx.take_commands().unwrap().is_empty()); +} + +#[test] +fn cooperative_parallel_does_not_hide_changed_later_leaf_details() { + let mut events = cancelled_parallel_history(); + events[2].payload["activity_type"] = json!("changed-third"); + let ctx = workflow_context(events); + let mut call = Box::pin(ctx.parallel(nested_parallel_operations())); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + assert!( + matches!(call.as_mut().poll(&mut cx), Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure { reason, .. }))) if reason == "recorded_command_detail_mismatch") + ); + assert!(ctx.take_commands().unwrap().is_empty()); +} + +#[test] +fn cooperative_parallel_rejects_changed_span_nesting_scalar_and_cleanup_scope() { + for change in 0..4 { + let ctx = workflow_context(cancelled_parallel_history()); + let _shield = if change == 3 { + Some(ctx.cancellation_shield().unwrap()) + } else { + None + }; + let operations = match change { + 0 => vec![ParallelOperation::activity("first", json!([]))], + 1 => vec![ + ParallelOperation::activity("first", json!([])), + ParallelOperation::child_workflow( + "second", + ChildWorkflowOptions::new("child-workers"), + json!([]), + ), + ParallelOperation::activity("third", json!([])), + ], + _ => nested_parallel_operations(), + }; + let mut cx = TaskContext::from_waker(noop_waker_ref()); + let outcome = if change == 2 { + Box::pin(ctx.activity("first", json!([]))) + .as_mut() + .poll(&mut cx) + .map_ok(|_| ()) + } else { + Box::pin(ctx.parallel(operations)) + .as_mut() + .poll(&mut cx) + .map_ok(|_| ()) + }; + assert!(matches!( + outcome, + Poll::Ready(Err(Error::NonDeterministicReplay(_))) + )); + assert!(ctx.take_commands().unwrap().is_empty()); + } +} + +#[test] +fn cooperative_parallel_cold_delivery_covers_all_scalar_leaf_kinds_without_scheduling() { + let mut delivery = canonical_delivery(1, "parallel"); + delivery.payload["sequence_span"] = json!(5); + let ctx = workflow_context(vec![canonical_request(), delivery]); + let mut call = Box::pin(ctx.parallel(vec![ + ParallelOperation::activity("work", json!([])), + ParallelOperation::child_workflow( + "child", + ChildWorkflowOptions::new("child-workers"), + json!([]), + ), + ParallelOperation::timer(Duration::from_secs(5)), + ParallelOperation::signal("ready"), + ParallelOperation::condition(ConditionWaitOptions::new("ready", "sha256:ready"), || { + Ok(false) + }), + ])); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + assert!(matches!( + call.as_mut().poll(&mut cx), + Poll::Ready(Err(Error::CooperativeCancellationRequested(_))) + )); + assert_eq!(ctx.state.lock().unwrap().command_cursor, 5); + ctx.ensure_history_consumed().unwrap(); + assert!(ctx.take_commands().unwrap().is_empty()); +} + +#[test] +fn cooperative_parallel_validates_recorded_timer_signal_and_condition_leaves() { + for changed in 0..3 { + let names = [ + ("ActivityScheduled", "activity_type", "work"), + ("ChildWorkflowScheduled", "workflow_type", "child"), + ("TimerScheduled", "timer_id", "timer"), + ("SignalWaitOpened", "signal_name", "ready"), + ("ConditionWaitOpened", "condition_wait_id", "condition"), + ]; + let mut events = names + .into_iter() + .enumerate() + .map(|(index, (event_type, field, name))| { + parallel_history_event( + event_type, + index as u64 + 1, + field, + name, + vec![parallel_group_entry(1, 5, index, "mixed")], + None, + ) + }) + .collect::>(); + events[2].payload["delay_seconds"] = json!(if changed == 1 { 500 } else { 5 }); + events[4].payload["condition_wait_occurrence_id"] = json!("rust:condition-wait:0"); + events[4].payload["condition_key"] = json!("ready"); + events[4].payload["condition_definition_fingerprint"] = json!(if changed == 2 { + "changed" + } else { + "sha256:ready" + }); + let mut delivery = canonical_delivery(1, "parallel"); + delivery.payload["sequence_span"] = json!(5); + events.extend([canonical_request(), delivery]); + let ctx = workflow_context(events); + let mut call = Box::pin(ctx.parallel(vec![ + ParallelOperation::activity("work", json!([])), + ParallelOperation::child_workflow( + "child", + ChildWorkflowOptions::new("child-workers"), + json!([]), + ), + ParallelOperation::timer(Duration::from_secs(5)), + ParallelOperation::signal("ready"), + ParallelOperation::condition( + ConditionWaitOptions::new("ready", "sha256:ready"), + || panic!("delivered condition must not reevaluate its predicate"), + ), + ])); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + match call.as_mut().poll(&mut cx) { + Poll::Ready(Err(Error::CooperativeCancellationRequested(_))) if changed == 0 => { + ctx.ensure_history_consumed().unwrap(); + } + Poll::Ready(Err(Error::NonDeterministicReplay(failure))) if changed == 1 => { + assert_eq!(failure.reason, "timer_delay_mismatch"); + } + Poll::Ready(Err(Error::NonDeterministicReplay(failure))) if changed == 2 => { + assert_eq!(failure.reason, "condition_wait_predicate_mismatch"); + } + other => panic!("{changed}: {other:?}"), + } + assert!(ctx.take_commands().unwrap().is_empty()); + } +} + +#[test] +fn cooperative_selection_group_delivers_before_an_uncommitted_winner() { + let mut delivery = canonical_delivery(1, "parallel"); + delivery.payload["sequence_span"] = json!(2); + let events = vec![ + selection_activity_event("ActivityScheduled", 0, "slow", None), + selection_activity_event("ActivityScheduled", 1, "fast", None), + canonical_request(), + delivery, + selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("too-late"))), + selection_winner_marker(), + ]; + let ctx = workflow_context(events); + let mut select = Box::pin(keyed_activity_selection(&ctx)); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + assert!(matches!( + select.as_mut().poll(&mut cx), + Poll::Ready(Err(Error::CooperativeCancellationRequested(_))) + )); + ctx.ensure_history_consumed().unwrap(); + assert!(ctx.take_commands().unwrap().is_empty()); +} + +#[test] +fn cooperative_parallel_saga_cleanup_uses_the_slot_after_the_entire_group() { + let mut events = cancelled_parallel_history(); + events.extend(completed_activity(4, "undo", Value::Null)); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + let mut saga = ctx.saga(); + saga.add_compensation("undo", json!([]))?; + saga.finish(ctx.parallel(nested_parallel_operations()).await) + .await?; + Ok(Value::Null) + }); + let commands = worker + .execute_workflow_task(cancellation_task(events)) + .unwrap(); + assert_eq!(commands.len(), 1); + assert_eq!( + commands[0]["exception_type"], + "WorkflowCancellationRequested" + ); + assert_eq!( + commands[0]["exception"]["properties"]["request_id"], + "original-request" + ); +} + #[test] fn cooperative_selection_handle_replays_original_request_after_committed_winner() { for _cold_restart in 0..2 { diff --git a/tests/fixtures/replay-regressions/cooperative-cold-parallel-group.json b/tests/fixtures/replay-regressions/cooperative-cold-parallel-group.json new file mode 100644 index 0000000..ac673ab --- /dev/null +++ b/tests/fixtures/replay-regressions/cooperative-cold-parallel-group.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-cooperative-cold-parallel-group", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": {"type":"corpus.nested-parallel", "input":[], "payload_codec":"avro"}, + "history": [ + { + "event_type":"CooperativeCancellationRequested", "workflow_command_id":"original-request", + "recorded_at":"2026-10-01T08:00:00Z", "payload": { + "workflow_command_id":"original-request", "workflow_run_id":"regression-corpus-run", + "cleanup_deadline_at":"2026-10-01T08:10:00Z" + } + }, + { + "event_type":"CooperativeCancellationDelivered", "workflow_command_id":"original-request", "payload": { + "workflow_command_id":"original-request", "workflow_run_id":"regression-corpus-run", + "sequence":1, "call_kind":"parallel", "sequence_span":3 + } + } + ], + "command_sequence": [{ + "type":"fail_workflow", "exception_type":"WorkflowCancellationRequested", "non_retryable":true, + "exception": {"properties": {"request_id":"original-request", "cleanup_deadline_at":"2026-10-01T08:10:00Z"}} + }], + "expected": { + "type":"fail_workflow", "exception_type":"WorkflowCancellationRequested", "non_retryable":true, + "exception": {"properties": {"request_id":"original-request", "cleanup_deadline_at":"2026-10-01T08:10:00Z"}} + } +} From 5b5f082b322cc1dbbbc076eec31dfedc23afacd3 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 09:35:35 +0000 Subject: [PATCH 07/58] Add fenced cancellation delivery and bounded history refresh --- src/cooperative_cancellation.rs | 168 ++++++++++++ src/lib.rs | 6 +- src/tests/cooperative_cancellation.rs | 360 ++++++++++++++++++++++++++ 3 files changed, 531 insertions(+), 3 deletions(-) diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index 3f2f950..977d7eb 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -2,6 +2,15 @@ use super::*; use std::collections::BTreeSet; const CONTROL_BUDGET: Duration = Duration::from_secs(5); +const MAX_REFRESH_PAGES: usize = 128; + +/// A delivery acknowledgment bound to one workflow task and durable run. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CancellationDeliveryReceipt { + pub task_id: String, + pub run_id: String, + pub delivery: CancellationDelivery, +} /// Optional reason and runtime-owned cleanup limit for a cooperative request. #[derive(Clone, Debug, Default, Serialize)] @@ -759,6 +768,145 @@ fn acknowledgment( } impl Client { + /// Commit cooperative delivery on the exact selected workflow-task claim. + /// + /// This explicit worker-protocol 1.20 operation renews no lease and advertises + /// no worker capability. The Server requires a previously admitted capable + /// claim. Reload canonical history before exposing cancellation to workflow + /// code, including when an acknowledgment is lost. + pub async fn deliver_workflow_cancellation( + &self, + task: &WorkflowTask, + delivery: &CancellationDelivery, + ) -> Result { + let (owner, run_id) = cancellation_claim(task)?; + let body = json!({ + "lease_owner":owner, "workflow_task_attempt":task.workflow_task_attempt, + "request_id":delivery.request_id, "sequence":delivery.sequence, + "call_kind":delivery.call_kind, "sequence_span":delivery.sequence_span, + "operation_sequence":delivery.operation_sequence, + "operation_sequence_span":delivery.operation_sequence_span + }); + let mut payload = body.clone(); + payload["workflow_command_id"] = json!(delivery.request_id); + if CancellationDelivery::from_payload(&payload)? != *delivery { + return Err(invalid( + "delivery must name a valid authored operation range", + )); + } + tokio::time::timeout(CONTROL_BUDGET, async { + let path = format!( + "/worker/workflow-tasks/{}/deliver-cancellation", + percent_encode_path_segment(&task.task_id) + ); + let response: Value = self + .request_json( + reqwest::Method::POST, + &path, + RequestProtocol::Worker("1.20"), + Some(&body), + ) + .await?; + if response["delivered"].as_bool() != Some(true) + || response["task_id"].as_str() != Some(task.task_id.as_str()) + || response["workflow_run_id"].as_str() != Some(run_id) + || response.get("reason") != Some(&Value::Null) + || [ + "sequence_span", + "operation_sequence", + "operation_sequence_span", + ] + .iter() + .any(|field| response.get(*field).is_none()) + { + return Err(invalid( + "delivery acknowledgment does not match the selected task/run", + )); + } + let mut recorded = response.clone(); + recorded["workflow_command_id"] = response["request_id"].clone(); + if CancellationDelivery::from_payload(&recorded)? != *delivery { + return Err(invalid( + "delivery acknowledgment changed its authored operation range", + )); + } + Ok(CancellationDeliveryReceipt { + task_id: task.task_id.clone(), + run_id: run_id.to_owned(), + delivery: delivery.clone(), + }) + }) + .await + .map_err(|_| Error::Timeout)? + } + + /// Reload canonical history with the Server-issued token and exact claim. + /// + /// The refresh has one five-second budget, at most 128 pages, and the existing + /// SDK page-size limit. Invalid, repeated or non-progressing tokens and pages + /// fail closed. This returns fresh history without modifying the caller's + /// previous task snapshot or its original request identity. + pub async fn refresh_workflow_cancellation_history( + &self, + task: &WorkflowTask, + observation: &CancellationRequest, + ) -> Result> { + let (owner, run_id) = cancellation_claim(task)?; + CancellationRequest::from_observation(&json!({ + "request_id":observation.request_id, "requested_at":observation.requested_at, + "cleanup_deadline_at":observation.cleanup_deadline_at, + "history_refresh_page_token":observation.history_refresh_page_token, + }))?; + let first_token = observation + .history_refresh_page_token + .clone() + .expect("validated history token"); + tokio::time::timeout(CONTROL_BUDGET, async { + let path = format!("/worker/workflow-tasks/{}/history", percent_encode_path_segment(&task.task_id)); + let mut token = Some(first_token); + let mut seen = BTreeSet::new(); + let mut history = Vec::new(); + while let Some(current) = token.take() { + if seen.len() >= MAX_REFRESH_PAGES || !seen.insert(current.clone()) { + return Err(invalid("canonical history refresh exceeded its page bound or repeated a token")); + } + let body = json!({ + "lease_owner":owner, "workflow_task_attempt":task.workflow_task_attempt, + "next_history_page_token":current, "history_page_size":WORKFLOW_HISTORY_PAGE_SIZE, + }); + let page: Value = self.request_json( + reqwest::Method::POST, &path, RequestProtocol::Worker("1.20"), Some(&body), + ).await?; + if page["task_id"].as_str() != Some(task.task_id.as_str()) + || page["workflow_task_attempt"].as_u64() != Some(task.workflow_task_attempt) + { + return Err(invalid("canonical history page changed the selected task/attempt")); + } + let events = page["history_events"].as_array().ok_or_else(|| invalid("canonical history page must contain an event array"))?; + if events.len() > WORKFLOW_HISTORY_PAGE_SIZE as usize { + return Err(invalid("canonical history page exceeds its requested event limit")); + } + token = match page.get("next_history_page_token") { + Some(Value::Null) => None, + Some(Value::String(next)) if !next.is_empty() && !events.is_empty() => Some(next.clone()), + _ => return Err(invalid("canonical history page token or progress is invalid")), + }; + for event in events { + let event: HistoryEvent = serde_json::from_value(event.clone()).map_err(|_| invalid("canonical history page contains a malformed event"))?; + if event.event_type.trim().is_empty() { + return Err(invalid("canonical history event type must be non-empty")); + } + history.push(event); + } + } + let canonical = CancellationHistory::from_events(&history, run_id, Some(observation))?; + if canonical.request_index >= history.len() { + return Err(invalid("canonical refresh omitted the original request event")); + } + Ok(history) + }).await.map_err(|_| Error::Timeout)? + } + /// Request bounded workflow-authored cleanup on a capable runtime. /// /// This is separate from terminal cancellation. Discovery must advertise @@ -840,6 +988,26 @@ impl Client { } } +fn cancellation_claim(task: &WorkflowTask) -> Result<(&str, &str)> { + let owner = task + .lease_owner + .as_deref() + .filter(|owner| !owner.trim().is_empty()); + let run = task.run_id.as_deref().filter(|run| !run.trim().is_empty()); + if task.task_id.trim().is_empty() + || task.workflow_task_attempt == 0 + || task.workflow_task_attempt > MAX_SEQUENCE + { + return Err(invalid( + "cancellation transport requires a valid task and attempt", + )); + } + Ok(( + owner.ok_or_else(|| invalid("cancellation transport requires the actual lease owner"))?, + run.ok_or_else(|| invalid("cancellation transport requires the selected durable run"))?, + )) +} + impl WorkflowHandle { /// Request bounded cleanup for whichever run is current. pub async fn request_cancellation( diff --git a/src/lib.rs b/src/lib.rs index c79f52d..c587007 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,9 +5,9 @@ mod runtime_payloads; mod runtime_uploads; pub use cooperative_cancellation::{ - CancellationCallKind, CancellationDelivery, CancellationHistory, CancellationRequest, - CancellationShield, CooperativeCancellationOptions, CooperativeCancellationRequested, - WorkflowCancellationRequest, + CancellationCallKind, CancellationDelivery, CancellationDeliveryReceipt, CancellationHistory, + CancellationRequest, CancellationShield, CooperativeCancellationOptions, + CooperativeCancellationRequested, WorkflowCancellationRequest, }; use std::{ diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs index d0870ba..f8d5706 100644 --- a/src/tests/cooperative_cancellation.rs +++ b/src/tests/cooperative_cancellation.rs @@ -1232,6 +1232,366 @@ fn client(server: &MockWorkerServer, case: &str) -> Client { .unwrap() } +fn transport_task() -> WorkflowTask { + let mut task = cancellation_task(vec![scheduled_timer(1)]); + task.task_id = "task/selected".into(); + task.workflow_task_attempt = 7; + task.lease_owner = Some("actual-owner".into()); + task +} + +fn transport_delivery() -> CancellationDelivery { + CancellationDelivery::from_payload(&canonical_delivery(1, "timer").payload).unwrap() +} + +fn transport_responses(path: &str, body: &str, number: usize) -> Option<(&'static str, String)> { + let case = path.split('/').nth(1).unwrap_or_default(); + let body: Value = serde_json::from_str(body).unwrap(); + if case == "refused" || (case == "lost-ack" && path.ends_with("/deliver-cancellation")) { + return Some(( + "409 Conflict", + json!({"reason":"lease_owner_mismatch"}).to_string(), + )); + } + if case == "budget" { + thread::sleep(Duration::from_secs(3)); + } + let mut response = if path.ends_with("/deliver-cancellation") { + json!({"delivered":true, "task_id":"task/selected", "workflow_run_id":"run", + "request_id":body["request_id"], "sequence":body["sequence"], "call_kind":body["call_kind"], + "sequence_span":body["sequence_span"], "operation_sequence":body["operation_sequence"], + "operation_sequence_span":body["operation_sequence_span"], "reason":null}) + } else if path.ends_with("/history") { + let first = body["next_history_page_token"] == "opaque-server-token"; + let events = if first { + vec![scheduled_timer(1), canonical_request()] + } else { + vec![canonical_delivery(1, "timer")] + }; + let events = events + .into_iter() + .map(|event| { + let mut value = event.raw; + value.insert("event_type".into(), json!(event.event_type)); + value.insert("payload".into(), event.payload); + Value::Object(value.into_iter().collect()) + }) + .collect::>(); + let mut page = json!({"task_id":"task/selected", "workflow_task_attempt":7, + "history_events":events, "total_history_events":3, + "next_history_page_token":if first { json!("page-next") } else { Value::Null }}); + match case { + "cycle" if !first => page["next_history_page_token"] = json!("opaque-server-token"), + "page-bound" => { + page["history_events"] = + json!([{"event_type":"OpaqueUnrelatedEvent", "payload":{}}]); + page["next_history_page_token"] = json!(format!("next-{number}")); + } + "empty-progress" => page["history_events"] = json!([]), + "oversized-page" => { + page["history_events"] = json!(vec![ + json!({"event_type":"OpaqueUnrelatedEvent"}); + WORKFLOW_HISTORY_PAGE_SIZE as usize + 1 + ]) + } + "missing-request" => { + page["history_events"] = if first { + json!([{"event_type":"TimerScheduled", "payload":{"sequence":1,"delay_seconds":5}}]) + } else { + json!([]) + }; + } + "changed-canonical" if !first => { + page["history_events"][0]["payload"]["workflow_command_id"] = json!("changed") + } + "malformed-event" => page["history_events"] = json!([{"event_type":false}]), + "wrong-attempt" => page["workflow_task_attempt"] = json!(8), + "empty-token" => page["next_history_page_token"] = json!(""), + "missing-token" => { + page.as_object_mut() + .unwrap() + .remove("next_history_page_token"); + } + _ => {} + } + page + } else { + return None; + }; + match case { + "wrong-task" => response["task_id"] = json!("other-task"), + "wrong-run" => response["workflow_run_id"] = json!("other-run"), + "wrong-request" => response["request_id"] = json!("other-request"), + "wrong-sequence" => response["sequence"] = json!(2), + "wrong-kind" => response["call_kind"] = json!("activity"), + "wrong-span" => response["sequence_span"] = json!(2), + "false-delivered" => response["delivered"] = json!(false), + "string-delivered" => response["delivered"] = json!("true"), + "wrong-reason" => response["reason"] = json!("not_committed"), + "missing-span" => { + response.as_object_mut().unwrap().remove("sequence_span"); + } + "missing-operation" => { + response + .as_object_mut() + .unwrap() + .remove("operation_sequence"); + } + "missing-operation-span" => { + response + .as_object_mut() + .unwrap() + .remove("operation_sequence_span"); + } + _ => {} + } + Some(("200 OK", response.to_string())) +} + +fn transport_server() -> MockWorkerServer { + MockWorkerServer::start_with_behavior(MockWorkerBehavior { + request_override: Some(transport_responses), + ..Default::default() + }) +} + +#[tokio::test] +async fn cooperative_transport_delivers_the_exact_claim_with_worker_credentials() { + let server = transport_server(); + let client = client(&server, "valid"); + let task = transport_task(); + let delivery = transport_delivery(); + let receipt = client + .deliver_workflow_cancellation(&task, &delivery) + .await + .unwrap(); + assert_eq!(receipt.task_id, task.task_id); + assert_eq!(receipt.run_id, "run"); + assert_eq!(receipt.delivery, delivery); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 1); + let request = &requests[0]; + assert_eq!( + request.path, + "/valid/api/worker/workflow-tasks/task%2Fselected/deliver-cancellation" + ); + assert_eq!(request.worker_protocol.as_deref(), Some("1.20")); + assert_eq!(request.authorization.as_deref(), Some("Bearer worker-only")); + assert_eq!(request.namespace.as_deref(), Some("caller-namespace")); + let body: Value = serde_json::from_str(&request.body).unwrap(); + assert_eq!(body["lease_owner"], "actual-owner"); + assert_eq!(body["workflow_task_attempt"], 7); + assert_eq!(body["request_id"], "original-request"); +} + +#[tokio::test] +async fn cooperative_transport_rejects_malformed_or_changed_delivery_acknowledgments() { + for case in [ + "wrong-task", + "wrong-run", + "wrong-request", + "wrong-sequence", + "wrong-kind", + "wrong-span", + "false-delivered", + "string-delivered", + "wrong-reason", + "missing-span", + "missing-operation", + "missing-operation-span", + ] { + let server = transport_server(); + let result = client(&server, case) + .deliver_workflow_cancellation(&transport_task(), &transport_delivery()) + .await; + assert!( + matches!(result, Err(Error::InvalidCooperativeCancellation(_))), + "{case}: {result:?}" + ); + } +} + +#[tokio::test] +async fn cooperative_transport_refreshes_with_the_opaque_token_and_preserves_the_snapshot() { + let server = transport_server(); + let client = client(&server, "valid"); + let task = transport_task(); + let observation = CancellationRequest::from_observation(&request_observation()).unwrap(); + let original = observation.clone(); + let fresh = client + .refresh_workflow_cancellation_history(&task, &observation) + .await + .unwrap(); + assert_eq!(fresh.len(), 3); + assert_eq!(task.history_events.len(), 1); + assert_eq!(observation, original); + let canonical = CancellationHistory::from_events(&fresh, "run", Some(&observation)).unwrap(); + assert_eq!(canonical.delivery, Some(transport_delivery())); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 2); + for (index, request) in requests.iter().enumerate() { + assert_eq!(request.worker_protocol.as_deref(), Some("1.20")); + let body: Value = serde_json::from_str(&request.body).unwrap(); + assert_eq!(body["lease_owner"], "actual-owner"); + assert_eq!(body["workflow_task_attempt"], 7); + assert_eq!(body["history_page_size"], WORKFLOW_HISTORY_PAGE_SIZE); + assert_eq!( + body["next_history_page_token"], + if index == 0 { + "opaque-server-token" + } else { + "page-next" + } + ); + } +} + +#[tokio::test] +async fn cooperative_transport_rejects_changed_claims_bad_pages_and_token_cycles() { + for case in [ + "wrong-task", + "wrong-attempt", + "cycle", + "empty-progress", + "oversized-page", + "missing-request", + "changed-canonical", + "malformed-event", + "empty-token", + "missing-token", + ] { + let server = transport_server(); + let observation = CancellationRequest::from_observation(&request_observation()).unwrap(); + let result = client(&server, case) + .refresh_workflow_cancellation_history(&transport_task(), &observation) + .await; + assert!( + matches!( + result, + Err(Error::InvalidCooperativeCancellation(_) | Error::NonDeterministicReplay(_)) + ), + "{case}: {result:?}" + ); + assert!(server.requests.lock().unwrap().len() <= 2); + } +} + +#[tokio::test] +async fn cooperative_transport_bounds_unique_history_pages() { + let server = transport_server(); + let observation = CancellationRequest::from_observation(&request_observation()).unwrap(); + let result = client(&server, "page-bound") + .refresh_workflow_cancellation_history(&transport_task(), &observation) + .await; + assert!(matches!( + result, + Err(Error::InvalidCooperativeCancellation(_)) + )); + assert_eq!(server.requests.lock().unwrap().len(), 128); +} + +#[tokio::test] +async fn cooperative_transport_can_prove_committed_history_after_a_lost_acknowledgment() { + let server = transport_server(); + let client = client(&server, "lost-ack"); + let task = transport_task(); + let intent = transport_delivery(); + assert!(matches!( + client.deliver_workflow_cancellation(&task, &intent).await, + Err(Error::Http { status, .. }) if status.as_u16() == 409 + )); + let observation = CancellationRequest::from_observation(&request_observation()).unwrap(); + let fresh = client + .refresh_workflow_cancellation_history(&task, &observation) + .await + .unwrap(); + assert_eq!( + CancellationHistory::from_events(&fresh, "run", Some(&observation)) + .unwrap() + .delivery, + Some(intent) + ); +} + +#[tokio::test] +async fn cooperative_transport_validates_inputs_and_never_substitutes_control_credentials() { + let server = transport_server(); + let client = client(&server, "valid"); + for changed in 0..6 { + let mut task = transport_task(); + match changed { + 0 => task.task_id.clear(), + 1 => task.workflow_task_attempt = 0, + 2 => task.workflow_task_attempt = u64::MAX, + 3 => task.run_id = None, + 4 => task.lease_owner = None, + 5 => task.lease_owner = Some(" ".into()), + _ => unreachable!(), + } + assert!(matches!( + client + .deliver_workflow_cancellation(&task, &transport_delivery()) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + } + let mut invalid = transport_delivery(); + invalid.sequence_span = 2; + assert!(matches!( + client + .deliver_workflow_cancellation(&transport_task(), &invalid) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + let mut observation = CancellationRequest::from_observation(&request_observation()).unwrap(); + observation.history_refresh_page_token = None; + assert!(matches!( + client + .refresh_workflow_cancellation_history(&transport_task(), &observation) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + let control_only = Client::builder(server.base_url()) + .control_token(Some("control-only".into())) + .build() + .unwrap(); + assert!(matches!( + control_only + .deliver_workflow_cancellation(&transport_task(), &transport_delivery()) + .await, + Err(Error::MissingRoleCredentials { role: "worker", .. }) + )); + assert!(server.requests.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn cooperative_transport_preserves_claim_refusals_and_bounds_the_total_exchange() { + let server = transport_server(); + assert!(matches!( + client(&server, "refused") + .deliver_workflow_cancellation(&transport_task(), &transport_delivery()) + .await, + Err(Error::Http { status, .. }) if status.as_u16() == 409 + )); + let started = Instant::now(); + let observation = CancellationRequest::from_observation(&request_observation()).unwrap(); + let result = client(&server, "budget") + .refresh_workflow_cancellation_history(&transport_task(), &observation) + .await; + assert!(matches!(result, Err(Error::Timeout))); + assert!(started.elapsed() < Duration::from_secs(6)); + assert_eq!( + server + .requests + .lock() + .unwrap() + .iter() + .filter(|request| request.path.ends_with("/history")) + .count(), + 2 + ); +} + #[tokio::test] async fn cooperative_request_uses_control_role_namespace_and_preserves_server_identity() { let server = server(); From 3df4223be487d83c5744abc8dbacc68d6435c6dd Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 09:47:43 +0000 Subject: [PATCH 08/58] Suspend authored cancellation calls with private delivery intent --- src/cooperative_cancellation.rs | 144 +++++++- src/lib.rs | 280 +++++++++++++- src/tests/cooperative_cancellation.rs | 508 ++++++++++++++++++++++++++ 3 files changed, 914 insertions(+), 18 deletions(-) diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index 977d7eb..42a2633 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -76,6 +76,15 @@ fn text<'a>(value: &'a Value, field: &str) -> Result<&'a str> { } impl CancellationRequest { + pub(super) fn validate_observation(&self) -> Result<()> { + Self::from_observation(&json!({ + "request_id":self.request_id, "requested_at":self.requested_at, + "cleanup_deadline_at":self.cleanup_deadline_at, + "history_refresh_page_token":self.history_refresh_page_token, + })) + .map(|_| ()) + } + pub(crate) fn from_observation(value: &Value) -> Result { let requested_at = text(value, "requested_at")?; let cleanup_deadline_at = text(value, "cleanup_deadline_at")?; @@ -512,6 +521,133 @@ impl CancellationHistory { } impl WorkflowState { + fn next_cancellation_sequence(&self) -> Result { + self.recorded_commands.get(self.command_cursor).map_or_else( + || { + self.recorded_commands + .last() + .map_or(0, RecordedCommand::sequence) + .checked_add(self.commands.len() as u64) + .and_then(|sequence| sequence.checked_add(1)) + .filter(|sequence| *sequence < MAX_SEQUENCE) + .ok_or_else(|| invalid("authored cancellation sequence overflowed")) + }, + |command| Ok(command.sequence()), + ) + } + + fn prepare_cancellation_delivery(&mut self, delivery: CancellationDelivery) -> Result { + if !self.cancellation_delivery_enabled || self.cancellation_shield_depth > 0 { + return Ok(false); + } + if self.cancellation_delivery_intent.is_some() { + self.matched_recorded_pending = true; + return Ok(true); + } + let (base, span) = delivery.range(); + if !self.cancellation_history.eligible(base, span) { + return Ok(false); + } + let payload = json!({ + "workflow_command_id":delivery.request_id, "sequence":delivery.sequence, + "call_kind":delivery.call_kind, "sequence_span":delivery.sequence_span, + "operation_sequence":delivery.operation_sequence, + "operation_sequence_span":delivery.operation_sequence_span, + }); + CancellationDelivery::from_payload(&payload)?; + self.cancellation_delivery_command_count = self.commands.len(); + self.cancellation_delivery_intent = Some(delivery); + self.matched_recorded_pending = true; + Ok(true) + } + + pub(super) fn prepare_scalar_cancellation( + &mut self, + index: usize, + kind: CancellationCallKind, + group_path: &[ParallelGroupMetadata], + ) -> Result { + if !self.cancellation_delivery_enabled || self.cancellation_shield_depth > 0 { + return Ok(false); + } + // Group leaves validate their recorded calls before the enclosing + // parallel/select future suspends. They never choose scalar delivery. + if !group_path.is_empty() && self.cancellation_delivery_intent.is_none() { + return Ok(false); + } + let Some(request) = self.cancellation_history.request.as_ref() else { + return Ok(false); + }; + let sequence = self.recorded_commands.get(index).map_or_else( + || self.next_cancellation_sequence(), + |command| Ok(command.sequence()), + )?; + let pending = self.prepare_cancellation_delivery(CancellationDelivery { + request_id: request.request_id.clone(), + sequence, + call_kind: kind, + sequence_span: 1, + operation_sequence: None, + operation_sequence_span: 1, + })?; + if pending && index < self.recorded_commands.len() { + self.command_cursor = index + 1; + } + Ok(pending) + } + + pub(super) fn prepare_group_cancellation( + &mut self, + descriptors: &[ParallelDescriptor], + ) -> Result<()> { + let Some(request) = self.cancellation_history.request.as_ref() else { + return Ok(()); + }; + let Some(group) = descriptors.first().and_then(|leaf| leaf.group_path.first()) else { + return Ok(()); + }; + self.prepare_cancellation_delivery(CancellationDelivery { + request_id: request.request_id.clone(), + sequence: group.parallel_group_base_sequence, + call_kind: CancellationCallKind::Parallel, + sequence_span: descriptors.len() as u64, + operation_sequence: None, + operation_sequence_span: 1, + })?; + Ok(()) + } + + pub(super) fn prepare_selection_handle_cancellation( + &mut self, + handle: &DurableOperationHandle, + ) -> Result { + if !self.cancellation_delivery_enabled || self.cancellation_shield_depth > 0 { + return Ok(false); + } + validate_selection_delivery_handle(self, handle)?; + if !self + .cancellation_history + .eligible(handle.base_sequence, handle.size as u64) + { + return Ok(false); + } + let request_id = self + .cancellation_history + .request + .as_ref() + .expect("eligible request") + .request_id + .clone(); + self.prepare_cancellation_delivery(CancellationDelivery { + request_id, + sequence: self.next_cancellation_sequence()?, + call_kind: CancellationCallKind::SelectionHandle, + sequence_span: 1, + operation_sequence: Some(handle.base_sequence), + operation_sequence_span: handle.size as u64, + }) + } + pub(super) fn expand_cancellation_group( &mut self, descriptors: &[ParallelDescriptor], @@ -852,11 +988,7 @@ impl Client { observation: &CancellationRequest, ) -> Result> { let (owner, run_id) = cancellation_claim(task)?; - CancellationRequest::from_observation(&json!({ - "request_id":observation.request_id, "requested_at":observation.requested_at, - "cleanup_deadline_at":observation.cleanup_deadline_at, - "history_refresh_page_token":observation.history_refresh_page_token, - }))?; + observation.validate_observation()?; let first_token = observation .history_refresh_page_token .clone() @@ -988,7 +1120,7 @@ impl Client { } } -fn cancellation_claim(task: &WorkflowTask) -> Result<(&str, &str)> { +pub(super) fn cancellation_claim(task: &WorkflowTask) -> Result<(&str, &str)> { let owner = task .lease_owner .as_deref() diff --git a/src/lib.rs b/src/lib.rs index c587007..4b915b4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6063,6 +6063,7 @@ struct WorkflowTaskDecision { commands: Vec, message_stream_cursors: Vec, message_stream_waits: Vec, + cancellation_delivery: Option, } impl WorkflowTaskDecision { @@ -6071,6 +6072,7 @@ impl WorkflowTaskDecision { commands, message_stream_cursors: Vec::new(), message_stream_waits: Vec::new(), + cancellation_delivery: None, } } } @@ -7088,6 +7090,11 @@ impl Worker { .unwrap_or_else(|| self.worker_id.clone()); match self.execute_workflow_task_decision(task) { + Ok(decision) if decision.cancellation_delivery.is_some() => { + return Err(Error::CooperativeCancellationUnavailable( + "worker cancellation delivery has not been negotiated".into(), + )); + } Ok(decision) if commands_use_workflow_memo_updates(&decision.commands) && !memo_updates_supported => @@ -7605,7 +7612,32 @@ impl Worker { } fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result { + self.execute_workflow_task_decision_with_cancellation(task, None) + } + + fn execute_workflow_task_decision_with_cancellation( + &self, + task: WorkflowTask, + observation: Option<&CancellationRequest>, + ) -> Result { validate_workflow_task_payloads(&task)?; + let cancellation_history = if let Some(observation) = observation { + observation.validate_observation()?; + cooperative_cancellation::cancellation_claim(&task)?; + let canonical = CancellationHistory::from_events( + &task.history_events, + task.run_id.as_deref().unwrap_or_default(), + Some(observation), + )?; + if canonical.request_index >= task.history_events.len() { + return Err(Error::InvalidCooperativeCancellation( + "workflow replay requires the observed canonical request".into(), + )); + } + Some(canonical) + } else { + None + }; if let Some(update_id) = task .workflow_update_id .as_deref() @@ -7650,6 +7682,10 @@ impl Worker { )?; workflow_state.history_budget = history_budget; workflow_state.workflow_command_identity = workflow_command_identity; + if let Some(canonical) = cancellation_history { + workflow_state.cancellation_history = canonical; + workflow_state.cancellation_delivery_enabled = true; + } if workflow_state.cancellation_history.request.is_none() { workflow_state.cancel_requested = task.cancel_requested; } @@ -7658,7 +7694,26 @@ impl Worker { let mut future = (workflow.execute)(ctx.clone(), input); let mut cx = TaskContext::from_waker(noop_waker_ref()); - match future.as_mut().poll(&mut cx) { + let outcome = future.as_mut().poll(&mut cx); + if ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)? + .cancellation_delivery_intent + .is_some() + { + // A private delivery intent never becomes an application error. + // Even a custom future which ignores a pending durable call cannot + // publish terminal commands before canonical delivery and replay. + if let Poll::Ready(Err(error)) = outcome { + if workflow_task_integrity_error(&error) { + return Err(error); + } + } + return self.message_stream_decision(&ctx, ctx.take_commands()?); + } + + match outcome { Poll::Ready(Ok(result)) => { ctx.ensure_history_consumed()?; let result = encode_typed_envelope(&result, &task.payload_codec)?; @@ -7712,10 +7767,27 @@ impl Worker { commands: Vec, ) -> Result { let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?; + let state = ctx.state.lock().map_err(|_| Error::WorkflowStatePoisoned)?; + if state.cancellation_delivery_intent.is_some() + && commands.len() != state.cancellation_delivery_command_count + { + return Err(invalid_recorded_history( + "cooperative_cancellation_pending_call_escaped", + state + .cancellation_delivery_intent + .as_ref() + .expect("pending delivery") + .sequence, + "only commands preceding the pending cancellation call", + "commands authored after a suspended call", + "workflow code cannot publish work beyond an uncommitted cancellation boundary", + )); + } Ok(WorkflowTaskDecision { commands, message_stream_cursors, message_stream_waits, + cancellation_delivery: state.cancellation_delivery_intent.clone(), }) } @@ -9366,6 +9438,9 @@ struct WorkflowState { cancellation_history: CancellationHistory, cancellation_consumed: bool, cancellation_shield_depth: u64, + cancellation_delivery_enabled: bool, + cancellation_delivery_intent: Option, + cancellation_delivery_command_count: usize, resume_signal: Option, recorded_commands: Vec, selection_markers: Vec, @@ -9516,6 +9591,9 @@ impl WorkflowState { cancellation_history, cancellation_consumed: false, cancellation_shield_depth: 0, + cancellation_delivery_enabled: false, + cancellation_delivery_intent: None, + cancellation_delivery_command_count: 0, resume_signal, recorded_commands, selection_markers, @@ -10696,11 +10774,15 @@ impl ParallelCall { }; let descriptors = parallel_descriptors(operations, base_sequence)?; - self.ctx - .state - .lock() - .map_err(|_| Error::WorkflowStatePoisoned)? - .expand_cancellation_group(&descriptors)?; + { + let mut state = self + .ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)?; + state.expand_cancellation_group(&descriptors)?; + state.prepare_group_cancellation(&descriptors)?; + } self.leaves = descriptors .into_iter() .map(|descriptor| { @@ -10753,6 +10835,16 @@ impl ParallelCall { { return Poll::Ready(Err(failures.remove(position).1)); } + if self + .ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)? + .cancellation_delivery_intent + .is_some() + { + return Poll::Pending; + } if let Some(position) = failures .iter() .position(|(_, error)| matches!(error, Error::CooperativeCancellationRequested(_))) @@ -10788,7 +10880,15 @@ impl ParallelCall { cause: Box::new(cause), }))); } - if pending { + if pending + || self + .ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)? + .cancellation_delivery_intent + .is_some() + { return Poll::Pending; } @@ -11169,11 +11269,15 @@ impl SelectCall { }; let (descriptors, members) = selection_descriptors(operations, base_sequence)?; let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len()); - self.ctx - .state - .lock() - .map_err(|_| Error::WorkflowStatePoisoned)? - .expand_cancellation_group(&descriptors)?; + { + let mut state = self + .ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)?; + state.expand_cancellation_group(&descriptors)?; + state.prepare_group_cancellation(&descriptors)?; + } self.leaves = descriptors .into_iter() .map(|descriptor| SelectionLeaf { @@ -11213,6 +11317,17 @@ impl Future for SelectCall { } } + if self + .ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)? + .cancellation_delivery_intent + .is_some() + { + return Poll::Pending; + } + if let Some(leaf) = self.leaves.iter_mut().find(|leaf| { matches!( leaf.outcome, @@ -11807,6 +11922,11 @@ impl Future for DurableOperationAwaitCall { if let Err(error) = state.replay_selection_handle_cancellation(&self.handle) { return Poll::Ready(Err(error)); } + match state.prepare_selection_handle_cancellation(&self.handle) { + Ok(true) => return Poll::Pending, + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } match selection_cancellation_for_handle(&state, &self.handle) { Err(error) => return Poll::Ready(Err(error)), Ok(false) => {} @@ -12169,6 +12289,18 @@ impl ActivityCall { { return Poll::Ready(Err(error)); } + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Activity, + &self.parallel_group_path, + ) { + Ok(true) => { + self.scheduled = true; + return Poll::Pending; + } + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } state.command_cursor += 1; if let Some(outcome) = outcome { return Poll::Ready(outcome.map_err(Error::ActivityFailed)); @@ -12226,6 +12358,18 @@ impl ActivityCall { command.insert("retry_policy".to_string(), retry_policy); } apply_parallel_group_path(&mut command, &self.parallel_group_path); + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Activity, + &self.parallel_group_path, + ) { + Ok(true) => { + self.scheduled = true; + return Poll::Pending; + } + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } state.commands.push(Value::Object(command)); self.scheduled = true; } @@ -12310,6 +12454,19 @@ impl Future for TimerCall { { return Poll::Ready(Err(error)); } + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Timer, + &self.parallel_group_path, + ) { + Ok(true) => { + self.scheduled = true; + self.matched_pending = true; + return Poll::Pending; + } + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } state.command_cursor += 1; if fired { return Poll::Ready(Ok(())); @@ -12329,6 +12486,19 @@ impl Future for TimerCall { ("delay_seconds".to_string(), json!(requested_delay)), ]); apply_parallel_group_path(&mut command, &self.parallel_group_path); + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Timer, + &self.parallel_group_path, + ) { + Ok(true) => { + self.scheduled = true; + self.matched_pending = true; + return Poll::Pending; + } + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } state.commands.push(Value::Object(command)); self.scheduled = true; } @@ -12405,6 +12575,7 @@ impl Future for ConditionWaitCall { let mut cursor = state.command_cursor; let mut result = None; + let mut pending_delivery = false; loop { if cursor > initial_cursor && matches!( @@ -12469,10 +12640,22 @@ impl Future for ConditionWaitCall { { return Poll::Ready(Err(error)); } + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Condition, + &self.parallel_group_path, + ) { + Ok(pending) => pending_delivery |= pending, + Err(error) => return Poll::Ready(Err(error)), + } result = recorded_result; cursor += 1; } state.command_cursor = cursor; + if pending_delivery { + self.opened_wait = true; + return Poll::Pending; + } result }; @@ -12496,6 +12679,26 @@ impl ConditionWaitCall { mut self: Pin<&mut Self>, options: ValidatedConditionWaitOptions, ) -> Poll> { + { + let ctx = self.ctx.clone(); + let mut state = match ctx.state.lock() { + Ok(state) => state, + Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)), + }; + let cursor = state.command_cursor; + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Condition, + &self.parallel_group_path, + ) { + Ok(true) => { + self.opened_wait = true; + return Poll::Pending; + } + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } + } let selection_member = self .parallel_group_path .first() @@ -12653,6 +12856,19 @@ impl ChildWorkflowCall { { return Poll::Ready(Err(error)); } + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Child, + &self.parallel_group_path, + ) { + Ok(true) => { + self.scheduled = true; + self.matched_pending = true; + return Poll::Pending; + } + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } state.command_cursor += 1; if let Some(outcome) = outcome { return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed)); @@ -12746,6 +12962,19 @@ impl ChildWorkflowCall { object.insert("run_timeout_seconds".to_string(), json!(seconds)); } apply_parallel_group_path(object, &self.parallel_group_path); + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Child, + &self.parallel_group_path, + ) { + Ok(true) => { + self.scheduled = true; + self.matched_pending = true; + return Poll::Pending; + } + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } state.commands.push(command); self.scheduled = true; } @@ -12851,6 +13080,19 @@ impl SignalCall { { return Poll::Ready(Err(error)); } + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Signal, + &self.parallel_group_path, + ) { + Ok(true) => { + self.opened_wait = true; + self.matched_pending = true; + return Poll::Pending; + } + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } state.command_cursor += 1; if let Some(value) = value { return Poll::Ready(Ok(value)); @@ -12881,6 +13123,20 @@ impl SignalCall { } } + match state.prepare_scalar_cancellation( + cursor, + CancellationCallKind::Signal, + &self.parallel_group_path, + ) { + Ok(true) => { + self.opened_wait = true; + self.matched_pending = true; + return Poll::Pending; + } + Ok(false) => {} + Err(error) => return Poll::Ready(Err(error)), + } + if state .resume_signal .as_ref() diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs index f8d5706..d932206 100644 --- a/src/tests/cooperative_cancellation.rs +++ b/src/tests/cooperative_cancellation.rs @@ -118,6 +118,514 @@ fn cancelled_parallel_history() -> Vec { ] } +fn intent_task(events: Vec) -> WorkflowTask { + let mut task = transport_task(); + task.history_events = events; + task +} + +fn original_observation() -> CancellationRequest { + CancellationRequest::from_observation(&request_observation()).unwrap() +} + +async fn authored_scalar(ctx: WorkflowContext, kind: &str) -> Result<()> { + match kind { + "activity" => { + ctx.activity("forward", json!([])).await?; + } + "timer" => { + ctx.sleep(Duration::from_secs(5)).await?; + } + "child" => { + ctx.start_child_workflow("child", ChildWorkflowOptions::new("queue"), json!([])) + .await?; + } + "signal" => { + ctx.wait_signal("resume").await?; + } + "condition" => { + ctx.wait_condition(ConditionWaitOptions::new("ready", "sha256:ready"), || { + Ok(false) + }) + .await?; + } + _ => unreachable!(), + } + Ok(()) +} + +fn pending_scalar_event(kind: &str) -> HistoryEvent { + match kind { + "activity" => event( + "ActivityScheduled", + json!({"sequence":1,"activity_type":"forward","task_queue":"queue"}), + ), + "timer" => scheduled_timer(1), + "child" => event( + "ChildWorkflowScheduled", + json!({"sequence":1,"workflow_type":"child"}), + ), + "signal" => event( + "SignalWaitOpened", + json!({"sequence":1,"signal_name":"resume"}), + ), + "condition" => event( + "ConditionWaitOpened", + json!({"sequence":1,"condition_wait_id":"condition-1", + "condition_wait_occurrence_id":"rust:condition-wait:0","condition_key":"ready","condition_definition_fingerprint":"sha256:ready"}), + ), + _ => unreachable!(), + } +} + +#[test] +fn cooperative_intent_suspends_actual_worker_scalar_calls_without_exposing_an_error() { + for kind in ["activity", "timer", "child", "signal", "condition"] { + for scheduled in [false, true] { + let resumed = Arc::new(AtomicBool::new(false)); + let observed = resumed.clone(); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _input| { + let observed = observed.clone(); + async move { + assert!(!ctx.is_cancellation_requested()?); + // Even catching every error cannot expose cancellation + // before the worker commits its delivery and replays. + let _ = authored_scalar(ctx, kind).await; + observed.store(true, Ordering::SeqCst); + Ok(Value::Null) + } + }); + let mut events = Vec::new(); + if scheduled { + events.push(pending_scalar_event(kind)); + } + events.push(canonical_request()); + let decision = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(events), + Some(&original_observation()), + ) + .unwrap(); + assert!(!resumed.load(Ordering::SeqCst), "{kind}:{scheduled}"); + assert!(decision.commands.is_empty(), "{kind}:{scheduled}"); + let delivery = decision.cancellation_delivery.unwrap(); + assert_eq!(delivery.sequence, 1); + assert_eq!(serde_json::to_value(delivery.call_kind).unwrap(), kind); + assert_eq!(delivery.request_id, "original-request"); + } + } +} + +#[test] +fn cooperative_intent_suspends_a_resolution_committed_after_the_request() { + let resumed = Arc::new(AtomicBool::new(false)); + let observed = resumed.clone(); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _input| { + let observed = observed.clone(); + async move { + let _ = ctx.sleep(Duration::from_secs(5)).await; + observed.store(true, Ordering::SeqCst); + Ok(Value::Null) + } + }); + let decision = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(vec![ + scheduled_timer(1), + canonical_request(), + event( + "TimerFired", + json!({"sequence":1,"timer_id":"timer-1","delay_seconds":5}), + ), + ]), + Some(&original_observation()), + ) + .unwrap(); + assert!(!resumed.load(Ordering::SeqCst)); + assert!(decision.commands.is_empty()); + assert_eq!(decision.cancellation_delivery, Some(transport_delivery())); +} + +#[test] +fn cooperative_intent_preserves_earlier_commands_and_replayed_side_effects() { + let side_effects = Arc::new(Mutex::new(0)); + let observed = side_effects.clone(); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _input| { + let observed = observed.clone(); + async move { + let value: Value = ctx.side_effect(|| { + *observed.lock().unwrap() += 1; + json!(7) + })?; + assert_eq!(value, 7); + ctx.sleep(Duration::from_secs(5)).await?; + Ok(Value::Null) + } + }); + let first = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(vec![canonical_request()]), + Some(&original_observation()), + ) + .unwrap(); + assert_eq!(first.commands.len(), 1); + assert_eq!(first.commands[0]["type"], "record_side_effect"); + assert_eq!(first.cancellation_delivery.unwrap().sequence, 2); + let second = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(vec![ + event( + "SideEffectRecorded", + json!({"sequence":1,"result":fixture_envelope(json!(7))}), + ), + canonical_request(), + ]), + Some(&original_observation()), + ) + .unwrap(); + assert!(second.commands.is_empty()); + assert_eq!(second.cancellation_delivery.unwrap().sequence, 2); + assert_eq!(*side_effects.lock().unwrap(), 1); +} + +#[test] +fn cooperative_intent_replays_committed_delivery_with_the_original_observation() { + let delivered = Arc::new(Mutex::new(None)); + let observed = delivered.clone(); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _input| { + let observed = observed.clone(); + async move { + match ctx.sleep(Duration::from_secs(5)).await { + Err(Error::CooperativeCancellationRequested(cancellation)) => { + *observed.lock().unwrap() = Some(cancellation.request); + } + other => panic!("{other:?}"), + } + Ok(Value::Null) + } + }); + let observation = original_observation(); + let decision = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(vec![ + scheduled_timer(1), + canonical_request(), + canonical_delivery(1, "timer"), + ]), + Some(&observation), + ) + .unwrap(); + assert!(decision.cancellation_delivery.is_none()); + assert_eq!(decision.commands.len(), 1); + assert_eq!(decision.commands[0]["type"], "complete_workflow"); + assert_eq!(*delivered.lock().unwrap(), Some(observation)); +} + +#[test] +fn cooperative_intent_cannot_publish_a_terminal_result_after_an_ignored_pending_call() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5))); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + assert!(matches!(timer.as_mut().poll(&mut cx), Poll::Pending)); + Ok(json!("ignored pending timer")) + }); + let decision = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(vec![canonical_request()]), + Some(&original_observation()), + ) + .unwrap(); + assert!(decision.commands.is_empty()); + assert_eq!(decision.cancellation_delivery, Some(transport_delivery())); +} + +#[test] +fn cooperative_intent_cannot_publish_commands_after_an_ignored_pending_call() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5))); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + assert!(matches!(timer.as_mut().poll(&mut cx), Poll::Pending)); + let _: Value = ctx.side_effect(|| json!("authored after pending timer"))?; + Ok(Value::Null) + }); + assert!( + matches!(worker.execute_workflow_task_decision_with_cancellation( + intent_task(vec![canonical_request()]), Some(&original_observation()), + ), Err(Error::NonDeterministicReplay(ReplayFailure { reason, .. })) + if reason == "cooperative_cancellation_pending_call_escaped") + ); +} + +#[test] +fn cooperative_intent_preserves_a_prior_selection_winner_then_suspends_the_loser_await() { + for late in [false, true] { + let resumed = Arc::new(AtomicBool::new(false)); + let observed = resumed.clone(); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _input| { + let observed = observed.clone(); + async move { + let selected = keyed_activity_selection(&ctx).await?; + assert_eq!(selected.key, SelectionKey::Name("fast".into())); + assert_eq!( + selected.value, + Some(ParallelResult::Activity(json!("winner-value"))) + ); + let _ = selected + .handle(&SelectionKey::Name("slow".into())) + .unwrap() + .await_result() + .await; + observed.store(true, Ordering::SeqCst); + Ok(Value::Null) + } + }); + let mut events = cancelled_selection_history(); + events.pop(); + if late { + events.push(selection_activity_event( + "ActivityCompleted", + 0, + "slow", + Some(json!("late")), + )); + } + let decision = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(events), + Some(&original_observation()), + ) + .unwrap(); + assert!(!resumed.load(Ordering::SeqCst)); + assert!(decision.commands.is_empty()); + let delivery = decision.cancellation_delivery.unwrap(); + assert_eq!(delivery.call_kind, CancellationCallKind::SelectionHandle); + assert_eq!(delivery.sequence, 3); + assert_eq!(delivery.operation_sequence, Some(1)); + assert_eq!(delivery.operation_sequence_span, 1); + } +} + +#[test] +fn cooperative_intent_validates_all_handle_fields_before_checking_prior_resolution() { + for changed in 0..7 { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _input| async move { + let selected = keyed_activity_selection(&ctx).await?; + let mut handle = selected + .handle(&SelectionKey::Name("slow".into())) + .unwrap() + .clone(); + match changed { + 0 => handle.key = SelectionKey::Name("changed".into()), + 1 => handle.index = 9, + 2 => handle.kind = "timer".into(), + 3 => handle.identity = "changed".into(), + 4 => handle.selection_group_id = "changed".into(), + 5 => handle.base_sequence = 2, + 6 => handle.size = 2, + _ => unreachable!(), + } + handle.await_result().await?; + Ok(Value::Null) + }); + let mut events = cancelled_selection_history(); + events.pop(); + assert!( + matches!( + worker.execute_workflow_task_decision_with_cancellation( + intent_task(events), + Some(&original_observation()), + ), + Err(Error::NonDeterministicReplay(_)) + ), + "{changed}" + ); + } +} + +#[test] +fn cooperative_intent_suspends_nested_parallel_and_an_uncommitted_selection_winner() { + for (selection, mode) in [ + (false, "new"), + (false, "scheduled"), + (false, "partial"), + (true, "new"), + (true, "scheduled"), + ] { + let resumed = Arc::new(AtomicBool::new(false)); + let observed = resumed.clone(); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _input| { + let observed = observed.clone(); + async move { + if selection { + let _ = keyed_activity_selection(&ctx).await; + } else { + let _ = ctx.parallel(nested_parallel_operations()).await; + } + observed.store(true, Ordering::SeqCst); + Ok(Value::Null) + } + }); + let events = if mode == "new" { + vec![canonical_request()] + } else if selection { + vec![ + selection_activity_event("ActivityScheduled", 0, "slow", None), + selection_activity_event("ActivityScheduled", 1, "fast", None), + canonical_request(), + selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("late"))), + selection_winner_marker(), + ] + } else { + let mut events = cancelled_parallel_history(); + events.pop(); + if mode == "partial" { + events.insert( + 3, + parallel_history_event( + "ActivityCompleted", + 1, + "activity_type", + "first", + nested_parallel_paths()[0].clone(), + Some(json!(7)), + ), + ); + } + events.push(event("ActivityFailed", json!({"sequence":3,"activity_type":"third", + "exception_type":"LateFailure","exception":{"class":"LateFailure","message":"too late"}}))); + events + }; + let decision = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(events), + Some(&original_observation()), + ) + .unwrap(); + assert!(!resumed.load(Ordering::SeqCst)); + assert!(decision.commands.is_empty()); + let delivery = decision.cancellation_delivery.unwrap(); + assert_eq!(delivery.call_kind, CancellationCallKind::Parallel); + assert_eq!(delivery.sequence, 1); + assert_eq!(delivery.sequence_span, if selection { 2 } else { 3 }); + } +} + +#[test] +fn cooperative_intent_rejects_changed_later_leaf_details_before_exporting_delivery() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + ctx.parallel(vec![ + ParallelOperation::activity("first", json!([])), + ParallelOperation::group(vec![ + ParallelOperation::child_workflow( + "second", + ChildWorkflowOptions::new("child-workers"), + json!([]), + ), + ParallelOperation::activity("changed-third", json!([])), + ]), + ]) + .await?; + Ok(Value::Null) + }); + let mut events = cancelled_parallel_history(); + events.pop(); + assert!(matches!( + worker.execute_workflow_task_decision_with_cancellation( + intent_task(events), + Some(&original_observation()), + ), + Err(Error::NonDeterministicReplay(_)) + )); +} + +#[test] +fn cooperative_intent_uses_the_pending_physical_condition_reopen() { + let payload = |sequence| { + json!({"sequence":sequence,"condition_wait_id":format!("condition:{sequence}"), + "condition_wait_occurrence_id":"rust:condition-wait:0","condition_key":"ready", + "condition_definition_fingerprint":"sha256:ready"}) + }; + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + ctx.wait_condition(ConditionWaitOptions::new("ready", "sha256:ready"), || { + panic!("delivery proposal must suspend before reevaluating the predicate") + }) + .await?; + Ok(Value::Null) + }); + let decision = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(vec![ + event("ConditionWaitOpened", payload(1)), + event("ConditionWaitSatisfied", payload(1)), + event("ConditionWaitOpened", payload(2)), + canonical_request(), + ]), + Some(&original_observation()), + ) + .unwrap(); + assert!(decision.commands.is_empty()); + let delivery = decision.cancellation_delivery.unwrap(); + assert_eq!(delivery.call_kind, CancellationCallKind::Condition); + assert_eq!(delivery.sequence, 2); +} + +#[test] +fn cooperative_intent_keeps_explicit_shielded_cleanup_pending_without_delivery() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + let _shield = ctx.cancellation_shield()?; + ctx.sleep(Duration::from_secs(5)).await?; + Ok(Value::Null) + }); + let decision = worker + .execute_workflow_task_decision_with_cancellation( + intent_task(vec![scheduled_timer(1), canonical_request()]), + Some(&original_observation()), + ) + .unwrap(); + assert!(decision.commands.is_empty()); + assert!(decision.cancellation_delivery.is_none()); +} + +#[test] +fn cooperative_intent_rejects_unproven_observation_or_claim_before_workflow_code() { + let invoked = Arc::new(AtomicBool::new(false)); + let observed = invoked.clone(); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |_ctx, _input| { + observed.store(true, Ordering::SeqCst); + async { Ok(Value::Null) } + }); + for changed in 0..6 { + let mut task = intent_task(vec![canonical_request()]); + let mut observation = original_observation(); + match changed { + 0 => task.history_events.clear(), + 1 => task.lease_owner = None, + 2 => task.run_id = Some("other-run".into()), + 3 => observation.request_id = "other-request".into(), + 4 => observation.cleanup_deadline_at = "2026-10-01T08:11:00Z".into(), + 5 => observation.history_refresh_page_token = None, + _ => unreachable!(), + } + assert!(worker + .execute_workflow_task_decision_with_cancellation(task, Some(&observation)) + .is_err()); + assert!(!invoked.load(Ordering::SeqCst)); + } +} + #[test] fn cooperative_parallel_replays_one_original_cancellation_for_nested_mixed_leaves() { for _cold_restart in 0..2 { From 5ac3677db322b1b16e97dfade2f477f45f134626 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 10:01:03 +0000 Subject: [PATCH 09/58] Validate workflow claim heartbeat and original cancellation observation --- src/cooperative_cancellation.rs | 81 +++++++++++++ src/lib.rs | 2 +- src/tests/cooperative_cancellation.rs | 164 ++++++++++++++++++++++++++ 3 files changed, 246 insertions(+), 1 deletion(-) diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index 42a2633..b8ec814 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -12,6 +12,17 @@ pub struct CancellationDeliveryReceipt { pub delivery: CancellationDelivery, } +/// Successful renewal of the exact workflow-task claim with optional observation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WorkflowTaskHeartbeat { + pub task_id: String, + pub workflow_task_attempt: u64, + pub lease_owner: String, + pub lease_expires_at: String, + pub run_status: String, + pub cancellation_request: Option, +} + /// Optional reason and runtime-owned cleanup limit for a cooperative request. #[derive(Clone, Debug, Default, Serialize)] pub struct CooperativeCancellationOptions { @@ -76,6 +87,22 @@ fn text<'a>(value: &'a Value, field: &str) -> Result<&'a str> { } impl CancellationRequest { + pub(super) fn preserve_observation(&self, current: &Self) -> Result { + self.validate_observation()?; + current.validate_observation()?; + if self.request_id != current.request_id + || DateTime::parse_from_rfc3339(&self.requested_at).unwrap() + != DateTime::parse_from_rfc3339(¤t.requested_at).unwrap() + || DateTime::parse_from_rfc3339(&self.cleanup_deadline_at).unwrap() + != DateTime::parse_from_rfc3339(¤t.cleanup_deadline_at).unwrap() + { + return Err(invalid( + "observation changed the original request or cleanup deadline", + )); + } + Ok(self.clone()) + } + pub(super) fn validate_observation(&self) -> Result<()> { Self::from_observation(&json!({ "request_id":self.request_id, "requested_at":self.requested_at, @@ -904,6 +931,60 @@ fn acknowledgment( } impl Client { + /// Renew the exact selected workflow-task lease using worker protocol 1.20. + /// + /// The reply must acknowledge the actual task, owner and attempt. Renewal + /// neither commits cancellation delivery nor extends its cleanup deadline. + pub async fn heartbeat_workflow_task( + &self, + task: &WorkflowTask, + original: Option<&CancellationRequest>, + ) -> Result { + let (owner, _) = cancellation_claim(task)?; + if let Some(original) = original { + original.validate_observation()?; + } + tokio::time::timeout(CONTROL_BUDGET, async { + let response: Value = self.request_json( + reqwest::Method::POST, + &format!("/worker/workflow-tasks/{}/heartbeat", percent_encode_path_segment(&task.task_id)), + RequestProtocol::Worker("1.20"), + Some(&json!({"lease_owner":owner,"workflow_task_attempt":task.workflow_task_attempt})), + ).await?; + if response["task_id"].as_str() != Some(task.task_id.as_str()) + || response["workflow_task_attempt"].as_u64() != Some(task.workflow_task_attempt) + || response["lease_owner"].as_str() != Some(owner) + || response["renewed"].as_bool() != Some(true) + || response.get("reason") != Some(&Value::Null) + || response["task_status"].as_str() != Some("leased") + { + return Err(invalid("workflow heartbeat did not acknowledge the exact leased claim")); + } + let expires = text(&response, "lease_expires_at")?; + DateTime::parse_from_rfc3339(expires).map_err(|_| invalid("workflow heartbeat lease expiry must include a timezone"))?; + let run_status = text(&response, "run_status")?; + if !matches!(run_status, "pending" | "running" | "waiting") { + return Err(invalid("workflow heartbeat did not acknowledge an active run")); + } + let cancellation_request = match response.get("cancellation_request") { + None | Some(Value::Null) if original.is_none() => None, + None | Some(Value::Null) => return Err(invalid("workflow heartbeat omitted its original pending request")), + Some(observation) => { + let current = CancellationRequest::from_observation(observation)?; + Some(match original { + Some(original) => original.preserve_observation(¤t)?, + None => current, + }) + }, + }; + Ok(WorkflowTaskHeartbeat { + task_id:task.task_id.clone(), workflow_task_attempt:task.workflow_task_attempt, + lease_owner:owner.to_owned(), lease_expires_at:expires.to_owned(), + run_status:run_status.to_owned(), cancellation_request, + }) + }).await.map_err(|_| Error::Timeout)? + } + /// Commit cooperative delivery on the exact selected workflow-task claim. /// /// This explicit worker-protocol 1.20 operation renews no lease and advertises diff --git a/src/lib.rs b/src/lib.rs index 4b915b4..27726ea 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,7 +7,7 @@ mod runtime_uploads; pub use cooperative_cancellation::{ CancellationCallKind, CancellationDelivery, CancellationDeliveryReceipt, CancellationHistory, CancellationRequest, CancellationShield, CooperativeCancellationOptions, - CooperativeCancellationRequested, WorkflowCancellationRequest, + CooperativeCancellationRequested, WorkflowCancellationRequest, WorkflowTaskHeartbeat, }; use std::{ diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs index d932206..4292892 100644 --- a/src/tests/cooperative_cancellation.rs +++ b/src/tests/cooperative_cancellation.rs @@ -1764,11 +1764,19 @@ fn transport_responses(path: &str, body: &str, number: usize) -> Option<(&'stati if case == "budget" { thread::sleep(Duration::from_secs(3)); } + if case == "heartbeat-budget" { + thread::sleep(Duration::from_millis(5200)); + } let mut response = if path.ends_with("/deliver-cancellation") { json!({"delivered":true, "task_id":"task/selected", "workflow_run_id":"run", "request_id":body["request_id"], "sequence":body["sequence"], "call_kind":body["call_kind"], "sequence_span":body["sequence_span"], "operation_sequence":body["operation_sequence"], "operation_sequence_span":body["operation_sequence_span"], "reason":null}) + } else if path.ends_with("/heartbeat") { + json!({"task_id":"task/selected", "workflow_task_attempt":7, + "lease_owner":"actual-owner", "renewed":true, + "lease_expires_at":"2026-10-01T08:00:30Z", "run_status":"running", + "task_status":"leased", "reason":null, "cancellation_request":request_observation()}) } else if path.ends_with("/history") { let first = body["next_history_page_token"] == "opaque-server-token"; let events = if first { @@ -1829,6 +1837,44 @@ fn transport_responses(path: &str, body: &str, number: usize) -> Option<(&'stati match case { "wrong-task" => response["task_id"] = json!("other-task"), "wrong-run" => response["workflow_run_id"] = json!("other-run"), + "heartbeat-attempt" => response["workflow_task_attempt"] = json!(8), + "heartbeat-owner" => response["lease_owner"] = json!("replacement-owner"), + "heartbeat-not-renewed" => response["renewed"] = json!(false), + "heartbeat-string-renewed" => response["renewed"] = json!("true"), + "heartbeat-closed" => response["run_status"] = json!("cancelled"), + "heartbeat-unknown-run" => response["run_status"] = json!("unknown"), + "heartbeat-finished-task" => response["task_status"] = json!("completed"), + "heartbeat-expiry" => response["lease_expires_at"] = json!("bad"), + "heartbeat-expiry-no-zone" => response["lease_expires_at"] = json!("2026-10-01T08:00:30"), + "heartbeat-malformed-request" => response["cancellation_request"] = json!(false), + "heartbeat-missing-token" => { + response["cancellation_request"] + .as_object_mut() + .unwrap() + .remove("history_refresh_page_token"); + } + "heartbeat-changed-request" => { + response["cancellation_request"]["request_id"] = json!("changed") + } + "heartbeat-changed-request-time" => { + response["cancellation_request"]["requested_at"] = json!("2026-10-01T08:00:01Z") + } + "heartbeat-changed-deadline" => { + response["cancellation_request"]["cleanup_deadline_at"] = json!("2026-10-01T08:11:00Z") + } + "heartbeat-equivalent-time" => { + response["cancellation_request"]["requested_at"] = json!("2026-10-01T10:00:00+02:00"); + response["cancellation_request"]["cleanup_deadline_at"] = + json!("2026-10-01T10:10:00+02:00"); + response["cancellation_request"]["history_refresh_page_token"] = + json!("fresh-server-token"); + } + "heartbeat-no-request" => { + response + .as_object_mut() + .unwrap() + .remove("cancellation_request"); + } "wrong-request" => response["request_id"] = json!("other-request"), "wrong-sequence" => response["sequence"] = json!(2), "wrong-kind" => response["call_kind"] = json!("activity"), @@ -1863,6 +1909,124 @@ fn transport_server() -> MockWorkerServer { }) } +#[tokio::test] +async fn cooperative_heartbeat_renews_the_exact_worker_claim_and_retains_original_observation() { + let server = transport_server(); + let original = original_observation(); + let receipt = client(&server, "heartbeat-equivalent-time") + .heartbeat_workflow_task(&transport_task(), Some(&original)) + .await + .unwrap(); + assert_eq!(receipt.task_id, "task/selected"); + assert_eq!(receipt.workflow_task_attempt, 7); + assert_eq!(receipt.lease_owner, "actual-owner"); + assert_eq!(receipt.cancellation_request, Some(original)); + let requests = server.requests.lock().unwrap(); + let request = &requests[0]; + assert_eq!( + request.path, + "/heartbeat-equivalent-time/api/worker/workflow-tasks/task%2Fselected/heartbeat" + ); + assert_eq!(request.worker_protocol.as_deref(), Some("1.20")); + assert_eq!(request.authorization.as_deref(), Some("Bearer worker-only")); + assert_eq!(request.namespace.as_deref(), Some("caller-namespace")); + let body: Value = serde_json::from_str(&request.body).unwrap(); + assert_eq!( + body, + json!({"lease_owner":"actual-owner", "workflow_task_attempt":7}) + ); +} + +#[tokio::test] +async fn cooperative_heartbeat_rejects_changed_claims_malformed_renewal_and_changed_identity() { + for case in [ + "wrong-task", + "heartbeat-attempt", + "heartbeat-owner", + "heartbeat-not-renewed", + "heartbeat-string-renewed", + "heartbeat-closed", + "heartbeat-unknown-run", + "heartbeat-finished-task", + "heartbeat-expiry", + "heartbeat-expiry-no-zone", + "heartbeat-malformed-request", + "heartbeat-missing-token", + "heartbeat-changed-request", + "heartbeat-changed-request-time", + "heartbeat-changed-deadline", + "heartbeat-no-request", + "wrong-reason", + ] { + let server = transport_server(); + let result = client(&server, case) + .heartbeat_workflow_task(&transport_task(), Some(&original_observation())) + .await; + assert!( + matches!(result, Err(Error::InvalidCooperativeCancellation(_))), + "{case}: {result:?}" + ); + } +} + +#[tokio::test] +async fn cooperative_heartbeat_accepts_new_observation_and_does_not_invent_a_request() { + for case in ["valid", "heartbeat-no-request"] { + let server = transport_server(); + let receipt = client(&server, case) + .heartbeat_workflow_task(&transport_task(), None) + .await + .unwrap(); + assert_eq!( + receipt.cancellation_request, + if case == "valid" { + Some(original_observation()) + } else { + None + } + ); + assert_eq!(receipt.lease_expires_at, "2026-10-01T08:00:30Z"); + } +} + +#[tokio::test] +async fn cooperative_heartbeat_validates_the_claim_and_original_observation_before_network_io() { + let server = transport_server(); + let client = client(&server, "valid"); + let mut task = transport_task(); + task.lease_owner = None; + assert!(matches!( + client.heartbeat_workflow_task(&task, None).await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + let mut original = original_observation(); + original.history_refresh_page_token = None; + assert!(matches!( + client + .heartbeat_workflow_task(&transport_task(), Some(&original)) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + assert!(server.requests.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn cooperative_heartbeat_preserves_ownership_refusal_and_the_total_budget() { + let server = transport_server(); + assert!( + matches!(client(&server, "refused").heartbeat_workflow_task(&transport_task(), None).await, + Err(Error::Http { status, .. }) if status.as_u16() == 409) + ); + let started = Instant::now(); + assert!(matches!( + client(&server, "heartbeat-budget") + .heartbeat_workflow_task(&transport_task(), None) + .await, + Err(Error::Timeout) + )); + assert!(started.elapsed() < Duration::from_secs(6)); +} + #[tokio::test] async fn cooperative_transport_delivers_the_exact_claim_with_worker_credentials() { let server = transport_server(); From 936303ddcea57789daf76889f5944d243758d7ce Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 13:42:35 +0000 Subject: [PATCH 10/58] feat: retain cooperative workflow claims and observations --- src/cooperative_cancellation.rs | 182 ++++++++++++++++++ src/lib.rs | 3 +- src/tests/cooperative_cancellation.rs | 261 ++++++++++++++++++++++++++ 3 files changed, 445 insertions(+), 1 deletion(-) diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index b8ec814..6975002 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -23,6 +23,46 @@ pub struct WorkflowTaskHeartbeat { pub cancellation_request: Option, } +/// One actual protocol 1.20 claim and its original pending observation. +/// +/// The claim is read-only. Observing a request does not deliver cancellation to +/// workflow code. Canonical history and a committed delivery are still required. +#[derive(Clone, Debug)] +pub struct CooperativeWorkflowTask { + task: WorkflowTask, + cancellation_request: Option, +} + +impl CooperativeWorkflowTask { + pub fn task(&self) -> &WorkflowTask { + &self.task + } + + pub fn cancellation_request(&self) -> Option<&CancellationRequest> { + self.cancellation_request.as_ref() + } + + /// Renew this exact claim and retain the first observation's identity/token. + /// + /// A refused or malformed renewal leaves the claim and observation intact. + pub async fn heartbeat(&mut self, client: &Client) -> Result { + let receipt = client + .heartbeat_workflow_task(&self.task, self.cancellation_request.as_ref()) + .await?; + self.cancellation_request = receipt.cancellation_request.clone(); + Ok(receipt) + } +} + +/// An explicit cooperative poll, including ordinary idle and stop outcomes. +#[derive(Clone, Debug)] +pub struct CooperativeWorkflowTaskPoll { + pub task: Option, + pub outcome: WorkerPollOutcome, + pub protocol_version: Option, + pub server_capabilities: Option, +} + /// Optional reason and runtime-owned cleanup limit for a cooperative request. #[derive(Clone, Debug, Default, Serialize)] pub struct CooperativeCancellationOptions { @@ -931,6 +971,148 @@ fn acknowledgment( } impl Client { + /// Acquire a claim with protocol 1.20 and retain its pending observation. + /// + /// The Server must have admitted a capable worker registration. This method + /// registers no capability and does not change ordinary [`Worker`] polling. + /// It never substitutes a worker ID for a missing lease owner or invents a + /// task attempt. Poll retries reuse one request ID. Poll and history loading + /// share one long-poll budget plus five seconds, with at most 128 pages. + pub async fn poll_cooperative_workflow_task( + &self, + worker_id: &str, + task_queue: &str, + timeout: Duration, + ) -> Result { + if worker_id.trim().is_empty() || task_queue.trim().is_empty() { + return Err(invalid( + "cooperative polling requires a worker and task queue", + )); + } + let budget = timeout + .checked_add(CONTROL_BUDGET) + .ok_or_else(|| invalid("cooperative poll timeout exceeds its supported budget"))?; + let body = json!({ + "worker_id":worker_id, "task_queue":task_queue, + "poll_request_id":unique_request_id("rust-workflow-poll"), + "timeout_seconds":long_poll_timeout_seconds(timeout), + "history_page_size":WORKFLOW_HISTORY_PAGE_SIZE, + }); + tokio::time::timeout(budget, async { + let value: Value = self + .poll_request_json( + "/worker/workflow-tasks/poll", + RequestProtocol::Worker("1.20"), + &body, + budget, + 1, + ) + .await?; + let mut response: PollWorkflowTaskResponse = serde_json::from_value(value.clone()) + .map_err(|_| invalid("cooperative poll returned a malformed envelope or task"))?; + let outcome = response.outcome(); + let task = if let Some(task) = response.task.take() { + let (owner, _) = cancellation_claim(&task)?; + if owner != worker_id + || value["task"]["workflow_task_attempt"].as_u64() + != Some(task.workflow_task_attempt) + { + return Err(invalid( + "cooperative poll did not return the caller's actual owner and attempt", + )); + } + let cancellation_request = match value["task"].get("cancellation_request") { + None | Some(Value::Null) => None, + Some(observation) => Some(CancellationRequest::from_observation(observation)?), + }; + let mut claim = CooperativeWorkflowTask { + task, + cancellation_request, + }; + self.load_cooperative_claim_history(&mut claim).await?; + Some(claim) + } else { + None + }; + Ok(CooperativeWorkflowTaskPoll { + task, + outcome, + protocol_version: response.protocol_version, + server_capabilities: response.server_capabilities, + }) + }) + .await + .map_err(|_| Error::Timeout)? + } + + async fn load_cooperative_claim_history( + &self, + claim: &mut CooperativeWorkflowTask, + ) -> Result<()> { + let (owner, _) = cancellation_claim(&claim.task)?; + let owner = owner.to_owned(); + let mut token = claim.task.next_history_page_token.clone(); + let mut seen = BTreeSet::new(); + while let Some(current) = token.take() { + if current.trim().is_empty() + || seen.len() >= MAX_REFRESH_PAGES + || !seen.insert(current.clone()) + { + return Err(invalid( + "claim history exceeded its page bound or returned an invalid/repeated token", + )); + } + let body = json!({ + "lease_owner":owner,"workflow_task_attempt":claim.task.workflow_task_attempt, + "next_history_page_token":current,"history_page_size":WORKFLOW_HISTORY_PAGE_SIZE, + }); + let value: Value = self + .request_json( + reqwest::Method::POST, + &format!( + "/worker/workflow-tasks/{}/history", + percent_encode_path_segment(&claim.task.task_id) + ), + RequestProtocol::Worker("1.20"), + Some(&body), + ) + .await?; + if value["task_id"].as_str() != Some(claim.task.task_id.as_str()) + || value["workflow_task_attempt"].as_u64() != Some(claim.task.workflow_task_attempt) + { + return Err(invalid( + "claim history changed the selected task or attempt", + )); + } + let events = value["history_events"] + .as_array() + .ok_or_else(|| invalid("claim history page must contain an event array"))?; + if events.len() > WORKFLOW_HISTORY_PAGE_SIZE as usize { + return Err(invalid( + "claim history page exceeds its requested event limit", + )); + } + token = match value.get("next_history_page_token") { + Some(Value::Null) => None, + Some(Value::String(next)) if !next.trim().is_empty() && !events.is_empty() => { + Some(next.clone()) + } + _ => return Err(invalid("claim history page token or progress is invalid")), + }; + let page: WorkflowTaskHistoryPage = serde_json::from_value(value) + .map_err(|_| invalid("claim history page contains malformed fields or events"))?; + if page + .history_events + .iter() + .any(|event| event.event_type.trim().is_empty()) + { + return Err(invalid("claim history event type must be non-empty")); + } + claim.task.append_history_page(page); + } + Ok(()) + } + /// Renew the exact selected workflow-task lease using worker protocol 1.20. /// /// The reply must acknowledge the actual task, owner and attempt. Renewal diff --git a/src/lib.rs b/src/lib.rs index 27726ea..5d404da 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,7 +7,8 @@ mod runtime_uploads; pub use cooperative_cancellation::{ CancellationCallKind, CancellationDelivery, CancellationDeliveryReceipt, CancellationHistory, CancellationRequest, CancellationShield, CooperativeCancellationOptions, - CooperativeCancellationRequested, WorkflowCancellationRequest, WorkflowTaskHeartbeat, + CooperativeCancellationRequested, CooperativeWorkflowTask, CooperativeWorkflowTaskPoll, + WorkflowCancellationRequest, WorkflowTaskHeartbeat, }; use std::{ diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs index 4292892..ed8f412 100644 --- a/src/tests/cooperative_cancellation.rs +++ b/src/tests/cooperative_cancellation.rs @@ -1767,6 +1767,76 @@ fn transport_responses(path: &str, body: &str, number: usize) -> Option<(&'stati if case == "heartbeat-budget" { thread::sleep(Duration::from_millis(5200)); } + if case == "claim-budget" { + thread::sleep(Duration::from_secs(3)); + } + if path.ends_with("/workflow-tasks/poll") { + if case == "claim-retry" && number == 1 { + // The first HTTP reply is unreadable after acquisition. Retrying + // must reuse the same durable poll identity rather than claim twice. + return Some(("invalid-status", String::new())); + } + if matches!(case, "claim-idle" | "claim-stop") { + return Some(( + "200 OK", + json!({"task":null, + "poll_status":if case == "claim-stop" { "draining" } else { "idle" }, + "reason":if case == "claim-stop" { "worker_draining" } else { "no_tasks" }, + "protocol_version":"1.20" + }) + .to_string(), + )); + } + let mut task = json!({ + "task_id":"task/selected", "workflow_type":"cancel", "run_id":"run", + "lease_owner":"actual-owner", "workflow_task_attempt":7, + "payload_codec":DEFAULT_CODEC, "history_events":[], + "cancel_requested":true, "cancellation_request":request_observation() + }); + match case { + "claim-no-observation" => { + task.as_object_mut().unwrap().remove("cancellation_request"); + } + "claim-missing-attempt" => { + task.as_object_mut() + .unwrap() + .remove("workflow_task_attempt"); + } + "claim-zero-attempt" => task["workflow_task_attempt"] = json!(0), + "claim-owner" => task["lease_owner"] = json!("replacement"), + "claim-missing-owner" => task["lease_owner"] = Value::Null, + "claim-missing-run" => task["run_id"] = Value::Null, + "claim-missing-id" => task["task_id"] = json!(" "), + "claim-bad-observation" => task["cancellation_request"] = json!(false), + "claim-bad-token" => { + task["cancellation_request"]["history_refresh_page_token"] = json!(" ") + } + _ => {} + } + if matches!( + case, + "claim-history" + | "claim-budget" + | "cycle" + | "page-bound" + | "empty-progress" + | "oversized-page" + | "wrong-task" + | "wrong-attempt" + | "malformed-event" + | "empty-token" + | "missing-token" + ) { + task["next_history_page_token"] = json!("opaque-server-token"); + } + return Some(( + "200 OK", + json!({"task":task,"protocol_version":"1.20", + "server_capabilities":{"workflow_memo_updates":{"supported":true}} + }) + .to_string(), + )); + } let mut response = if path.ends_with("/deliver-cancellation") { json!({"delivered":true, "task_id":"task/selected", "workflow_run_id":"run", "request_id":body["request_id"], "sequence":body["sequence"], "call_kind":body["call_kind"], @@ -1909,6 +1979,197 @@ fn transport_server() -> MockWorkerServer { }) } +#[tokio::test] +async fn cooperative_poll_retains_the_actual_claim_observation_and_fenced_history() { + let server = transport_server(); + let response = client(&server, "claim-history") + .poll_cooperative_workflow_task("actual-owner", "queue", Duration::ZERO) + .await + .unwrap(); + assert_eq!(response.outcome, WorkerPollOutcome::Task); + assert_eq!(response.protocol_version.as_deref(), Some("1.20")); + assert_eq!( + response.server_capabilities.unwrap()["workflow_memo_updates"]["supported"], + true + ); + let claim = response.task.unwrap(); + assert_eq!(claim.task().task_id, "task/selected"); + assert_eq!(claim.task().lease_owner.as_deref(), Some("actual-owner")); + assert_eq!(claim.task().workflow_task_attempt, 7); + assert_eq!(claim.cancellation_request(), Some(&original_observation())); + assert_eq!(claim.task().history_events.len(), 3); + assert_eq!(claim.task().next_history_page_token, None); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 3); + for request in requests.iter() { + assert_eq!(request.worker_protocol.as_deref(), Some("1.20")); + assert_eq!(request.authorization.as_deref(), Some("Bearer worker-only")); + assert_eq!(request.namespace.as_deref(), Some("caller-namespace")); + let body: Value = serde_json::from_str(&request.body).unwrap(); + if request.path.ends_with("/poll") { + assert_eq!(body["worker_id"], "actual-owner"); + assert_eq!(body["task_queue"], "queue"); + assert_eq!(body["history_page_size"], WORKFLOW_HISTORY_PAGE_SIZE); + assert!(!body["poll_request_id"].as_str().unwrap().is_empty()); + } else { + assert!(request.path.ends_with("/task%2Fselected/history")); + assert_eq!(body["lease_owner"], "actual-owner"); + assert_eq!(body["workflow_task_attempt"], 7); + } + } +} + +#[tokio::test] +async fn cooperative_poll_rejects_invented_claims_and_malformed_observations() { + for case in [ + "claim-missing-attempt", + "claim-zero-attempt", + "claim-owner", + "claim-missing-owner", + "claim-missing-run", + "claim-missing-id", + "claim-bad-observation", + "claim-bad-token", + ] { + let server = transport_server(); + let result = client(&server, case) + .poll_cooperative_workflow_task("actual-owner", "queue", Duration::ZERO) + .await; + assert!( + matches!(result, Err(Error::InvalidCooperativeCancellation(_))), + "{case}: {result:?}" + ); + assert_eq!(server.requests.lock().unwrap().len(), 1); + } +} + +#[tokio::test] +async fn cooperative_poll_transport_retry_preserves_the_same_acquisition_identity() { + let server = transport_server(); + let response = client(&server, "claim-retry") + .poll_cooperative_workflow_task("actual-owner", "queue", Duration::ZERO) + .await + .unwrap(); + assert_eq!(response.task.unwrap().task().workflow_task_attempt, 7); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].body, requests[1].body); + let body: Value = serde_json::from_str(&requests[0].body).unwrap(); + assert!(!body["poll_request_id"].as_str().unwrap().is_empty()); +} + +#[tokio::test] +async fn cooperative_claim_history_rejects_changed_claims_bad_progress_and_unbounded_pages() { + for case in [ + "cycle", + "page-bound", + "empty-progress", + "oversized-page", + "wrong-task", + "wrong-attempt", + "malformed-event", + "empty-token", + "missing-token", + ] { + let server = transport_server(); + let result = client(&server, case) + .poll_cooperative_workflow_task("actual-owner", "queue", Duration::ZERO) + .await; + assert!( + matches!(result, Err(Error::InvalidCooperativeCancellation(_))), + "{case}: {result:?}" + ); + assert!(server.requests.lock().unwrap().len() <= 129); + } +} + +#[tokio::test] +async fn cooperative_claim_heartbeat_preserves_the_first_observation_and_refuses_replacement() { + let server = transport_server(); + let mut claim = client(&server, "valid") + .poll_cooperative_workflow_task("actual-owner", "queue", Duration::ZERO) + .await + .unwrap() + .task + .unwrap(); + assert!(claim.task().history_events.is_empty()); + claim + .heartbeat(&client(&server, "heartbeat-equivalent-time")) + .await + .unwrap(); + assert_eq!(claim.cancellation_request(), Some(&original_observation())); + assert!(matches!( + claim.heartbeat(&client(&server, "heartbeat-owner")).await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + assert_eq!(claim.cancellation_request(), Some(&original_observation())); + assert_eq!(claim.task().lease_owner.as_deref(), Some("actual-owner")); + assert_eq!(claim.task().workflow_task_attempt, 7); + assert!(claim.task().history_events.is_empty()); +} + +#[tokio::test] +async fn cooperative_poll_preserves_idle_stop_and_no_observation_despite_task_flags() { + let server = transport_server(); + for case in ["claim-idle", "claim-stop"] { + let response = client(&server, case) + .poll_cooperative_workflow_task("actual-owner", "queue", Duration::ZERO) + .await + .unwrap(); + assert!(response.task.is_none()); + assert_eq!(response.outcome.should_stop(), case == "claim-stop"); + } + let mut claim = client(&server, "claim-no-observation") + .poll_cooperative_workflow_task("actual-owner", "queue", Duration::ZERO) + .await + .unwrap() + .task + .unwrap(); + assert!(claim.task().cancel_requested); + assert!(claim.cancellation_request().is_none()); + claim.heartbeat(&client(&server, "valid")).await.unwrap(); + assert_eq!(claim.cancellation_request(), Some(&original_observation())); +} + +#[tokio::test] +async fn cooperative_poll_preflights_inputs_and_shares_the_poll_and_history_budget() { + let server = transport_server(); + let client = client(&server, "claim-budget"); + for (owner, queue, timeout) in [ + (" ", "queue", Duration::ZERO), + ("actual-owner", " ", Duration::ZERO), + ("actual-owner", "queue", Duration::MAX), + ] { + assert!(matches!( + client + .poll_cooperative_workflow_task(owner, queue, timeout) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + } + assert!(server.requests.lock().unwrap().is_empty()); + let control_only = Client::builder(server.base_url()) + .control_token(Some("control-only".into())) + .build() + .unwrap(); + assert!(matches!( + control_only + .poll_cooperative_workflow_task("actual-owner", "queue", Duration::ZERO) + .await, + Err(Error::MissingRoleCredentials { role: "worker", .. }) + )); + assert!(server.requests.lock().unwrap().is_empty()); + let started = Instant::now(); + assert!(matches!( + client + .poll_cooperative_workflow_task("actual-owner", "queue", Duration::ZERO) + .await, + Err(Error::Timeout) + )); + assert!(started.elapsed() < Duration::from_secs(6)); + assert_eq!(server.requests.lock().unwrap().len(), 2); +} + #[tokio::test] async fn cooperative_heartbeat_renews_the_exact_worker_claim_and_retains_original_observation() { let server = transport_server(); From fcf7259ab3242a58c9e86b03e33f5e6935fbdef8 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 14:07:18 +0000 Subject: [PATCH 11/58] Coordinate canonical cancellation delivery before worker publication --- src/cooperative_cancellation.rs | 135 +++++++- src/lib.rs | 47 ++- src/tests/cooperative_cancellation.rs | 439 ++++++++++++++++++++++++++ 3 files changed, 606 insertions(+), 15 deletions(-) diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index 6975002..e528524 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -984,7 +984,28 @@ impl Client { task_queue: &str, timeout: Duration, ) -> Result { - if worker_id.trim().is_empty() || task_queue.trim().is_empty() { + self.poll_cooperative_workflow_task_with_request_id( + worker_id, + task_queue, + timeout, + &unique_request_id("rust-workflow-poll"), + 1, + ) + .await + } + + async fn poll_cooperative_workflow_task_with_request_id( + &self, + worker_id: &str, + task_queue: &str, + timeout: Duration, + poll_request_id: &str, + transport_retries: usize, + ) -> Result { + if worker_id.trim().is_empty() + || task_queue.trim().is_empty() + || poll_request_id.trim().is_empty() + { return Err(invalid( "cooperative polling requires a worker and task queue", )); @@ -994,7 +1015,7 @@ impl Client { .ok_or_else(|| invalid("cooperative poll timeout exceeds its supported budget"))?; let body = json!({ "worker_id":worker_id, "task_queue":task_queue, - "poll_request_id":unique_request_id("rust-workflow-poll"), + "poll_request_id":poll_request_id, "timeout_seconds":long_poll_timeout_seconds(timeout), "history_page_size":WORKFLOW_HISTORY_PAGE_SIZE, }); @@ -1005,7 +1026,7 @@ impl Client { RequestProtocol::Worker("1.20"), &body, budget, - 1, + transport_retries, ) .await?; let mut response: PollWorkflowTaskResponse = serde_json::from_value(value.clone()) @@ -1383,6 +1404,114 @@ impl Client { } } +impl Worker { + pub(super) async fn poll_cooperative_workflow_once(&self) -> Result { + let poll_request_id = unique_request_id("rust-workflow-poll"); + let response = self + .retry_worker_operation(|| { + self.client.poll_cooperative_workflow_task_with_request_id( + &self.worker_id, + &self.task_queue, + self.poll_timeout, + &poll_request_id, + 0, + ) + }) + .await; + let Some(response) = self.settle_worker_poll_response(response).await? else { + return Ok(ManagedPollOutcome::Idle); + }; + if response.outcome.should_stop() { + return Ok(ManagedPollOutcome::Stop); + } + let memo_updates_supported = + runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref()); + let Some(claim) = response.task else { + return Ok(ManagedPollOutcome::Idle); + }; + // An uncertain observation, delivery or canonical refresh cannot become + // an application failure or be published as a workflow-task decision. + let decision = self.replay_cooperative_workflow_claim(&claim).await?; + let (owner, _) = cancellation_claim(&claim.task)?; + self.settle_workflow_task_decision( + &claim.task.task_id, + owner, + claim.task.workflow_task_attempt, + claim.task.run_id.as_deref(), + Ok(decision), + memo_updates_supported, + ) + .await + } + + async fn replay_cooperative_workflow_claim( + &self, + claim: &CooperativeWorkflowTask, + ) -> Result { + let (owner, run_id) = cancellation_claim(&claim.task)?; + if owner != self.worker_id { + return Err(invalid( + "cooperative replay requires this worker's actual claim", + )); + } + let mut task = claim.task.clone(); + let Some(observation) = claim.cancellation_request.as_ref() else { + let canonical = CancellationHistory::from_events(&task.history_events, run_id, None)?; + if canonical.request.is_some() { + return Err(invalid( + "cooperative replay omitted its original pending observation", + )); + } + return self.execute_workflow_task_decision(task); + }; + task.history_events = self + .client + .refresh_workflow_cancellation_history(&claim.task, observation) + .await?; + for _ in 0..3 { + // Count the actual refreshed snapshot. Its byte budget was not + // remeasured, so do not expose the old snapshot's size as current. + task.total_history_events = None; + task.history_size_bytes = None; + let mut decision = self.execute_workflow_task_decision_with_cancellation( + task.clone(), + Some(observation), + )?; + let Some(intent) = decision.cancellation_delivery.as_ref() else { + return Ok(decision); + }; + if !decision.commands.is_empty() { + // Native permits delivery only at/before its next durable call. + // Commit earlier commands first. Completion releases this claim + // and its successor replays their durable results before delivery. + decision.cancellation_delivery = None; + return Ok(decision); + } + let delivery_error = self + .client + .deliver_workflow_cancellation(&claim.task, intent) + .await + .err(); + task.history_events = self + .client + .refresh_workflow_cancellation_history(&claim.task, observation) + .await?; + let canonical = + CancellationHistory::from_events(&task.history_events, run_id, Some(observation))?; + if canonical.delivery.as_ref() != Some(intent) { + return Err(delivery_error.unwrap_or_else(|| { + invalid( + "canonical history did not prove the exact intended cancellation delivery", + ) + })); + } + } + Err(invalid( + "workflow replay did not converge on its canonical cancellation delivery", + )) + } +} + pub(super) fn cancellation_claim(task: &WorkflowTask) -> Result<(&str, &str)> { let owner = task .lease_owner diff --git a/src/lib.rs b/src/lib.rs index 5d404da..e970a0c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6168,6 +6168,8 @@ pub struct Worker { heartbeat_interval: Duration, retry_policy: WorkerRetryPolicy, heartbeat_observer: Option, + // Registration and callback lifetime qualification must precede activation. + cooperative_cancellation_enabled: bool, } impl Worker { @@ -6186,6 +6188,7 @@ impl Worker { heartbeat_interval: Duration::from_secs(60), retry_policy: WorkerRetryPolicy::default(), heartbeat_observer: None, + cooperative_cancellation_enabled: false, } } @@ -7058,6 +7061,9 @@ impl Worker { } async fn poll_workflow_once(&self) -> Result { + if self.cooperative_cancellation_enabled { + return self.poll_cooperative_workflow_once().await; + } let poll_request_id = unique_request_id("rust-workflow-poll"); let response = self .retry_worker_operation(|| { @@ -7090,7 +7096,27 @@ impl Worker { .clone() .unwrap_or_else(|| self.worker_id.clone()); - match self.execute_workflow_task_decision(task) { + self.settle_workflow_task_decision( + &task_id, + &lease_owner, + attempt, + run_id.as_deref(), + self.execute_workflow_task_decision(task), + memo_updates_supported, + ) + .await + } + + async fn settle_workflow_task_decision( + &self, + task_id: &str, + lease_owner: &str, + attempt: u64, + run_id: Option<&str>, + decision: Result, + memo_updates_supported: bool, + ) -> Result { + match decision { Ok(decision) if decision.cancellation_delivery.is_some() => { return Err(Error::CooperativeCancellationUnavailable( "worker cancellation delivery has not been negotiated".into(), @@ -7102,8 +7128,8 @@ impl Worker { { self.client .fail_workflow_task( - &task_id, - &lease_owner, + task_id, + lease_owner, attempt, Error::WorkflowMemoUpdatesUnavailable.to_string(), ) @@ -7117,8 +7143,8 @@ impl Worker { // least one executable command. self.client .fail_workflow_task_with_type( - &task_id, - &lease_owner, + task_id, + lease_owner, attempt, WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE, WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE, @@ -7129,8 +7155,8 @@ impl Worker { let completion = self .client .complete_workflow_task_with_message_streams( - &task_id, - &lease_owner, + task_id, + lease_owner, attempt, decision.commands, decision.message_stream_cursors, @@ -7139,10 +7165,7 @@ impl Worker { .await; if let Err(error) = completion { if !workflow_task_completion_is_terminal_timeout( - &error, - &task_id, - attempt, - run_id.as_deref(), + &error, task_id, attempt, run_id, ) { return Err(error); } @@ -7150,7 +7173,7 @@ impl Worker { } Err(error) => { self.client - .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string()) + .fail_workflow_task(task_id, lease_owner, attempt, error.to_string()) .await?; } } diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs index ed8f412..3a0e876 100644 --- a/src/tests/cooperative_cancellation.rs +++ b/src/tests/cooperative_cancellation.rs @@ -1979,6 +1979,445 @@ fn transport_server() -> MockWorkerServer { }) } +fn coordinator_responses(path: &str, body: &str, number: usize) -> Option<(&'static str, String)> { + let case = path.split('/').nth(1).unwrap_or_default(); + if path.ends_with("/cluster/info") { + return responses(path, body, number); + } + if path.ends_with("/poll") { + if case == "coordinator-retry" && number == 1 { + return Some(("invalid-status", String::new())); + } + let (status, response) = transport_responses(path, body, number)?; + let mut response: Value = serde_json::from_str(&response).unwrap(); + if case == "coordinator-prefix" && number >= 2 { + response["task"]["task_id"] = json!("successor"); + response["task"]["workflow_task_attempt"] = json!(8); + } + if case == "coordinator-saga" && number >= 2 { + response["task"]["task_id"] = json!("cleanup"); + response["task"]["workflow_task_attempt"] = json!(9); + } + return Some((status, response.to_string())); + } + if path.ends_with("/history") { + if case == "coordinator-owner-lost" && number >= 2 { + return Some(( + "409 Conflict", + json!({"reason":"lease_owner_mismatch"}).to_string(), + )); + } + let prefix = case == "coordinator-prefix"; + let successor = prefix && path.ends_with("/successor/history"); + let mut events = if prefix { + vec![canonical_request()] + } else { + vec![scheduled_timer(1), canonical_request()] + }; + if successor { + events.push(event( + "SideEffectRecorded", + json!({"sequence":1,"result":fixture_envelope(json!(7))}), + )); + } + let cleanup = case == "coordinator-saga" && path.ends_with("/cleanup/history"); + let delivered = if prefix { + successor && number >= 2 + } else { + number >= 2 || case == "coordinator-cold" || cleanup + }; + if delivered && !matches!(case, "coordinator-unproved" | "coordinator-shield") { + events.push(canonical_delivery( + if prefix { 2 } else { 1 }, + if case == "coordinator-mismatch" { + "activity" + } else { + "timer" + }, + )); + } + if cleanup { + events.extend(completed_activity(2, "undo", Value::Null)); + } + if case == "coordinator-missing-request" { + events.retain(|event| event.event_type != "CooperativeCancellationRequested"); + } + let events = events + .into_iter() + .map(|event| { + let mut value = event.raw; + value.insert("event_type".into(), json!(event.event_type)); + value.insert("payload".into(), event.payload); + Value::Object(value.into_iter().collect()) + }) + .collect::>(); + return Some(( + "200 OK", + json!({ + "task_id":if successor { "successor" } else if cleanup { "cleanup" } else { "task/selected" }, + "workflow_task_attempt":if successor { 8 } else if cleanup { 9 } else { 7 }, + "total_history_events":events.len(),"history_events":events, + "next_history_page_token":null, + }) + .to_string(), + )); + } + if path.ends_with("/deliver-cancellation") { + if case == "coordinator-lost-ack" { + return Some(( + "409 Conflict", + json!({"reason":"lease_owner_mismatch"}).to_string(), + )); + } + let (status, response) = transport_responses(path, body, number)?; + let mut response: Value = serde_json::from_str(&response).unwrap(); + if case == "coordinator-prefix" { + response["task_id"] = json!("successor"); + } + return Some((status, response.to_string())); + } + if path.ends_with("/complete") || path.ends_with("/fail") { + return Some(("200 OK", json!({"recorded":true}).to_string())); + } + None +} + +fn coordinator_server() -> MockWorkerServer { + MockWorkerServer::start_with_behavior(MockWorkerBehavior { + request_override: Some(coordinator_responses), + ..Default::default() + }) +} + +fn coordinator_worker(server: &MockWorkerServer, case: &str) -> Worker { + let mut worker = Worker::new(client(server, case), "queue").worker_id("actual-owner"); + worker.cooperative_cancellation_enabled = true; + worker.poll_timeout = Duration::ZERO; + worker.retry_policy = WorkerRetryPolicy { + max_retries: 1, + initial_backoff: Duration::from_millis(1), + max_backoff: Duration::from_millis(1), + }; + worker +} + +fn register_coordinator_timer( + worker: &mut Worker, + observed: Arc>>, +) { + worker.register_workflow("cancel", move |ctx, _| { + let observed = observed.clone(); + async move { + match ctx.sleep(Duration::from_secs(5)).await { + Err(Error::CooperativeCancellationRequested(request)) => { + observed.lock().unwrap().push(request) + } + result => result?, + } + Ok(Value::Null) + } + }); +} + +#[tokio::test] +async fn cooperative_coordinator_commits_delivery_proves_history_and_replays_before_completion() { + for case in [ + "coordinator-valid", + "coordinator-lost-ack", + "coordinator-cold", + ] { + let server = coordinator_server(); + let mut worker = coordinator_worker(&server, case); + let observed = Arc::new(Mutex::new(Vec::new())); + register_coordinator_timer(&mut worker, observed.clone()); + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + let observed = observed.lock().unwrap(); + assert_eq!(observed.len(), 1); + assert_eq!(observed[0].request, original_observation()); + assert_eq!(observed[0].delivery, transport_delivery()); + let requests = server.requests.lock().unwrap(); + let delivery_count = requests + .iter() + .filter(|request| request.path.ends_with("/deliver-cancellation")) + .count(); + assert_eq!( + delivery_count, + if case == "coordinator-cold" { 0 } else { 1 } + ); + let completion = requests.last().unwrap(); + assert!(completion.path.ends_with("/complete")); + assert!(requests[requests.len() - 2].path.ends_with("/history")); + let body: Value = serde_json::from_str(&completion.body).unwrap(); + assert_eq!(body["lease_owner"], "actual-owner"); + assert_eq!(body["workflow_task_attempt"], 7); + assert_eq!(body["commands"][0]["type"], "complete_workflow"); + assert!(!requests + .iter() + .any(|request| request.path.ends_with("/fail"))); + } +} + +#[tokio::test] +async fn cooperative_coordinator_refuses_unproved_changed_or_lost_claim_history_without_publication( +) { + for case in [ + "coordinator-unproved", + "coordinator-mismatch", + "coordinator-owner-lost", + "coordinator-missing-request", + ] { + let server = coordinator_server(); + let mut worker = coordinator_worker(&server, case); + let observed = Arc::new(Mutex::new(Vec::new())); + register_coordinator_timer(&mut worker, observed.clone()); + let result = worker.poll_workflow_once().await; + if case == "coordinator-owner-lost" { + assert!(matches!(result, Err(Error::Http { status, .. }) if status.as_u16() == 409)); + } else { + assert!( + matches!(result, Err(Error::InvalidCooperativeCancellation(_))), + "{case}: {result:?}" + ); + } + assert!(observed.lock().unwrap().is_empty()); + let requests = server.requests.lock().unwrap(); + assert!(!requests + .iter() + .any(|request| request.path.ends_with("/complete") || request.path.ends_with("/fail"))); + } +} + +#[tokio::test] +async fn cooperative_coordinator_commits_earlier_prefix_and_delivers_only_on_its_successor_claim() { + let server = coordinator_server(); + let mut worker = coordinator_worker(&server, "coordinator-prefix"); + let side_effects = Arc::new(Mutex::new(0)); + let executions = side_effects.clone(); + let observed = Arc::new(Mutex::new(Vec::new())); + let errors = observed.clone(); + worker.register_workflow("cancel", move |ctx, _| { + let executions = executions.clone(); + let errors = errors.clone(); + async move { + let value: Value = ctx.side_effect(|| { + *executions.lock().unwrap() += 1; + json!(7) + })?; + assert_eq!(value, 7); + match ctx.sleep(Duration::from_secs(5)).await { + Err(Error::CooperativeCancellationRequested(request)) => { + errors.lock().unwrap().push(request) + } + result => result?, + } + Ok(Value::Null) + } + }); + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + assert!(observed.lock().unwrap().is_empty()); + { + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 3); + let completion: Value = serde_json::from_str(&requests[2].body).unwrap(); + assert_eq!(completion["commands"].as_array().unwrap().len(), 1); + assert_eq!(completion["commands"][0]["type"], "record_side_effect"); + assert_eq!(completion["workflow_task_attempt"], 7); + assert!(!requests + .iter() + .any(|request| request.path.ends_with("/deliver-cancellation"))); + } + let result = worker.poll_workflow_once().await; + assert!( + matches!(result, Ok(ManagedPollOutcome::Handled)), + "{result:?}, paths: {:?}", + server + .requests + .lock() + .unwrap() + .iter() + .map(|request| &request.path) + .collect::>() + ); + assert_eq!(*side_effects.lock().unwrap(), 1); + let observed = observed.lock().unwrap(); + assert_eq!(observed.len(), 1); + assert_eq!(observed[0].delivery.sequence, 2); + assert_eq!(observed[0].request, original_observation()); + let requests = server.requests.lock().unwrap(); + let delivery = requests + .iter() + .find(|request| request.path.ends_with("/deliver-cancellation")) + .unwrap(); + assert!(delivery.path.ends_with("/successor/deliver-cancellation")); + let body: Value = serde_json::from_str(&delivery.body).unwrap(); + assert_eq!(body["workflow_task_attempt"], 8); + assert_eq!(body["sequence"], 2); + let body: Value = serde_json::from_str(&requests.last().unwrap().body).unwrap(); + assert_eq!(body["commands"].as_array().unwrap().len(), 1); + assert_eq!(body["commands"][0]["type"], "complete_workflow"); +} + +#[tokio::test] +async fn cooperative_coordinator_leaves_shielded_cleanup_pending_without_delivery() { + let server = coordinator_server(); + let mut worker = coordinator_worker(&server, "coordinator-shield"); + worker.register_workflow("cancel", |ctx, _| async move { + let _shield = ctx.cancellation_shield()?; + ctx.sleep(Duration::from_secs(5)).await?; + Ok(Value::Null) + }); + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 3); + assert!(requests.last().unwrap().path.ends_with("/fail")); + let body: Value = serde_json::from_str(&requests.last().unwrap().body).unwrap(); + assert_eq!( + body["failure"]["type"], + WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE + ); + assert!(!requests + .iter() + .any(|request| request.path.ends_with("/deliver-cancellation"))); +} + +#[tokio::test] +async fn cooperative_coordinator_unhandled_canonical_request_publishes_typed_workflow_cancellation() +{ + let server = coordinator_server(); + let mut worker = coordinator_worker(&server, "coordinator-valid"); + worker.register_workflow("cancel", |ctx, _| async move { + ctx.sleep(Duration::from_secs(5)).await?; + Ok(Value::Null) + }); + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + let requests = server.requests.lock().unwrap(); + let completion = requests.last().unwrap(); + assert!(completion.path.ends_with("/complete")); + let body: Value = serde_json::from_str(&completion.body).unwrap(); + assert_eq!(body["commands"].as_array().unwrap().len(), 1); + let command = &body["commands"][0]; + assert_eq!(command["type"], "fail_workflow"); + assert_eq!(command["exception_type"], "WorkflowCancellationRequested"); + assert_eq!( + command["exception_class"], + "durable_workflow::CooperativeCancellationRequested" + ); + assert_eq!(command["non_retryable"], true); + assert_eq!(command["exception"]["properties"]["reason"], "cancelled"); + assert_eq!( + command["exception"]["properties"]["request_id"], + "original-request" + ); + assert_eq!( + command["exception"]["properties"]["cleanup_deadline_at"], + "2026-10-01T08:10:00Z" + ); + assert!(!requests + .iter() + .any(|request| request.path.ends_with("/fail"))); +} + +#[tokio::test] +async fn cooperative_coordinator_replays_saga_cleanup_on_a_cold_successor_before_terminal_cancellation( +) { + let server = coordinator_server(); + let mut worker = coordinator_worker(&server, "coordinator-saga"); + worker.register_workflow("cancel", |ctx, _| async move { + let mut saga = ctx.saga(); + saga.add_compensation("undo", json!([]))?; + let result = ctx.sleep(Duration::from_secs(5)).await; + saga.finish(result).await?; + Ok(Value::Null) + }); + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + { + let requests = server.requests.lock().unwrap(); + let body: Value = serde_json::from_str(&requests.last().unwrap().body).unwrap(); + assert_eq!(body["commands"].as_array().unwrap().len(), 1); + assert_eq!(body["commands"][0]["type"], "schedule_activity"); + assert_eq!(body["commands"][0]["activity_type"], "undo"); + assert_eq!(body["workflow_task_attempt"], 7); + } + // A fresh Worker represents process loss after cleanup was scheduled. The + // Server history proves its activity completion before terminal publication. + let mut replacement = coordinator_worker(&server, "coordinator-saga"); + replacement.register_workflow("cancel", |ctx, _| async move { + let mut saga = ctx.saga(); + saga.add_compensation("undo", json!([]))?; + let result = ctx.sleep(Duration::from_secs(5)).await; + saga.finish(result).await?; + Ok(Value::Null) + }); + assert_eq!( + replacement.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + let requests = server.requests.lock().unwrap(); + assert_eq!( + requests + .iter() + .filter(|request| request.path.ends_with("/deliver-cancellation")) + .count(), + 1 + ); + let completion = requests.last().unwrap(); + assert!(completion.path.ends_with("/cleanup/complete")); + let body: Value = serde_json::from_str(&completion.body).unwrap(); + assert_eq!(body["workflow_task_attempt"], 9); + assert_eq!(body["commands"].as_array().unwrap().len(), 1); + let command = &body["commands"][0]; + assert_eq!(command["type"], "fail_workflow"); + assert_eq!(command["exception_type"], "WorkflowCancellationRequested"); + assert_eq!( + command["exception_class"], + "durable_workflow::CooperativeCancellationRequested" + ); + assert_eq!(command["non_retryable"], true); + assert_eq!(command["exception"]["properties"]["reason"], "cancelled"); + assert_eq!( + command["exception"]["properties"]["request_id"], + "original-request" + ); + assert_eq!( + command["exception"]["properties"]["cleanup_deadline_at"], + "2026-10-01T08:10:00Z" + ); +} + +#[tokio::test] +async fn cooperative_coordinator_poll_retry_retains_the_same_claim_acquisition_identity() { + let server = coordinator_server(); + let mut worker = coordinator_worker(&server, "coordinator-retry"); + register_coordinator_timer(&mut worker, Arc::new(Mutex::new(Vec::new()))); + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + let requests = server.requests.lock().unwrap(); + let polls = requests + .iter() + .filter(|request| request.path.ends_with("/poll")) + .collect::>(); + assert_eq!(polls.len(), 2); + assert_eq!(polls[0].body, polls[1].body); + assert_eq!(polls[0].worker_protocol.as_deref(), Some("1.20")); +} + #[tokio::test] async fn cooperative_poll_retains_the_actual_claim_observation_and_fenced_history() { let server = transport_server(); From 8aba06432c250f7dcda659a6afb3870333c59fcf Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 14:18:44 +0000 Subject: [PATCH 12/58] Fence cooperative activity callbacks to their owned attempt and lifetime --- src/cooperative_cancellation.rs | 345 +++++++++++++++++ src/lib.rs | 10 + src/tests/cooperative_cancellation.rs | 516 ++++++++++++++++++++++++++ 3 files changed, 871 insertions(+) diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index e528524..67e008d 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -971,6 +971,58 @@ fn acknowledgment( } impl Client { + /// Observe an exact activity attempt without renewing its lease or progress. + /// + /// This explicit worker-protocol 1.20 operation has one five-second budget. + /// A pending workflow request alone does not cancel an activity. Its reply + /// reports the runtime's current continuation authority after delivery. + pub async fn activity_task_status( + &self, + task_id: &str, + activity_attempt_id: &str, + lease_owner: &str, + ) -> Result { + if [task_id, activity_attempt_id, lease_owner] + .iter() + .any(|value| value.trim().is_empty()) + { + return Err(invalid("activity observation requires the actual claim")); + } + tokio::time::timeout(CONTROL_BUDGET, async { + let value: Value = activity_task_response( + self.request_json( + reqwest::Method::POST, + &format!( + "/worker/activity-tasks/{}/status", + percent_encode_path_segment(task_id) + ), + RequestProtocol::Worker("1.20"), + Some(&json!({ + "activity_attempt_id":activity_attempt_id,"lease_owner":lease_owner + })), + ) + .await, + "status", + task_id, + activity_attempt_id, + )?; + if value["task_id"].as_str() != Some(task_id) + || value["activity_attempt_id"].as_str() != Some(activity_attempt_id) + || value["lease_owner"].as_str() != Some(lease_owner) + || value["can_continue"].as_bool().is_none() + || value["cancel_requested"].as_bool().is_none() + || value["heartbeat_recorded"].as_bool() != Some(false) + { + return Err(invalid( + "activity observation did not acknowledge the exact claim", + )); + } + Ok(value) + }) + .await + .map_err(|_| Error::Timeout)? + } + /// Acquire a claim with protocol 1.20 and retain its pending observation. /// /// The Server must have admitted a capable worker registration. This method @@ -1405,6 +1457,116 @@ impl Client { } impl Worker { + pub(super) async fn poll_cooperative_activity_once(&self) -> Result { + let poll_request_id = unique_request_id("rust-activity-poll"); + let response = self + .retry_worker_operation(|| { + self.client.poll_activity_task_response_with_request_id( + &self.worker_id, + &self.task_queue, + self.poll_timeout, + &poll_request_id, + 0, + ) + }) + .await; + let Some(response) = self.settle_worker_poll_response(response).await? else { + return Ok(ManagedPollOutcome::Idle); + }; + if response.outcome().should_stop() { + return Ok(ManagedPollOutcome::Stop); + } + let Some(task) = response.task else { + return Ok(ManagedPollOutcome::Idle); + }; + let guard = ActivityClaimGuard::new(&self.client, &task, &self.worker_id)?; + let _abandon_on_drop = AbandonActivityOnDrop(guard.clone()); + if guard.observe().await.is_err() { + return Ok(ManagedPollOutcome::Handled); + } + let invocation = self.execute_cooperative_activity_task(&task, &guard); + tokio::pin!(invocation); + let result = loop { + tokio::select! { + biased; + _ = guard.wait_for_shutdown() => { + guard.abandon(); + return Ok(ManagedPollOutcome::Handled); + } + result = &mut invocation => break result, + _ = tokio::time::sleep(Duration::from_secs(1)) => { + if guard.observe().await.is_err() { + return Ok(ManagedPollOutcome::Handled); + } + } + } + }; + // Both genuine application failures and successful results need current + // authority before result upload or completion/failure publication. + if matches!(result, Err(Error::ActivityExecutionAbandoned(_))) + || guard.observe().await.is_err() + { + return Ok(ManagedPollOutcome::Handled); + } + let settlement = match result { + Ok(value) => { + self.client + .complete_activity_task( + &guard.task_id, + &guard.attempt_id, + &guard.owner, + value, + &task.payload_codec, + ) + .await + } + Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error), + Err(error) => { + self.client + .fail_activity_task( + &guard.task_id, + &guard.attempt_id, + &guard.owner, + error.to_string(), + false, + ) + .await + } + }; + if let Err(error) = settlement { + if !activity_task_rejection_is_final(&error) { + return Err(error); + } + } + Ok(ManagedPollOutcome::Handled) + } + + async fn execute_cooperative_activity_task( + &self, + task: &ActivityTask, + guard: &ActivityClaimGuard, + ) -> Result { + validate_activity_task_payloads(task)?; + let handler = self + .activities + .get(&task.activity_type) + .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?; + let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?; + let context = ActivityContext { + client: self.client.clone(), + task_id: guard.task_id.clone(), + activity_attempt_id: guard.attempt_id.clone(), + lease_owner: guard.owner.clone(), + activity_type: task.activity_type.clone(), + attempt_number: task.attempt_number, + task_queue: self.task_queue.clone(), + worker_id: self.worker_id.clone(), + claim_guard: Some(guard.clone()), + }; + guard.boundary()?; + handler(context, args).await + } + pub(super) async fn poll_cooperative_workflow_once(&self) -> Result { let poll_request_id = unique_request_id("rust-workflow-poll"); let response = self @@ -1512,6 +1674,189 @@ impl Worker { } } +#[derive(Clone, Debug)] +pub(super) struct ActivityClaimGuard { + client: Client, + task_id: String, + attempt_id: String, + owner: String, + active: Arc, + stop: Option>, +} + +struct AbandonActivityOnDrop(ActivityClaimGuard); + +impl Drop for AbandonActivityOnDrop { + fn drop(&mut self) { + self.0.abandon(); + } +} + +impl ActivityClaimGuard { + fn new(client: &Client, task: &ActivityTask, worker_id: &str) -> Result { + let owner = task.lease_owner.as_deref().unwrap_or_default(); + let attempt = task + .activity_attempt_id + .as_deref() + .or(task.attempt_id.as_deref()) + .unwrap_or_default(); + if task.task_id.trim().is_empty() + || attempt.trim().is_empty() + || owner.trim().is_empty() + || owner != worker_id + || matches!((&task.activity_attempt_id, &task.attempt_id), (Some(left), Some(right)) if left != right) + { + return Err(invalid( + "activity execution requires the worker's actual immutable claim", + )); + } + Ok(Self { + client: client.clone(), + task_id: task.task_id.clone(), + attempt_id: attempt.to_owned(), + owner: owner.to_owned(), + active: Arc::new(AtomicBool::new(true)), + stop: client + .worker_storage_admission + .as_ref() + .map(|admission| Arc::clone(&admission.stop)), + }) + } + + fn abandon(&self) { + self.active.store(false, Ordering::SeqCst); + } + + fn boundary(&self) -> Result<()> { + if !self.active.load(Ordering::SeqCst) + || self + .stop + .as_ref() + .is_some_and(|stop| stop.load(Ordering::SeqCst)) + { + self.abandon(); + return Err(Error::ActivityExecutionAbandoned( + "callback completed, was abandoned, or its worker is stopping".into(), + )); + } + Ok(()) + } + + async fn wait_for_shutdown(&self) { + if let Some(stop) = &self.stop { + wait_for_worker_stop(stop).await; + } else { + std::future::pending::<()>().await; + } + } + + async fn observe(&self) -> Result<()> { + self.boundary()?; + let result = self + .client + .activity_task_status(&self.task_id, &self.attempt_id, &self.owner) + .await + .and_then(|value| { + if value["can_continue"].as_bool() != Some(true) + || value["cancel_requested"].as_bool() != Some(false) + || value.get("reason") != Some(&Value::Null) + || value["task_status"].as_str() != Some("leased") + || value["attempt_status"].as_str() != Some("running") + || value["activity_status"].as_str() != Some("running") + { + return Err(invalid("activity observation refused continuation")); + } + let mut bounds = vec![text(&value, "lease_expires_at")?]; + if let Some(deadlines) = value.get("deadlines").filter(|v| !v.is_null()) { + if !deadlines.is_object() { + return Err(invalid("activity execution deadlines must be an object")); + } + for kind in ["heartbeat", "start_to_close", "schedule_to_close"] { + if let Some(deadline) = deadlines.get(kind).filter(|v| !v.is_null()) { + bounds.push( + deadline + .as_str() + .ok_or_else(|| invalid("invalid activity deadline"))?, + ); + } + } + } + if let Some(session) = value.get("worker_session").filter(|v| !v.is_null()) { + if session["status"].as_str() != Some("active") + || session["lease_owner"].as_str() != Some(self.owner.as_str()) + { + return Err(invalid( + "activity no longer owns its required worker session", + )); + } + bounds.push(text(session, "lease_expires_at")?); + bounds.push(text(session, "ttl_expires_at")?); + } + for bound in bounds { + let deadline = DateTime::parse_from_rfc3339(bound) + .map_err(|_| invalid("activity deadline must include a timezone"))?; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| invalid("activity observation clock precedes the epoch"))?; + if !u128::try_from(deadline.timestamp_millis()) + .is_ok_and(|millis| millis > now.as_millis()) + { + return Err(invalid("activity ownership or execution deadline elapsed")); + } + } + self.boundary() + }); + result.map_err(|error| { + self.abandon(); + Error::ActivityExecutionAbandoned(error.to_string()) + }) + } + + pub(super) async fn heartbeat( + &self, + context: &ActivityContext, + details: T, + ) -> Result { + if context.task_id != self.task_id + || context.activity_attempt_id != self.attempt_id + || context.lease_owner != self.owner + || context.worker_id != self.owner + { + self.abandon(); + return Err(Error::ActivityExecutionAbandoned( + "activity context changed its original claim".into(), + )); + } + self.observe().await?; + let result = tokio::time::timeout(CONTROL_BUDGET, async { + let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?; + self.boundary()?; + let value: Value = self.client.request_json( + reqwest::Method::POST, + &format!("/worker/activity-tasks/{}/heartbeat", percent_encode_path_segment(&self.task_id)), + RequestProtocol::Worker("1.20"), + Some(&json!({"activity_attempt_id":self.attempt_id,"lease_owner":self.owner,"details":details})), + ).await?; + if value["task_id"].as_str() != Some(self.task_id.as_str()) + || value["activity_attempt_id"].as_str() != Some(self.attempt_id.as_str()) + || value["lease_owner"].as_str() != Some(self.owner.as_str()) + || value["can_continue"].as_bool() != Some(true) + || value["cancel_requested"].as_bool() != Some(false) + || value["heartbeat_recorded"].as_bool() != Some(true) + { + return Err(invalid("activity heartbeat lost its original claim")); + } + serde_json::from_value(value).map_err(Error::from) + }).await.map_err(|_| Error::Timeout).and_then(|result| result); + let response = result.map_err(|error| { + self.abandon(); + Error::ActivityExecutionAbandoned(error.to_string()) + })?; + self.observe().await?; + Ok(response) + } +} + pub(super) fn cancellation_claim(task: &WorkflowTask) -> Result<(&str, &str)> { let owner = task .lease_owner diff --git a/src/lib.rs b/src/lib.rs index e970a0c..d24e4ff 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -266,6 +266,8 @@ pub enum Error { CooperativeCancellationUnavailable(String), #[error("invalid cooperative cancellation: {0}")] InvalidCooperativeCancellation(String), + #[error("activity execution no longer owns its claim: {0}")] + ActivityExecutionAbandoned(String), #[error(transparent)] InvalidActivityOptions(ActivityOptionsError), #[error(transparent)] @@ -7193,6 +7195,9 @@ impl Worker { } async fn poll_activity_once(&self) -> Result { + if self.cooperative_cancellation_enabled { + return self.poll_cooperative_activity_once().await; + } let poll_request_id = unique_request_id("rust-activity-poll"); let response = self .retry_worker_operation(|| { @@ -7906,6 +7911,7 @@ impl Worker { attempt_number: task.attempt_number, task_queue: self.task_queue.clone(), worker_id: self.worker_id.clone(), + claim_guard: None, }; handler(ctx, args).await @@ -13214,10 +13220,14 @@ pub struct ActivityContext { pub attempt_number: u64, pub task_queue: String, pub worker_id: String, + claim_guard: Option, } impl ActivityContext { pub async fn heartbeat(&self, details: T) -> Result { + if let Some(guard) = &self.claim_guard { + return guard.heartbeat(self, details).await; + } self.client .heartbeat_activity_task( &self.task_id, diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs index 3a0e876..f304c6b 100644 --- a/src/tests/cooperative_cancellation.rs +++ b/src/tests/cooperative_cancellation.rs @@ -1979,6 +1979,522 @@ fn transport_server() -> MockWorkerServer { }) } +fn remote_activity_responses( + path: &str, + body: &str, + number: usize, +) -> Option<(&'static str, String)> { + let case = path.split('/').nth(1).unwrap_or_default(); + if path.ends_with("/cluster/info") { + return responses(path, body, number); + } + if path.ends_with("/worker/register") { + return Some(( + "200 OK", + json!({"registered":true,"worker_id":"actual-owner","heartbeat_interval_seconds":3600}) + .to_string(), + )); + } + if path.ends_with("/worker/heartbeat") { + return Some(("200 OK", json!({}).to_string())); + } + if path.ends_with("/worker/registrations/actual-owner") { + return Some(("200 OK", json!({"worker_id":"actual-owner","outcome":"deregistered","recovered_workflow_task_count":1}).to_string())); + } + if path.ends_with("/activity-tasks/poll") { + if number > 1 { + return Some(( + "200 OK", + json!({"task":null,"poll_status":"timeout"}).to_string(), + )); + } + let mut task = json!({"task_id":"activity","activity_attempt_id":"attempt-original", + "activity_type":"work","payload_codec":"avro","lease_owner":"actual-owner","attempt_number":3}); + match case { + "remote-missing-owner" => { + task.as_object_mut().unwrap().remove("lease_owner"); + } + "remote-missing-attempt" => { + task.as_object_mut().unwrap().remove("activity_attempt_id"); + } + "remote-invented-owner" => task["lease_owner"] = json!("other-worker"), + "remote-conflicting-attempt" => task["attempt_id"] = json!("other-attempt"), + _ => {} + } + return Some(( + "200 OK", + json!({"task":task,"poll_status":"leased"}).to_string(), + )); + } + if path.ends_with("/status") { + let request: Value = serde_json::from_str(body).unwrap(); + if case == "remote-refused" || (case == "remote-replaced" && number >= 2) { + return Some(( + "409 Conflict", + json!({"reason":"lease_owner_mismatch"}).to_string(), + )); + } + if case == "remote-offline" { + return Some(( + "503 Service Unavailable", + json!({"reason":"backend_unavailable"}).to_string(), + )); + } + if case == "remote-status-slow" { + std::thread::sleep(Duration::from_millis(5250)); + } + let mut reply = json!({"task_id":if path.ends_with("/activity%2Fselected/status") { "activity/selected" } else { "activity" }, + "activity_attempt_id":request["activity_attempt_id"],"lease_owner":request["lease_owner"], + "can_continue":true,"cancel_requested":false,"reason":null,"heartbeat_recorded":false, + "lease_expires_at":"2099-01-01T00:00:00Z","deadlines":null,"worker_session":null, + "task_status":"leased","attempt_status":"running","activity_status":"running"}); + match case { + "remote-wrong-task" => reply["task_id"] = json!("other-task"), + "remote-wrong-attempt" => reply["activity_attempt_id"] = json!("other-attempt"), + "remote-wrong-owner" => reply["lease_owner"] = json!("other-worker"), + "remote-malformed" => reply["can_continue"] = json!("true"), + "remote-recorded-progress" => reply["heartbeat_recorded"] = json!(true), + "remote-closed-task" => reply["task_status"] = json!("completed"), + "remote-closed-attempt" => reply["attempt_status"] = json!("completed"), + "remote-closed-activity" => reply["activity_status"] = json!("completed"), + "remote-expired" => reply["lease_expires_at"] = json!("2000-01-01T00:00:00Z"), + "remote-no-timezone" => reply["lease_expires_at"] = json!("2099-01-01T00:00:00"), + "remote-deadline" => reply["deadlines"] = json!({"heartbeat":"2000-01-01T00:00:00Z"}), + "remote-bad-deadlines" => reply["deadlines"] = json!(false), + "remote-session-expired" => { + reply["worker_session"] = json!({"status":"active","lease_owner":"actual-owner","lease_expires_at":"2099-01-01T00:00:00Z","ttl_expires_at":"2000-01-01T00:00:00Z"}) + } + "remote-session-replaced" => { + reply["worker_session"] = json!({"status":"active","lease_owner":"replacement","lease_expires_at":"2099-01-01T00:00:00Z","ttl_expires_at":"2099-01-01T00:00:00Z"}) + } + "remote-cancelled" | "remote-late-result" + if case == "remote-cancelled" || number >= 2 => + { + reply["can_continue"] = json!(false); + reply["cancel_requested"] = json!(true); + reply["reason"] = json!("activity_cancelled"); + } + "remote-heartbeat-post-loss" if number >= 3 => reply["can_continue"] = json!(false), + _ => {} + } + return Some(("200 OK", reply.to_string())); + } + if path.ends_with("/activity/heartbeat") { + let mut reply = json!({"task_id":"activity","activity_attempt_id":"attempt-original", + "lease_owner":"actual-owner","can_continue":true,"cancel_requested":false,"heartbeat_recorded":true}); + match case { + "remote-heartbeat-wrong-attempt" => reply["activity_attempt_id"] = json!("replacement"), + "remote-heartbeat-cancelled" => reply["cancel_requested"] = json!(true), + _ => {} + } + return Some(("200 OK", reply.to_string())); + } + if path.ends_with("/complete") || path.ends_with("/fail") { + return Some(("200 OK", json!({"recorded":true}).to_string())); + } + None +} + +fn remote_activity_server() -> MockWorkerServer { + MockWorkerServer::start_with_behavior(MockWorkerBehavior { + request_override: Some(remote_activity_responses), + ..Default::default() + }) +} + +struct PendingActivityCallback(Arc); + +impl Future for PendingActivityCallback { + type Output = Result; + + fn poll(self: Pin<&mut Self>, _: &mut TaskContext<'_>) -> Poll { + Poll::Pending + } +} + +impl Drop for PendingActivityCallback { + fn drop(&mut self) { + self.0.store(true, Ordering::SeqCst); + } +} + +#[tokio::test] +async fn cooperative_activity_observation_uses_the_exact_worker_claim_without_renewal() { + let server = remote_activity_server(); + let reply = client(&server, "remote-valid") + .activity_task_status("activity/selected", "attempt-original", "actual-owner") + .await + .unwrap(); + assert_eq!(reply["heartbeat_recorded"], false); + assert_eq!(reply["can_continue"], true); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert_eq!( + requests[0].path, + "/remote-valid/api/worker/activity-tasks/activity%2Fselected/status" + ); + assert_eq!(requests[0].worker_protocol.as_deref(), Some("1.20")); + assert_eq!( + requests[0].authorization.as_deref(), + Some("Bearer worker-only") + ); + assert_eq!(requests[0].namespace.as_deref(), Some("caller-namespace")); + let body: Value = serde_json::from_str(&requests[0].body).unwrap(); + assert_eq!( + body, + json!({"activity_attempt_id":"attempt-original","lease_owner":"actual-owner"}) + ); +} + +#[tokio::test] +async fn cooperative_activity_observation_rejects_changed_receipts_and_preserves_refusals() { + for case in [ + "remote-wrong-task", + "remote-wrong-attempt", + "remote-wrong-owner", + "remote-malformed", + "remote-recorded-progress", + ] { + let server = remote_activity_server(); + assert!( + matches!( + client(&server, case) + .activity_task_status("activity", "attempt-original", "actual-owner") + .await, + Err(Error::InvalidCooperativeCancellation(_)) + ), + "{case}" + ); + } + let server = remote_activity_server(); + let result = client(&server, "remote-refused") + .activity_task_status("activity", "attempt-original", "actual-owner") + .await; + assert!( + matches!(result, Err(Error::ActivityTaskRejected(ref error)) if error.operation == "status" + && error.status == 409 && error.reason == "lease_owner_mismatch") + ); + for (task, attempt, owner) in [ + ("", "attempt", "owner"), + ("task", "", "owner"), + ("task", "attempt", ""), + ] { + assert!(client(&server, "remote-valid") + .activity_task_status(task, attempt, owner) + .await + .is_err()); + } + let control_only = Client::builder(format!("{}/remote-valid", server.base_url())) + .control_token(Some("control-only".to_string())) + .build() + .unwrap(); + assert!(matches!( + control_only + .activity_task_status("activity", "attempt", "owner") + .await, + Err(Error::MissingRoleCredentials { role: "worker", .. }) + )); + assert_eq!(server.requests.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn cooperative_activity_observation_has_one_five_second_budget() { + let server = remote_activity_server(); + let start = Instant::now(); + assert!(matches!( + client(&server, "remote-status-slow") + .activity_task_status("activity", "attempt-original", "actual-owner") + .await, + Err(Error::Timeout) + )); + assert!(start.elapsed() < Duration::from_millis(5200)); + assert_eq!(server.requests.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn cooperative_activity_refuses_execution_without_current_claim_deadlines_and_session() { + for case in [ + "remote-refused", + "remote-offline", + "remote-cancelled", + "remote-wrong-task", + "remote-wrong-attempt", + "remote-wrong-owner", + "remote-malformed", + "remote-recorded-progress", + "remote-expired", + "remote-closed-task", + "remote-closed-attempt", + "remote-closed-activity", + "remote-no-timezone", + "remote-deadline", + "remote-bad-deadlines", + "remote-session-expired", + "remote-session-replaced", + ] { + let server = remote_activity_server(); + let mut worker = coordinator_worker(&server, case); + let called = Arc::new(AtomicBool::new(false)); + let invoked = Arc::clone(&called); + worker.register_activity("work", move |_, _| { + invoked.store(true, Ordering::SeqCst); + async { Ok(Value::Null) } + }); + assert_eq!( + worker.poll_activity_once().await.unwrap(), + ManagedPollOutcome::Handled, + "{case}" + ); + assert!(!called.load(Ordering::SeqCst), "{case}"); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 2, "{case}"); + assert!(requests.last().unwrap().path.ends_with("/status"), "{case}"); + } + for case in [ + "remote-missing-owner", + "remote-missing-attempt", + "remote-invented-owner", + "remote-conflicting-attempt", + ] { + let server = remote_activity_server(); + let worker = coordinator_worker(&server, case); + assert!( + matches!( + worker.poll_activity_once().await, + Err(Error::InvalidCooperativeCancellation(_)) + ), + "{case}" + ); + assert_eq!(server.requests.lock().unwrap().len(), 1, "{case}"); + } +} + +#[tokio::test] +async fn cooperative_activity_fences_success_failure_heartbeat_and_late_context() { + for fail in [false, true] { + let server = remote_activity_server(); + let mut worker = coordinator_worker(&server, "remote-valid"); + let saved = Arc::new(Mutex::new(None::)); + let capture = Arc::clone(&saved); + worker.register_activity("work", move |ctx, _| { + *capture.lock().unwrap() = Some(ctx.clone()); + async move { + assert!( + ctx.heartbeat(json!({"completed":1})) + .await? + .heartbeat_recorded + ); + if fail { + return Err(Error::WorkerLoop("application failure".into())); + } + Ok(json!("done")) + } + }); + assert_eq!( + worker.poll_activity_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + let context = saved.lock().unwrap().as_ref().unwrap().clone(); + let before = server.requests.lock().unwrap().len(); + assert!(matches!( + context.heartbeat(json!({"late":true})).await, + Err(Error::ActivityExecutionAbandoned(_)) + )); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), before); + let paths = requests + .iter() + .map(|r| r.path.rsplit('/').next().unwrap()) + .collect::>(); + let mut expected = vec!["poll", "status", "status", "heartbeat", "status", "status"]; + if !fail { + expected.push("info"); + } + expected.push(if fail { "fail" } else { "complete" }); + assert_eq!(paths, expected); + let body: Value = serde_json::from_str(&requests.last().unwrap().body).unwrap(); + assert_eq!(body["activity_attempt_id"], "attempt-original"); + assert_eq!(body["lease_owner"], "actual-owner"); + if fail { + assert_eq!( + body["failure"]["message"], + "worker loop error: application failure" + ); + } + } +} + +#[tokio::test] +async fn cooperative_activity_discards_results_and_failure_after_attempt_replacement() { + for (case, fail) in [ + ("remote-replaced", false), + ("remote-replaced", true), + ("remote-late-result", false), + ] { + let server = remote_activity_server(); + let mut worker = coordinator_worker(&server, case); + worker.register_activity("work", move |_, _| async move { + if fail { + Err(Error::WorkerLoop("application failure".into())) + } else { + Ok(json!("late")) + } + }); + assert_eq!( + worker.poll_activity_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 3); + assert!(requests.last().unwrap().path.ends_with("/status")); + assert!(!requests.iter().any(|r| r.path.ends_with("/complete") + || r.path.ends_with("/fail") + || r.path.ends_with("/cluster/info"))); + } +} + +#[tokio::test] +async fn cooperative_activity_heartbeat_abandons_changed_context_or_lost_authority() { + for case in [ + "remote-context-changed", + "remote-heartbeat-wrong-attempt", + "remote-heartbeat-cancelled", + "remote-heartbeat-post-loss", + ] { + let server = remote_activity_server(); + let mut worker = coordinator_worker(&server, case); + let changed = case == "remote-context-changed"; + worker.register_activity("work", move |mut ctx, _| async move { + if changed { + ctx.activity_attempt_id = "replacement".into(); + } + assert!(matches!( + ctx.heartbeat(json!({"progress":1})).await, + Err(Error::ActivityExecutionAbandoned(_)) + )); + // Catching abandonment cannot restore publication authority. + Ok(json!("late")) + }); + assert_eq!( + worker.poll_activity_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + let requests = server.requests.lock().unwrap(); + assert!( + !requests + .iter() + .any(|r| r.path.ends_with("/complete") || r.path.ends_with("/fail")), + "{case}" + ); + assert_eq!( + requests + .iter() + .filter(|r| r.path.ends_with("/activity/heartbeat")) + .count(), + if changed { 0 } else { 1 } + ); + } +} + +#[tokio::test] +async fn cooperative_activity_observer_drops_a_pending_callback_and_fences_its_context() { + let server = remote_activity_server(); + let mut worker = coordinator_worker(&server, "remote-replaced"); + let saved = Arc::new(Mutex::new(None::)); + let capture = Arc::clone(&saved); + let dropped = Arc::new(AtomicBool::new(false)); + let finished = Arc::clone(&dropped); + worker.register_activity("work", move |ctx, _| { + *capture.lock().unwrap() = Some(ctx); + PendingActivityCallback(Arc::clone(&finished)) + }); + assert_eq!( + tokio::time::timeout(Duration::from_secs(3), worker.poll_activity_once()) + .await + .unwrap() + .unwrap(), + ManagedPollOutcome::Handled + ); + let context = saved.lock().unwrap().as_ref().unwrap().clone(); + assert!(dropped.load(Ordering::SeqCst)); + let before = server.requests.lock().unwrap().len(); + assert!(matches!( + context.heartbeat(Value::Null).await, + Err(Error::ActivityExecutionAbandoned(_)) + )); + assert_eq!(server.requests.lock().unwrap().len(), before); + assert_eq!(before, 3); +} + +#[tokio::test] +async fn cooperative_activity_worker_shutdown_abandons_a_pending_callback_before_deregistration() { + let server = remote_activity_server(); + let mut worker = coordinator_worker(&server, "remote-valid"); + let saved = Arc::new(Mutex::new(None::)); + let capture = Arc::clone(&saved); + let started = Arc::new(tokio::sync::Notify::new()); + let invoked = Arc::clone(&started); + let dropped = Arc::new(AtomicBool::new(false)); + let finished = Arc::clone(&dropped); + worker.register_activity("work", move |ctx, _| { + *capture.lock().unwrap() = Some(ctx); + invoked.notify_one(); + PendingActivityCallback(Arc::clone(&finished)) + }); + tokio::time::timeout(Duration::from_secs(3), worker.run_until(started.notified())) + .await + .unwrap() + .unwrap(); + let context = saved.lock().unwrap().as_ref().unwrap().clone(); + assert!(dropped.load(Ordering::SeqCst)); + let before = server.requests.lock().unwrap().len(); + assert!(matches!( + context.heartbeat(Value::Null).await, + Err(Error::ActivityExecutionAbandoned(_)) + )); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), before); + let deregistration = requests.last().unwrap(); + assert!(deregistration + .path + .ends_with("/worker/registrations/actual-owner")); + assert_eq!(deregistration.method, "DELETE"); + assert!(!requests + .iter() + .any(|r| r.path.ends_with("/complete") || r.path.ends_with("/fail"))); +} + +#[tokio::test] +async fn cooperative_activity_dropped_poll_future_abandons_cloned_context() { + let server = remote_activity_server(); + let mut worker = coordinator_worker(&server, "remote-valid"); + let saved = Arc::new(Mutex::new(None::)); + let capture = Arc::clone(&saved); + let started = Arc::new(tokio::sync::Notify::new()); + let invoked = Arc::clone(&started); + let dropped = Arc::new(AtomicBool::new(false)); + let finished = Arc::clone(&dropped); + worker.register_activity("work", move |ctx, _| { + *capture.lock().unwrap() = Some(ctx); + invoked.notify_one(); + PendingActivityCallback(Arc::clone(&finished)) + }); + let poll = tokio::spawn(async move { worker.poll_activity_once().await }); + tokio::time::timeout(Duration::from_secs(3), started.notified()) + .await + .unwrap(); + poll.abort(); + assert!(poll.await.unwrap_err().is_cancelled()); + assert!(dropped.load(Ordering::SeqCst)); + let context = saved.lock().unwrap().as_ref().unwrap().clone(); + let before = server.requests.lock().unwrap().len(); + assert!(matches!( + context.heartbeat(Value::Null).await, + Err(Error::ActivityExecutionAbandoned(_)) + )); + assert_eq!(server.requests.lock().unwrap().len(), before); +} + fn coordinator_responses(path: &str, body: &str, number: usize) -> Option<(&'static str, String)> { let case = path.split('/').nth(1).unwrap_or_default(); if path.ends_with("/cluster/info") { From 03a41c8ec0b426ce3764883d036ae4454baa54b3 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 14:30:58 +0000 Subject: [PATCH 13/58] Negotiate explicit cooperative worker registration and protocol scope --- src/cooperative_cancellation.rs | 2 +- src/lib.rs | 111 ++++++++++-- src/tests/cooperative_cancellation.rs | 240 +++++++++++++++++++++++++- 3 files changed, 335 insertions(+), 18 deletions(-) diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index 67e008d..a5f052e 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -922,7 +922,7 @@ impl WorkflowContext { } } -fn supports_protocol(version: &str) -> bool { +pub(super) fn supports_protocol(version: &str) -> bool { let Some((major, minor)) = version.split_once('.') else { return false; }; diff --git a/src/lib.rs b/src/lib.rs index d24e4ff..0526c1f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2652,6 +2652,7 @@ pub struct Client { namespace: String, max_external_payload_bytes: usize, worker_storage_admission: Option, + cooperative_worker_protocol: bool, runtime_upload_policy: Arc>, } @@ -3570,13 +3571,51 @@ impl Client { body["workflow_definition_fingerprints"] = json!(fingerprints); } - self.request_json( - reqwest::Method::POST, - "/worker/register", - RequestProtocol::Worker(WORKER_PROTOCOL_VERSION), - Some(&body), - ) - .await + let response: Value = self + .request_json( + reqwest::Method::POST, + "/worker/register", + RequestProtocol::Worker(WORKER_PROTOCOL_VERSION), + Some(&body), + ) + .await?; + if self.cooperative_worker_protocol { + let compatible = response["registered"].as_bool() == Some(true) + && response["worker_id"].as_str() == Some(worker_id) + && response["namespace"].as_str() == Some(self.namespace.as_str()) + && response["task_queue"].as_str() == Some(task_queue) + && response["protocol_version"] + .as_str() + .is_some_and(cooperative_cancellation::supports_protocol) + && response["server_capabilities"]["cooperative_cancellation"].as_bool() + == Some(true) + && response["capabilities"].as_array().is_some_and(|accepted| { + capabilities.iter().all(|capability| { + accepted + .iter() + .any(|value| value.as_str() == Some(capability.as_str())) + }) + }); + if !compatible { + let error = Error::CooperativeCancellationUnavailable( + "registration must acknowledge this namespace, queue, worker, capabilities and compatible runtime protocol 1.20".into(), + ); + if response["registered"].as_bool() == Some(true) + && response["worker_id"].as_str() == Some(worker_id) + { + if let Err(deregistration) = + self.deregister_worker_registration(worker_id).await + { + return Err(Error::WorkerShutdown { + primary: Box::new(error), + deregistration: Box::new(deregistration), + }); + } + } + return Err(error); + } + } + Ok(serde_json::from_value(response)?) } /// Gracefully remove one worker's registration through the worker plane. @@ -4152,6 +4191,18 @@ impl Client { body: Option<&B>, timeout: Duration, ) -> Result { + let protocol = match protocol { + RequestProtocol::Worker(version) + if self.cooperative_worker_protocol + && version + .strip_prefix("1.") + .and_then(|minor| minor.parse::().ok()) + .is_some_and(|minor| minor < 20) => + { + RequestProtocol::Worker("1.20") + } + protocol => protocol, + }; let auth_token = self.auth_token(protocol)?; let mut request = self .http @@ -4901,6 +4952,7 @@ impl ClientBuilder { namespace: self.namespace, max_external_payload_bytes: self.max_external_payload_bytes, worker_storage_admission: None, + cooperative_worker_protocol: false, runtime_upload_policy: Arc::new(Mutex::new([None, None])), }) } @@ -6170,8 +6222,8 @@ pub struct Worker { heartbeat_interval: Duration, retry_policy: WorkerRetryPolicy, heartbeat_observer: Option, - // Registration and callback lifetime qualification must precede activation. cooperative_cancellation_enabled: bool, + cooperative_registration_confirmed: Arc, } impl Worker { @@ -6191,11 +6243,13 @@ impl Worker { retry_policy: WorkerRetryPolicy::default(), heartbeat_observer: None, cooperative_cancellation_enabled: false, + cooperative_registration_confirmed: Arc::new(AtomicBool::new(false)), } } pub fn worker_id(mut self, worker_id: impl Into) -> Self { self.worker_id = worker_id.into(); + self.cooperative_registration_confirmed = Arc::new(AtomicBool::new(false)); self } @@ -6209,6 +6263,22 @@ impl Worker { self } + /// Opt in to separate cooperative workflow cancellation requests. + /// + /// Registration requires the Server to acknowledge this exact worker, + /// namespace, queue and capabilities with compatible protocol 1.20. Task + /// processing uses canonical delivery and activity ownership fences. + /// Existing terminal cancel and terminate operations remain terminal. + /// The default is disabled, preserving ordinary protocol 1.19 workers. + /// Call [`Worker::register`] before [`Worker::run_once`]. [`Worker::run`] + /// and [`Worker::run_until`] register automatically. + pub fn cooperative_cancellation(mut self, enabled: bool) -> Self { + self.cooperative_cancellation_enabled = enabled; + self.client.cooperative_worker_protocol = enabled; + self.cooperative_registration_confirmed = Arc::new(AtomicBool::new(false)); + self + } + /// Configure bounded retries for task-poll acquisition and worker heartbeats. pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self { self.retry_policy = policy; @@ -6788,7 +6858,10 @@ impl Worker { ); } - self.client + self.cooperative_registration_confirmed + .store(false, Ordering::SeqCst); + let response = self + .client .register_worker_with_definition_fingerprints( &self.worker_id, &self.task_queue, @@ -6801,6 +6874,8 @@ impl Worker { Some(DURABLE_SELECTION_CAPABILITY.to_string()), Some(MEMO_UPSERTS_CAPABILITY.to_string()), Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()), + self.cooperative_cancellation_enabled + .then(|| "cooperative_cancellation".to_string()), (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()), (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()), worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION) @@ -6822,7 +6897,12 @@ impl Worker { .collect(), ), ) - .await + .await?; + if self.cooperative_cancellation_enabled && response.registered { + self.cooperative_registration_confirmed + .store(true, Ordering::SeqCst); + } + Ok(response) } /// Run until shutdown or a terminal worker error occurs. @@ -6875,6 +6955,8 @@ impl Worker { } let registered_worker_id = registration.worker_id.clone(); let primary = self.run_registered_until(stop, registration).await; + self.cooperative_registration_confirmed + .store(false, Ordering::SeqCst); let deregistration = self .client .deregister_worker_registration(®istered_worker_id) @@ -7040,6 +7122,15 @@ impl Worker { /// Direct callers of [`Client::complete_workflow_task`] continue to receive /// the original [`Error::Http`] status and response body. pub async fn run_once(&self) -> Result { + if self.cooperative_cancellation_enabled + && !self + .cooperative_registration_confirmed + .load(Ordering::SeqCst) + { + return Err(Error::CooperativeCancellationUnavailable( + "register this cooperative worker before polling tasks".into(), + )); + } let worker = self.with_storage_admission(Arc::new(AtomicBool::new(false))); let mut handled = 0; match worker.poll_workflow_once().await? { diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs index f304c6b..51acd00 100644 --- a/src/tests/cooperative_cancellation.rs +++ b/src/tests/cooperative_cancellation.rs @@ -1989,16 +1989,52 @@ fn remote_activity_responses( return responses(path, body, number); } if path.ends_with("/worker/register") { - return Some(( - "200 OK", - json!({"registered":true,"worker_id":"actual-owner","heartbeat_interval_seconds":3600}) - .to_string(), - )); + let request: Value = serde_json::from_str(body).unwrap(); + let mut reply = json!({"registered":true,"worker_id":request["worker_id"], + "namespace":"caller-namespace","task_queue":request["task_queue"], + "capabilities":request["capabilities"],"heartbeat_interval_seconds":3600, + "protocol_version":"1.20","server_capabilities":{"cooperative_cancellation":true}}); + match case { + "register-old-protocol" => reply["protocol_version"] = json!("1.19"), + "register-wrong-major" => reply["protocol_version"] = json!("2.0"), + "register-malformed-protocol" => reply["protocol_version"] = json!("1.20garbage"), + "register-missing-protocol" => { + reply.as_object_mut().unwrap().remove("protocol_version"); + } + "register-unsupported" | "register-cleanup-refused" => { + reply["server_capabilities"]["cooperative_cancellation"] = json!(false) + } + "register-string-support" => { + reply["server_capabilities"]["cooperative_cancellation"] = json!("true") + } + "register-missing-support" => reply["server_capabilities"] = json!({}), + "register-wrong-worker" => reply["worker_id"] = json!("other-worker"), + "register-wrong-namespace" => reply["namespace"] = json!("other-namespace"), + "register-wrong-queue" => reply["task_queue"] = json!("other-queue"), + "register-missing-capability" => { + reply["capabilities"] = json!(["cooperative_cancellation"]) + } + "register-no-cooperative" => reply["capabilities"] + .as_array_mut() + .unwrap() + .retain(|value| value != "cooperative_cancellation"), + "register-declined" => reply["registered"] = json!(false), + "register-newer" => reply["protocol_version"] = json!("1.21"), + "register-replaced" if number >= 2 => reply["worker_id"] = json!("replacement"), + _ => {} + } + return Some(("200 OK", reply.to_string())); } if path.ends_with("/worker/heartbeat") { return Some(("200 OK", json!({}).to_string())); } if path.ends_with("/worker/registrations/actual-owner") { + if case == "register-cleanup-refused" { + return Some(( + "503 Service Unavailable", + json!({"reason":"backend_unavailable"}).to_string(), + )); + } return Some(("200 OK", json!({"worker_id":"actual-owner","outcome":"deregistered","recovered_workflow_task_count":1}).to_string())); } if path.ends_with("/activity-tasks/poll") { @@ -2102,6 +2138,195 @@ fn remote_activity_server() -> MockWorkerServer { }) } +#[tokio::test] +async fn cooperative_registration_uses_explicit_protocol_and_preserves_other_clients() { + for case in ["register-valid", "register-newer"] { + let server = remote_activity_server(); + let original = client(&server, case); + let worker = Worker::new(original.clone(), "queue") + .worker_id("actual-owner") + .cooperative_cancellation(true); + let registration = worker.register().await.unwrap(); + assert!(registration.registered); + assert!(worker + .cooperative_registration_confirmed + .load(Ordering::SeqCst)); + original + .poll_activity_task("actual-owner", "queue", Duration::ZERO) + .await + .unwrap(); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].worker_protocol.as_deref(), Some("1.20")); + assert_eq!(requests[1].worker_protocol.as_deref(), Some("1.19")); + assert_eq!( + requests[0].authorization.as_deref(), + Some("Bearer worker-only") + ); + assert_eq!(requests[0].namespace.as_deref(), Some("caller-namespace")); + let body: Value = serde_json::from_str(&requests[0].body).unwrap(); + assert!(body["capabilities"] + .as_array() + .unwrap() + .contains(&json!("cooperative_cancellation"))); + assert_eq!( + body["capability_manifest"], + portable_worker_affinity_capability_manifest() + ); + } +} + +#[tokio::test] +async fn cooperative_registration_refuses_incompatible_receipts_before_polling() { + for case in [ + "register-old-protocol", + "register-wrong-major", + "register-malformed-protocol", + "register-missing-protocol", + "register-unsupported", + "register-string-support", + "register-missing-support", + "register-wrong-worker", + "register-wrong-namespace", + "register-wrong-queue", + "register-missing-capability", + "register-no-cooperative", + "register-declined", + ] { + let server = remote_activity_server(); + let worker = coordinator_worker(&server, case); + assert!( + matches!( + worker.register().await, + Err(Error::CooperativeCancellationUnavailable(_)) + ), + "{case}" + ); + assert!( + matches!( + worker.run_once().await, + Err(Error::CooperativeCancellationUnavailable(_)) + ), + "{case}" + ); + let requests = server.requests.lock().unwrap(); + assert_eq!( + requests.len(), + if matches!(case, "register-wrong-worker" | "register-declined") { + 1 + } else { + 2 + }, + "{case}" + ); + assert!(requests[0].path.ends_with("/register")); + assert!( + !requests + .iter() + .any(|request| request.path.ends_with("/poll")), + "{case}" + ); + if requests.len() > 1 { + assert_eq!(requests[1].method, "DELETE"); + assert!(requests[1].path.ends_with("/registrations/actual-owner")); + assert_eq!(requests[1].worker_protocol.as_deref(), Some("1.20")); + } + } +} + +#[tokio::test] +async fn cooperative_registration_preserves_cleanup_failure_and_requires_registration() { + let server = remote_activity_server(); + let worker = coordinator_worker(&server, "register-cleanup-refused"); + assert!(matches!( + worker.run_once().await, + Err(Error::CooperativeCancellationUnavailable(_)) + )); + assert!(server.requests.lock().unwrap().is_empty()); + let error = worker.register().await.unwrap_err(); + assert!( + matches!(error, Error::WorkerShutdown {primary, deregistration} + if matches!(*primary, Error::CooperativeCancellationUnavailable(_)) + && matches!(*deregistration, Error::Http { status, .. } if status.as_u16() == 503)) + ); + assert!(!worker + .cooperative_registration_confirmed + .load(Ordering::SeqCst)); +} + +#[tokio::test] +async fn cooperative_registration_refusal_invalidates_previous_confirmation_and_changed_identity() { + let server = remote_activity_server(); + let worker = coordinator_worker(&server, "register-replaced"); + worker.register().await.unwrap(); + assert!(worker + .cooperative_registration_confirmed + .load(Ordering::SeqCst)); + assert!(matches!( + worker.register().await, + Err(Error::CooperativeCancellationUnavailable(_)) + )); + assert!(matches!( + worker.run_once().await, + Err(Error::CooperativeCancellationUnavailable(_)) + )); + assert_eq!(server.requests.lock().unwrap().len(), 2); + + let worker = coordinator_worker(&server, "register-valid"); + worker.register().await.unwrap(); + let changed = worker.worker_id("replacement"); + assert!(matches!( + changed.run_once().await, + Err(Error::CooperativeCancellationUnavailable(_)) + )); + assert_eq!(server.requests.lock().unwrap().len(), 3); +} + +#[tokio::test] +async fn cooperative_registration_default_and_disabled_workers_keep_ordinary_protocol() { + for explicit in [false, true] { + let server = remote_activity_server(); + let worker = + Worker::new(client(&server, "register-valid"), "queue").worker_id("actual-owner"); + let worker = if explicit { + worker + .cooperative_cancellation(true) + .cooperative_cancellation(false) + } else { + worker + }; + worker.register().await.unwrap(); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests[0].worker_protocol.as_deref(), Some("1.19")); + let body: Value = serde_json::from_str(&requests[0].body).unwrap(); + assert!(!body["capabilities"] + .as_array() + .unwrap() + .contains(&json!("cooperative_cancellation"))); + } +} + +#[tokio::test] +async fn cooperative_registration_requires_only_worker_credentials() { + let server = remote_activity_server(); + let client = Client::builder(format!("{}/register-valid", server.base_url())) + .worker_token(Some("worker-only".into())) + .namespace("caller-namespace") + .build() + .unwrap(); + let worker = Worker::new(client, "queue") + .worker_id("actual-owner") + .cooperative_cancellation(true); + worker.register().await.unwrap(); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert_eq!( + requests[0].authorization.as_deref(), + Some("Bearer worker-only") + ); + assert!(requests[0].control_protocol.is_none()); +} + struct PendingActivityCallback(Arc); impl Future for PendingActivityCallback { @@ -2606,8 +2831,9 @@ fn coordinator_server() -> MockWorkerServer { } fn coordinator_worker(server: &MockWorkerServer, case: &str) -> Worker { - let mut worker = Worker::new(client(server, case), "queue").worker_id("actual-owner"); - worker.cooperative_cancellation_enabled = true; + let mut worker = Worker::new(client(server, case), "queue") + .worker_id("actual-owner") + .cooperative_cancellation(true); worker.poll_timeout = Duration::ZERO; worker.retry_policy = WorkerRetryPolicy { max_retries: 1, From 0ca459c655433b77a8b81ee70f0e0e0cf79c2e3a Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 14:51:03 +0000 Subject: [PATCH 14/58] Qualify cooperative Rust Workers against exact Server candidates --- .github/workflows/ci.yml | 76 ++++++++- docker-compose.cooperative.yml | 71 ++++++++ tests/cooperative_server.rs | 289 +++++++++++++++++++++++++++++++++ 3 files changed, 435 insertions(+), 1 deletion(-) create mode 100644 docker-compose.cooperative.yml create mode 100644 tests/cooperative_server.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38c9c7a..e8826ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,15 @@ on: push: branches: [main] workflow_dispatch: + inputs: + cooperative_qualification: + description: Run cooperative protocol 1.20 cases against an isolated Server candidate + type: boolean + default: false + server_commit: + description: Exact 40-character public Server candidate SHA + type: string + default: "" permissions: contents: read @@ -181,10 +190,70 @@ jobs: python scripts/ci/test-release-package.py python scripts/ci/test-example-base-urls.py + cooperative-server: + name: Cooperative Server source qualification + if: ${{ github.event_name == 'workflow_dispatch' && inputs.cooperative_qualification }} + needs: [action-policy, test] + runs-on: ubuntu-latest + timeout-minutes: 25 + env: + COMPOSE_PROJECT_NAME: rust-cooperative-${{ github.run_id }}-${{ github.run_attempt }} + SERVER_CANDIDATE_SHA: ${{ inputs.server_commit }} + DURABLE_WORKFLOW_SERVER_SOURCE: ${{ github.workspace }}/.cooperative-server-source + DURABLE_WORKFLOW_AUTH_TOKEN: test-token + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Require an exact Server candidate + run: '[[ "$SERVER_CANDIDATE_SHA" =~ ^[0-9a-f]{40}$ ]]' + - name: Check out the public Server candidate + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: durable-workflow/server + ref: ${{ inputs.server_commit }} + path: .cooperative-server-source + persist-credentials: false + - name: Verify checked-out candidate identity + run: test "$(git -C .cooperative-server-source rev-parse HEAD)" = "$SERVER_CANDIDATE_SHA" + - name: Install Rust 1.86 + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable + with: + toolchain: 1.86.0 + - name: Start isolated Server stack + run: docker compose -f docker-compose.cooperative.yml up -d --build --wait --wait-timeout 300 + - name: Retain runtime identity and select endpoint + run: | + docker compose -f docker-compose.cooperative.yml exec -T server cat /app/.package-provenance > cooperative-package-provenance.txt + docker compose -f docker-compose.cooperative.yml images --format json > cooperative-images.json + endpoint="$(docker compose -f docker-compose.cooperative.yml port server 8080)" + echo "DURABLE_WORKFLOW_SERVER_URL=http://$endpoint" >> "$GITHUB_ENV" + - name: Run actual Worker cooperative scenarios + run: cargo test --test cooperative_server -- --ignored --nocapture --test-threads=1 2>&1 | tee cooperative-results.log + - name: Retain bounded scenario evidence + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: cooperative-server-${{ github.sha }} + path: | + cooperative-results.log + cooperative-package-provenance.txt + cooperative-images.json + retention-days: 7 + if-no-files-found: warn + - name: Emit failure diagnostics + if: ${{ failure() }} + run: docker compose -f docker-compose.cooperative.yml logs --no-color --tail 100 bootstrap server worker + - name: Remove task runtime resources + if: ${{ always() }} + run: | + docker compose -f docker-compose.cooperative.yml down -v + docker image rm "dw-rust-cooperative-server:$SERVER_CANDIDATE_SHA" + target-branch-qualification: name: Target branch qualification if: ${{ always() }} - needs: [action-policy, regression-corpus, test, protocol, docs, package] + needs: [action-policy, regression-corpus, test, protocol, docs, package, cooperative-server] runs-on: ubuntu-latest timeout-minutes: 2 steps: @@ -200,9 +269,14 @@ jobs: PACKAGE_RESULT: ${{ needs.package.result }} PROTOCOL_RESULT: ${{ needs.protocol.result }} TEST_RESULT: ${{ needs.test.result }} + COOPERATIVE_REQUESTED: ${{ inputs.cooperative_qualification }} + COOPERATIVE_RESULT: ${{ needs.cooperative-server.result }} run: | test "$CORPUS_RESULT" = success test "$DOCS_RESULT" = success test "$PACKAGE_RESULT" = success test "$PROTOCOL_RESULT" = success test "$TEST_RESULT" = success + if [ "$COOPERATIVE_REQUESTED" = true ]; then + test "$COOPERATIVE_RESULT" = success + fi diff --git a/docker-compose.cooperative.yml b/docker-compose.cooperative.yml new file mode 100644 index 0000000..a7dc1d9 --- /dev/null +++ b/docker-compose.cooperative.yml @@ -0,0 +1,71 @@ +services: + bootstrap: + image: dw-rust-cooperative-server:${SERVER_CANDIDATE_SHA:-candidate} + build: + context: ${DURABLE_WORKFLOW_SERVER_SOURCE:-../server} + command: ["server-bootstrap"] + environment: &runtime-env + APP_ENV: testing + APP_KEY: "base64:dGVzdGluZy1rZXktMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMg==" + APP_DEBUG: "false" + DB_CONNECTION: mysql + DB_HOST: mysql + DB_DATABASE: durable_workflow + DB_USERNAME: workflow + DB_PASSWORD: workflow + REDIS_HOST: redis + QUEUE_CONNECTION: redis + CACHE_STORE: redis + WORKFLOW_SERVER_AUTH_DRIVER: token + WORKFLOW_SERVER_AUTH_TOKEN: test-token + DW_WORKFLOW_TASK_TIMEOUT: 10 + DW_WORKER_PROTOCOL_VERSION: "1.20" + depends_on: + mysql: + condition: service_healthy + redis: + condition: service_healthy + server: + image: dw-rust-cooperative-server:${SERVER_CANDIDATE_SHA:-candidate} + build: + context: ${DURABLE_WORKFLOW_SERVER_SOURCE:-../server} + environment: *runtime-env + ports: + - "127.0.0.1::8080" + depends_on: + bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/api/health"] + interval: 5s + timeout: 3s + retries: 12 + worker: + image: dw-rust-cooperative-server:${SERVER_CANDIDATE_SHA:-candidate} + build: + context: ${DURABLE_WORKFLOW_SERVER_SOURCE:-../server} + command: ["php", "artisan", "queue:work", "--sleep=1", "--tries=3", "--max-time=1800"] + environment: *runtime-env + depends_on: + server: + condition: service_healthy + mysql: + image: mysql:8.0 + environment: + MYSQL_DATABASE: durable_workflow + MYSQL_USER: workflow + MYSQL_PASSWORD: workflow + MYSQL_ROOT_PASSWORD: root + tmpfs: ["/var/lib/mysql"] + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] + interval: 3s + timeout: 2s + retries: 20 + redis: + image: redis:7-alpine + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 2s + retries: 10 diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs new file mode 100644 index 0000000..9dd125c --- /dev/null +++ b/tests/cooperative_server.rs @@ -0,0 +1,289 @@ +//! Explicit source qualification against an isolated Server protocol 1.20 stack. +//! Ordinary cargo tests leave these cases ignored. They require a real runtime. + +use durable_workflow::{ + json, Client, CooperativeCancellationOptions, Error, Value, Worker, WorkflowHandle, + WorkflowResultOptions, +}; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; + +const WORKFLOW: &str = "tests.rust-cooperative-timer"; +const UNDO: &str = "tests.rust-cooperative-undo"; + +fn client() -> Client { + Client::builder(std::env::var("DURABLE_WORKFLOW_SERVER_URL").expect("isolated Server URL")) + .token(Some( + std::env::var("DURABLE_WORKFLOW_AUTH_TOKEN").expect("isolated Server token"), + )) + .namespace("default") + .build() + .unwrap() +} + +fn queue() -> String { + format!("rust-cooperative-{}", durable_workflow::Uuid::new_v4()) +} + +fn worker(client: &Client, queue: &str, saga: bool, prefix: bool) -> Worker { + let mut worker = Worker::new(client.clone(), queue) + .worker_id(format!("{queue}-{}", durable_workflow::Uuid::new_v4())) + .cooperative_cancellation(true) + .poll_timeout(Duration::from_secs(1)); + worker.register_workflow(WORKFLOW, move |ctx, _| async move { + if prefix { + assert_eq!(ctx.side_effect(|| json!(7))?, json!(7)); + } + if saga { + let mut saga = ctx.saga(); + saga.add_compensation(UNDO, json!([]))?; + let result = ctx.sleep(Duration::from_secs(300)).await; + saga.finish(result).await?; + } else { + ctx.sleep(Duration::from_secs(300)).await?; + } + Ok(Value::Null) + }); + worker +} + +async fn history(handle: &WorkflowHandle) -> Value { + let url = std::env::var("DURABLE_WORKFLOW_SERVER_URL").unwrap(); + let token = std::env::var("DURABLE_WORKFLOW_AUTH_TOKEN").unwrap(); + let response = reqwest::Client::new() + .get(format!( + "{url}/api/workflows/{}/runs/{}/history", + handle.workflow_id, + handle.run_id.as_deref().unwrap() + )) + .query(&[("page_size", "1000")]) + .bearer_auth(token) + .header("X-Namespace", "default") + .header("X-Durable-Workflow-Control-Plane-Version", "2") + .send() + .await + .unwrap(); + let status = response.status(); + let body: Value = response.json().await.unwrap(); + assert!(status.is_success(), "history {status}: {body}"); + assert!( + body["next_page_token"].is_null(), + "bounded scenario history was paginated" + ); + body +} + +fn count(history: &Value, kind: &str) -> usize { + history["events"] + .as_array() + .unwrap() + .iter() + .filter(|event| event["event_type"] == kind) + .count() +} + +async fn tick_until(worker: &Worker, handle: &WorkflowHandle, kind: &str) -> Value { + let mut last_snapshot = Value::Null; + let observed = tokio::time::timeout(Duration::from_secs(30), async { + loop { + worker.run_once().await.expect("actual Worker tick"); + let snapshot = history(handle).await; + if count(&snapshot, kind) > 0 { + return snapshot; + } + last_snapshot = snapshot; + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await; + observed.unwrap_or_else(|_| { + panic!( + "expected {kind} within 30s for {}/{:?}, last history: {last_snapshot}", + handle.workflow_id, handle.run_id + ) + }) +} + +async fn assert_cancelled(handle: &WorkflowHandle, request_id: &str, cleanup: bool) -> Value { + let result = handle + .result_selected_run(WorkflowResultOptions { + timeout: Duration::from_secs(5), + poll_interval: Duration::from_millis(50), + }) + .await; + assert!( + matches!(result, Err(Error::WorkflowCancelled(_))), + "terminal result: {result:?}" + ); + let snapshot = history(handle).await; + for kind in [ + "CooperativeCancellationRequested", + "CooperativeCancellationDelivered", + "WorkflowCancelled", + ] { + assert_eq!(count(&snapshot, kind), 1, "{kind}: {snapshot}"); + let event = snapshot["events"] + .as_array() + .unwrap() + .iter() + .find(|event| event["event_type"] == kind) + .unwrap(); + assert_eq!( + event["payload"]["workflow_command_id"], request_id, + "{kind}: {event}" + ); + } + for kind in [ + "WorkflowCompleted", + "WorkflowFailed", + "ActivityFailed", + "ActivityTimedOut", + ] { + assert_eq!(count(&snapshot, kind), 0, "{kind}: {snapshot}"); + } + assert_eq!(count(&snapshot, "ActivityCompleted"), usize::from(cleanup)); + snapshot +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_request_before_claim_replays_canonical_typed_cancellation() { + let client = client(); + let queue = queue(); + let worker = worker(&client, &queue, false, false); + worker.register().await.unwrap(); + let handle = client + .start_workflow(WORKFLOW, &queue, &queue, json!([])) + .await + .unwrap(); + let request = handle + .request_cancellation(CooperativeCancellationOptions::default()) + .await + .unwrap(); + tick_until(&worker, &handle, "WorkflowCancelled").await; + assert_cancelled(&handle, &request.cancellation_request.request_id, false).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_waiting_timer_observes_the_original_request_and_delivery() { + let client = client(); + let queue = queue(); + let worker = worker(&client, &queue, false, false); + worker.register().await.unwrap(); + let handle = client + .start_workflow(WORKFLOW, &queue, &queue, json!([])) + .await + .unwrap(); + tick_until(&worker, &handle, "TimerScheduled").await; + let request = handle + .request_cancellation(CooperativeCancellationOptions::default()) + .await + .unwrap(); + tick_until(&worker, &handle, "WorkflowCancelled").await; + assert_cancelled(&handle, &request.cancellation_request.request_id, false).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_duplicate_request_preserves_identity_and_cleanup_deadline() { + let client = client(); + let queue = queue(); + let worker = worker(&client, &queue, false, false); + worker.register().await.unwrap(); + let handle = client + .start_workflow(WORKFLOW, &queue, &queue, json!([])) + .await + .unwrap(); + let first = handle + .request_cancellation(CooperativeCancellationOptions { + cleanup_timeout_seconds: Some(60), + reason: Some("first".into()), + }) + .await + .unwrap(); + let duplicate = handle + .request_cancellation(CooperativeCancellationOptions { + cleanup_timeout_seconds: Some(120), + reason: Some("duplicate".into()), + }) + .await + .unwrap(); + assert!(duplicate.duplicate); + assert_eq!(first.cancellation_request, duplicate.cancellation_request); + tick_until(&worker, &handle, "WorkflowCancelled").await; + assert_cancelled(&handle, &first.cancellation_request.request_id, false).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_commits_earlier_side_effect_before_cancellation_delivery() { + let client = client(); + let queue = queue(); + let original = worker(&client, &queue, false, true); + original.register().await.unwrap(); + let handle = client + .start_workflow(WORKFLOW, &queue, &queue, json!([])) + .await + .unwrap(); + let request = handle + .request_cancellation(CooperativeCancellationOptions::default()) + .await + .unwrap(); + let prefix = tick_until(&original, &handle, "SideEffectRecorded").await; + assert_eq!(count(&prefix, "CooperativeCancellationDelivered"), 0); + drop(original); + let successor = worker(&client, &queue, false, true); + successor.register().await.unwrap(); + tick_until(&successor, &handle, "WorkflowCancelled").await; + let snapshot = assert_cancelled(&handle, &request.cancellation_request.request_id, false).await; + assert_eq!(count(&snapshot, "SideEffectRecorded"), 1); + let delivery = snapshot["events"] + .as_array() + .unwrap() + .iter() + .find(|event| event["event_type"] == "CooperativeCancellationDelivered") + .unwrap(); + assert_eq!(delivery["payload"]["sequence"], 2); +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_cold_successor_runs_saga_cleanup_before_terminal_cancellation() { + let client = client(); + let queue = queue(); + let original = worker(&client, &queue, true, false); + original.register().await.unwrap(); + let handle = client + .start_workflow(WORKFLOW, &queue, &queue, json!([])) + .await + .unwrap(); + let request = handle + .request_cancellation(CooperativeCancellationOptions::default()) + .await + .unwrap(); + let pending = tick_until(&original, &handle, "ActivityScheduled").await; + assert_eq!(count(&pending, "ActivityCompleted"), 0); + assert_eq!(count(&pending, "WorkflowCancelled"), 0); + drop(original); + let mut successor = worker(&client, &queue, true, false); + let completed = Arc::new(AtomicUsize::new(0)); + let called = Arc::clone(&completed); + successor.register_activity(UNDO, move |ctx, _| { + let called = Arc::clone(&called); + async move { + ctx.heartbeat(json!({"cleanup":true})).await?; + called.fetch_add(1, Ordering::SeqCst); + Ok(Value::Null) + } + }); + successor.register().await.unwrap(); + tick_until(&successor, &handle, "WorkflowCancelled").await; + assert_cancelled(&handle, &request.cancellation_request.request_id, true).await; + assert_eq!(completed.load(Ordering::SeqCst), 1); +} From dd8a2d5d92ea622e3acacabaa27add2af2196536 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 15:07:00 +0000 Subject: [PATCH 15/58] Qualify managed Rust callback fencing and cleanup capacity --- tests/cooperative_server.rs | 185 +++++++++++++++++++++++++++++++++++- 1 file changed, 183 insertions(+), 2 deletions(-) diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index 9dd125c..141f96d 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -2,8 +2,8 @@ //! Ordinary cargo tests leave these cases ignored. They require a real runtime. use durable_workflow::{ - json, Client, CooperativeCancellationOptions, Error, Value, Worker, WorkflowHandle, - WorkflowResultOptions, + json, ActivityContext, Client, CooperativeCancellationOptions, Error, Value, Worker, + WorkflowHandle, WorkflowResultOptions, }; use std::{ sync::{ @@ -15,6 +15,15 @@ use std::{ const WORKFLOW: &str = "tests.rust-cooperative-timer"; const UNDO: &str = "tests.rust-cooperative-undo"; +const BLOCKED: &str = "tests.rust-cooperative-blocked"; + +struct CallbackDrop(Arc); + +impl Drop for CallbackDrop { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} fn client() -> Client { Client::builder(std::env::var("DURABLE_WORKFLOW_SERVER_URL").expect("isolated Server URL")) @@ -287,3 +296,175 @@ async fn server_cold_successor_runs_saga_cleanup_before_terminal_cancellation() assert_cancelled(&handle, &request.cancellation_request.request_id, true).await; assert_eq!(completed.load(Ordering::SeqCst), 1); } + +async fn managed_remote_cancellation(user_heartbeat: bool) { + let client = client(); + let queue = queue(); + let dropped = Arc::new(AtomicUsize::new(0)); + let completed = Arc::new(AtomicUsize::new(0)); + let (entered_tx, mut entered_rx) = tokio::sync::mpsc::unbounded_channel::(); + let (heartbeat_tx, mut heartbeat_rx) = tokio::sync::mpsc::unbounded_channel(); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let mut worker = Worker::new(client.clone(), &queue) + .worker_id(format!("{queue}-managed")) + .cooperative_cancellation(true) + .max_concurrent_workflow_tasks(1) + .max_concurrent_activity_tasks(1) + .poll_timeout(Duration::from_secs(1)) + .on_worker_heartbeat(move |observation| { + if observation.acknowledgement.acknowledged { + let _ = heartbeat_tx.send(()); + } + }); + worker.register_workflow(WORKFLOW, |ctx, _| async move { + let mut saga = ctx.saga(); + saga.add_compensation(UNDO, json!([]))?; + let result = ctx.activity(BLOCKED, json!([])).await; + saga.finish(result).await?; + Ok(Value::Null) + }); + let callback_dropped = Arc::clone(&dropped); + worker.register_activity(BLOCKED, move |ctx, _| { + let entered_tx = entered_tx.clone(); + let callback_dropped = Arc::clone(&callback_dropped); + async move { + let _drop = CallbackDrop(callback_dropped); + if user_heartbeat { + ctx.heartbeat(json!({"qualification":"blocked"})).await?; + } + entered_tx.send(ctx.clone()).unwrap(); + if user_heartbeat { + loop { + tokio::time::sleep(Duration::from_millis(100)).await; + ctx.heartbeat(json!({"qualification":"blocked"})).await?; + } + } else { + std::future::pending::>().await + } + } + }); + let cleanup_completed = Arc::clone(&completed); + worker.register_activity(UNDO, move |ctx, _| { + let cleanup_completed = Arc::clone(&cleanup_completed); + async move { + ctx.heartbeat(json!({"qualification":"cleanup"})).await?; + cleanup_completed.fetch_add(1, Ordering::SeqCst); + Ok(Value::Null) + } + }); + let run = tokio::spawn(async move { + worker + .run_until(async { + let _ = shutdown_rx.await; + }) + .await + }); + tokio::time::timeout(Duration::from_secs(10), heartbeat_rx.recv()) + .await + .unwrap() + .expect("actual accepted worker heartbeat"); + let handle = client + .start_workflow(WORKFLOW, &queue, &queue, json!([])) + .await + .unwrap(); + let original = tokio::time::timeout(Duration::from_secs(10), entered_rx.recv()) + .await + .unwrap() + .expect("actual blocked callback entered"); + let request = handle + .request_cancellation(CooperativeCancellationOptions::default()) + .await + .unwrap(); + let result = handle + .result_selected_run(WorkflowResultOptions { + timeout: Duration::from_secs(20), + poll_interval: Duration::from_millis(50), + }) + .await; + assert!( + matches!(result, Err(Error::WorkflowCancelled(_))), + "managed result: {result:?}" + ); + let snapshot = assert_cancelled(&handle, &request.cancellation_request.request_id, true).await; + assert_eq!( + dropped.load(Ordering::SeqCst), + 1, + "blocked callback future must drop" + ); + assert_eq!( + completed.load(Ordering::SeqCst), + 1, + "cleanup runs once after freeing its only activity slot" + ); + assert_eq!(count(&snapshot, "ActivityCancelled"), 1); + let original_progress = snapshot["events"] + .as_array() + .unwrap() + .iter() + .filter(|event| { + event["event_type"] == "ActivityHeartbeatRecorded" + && event["payload"]["activity_type"] == BLOCKED + }) + .count(); + assert_eq!( + original_progress > 0, + user_heartbeat, + "readonly checks must not record progress" + ); + assert!( + matches!( + original.heartbeat(json!({"late":true})).await, + Err(Error::ActivityExecutionAbandoned(_)) + ), + "cloned context remains abandoned" + ); + let completion = client + .complete_activity_task( + &original.task_id, + &original.activity_attempt_id, + &original.lease_owner, + json!("late"), + "avro", + ) + .await; + assert!( + matches!(completion, Err(Error::ActivityTaskRejected(rejection)) if rejection.status == 409), + "Server must reject the cancelled attempt's late completion" + ); + let failure = client + .fail_activity_task( + &original.task_id, + &original.activity_attempt_id, + &original.lease_owner, + "late failure", + true, + ) + .await; + assert!( + matches!(failure, Err(Error::ActivityTaskRejected(rejection)) if rejection.status == 409), + "Server must reject the cancelled attempt's late failure" + ); + assert_eq!( + snapshot, + history(&handle).await, + "late publication changed canonical history" + ); + shutdown_tx.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(10), run) + .await + .unwrap() + .unwrap() + .unwrap(); +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_managed_worker_fences_blocked_callback_without_user_heartbeats() { + managed_remote_cancellation(false).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_managed_worker_fences_blocked_callback_with_user_heartbeats() { + managed_remote_cancellation(true).await; +} From f6beafac9c0890c476db273cc1dfcaceea27e0c9 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 15:11:32 +0000 Subject: [PATCH 16/58] Read managed heartbeat acknowledgment from its JSON envelope --- tests/cooperative_server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index 141f96d..c27eb6a 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -312,7 +312,7 @@ async fn managed_remote_cancellation(user_heartbeat: bool) { .max_concurrent_activity_tasks(1) .poll_timeout(Duration::from_secs(1)) .on_worker_heartbeat(move |observation| { - if observation.acknowledgement.acknowledged { + if observation.acknowledgement["acknowledged"] == true { let _ = heartbeat_tx.send(()); } }); From c8abb391ed86ea6a5a6c4c86435ab621f89d17d8 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 15:47:37 +0000 Subject: [PATCH 17/58] Qualify reopened cancellation waits against the Server --- tests/cooperative_server.rs | 202 +++++++++++++++++++++++++++++++++++- 1 file changed, 198 insertions(+), 4 deletions(-) diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index c27eb6a..a0aadf5 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -2,8 +2,9 @@ //! Ordinary cargo tests leave these cases ignored. They require a real runtime. use durable_workflow::{ - json, ActivityContext, Client, CooperativeCancellationOptions, Error, Value, Worker, - WorkflowHandle, WorkflowResultOptions, + json, ActivityContext, Client, ConditionWaitOptions, CooperativeCancellationOptions, Error, + ParallelOperation, ParallelResult, SelectionKey, Value, Worker, WorkflowHandle, + WorkflowResultOptions, }; use std::{ sync::{ @@ -16,6 +17,7 @@ use std::{ const WORKFLOW: &str = "tests.rust-cooperative-timer"; const UNDO: &str = "tests.rust-cooperative-undo"; const BLOCKED: &str = "tests.rust-cooperative-blocked"; +const REPLAY: &str = "tests.rust-cooperative-replay"; struct CallbackDrop(Arc); @@ -97,12 +99,21 @@ fn count(history: &Value, kind: &str) -> usize { } async fn tick_until(worker: &Worker, handle: &WorkflowHandle, kind: &str) -> Value { + tick_until_count(worker, handle, kind, 1).await +} + +async fn tick_until_count( + worker: &Worker, + handle: &WorkflowHandle, + kind: &str, + target: usize, +) -> Value { let mut last_snapshot = Value::Null; let observed = tokio::time::timeout(Duration::from_secs(30), async { loop { worker.run_once().await.expect("actual Worker tick"); let snapshot = history(handle).await; - if count(&snapshot, kind) > 0 { + if count(&snapshot, kind) >= target { return snapshot; } last_snapshot = snapshot; @@ -112,7 +123,7 @@ async fn tick_until(worker: &Worker, handle: &WorkflowHandle, kind: &str) -> Val .await; observed.unwrap_or_else(|_| { panic!( - "expected {kind} within 30s for {}/{:?}, last history: {last_snapshot}", + "expected {target} {kind} within 30s for {}/{:?}, last history: {last_snapshot}", handle.workflow_id, handle.run_id ) }) @@ -468,3 +479,186 @@ async fn server_managed_worker_fences_blocked_callback_without_user_heartbeats() async fn server_managed_worker_fences_blocked_callback_with_user_heartbeats() { managed_remote_cancellation(true).await; } + +fn replay_worker(client: &Client, queue: &str, mode: &'static str) -> Worker { + let mut worker = Worker::new(client.clone(), queue) + .worker_id(format!("{queue}-{}", durable_workflow::Uuid::new_v4())) + .cooperative_cancellation(true) + .poll_timeout(Duration::from_secs(1)); + worker.register_workflow(REPLAY, move |ctx, _| async move { + let predicate_ctx = ctx.clone(); + let condition = || ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"); + match mode { + "condition" => { + ctx.wait_condition(condition(), move || { + Ok(predicate_ctx.signals("vote")?.len() >= 2) + }) + .await?; + } + "parallel" => { + ctx.parallel(vec![ + ParallelOperation::timer(Duration::from_secs(300)), + ParallelOperation::group(vec![ + ParallelOperation::signal("never"), + ParallelOperation::condition(condition(), move || { + Ok(predicate_ctx.signals("vote")?.len() >= 2) + }), + ]), + ]) + .await?; + } + "selection" => { + ctx.select_keyed(vec![ + ("timer", ParallelOperation::timer(Duration::from_secs(300))), + ( + "votes", + ParallelOperation::condition(condition(), move || { + Ok(predicate_ctx.signals("vote")?.len() >= 2) + }), + ), + ]) + .await?; + } + "winner" => { + let selected = ctx + .select_keyed(vec![ + ("slow", ParallelOperation::signal("never")), + ("fast", ParallelOperation::timer(Duration::from_secs(1))), + ]) + .await?; + assert_eq!(selected.key, SelectionKey::Name("fast".into())); + let slow = selected + .handle(&SelectionKey::Name("slow".into())) + .unwrap() + .clone(); + assert_eq!(selected.into_result()?, ParallelResult::Timer); + slow.await_result().await?; + } + _ => panic!("unknown fixture mode"), + } + Ok(Value::Null) + }); + worker +} + +async fn reopened_condition_cancellation(mode: &'static str, span: u64) { + let client = client(); + let queue = queue(); + let original = replay_worker(&client, &queue, mode); + original.register().await.unwrap(); + let handle = client + .start_workflow(REPLAY, &queue, &queue, json!([])) + .await + .unwrap(); + let first = tick_until(&original, &handle, "ConditionWaitOpened").await; + assert_eq!(count(&first, "ConditionWaitOpened"), 1); + handle + .signal_selected_run("vote", json!(["first"])) + .await + .unwrap(); + let reopened = tick_until_count(&original, &handle, "ConditionWaitOpened", 2).await; + assert_eq!(count(&reopened, "ConditionWaitOpened"), 2); + assert_eq!(count(&reopened, "ConditionWaitSatisfied"), 1); + assert_eq!(count(&reopened, "SelectionResolved"), 0); + let opens = reopened["events"] + .as_array() + .unwrap() + .iter() + .filter(|event| event["event_type"] == "ConditionWaitOpened") + .collect::>(); + assert_eq!( + opens[0]["payload"]["condition_wait_occurrence_id"], + opens[1]["payload"]["condition_wait_occurrence_id"] + ); + assert_ne!( + opens[0]["payload"]["sequence"], + opens[1]["payload"]["sequence"] + ); + let pending_sequence = opens[1]["payload"]["sequence"].as_u64().unwrap(); + let request = handle + .request_cancellation(CooperativeCancellationOptions::default()) + .await + .unwrap(); + drop(original); + let successor = replay_worker(&client, &queue, mode); + successor.register().await.unwrap(); + tick_until(&successor, &handle, "WorkflowCancelled").await; + let snapshot = assert_cancelled(&handle, &request.cancellation_request.request_id, false).await; + assert_eq!(count(&snapshot, "ConditionWaitOpened"), 2); + assert_eq!(count(&snapshot, "SelectionResolved"), 0); + let delivery = snapshot["events"] + .as_array() + .unwrap() + .iter() + .find(|event| event["event_type"] == "CooperativeCancellationDelivered") + .unwrap(); + assert_eq!( + delivery["payload"]["sequence_span"].as_u64().unwrap_or(1), + span + ); + if mode == "condition" { + assert_eq!(delivery["payload"]["call_kind"], "condition"); + assert_eq!(delivery["payload"]["sequence"], pending_sequence); + } else { + assert_eq!(delivery["payload"]["call_kind"], "parallel"); + assert_eq!(delivery["payload"]["sequence"], 1); + } +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_cold_condition_delivery_uses_the_pending_physical_reopen() { + reopened_condition_cancellation("condition", 1).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_cold_nested_parallel_cancellation_replays_reopened_condition() { + reopened_condition_cancellation("parallel", 3).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_cold_selection_cancellation_replays_reopened_condition() { + reopened_condition_cancellation("selection", 2).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_cold_selection_preserves_committed_winner_before_loser_cancellation() { + let client = client(); + let queue = queue(); + let original = replay_worker(&client, &queue, "winner"); + original.register().await.unwrap(); + let handle = client + .start_workflow(REPLAY, &queue, &queue, json!([])) + .await + .unwrap(); + let winner = tick_until(&original, &handle, "SelectionResolved").await; + assert_eq!(count(&winner, "SelectionResolved"), 1); + assert_eq!(count(&winner, "WorkflowCompleted"), 0); + let request = handle + .request_cancellation(CooperativeCancellationOptions::default()) + .await + .unwrap(); + drop(original); + let successor = replay_worker(&client, &queue, "winner"); + successor.register().await.unwrap(); + tick_until(&successor, &handle, "WorkflowCancelled").await; + let snapshot = assert_cancelled(&handle, &request.cancellation_request.request_id, false).await; + assert_eq!(count(&snapshot, "SelectionResolved"), 1); + let delivery = snapshot["events"] + .as_array() + .unwrap() + .iter() + .find(|event| event["event_type"] == "CooperativeCancellationDelivered") + .unwrap(); + assert_eq!(delivery["payload"]["call_kind"], "selection_handle"); + assert_eq!(delivery["payload"]["operation_sequence"], 1); + assert_eq!( + delivery["payload"]["operation_sequence_span"] + .as_u64() + .unwrap_or(1), + 1 + ); +} From 78bb07eff0cfe4b7cc56b7b5ad5049da0849f379 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 15:54:29 +0000 Subject: [PATCH 18/58] Propagate connected Cargo failures through retained logs --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8826ed..8c1d025 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -229,6 +229,7 @@ jobs: endpoint="$(docker compose -f docker-compose.cooperative.yml port server 8080)" echo "DURABLE_WORKFLOW_SERVER_URL=http://$endpoint" >> "$GITHUB_ENV" - name: Run actual Worker cooperative scenarios + shell: bash run: cargo test --test cooperative_server -- --ignored --nocapture --test-threads=1 2>&1 | tee cooperative-results.log - name: Retain bounded scenario evidence if: ${{ always() }} From f5758d5ad2373b20fc70f6859781411584552dc8 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 18:14:59 +0000 Subject: [PATCH 19/58] Check authored wait identity and delivery ordering after cancellation prefix work --- tests/cooperative_server.rs | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index a0aadf5..a022317 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -584,7 +584,35 @@ async fn reopened_condition_cancellation(mode: &'static str, span: u64) { successor.register().await.unwrap(); tick_until(&successor, &handle, "WorkflowCancelled").await; let snapshot = assert_cancelled(&handle, &request.cancellation_request.request_id, false).await; - assert_eq!(count(&snapshot, "ConditionWaitOpened"), 2); + let events = snapshot["events"].as_array().unwrap(); + let delivery_index = events + .iter() + .position(|event| event["event_type"] == "CooperativeCancellationDelivered") + .unwrap(); + // Recorded command-prefix work can reopen a grouped physical wait before + // delivery. Cancellation must retain its authored identity and prevent any + // new wait after the canonical delivery boundary. + assert!(count(&snapshot, "ConditionWaitOpened") >= 2); + for event in events + .iter() + .filter(|event| event["event_type"] == "ConditionWaitOpened") + { + assert_eq!( + event["payload"]["condition_wait_occurrence_id"], + opens[0]["payload"]["condition_wait_occurrence_id"], + "physical prefix reopen changed authored identity: {snapshot}" + ); + } + assert!( + events[delivery_index + 1..] + .iter() + .all(|event| event["event_type"] != "ConditionWaitOpened"), + "condition reopened after cancellation delivery: {snapshot}" + ); + eprintln!("cooperative reopened condition {mode}: {snapshot}"); + if mode == "condition" { + assert_eq!(count(&snapshot, "ConditionWaitOpened"), 2); + } assert_eq!(count(&snapshot, "SelectionResolved"), 0); let delivery = snapshot["events"] .as_array() From 4f657b53d63bbe872505c0d3ce5d517f1eebf164 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 18:25:24 +0000 Subject: [PATCH 20/58] Fence the latest physical condition at grouped cancellation delivery --- src/cooperative_cancellation.rs | 46 ++++++++++++- src/tests/cooperative_cancellation.rs | 65 +++++++++++++++++ tests/cooperative_server.rs | 10 +-- .../cooperative-grouped-condition-reopen.json | 69 +++++++++++++++++++ 4 files changed, 181 insertions(+), 9 deletions(-) create mode 100644 tests/fixtures/replay-regressions/cooperative-grouped-condition-reopen.json diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index a5f052e..b943ba3 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -291,6 +291,7 @@ pub struct CancellationHistory { pub request_index: usize, pub delivery_index: Option, resolved_before_request: BTreeSet, + latest_condition_sequences_before_request: BTreeMap, failed_before_request: BTreeSet, selected_before_request: BTreeSet<(u64, u64)>, } @@ -331,6 +332,32 @@ impl CancellationHistory { original, }, ); + // Physical condition reopens retain the group's authored identity + // but can lie beyond its original sequence span. Bind unresolved + // reopens as well, so replay consumes the latest wait at delivery. + for command in &mut commands[index + 1..] { + let RecordedCommand::ConditionWait { + sequence, + parallel_group_path: Some(path), + .. + } = command + else { + continue; + }; + if self.resolved_before_request.contains(sequence) + || !path.first().is_some_and(|group| { + group.parallel_group_base_sequence == delivery.sequence + && group.parallel_group_size as u64 == delivery.sequence_span + }) + { + continue; + } + *command = RecordedCommand::CancellationBoundary { + sequence: *sequence, + call_kind: CancellationCallKind::Condition, + original: Some(Box::new(command.clone())), + }; + } return Ok(commands); } if !matches!( @@ -421,6 +448,7 @@ impl CancellationHistory { request_index: events.len(), delivery_index: None, resolved_before_request: BTreeSet::new(), + latest_condition_sequences_before_request: BTreeMap::new(), failed_before_request: BTreeSet::new(), selected_before_request: BTreeSet::new(), }; @@ -502,6 +530,7 @@ impl CancellationHistory { if state.request.is_none() { return Ok(state); } + let mut condition_occurrences = BTreeMap::new(); for event in &events[..state.request_index] { if matches!( event.event_type.as_str(), @@ -529,6 +558,14 @@ impl CancellationHistory { else { continue; }; + if event.event_type == "ConditionWaitOpened" { + if let Some(occurrence) = event.payload["condition_wait_occurrence_id"].as_str() { + let authored = *condition_occurrences.entry(occurrence).or_insert(sequence); + state + .latest_condition_sequences_before_request + .insert(authored, sequence); + } + } if matches!( event.event_type.as_str(), "ActivityCompleted" @@ -574,8 +611,13 @@ impl CancellationHistory { (1..=1000).contains(&span) && sequence > 0 && sequence <= MAX_SEQUENCE - span - && !(sequence..sequence + span) - .all(|sequence| self.resolved_before_request.contains(&sequence)) + && !(sequence..sequence + span).all(|sequence| { + let latest = self + .latest_condition_sequences_before_request + .get(&sequence) + .unwrap_or(&sequence); + self.resolved_before_request.contains(latest) + }) && !(sequence..sequence + span) .any(|sequence| self.failed_before_request.contains(&sequence)) && !self.selected_before_request.contains(&(sequence, span)) diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs index 51acd00..72cc385 100644 --- a/src/tests/cooperative_cancellation.rs +++ b/src/tests/cooperative_cancellation.rs @@ -1171,6 +1171,71 @@ fn cooperative_replay_consumes_reopened_condition_once_at_its_physical_delivery( } } +fn cancelled_reopened_group_history() -> Vec { + let fixture: Value = serde_json::from_str(include_str!( + "../../tests/fixtures/replay-regressions/cooperative-grouped-condition-reopen.json" + )) + .unwrap(); + let mut events: Vec = serde_json::from_value(fixture["history"].clone()).unwrap(); + for event in &mut events { + if matches!( + event.event_type.as_str(), + "CooperativeCancellationRequested" | "CooperativeCancellationDelivered" + ) { + event.payload["workflow_run_id"] = json!("run"); + } + } + events +} + +#[test] +fn cooperative_group_delivery_consumes_pending_physical_condition_without_reopening() { + for reopens in 1..=2 { + let mut events = cancelled_reopened_group_history(); + if reopens == 2 { + let mut satisfied = events[3].clone(); + satisfied.event_type = "ConditionWaitSatisfied".into(); + let mut reopened = events[3].clone(); + reopened.payload["sequence"] = json!(4); + reopened.payload["condition_wait_id"] = json!("condition-4"); + events.splice(4..4, [satisfied, reopened]); + } + let ctx = workflow_context(events); + let mut call = Box::pin(ctx.parallel(vec![ + ParallelOperation::timer(Duration::from_secs(300)), + ParallelOperation::condition( + ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"), + || panic!("canonical cancellation must not reevaluate or reopen the condition"), + ), + ])); + let mut cx = TaskContext::from_waker(noop_waker_ref()); + assert!(matches!( + call.as_mut().poll(&mut cx), + Poll::Ready(Err(Error::CooperativeCancellationRequested(_))) + )); + ctx.ensure_history_consumed().unwrap(); + assert!(ctx.take_commands().unwrap().is_empty()); + } +} + +#[test] +fn cooperative_group_eligibility_uses_the_latest_physical_condition_result() { + for latest_satisfied in [false, true] { + let mut events = cancelled_reopened_group_history(); + events.pop(); + let mut fired = events[0].clone(); + fired.event_type = "TimerFired".into(); + events.insert(4, fired); + if latest_satisfied { + let mut satisfied = events[3].clone(); + satisfied.event_type = "ConditionWaitSatisfied".into(); + events.insert(5, satisfied); + } + let cancellation = history(&events).unwrap(); + assert_eq!(cancellation.eligible(1, 2), !latest_satisfied); + } +} + #[test] fn cooperative_replay_keeps_adjacent_condition_occurrences_and_cleanup_scopes_separate() { let payload = |sequence, occurrence| { diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index a022317..e488f98 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -589,10 +589,9 @@ async fn reopened_condition_cancellation(mode: &'static str, span: u64) { .iter() .position(|event| event["event_type"] == "CooperativeCancellationDelivered") .unwrap(); - // Recorded command-prefix work can reopen a grouped physical wait before - // delivery. Cancellation must retain its authored identity and prevent any - // new wait after the canonical delivery boundary. - assert!(count(&snapshot, "ConditionWaitOpened") >= 2); + // Delivery consumes the latest physical wait under the original authored + // identity. It must not create another opening at either side of delivery. + assert_eq!(count(&snapshot, "ConditionWaitOpened"), 2); for event in events .iter() .filter(|event| event["event_type"] == "ConditionWaitOpened") @@ -610,9 +609,6 @@ async fn reopened_condition_cancellation(mode: &'static str, span: u64) { "condition reopened after cancellation delivery: {snapshot}" ); eprintln!("cooperative reopened condition {mode}: {snapshot}"); - if mode == "condition" { - assert_eq!(count(&snapshot, "ConditionWaitOpened"), 2); - } assert_eq!(count(&snapshot, "SelectionResolved"), 0); let delivery = snapshot["events"] .as_array() diff --git a/tests/fixtures/replay-regressions/cooperative-grouped-condition-reopen.json b/tests/fixtures/replay-regressions/cooperative-grouped-condition-reopen.json new file mode 100644 index 0000000..80adff9 --- /dev/null +++ b/tests/fixtures/replay-regressions/cooperative-grouped-condition-reopen.json @@ -0,0 +1,69 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-cooperative-grouped-condition-reopen", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": { + "type": "corpus.grouped-condition-reopen", + "input": [], + "payload_codec": "avro" + }, + "history": [ + { + "event_type": "TimerScheduled", + "payload": { + "sequence": 1, "timer_id": "timer-1", "delay_seconds": 300, + "parallel_group_id": "parallel-calls:1:2", "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 1, "parallel_group_size": 2, "parallel_group_index": 0 + } + }, + { + "event_type": "ConditionWaitOpened", + "payload": { + "sequence": 2, "condition_wait_id": "condition-2", + "condition_wait_occurrence_id": "rust:condition-wait:0", "condition_key": "two-votes", + "condition_definition_fingerprint": "sha256:two-votes-v1", + "parallel_group_id": "parallel-calls:1:2", "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 1, "parallel_group_size": 2, "parallel_group_index": 1 + } + }, + { + "event_type": "ConditionWaitSatisfied", + "payload": { + "sequence": 2, "condition_wait_id": "condition-2", + "condition_wait_occurrence_id": "rust:condition-wait:0", "condition_key": "two-votes", + "condition_definition_fingerprint": "sha256:two-votes-v1", + "parallel_group_id": "parallel-calls:1:2", "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 1, "parallel_group_size": 2, "parallel_group_index": 1 + } + }, + { + "event_type": "ConditionWaitOpened", + "payload": { + "sequence": 3, "condition_wait_id": "condition-3", + "condition_wait_occurrence_id": "rust:condition-wait:0", "condition_key": "two-votes", + "condition_definition_fingerprint": "sha256:two-votes-v1", + "parallel_group_id": "parallel-calls:1:2", "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 1, "parallel_group_size": 2, "parallel_group_index": 1 + } + }, + { + "event_type": "CooperativeCancellationRequested", + "recorded_at": "2026-10-01T08:00:00Z", + "payload": { + "workflow_command_id": "original-request", "workflow_run_id": "regression-corpus-run", + "cleanup_deadline_at": "2026-10-01T08:10:00Z" + } + }, + { + "event_type": "CooperativeCancellationDelivered", + "payload": { + "workflow_command_id": "original-request", "workflow_run_id": "regression-corpus-run", + "sequence": 1, "call_kind": "parallel", "sequence_span": 2 + } + } + ], + "command_sequence": [{"type": "fail_workflow", "exception_type": "WorkflowCancellationRequested"}], + "expected": {"type": "fail_workflow", "exception_type": "WorkflowCancellationRequested"} +} From 7eb06f6e65565ed94e8976fe06300cd340352908 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 18:28:25 +0000 Subject: [PATCH 21/58] Preserve scalar delivery at the pending physical condition --- src/cooperative_cancellation.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index b943ba3..0672e4d 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -558,7 +558,14 @@ impl CancellationHistory { else { continue; }; - if event.event_type == "ConditionWaitOpened" { + // Groups deliver at their original authored span. Scalar condition + // delivery already names the pending physical sequence directly. + if event.event_type == "ConditionWaitOpened" + && (event.payload["parallel_group_id"].is_string() + || event.payload["parallel_group_path"] + .as_array() + .is_some_and(|path| !path.is_empty())) + { if let Some(occurrence) = event.payload["condition_wait_occurrence_id"].as_str() { let authored = *condition_occurrences.entry(occurrence).or_insert(sequence); state From f40e63ec8c63df2b89b03ada0f3ee12aec28fd59 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 18:30:30 +0000 Subject: [PATCH 22/58] Qualify actual activity worker SIGKILL and lease reclamation --- .github/workflows/ci.yml | 2 +- docker-compose.cooperative.yml | 8 ++ tests/cooperative_server.rs | 253 +++++++++++++++++++++++++++++++++ 3 files changed, 262 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c1d025..85f3915 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -244,7 +244,7 @@ jobs: if-no-files-found: warn - name: Emit failure diagnostics if: ${{ failure() }} - run: docker compose -f docker-compose.cooperative.yml logs --no-color --tail 100 bootstrap server worker + run: docker compose -f docker-compose.cooperative.yml logs --no-color --tail 100 bootstrap server worker repair - name: Remove task runtime resources if: ${{ always() }} run: | diff --git a/docker-compose.cooperative.yml b/docker-compose.cooperative.yml index a7dc1d9..ed06456 100644 --- a/docker-compose.cooperative.yml +++ b/docker-compose.cooperative.yml @@ -19,6 +19,7 @@ services: WORKFLOW_SERVER_AUTH_DRIVER: token WORKFLOW_SERVER_AUTH_TOKEN: test-token DW_WORKFLOW_TASK_TIMEOUT: 10 + DW_ACTIVITY_TASK_TIMEOUT: 10 DW_WORKER_PROTOCOL_VERSION: "1.20" depends_on: mysql: @@ -49,6 +50,13 @@ services: depends_on: server: condition: service_healthy + repair: + image: dw-rust-cooperative-server:${SERVER_CANDIDATE_SHA:-candidate} + command: ["php", "artisan", "workflow:v2:repair-pass", "--loop", "--sleep-seconds=1", "--json"] + environment: *runtime-env + depends_on: + server: + condition: service_healthy mysql: image: mysql:8.0 environment: diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index e488f98..f31649d 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -18,6 +18,259 @@ const WORKFLOW: &str = "tests.rust-cooperative-timer"; const UNDO: &str = "tests.rust-cooperative-undo"; const BLOCKED: &str = "tests.rust-cooperative-blocked"; const REPLAY: &str = "tests.rust-cooperative-replay"; +const PROCESS: &str = "tests.rust-cooperative-process-reclaim"; + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +struct ActivityGrant { + task_id: String, + attempt_id: String, + owner: String, + attempt_number: u64, +} + +struct WorkerProcess(std::process::Child); + +impl Drop for WorkerProcess { + fn drop(&mut self) { + if !matches!(self.0.try_wait(), Ok(Some(_))) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } +} + +struct ProcessScratch(std::path::PathBuf); + +impl Drop for ProcessScratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +struct AbortWorkerOnDrop(tokio::task::AbortHandle); + +impl Drop for AbortWorkerOnDrop { + fn drop(&mut self) { + self.0.abort(); + } +} + +fn register_process_workflow(worker: &mut Worker) { + worker.register_workflow(PROCESS, |ctx, _| async move { + let mut saga = ctx.saga(); + saga.add_compensation(UNDO, json!([]))?; + let result = ctx.activity(BLOCKED, json!([])).await; + saga.finish(result).await?; + Ok(Value::Null) + }); +} + +// Invoked only by the isolated process-death case. Ordinary tests return +// without starting a worker. The parent kills this actual Worker with SIGKILL. +#[tokio::test] +async fn cooperative_process_worker_child() { + let Ok(queue) = std::env::var("DURABLE_WORKFLOW_PROCESS_CHILD_QUEUE") else { + return; + }; + let ready = std::env::var("DURABLE_WORKFLOW_PROCESS_CHILD_READY").unwrap(); + let grant = std::env::var("DURABLE_WORKFLOW_PROCESS_CHILD_GRANT").unwrap(); + let mut worker = Worker::new(client(), &queue) + .worker_id(format!("{queue}-killed")) + .cooperative_cancellation(true) + .poll_timeout(Duration::from_secs(1)); + register_process_workflow(&mut worker); + worker.register_activity(BLOCKED, move |ctx, _| { + let grant = grant.clone(); + async move { + let bytes = serde_json::to_vec(&ActivityGrant { + task_id: ctx.task_id, + attempt_id: ctx.activity_attempt_id, + owner: ctx.lease_owner, + attempt_number: ctx.attempt_number, + }) + .unwrap(); + let pending = format!("{grant}.pending"); + std::fs::write(&pending, bytes).unwrap(); + std::fs::rename(pending, grant).unwrap(); + std::future::pending::>().await + } + }); + worker.register().await.unwrap(); + std::fs::write(ready, b"registered").unwrap(); + loop { + worker.run_once().await.expect("child actual Worker tick"); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +async fn await_child_file(process: &mut WorkerProcess, path: &std::path::Path) { + tokio::time::timeout(Duration::from_secs(15), async { + while !path.is_file() { + assert!( + process.0.try_wait().unwrap().is_none(), + "actual child Worker exited before publishing {}", + path.display() + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("actual child Worker claim within 15 seconds"); +} + +#[cfg(unix)] +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with ten-second leases and the repair daemon"] +async fn server_sigkill_activity_worker_reclaims_attempt_and_fences_old_publication() { + use std::os::unix::process::ExitStatusExt; + + let client = client(); + let queue = queue(); + let scratch = ProcessScratch(std::env::temp_dir().join(format!("{queue}-process"))); + std::fs::create_dir(&scratch.0).unwrap(); + let ready = scratch.0.join("ready"); + let grant_path = scratch.0.join("grant.json"); + let mut original_process = WorkerProcess( + std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "cooperative_process_worker_child", "--nocapture"]) + .env("DURABLE_WORKFLOW_PROCESS_CHILD_QUEUE", &queue) + .env("DURABLE_WORKFLOW_PROCESS_CHILD_READY", &ready) + .env("DURABLE_WORKFLOW_PROCESS_CHILD_GRANT", &grant_path) + .spawn() + .unwrap(), + ); + await_child_file(&mut original_process, &ready).await; + let handle = client + .start_workflow(PROCESS, &queue, &queue, json!([])) + .await + .unwrap(); + await_child_file(&mut original_process, &grant_path).await; + let original: ActivityGrant = + serde_json::from_slice(&std::fs::read(&grant_path).unwrap()).unwrap(); + assert_eq!(original.attempt_number, 1); + original_process.0.kill().unwrap(); + let status = original_process.0.wait().unwrap(); + assert_eq!( + status.signal(), + Some(9), + "expected actual SIGKILL: {status}" + ); + + let (entered_tx, mut entered_rx) = tokio::sync::mpsc::unbounded_channel(); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let cleanups = Arc::new(AtomicUsize::new(0)); + let mut successor = Worker::new(client.clone(), &queue) + .worker_id(format!("{queue}-successor")) + .cooperative_cancellation(true) + .max_concurrent_workflow_tasks(1) + .max_concurrent_activity_tasks(1) + .poll_timeout(Duration::from_secs(1)); + register_process_workflow(&mut successor); + successor.register_activity(BLOCKED, move |ctx, _| { + let entered_tx = entered_tx.clone(); + async move { + ctx.heartbeat(json!({"qualification":"reclaimed"})).await?; + entered_tx.send(ctx).unwrap(); + std::future::pending::>().await + } + }); + let completed = Arc::clone(&cleanups); + successor.register_activity(UNDO, move |ctx, _| { + let completed = Arc::clone(&completed); + async move { + ctx.heartbeat(json!({"qualification":"cleanup"})).await?; + completed.fetch_add(1, Ordering::SeqCst); + Ok(Value::Null) + } + }); + let run = tokio::spawn(async move { + successor + .run_until(async { + let _ = shutdown_rx.await; + }) + .await + }); + let _abort_on_failure = AbortWorkerOnDrop(run.abort_handle()); + let reclaimed: ActivityContext = + tokio::time::timeout(Duration::from_secs(25), entered_rx.recv()) + .await + .expect("repair daemon and actual successor reclaim within 25 seconds") + .expect("actual successor callback entered"); + assert_eq!(reclaimed.task_id, original.task_id); + assert_ne!(reclaimed.activity_attempt_id, original.attempt_id); + assert_ne!(reclaimed.lease_owner, original.owner); + assert_eq!(reclaimed.attempt_number, 2); + + let before = history(&handle).await; + for result in [ + client + .complete_activity_task( + &original.task_id, + &original.attempt_id, + &original.owner, + json!("late"), + "avro", + ) + .await, + client + .fail_activity_task( + &original.task_id, + &original.attempt_id, + &original.owner, + "late failure", + true, + ) + .await, + ] { + assert!( + matches!(result, Err(Error::ActivityTaskRejected(rejection)) if rejection.status == 409) + ); + } + let heartbeat = client + .heartbeat_activity_task( + &original.task_id, + &original.attempt_id, + &original.owner, + json!({"late":true}), + ) + .await; + assert!( + matches!(heartbeat, Err(Error::ActivityTaskRejected(rejection)) if rejection.status == 409) + ); + assert_eq!( + before, + history(&handle).await, + "dead attempt changed canonical history" + ); + + let request = handle + .request_cancellation(CooperativeCancellationOptions::default()) + .await + .unwrap(); + let result = handle + .result_selected_run(WorkflowResultOptions { + timeout: Duration::from_secs(20), + poll_interval: Duration::from_millis(50), + }) + .await; + assert!( + matches!(result, Err(Error::WorkflowCancelled(_))), + "reclaimed result: {result:?}" + ); + let snapshot = assert_cancelled(&handle, &request.cancellation_request.request_id, true).await; + assert_eq!(cleanups.load(Ordering::SeqCst), 1); + assert_eq!(count(&snapshot, "ActivityCancelled"), 1); + eprintln!( + "SIGKILL activity recovery: old={original:?}, successor={}/{}/{}, history={snapshot}", + reclaimed.task_id, reclaimed.activity_attempt_id, reclaimed.lease_owner + ); + shutdown_tx.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(10), run) + .await + .unwrap() + .unwrap() + .unwrap(); +} struct CallbackDrop(Arc); From 768c5ec2a191548e4cdcd239520b098a0333d53a Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 18:37:11 +0000 Subject: [PATCH 23/58] Wait for the real Native activity lease in process recovery qualification --- docker-compose.cooperative.yml | 1 - tests/cooperative_server.rs | 14 +++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docker-compose.cooperative.yml b/docker-compose.cooperative.yml index ed06456..52709d7 100644 --- a/docker-compose.cooperative.yml +++ b/docker-compose.cooperative.yml @@ -19,7 +19,6 @@ services: WORKFLOW_SERVER_AUTH_DRIVER: token WORKFLOW_SERVER_AUTH_TOKEN: test-token DW_WORKFLOW_TASK_TIMEOUT: 10 - DW_ACTIVITY_TASK_TIMEOUT: 10 DW_WORKER_PROTOCOL_VERSION: "1.20" depends_on: mysql: diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index f31649d..bbe2771 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -120,7 +120,7 @@ async fn await_child_file(process: &mut WorkerProcess, path: &std::path::Path) { #[cfg(unix)] #[tokio::test] -#[ignore = "requires an isolated cooperative Server with ten-second leases and the repair daemon"] +#[ignore = "requires an isolated cooperative Server with its real activity lease and repair daemon"] async fn server_sigkill_activity_worker_reclaims_attempt_and_fences_old_publication() { use std::os::unix::process::ExitStatusExt; @@ -148,6 +148,12 @@ async fn server_sigkill_activity_worker_reclaims_attempt_and_fences_old_publicat let original: ActivityGrant = serde_json::from_slice(&std::fs::read(&grant_path).unwrap()).unwrap(); assert_eq!(original.attempt_number, 1); + let leased = client + .activity_task_status(&original.task_id, &original.attempt_id, &original.owner) + .await + .unwrap(); + assert_eq!(leased["can_continue"], true); + eprintln!("SIGKILL original grant: {original:?}, actual lease: {leased}"); original_process.0.kill().unwrap(); let status = original_process.0.wait().unwrap(); assert_eq!( @@ -192,9 +198,11 @@ async fn server_sigkill_activity_worker_reclaims_attempt_and_fences_old_publicat }); let _abort_on_failure = AbortWorkerOnDrop(run.abort_handle()); let reclaimed: ActivityContext = - tokio::time::timeout(Duration::from_secs(25), entered_rx.recv()) + // Native's actual activity-task lease is five minutes. This case waits + // for real wall-clock expiry, without changing timestamps or storage. + tokio::time::timeout(Duration::from_secs(330), entered_rx.recv()) .await - .expect("repair daemon and actual successor reclaim within 25 seconds") + .expect("repair daemon and actual successor reclaim within 330 seconds") .expect("actual successor callback entered"); assert_eq!(reclaimed.task_id, original.task_id); assert_ne!(reclaimed.activity_attempt_id, original.attempt_id); From 082086de148dc3fdc979425bcaffc78ceeef1a14 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 18:50:52 +0000 Subject: [PATCH 24/58] Assert dead activity heartbeat stop status after real reclaim --- tests/cooperative_server.rs | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index bbe2771..82e72dc 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -208,6 +208,13 @@ async fn server_sigkill_activity_worker_reclaims_attempt_and_fences_old_publicat assert_ne!(reclaimed.activity_attempt_id, original.attempt_id); assert_ne!(reclaimed.lease_owner, original.owner); assert_eq!(reclaimed.attempt_number, 2); + eprintln!( + "SIGKILL successor grant: {}/{}/{}/{}", + reclaimed.task_id, + reclaimed.activity_attempt_id, + reclaimed.lease_owner, + reclaimed.attempt_number + ); let before = history(&handle).await; for result in [ @@ -241,10 +248,20 @@ async fn server_sigkill_activity_worker_reclaims_attempt_and_fences_old_publicat &original.owner, json!({"late":true}), ) - .await; - assert!( - matches!(heartbeat, Err(Error::ActivityTaskRejected(rejection)) if rejection.status == 409) + .await + .expect("dead-attempt heartbeat returns its stop status"); + assert_eq!(heartbeat.can_continue, Some(false)); + assert!(!heartbeat.heartbeat_recorded); + assert!(heartbeat.should_stop()); + assert_eq!(heartbeat.reason.as_deref(), Some("attempt_closed")); + assert!(!heartbeat.cancel_requested); + assert_eq!(heartbeat.last_heartbeat_at, None); + assert_eq!( + heartbeat.lease_expires_at.as_deref(), + leased["lease_expires_at"].as_str(), + "dead attempt renewed its lease" ); + eprintln!("SIGKILL dead-attempt heartbeat: {heartbeat:?}"); assert_eq!( before, history(&handle).await, From a4897f82dac734d754d694ecafb7a91e0ebaea67 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 18:57:45 +0000 Subject: [PATCH 25/58] Qualify actual Rust cleanup deadline and termination fences --- tests/cooperative_server.rs | 170 +++++++++++++++++++++++++++++++++++- 1 file changed, 168 insertions(+), 2 deletions(-) diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index 82e72dc..3b3a6d6 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -3,8 +3,8 @@ use durable_workflow::{ json, ActivityContext, Client, ConditionWaitOptions, CooperativeCancellationOptions, Error, - ParallelOperation, ParallelResult, SelectionKey, Value, Worker, WorkflowHandle, - WorkflowResultOptions, + ParallelOperation, ParallelResult, SelectionKey, Value, Worker, WorkflowCommandOptions, + WorkflowHandle, WorkflowResultOptions, }; use std::{ sync::{ @@ -299,6 +299,172 @@ async fn server_sigkill_activity_worker_reclaims_attempt_and_fences_old_publicat struct CallbackDrop(Arc); +async fn blocked_cleanup_cutoff(terminate: bool) { + let client = client(); + let queue = queue(); + let dropped = Arc::new(AtomicUsize::new(0)); + let cleanup_dropped = Arc::clone(&dropped); + let (entered_tx, mut entered_rx) = tokio::sync::mpsc::unbounded_channel(); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let mut worker = Worker::new(client.clone(), &queue) + .cooperative_cancellation(true) + .max_concurrent_workflow_tasks(1) + .max_concurrent_activity_tasks(1) + .poll_timeout(Duration::from_secs(1)); + register_process_workflow(&mut worker); + worker.register_activity(UNDO, move |ctx, _| { + let entered_tx = entered_tx.clone(); + let cleanup_dropped = Arc::clone(&cleanup_dropped); + async move { + let _drop = CallbackDrop(cleanup_dropped); + entered_tx.send(ctx).unwrap(); + std::future::pending::>().await + } + }); + let handle = client + .start_workflow(PROCESS, &queue, &queue, json!([])) + .await + .unwrap(); + let request = handle + .request_cancellation(CooperativeCancellationOptions { + cleanup_timeout_seconds: Some(if terminate { 60 } else { 5 }), + ..CooperativeCancellationOptions::default() + }) + .await + .unwrap(); + let run = tokio::spawn(async move { + worker + .run_until(async { + let _ = shutdown_rx.await; + }) + .await + }); + let _abort_on_failure = AbortWorkerOnDrop(run.abort_handle()); + let cleanup: ActivityContext = tokio::time::timeout(Duration::from_secs(10), entered_rx.recv()) + .await + .unwrap() + .expect("actual shielded cleanup entered"); + if terminate { + handle + .terminate_selected_run(WorkflowCommandOptions::default()) + .await + .unwrap(); + } + let result = handle + .result_selected_run(WorkflowResultOptions { + timeout: Duration::from_secs(15), + poll_interval: Duration::from_millis(50), + }) + .await; + if terminate { + assert!( + matches!(result, Err(Error::WorkflowTerminated(_))), + "{result:?}" + ); + } else { + assert!( + matches!(result, Err(Error::WorkflowCancelled(_))), + "{result:?}" + ); + } + tokio::time::timeout(Duration::from_secs(10), async { + while dropped.load(Ordering::SeqCst) == 0 { + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("blocked cleanup future drops after terminal authority loss"); + assert_eq!(dropped.load(Ordering::SeqCst), 1); + assert!(matches!( + cleanup.heartbeat(json!({"late":true})).await, + Err(Error::ActivityExecutionAbandoned(_)) + )); + let snapshot = history(&handle).await; + for kind in [ + "CooperativeCancellationRequested", + "CooperativeCancellationDelivered", + ] { + assert_eq!(count(&snapshot, kind), 1, "{snapshot}"); + let event = snapshot["events"] + .as_array() + .unwrap() + .iter() + .find(|e| e["event_type"] == kind) + .unwrap(); + assert_eq!( + event["payload"]["workflow_command_id"], + request.cancellation_request.request_id + ); + } + assert_eq!( + count(&snapshot, "WorkflowCancelled"), + usize::from(!terminate) + ); + assert_eq!( + count(&snapshot, "WorkflowTerminated"), + usize::from(terminate) + ); + assert_eq!(count(&snapshot, "ActivityScheduled"), 1); + assert_eq!(count(&snapshot, "ActivityCancelled"), 1); + for kind in [ + "ActivityCompleted", + "ActivityFailed", + "ActivityTimedOut", + "WorkflowCompleted", + "WorkflowFailed", + ] { + assert_eq!(count(&snapshot, kind), 0, "{snapshot}"); + } + for result in [ + client + .complete_activity_task( + &cleanup.task_id, + &cleanup.activity_attempt_id, + &cleanup.lease_owner, + json!("late"), + "avro", + ) + .await, + client + .fail_activity_task( + &cleanup.task_id, + &cleanup.activity_attempt_id, + &cleanup.lease_owner, + "late", + true, + ) + .await, + ] { + assert!( + matches!(result, Err(Error::ActivityTaskRejected(rejection)) if rejection.status == 409) + ); + } + assert_eq!( + snapshot, + history(&handle).await, + "late cleanup changed canonical history" + ); + eprintln!("blocked cleanup cutoff terminate={terminate}: {snapshot}"); + shutdown_tx.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(10), run) + .await + .unwrap() + .unwrap() + .unwrap(); +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_cleanup_deadline_stops_blocked_compensation() { + blocked_cleanup_cutoff(false).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_termination_stops_blocked_compensation() { + blocked_cleanup_cutoff(true).await; +} + impl Drop for CallbackDrop { fn drop(&mut self) { self.0.fetch_add(1, Ordering::SeqCst); From 745d476bf1f37a3734f6072c189f23b1c07143e8 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 18:58:54 +0000 Subject: [PATCH 26/58] Compare reclaimed attempt status around refused heartbeat --- tests/cooperative_server.rs | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index 3b3a6d6..52fbc33 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -216,6 +216,12 @@ async fn server_sigkill_activity_worker_reclaims_attempt_and_fences_old_publicat reclaimed.attempt_number ); + let closed = client + .activity_task_status(&original.task_id, &original.attempt_id, &original.owner) + .await + .unwrap(); + assert_eq!(closed["attempt_status"], "expired"); + assert_eq!(closed["can_continue"], false); let before = history(&handle).await; for result in [ client @@ -255,13 +261,24 @@ async fn server_sigkill_activity_worker_reclaims_attempt_and_fences_old_publicat assert!(heartbeat.should_stop()); assert_eq!(heartbeat.reason.as_deref(), Some("attempt_closed")); assert!(!heartbeat.cancel_requested); - assert_eq!(heartbeat.last_heartbeat_at, None); + assert_eq!( + heartbeat.last_heartbeat_at.as_deref(), + closed["last_heartbeat_at"].as_str() + ); assert_eq!( heartbeat.lease_expires_at.as_deref(), - leased["lease_expires_at"].as_str(), + closed["lease_expires_at"].as_str(), "dead attempt renewed its lease" ); eprintln!("SIGKILL dead-attempt heartbeat: {heartbeat:?}"); + assert_eq!( + closed, + client + .activity_task_status(&original.task_id, &original.attempt_id, &original.owner) + .await + .unwrap(), + "dead heartbeat changed attempt authority or lifetime" + ); assert_eq!( before, history(&handle).await, From 91bcba22edb830fa5ed777eb551c9728f809f742 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 19:16:53 +0000 Subject: [PATCH 27/58] test: match production repair cadence for cancellation recovery --- docker-compose.cooperative.yml | 3 ++- tests/cooperative_server.rs | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docker-compose.cooperative.yml b/docker-compose.cooperative.yml index 52709d7..880b80a 100644 --- a/docker-compose.cooperative.yml +++ b/docker-compose.cooperative.yml @@ -51,7 +51,8 @@ services: condition: service_healthy repair: image: dw-rust-cooperative-server:${SERVER_CANDIDATE_SHA:-candidate} - command: ["php", "artisan", "workflow:v2:repair-pass", "--loop", "--sleep-seconds=1", "--json"] + # Match the production scheduler's unscoped, unthrottled repair cadence. + command: sh -c 'while true; do php artisan workflow:v2:repair-pass --json; sleep 10; done' environment: *runtime-env depends_on: server: diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index 52fbc33..fb4821c 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -369,7 +369,9 @@ async fn blocked_cleanup_cutoff(terminate: bool) { } let result = handle .result_selected_run(WorkflowResultOptions { - timeout: Duration::from_secs(15), + // Include the production ten-second repair cadence without extending + // the original cleanup deadline. + timeout: Duration::from_secs(25), poll_interval: Duration::from_millis(50), }) .await; From dfb8864d7c64c7b7c259484f34480f04e2a4653e Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 21:48:39 +0000 Subject: [PATCH 28/58] fix: return Rust worker to polling during child cancellation waits --- src/cooperative_cancellation.rs | 76 ++++++++-- src/lib.rs | 8 +- src/tests/cooperative_cancellation.rs | 211 +++++++++++++++++++++++++- 3 files changed, 271 insertions(+), 24 deletions(-) diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index 0672e4d..0165d2f 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -12,6 +12,18 @@ pub struct CancellationDeliveryReceipt { pub delivery: CancellationDelivery, } +/// Delivery either commits or parks the parent until canonical child cleanup. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CancellationDeliveryReply { + Delivered(CancellationDeliveryReceipt), + /// The current claim is completed. Return to polling without publishing + /// completion or failure. A child terminal event wakes a successor claim. + ClaimReleased { + task_id: String, + run_id: String, + }, +} + /// Successful renewal of the exact workflow-task claim with optional observation. #[derive(Clone, Debug, PartialEq, Eq)] pub struct WorkflowTaskHeartbeat { @@ -1294,12 +1306,13 @@ impl Client { /// This explicit worker-protocol 1.20 operation renews no lease and advertises /// no worker capability. The Server requires a previously admitted capable /// claim. Reload canonical history before exposing cancellation to workflow - /// code, including when an acknowledgment is lost. + /// code, including when an acknowledgment is lost. A child wait explicitly + /// releases the claim and must return the worker to polling. pub async fn deliver_workflow_cancellation( &self, task: &WorkflowTask, delivery: &CancellationDelivery, - ) -> Result { + ) -> Result { let (owner, run_id) = cancellation_claim(task)?; let body = json!({ "lease_owner":owner, "workflow_task_attempt":task.workflow_task_attempt, @@ -1328,6 +1341,33 @@ impl Client { Some(&body), ) .await?; + if response["delivered"].as_bool() == Some(false) + && matches!( + delivery.call_kind, + CancellationCallKind::Child + | CancellationCallKind::Parallel + | CancellationCallKind::SelectionHandle + ) + && response["reason"].as_str() == Some("cancellation_waiting_for_child") + && response["claim_released"].as_bool() == Some(true) + && response["task_id"].as_str() == Some(task.task_id.as_str()) + && response["workflow_run_id"].as_str() == Some(run_id) + && [ + "request_id", + "sequence", + "call_kind", + "sequence_span", + "operation_sequence", + "operation_sequence_span", + ] + .iter() + .all(|field| response.get(*field).is_none_or(Value::is_null)) + { + return Ok(CancellationDeliveryReply::ClaimReleased { + task_id: task.task_id.clone(), + run_id: run_id.to_owned(), + }); + } if response["delivered"].as_bool() != Some(true) || response["task_id"].as_str() != Some(task.task_id.as_str()) || response["workflow_run_id"].as_str() != Some(run_id) @@ -1351,11 +1391,13 @@ impl Client { "delivery acknowledgment changed its authored operation range", )); } - Ok(CancellationDeliveryReceipt { - task_id: task.task_id.clone(), - run_id: run_id.to_owned(), - delivery: delivery.clone(), - }) + Ok(CancellationDeliveryReply::Delivered( + CancellationDeliveryReceipt { + task_id: task.task_id.clone(), + run_id: run_id.to_owned(), + delivery: delivery.clone(), + }, + )) }) .await .map_err(|_| Error::Timeout)? @@ -1642,7 +1684,9 @@ impl Worker { }; // An uncertain observation, delivery or canonical refresh cannot become // an application failure or be published as a workflow-task decision. - let decision = self.replay_cooperative_workflow_claim(&claim).await?; + let Some(decision) = self.replay_cooperative_workflow_claim(&claim).await? else { + return Ok(ManagedPollOutcome::Handled); + }; let (owner, _) = cancellation_claim(&claim.task)?; self.settle_workflow_task_decision( &claim.task.task_id, @@ -1658,7 +1702,7 @@ impl Worker { async fn replay_cooperative_workflow_claim( &self, claim: &CooperativeWorkflowTask, - ) -> Result { + ) -> Result> { let (owner, run_id) = cancellation_claim(&claim.task)?; if owner != self.worker_id { return Err(invalid( @@ -1673,7 +1717,7 @@ impl Worker { "cooperative replay omitted its original pending observation", )); } - return self.execute_workflow_task_decision(task); + return self.execute_workflow_task_decision(task).map(Some); }; task.history_events = self .client @@ -1689,20 +1733,24 @@ impl Worker { Some(observation), )?; let Some(intent) = decision.cancellation_delivery.as_ref() else { - return Ok(decision); + return Ok(Some(decision)); }; if !decision.commands.is_empty() { // Native permits delivery only at/before its next durable call. // Commit earlier commands first. Completion releases this claim // and its successor replays their durable results before delivery. decision.cancellation_delivery = None; - return Ok(decision); + return Ok(Some(decision)); } - let delivery_error = self + let delivery_error = match self .client .deliver_workflow_cancellation(&claim.task, intent) .await - .err(); + { + Ok(CancellationDeliveryReply::ClaimReleased { .. }) => return Ok(None), + Ok(CancellationDeliveryReply::Delivered(_)) => None, + Err(error) => Some(error), + }; task.history_events = self .client .refresh_workflow_cancellation_history(&claim.task, observation) diff --git a/src/lib.rs b/src/lib.rs index c276f6d..a036e9c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,10 +5,10 @@ mod runtime_payloads; mod runtime_uploads; pub use cooperative_cancellation::{ - CancellationCallKind, CancellationDelivery, CancellationDeliveryReceipt, CancellationHistory, - CancellationRequest, CancellationShield, CooperativeCancellationOptions, - CooperativeCancellationRequested, CooperativeWorkflowTask, CooperativeWorkflowTaskPoll, - WorkflowCancellationRequest, WorkflowTaskHeartbeat, + CancellationCallKind, CancellationDelivery, CancellationDeliveryReceipt, + CancellationDeliveryReply, CancellationHistory, CancellationRequest, CancellationShield, + CooperativeCancellationOptions, CooperativeCancellationRequested, CooperativeWorkflowTask, + CooperativeWorkflowTaskPoll, WorkflowCancellationRequest, WorkflowTaskHeartbeat, }; use std::{ diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs index 72cc385..08df0ef 100644 --- a/src/tests/cooperative_cancellation.rs +++ b/src/tests/cooperative_cancellation.rs @@ -1817,6 +1817,13 @@ fn transport_delivery() -> CancellationDelivery { CancellationDelivery::from_payload(&canonical_delivery(1, "timer").payload).unwrap() } +fn pending_child_delivery(task_id: &str) -> Value { + json!({"delivered":false, "task_id":task_id, "workflow_run_id":"run", + "reason":"cancellation_waiting_for_child", "claim_released":true, + "request_id":null, "sequence":null, "call_kind":null, "sequence_span":null, + "operation_sequence":null, "operation_sequence_span":null}) +} + fn transport_responses(path: &str, body: &str, number: usize) -> Option<(&'static str, String)> { let case = path.split('/').nth(1).unwrap_or_default(); let body: Value = serde_json::from_str(body).unwrap(); @@ -1902,6 +1909,28 @@ fn transport_responses(path: &str, body: &str, number: usize) -> Option<(&'stati .to_string(), )); } + if path.ends_with("/deliver-cancellation") && case.starts_with("child-wait") { + let mut pending = pending_child_delivery("task/selected"); + match case { + "child-wait-wrong-task" => pending["task_id"] = json!("other"), + "child-wait-wrong-run" => pending["workflow_run_id"] = json!("other"), + "child-wait-not-released" => pending["claim_released"] = json!(false), + "child-wait-string-released" => pending["claim_released"] = json!("true"), + "child-wait-missing-release" => { + pending.as_object_mut().unwrap().remove("claim_released"); + } + "child-wait-wrong-reason" => pending["reason"] = json!("other"), + "child-wait-request" => pending["request_id"] = body["request_id"].clone(), + "child-wait-sequence" => pending["sequence"] = json!(1), + "child-wait-kind" => pending["call_kind"] = json!("child"), + "child-wait-span" => pending["sequence_span"] = json!(1), + "child-wait-operation" => pending["operation_sequence"] = json!(1), + "child-wait-operation-span" => pending["operation_sequence_span"] = json!(1), + "child-wait-string-delivered" => pending["delivered"] = json!("false"), + _ => {} + } + return Some(("200 OK", pending.to_string())); + } let mut response = if path.ends_with("/deliver-cancellation") { json!({"delivered":true, "task_id":"task/selected", "workflow_run_id":"run", "request_id":body["request_id"], "sequence":body["sequence"], "call_kind":body["call_kind"], @@ -2804,6 +2833,20 @@ fn coordinator_responses(path: &str, body: &str, number: usize) -> Option<(&'sta response["task"]["task_id"] = json!("cleanup"); response["task"]["workflow_task_attempt"] = json!(9); } + if case == "coordinator-child-wait" && number == 2 { + response["task"]["task_id"] = json!("other-task"); + response["task"]["workflow_type"] = json!("other"); + response["task"]["run_id"] = json!("other-run"); + response["task"]["cancel_requested"] = json!(false); + response["task"] + .as_object_mut() + .unwrap() + .remove("cancellation_request"); + } + if case == "coordinator-child-wait" && number >= 3 { + response["task"]["task_id"] = json!("parent-resume-task"); + response["task"]["workflow_task_attempt"] = json!(8); + } return Some((status, response.to_string())); } if path.ends_with("/history") { @@ -2815,7 +2858,11 @@ fn coordinator_responses(path: &str, body: &str, number: usize) -> Option<(&'sta } let prefix = case == "coordinator-prefix"; let successor = prefix && path.ends_with("/successor/history"); - let mut events = if prefix { + let child_wait = case == "coordinator-child-wait"; + let child_resume = child_wait && path.ends_with("/parent-resume-task/history"); + let mut events = if child_wait { + vec![pending_scalar_event("child"), canonical_request()] + } else if prefix { vec![canonical_request()] } else { vec![scheduled_timer(1), canonical_request()] @@ -2827,7 +2874,9 @@ fn coordinator_responses(path: &str, body: &str, number: usize) -> Option<(&'sta )); } let cleanup = case == "coordinator-saga" && path.ends_with("/cleanup/history"); - let delivered = if prefix { + let delivered = if child_wait { + child_resume && number >= 2 + } else if prefix { successor && number >= 2 } else { number >= 2 || case == "coordinator-cold" || cleanup @@ -2835,7 +2884,9 @@ fn coordinator_responses(path: &str, body: &str, number: usize) -> Option<(&'sta if delivered && !matches!(case, "coordinator-unproved" | "coordinator-shield") { events.push(canonical_delivery( if prefix { 2 } else { 1 }, - if case == "coordinator-mismatch" { + if child_wait { + "child" + } else if case == "coordinator-mismatch" { "activity" } else { "timer" @@ -2860,8 +2911,8 @@ fn coordinator_responses(path: &str, body: &str, number: usize) -> Option<(&'sta return Some(( "200 OK", json!({ - "task_id":if successor { "successor" } else if cleanup { "cleanup" } else { "task/selected" }, - "workflow_task_attempt":if successor { 8 } else if cleanup { 9 } else { 7 }, + "task_id":if successor { "successor" } else if child_resume { "parent-resume-task" } else if cleanup { "cleanup" } else { "task/selected" }, + "workflow_task_attempt":if successor || child_resume { 8 } else if cleanup { 9 } else { 7 }, "total_history_events":events.len(),"history_events":events, "next_history_page_token":null, }) @@ -2869,6 +2920,14 @@ fn coordinator_responses(path: &str, body: &str, number: usize) -> Option<(&'sta )); } if path.ends_with("/deliver-cancellation") { + if case == "coordinator-child-wait" + && !path.ends_with("/parent-resume-task/deliver-cancellation") + { + return Some(( + "200 OK", + pending_child_delivery("task/selected").to_string(), + )); + } if case == "coordinator-lost-ack" { return Some(( "409 Conflict", @@ -2880,6 +2939,9 @@ fn coordinator_responses(path: &str, body: &str, number: usize) -> Option<(&'sta if case == "coordinator-prefix" { response["task_id"] = json!("successor"); } + if case == "coordinator-child-wait" { + response["task_id"] = json!("parent-resume-task"); + } return Some((status, response.to_string())); } if path.ends_with("/complete") || path.ends_with("/fail") { @@ -2967,6 +3029,80 @@ async fn cooperative_coordinator_commits_delivery_proves_history_and_replays_bef } } +#[tokio::test] +async fn cooperative_coordinator_returns_to_other_work_then_replays_the_parent_on_a_new_claim() { + let server = coordinator_server(); + let mut worker = coordinator_worker(&server, "coordinator-child-wait"); + let observed = Arc::new(Mutex::new(Vec::new())); + let cleanup = Arc::clone(&observed); + worker.register_workflow("cancel", move |ctx, _| { + let cleanup = Arc::clone(&cleanup); + async move { + match ctx + .start_child_workflow("child", ChildWorkflowOptions::new("queue"), json!([])) + .await + { + Err(Error::CooperativeCancellationRequested(request)) => { + cleanup.lock().unwrap().push(request) + } + result => { + result?; + } + } + Ok(Value::Null) + } + }); + worker.register_workflow("other", |_, _| async { Ok(Value::Null) }); + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + assert!(observed.lock().unwrap().is_empty()); + assert!(!server + .requests + .lock() + .unwrap() + .iter() + .any(|request| request.path.ends_with("/complete") || request.path.ends_with("/fail"))); + + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + assert!(observed.lock().unwrap().is_empty()); + assert!(server + .requests + .lock() + .unwrap() + .iter() + .any(|request| request.path.ends_with("/other-task/complete"))); + + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + let observed = observed.lock().unwrap(); + assert_eq!(observed.len(), 1); + assert_eq!(observed[0].request, original_observation()); + assert_eq!(observed[0].delivery.call_kind, CancellationCallKind::Child); + assert_eq!(observed[0].delivery.sequence, 1); + let requests = server.requests.lock().unwrap(); + let deliveries = requests + .iter() + .filter(|request| request.path.ends_with("/deliver-cancellation")) + .map(|request| serde_json::from_str::(&request.body).unwrap()) + .collect::>(); + assert_eq!(deliveries.len(), 2); + assert_eq!(deliveries[0]["workflow_task_attempt"], 7); + assert_eq!(deliveries[1]["workflow_task_attempt"], 8); + assert!(deliveries + .iter() + .all(|body| body["request_id"] == "original-request")); + assert!(!requests + .iter() + .any(|request| request.path.ends_with("/fail"))); +} + #[tokio::test] async fn cooperative_coordinator_refuses_unproved_changed_or_lost_claim_history_without_publication( ) { @@ -3540,10 +3676,13 @@ async fn cooperative_transport_delivers_the_exact_claim_with_worker_credentials( let client = client(&server, "valid"); let task = transport_task(); let delivery = transport_delivery(); - let receipt = client + let reply = client .deliver_workflow_cancellation(&task, &delivery) .await .unwrap(); + let CancellationDeliveryReply::Delivered(receipt) = reply else { + panic!("delivery must be committed") + }; assert_eq!(receipt.task_id, task.task_id); assert_eq!(receipt.run_id, "run"); assert_eq!(receipt.delivery, delivery); @@ -3563,6 +3702,66 @@ async fn cooperative_transport_delivers_the_exact_claim_with_worker_credentials( assert_eq!(body["request_id"], "original-request"); } +#[tokio::test] +async fn cooperative_transport_accepts_only_explicit_child_claim_release() { + for kind in [ + CancellationCallKind::Child, + CancellationCallKind::Parallel, + CancellationCallKind::SelectionHandle, + ] { + let server = transport_server(); + let mut delivery = transport_delivery(); + delivery.call_kind = kind; + if kind == CancellationCallKind::SelectionHandle { + delivery.sequence = 2; + delivery.operation_sequence = Some(1); + } + assert_eq!( + client(&server, "child-wait") + .deliver_workflow_cancellation(&transport_task(), &delivery) + .await + .unwrap(), + CancellationDeliveryReply::ClaimReleased { + task_id: "task/selected".into(), + run_id: "run".into() + } + ); + } + let server = transport_server(); + assert!(matches!( + client(&server, "child-wait") + .deliver_workflow_cancellation(&transport_task(), &transport_delivery()) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); + for case in [ + "child-wait-wrong-task", + "child-wait-wrong-run", + "child-wait-not-released", + "child-wait-string-released", + "child-wait-missing-release", + "child-wait-wrong-reason", + "child-wait-request", + "child-wait-sequence", + "child-wait-kind", + "child-wait-span", + "child-wait-operation", + "child-wait-operation-span", + "child-wait-string-delivered", + ] { + let server = transport_server(); + let mut delivery = transport_delivery(); + delivery.call_kind = CancellationCallKind::Child; + let result = client(&server, case) + .deliver_workflow_cancellation(&transport_task(), &delivery) + .await; + assert!( + matches!(result, Err(Error::InvalidCooperativeCancellation(_))), + "{case}: {result:?}" + ); + } +} + #[tokio::test] async fn cooperative_transport_rejects_malformed_or_changed_delivery_acknowledgments() { for case in [ From 85a5a9f1de91d6dbaa3ac65e373272f4e24e50fc Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 22:47:49 +0000 Subject: [PATCH 29/58] feat: expose canonical cancellation context in Rust cleanup --- Cargo.toml | 2 +- docs/cooperative-cancellation-design.md | 44 ++++ src/cancellation_context.rs | 218 +++++++++++++++ src/cooperative_cancellation.rs | 85 +++++- src/lib.rs | 3 + src/tests/cooperative_cancellation.rs | 248 ++++++++++++++++++ .../cooperative-cancellation-context.json | 17 ++ 7 files changed, 611 insertions(+), 6 deletions(-) create mode 100644 docs/cooperative-cancellation-design.md create mode 100644 src/cancellation_context.rs create mode 100644 src/tests/fixtures/cooperative-cancellation-context.json diff --git a/Cargo.toml b/Cargo.toml index 4155151..375d780 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,7 +110,7 @@ apache-avro = { version = "0.21", default-features = false } base64 = "0.22" # Keep apache-avro's dependency resolution compatible with the Rust 1.86 MSRV. bon = { version = "=3.8.1", default-features = false } -chrono = { version = "0.4", default-features = false } +chrono = { version = "0.4", default-features = false, features = ["alloc"] } futures-util = "0.3" # Keep reqwest's IDNA dependency resolution compatible with the Rust 1.86 MSRV. icu_collections = { version = "=2.2.0", default-features = false } diff --git a/docs/cooperative-cancellation-design.md b/docs/cooperative-cancellation-design.md new file mode 100644 index 0000000..a877a0a --- /dev/null +++ b/docs/cooperative-cancellation-design.md @@ -0,0 +1,44 @@ +# Cooperative cancellation source design + +This document describes the unfinished source candidate for shared cancellation +issue 136. Default worker protocol remains 1.19. Cooperation requires explicit +protocol 1.20 opt-in and a compatible Server and Native backend. The shared +specification and published mixed-language qualification remain pending. + +## Recorded context + +`WorkflowContext::cancellation_context()` returns the original immutable metadata +after committed authored cancellation delivery. Earlier workflow code receives +`None`. The existing `Error::CooperativeCancellationRequested` carries the same +snapshot in `request.context`. Poll and heartbeat observations do not supply +workflow-facing context, even when their transport object contains extra fields. + +`CancellationContext` and `CancellationLineage` expose read-only accessors. +Metadata includes local and root request IDs, root workflow instance and run IDs, +the immediate parent request ID, original reason, requester/source, root request +time, original deadline and ordered lineage. Requester fields are limited to +caller type, ID and label. Timestamp helpers return immutable UTC dates. +`to_value()` produces a detached portable snapshot. + +A child keeps the root request time and cleanup budget even if its local request +is accepted later. Cold replay restores the object from canonical request history. +Explicit checks and shielded cleanup retain it. The parser validates local +request/run binding, cycles, budget and delivery snapshot equality. Older +cancellation histories without rich context keep their existing delivery behavior +with `context == None`. + +## Remaining qualification + +Portable operation policies, nested scopes and deterministic remaining-time +helpers still need completion. Remaining time must use the replayed workflow +clock. Do not subtract the host clock from the deadline in workflow code. The +runtime continues enforcing the original deadline and fencing task and activity +ownership. Rust local activities and worker affinity remain unsupported. + +Connected qualification must cover the PHP parent, Python child, Rust remote +activity and PHP local activity together. Both callbacks must stop without +application heartbeats. A replacement after SIGKILL during cleanup must replay +the same boundary and finish before the original 30-second deadline. Record +supported workflow lease, heartbeat and repair settings with that scenario. +Exact published artifacts and one cascade inspection view remain required before +release claims. diff --git a/src/cancellation_context.rs b/src/cancellation_context.rs new file mode 100644 index 0000000..5937454 --- /dev/null +++ b/src/cancellation_context.rs @@ -0,0 +1,218 @@ +use super::*; +use chrono::{SecondsFormat, Utc}; + +/// One immutable local request in the ordered cancellation lineage. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CancellationLineage { + request_id: String, + workflow_instance_id: String, + workflow_run_id: String, +} + +impl CancellationLineage { + pub fn request_id(&self) -> &str { + &self.request_id + } + + pub fn workflow_instance_id(&self) -> &str { + &self.workflow_instance_id + } + + pub fn workflow_run_id(&self) -> &str { + &self.workflow_run_id + } + + fn to_value(&self) -> Value { + json!({ + "request_id": self.request_id, + "workflow_instance_id": self.workflow_instance_id, + "workflow_run_id": self.workflow_run_id, + }) + } +} + +/// Original cancellation metadata restored from canonical request history. +/// +/// Fields and nested metadata have read-only accessors. Descendants retain +/// the original root identity, request time and cleanup budget. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CancellationContext { + request_id: String, + root_request_id: String, + root_workflow_instance_id: String, + root_workflow_run_id: String, + parent_request_id: Option, + reason: Option, + requester: BTreeMap, + source: String, + requested_at: DateTime, + cleanup_deadline_at: DateTime, + lineage: Vec, +} + +fn invalid_context(message: &str) -> Error { + Error::InvalidCooperativeCancellation(message.to_owned()) +} + +fn context_text<'a>(value: &'a Value, key: &str) -> Result<&'a str> { + value[key] + .as_str() + .filter(|text| !text.trim().is_empty()) + .ok_or_else(|| invalid_context("cancellation context identity must be a non-empty string")) +} + +fn nullable_text(value: &Value, key: &str, allow_empty: bool) -> Result> { + match value.get(key) { + None | Some(Value::Null) => Ok(None), + Some(Value::String(text)) if allow_empty || !text.is_empty() => Ok(Some(text.clone())), + _ => Err(invalid_context( + "cancellation parent identity or reason is invalid", + )), + } +} + +impl CancellationContext { + /// Validate the portable v1 snapshot without granting a new cleanup budget. + pub fn from_value(value: &Value) -> Result { + if value["schema"] != "durable-workflow.cancellation-context/v1" { + return Err(invalid_context("unsupported cancellation context schema")); + } + let requester = value["requester"] + .as_object() + .filter(|requester| !requester.is_empty()) + .ok_or_else(|| invalid_context("cancellation requester must identify its caller"))?; + let mut normalized_requester = BTreeMap::new(); + for (key, value) in requester { + let text = value + .as_str() + .filter(|text| !text.is_empty()) + .ok_or_else(|| { + invalid_context("cancellation requester contains unsupported metadata") + })?; + if !matches!(key.as_str(), "type" | "id" | "label") { + return Err(invalid_context( + "cancellation requester contains unsupported metadata", + )); + } + normalized_requester.insert(key.clone(), text.to_owned()); + } + let lineage = value["lineage"] + .as_array() + .filter(|lineage| !lineage.is_empty()) + .ok_or_else(|| invalid_context("cancellation lineage must contain the root request"))?; + let mut normalized = Vec::new(); + for entry in lineage { + normalized.push(CancellationLineage { + request_id: context_text(entry, "request_id")?.to_owned(), + workflow_instance_id: context_text(entry, "workflow_instance_id")?.to_owned(), + workflow_run_id: context_text(entry, "workflow_run_id")?.to_owned(), + }); + } + let request_id = context_text(value, "request_id")?.to_owned(); + let root_request_id = context_text(value, "root_request_id")?.to_owned(); + let root_workflow_instance_id = + context_text(value, "root_workflow_instance_id")?.to_owned(); + let root_workflow_run_id = context_text(value, "root_workflow_run_id")?.to_owned(); + let parent_request_id = nullable_text(value, "parent_request_id", false)?; + let reason = nullable_text(value, "reason", true)?; + let requests: BTreeSet<_> = normalized.iter().map(|entry| &entry.request_id).collect(); + let runs: BTreeSet<_> = normalized + .iter() + .map(|entry| &entry.workflow_run_id) + .collect(); + if requests.len() != normalized.len() || runs.len() != normalized.len() { + return Err(invalid_context( + "cancellation lineage cannot contain a cycle", + )); + } + let expected_parent = normalized + .iter() + .rev() + .nth(1) + .map(|entry| &entry.request_id); + if normalized[0].request_id != root_request_id + || normalized[0].workflow_instance_id != root_workflow_instance_id + || normalized[0].workflow_run_id != root_workflow_run_id + || normalized.last().unwrap().request_id != request_id + || parent_request_id.as_ref() != expected_parent + { + return Err(invalid_context( + "cancellation lineage does not match its request identities", + )); + } + let requested_at = DateTime::parse_from_rfc3339(context_text(value, "requested_at")?) + .map_err(|_| invalid_context("cancellation request timestamp is invalid"))? + .with_timezone(&Utc); + let cleanup_deadline_at = + DateTime::parse_from_rfc3339(context_text(value, "cleanup_deadline_at")?) + .map_err(|_| invalid_context("cancellation deadline timestamp is invalid"))? + .with_timezone(&Utc); + if cleanup_deadline_at <= requested_at { + return Err(invalid_context( + "cancellation deadline must follow the original request", + )); + } + Ok(Self { + request_id, + root_request_id, + root_workflow_instance_id, + root_workflow_run_id, + parent_request_id, + reason, + requester: normalized_requester, + source: context_text(value, "source")?.to_owned(), + requested_at, + cleanup_deadline_at, + lineage: normalized, + }) + } + + pub fn request_id(&self) -> &str { + &self.request_id + } + pub fn root_request_id(&self) -> &str { + &self.root_request_id + } + pub fn root_workflow_instance_id(&self) -> &str { + &self.root_workflow_instance_id + } + pub fn root_workflow_run_id(&self) -> &str { + &self.root_workflow_run_id + } + pub fn parent_request_id(&self) -> Option<&str> { + self.parent_request_id.as_deref() + } + pub fn reason(&self) -> Option<&str> { + self.reason.as_deref() + } + pub fn requester(&self) -> &BTreeMap { + &self.requester + } + pub fn source(&self) -> &str { + &self.source + } + pub fn requested_at(&self) -> DateTime { + self.requested_at + } + pub fn deadline(&self) -> DateTime { + self.cleanup_deadline_at + } + pub fn lineage(&self) -> &[CancellationLineage] { + &self.lineage + } + + /// Detached metadata in the portable context schema. + pub fn to_value(&self) -> Value { + json!({ + "schema": "durable-workflow.cancellation-context/v1", + "request_id": self.request_id, "root_request_id": self.root_request_id, + "root_workflow_instance_id": self.root_workflow_instance_id, + "root_workflow_run_id": self.root_workflow_run_id, + "parent_request_id": self.parent_request_id, "reason": self.reason, + "requester": self.requester, "source": self.source, + "requested_at": self.requested_at.to_rfc3339_opts(SecondsFormat::Micros, true), + "cleanup_deadline_at": self.cleanup_deadline_at.to_rfc3339_opts(SecondsFormat::Micros, true), + "lineage": self.lineage.iter().map(CancellationLineage::to_value).collect::>(), + }) + } +} diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index 0165d2f..c9cc805 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -91,6 +91,7 @@ pub struct CancellationRequest { pub requested_at: String, pub cleanup_deadline_at: String, pub history_refresh_page_token: Option, + pub context: Option, } /// Acknowledgment of a request targeting the current durable run. @@ -179,6 +180,7 @@ impl CancellationRequest { requested_at: requested_at.to_owned(), cleanup_deadline_at: cleanup_deadline_at.to_owned(), history_refresh_page_token: Some(text(value, "history_refresh_page_token")?.to_owned()), + context: None, }) } } @@ -455,7 +457,10 @@ impl CancellationHistory { } } let mut state = Self { - request: observation.cloned(), + request: observation.cloned().map(|mut request| { + request.context = None; + request + }), delivery: None, request_index: events.len(), delivery_index: None, @@ -496,7 +501,37 @@ impl CancellationHistory { let deadline_text = text(&event.payload, "cleanup_deadline_at")?; let deadline = DateTime::parse_from_rfc3339(deadline_text) .map_err(|_| invalid("canonical cleanup deadline is invalid"))?; - if deadline <= requested { + let context = event + .payload + .get("cancellation") + .map(CancellationContext::from_value) + .transpose()?; + if let Some(context) = &context { + let local = context.lineage().last().unwrap(); + if context.request_id() != request_id + || local.workflow_run_id() != event_run + || event + .payload + .get("workflow_instance_id") + .is_some_and(|instance| { + instance.as_str() != Some(local.workflow_instance_id()) + }) + || context.deadline() != deadline + || context.requested_at() > requested + || event + .payload + .get("reason") + .is_some_and(|reason| match reason { + Value::Null => context.reason().is_some(), + Value::String(reason) => Some(reason.as_str()) != context.reason(), + _ => true, + }) + { + return Err(invalid( + "canonical cancellation context does not match its request event", + )); + } + } else if deadline <= requested { return Err(invalid( "canonical cleanup deadline must follow the request", )); @@ -510,12 +545,22 @@ impl CancellationHistory { "observation changes the original request or cleanup deadline", )); } - } else { + } + if context.is_some() || observation.is_none() { state.request = Some(CancellationRequest { request_id: request_id.to_owned(), - requested_at: recorded_at.to_owned(), + requested_at: context.as_ref().map_or_else( + || recorded_at.to_owned(), + |context| { + context + .requested_at() + .to_rfc3339_opts(chrono::SecondsFormat::Micros, true) + }, + ), cleanup_deadline_at: deadline_text.to_owned(), - history_refresh_page_token: None, + history_refresh_page_token: observation + .and_then(|request| request.history_refresh_page_token.clone()), + context, }); } state.request_index = index; @@ -535,6 +580,19 @@ impl CancellationHistory { { return Err(invalid("delivery names a different original request")); } + if let Some(snapshot) = event.payload.get("cancellation") { + let context = CancellationContext::from_value(snapshot)?; + if state + .request + .as_ref() + .and_then(|request| request.context.as_ref()) + != Some(&context) + { + return Err(invalid( + "delivery changes the canonical cancellation context", + )); + } + } state.delivery = Some(delivery); state.delivery_index = Some(index); } @@ -964,6 +1022,23 @@ impl WorkflowState { } impl WorkflowContext { + /// Original immutable metadata, visible only at committed delivery. + pub fn cancellation_context(&self) -> Result> { + let state = self + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)?; + Ok(if state.cancellation_consumed { + state + .cancellation_history + .request + .as_ref() + .and_then(|request| request.context.clone()) + } else { + None + }) + } + /// Shield explicit cleanup from repeated cancellation checks. /// /// Keep the returned guard alive across cleanup awaits. Scopes can nest. diff --git a/src/lib.rs b/src/lib.rs index a036e9c..1e743a3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,9 +1,12 @@ #![doc = include_str!("../README.md")] +mod cancellation_context; mod cooperative_cancellation; mod runtime_payloads; mod runtime_uploads; +pub use cancellation_context::{CancellationContext, CancellationLineage}; + pub use cooperative_cancellation::{ CancellationCallKind, CancellationDelivery, CancellationDeliveryReceipt, CancellationDeliveryReply, CancellationHistory, CancellationRequest, CancellationShield, diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs index 08df0ef..9f1aac4 100644 --- a/src/tests/cooperative_cancellation.rs +++ b/src/tests/cooperative_cancellation.rs @@ -1,5 +1,253 @@ use super::*; +fn context_snapshot() -> Value { + serde_json::from_str(include_str!( + "fixtures/cooperative-cancellation-context.json" + )) + .unwrap() +} + +fn context_request() -> HistoryEvent { + serde_json::from_value(json!({ + "event_type": "CooperativeCancellationRequested", "recorded_at": "2026-10-01T00:00:05Z", + "payload": { + "workflow_command_id": "request-1", "workflow_run_id": "run-1", "workflow_instance_id": "child-instance", + "reason": "maintenance", "cleanup_deadline_at": "2026-10-01T00:00:30.123456Z", + "cancellation": context_snapshot(), + } + })).unwrap() +} + +fn context_delivery() -> HistoryEvent { + event( + "CooperativeCancellationDelivered", + json!({ + "workflow_command_id": "request-1", "workflow_run_id": "run-1", + "sequence": 1, "call_kind": "timer", "cancellation": context_snapshot(), + }), + ) +} + +fn context_observation() -> CancellationRequest { + CancellationRequest::from_observation(&json!({ + "request_id": "request-1", "requested_at": "2026-10-01T00:00:05Z", + "cleanup_deadline_at": "2026-10-01T00:00:30.123456Z", + "history_refresh_page_token": "opaque-first-page", "cancellation": context_snapshot(), + })) + .unwrap() +} + +fn context_task(events: Vec) -> WorkflowTask { + let mut task = cancellation_task(events); + task.run_id = Some("run-1".into()); + task +} + +#[test] +fn cooperative_context_retains_original_snapshot_and_detached_copies() { + let mut original = context_snapshot(); + let context = CancellationContext::from_value(&original).unwrap(); + original["requester"]["id"] = json!("changed"); + original["lineage"][0]["request_id"] = json!("changed"); + let mut detached = context.to_value(); + detached["reason"] = json!("changed"); + assert_eq!(context.to_value(), context_snapshot()); + assert_eq!(context.request_id(), "request-1"); + assert_eq!(context.root_request_id(), "root-1"); + assert_eq!(context.parent_request_id(), Some("root-1")); + assert_eq!(context.root_workflow_instance_id(), "parent-instance"); + assert_eq!(context.root_workflow_run_id(), "parent-run"); + assert_eq!(context.reason(), Some("maintenance")); + assert_eq!(context.source(), "control_plane"); + assert_eq!(context.requester()["id"], "operator-1"); + assert_eq!( + context.lineage()[1].workflow_instance_id(), + "child-instance" + ); + assert_eq!( + context.deadline() - context.requested_at(), + chrono::Duration::seconds(30) + ); +} + +#[test] +fn cooperative_context_normalizes_equivalent_timezone_and_object_order() { + let mut value = context_snapshot(); + value["requested_at"] = json!("2026-09-30T20:00:00.123456-04:00"); + value["cleanup_deadline_at"] = json!("2026-09-30T20:00:30.123456-04:00"); + assert_eq!( + CancellationContext::from_value(&value).unwrap(), + CancellationContext::from_value(&context_snapshot()).unwrap() + ); +} + +#[test] +fn cooperative_context_rejects_invalid_snapshots() { + for (field, value) in [ + ("schema", json!("unknown")), + ("request_id", json!("")), + ("source", json!(" ")), + ("parent_request_id", json!("wrong")), + ("root_request_id", json!("wrong")), + ("root_workflow_run_id", json!("wrong")), + ("reason", json!([])), + ("requester", json!([])), + ("lineage", json!([])), + ("requested_at", json!("2026-02-30T00:00:00Z")), + ("cleanup_deadline_at", json!("2026-10-01T00:00:00.123456Z")), + ] { + let mut snapshot = context_snapshot(); + snapshot[field] = value; + assert!( + CancellationContext::from_value(&snapshot).is_err(), + "{field}" + ); + } +} + +#[test] +fn cooperative_context_rejects_lineage_cycles_order_and_unrelated_requester_metadata() { + for field in ["request_id", "workflow_run_id"] { + let mut value = context_snapshot(); + value["lineage"][1][field] = value["lineage"][0][field].clone(); + assert!(CancellationContext::from_value(&value).is_err()); + } + let mut value = context_snapshot(); + value["lineage"].as_array_mut().unwrap().reverse(); + assert!(CancellationContext::from_value(&value).is_err()); + let mut value = context_snapshot(); + value["requester"]["authorization"] = json!("unsupported"); + assert!(CancellationContext::from_value(&value).is_err()); +} + +#[test] +fn cooperative_context_child_keeps_root_time_after_expired_local_admission() { + let mut request = context_request(); + request + .raw + .insert("recorded_at".into(), json!("2026-10-01T00:00:31Z")); + let state = CancellationHistory::from_events(&[request], "run-1", None).unwrap(); + let request = state.request.unwrap(); + assert_eq!(request.requested_at, "2026-10-01T00:00:00.123456Z"); + assert_eq!(request.cleanup_deadline_at, "2026-10-01T00:00:30.123456Z"); + assert_eq!(request.context.unwrap().to_value(), context_snapshot()); + assert!(state.delivery.is_none()); +} + +#[test] +fn cooperative_context_observation_keeps_refresh_route_and_history_supplies_metadata() { + let mut observation = context_observation(); + observation.context = Some(CancellationContext::from_value(&context_snapshot()).unwrap()); + let pending = CancellationHistory::from_events(&[], "run-1", Some(&observation)).unwrap(); + assert!(pending.request.unwrap().context.is_none()); + let state = CancellationHistory::from_events( + &[context_request(), context_delivery()], + "run-1", + Some(&observation), + ) + .unwrap(); + let request = state.request.unwrap(); + assert_eq!( + request.history_refresh_page_token.as_deref(), + Some("opaque-first-page") + ); + assert_eq!(request.requested_at, "2026-10-01T00:00:00.123456Z"); + assert_eq!(request.context.unwrap().to_value(), context_snapshot()); +} + +#[test] +fn cooperative_context_must_match_canonical_local_request_and_run() { + for (field, value) in [ + ("workflow_command_id", json!("other")), + ("workflow_instance_id", json!("other")), + ("cleanup_deadline_at", json!("2026-10-01T00:00:35Z")), + ("reason", json!("changed")), + ("cancellation", Value::Null), + ] { + let mut request = context_request(); + request.payload[field] = value; + assert!( + CancellationHistory::from_events(&[request], "run-1", None).is_err(), + "{field}" + ); + } + let mut request = context_request(); + request.payload["cancellation"]["lineage"][1]["workflow_run_id"] = json!("other"); + assert!(CancellationHistory::from_events(&[request], "run-1", None).is_err()); + let mut request = context_request(); + request + .raw + .insert("recorded_at".into(), json!("2026-09-30T23:59:59Z")); + assert!(CancellationHistory::from_events(&[request], "run-1", None).is_err()); +} + +#[test] +fn cooperative_context_delivery_cannot_change_the_accepted_snapshot() { + let mut delivery = context_delivery(); + delivery.payload["cancellation"]["reason"] = json!("changed"); + assert!( + CancellationHistory::from_events(&[context_request(), delivery], "run-1", None).is_err() + ); +} + +#[test] +fn cooperative_context_cold_worker_replay_restores_same_delivery_and_cleanup() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + assert!(ctx.cancellation_context()?.is_none()); + let Err(Error::CooperativeCancellationRequested(cancelled)) = + ctx.sleep(Duration::from_secs(10)).await + else { + panic!("expected canonical cancellation"); + }; + let context = cancelled.request.context.unwrap(); + assert_eq!(ctx.cancellation_context()?, Some(context.clone())); + let _shield = ctx.cancellation_shield()?; + ctx.throw_if_cancellation_requested()?; + ctx.activity("cleanup", json!([])).await?; + Ok(context.to_value()) + }); + let first = worker + .execute_workflow_task(context_task(vec![context_request(), context_delivery()])) + .unwrap(); + assert_eq!(first.len(), 1); + assert_eq!(first[0]["type"], "schedule_activity"); + assert_eq!(first[0]["activity_type"], "cleanup"); + let mut history = vec![context_request(), context_delivery()]; + history.extend(completed_activity(2, "cleanup", json!("cleaned"))); + for _restart in 0..2 { + let result = worker + .execute_workflow_task(context_task(history.clone())) + .unwrap(); + assert_eq!(result[0]["type"], "complete_workflow"); + assert_eq!( + decode_wire_value(&result[0]["result"], DEFAULT_CODEC).unwrap(), + context_snapshot() + ); + } +} + +#[test] +fn cooperative_context_explicit_check_retains_delivered_metadata() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _input| async move { + let _ = ctx.sleep(Duration::from_secs(10)).await; + let Err(Error::CooperativeCancellationRequested(cancelled)) = + ctx.throw_if_cancellation_requested() + else { + panic!("expected the delivered cancellation"); + }; + assert_eq!( + cancelled.request.context.unwrap().to_value(), + context_snapshot() + ); + Ok(Value::Null) + }); + worker + .execute_workflow_task(context_task(vec![context_request(), context_delivery()])) + .unwrap(); +} + fn request_observation() -> Value { json!({ "request_id": "original-request", diff --git a/src/tests/fixtures/cooperative-cancellation-context.json b/src/tests/fixtures/cooperative-cancellation-context.json new file mode 100644 index 0000000..dbe7e89 --- /dev/null +++ b/src/tests/fixtures/cooperative-cancellation-context.json @@ -0,0 +1,17 @@ +{ + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "request-1", + "root_request_id": "root-1", + "root_workflow_instance_id": "parent-instance", + "root_workflow_run_id": "parent-run", + "parent_request_id": "root-1", + "reason": "maintenance", + "requester": {"type": "operator", "id": "operator-1", "label": "Maintainer"}, + "source": "control_plane", + "requested_at": "2026-10-01T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-01T00:00:30.123456Z", + "lineage": [ + {"request_id": "root-1", "workflow_instance_id": "parent-instance", "workflow_run_id": "parent-run"}, + {"request_id": "request-1", "workflow_instance_id": "child-instance", "workflow_run_id": "run-1"} + ] +} From 8cb36ec547bfa4a3a976482f8b2d4a0d46ba84a2 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 1 Oct 2026 23:21:43 +0000 Subject: [PATCH 30/58] Bind typed child cancellation policies across Rust replay --- docs/cooperative-cancellation-design.md | 39 ++- src/lib.rs | 171 ++++++++++ src/tests/child_workflow_policies.rs | 296 ++++++++++++++++++ .../child-cancellation-policy-changed.json | 14 + .../child-start-policy-conflict.json | 15 + tests/replay_regression_corpus.rs | 12 + 6 files changed, 546 insertions(+), 1 deletion(-) create mode 100644 src/tests/child_workflow_policies.rs create mode 100644 tests/fixtures/replay-regressions/child-cancellation-policy-changed.json create mode 100644 tests/fixtures/replay-regressions/child-start-policy-conflict.json diff --git a/docs/cooperative-cancellation-design.md b/docs/cooperative-cancellation-design.md index a877a0a..25bad4e 100644 --- a/docs/cooperative-cancellation-design.md +++ b/docs/cooperative-cancellation-design.md @@ -27,9 +27,46 @@ request/run binding, cycles, budget and delivery snapshot equality. Older cancellation histories without rich context keep their existing delivery behavior with `context == None`. +## Child policies + +`ChildWorkflowOptions` accepts `CancellationPolicy` through its builder for +ordinary, parallel and selected child calls. + +```rust +ChildWorkflowOptions::new("python-workers") + .cancellation_policy(CancellationPolicy::WaitCancellationCompleted) + .parent_close_policy(ParentClosePolicy::RequestCancellation) +``` + +`TryCancel` requests child cleanup and delivers parent cancellation without +waiting. `WaitCancellationCompleted` parks the parent until the child reaches a +recorded terminal outcome, releasing its task claim for other work. `Abandon` +leaves the child independent and preserves the historical default. Parent +closure remains separate. `RequestCancellation` requests genuine cooperative +cleanup with the original lineage and budget. `RequestCancel` retains its +legacy terminal behavior. + +Cold replay compares both policies, including cancellation delivery and groups. +Omitted historical fields preserve the `Abandon` defaults. Later events with +missing fields retain the scheduled snapshot. Invalid or conflicting policy +history fails explicitly. A worker without the cooperation opt-in refuses the +commands before completion with its identity and required protocol. Server also +checks the immutable task claim and installed backend. + +## Rust API release boundary + +The candidate adds a variant to the existing exhaustive `ParentClosePolicy` +enum and a field to the public `ChildWorkflowOptions` struct. Existing struct +literals and exhaustive matches need updates. The cooperative error variants +also extend an existing exhaustive public enum. These changes require a major +Rust SDK release if retained. This draft does not authorize that release or +change the published package version. The qualified release proposal must +include its migration notes and receive the major-release decision before +publication. + ## Remaining qualification -Portable operation policies, nested scopes and deterministic remaining-time +Portable activity policies, nested scopes and deterministic remaining-time helpers still need completion. Remaining time must use the replayed workflow clock. Do not subtract the host clock from the deadline in workflow code. The runtime continues enforcing the original deadline and fencing task and activity diff --git a/src/lib.rs b/src/lib.rs index 1e743a3..56f6414 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1273,7 +1273,10 @@ pub struct SagaCompensationFailure { pub enum ParentClosePolicy { #[default] Abandon, + /// Legacy terminal cancellation without cooperative cleanup. RequestCancel, + /// Request cooperative cleanup with the original lineage and deadline. + RequestCancellation, Terminate, } @@ -1282,11 +1285,35 @@ impl ParentClosePolicy { match self { Self::Abandon => "abandon", Self::RequestCancel => "request_cancel", + Self::RequestCancellation => "request_cancellation", Self::Terminate => "terminate", } } } +/// Cancellation at an awaiting child call. +#[non_exhaustive] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum CancellationPolicy { + /// Request child cleanup and deliver parent cancellation without waiting. + TryCancel, + /// Wait for the child's recorded terminal outcome before parent delivery. + WaitCancellationCompleted, + /// Leave the child independent. This preserves the historical default. + #[default] + Abandon, +} + +impl CancellationPolicy { + fn as_str(self) -> &'static str { + match self { + Self::TryCancel => "try_cancel", + Self::WaitCancellationCompleted => "wait_cancellation_completed", + Self::Abandon => "abandon", + } + } +} + /// Durable retry policy for one child workflow invocation. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ChildWorkflowRetryPolicy { @@ -1302,6 +1329,7 @@ pub struct ChildWorkflowRetryPolicy { pub struct ChildWorkflowOptions { pub task_queue: String, pub parent_close_policy: ParentClosePolicy, + pub cancellation_policy: CancellationPolicy, pub retry_policy: Option, pub execution_timeout_seconds: Option, pub run_timeout_seconds: Option, @@ -1312,6 +1340,7 @@ impl ChildWorkflowOptions { Self { task_queue: task_queue.into(), parent_close_policy: ParentClosePolicy::Abandon, + cancellation_policy: CancellationPolicy::Abandon, retry_policy: None, execution_timeout_seconds: None, run_timeout_seconds: None, @@ -1323,6 +1352,11 @@ impl ChildWorkflowOptions { self } + pub fn cancellation_policy(mut self, policy: CancellationPolicy) -> Self { + self.cancellation_policy = policy; + self + } + pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self { self.retry_policy = Some(policy); self @@ -7889,6 +7923,21 @@ impl Worker { ctx: &WorkflowContext, commands: Vec, ) -> Result { + if !self.cooperative_cancellation_enabled + && commands.iter().any(|command| { + command["type"] == "start_child_workflow" + && (command["parent_close_policy"] == "request_cancellation" + || matches!( + command["cancellation_policy"].as_str(), + Some("try_cancel" | "wait_cancellation_completed") + )) + }) + { + return Err(Error::CooperativeCancellationUnavailable(format!( + "child_cancellation_policy_not_supported: Rust worker {} must enable cooperative cancellation with worker protocol 1.20 and a compatible Server/Native backend", + self.worker_id, + ))); + } let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?; let state = ctx.state.lock().map_err(|_| Error::WorkflowStatePoisoned)?; if state.cancellation_delivery_intent.is_some() @@ -9829,6 +9878,7 @@ enum RecordedCommand { ChildWorkflow { sequence: u64, workflow_type: Option, + policies: RecordedChildPolicies, outcome: Option, parallel_group_path: Option>, }, @@ -10167,6 +10217,109 @@ struct RecordedActivityOptions { retry_policy: ActivityRetrySnapshot, } +#[derive(Clone, Debug)] +struct RecordedChildPolicies { + parent_close_policy: String, + cancellation_policy: String, +} + +fn recorded_child_policy_value<'a>( + payload: &'a Value, + field: &str, + sequence: u64, +) -> Result> { + let Some(value) = payload.get(field).filter(|value| !value.is_null()) else { + return Ok(None); + }; + let valid = match (field, value.as_str()) { + ( + "parent_close_policy", + Some("abandon" | "request_cancel" | "request_cancellation" | "terminate"), + ) => true, + ("cancellation_policy", Some("abandon" | "try_cancel" | "wait_cancellation_completed")) => { + true + } + _ => false, + }; + if !valid { + return Err(invalid_recorded_history( + "invalid_child_workflow_policy_history", + sequence, + "supported child workflow policy", + &value.to_string(), + "child workflow history contains an invalid policy", + )); + } + Ok(value.as_str()) +} + +fn recorded_child_policies( + events: &[&HistoryEvent], + scheduled: &HistoryEvent, + sequence: u64, +) -> Result { + let parent_close_policy = + recorded_child_policy_value(&scheduled.payload, "parent_close_policy", sequence)? + .unwrap_or("abandon") + .to_string(); + let cancellation_policy = + recorded_child_policy_value(&scheduled.payload, "cancellation_policy", sequence)? + .unwrap_or("abandon") + .to_string(); + for event in events { + for (field, expected) in [ + ("parent_close_policy", parent_close_policy.as_str()), + ("cancellation_policy", cancellation_policy.as_str()), + ] { + if let Some(actual) = recorded_child_policy_value(&event.payload, field, sequence)? { + if actual != expected { + return Err(invalid_recorded_history( + "child_workflow_policy_history_conflict", + sequence, + expected, + actual, + "child workflow policy changed between history events", + )); + } + } + } + } + Ok(RecordedChildPolicies { + parent_close_policy, + cancellation_policy, + }) +} + +fn ensure_child_policies_match( + sequence: u64, + recorded: &RecordedChildPolicies, + current: &ChildWorkflowOptions, +) -> Result<()> { + for (field, expected, actual) in [ + ( + "parent_close_policy", + recorded.parent_close_policy.as_str(), + current.parent_close_policy.as_str(), + ), + ( + "cancellation_policy", + recorded.cancellation_policy.as_str(), + current.cancellation_policy.as_str(), + ), + ] { + if expected != actual { + return Err(Error::NonDeterministicReplay(ReplayFailure::new( + "child_workflow_policy_changed", + Some(sequence), + Some(expected.to_string()), + Some(actual.to_string()), + format!("child workflow {field} changed during replay"), + ))); + } + } + Ok(()) +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize)] enum RecordedSnapshotValue { /// Older history did not persist this field, so it cannot constrain replay. @@ -12994,6 +13147,7 @@ impl ChildWorkflowCall { match recorded { RecordedCommand::ChildWorkflow { workflow_type, + policies, outcome, parallel_group_path, .. @@ -13018,6 +13172,11 @@ impl ChildWorkflowCall { ))); } } + if let Err(error) = + ensure_child_policies_match(sequence, &policies, &self.options) + { + return Poll::Ready(Err(error)); + } if let Err(error) = state.replay_cancellation_at(cursor, CancellationCallKind::Child) { @@ -13095,6 +13254,12 @@ impl ChildWorkflowCall { let object = command .as_object_mut() .expect("child workflow command is always an object"); + if self.options.cancellation_policy != CancellationPolicy::Abandon { + object.insert( + "cancellation_policy".to_string(), + json!(self.options.cancellation_policy.as_str()), + ); + } if let Some(policy) = &self.options.retry_policy { let mut retry_policy = serde_json::Map::new(); if let Some(max_attempts) = policy.max_attempts { @@ -13885,6 +14050,7 @@ fn recorded_commands( let is_child_workflow = matches!( event.event_type.as_str(), "ChildWorkflowScheduled" + | "ChildRunStarted" | "ChildRunCompleted" | "ChildRunFailed" | "ChildRunCancelled" @@ -14137,6 +14303,7 @@ fn recorded_commands( "child workflow replay requires exactly one recorded schedule event", )); } + let policies = recorded_child_policies(&child_events, scheduled[0], sequence)?; let workflow_type = child_events.iter().find_map(|event| { event .payload @@ -14194,6 +14361,7 @@ fn recorded_commands( return Ok(RecordedCommand::ChildWorkflow { sequence, workflow_type, + policies, outcome: outcomes.pop(), parallel_group_path, }); @@ -15830,6 +15998,7 @@ fn value_as_u64(value: &Value) -> Option { #[cfg(test)] mod tests { use super::*; + mod child_workflow_policies; mod cooperative_cancellation; mod runtime_payloads; mod runtime_uploads; @@ -22643,6 +22812,7 @@ mod tests { "child_workflow_instance_id": "wf-child", "child_workflow_run_id": "run-child", "child_workflow_type": "python.child", + "parent_close_policy": "terminate", }), raw: HashMap::new(), }, @@ -22866,6 +23036,7 @@ mod tests { }), ); task.workflow_type = "rust.handled-parent".to_string(); + task.history_events[0].payload["parent_close_policy"] = json!("abandon"); let commands = worker.execute_workflow_task(task).expect("handled failure"); assert_eq!(commands[0]["type"], "complete_workflow"); diff --git a/src/tests/child_workflow_policies.rs b/src/tests/child_workflow_policies.rs new file mode 100644 index 0000000..e37df14 --- /dev/null +++ b/src/tests/child_workflow_policies.rs @@ -0,0 +1,296 @@ +use super::*; + +fn policy_worker( + parent: ParentClosePolicy, + operation: CancellationPolicy, + mode: &'static str, + enabled: bool, +) -> Worker { + let mut worker = Worker::new( + Client::builder("http://127.0.0.1:1").build().unwrap(), + "queue", + ) + .cooperative_cancellation(enabled); + worker.register_workflow("child-policy", move |ctx, _input| async move { + let options = ChildWorkflowOptions::new("queue") + .parent_close_policy(parent) + .cancellation_policy(operation); + match mode { + "parallel" => { + ctx.parallel(vec![ParallelOperation::child_workflow( + "child", + options, + json!(["argument"]), + )]) + .await?; + } + "selection" => { + ctx.select_keyed(vec![( + SelectionKey::from("child"), + ParallelOperation::child_workflow("child", options, json!(["argument"])), + )]) + .await?; + } + _ => { + ctx.start_child_workflow("child", options, json!(["argument"])) + .await?; + } + } + Ok(json!("finished")) + }); + worker +} + +fn policy_task(events: Vec) -> WorkflowTask { + let mut task = workflow_task("child-policy", events, DEFAULT_CODEC); + task.run_id = Some("run".into()); + task +} + +fn policy_event(kind: &str, payload: Value) -> HistoryEvent { + serde_json::from_value(json!({"event_type": kind, "payload": payload})).unwrap() +} + +fn scheduled_policy_history(command: &Value) -> Vec { + let mut payload = command.clone(); + let object = payload.as_object_mut().unwrap(); + object.remove("type"); + object.remove("arguments"); + object.remove("queue"); + object.remove("workflow_type"); + object.insert("sequence".into(), json!(1)); + object.insert("child_workflow_type".into(), json!("child")); + object.insert("child_workflow_run_id".into(), json!("child-run")); + vec![policy_event("ChildWorkflowScheduled", payload)] +} + +fn policy_cancellation_history(mut events: Vec, mode: &str) -> Vec { + events.push(serde_json::from_value(json!({ + "event_type": "CooperativeCancellationRequested", "recorded_at": "2026-10-01T00:00:00Z", + "payload": {"workflow_run_id": "run", "workflow_command_id": "request-1", "cleanup_deadline_at": "2026-10-01T00:00:30Z"}, + })).unwrap()); + events.push(policy_event("CooperativeCancellationDelivered", json!({ + "workflow_run_id": "run", "workflow_command_id": "request-1", "sequence": 1, + "call_kind": if mode == "sequential" { "child" } else { "parallel" }, "sequence_span": 1, + }))); + events +} + +fn assert_policy_failure(error: Error, reason: &str) { + let Error::NonDeterministicReplay(failure) = error else { + panic!("expected replay failure, got {error:?}"); + }; + assert_eq!(failure.reason, reason); + assert_eq!(failure.sequence, Some(1)); +} + +#[test] +fn child_policies_encode_all_choices_and_replay_the_same_snapshot() { + for parent in [ + ParentClosePolicy::Abandon, + ParentClosePolicy::RequestCancel, + ParentClosePolicy::RequestCancellation, + ParentClosePolicy::Terminate, + ] { + for operation in [ + CancellationPolicy::Abandon, + CancellationPolicy::TryCancel, + CancellationPolicy::WaitCancellationCompleted, + ] { + let worker = policy_worker(parent, operation, "sequential", true); + let commands = worker.execute_workflow_task(policy_task(vec![])).unwrap(); + assert_eq!(commands[0]["parent_close_policy"], parent.as_str()); + if operation == CancellationPolicy::Abandon { + assert!(commands[0].get("cancellation_policy").is_none()); + } else { + assert_eq!(commands[0]["cancellation_policy"], operation.as_str()); + } + assert_eq!( + decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).unwrap(), + json!(["argument"]) + ); + let mut events = scheduled_policy_history(&commands[0]); + events.push(policy_event( + "ChildRunStarted", + json!({"sequence": 1, "child_workflow_type": "child"}), + )); + events.push(policy_event( + "ChildRunCompleted", + json!({"sequence": 1, "result": fixture_envelope(json!("child-result"))}), + )); + let completed = worker.execute_workflow_task(policy_task(events)).unwrap(); + assert_eq!( + decode_wire_value(&completed[0]["result"], DEFAULT_CODEC).unwrap(), + json!("finished") + ); + } + } +} + +#[test] +fn child_policies_are_checked_in_ordinary_parallel_selection_and_cancellation_replay() { + for mode in ["sequential", "parallel", "selection"] { + let original = policy_worker( + ParentClosePolicy::RequestCancellation, + CancellationPolicy::WaitCancellationCompleted, + mode, + true, + ); + let commands = original.execute_workflow_task(policy_task(vec![])).unwrap(); + let scheduled = scheduled_policy_history(&commands[0]); + for delivered in [false, true] { + let events = if delivered { + policy_cancellation_history(scheduled.clone(), mode) + } else { + scheduled.clone() + }; + let unchanged = original + .execute_workflow_task(policy_task(events.clone())) + .unwrap(); + if delivered { + assert_eq!(unchanged[0]["type"], "fail_workflow"); + } else { + assert!(unchanged.is_empty()); + } + for (parent, operation) in [ + ( + ParentClosePolicy::Abandon, + CancellationPolicy::WaitCancellationCompleted, + ), + ( + ParentClosePolicy::RequestCancellation, + CancellationPolicy::TryCancel, + ), + ] { + let changed = policy_worker(parent, operation, mode, true); + assert_policy_failure( + changed + .execute_workflow_task(policy_task(events.clone())) + .unwrap_err(), + "child_workflow_policy_changed", + ); + } + } + } +} + +#[test] +fn child_policies_preserve_historical_defaults_and_refuse_later_changes() { + let worker = policy_worker( + ParentClosePolicy::Abandon, + CancellationPolicy::Abandon, + "sequential", + false, + ); + let commands = worker.execute_workflow_task(policy_task(vec![])).unwrap(); + assert!(commands[0].get("cancellation_policy").is_none()); + let mut events = scheduled_policy_history(&commands[0]); + events[0] + .payload + .as_object_mut() + .unwrap() + .remove("parent_close_policy"); + assert!(worker + .execute_workflow_task(policy_task(events.clone())) + .unwrap() + .is_empty()); + let changed = policy_worker( + ParentClosePolicy::RequestCancellation, + CancellationPolicy::WaitCancellationCompleted, + "sequential", + true, + ); + assert_policy_failure( + changed + .execute_workflow_task(policy_task(events)) + .unwrap_err(), + "child_workflow_policy_changed", + ); +} + +#[test] +fn child_policies_reject_invalid_and_conflicting_history() { + let worker = policy_worker( + ParentClosePolicy::RequestCancellation, + CancellationPolicy::WaitCancellationCompleted, + "sequential", + true, + ); + let commands = worker.execute_workflow_task(policy_task(vec![])).unwrap(); + let scheduled = scheduled_policy_history(&commands[0]); + for event_type in [ + "ChildRunStarted", + "ChildRunCompleted", + "ChildRunFailed", + "ChildRunCancelled", + "ChildRunTerminated", + ] { + for (field, conflicting) in [ + ("parent_close_policy", "terminate"), + ("cancellation_policy", "try_cancel"), + ] { + let mut events = scheduled.clone(); + let mut payload = json!({"sequence": 1}); + payload[field] = json!(conflicting); + events.push(policy_event(event_type, payload)); + assert_policy_failure( + worker + .execute_workflow_task(policy_task(events)) + .unwrap_err(), + "child_workflow_policy_history_conflict", + ); + } + } + for field in ["parent_close_policy", "cancellation_policy"] { + for invalid in [ + json!("unknown"), + json!(true), + json!(1), + json!([]), + json!({"type": "abandon"}), + ] { + let mut events = scheduled.clone(); + events[0].payload[field] = invalid; + assert_policy_failure( + worker + .execute_workflow_task(policy_task(events)) + .unwrap_err(), + "invalid_child_workflow_policy_history", + ); + } + } +} + +#[test] +fn child_policies_require_worker_opt_in_but_legacy_choices_remain_available() { + for (parent, operation) in [ + ( + ParentClosePolicy::RequestCancellation, + CancellationPolicy::Abandon, + ), + (ParentClosePolicy::Abandon, CancellationPolicy::TryCancel), + ( + ParentClosePolicy::Abandon, + CancellationPolicy::WaitCancellationCompleted, + ), + ] { + let worker = policy_worker(parent, operation, "sequential", false); + let Error::CooperativeCancellationUnavailable(message) = worker + .execute_workflow_task(policy_task(vec![])) + .unwrap_err() + else { + panic!("expected capability diagnostic"); + }; + assert!(message.contains("child_cancellation_policy_not_supported")); + assert!(message.contains(&worker.worker_id)); + assert!(message.contains("1.20")); + } + for parent in [ + ParentClosePolicy::Abandon, + ParentClosePolicy::RequestCancel, + ParentClosePolicy::Terminate, + ] { + let worker = policy_worker(parent, CancellationPolicy::Abandon, "sequential", false); + assert!(worker.execute_workflow_task(policy_task(vec![])).is_ok()); + } +} diff --git a/tests/fixtures/replay-regressions/child-cancellation-policy-changed.json b/tests/fixtures/replay-regressions/child-cancellation-policy-changed.json new file mode 100644 index 0000000..d3089d7 --- /dev/null +++ b/tests/fixtures/replay-regressions/child-cancellation-policy-changed.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-child-cancellation-policy-changed", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": {"type": "corpus.child-policy-author", "input": [], "payload_codec": "avro"}, + "history": [ + {"event_type": "ChildWorkflowScheduled", "payload": {"sequence": 1, "child_workflow_type": "child", "parent_close_policy": "abandon", "cancellation_policy": "wait_cancellation_completed"}} + ], + "command_sequence": [{"type": "complete_workflow", "result": "unreachable"}], + "expected": {"type": "complete_workflow", "result": "unreachable"}, + "expected_failure": "child_workflow_policy_changed" +} diff --git a/tests/fixtures/replay-regressions/child-start-policy-conflict.json b/tests/fixtures/replay-regressions/child-start-policy-conflict.json new file mode 100644 index 0000000..d1523d0 --- /dev/null +++ b/tests/fixtures/replay-regressions/child-start-policy-conflict.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-child-start-policy-conflict", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": {"type": "corpus.child-policy-author", "input": [], "payload_codec": "avro"}, + "history": [ + {"event_type": "ChildWorkflowScheduled", "payload": {"sequence": 1, "child_workflow_type": "child", "parent_close_policy": "abandon", "cancellation_policy": "abandon"}}, + {"event_type": "ChildRunStarted", "payload": {"sequence": 1, "child_workflow_type": "child", "cancellation_policy": "try_cancel"}} + ], + "command_sequence": [{"type": "complete_workflow", "result": "unreachable"}], + "expected": {"type": "complete_workflow", "result": "unreachable"}, + "expected_failure": "child_workflow_policy_history_conflict" +} diff --git a/tests/replay_regression_corpus.rs b/tests/replay_regression_corpus.rs index 8a7a17d..6df0226 100644 --- a/tests/replay_regression_corpus.rs +++ b/tests/replay_regression_corpus.rs @@ -411,6 +411,7 @@ async fn execute_fixture_delivery(fixture: &Value, delivery_id: &str) -> Result< | "corpus.durable-selection" | "corpus.durable-selection-portable-affinity" | "corpus.redrive-boundary" + | "corpus.child-policy-author" ) { return Err(format!( "replay fixture {fixture_id} has no registered Rust workflow {workflow_type:?}" @@ -607,6 +608,17 @@ async fn execute_fixture_delivery(fixture: &Value, delivery_id: &str) -> Result< }, ); } + "corpus.child-policy-author" => { + worker.register_workflow(workflow_type, |ctx, _input| async move { + ctx.start_child_workflow( + "child", + ChildWorkflowOptions::new("regression-corpus"), + json!([]), + ) + .await?; + Ok(json!("unreachable")) + }); + } "corpus.redrive-boundary" => { worker.register_workflow(workflow_type, |ctx, _input| async move { ctx.activity("corpus.redrive.activity", json!([])).await?; From b9998173741d34a5b66eb60c41660790ff108331 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 00:32:53 +0000 Subject: [PATCH 31/58] Report remote cancellation after dropping the Rust callback --- .github/workflows/ci.yml | 40 ++++- docker-compose.native-cancellation.yml | 14 ++ docs/cooperative-cancellation-design.md | 24 +++ src/cooperative_cancellation.rs | 163 ++++++++++++++++-- src/tests/cooperative_cancellation.rs | 209 ++++++++++++++++++++++++ tests/cooperative_server.rs | 83 +++++++++- 6 files changed, 512 insertions(+), 21 deletions(-) create mode 100644 docker-compose.native-cancellation.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 85f3915..3062ace 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,10 @@ on: description: Exact 40-character public Server candidate SHA type: string default: "" + native_commit: + description: Optional exact public Native candidate SHA for readonly source qualification + type: string + default: "" permissions: contents: read @@ -199,8 +203,11 @@ jobs: env: COMPOSE_PROJECT_NAME: rust-cooperative-${{ github.run_id }}-${{ github.run_attempt }} SERVER_CANDIDATE_SHA: ${{ inputs.server_commit }} + NATIVE_CANDIDATE_SHA: ${{ inputs.native_commit }} DURABLE_WORKFLOW_SERVER_SOURCE: ${{ github.workspace }}/.cooperative-server-source + DURABLE_WORKFLOW_NATIVE_SOURCE: ${{ github.workspace }}/.cooperative-native-source DURABLE_WORKFLOW_AUTH_TOKEN: test-token + COMPOSE_FILE: docker-compose.cooperative.yml steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: @@ -216,17 +223,37 @@ jobs: persist-credentials: false - name: Verify checked-out candidate identity run: test "$(git -C .cooperative-server-source rev-parse HEAD)" = "$SERVER_CANDIDATE_SHA" + - name: Require an exact optional Native candidate + if: ${{ inputs.native_commit != '' }} + run: '[[ "$NATIVE_CANDIDATE_SHA" =~ ^[0-9a-f]{40}$ ]]' + - name: Check out the public Native candidate + if: ${{ inputs.native_commit != '' }} + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: durable-workflow/workflow + ref: ${{ inputs.native_commit }} + path: .cooperative-native-source + persist-credentials: false + - name: Verify Native identity and select its readonly source overlay + if: ${{ inputs.native_commit != '' }} + run: | + test "$(git -C .cooperative-native-source rev-parse HEAD)" = "$NATIVE_CANDIDATE_SHA" + printf 'COMPOSE_FILE=docker-compose.cooperative.yml:docker-compose.native-cancellation.yml\n' >> "$GITHUB_ENV" + printf 'DURABLE_WORKFLOW_NATIVE_SOURCE_QUALIFICATION=1\n' >> "$GITHUB_ENV" - name: Install Rust 1.86 uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable with: toolchain: 1.86.0 - name: Start isolated Server stack - run: docker compose -f docker-compose.cooperative.yml up -d --build --wait --wait-timeout 300 + run: docker compose up -d --build --wait --wait-timeout 300 - name: Retain runtime identity and select endpoint run: | - docker compose -f docker-compose.cooperative.yml exec -T server cat /app/.package-provenance > cooperative-package-provenance.txt - docker compose -f docker-compose.cooperative.yml images --format json > cooperative-images.json - endpoint="$(docker compose -f docker-compose.cooperative.yml port server 8080)" + docker compose exec -T server cat /app/.package-provenance > cooperative-package-provenance.txt + docker compose images --format json > cooperative-images.json + jq --null-input --arg sdk "$GITHUB_SHA" --arg server "$SERVER_CANDIDATE_SHA" --arg native "$NATIVE_CANDIDATE_SHA" \ + '{qualification:"source",sdk_commit:$sdk,server_commit:$server,native_source_overlay:($native | if length > 0 then . else null end)}' \ + > cooperative-source-provenance.json + endpoint="$(docker compose port server 8080)" echo "DURABLE_WORKFLOW_SERVER_URL=http://$endpoint" >> "$GITHUB_ENV" - name: Run actual Worker cooperative scenarios shell: bash @@ -240,15 +267,16 @@ jobs: cooperative-results.log cooperative-package-provenance.txt cooperative-images.json + cooperative-source-provenance.json retention-days: 7 if-no-files-found: warn - name: Emit failure diagnostics if: ${{ failure() }} - run: docker compose -f docker-compose.cooperative.yml logs --no-color --tail 100 bootstrap server worker repair + run: docker compose logs --no-color --tail 100 bootstrap server worker repair - name: Remove task runtime resources if: ${{ always() }} run: | - docker compose -f docker-compose.cooperative.yml down -v + docker compose down -v docker image rm "dw-rust-cooperative-server:$SERVER_CANDIDATE_SHA" target-branch-qualification: diff --git a/docker-compose.native-cancellation.yml b/docker-compose.native-cancellation.yml new file mode 100644 index 0000000..4393085 --- /dev/null +++ b/docker-compose.native-cancellation.yml @@ -0,0 +1,14 @@ +# Source qualification only. Published Composer authority is retained in the image. +services: + bootstrap: + volumes: + - ${DURABLE_WORKFLOW_NATIVE_SOURCE:?exact Native checkout}:/app/vendor/durable-workflow/workflow:ro + server: + volumes: + - ${DURABLE_WORKFLOW_NATIVE_SOURCE:?exact Native checkout}:/app/vendor/durable-workflow/workflow:ro + worker: + volumes: + - ${DURABLE_WORKFLOW_NATIVE_SOURCE:?exact Native checkout}:/app/vendor/durable-workflow/workflow:ro + repair: + volumes: + - ${DURABLE_WORKFLOW_NATIVE_SOURCE:?exact Native checkout}:/app/vendor/durable-workflow/workflow:ro diff --git a/docs/cooperative-cancellation-design.md b/docs/cooperative-cancellation-design.md index 25bad4e..5017c55 100644 --- a/docs/cooperative-cancellation-design.md +++ b/docs/cooperative-cancellation-design.md @@ -53,6 +53,30 @@ history fails explicitly. A worker without the cooperation opt-in refuses the commands before completion with its identity and required protocol. Server also checks the immutable task claim and installed backend. +## Remote callback-stop receipts + +The managed worker drops its activity callback future before reporting a stop. +Cloned `ActivityContext` values remain fenced. Only a canonical cancellation +observation on the original task, attempt and owner supplies the local request ID +for `Client::acknowledge_activity_cancellation()`. A lost lease, transport error, +shutdown or malformed observation does not supply a cooperative stop receipt. +Callbacks that never started are not reported as dropped. + +The explicit protocol 1.20 receipt request has a five-second budget. It validates +the Server's original claim, request and history receipt. A failed acknowledgment +remains an error and cannot become a completion or a new cleanup budget. A +duplicate returns the original event. The report covers the managed callback +future. Detached threads, processes and downstream effects need their own +cooperating cancellation and fencing. + +Connected source qualification accepts an optional exact `native_commit` in +addition to `server_commit`. The Native checkout is mounted read-only while the +image retains its published Composer authority. The source lane verifies the +durable receipt for callbacks with and without application heartbeats, +duplicates, the original deadline and stale result refusal. Actions retains all +three source identities and scenario history. This does not qualify a published +Native image or package. + ## Rust API release boundary The candidate adds a variant to the existing exhaustive `ParentClosePolicy` diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index c9cc805..32c57ca 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -1107,6 +1107,52 @@ fn acknowledgment( } impl Client { + /// Report a stopped and dropped remote callback under its original claim. + /// + /// Call only after callback execution has ended. This explicit protocol 1.20 + /// operation preserves the original cancellation request and grants no lease + /// or result publication authority. The request has one five-second budget. + pub async fn acknowledge_activity_cancellation( + &self, + task_id: &str, + activity_attempt_id: &str, + lease_owner: &str, + request_id: &str, + ) -> Result { + if [task_id, activity_attempt_id, lease_owner, request_id] + .iter() + .any(|value| value.trim().is_empty() || value.len() > 255) + { + return Err(invalid( + "activity stop receipt requires bounded original claim and request identities", + )); + } + tokio::time::timeout(CONTROL_BUDGET, async { + let value: Value = activity_task_response( + self.request_json( + reqwest::Method::POST, + &format!("/worker/activity-tasks/{}/acknowledge-cancellation", percent_encode_path_segment(task_id)), + RequestProtocol::Worker("1.20"), + Some(&json!({"activity_attempt_id":activity_attempt_id,"lease_owner":lease_owner,"request_id":request_id})), + ).await, + "acknowledge-cancellation", task_id, activity_attempt_id, + )?; + if value["task_id"].as_str() != Some(task_id) + || value["activity_attempt_id"].as_str() != Some(activity_attempt_id) + || value["lease_owner"].as_str() != Some(lease_owner) + || value["request_id"].as_str() != Some(request_id) + || value["acknowledged"].as_bool() != Some(true) + || value["duplicate"].as_bool().is_none() + || value.get("reason") != Some(&Value::Null) + || value["heartbeat_recorded"].as_bool() != Some(false) + || text(&value, "history_event_id").is_err() + { + return Err(invalid("activity callback-stop acknowledgment did not prove the original claim and receipt")); + } + Ok(value) + }).await.map_err(|_| Error::Timeout)? + } + /// Observe an exact activity attempt without renewing its lease or progress. /// /// This explicit worker-protocol 1.20 operation has one five-second budget. @@ -1650,28 +1696,42 @@ impl Worker { if guard.observe().await.is_err() { return Ok(ManagedPollOutcome::Handled); } - let invocation = self.execute_cooperative_activity_task(&task, &guard); - tokio::pin!(invocation); - let result = loop { - tokio::select! { - biased; - _ = guard.wait_for_shutdown() => { - guard.abandon(); - return Ok(ManagedPollOutcome::Handled); - } - result = &mut invocation => break result, - _ = tokio::time::sleep(Duration::from_secs(1)) => { - if guard.observe().await.is_err() { - return Ok(ManagedPollOutcome::Handled); + let mut callback_started = false; + let result = { + let invocation = + self.execute_cooperative_activity_task(&task, &guard, &mut callback_started); + tokio::pin!(invocation); + loop { + tokio::select! { + biased; + _ = guard.wait_for_shutdown() => { + guard.abandon(); + break None; + } + result = &mut invocation => break Some(result), + _ = tokio::time::sleep(Duration::from_secs(1)) => { + if guard.observe().await.is_err() { + break None; + } } } } }; + // The invocation and its callback future are dropped before any receipt. + let Some(result) = result else { + if callback_started { + guard.acknowledge_stopped().await?; + } + return Ok(ManagedPollOutcome::Handled); + }; // Both genuine application failures and successful results need current // authority before result upload or completion/failure publication. if matches!(result, Err(Error::ActivityExecutionAbandoned(_))) || guard.observe().await.is_err() { + if callback_started { + guard.acknowledge_stopped().await?; + } return Ok(ManagedPollOutcome::Handled); } let settlement = match result { @@ -1711,6 +1771,7 @@ impl Worker { &self, task: &ActivityTask, guard: &ActivityClaimGuard, + callback_started: &mut bool, ) -> Result { validate_activity_task_payloads(task)?; let handler = self @@ -1730,6 +1791,7 @@ impl Worker { claim_guard: Some(guard.clone()), }; guard.boundary()?; + *callback_started = true; handler(context, args).await } @@ -1853,6 +1915,7 @@ pub(super) struct ActivityClaimGuard { attempt_id: String, owner: String, active: Arc, + cancellation_receipt: Arc>>, stop: Option>, } @@ -1888,6 +1951,7 @@ impl ActivityClaimGuard { attempt_id: attempt.to_owned(), owner: owner.to_owned(), active: Arc::new(AtomicBool::new(true)), + cancellation_receipt: Arc::new(Mutex::new(None)), stop: client .worker_storage_admission .as_ref() @@ -1899,6 +1963,78 @@ impl ActivityClaimGuard { self.active.store(false, Ordering::SeqCst); } + /// Caller has already dropped the callback future. Never restores authority. + async fn acknowledge_stopped(&self) -> Result<()> { + self.abandon(); + let receipt = self + .cancellation_receipt + .lock() + .map_err(|_| invalid("activity cancellation receipt lock was poisoned"))? + .clone(); + if let Some(receipt) = receipt { + self.client + .acknowledge_activity_cancellation( + &self.task_id, + &self.attempt_id, + &self.owner, + text(&receipt, "request_id")?, + ) + .await?; + } + Ok(()) + } + + fn retain_cancellation_receipt(&self, value: &Value) -> Result<()> { + if value["can_continue"].as_bool() != Some(false) + || value["cancel_requested"].as_bool() != Some(true) + { + return Ok(()); + } + let Some(receipt) = value + .get("cancellation_acknowledgement") + .filter(|receipt| receipt.is_object()) + else { + return Ok(()); + }; + if !matches!( + receipt["callback_state"].as_str(), + Some("unknown" | "stopped") + ) || [ + "request_id", + "root_request_id", + "cleanup_deadline_at", + "cancellation_history_event_id", + ] + .iter() + .any(|field| text(receipt, field).is_err()) + || DateTime::parse_from_rfc3339(text(receipt, "cleanup_deadline_at")?).is_err() + { + return Ok(()); + } + let mut retained = self + .cancellation_receipt + .lock() + .map_err(|_| invalid("activity cancellation receipt lock was poisoned"))?; + if let Some(original) = retained.as_ref() { + if [ + "request_id", + "root_request_id", + "cleanup_deadline_at", + "cancellation_history_event_id", + ] + .iter() + .any(|field| original[*field] != receipt[*field]) + { + return Err(invalid( + "activity cancellation observation changed its original identity or deadline", + )); + } + } else { + *retained = Some(receipt.clone()); + } + Ok(()) + } + fn boundary(&self) -> Result<()> { if !self.active.load(Ordering::SeqCst) || self @@ -1929,6 +2065,7 @@ impl ActivityClaimGuard { .activity_task_status(&self.task_id, &self.attempt_id, &self.owner) .await .and_then(|value| { + self.retain_cancellation_receipt(&value)?; if value["can_continue"].as_bool() != Some(true) || value["cancel_requested"].as_bool() != Some(false) || value.get("reason") != Some(&Value::Null) diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs index 9f1aac4..0990975 100644 --- a/src/tests/cooperative_cancellation.rs +++ b/src/tests/cooperative_cancellation.rs @@ -2455,6 +2455,52 @@ fn remote_activity_responses( "remote-heartbeat-post-loss" if number >= 3 => reply["can_continue"] = json!(false), _ => {} } + if case.starts_with("remote-stop-joined-") && number >= 2 { + reply["can_continue"] = json!(false); + reply["cancel_requested"] = json!(true); + reply["reason"] = json!("activity_cancelled"); + reply["cancellation_acknowledgement"] = json!({"request_id":"local-request","root_request_id":"root-request", + "cleanup_deadline_at":"2099-01-01T00:00:30Z","cancellation_history_event_id":"cancel-history","callback_state":"unknown"}); + } + return Some(("200 OK", reply.to_string())); + } + if path.ends_with("/acknowledge-cancellation") { + let request: Value = serde_json::from_str(body).unwrap(); + if case.starts_with("remote-stop-joined-") + && std::fs::read_to_string(std::env::temp_dir().join(case)) + .ok() + .as_deref() + != Some("dropped") + { + return Some(( + "409 Conflict", + json!({"reason":"callback_not_dropped"}).to_string(), + )); + } + if case == "remote-ack-refused" { + return Some(( + "409 Conflict", + json!({"reason":"cancellation_request_mismatch"}).to_string(), + )); + } + let mut reply = json!({"task_id":"activity/selected","activity_attempt_id":request["activity_attempt_id"], + "lease_owner":request["lease_owner"],"request_id":request["request_id"],"acknowledged":true, + "duplicate":number > 1,"history_event_id":"stop-history","reason":null,"heartbeat_recorded":false}); + if case.starts_with("remote-stop-joined-") { + reply["task_id"] = json!("activity"); + } + match case { + "remote-ack-task" => reply["task_id"] = json!("wrong"), + "remote-ack-attempt" => reply["activity_attempt_id"] = json!("wrong"), + "remote-ack-owner" => reply["lease_owner"] = json!("wrong"), + "remote-ack-request" => reply["request_id"] = json!("wrong"), + "remote-ack-unproved" => reply["history_event_id"] = Value::Null, + "remote-ack-declined" => reply["acknowledged"] = json!(false), + "remote-ack-progress" => reply["heartbeat_recorded"] = json!(true), + "remote-ack-duplicate" => reply["duplicate"] = json!("true"), + "remote-ack-slow" => std::thread::sleep(Duration::from_millis(5250)), + _ => {} + } return Some(("200 OK", reply.to_string())); } if path.ends_with("/activity/heartbeat") { @@ -2764,6 +2810,169 @@ async fn cooperative_activity_observation_rejects_changed_receipts_and_preserves assert_eq!(server.requests.lock().unwrap().len(), 1); } +#[tokio::test] +async fn cooperative_activity_stop_receipt_preserves_worker_authentication_and_duplicate_identity() +{ + let server = remote_activity_server(); + let original = client(&server, "remote-ack-valid"); + let first = original + .acknowledge_activity_cancellation( + "activity/selected", + "attempt-original", + "actual-owner", + "local-request", + ) + .await + .unwrap(); + let duplicate = original + .acknowledge_activity_cancellation( + "activity/selected", + "attempt-original", + "actual-owner", + "local-request", + ) + .await + .unwrap(); + assert_eq!(first["history_event_id"], duplicate["history_event_id"]); + assert_eq!(first["duplicate"], false); + assert_eq!(duplicate["duplicate"], true); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 2); + for request in requests.iter() { + assert!(request + .path + .ends_with("/activity%2Fselected/acknowledge-cancellation")); + assert_eq!(request.worker_protocol.as_deref(), Some("1.20")); + assert_eq!(request.authorization.as_deref(), Some("Bearer worker-only")); + assert_eq!(request.namespace.as_deref(), Some("caller-namespace")); + let body: Value = serde_json::from_str(&request.body).unwrap(); + assert_eq!( + body, + json!({"activity_attempt_id":"attempt-original","lease_owner":"actual-owner","request_id":"local-request"}) + ); + } + assert_eq!(WORKER_PROTOCOL_VERSION, "1.19"); +} + +#[tokio::test] +async fn cooperative_activity_stop_receipt_refuses_changed_fences_unproved_receipts_and_missing_identity( +) { + for case in [ + "remote-ack-task", + "remote-ack-attempt", + "remote-ack-owner", + "remote-ack-request", + "remote-ack-unproved", + "remote-ack-declined", + "remote-ack-progress", + "remote-ack-duplicate", + ] { + let server = remote_activity_server(); + assert!( + matches!( + client(&server, case) + .acknowledge_activity_cancellation( + "activity/selected", + "attempt-original", + "actual-owner", + "local-request" + ) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + ), + "{case}" + ); + } + let server = remote_activity_server(); + assert!( + matches!(client(&server, "remote-ack-refused").acknowledge_activity_cancellation( + "activity/selected", "attempt-original", "actual-owner", "local-request").await, + Err(Error::ActivityTaskRejected(ref error)) if error.operation == "acknowledge-cancellation" && error.reason == "cancellation_request_mismatch") + ); + for (task, attempt, owner, request) in [ + ("", "attempt", "owner", "request"), + ("task", "", "owner", "request"), + ("task", "attempt", "", "request"), + ("task", "attempt", "owner", ""), + ] { + assert!(client(&server, "remote-ack-valid") + .acknowledge_activity_cancellation(task, attempt, owner, request) + .await + .is_err()); + } + assert_eq!(server.requests.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn cooperative_activity_stop_receipt_has_one_five_second_budget() { + let server = remote_activity_server(); + let started = Instant::now(); + assert!(matches!( + client(&server, "remote-ack-slow") + .acknowledge_activity_cancellation( + "activity/selected", + "attempt-original", + "actual-owner", + "local-request" + ) + .await, + Err(Error::Timeout) + )); + assert!(started.elapsed() < Duration::from_millis(5200)); + assert_eq!(server.requests.lock().unwrap().len(), 1); +} + +struct StoppedCallbackMarker(std::path::PathBuf); + +impl Drop for StoppedCallbackMarker { + fn drop(&mut self) { + std::fs::write(&self.0, "dropped").unwrap(); + } +} + +#[tokio::test] +async fn cooperative_activity_callback_is_dropped_before_stop_receipt_and_cloned_context_stays_fenced( +) { + let case = unique_request_id("remote-stop-joined"); + let marker = std::env::temp_dir().join(&case); + let server = remote_activity_server(); + let mut worker = coordinator_worker(&server, &case); + let saved = Arc::new(Mutex::new(None::)); + let capture = Arc::clone(&saved); + let callback_marker = marker.clone(); + worker.register_activity("work", move |ctx, _| { + *capture.lock().unwrap() = Some(ctx); + let stopped = StoppedCallbackMarker(callback_marker.clone()); + async move { + let _stopped = stopped; + std::future::pending::<()>().await; + Ok(Value::Null) + } + }); + let result = worker.poll_activity_once().await; + let dropped = std::fs::read_to_string(&marker); + let _ = std::fs::remove_file(&marker); + assert_eq!(result.unwrap(), ManagedPollOutcome::Handled); + assert_eq!(dropped.unwrap(), "dropped"); + let before = server.requests.lock().unwrap().len(); + let context = saved.lock().unwrap().as_ref().unwrap().clone(); + assert!(matches!( + context.heartbeat(json!({"late":true})).await, + Err(Error::ActivityExecutionAbandoned(_)) + )); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), before); + assert_eq!( + requests.last().unwrap().path.rsplit('/').next(), + Some("acknowledge-cancellation") + ); + assert!(!requests + .iter() + .any(|request| request.path.ends_with("/heartbeat") + || request.path.ends_with("/complete") + || request.path.ends_with("/fail"))); +} + #[tokio::test] async fn cooperative_activity_observation_has_one_five_second_budget() { let server = remote_activity_server(); diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index fb4821c..1360751 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -846,7 +846,10 @@ async fn managed_remote_cancellation(user_heartbeat: bool) { .unwrap() .expect("actual blocked callback entered"); let request = handle - .request_cancellation(CooperativeCancellationOptions::default()) + .request_cancellation(CooperativeCancellationOptions { + cleanup_timeout_seconds: Some(30), + ..CooperativeCancellationOptions::default() + }) .await .unwrap(); let result = handle @@ -859,7 +862,8 @@ async fn managed_remote_cancellation(user_heartbeat: bool) { matches!(result, Err(Error::WorkflowCancelled(_))), "managed result: {result:?}" ); - let snapshot = assert_cancelled(&handle, &request.cancellation_request.request_id, true).await; + let mut snapshot = + assert_cancelled(&handle, &request.cancellation_request.request_id, true).await; assert_eq!( dropped.load(Ordering::SeqCst), 1, @@ -892,6 +896,81 @@ async fn managed_remote_cancellation(user_heartbeat: bool) { ), "cloned context remains abandoned" ); + if std::env::var("DURABLE_WORKFLOW_NATIVE_SOURCE_QUALIFICATION").as_deref() == Ok("1") { + let status = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let status = client + .activity_task_status( + &original.task_id, + &original.activity_attempt_id, + &original.lease_owner, + ) + .await + .unwrap(); + if status["cancellation_acknowledgement"]["callback_state"] == "stopped" { + break status; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("dropped callback must leave a durable stop receipt"); + let receipt = &status["cancellation_acknowledgement"]; + assert_eq!( + receipt["request_id"], + request.cancellation_request.request_id + ); + assert_eq!( + receipt["root_request_id"], + request.cancellation_request.request_id + ); + assert_eq!( + receipt["cleanup_deadline_at"], + request.cancellation_request.cleanup_deadline_at + ); + assert_eq!(receipt["received_after_deadline"], false); + assert_eq!(status["heartbeat_recorded"], false); + assert_eq!(status["can_continue"], false); + let duplicate = client + .acknowledge_activity_cancellation( + &original.task_id, + &original.activity_attempt_id, + &original.lease_owner, + &request.cancellation_request.request_id, + ) + .await + .unwrap(); + assert_eq!(duplicate["duplicate"], true); + assert_eq!(duplicate["history_event_id"], receipt["history_event_id"]); + snapshot = history(&handle).await; + assert_eq!(count(&snapshot, "ActivityCancellationAcknowledged"), 1); + let recorded = snapshot["events"] + .as_array() + .unwrap() + .iter() + .find(|event| event["event_type"] == "ActivityCancellationAcknowledged") + .unwrap(); + assert_eq!(recorded["payload"]["callback_state"], "stopped"); + assert_eq!(recorded["payload"]["evidence_source"], "activity_worker"); + assert_eq!( + recorded["payload"]["activity_attempt_id"], + original.activity_attempt_id + ); + assert_eq!(recorded["payload"]["request_id"], receipt["request_id"]); + assert_eq!( + recorded["payload"]["root_request_id"], + receipt["root_request_id"] + ); + assert_eq!( + recorded["payload"]["cleanup_deadline_at"], + receipt["cleanup_deadline_at"] + ); + assert_eq!( + recorded["payload"]["cancellation_history_event_id"], + receipt["cancellation_history_event_id"] + ); + eprintln!("remote stop receipt: user_heartbeat={user_heartbeat}, status={status}, duplicate={duplicate}, history={recorded}"); + } let completion = client .complete_activity_task( &original.task_id, From 8a30461b04d638a03f6116ff2f5abd5dbdf98a56 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 01:14:10 +0000 Subject: [PATCH 32/58] Observe the cleanup stop receipt before checking stale publication --- tests/cooperative_server.rs | 45 +++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index 1360751..bdd58fd 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -398,7 +398,52 @@ async fn blocked_cleanup_cutoff(terminate: bool) { cleanup.heartbeat(json!({"late":true})).await, Err(Error::ActivityExecutionAbandoned(_)) )); + let receipt_expected = !terminate + && std::env::var("DURABLE_WORKFLOW_NATIVE_SOURCE_QUALIFICATION").as_deref() == Ok("1"); + if receipt_expected { + // Callback drop precedes the asynchronous Server receipt. Stabilize + // that diagnostic before checking stale result/failure publication. + let status = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let status = client + .activity_task_status( + &cleanup.task_id, + &cleanup.activity_attempt_id, + &cleanup.lease_owner, + ) + .await + .unwrap(); + if status["cancellation_acknowledgement"]["callback_state"] == "stopped" { + break status; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("dropped cleanup callback must leave its original stop receipt"); + let receipt = &status["cancellation_acknowledgement"]; + assert_eq!( + receipt["request_id"], + request.cancellation_request.request_id + ); + assert_eq!( + receipt["root_request_id"], + request.cancellation_request.request_id + ); + assert_eq!( + receipt["cleanup_deadline_at"], + request.cancellation_request.cleanup_deadline_at + ); + assert_eq!(receipt["received_after_deadline"], true); + assert_eq!(status["heartbeat_recorded"], false); + assert_eq!(status["can_continue"], false); + eprintln!("Blocked cleanup stop receipt: {status}"); + } let snapshot = history(&handle).await; + assert_eq!( + count(&snapshot, "ActivityCancellationAcknowledged"), + usize::from(receipt_expected) + ); for kind in [ "CooperativeCancellationRequested", "CooperativeCancellationDelivered", From 12f6903a0b3eb6e2469d4b37324ebb4f4637b161 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 12:50:39 +0000 Subject: [PATCH 33/58] Retain cooperative source qualification evidence for 90 days --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3062ace..3b9d5a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -268,7 +268,7 @@ jobs: cooperative-package-provenance.txt cooperative-images.json cooperative-source-provenance.json - retention-days: 7 + retention-days: 90 if-no-files-found: warn - name: Emit failure diagnostics if: ${{ failure() }} From aceea0cbb7aeec1d4fce0835ac3a505fcd4cc8c4 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 15:16:22 +0000 Subject: [PATCH 34/58] Validate activity cancellation waits before releasing claims --- src/cooperative_cancellation.rs | 25 ++-- src/tests/cooperative_cancellation.rs | 168 ++++++++++++++++---------- 2 files changed, 121 insertions(+), 72 deletions(-) diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index 32c57ca..25c3169 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -1427,7 +1427,7 @@ impl Client { /// This explicit worker-protocol 1.20 operation renews no lease and advertises /// no worker capability. The Server requires a previously admitted capable /// claim. Reload canonical history before exposing cancellation to workflow - /// code, including when an acknowledgment is lost. A child wait explicitly + /// code, including when an acknowledgment is lost. A cancellation wait explicitly /// releases the claim and must return the worker to polling. pub async fn deliver_workflow_cancellation( &self, @@ -1463,13 +1463,22 @@ impl Client { ) .await?; if response["delivered"].as_bool() == Some(false) - && matches!( - delivery.call_kind, - CancellationCallKind::Child - | CancellationCallKind::Parallel - | CancellationCallKind::SelectionHandle - ) - && response["reason"].as_str() == Some("cancellation_waiting_for_child") + && match response["reason"].as_str() { + Some("cancellation_waiting_for_child") => matches!( + delivery.call_kind, + CancellationCallKind::Child + | CancellationCallKind::Parallel + | CancellationCallKind::SelectionHandle + ), + Some("cancellation_waiting_for_activity") => matches!( + delivery.call_kind, + CancellationCallKind::Activity + | CancellationCallKind::LocalActivity + | CancellationCallKind::Parallel + | CancellationCallKind::SelectionHandle + ), + _ => false, + } && response["claim_released"].as_bool() == Some(true) && response["task_id"].as_str() == Some(task.task_id.as_str()) && response["workflow_run_id"].as_str() == Some(run_id) diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs index 0990975..cda8be7 100644 --- a/src/tests/cooperative_cancellation.rs +++ b/src/tests/cooperative_cancellation.rs @@ -2157,24 +2157,33 @@ fn transport_responses(path: &str, body: &str, number: usize) -> Option<(&'stati .to_string(), )); } - if path.ends_with("/deliver-cancellation") && case.starts_with("child-wait") { + if path.ends_with("/deliver-cancellation") + && (case.starts_with("child-wait") || case.starts_with("activity-wait")) + { let mut pending = pending_child_delivery("task/selected"); - match case { - "child-wait-wrong-task" => pending["task_id"] = json!("other"), - "child-wait-wrong-run" => pending["workflow_run_id"] = json!("other"), - "child-wait-not-released" => pending["claim_released"] = json!(false), - "child-wait-string-released" => pending["claim_released"] = json!("true"), - "child-wait-missing-release" => { + if case.starts_with("activity-wait") { + pending["reason"] = json!("cancellation_waiting_for_activity"); + } + let suffix = case + .strip_prefix("child-wait") + .or_else(|| case.strip_prefix("activity-wait")) + .unwrap(); + match suffix { + "-wrong-task" => pending["task_id"] = json!("other"), + "-wrong-run" => pending["workflow_run_id"] = json!("other"), + "-not-released" => pending["claim_released"] = json!(false), + "-string-released" => pending["claim_released"] = json!("true"), + "-missing-release" => { pending.as_object_mut().unwrap().remove("claim_released"); } - "child-wait-wrong-reason" => pending["reason"] = json!("other"), - "child-wait-request" => pending["request_id"] = body["request_id"].clone(), - "child-wait-sequence" => pending["sequence"] = json!(1), - "child-wait-kind" => pending["call_kind"] = json!("child"), - "child-wait-span" => pending["sequence_span"] = json!(1), - "child-wait-operation" => pending["operation_sequence"] = json!(1), - "child-wait-operation-span" => pending["operation_sequence_span"] = json!(1), - "child-wait-string-delivered" => pending["delivered"] = json!("false"), + "-wrong-reason" => pending["reason"] = json!("other"), + "-request" => pending["request_id"] = body["request_id"].clone(), + "-sequence" => pending["sequence"] = json!(1), + "-kind" => pending["call_kind"] = json!("child"), + "-span" => pending["sequence_span"] = json!(1), + "-operation" => pending["operation_sequence"] = json!(1), + "-operation-span" => pending["operation_sequence_span"] = json!(1), + "-string-delivered" => pending["delivered"] = json!("false"), _ => {} } return Some(("200 OK", pending.to_string())); @@ -4160,62 +4169,93 @@ async fn cooperative_transport_delivers_the_exact_claim_with_worker_credentials( } #[tokio::test] -async fn cooperative_transport_accepts_only_explicit_child_claim_release() { - for kind in [ - CancellationCallKind::Child, - CancellationCallKind::Parallel, - CancellationCallKind::SelectionHandle, +async fn cooperative_transport_accepts_only_explicit_cancellation_claim_release() { + for (case, kinds) in [ + ( + "child-wait", + vec![ + CancellationCallKind::Child, + CancellationCallKind::Parallel, + CancellationCallKind::SelectionHandle, + ], + ), + ( + "activity-wait", + vec![ + CancellationCallKind::Activity, + CancellationCallKind::LocalActivity, + CancellationCallKind::Parallel, + CancellationCallKind::SelectionHandle, + ], + ), + ] { + for kind in kinds { + let server = transport_server(); + let mut delivery = transport_delivery(); + delivery.call_kind = kind; + if kind == CancellationCallKind::SelectionHandle { + delivery.sequence = 2; + delivery.operation_sequence = Some(1); + } + assert_eq!( + client(&server, case) + .deliver_workflow_cancellation(&transport_task(), &delivery) + .await + .unwrap(), + CancellationDeliveryReply::ClaimReleased { + task_id: "task/selected".into(), + run_id: "run".into() + } + ); + } + } + for (case, kind) in [ + ("child-wait", CancellationCallKind::Timer), + ("activity-wait", CancellationCallKind::Timer), + ("child-wait", CancellationCallKind::Activity), + ("activity-wait", CancellationCallKind::Child), ] { let server = transport_server(); let mut delivery = transport_delivery(); delivery.call_kind = kind; - if kind == CancellationCallKind::SelectionHandle { - delivery.sequence = 2; - delivery.operation_sequence = Some(1); - } - assert_eq!( - client(&server, "child-wait") + assert!(matches!( + client(&server, case) .deliver_workflow_cancellation(&transport_task(), &delivery) - .await - .unwrap(), - CancellationDeliveryReply::ClaimReleased { - task_id: "task/selected".into(), - run_id: "run".into() - } - ); + .await, + Err(Error::InvalidCooperativeCancellation(_)) + )); } - let server = transport_server(); - assert!(matches!( - client(&server, "child-wait") - .deliver_workflow_cancellation(&transport_task(), &transport_delivery()) - .await, - Err(Error::InvalidCooperativeCancellation(_)) - )); - for case in [ - "child-wait-wrong-task", - "child-wait-wrong-run", - "child-wait-not-released", - "child-wait-string-released", - "child-wait-missing-release", - "child-wait-wrong-reason", - "child-wait-request", - "child-wait-sequence", - "child-wait-kind", - "child-wait-span", - "child-wait-operation", - "child-wait-operation-span", - "child-wait-string-delivered", + for (prefix, kind) in [ + ("child-wait", CancellationCallKind::Child), + ("activity-wait", CancellationCallKind::Activity), ] { - let server = transport_server(); - let mut delivery = transport_delivery(); - delivery.call_kind = CancellationCallKind::Child; - let result = client(&server, case) - .deliver_workflow_cancellation(&transport_task(), &delivery) - .await; - assert!( - matches!(result, Err(Error::InvalidCooperativeCancellation(_))), - "{case}: {result:?}" - ); + for suffix in [ + "wrong-task", + "wrong-run", + "not-released", + "string-released", + "missing-release", + "wrong-reason", + "request", + "sequence", + "kind", + "span", + "operation", + "operation-span", + "string-delivered", + ] { + let case = format!("{prefix}-{suffix}"); + let server = transport_server(); + let mut delivery = transport_delivery(); + delivery.call_kind = kind; + let result = client(&server, &case) + .deliver_workflow_cancellation(&transport_task(), &delivery) + .await; + assert!( + matches!(result, Err(Error::InvalidCooperativeCancellation(_))), + "{case}: {result:?}" + ); + } } } From 4622819d52fa7debf8b734c911f17ba8eca35f69 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 16:59:14 +0000 Subject: [PATCH 35/58] Preserve remote activity cancellation policies through Rust replay --- docs/cooperative-cancellation-design.md | 37 ++- src/lib.rs | 111 +++++++- src/tests/activity_cancellation_policies.rs | 251 ++++++++++++++++++ tests/cooperative_server.rs | 248 ++++++++++++++++- .../activity-cancellation-policy-changed.json | 14 + tests/replay_regression_corpus.rs | 7 + 6 files changed, 655 insertions(+), 13 deletions(-) create mode 100644 src/tests/activity_cancellation_policies.rs create mode 100644 tests/fixtures/replay-regressions/activity-cancellation-policy-changed.json diff --git a/docs/cooperative-cancellation-design.md b/docs/cooperative-cancellation-design.md index 5017c55..aadf8bb 100644 --- a/docs/cooperative-cancellation-design.md +++ b/docs/cooperative-cancellation-design.md @@ -53,6 +53,39 @@ history fails explicitly. A worker without the cooperation opt-in refuses the commands before completion with its identity and required protocol. Server also checks the immutable task claim and installed backend. +## Remote Activity policies + +`ActivityOptions::cancellation_policy()` accepts the same `CancellationPolicy` +enum for ordinary, parallel and selection Activity calls. Omission retains the +historical Try behavior and wire shape. + +```rust +ActivityOptions::new() + .cancellation_policy(CancellationPolicy::WaitCancellationCompleted) +``` + +Try requests cancellation and continues without waiting for the stop receipt. +Wait delays workflow delivery until the original remote attempt's callback drop +is durably acknowledged. Abandon leaves work independent after parent +cancellation. It requires a finite positive `schedule_to_close_timeout` and +keeps that original deadline as its total lifetime. It does not extend the +parent's cleanup budget. + +Explicit policies require cooperative worker opt-in, protocol 1.20 and a +compatible installed backend. Unsupported workers identify their operation, +identity and required protocol before submission. Server also checks the +original immutable claim. Replay compares the policy against canonical history +before ordinary/group/selection settlement or cancellation delivery. Later +events with missing fields retain the original policy. Invalid, conflicting +and changed history fails explicitly. Historical omission retains Try. + +Connected Source cases include explicit Try/Wait callback drop without app +heartbeats, receipt ordering for Wait and stale-result refusal. Bounded Abandon +checks that the callback future survives parent closure, commits independent +completion under its original total timeout and cannot publish a second outcome +or reopen the parent. These tests supervise async callback futures. Detached +threads, processes and downstream effects need their own cooperation and fencing. + ## Remote callback-stop receipts The managed worker drops its activity callback future before reporting a stop. @@ -80,7 +113,9 @@ Native image or package. ## Rust API release boundary The candidate adds a variant to the existing exhaustive `ParentClosePolicy` -enum and a field to the public `ChildWorkflowOptions` struct. Existing struct +enum and fields to the public `ChildWorkflowOptions` and `ActivityOptions` +structs. It also adds a total-lifetime error category to the exhaustive public +`ActivityOptionsErrorKind` enum. Existing struct literals and exhaustive matches need updates. The cooperative error variants also extend an existing exhaustive public enum. These changes require a major Rust SDK release if retained. This draft does not authorize that release or diff --git a/src/lib.rs b/src/lib.rs index 56f6414..ed18a88 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -986,6 +986,7 @@ pub enum ActivityOptionsErrorKind { BackoffOverflow, EmptyNonRetryableErrorType, TimeoutNotPositive, + MissingTotalTimeout, TimeoutOverflow, TimeoutOrder, } @@ -1291,15 +1292,15 @@ impl ParentClosePolicy { } } -/// Cancellation at an awaiting child call. +/// Cancellation at an awaiting operation. Activities default to Try, children to Abandon. #[non_exhaustive] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub enum CancellationPolicy { - /// Request child cleanup and deliver parent cancellation without waiting. + /// Request cancellation and continue without waiting. Historical Activity default. TryCancel, - /// Wait for the child's recorded terminal outcome before parent delivery. + /// Wait for recorded child termination or the original Activity attempt's stop receipt. WaitCancellationCompleted, - /// Leave the child independent. This preserves the historical default. + /// Leave work independent. Historical child default. Remote Activities require a total timeout. #[default] Abandon, } @@ -1450,6 +1451,7 @@ pub struct ActivityOptions { pub schedule_to_start_timeout: Option, pub schedule_to_close_timeout: Option, pub heartbeat_timeout: Option, + pub cancellation_policy: Option, } impl ActivityOptions { @@ -1487,7 +1489,21 @@ impl ActivityOptions { self } + pub fn cancellation_policy(mut self, policy: CancellationPolicy) -> Self { + self.cancellation_policy = Some(policy); + self + } + fn validate(&self) -> std::result::Result { + if self.cancellation_policy == Some(CancellationPolicy::Abandon) + && self.schedule_to_close_timeout.is_none() + { + return Err(ActivityOptionsError::new( + ActivityOptionsErrorKind::MissingTotalTimeout, + Some("schedule_to_close_timeout"), + "remote Activity Abandon requires a finite positive total timeout", + )); + } if self .task_queue .as_deref() @@ -1554,6 +1570,7 @@ impl ActivityOptions { self.schedule_to_close_timeout, )?, heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?, + cancellation_policy: self.cancellation_policy, }) } } @@ -1643,6 +1660,7 @@ struct ValidatedActivityOptions { schedule_to_start_timeout: Option, schedule_to_close_timeout: Option, heartbeat_timeout: Option, + cancellation_policy: Option, } fn validate_timeout_order( @@ -7923,6 +7941,17 @@ impl Worker { ctx: &WorkflowContext, commands: Vec, ) -> Result { + if !self.cooperative_cancellation_enabled + && commands.iter().any(|command| { + command["type"] == "schedule_activity" + && command.get("cancellation_policy").is_some() + }) + { + return Err(Error::CooperativeCancellationUnavailable(format!( + "activity_cancellation_policy_not_supported: Rust worker {} must enable cooperative cancellation with worker protocol 1.20 and a compatible Server/Native backend", + self.worker_id, + ))); + } if !self.cooperative_cancellation_enabled && commands.iter().any(|command| { command["type"] == "start_child_workflow" @@ -9865,6 +9894,7 @@ enum RecordedCommand { Activity { sequence: u64, activity_type: Option, + cancellation_policy: String, options: Option, outcome: Option, parallel_group_path: Option>, @@ -10223,6 +10253,58 @@ struct RecordedChildPolicies { cancellation_policy: String, } +fn recorded_activity_cancellation_policy( + events: &[&HistoryEvent], + sequence: u64, +) -> Result { + let mut policy: Option = None; + for event in events.iter().filter(|event| { + matches!( + event.event_type.as_str(), + "ActivityScheduled" + | "ActivityStarted" + | "ActivityCompleted" + | "ActivityFailed" + | "ActivityTimedOut" + | "ActivityCancelled" + ) + }) { + for source in [Some(&event.payload), event.payload.get("activity")] + .into_iter() + .flatten() + { + let Some(value) = source.get("cancellation_policy") else { + continue; + }; + let Some(incoming @ ("try_cancel" | "wait_cancellation_completed" | "abandon")) = + value.as_str() + else { + return Err(invalid_recorded_history( + "invalid_activity_cancellation_policy_history", + sequence, + "supported Activity cancellation policy", + &value.to_string(), + "Activity history contains an invalid cancellation policy", + )); + }; + if let Some(previous) = policy.as_deref() { + if previous != incoming { + return Err(invalid_recorded_history( + "activity_cancellation_policy_history_conflict", + sequence, + previous, + incoming, + "Activity cancellation policy changed between history events", + )); + } + } + policy = Some(incoming.to_string()); + } + policy.get_or_insert_with(|| "try_cancel".to_string()); + } + Ok(policy.unwrap_or_else(|| "try_cancel".to_string())) +} + fn recorded_child_policy_value<'a>( payload: &'a Value, field: &str, @@ -12534,11 +12616,27 @@ impl ActivityCall { match recorded { RecordedCommand::Activity { activity_type, + cancellation_policy, options: recorded_options, outcome, parallel_group_path, .. } => { + let current_policy = options + .cancellation_policy + .unwrap_or(CancellationPolicy::TryCancel) + .as_str(); + if cancellation_policy != current_policy { + return Poll::Ready(Err(Error::NonDeterministicReplay( + ReplayFailure::new( + "activity_cancellation_policy_changed", + Some(sequence), + Some(cancellation_policy), + Some(current_policy.to_string()), + "Activity cancellation policy changed during replay", + ), + ))); + } if let Err(error) = ensure_parallel_path_matches( sequence, parallel_group_path.as_deref(), @@ -12676,6 +12774,9 @@ impl ActivityCall { if let Some(retry_policy) = options.retry_policy { command.insert("retry_policy".to_string(), retry_policy); } + if let Some(policy) = options.cancellation_policy { + command.insert("cancellation_policy".to_string(), json!(policy.as_str())); + } apply_parallel_group_path(&mut command, &self.parallel_group_path); match state.prepare_scalar_cancellation( cursor, @@ -14281,6 +14382,7 @@ fn recorded_commands( return Ok(RecordedCommand::Activity { sequence, activity_type, + cancellation_policy: recorded_activity_cancellation_policy(&activity_events, sequence)?, options, outcome, parallel_group_path, @@ -15998,6 +16100,7 @@ fn value_as_u64(value: &Value) -> Option { #[cfg(test)] mod tests { use super::*; + mod activity_cancellation_policies; mod child_workflow_policies; mod cooperative_cancellation; mod runtime_payloads; diff --git a/src/tests/activity_cancellation_policies.rs b/src/tests/activity_cancellation_policies.rs new file mode 100644 index 0000000..e133a16 --- /dev/null +++ b/src/tests/activity_cancellation_policies.rs @@ -0,0 +1,251 @@ +use super::*; + +fn activity_policy_worker( + policy: Option, + mode: &'static str, + enabled: bool, +) -> Worker { + let mut worker = Worker::new( + Client::builder("http://127.0.0.1:1").build().unwrap(), + "queue", + ) + .cooperative_cancellation(enabled); + worker.register_workflow("activity-policy", move |ctx, _| async move { + let mut options = ActivityOptions::new(); + if let Some(policy) = policy { + options = options + .cancellation_policy(policy) + .schedule_to_close_timeout(Duration::from_secs(60)); + } + match mode { + "parallel" => { + ctx.parallel(vec![ParallelOperation::activity_with_options( + "work", + options, + json!([]), + )]) + .await?; + } + "selection" => { + ctx.select_keyed(vec![( + SelectionKey::from("work"), + ParallelOperation::activity_with_options("work", options, json!([])), + )]) + .await?; + } + _ => { + ctx.activity_with_options("work", options, json!([])) + .await?; + } + } + Ok(json!("finished")) + }); + worker +} + +fn activity_policy_event(kind: &str, payload: Value) -> HistoryEvent { + serde_json::from_value(json!({"event_type": kind, "payload": payload})).unwrap() +} + +fn activity_policy_history(command: &Value) -> Vec { + let mut payload = command.clone(); + let object = payload.as_object_mut().unwrap(); + object.remove("type"); + object.remove("arguments"); + object.insert("sequence".into(), json!(1)); + let mut activity = json!({"type": "work"}); + if let Some(policy) = command.get("cancellation_policy") { + activity["cancellation_policy"] = policy.clone(); + } + object.insert("activity".into(), activity); + vec![activity_policy_event("ActivityScheduled", payload)] +} + +fn assert_activity_policy_failure(error: Error, reason: &str) { + let Error::NonDeterministicReplay(failure) = error else { + panic!("expected replay failure, got {error:?}"); + }; + assert_eq!(failure.reason, reason); + assert_eq!(failure.sequence, Some(1)); +} + +#[test] +fn changed_activity_cancellation_policy_is_rejected_during_replay() { + let mut worker = Worker::new( + Client::builder("http://127.0.0.1:1").build().unwrap(), + "queue", + ) + .cooperative_cancellation(true); + worker.register_workflow("activity-policy", |ctx, _| async move { + ctx.activity("work", json!([])).await?; + Ok(json!("finished")) + }); + let event = serde_json::from_value(json!({ + "event_type": "ActivityScheduled", + "payload": {"sequence": 1, "activity_type": "work", "activity": { + "type": "work", "cancellation_policy": "wait_cancellation_completed", + }}, + })) + .unwrap(); + let error = worker + .execute_workflow_task(workflow_task("activity-policy", vec![event], DEFAULT_CODEC)) + .expect_err("changing the original Activity policy must fail replay"); + let Error::NonDeterministicReplay(failure) = error else { + panic!("expected replay failure, got {error:?}"); + }; + assert_eq!(failure.reason, "activity_cancellation_policy_changed"); + assert_eq!(failure.sequence, Some(1)); +} + +#[test] +fn activity_policies_encode_and_replay_original_history_through_completion() { + for policy in [ + None, + Some(CancellationPolicy::TryCancel), + Some(CancellationPolicy::WaitCancellationCompleted), + Some(CancellationPolicy::Abandon), + ] { + let worker = activity_policy_worker(policy, "sequential", true); + let commands = worker + .execute_workflow_task(workflow_task("activity-policy", vec![], DEFAULT_CODEC)) + .unwrap(); + assert_eq!( + commands[0] + .get("cancellation_policy") + .and_then(Value::as_str), + policy.map(CancellationPolicy::as_str) + ); + let mut history = activity_policy_history(&commands[0]); + history.push(activity_policy_event( + "ActivityStarted", + json!({"sequence":1, "activity_type":"work"}), + )); + history.push(activity_policy_event( + "ActivityCompleted", + json!({"sequence":1, "activity_type":"work", + "result": fixture_envelope(json!("recorded"))}), + )); + let completed = worker + .execute_workflow_task(workflow_task( + "activity-policy", + history.clone(), + DEFAULT_CODEC, + )) + .unwrap(); + assert_eq!( + decode_wire_value(&completed[0]["result"], DEFAULT_CODEC).unwrap(), + json!("finished") + ); + let changed = activity_policy_worker( + Some(CancellationPolicy::WaitCancellationCompleted), + "sequential", + true, + ); + if policy != Some(CancellationPolicy::WaitCancellationCompleted) { + assert_activity_policy_failure( + changed + .execute_workflow_task(workflow_task("activity-policy", history, DEFAULT_CODEC)) + .unwrap_err(), + "activity_cancellation_policy_changed", + ); + } + } +} + +#[test] +fn activity_policies_match_ordinary_groups_and_cancellation_delivery() { + for mode in ["sequential", "parallel", "selection"] { + let original = activity_policy_worker( + Some(CancellationPolicy::WaitCancellationCompleted), + mode, + true, + ); + let commands = original + .execute_workflow_task(workflow_task("activity-policy", vec![], DEFAULT_CODEC)) + .unwrap(); + for delivered in [false, true] { + let mut history = activity_policy_history(&commands[0]); + if delivered { + history.push(serde_json::from_value(json!({ + "event_type":"CooperativeCancellationRequested", "recorded_at":"2026-10-01T00:00:00Z", + "payload":{"workflow_run_id":"run", "workflow_command_id":"request-1", "cleanup_deadline_at":"2026-10-01T00:00:30Z"}, + })).unwrap()); + history.push(activity_policy_event("CooperativeCancellationDelivered", json!({ + "workflow_run_id":"run", "workflow_command_id":"request-1", "sequence":1, + "call_kind":if mode == "sequential" {"activity"} else {"parallel"}, "sequence_span":1, + }))); + } + let mut task = workflow_task("activity-policy", history, DEFAULT_CODEC); + task.run_id = Some("run".into()); + original.execute_workflow_task(task.clone()).unwrap(); + let changed = activity_policy_worker(Some(CancellationPolicy::TryCancel), mode, true); + assert_activity_policy_failure( + changed.execute_workflow_task(task).unwrap_err(), + "activity_cancellation_policy_changed", + ); + } + } +} + +#[test] +fn malformed_and_conflicting_activity_policy_history_is_rejected() { + let worker = activity_policy_worker(None, "sequential", true); + for policy in [Value::Null, json!(false), json!([]), json!("unknown")] { + let history = vec![activity_policy_event( + "ActivityScheduled", + json!({"sequence":1, "activity_type":"work", + "activity":{"cancellation_policy":policy}}), + )]; + assert_activity_policy_failure( + worker + .execute_workflow_task(workflow_task("activity-policy", history, DEFAULT_CODEC)) + .unwrap_err(), + "invalid_activity_cancellation_policy_history", + ); + } + let commands = worker + .execute_workflow_task(workflow_task("activity-policy", vec![], DEFAULT_CODEC)) + .unwrap(); + let mut history = activity_policy_history(&commands[0]); + history.push(activity_policy_event( + "ActivityStarted", + json!({"sequence":1, "activity_type":"work", + "activity":{"cancellation_policy":"abandon"}}), + )); + assert_activity_policy_failure( + worker + .execute_workflow_task(workflow_task("activity-policy", history, DEFAULT_CODEC)) + .unwrap_err(), + "activity_cancellation_policy_history_conflict", + ); +} + +#[test] +fn explicit_activity_policies_require_worker_opt_in_and_abandon_requires_total_lifetime() { + for policy in [ + CancellationPolicy::TryCancel, + CancellationPolicy::WaitCancellationCompleted, + CancellationPolicy::Abandon, + ] { + let worker = activity_policy_worker(Some(policy), "sequential", false); + let error = worker + .execute_workflow_task(workflow_task("activity-policy", vec![], DEFAULT_CODEC)) + .unwrap_err(); + let Error::CooperativeCancellationUnavailable(message) = error else { + panic!("unexpected error {error:?}"); + }; + assert!(message.contains("activity_cancellation_policy_not_supported")); + assert!(message.contains("1.20")); + } + let absent = ActivityOptions::new() + .cancellation_policy(CancellationPolicy::Abandon) + .validate() + .unwrap_err(); + assert_eq!(absent.kind, ActivityOptionsErrorKind::MissingTotalTimeout); + let zero = ActivityOptions::new() + .cancellation_policy(CancellationPolicy::Abandon) + .schedule_to_close_timeout(Duration::ZERO) + .validate() + .unwrap_err(); + assert_eq!(zero.kind, ActivityOptionsErrorKind::TimeoutNotPositive); +} diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index bdd58fd..f958fe1 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -2,9 +2,9 @@ //! Ordinary cargo tests leave these cases ignored. They require a real runtime. use durable_workflow::{ - json, ActivityContext, Client, ConditionWaitOptions, CooperativeCancellationOptions, Error, - ParallelOperation, ParallelResult, SelectionKey, Value, Worker, WorkflowCommandOptions, - WorkflowHandle, WorkflowResultOptions, + json, ActivityContext, ActivityOptions, CancellationPolicy, Client, ConditionWaitOptions, + CooperativeCancellationOptions, Error, ParallelOperation, ParallelResult, SelectionKey, Value, + Worker, WorkflowCommandOptions, WorkflowHandle, WorkflowResultOptions, }; use std::{ sync::{ @@ -816,7 +816,7 @@ async fn server_cold_successor_runs_saga_cleanup_before_terminal_cancellation() assert_eq!(completed.load(Ordering::SeqCst), 1); } -async fn managed_remote_cancellation(user_heartbeat: bool) { +async fn managed_remote_cancellation(user_heartbeat: bool, policy: Option) { let client = client(); let queue = queue(); let dropped = Arc::new(AtomicUsize::new(0)); @@ -835,10 +835,20 @@ async fn managed_remote_cancellation(user_heartbeat: bool) { let _ = heartbeat_tx.send(()); } }); - worker.register_workflow(WORKFLOW, |ctx, _| async move { + worker.register_workflow(WORKFLOW, move |ctx, _| async move { let mut saga = ctx.saga(); saga.add_compensation(UNDO, json!([]))?; - let result = ctx.activity(BLOCKED, json!([])).await; + let result = match policy { + Some(policy) => { + ctx.activity_with_options( + BLOCKED, + ActivityOptions::new().cancellation_policy(policy), + json!([]), + ) + .await + } + None => ctx.activity(BLOCKED, json!([])).await, + }; saga.finish(result).await?; Ok(Value::Null) }); @@ -989,6 +999,37 @@ async fn managed_remote_cancellation(user_heartbeat: bool) { assert_eq!(duplicate["history_event_id"], receipt["history_event_id"]); snapshot = history(&handle).await; assert_eq!(count(&snapshot, "ActivityCancellationAcknowledged"), 1); + if let Some(policy) = policy { + let events = snapshot["events"].as_array().unwrap(); + let scheduled = events + .iter() + .find(|event| event["event_type"] == "ActivityScheduled") + .unwrap(); + let expected = match policy { + CancellationPolicy::TryCancel => "try_cancel", + CancellationPolicy::WaitCancellationCompleted => "wait_cancellation_completed", + CancellationPolicy::Abandon => unreachable!(), + _ => unreachable!(), + }; + assert_eq!( + scheduled["payload"]["activity"]["cancellation_policy"], + expected + ); + if policy == CancellationPolicy::WaitCancellationCompleted { + let stop = events + .iter() + .position(|event| event["event_type"] == "ActivityCancellationAcknowledged") + .unwrap(); + let delivery = events + .iter() + .position(|event| event["event_type"] == "CooperativeCancellationDelivered") + .unwrap(); + assert!( + stop < delivery, + "Wait must record actual callback drop before workflow delivery" + ); + } + } let recorded = snapshot["events"] .as_array() .unwrap() @@ -1058,13 +1099,204 @@ async fn managed_remote_cancellation(user_heartbeat: bool) { #[tokio::test] #[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] async fn server_managed_worker_fences_blocked_callback_without_user_heartbeats() { - managed_remote_cancellation(false).await; + managed_remote_cancellation(false, None).await; } #[tokio::test] #[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] async fn server_managed_worker_fences_blocked_callback_with_user_heartbeats() { - managed_remote_cancellation(true).await; + managed_remote_cancellation(true, None).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_explicit_try_cancellation_stops_callback_without_user_heartbeats() { + managed_remote_cancellation(false, Some(CancellationPolicy::TryCancel)).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_explicit_wait_records_callback_stop_before_delivery() { + managed_remote_cancellation(false, Some(CancellationPolicy::WaitCancellationCompleted)).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_bounded_abandon_completes_independent_activity_after_parent_cancellation() { + let client = client(); + let queue = queue(); + let dropped = Arc::new(AtomicUsize::new(0)); + let (entered_tx, mut entered_rx) = tokio::sync::mpsc::unbounded_channel::(); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let mut worker = Worker::new(client.clone(), &queue) + .worker_id(format!("{queue}-abandon")) + .cooperative_cancellation(true) + .max_concurrent_workflow_tasks(1) + .max_concurrent_activity_tasks(2) + .poll_timeout(Duration::from_secs(1)); + worker.register_workflow(WORKFLOW, |ctx, _| async move { + let mut saga = ctx.saga(); + saga.add_compensation(UNDO, json!([]))?; + let result = ctx + .activity_with_options( + BLOCKED, + ActivityOptions::new() + .cancellation_policy(CancellationPolicy::Abandon) + .schedule_to_close_timeout(Duration::from_secs(60)), + json!([]), + ) + .await; + saga.finish(result).await?; + Ok(Value::Null) + }); + let callback_dropped = Arc::clone(&dropped); + worker.register_activity(BLOCKED, move |ctx, _| { + let entered_tx = entered_tx.clone(); + let callback_dropped = Arc::clone(&callback_dropped); + async move { + let _drop = CallbackDrop(callback_dropped); + entered_tx.send(ctx).unwrap(); + tokio::time::sleep(Duration::from_secs(25)).await; + Ok(json!("independent-completion")) + } + }); + worker.register_activity(UNDO, |_ctx, _| async { Ok(Value::Null) }); + let run = tokio::spawn(async move { + worker + .run_until(async { + let _ = shutdown_rx.await; + }) + .await + }); + let handle = client + .start_workflow(WORKFLOW, &queue, &queue, json!([])) + .await + .unwrap(); + let original = tokio::time::timeout(Duration::from_secs(10), entered_rx.recv()) + .await + .unwrap() + .unwrap(); + let request = handle + .request_cancellation(CooperativeCancellationOptions { + cleanup_timeout_seconds: Some(30), + ..CooperativeCancellationOptions::default() + }) + .await + .unwrap(); + let duplicate = handle + .request_cancellation(CooperativeCancellationOptions { + cleanup_timeout_seconds: Some(300), + ..CooperativeCancellationOptions::default() + }) + .await + .unwrap(); + assert!(duplicate.duplicate); + assert_eq!(request.cancellation_request, duplicate.cancellation_request); + let snapshot = assert_cancelled(&handle, &request.cancellation_request.request_id, true).await; + assert_eq!( + dropped.load(Ordering::SeqCst), + 0, + "Abandon must preserve the callback after parent closure" + ); + assert_eq!(count(&snapshot, "ActivityCancelled"), 0); + assert_eq!(count(&snapshot, "ActivityCancellationAcknowledged"), 0); + let scheduled = snapshot["events"] + .as_array() + .unwrap() + .iter() + .find(|event| event["event_type"] == "ActivityScheduled") + .unwrap(); + assert_eq!( + scheduled["payload"]["activity"]["cancellation_policy"], + "abandon" + ); + let total_deadline = scheduled["payload"]["activity"]["schedule_to_close_deadline_at"].clone(); + assert!(total_deadline.is_string()); + let status = client + .activity_task_status( + &original.task_id, + &original.activity_attempt_id, + &original.lease_owner, + ) + .await + .unwrap(); + assert_eq!(status["can_continue"], true); + let snapshot = tokio::time::timeout(Duration::from_secs(35), async { + loop { + let snapshot = history(&handle).await; + if snapshot["events"].as_array().unwrap().iter().any(|event| { + event["event_type"] == "ActivityCompleted" + && event["payload"]["activity_type"] == BLOCKED + }) { + break snapshot; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await + .expect("independent callback must complete within its original lifetime"); + let completed: Vec<_> = snapshot["events"] + .as_array() + .unwrap() + .iter() + .filter(|event| { + event["event_type"] == "ActivityCompleted" + && event["payload"]["activity_type"] == BLOCKED + }) + .collect(); + assert_eq!(completed.len(), 1); + let envelope: durable_workflow::PayloadEnvelope = + serde_json::from_value(completed[0]["payload"]["result"].clone()).unwrap(); + assert_eq!( + durable_workflow::decode_payload::(&envelope).unwrap(), + "independent-completion" + ); + assert_eq!( + completed[0]["payload"]["activity"]["schedule_to_close_deadline_at"], + total_deadline + ); + assert_eq!( + completed[0]["payload"]["activity_attempt_id"], + original.activity_attempt_id + ); + assert_eq!(count(&snapshot, "WorkflowCancelled"), 1); + assert_eq!(count(&snapshot, "WorkflowCompleted"), 0); + assert_eq!(count(&snapshot, "ActivityCancellationAcknowledged"), 0); + for outcome in [ + client + .complete_activity_task( + &original.task_id, + &original.activity_attempt_id, + &original.lease_owner, + json!("stale"), + "avro", + ) + .await, + client + .fail_activity_task( + &original.task_id, + &original.activity_attempt_id, + &original.lease_owner, + "stale", + true, + ) + .await, + ] { + assert!( + matches!(outcome, Err(Error::ActivityTaskRejected(rejection)) if rejection.status == 409) + ); + } + assert_eq!(snapshot, history(&handle).await); + eprintln!( + "Bounded remote Abandon: root={}, total_deadline={}, completed={}", + request.cancellation_request.request_id, total_deadline, completed[0] + ); + shutdown_tx.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(10), run) + .await + .unwrap() + .unwrap() + .unwrap(); } fn replay_worker(client: &Client, queue: &str, mode: &'static str) -> Worker { diff --git a/tests/fixtures/replay-regressions/activity-cancellation-policy-changed.json b/tests/fixtures/replay-regressions/activity-cancellation-policy-changed.json new file mode 100644 index 0000000..3fd68c2 --- /dev/null +++ b/tests/fixtures/replay-regressions/activity-cancellation-policy-changed.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-activity-cancellation-policy-changed", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": {"type": "corpus.activity-policy-author", "input": [], "payload_codec": "avro"}, + "history": [ + {"event_type": "ActivityScheduled", "payload": {"sequence": 1, "activity_type": "work", "activity": {"type": "work", "cancellation_policy": "wait_cancellation_completed"}}} + ], + "command_sequence": [{"type": "complete_workflow", "result": "unreachable"}], + "expected": {"type": "complete_workflow", "result": "unreachable"}, + "expected_failure": "activity_cancellation_policy_changed" +} diff --git a/tests/replay_regression_corpus.rs b/tests/replay_regression_corpus.rs index 6df0226..3b7a912 100644 --- a/tests/replay_regression_corpus.rs +++ b/tests/replay_regression_corpus.rs @@ -412,6 +412,7 @@ async fn execute_fixture_delivery(fixture: &Value, delivery_id: &str) -> Result< | "corpus.durable-selection-portable-affinity" | "corpus.redrive-boundary" | "corpus.child-policy-author" + | "corpus.activity-policy-author" ) { return Err(format!( "replay fixture {fixture_id} has no registered Rust workflow {workflow_type:?}" @@ -608,6 +609,12 @@ async fn execute_fixture_delivery(fixture: &Value, delivery_id: &str) -> Result< }, ); } + "corpus.activity-policy-author" => { + worker.register_workflow(workflow_type, |ctx, _input| async move { + ctx.activity("work", json!([])).await?; + Ok(json!("unreachable")) + }); + } "corpus.child-policy-author" => { worker.register_workflow(workflow_type, |ctx, _input| async move { ctx.start_child_workflow( From ecc5d2704f380ff9f14e81e538604b7edea0a2f1 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 17:21:04 +0000 Subject: [PATCH 36/58] Keep cooperative activity cleanup within bounded worker slots --- docs/cooperative-cancellation-design.md | 6 ++ src/lib.rs | 45 +++++++++-- src/tests/cooperative_cancellation.rs | 100 +++++++++++++++++++++++- 3 files changed, 143 insertions(+), 8 deletions(-) diff --git a/docs/cooperative-cancellation-design.md b/docs/cooperative-cancellation-design.md index aadf8bb..ad93043 100644 --- a/docs/cooperative-cancellation-design.md +++ b/docs/cooperative-cancellation-design.md @@ -71,6 +71,12 @@ cancellation. It requires a finite positive `schedule_to_close_timeout` and keeps that original deadline as its total lifetime. It does not extend the parent's cleanup budget. +The opted-in managed worker runs at most `max_concurrent_activity_tasks` +callbacks concurrently and joins all slots before deregistration. Reserve an +available activity slot for remote cleanup when independent Abandon work uses +the same worker. Workflow polling continues independently. Default protocol +1.19 workers retain their existing serial activity loop. + Explicit policies require cooperative worker opt-in, protocol 1.20 and a compatible installed backend. Unsupported workers identify their operation, identity and required protocol before submission. Server also checks the diff --git a/src/lib.rs b/src/lib.rs index ed18a88..59d59ed 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -31,7 +31,9 @@ use std::{ use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema}; use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; use chrono::DateTime; -use futures_util::{future::OptionFuture, task::noop_waker_ref}; +use futures_util::{ + future::OptionFuture, stream::FuturesUnordered, task::noop_waker_ref, StreamExt, +}; use serde::{ de::DeserializeOwned, ser::{SerializeMap, SerializeSeq}, @@ -7414,14 +7416,43 @@ impl Worker { } async fn poll_activities_until_stopped(self, stop: Arc) -> Result<()> { - while !stop.load(Ordering::SeqCst) { - if self.poll_activity_once().await? == ManagedPollOutcome::Stop { - stop.store(true, Ordering::SeqCst); - break; + // Each lane retains its leased response through callback settlement. + // A bounded independent activity must not occupy the cleanup lane too. + // Preserve the existing serial loop for default protocol 1.19 workers. + let lane_count = if self.cooperative_cancellation_enabled { + self.max_concurrent_activity_tasks + } else { + 1 + }; + let mut lanes = FuturesUnordered::new(); + for _ in 0..lane_count { + let worker = self.clone(); + let stop = Arc::clone(&stop); + lanes.push(async move { + while !stop.load(Ordering::SeqCst) { + match worker.poll_activity_once().await { + Ok(ManagedPollOutcome::Stop) => { + stop.store(true, Ordering::SeqCst); + break; + } + Err(error) => { + stop.store(true, Ordering::SeqCst); + return Err(error); + } + _ => {} + } + } + Ok(()) + }); + } + // Join every lane before deregistration, including after one fails. + let mut first_error = None; + while let Some(result) = lanes.next().await { + if let Err(error) = result { + first_error.get_or_insert(error); } } - - Ok(()) + first_error.map_or(Ok(()), Err) } async fn poll_query_once(&self) -> Result { diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs index cda8be7..36299ab 100644 --- a/src/tests/cooperative_cancellation.rs +++ b/src/tests/cooperative_cancellation.rs @@ -2389,7 +2389,7 @@ fn remote_activity_responses( return Some(("200 OK", json!({"worker_id":"actual-owner","outcome":"deregistered","recovered_workflow_task_count":1}).to_string())); } if path.ends_with("/activity-tasks/poll") { - if number > 1 { + if number > if case == "remote-concurrency" { 3 } else { 1 } { return Some(( "200 OK", json!({"task":null,"poll_status":"timeout"}).to_string(), @@ -2397,6 +2397,10 @@ fn remote_activity_responses( } let mut task = json!({"task_id":"activity","activity_attempt_id":"attempt-original", "activity_type":"work","payload_codec":"avro","lease_owner":"actual-owner","attempt_number":3}); + if case == "remote-concurrency" { + task["task_id"] = json!(format!("activity-{number}")); + task["activity_attempt_id"] = json!(format!("attempt-{number}")); + } match case { "remote-missing-owner" => { task.as_object_mut().unwrap().remove("lease_owner"); @@ -2435,6 +2439,9 @@ fn remote_activity_responses( "can_continue":true,"cancel_requested":false,"reason":null,"heartbeat_recorded":false, "lease_expires_at":"2099-01-01T00:00:00Z","deadlines":null,"worker_session":null, "task_status":"leased","attempt_status":"running","activity_status":"running"}); + if case == "remote-concurrency" { + reply["task_id"] = json!(path.rsplit('/').nth(1).unwrap()); + } match case { "remote-wrong-task" => reply["task_id"] = json!("other-task"), "remote-wrong-attempt" => reply["activity_attempt_id"] = json!("other-attempt"), @@ -3280,6 +3287,97 @@ async fn cooperative_activity_dropped_poll_future_abandons_cloned_context() { assert_eq!(server.requests.lock().unwrap().len(), before); } +#[tokio::test] +async fn cooperative_activity_concurrency_reuses_slots_and_stops_before_deregistering() { + struct ActiveCallback(Arc); + impl Drop for ActiveCallback { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } + } + let server = remote_activity_server(); + let mut worker = + coordinator_worker(&server, "remote-concurrency").max_concurrent_activity_tasks(2); + let active = Arc::new(AtomicUsize::new(0)); + let release = Arc::new(tokio::sync::Semaphore::new(0)); + let (entered_tx, mut entered_rx) = tokio::sync::mpsc::unbounded_channel(); + let callbacks = Arc::clone(&active); + let permits = Arc::clone(&release); + worker.register_activity("work", move |ctx, _| { + let callbacks = Arc::clone(&callbacks); + let permits = Arc::clone(&permits); + let entered_tx = entered_tx.clone(); + async move { + callbacks.fetch_add(1, Ordering::SeqCst); + let _active = ActiveCallback(callbacks); + entered_tx.send(ctx.task_id).unwrap(); + permits.acquire().await.unwrap().forget(); + Ok(Value::Null) + } + }); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let running = tokio::spawn(async move { + worker + .run_until(async { + let _ = shutdown_rx.await; + }) + .await + }); + let mut entered = Vec::new(); + for _ in 0..2 { + entered.push( + tokio::time::timeout(Duration::from_secs(3), entered_rx.recv()) + .await + .expect("both configured slots must start callbacks") + .unwrap(), + ); + } + assert_eq!(active.load(Ordering::SeqCst), 2); + assert!( + tokio::time::timeout(Duration::from_millis(100), entered_rx.recv()) + .await + .is_err(), + "a third callback cannot exceed the configured capacity" + ); + release.add_permits(1); + entered.push( + tokio::time::timeout(Duration::from_secs(3), entered_rx.recv()) + .await + .expect("a settled callback must release its slot") + .unwrap(), + ); + entered.sort(); + assert_eq!(entered, ["activity-1", "activity-2", "activity-3"]); + assert_eq!(active.load(Ordering::SeqCst), 2); + shutdown_tx.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(3), running) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + active.load(Ordering::SeqCst), + 0, + "shutdown must drop all managed callbacks" + ); + let requests = server.requests.lock().unwrap(); + assert_eq!( + requests + .iter() + .filter(|request| request.path.ends_with("/registrations/actual-owner")) + .count(), + 1 + ); + assert!( + requests + .last() + .unwrap() + .path + .ends_with("/registrations/actual-owner"), + "all lanes must stop before deregistration" + ); +} + fn coordinator_responses(path: &str, body: &str, number: usize) -> Option<(&'static str, String)> { let case = path.split('/').nth(1).unwrap_or_default(); if path.ends_with("/cluster/info") { From 2eaa9d5e81ece0d184eb0dd6a4ce6e17be0a701d Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 22:08:48 +0000 Subject: [PATCH 37/58] Add deterministic cancellation remaining time --- docs/cooperative-cancellation-design.md | 27 +- src/cancellation_context.rs | 50 +- src/cancellation_replay_clock.rs | 296 +++++++++++ src/cooperative_cancellation.rs | 12 +- src/lib.rs | 94 +++- src/tests/cooperative_cancellation.rs | 458 ++++++++++++++++++ tests/cooperative_server.rs | 88 +++- ...ancellation-remaining-final-condition.json | 56 +++ tests/replay_regression_corpus.rs | 19 + 9 files changed, 1082 insertions(+), 18 deletions(-) create mode 100644 src/cancellation_replay_clock.rs create mode 100644 tests/fixtures/replay-regressions/cancellation-remaining-final-condition.json diff --git a/docs/cooperative-cancellation-design.md b/docs/cooperative-cancellation-design.md index ad93043..42547dc 100644 --- a/docs/cooperative-cancellation-design.md +++ b/docs/cooperative-cancellation-design.md @@ -13,6 +13,23 @@ after committed authored cancellation delivery. Earlier workflow code receives snapshot in `request.context`. Poll and heartbeat observations do not supply workflow-facing context, even when their transport object contains extra fields. +`CancellationContext::remaining()` returns a `Result` from the original +deadline and the recorded blocking boundary consumed by this workflow replay. +Committed delivery starts the clock. Activity/child results, timer fires, signal +and condition resolutions advance it. Parallel groups use their consumed +members and exclude results later than the failure returned to workflow code. +Selection uses its committed winner marker, then the result or first cancellation +receipt of a handle that workflow code actually awaits. Synchronous side effects, +version markers, memo and search attributes preserve the same budget before and +after persistence. Recorded clock skew cannot increase the budget, microseconds +are preserved, and expiry returns zero. + +The helper reads no host clock and cannot renew a deadline. Contexts restored +from portable metadata, ended or unrelated workflow replays, and missing or +invalid committed timestamps return an explicit error. A weak replay binding +does not keep workflow history alive. The Server independently enforces the +original deadline while a worker is suspended or absent. + `CancellationContext` and `CancellationLineage` expose read-only accessors. Metadata includes local and root request IDs, root workflow instance and run IDs, the immediate parent request ID, original reason, requester/source, root request @@ -131,11 +148,11 @@ publication. ## Remaining qualification -Portable activity policies, nested scopes and deterministic remaining-time -helpers still need completion. Remaining time must use the replayed workflow -clock. Do not subtract the host clock from the deadline in workflow code. The -runtime continues enforcing the original deadline and fencing task and activity -ownership. Rust local activities and worker affinity remain unsupported. +Nested scopes, competitive qualification and exact published artifacts still +need completion. The remaining-time helper needs connected exact-head +qualification. Do not subtract the host clock from the deadline in workflow code. +The runtime continues enforcing the original deadline and fencing task and +activity ownership. Rust local activities and worker affinity remain unsupported. Connected qualification must cover the PHP parent, Python child, Rust remote activity and PHP local activity together. Both callbacks must stop without diff --git a/src/cancellation_context.rs b/src/cancellation_context.rs index 5937454..12bb91b 100644 --- a/src/cancellation_context.rs +++ b/src/cancellation_context.rs @@ -1,5 +1,6 @@ use super::*; use chrono::{SecondsFormat, Utc}; +use std::sync::Weak; /// One immutable local request in the ordered cancellation lineage. #[derive(Clone, Debug, PartialEq, Eq)] @@ -35,7 +36,7 @@ impl CancellationLineage { /// /// Fields and nested metadata have read-only accessors. Descendants retain /// the original root identity, request time and cleanup budget. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug)] pub struct CancellationContext { request_id: String, root_request_id: String, @@ -48,8 +49,27 @@ pub struct CancellationContext { requested_at: DateTime, cleanup_deadline_at: DateTime, lineage: Vec, + replay: Option>>, } +impl PartialEq for CancellationContext { + fn eq(&self, other: &Self) -> bool { + self.request_id == other.request_id + && self.root_request_id == other.root_request_id + && self.root_workflow_instance_id == other.root_workflow_instance_id + && self.root_workflow_run_id == other.root_workflow_run_id + && self.parent_request_id == other.parent_request_id + && self.reason == other.reason + && self.requester == other.requester + && self.source == other.source + && self.requested_at == other.requested_at + && self.cleanup_deadline_at == other.cleanup_deadline_at + && self.lineage == other.lineage + } +} + +impl Eq for CancellationContext {} + fn invalid_context(message: &str) -> Error { Error::InvalidCooperativeCancellation(message.to_owned()) } @@ -164,6 +184,7 @@ impl CancellationContext { requested_at, cleanup_deadline_at, lineage: normalized, + replay: None, }) } @@ -197,6 +218,33 @@ impl CancellationContext { pub fn deadline(&self) -> DateTime { self.cleanup_deadline_at } + + /// Remaining cleanup budget at the last blocking result consumed by this replay. + /// + /// Never reads host time. Detached metadata, an ended replay, and a missing or + /// invalid committed timestamp return an explicit error. Expiry returns zero. + pub fn remaining(&self) -> Result { + let state = self + .replay + .as_ref() + .and_then(Weak::upgrade) + .filter(cancellation_replay_clock::is_active) + .ok_or_else(|| { + invalid_context("remaining() is available only in its active workflow replay") + })?; + let time = state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)? + .cancellation_time()?; + Ok((self.cleanup_deadline_at - time) + .to_std() + .unwrap_or(Duration::ZERO)) + } + + pub(super) fn with_replay(mut self, replay: Option>>) -> Self { + self.replay = replay; + self + } pub fn lineage(&self) -> &[CancellationLineage] { &self.lineage } diff --git a/src/cancellation_replay_clock.rs b/src/cancellation_replay_clock.rs new file mode 100644 index 0000000..795e321 --- /dev/null +++ b/src/cancellation_replay_clock.rs @@ -0,0 +1,296 @@ +use super::*; +use chrono::Utc; +use std::{cell::RefCell, sync::Weak}; + +thread_local! { + static ACTIVE_REPLAY: RefCell>>> = const { RefCell::new(None) }; +} + +/// A binding is useful only while its original workflow future is being polled. +pub(super) struct ReplayGuard { + previous: Option>>, +} + +impl ReplayGuard { + pub(super) fn enter(ctx: &WorkflowContext) -> Result> { + let bound = ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)? + .cancellation_clock + .is_some(); + if !bound && ACTIVE_REPLAY.with(|active| active.borrow().is_none()) { + return Ok(None); + } + let previous = + ACTIVE_REPLAY.with(|active| active.replace(bound.then(|| Arc::downgrade(&ctx.state)))); + Ok(Some(Self { previous })) + } +} + +impl Drop for ReplayGuard { + fn drop(&mut self) { + ACTIVE_REPLAY.with(|active| active.replace(self.previous.take())); + } +} + +pub(super) fn active_binding() -> Option>> { + ACTIVE_REPLAY.with(|active| active.borrow().clone()) +} + +pub(super) fn is_active(state: &Arc>) -> bool { + ACTIVE_REPLAY.with(|active| { + active + .borrow() + .as_ref() + .and_then(Weak::upgrade) + .is_some_and(|current| Arc::ptr_eq(¤t, state)) + }) +} + +/// Index only committed blocking results. Synchronous metadata never enters this clock. +#[derive(Debug)] +pub(super) struct ReplayClock { + boundaries: HashMap, + failures: BTreeSet, + selections: HashMap, + cancellations: HashMap<(String, u64), usize>, + delivery_index: Option, + time: Option>, + available: bool, + started: bool, +} + +impl ReplayClock { + pub(super) fn new( + history: &[HistoryEvent], + commands: &[RecordedCommand], + delivery_index: Option, + ) -> Self { + let mut clock = Self { + boundaries: HashMap::new(), + failures: BTreeSet::new(), + selections: HashMap::new(), + cancellations: HashMap::new(), + delivery_index, + time: None, + available: false, + started: false, + }; + for (index, event) in history.iter().enumerate() { + if matches!( + event.event_type.as_str(), + "ActivityCompleted" + | "ActivityFailed" + | "ActivityCancelled" + | "ActivityTimedOut" + | "ChildRunCompleted" + | "ChildRunFailed" + | "ChildRunCancelled" + | "ChildRunTerminated" + | "TimerFired" + | "SignalApplied" + | "ConditionWaitSatisfied" + | "ConditionWaitTimedOut" + ) { + if let Some(sequence) = durable_event_sequence(event) { + if let std::collections::hash_map::Entry::Vacant(entry) = + clock.boundaries.entry(sequence) + { + entry.insert(index); + if matches!( + event.event_type.as_str(), + "ActivityFailed" + | "ActivityCancelled" + | "ActivityTimedOut" + | "ChildRunFailed" + | "ChildRunCancelled" + | "ChildRunTerminated" + ) { + clock.failures.insert(sequence); + } + } + } + } else if event.event_type == "SelectionResolved" { + if let Some(group) = event.payload["selection_group_id"].as_str() { + clock.selections.entry(group.to_owned()).or_insert(index); + } + } else if event.event_type == "SelectionOperationCancelled" { + if let (Some(group), Some(sequence)) = ( + event.payload["selection_group_id"].as_str(), + event.payload["member_base_sequence"].as_u64(), + ) { + clock + .cancellations + .entry((group.to_owned(), sequence)) + .or_insert(index); + } + } + } + // One logical grouped condition may have several physical waits. The + // group consumes its final resolved wait, as the operation replayer does. + let mut conditions = HashMap::new(); + for command in commands { + if let RecordedCommand::ConditionWait { + sequence, + occurrence_id, + parallel_group_path: Some(_), + .. + } = command + { + let original = *conditions.entry(occurrence_id).or_insert(*sequence); + if original != *sequence { + if let Some(index) = clock.boundaries.get(sequence).copied() { + clock.boundaries.insert(original, index); + } else { + clock.boundaries.remove(&original); + } + } + } + } + clock + } + + fn observe(&mut self, event: Option<&HistoryEvent>) { + let time = event.and_then(|event| { + event + .raw + .get("timestamp") + .filter(|value| !value.is_null()) + .or_else(|| event.raw.get("recorded_at")) + .and_then(Value::as_str) + .and_then(|time| DateTime::parse_from_rfc3339(time).ok()) + .map(|time| time.with_timezone(&Utc)) + }); + self.available = time.is_some(); + if let Some(time) = time { + self.time = Some(self.time.map_or(time, |previous| previous.max(time))); + } + } +} + +impl WorkflowState { + pub(super) fn start_cancellation_clock(&mut self) { + if let Some(clock) = &mut self.cancellation_clock { + if !clock.started { + clock.started = true; + clock.observe( + clock + .delivery_index + .and_then(|index| self.history_events.get(index)), + ); + } + } + } + + pub(super) fn cancellation_time(&self) -> Result> { + self.cancellation_clock + .as_ref() + .filter(|clock| clock.started && clock.available) + .and_then(|clock| clock.time) + .ok_or_else(|| { + Error::InvalidCooperativeCancellation( + "remaining() requires a recorded consumed blocking timestamp".to_owned(), + ) + }) + } + + fn advance_cancellation_indices(&mut self, mut indices: Vec>) { + let Some(clock) = &mut self.cancellation_clock else { + return; + }; + if !clock.started { + return; + } + // A missing timestamp/result cannot borrow a different operation's clock. + indices.sort_unstable(); + indices.dedup(); + let mut missing = false; + for index in indices { + clock.observe(index.and_then(|index| self.history_events.get(index))); + missing |= !clock.available; + } + if missing { + clock.available = false; + } + } + + pub(super) fn advance_cancellation_sequence( + &mut self, + sequence: u64, + path: &[ParallelGroupMetadata], + ) { + if path.is_empty() { + self.advance_cancellation_sequences(&[sequence], None); + } + } + + pub(super) fn unavailable_cancellation_boundary(&mut self, path: &[ParallelGroupMetadata]) { + if path.is_empty() { + self.advance_cancellation_indices(vec![None]); + } + } + + pub(super) fn advance_cancellation_sequences( + &mut self, + sequences: &[u64], + failed_sequence: Option, + ) { + let Some(clock) = &self.cancellation_clock else { + return; + }; + let cutoff = failed_sequence.and_then(|sequence| clock.boundaries.get(&sequence).copied()); + let indices = sequences + .iter() + .filter_map(|sequence| { + let index = clock.boundaries.get(sequence).copied(); + if cutoff.is_some_and(|cutoff| index.is_some_and(|index| index > cutoff)) { + None + } else { + Some(index) + } + }) + .collect(); + self.advance_cancellation_indices(indices); + } + + pub(super) fn advance_cancellation_selection(&mut self, group: &str) { + let index = self + .cancellation_clock + .as_ref() + .and_then(|clock| clock.selections.get(group).copied()); + self.advance_cancellation_indices(vec![index]); + } + + pub(super) fn advance_cancellation_receipt(&mut self, group: &str, sequence: u64) { + let index = self.cancellation_clock.as_ref().and_then(|clock| { + clock + .cancellations + .get(&(group.to_owned(), sequence)) + .copied() + }); + self.advance_cancellation_indices(vec![index]); + } + + pub(super) fn advance_cancellation_handle( + &mut self, + handle: &DurableOperationHandle, + failed: bool, + ) { + let sequences = (handle.base_sequence + ..handle.base_sequence.saturating_add(handle.size as u64)) + .collect::>(); + let failure = self + .cancellation_clock + .as_ref() + .filter(|_| failed) + .and_then(|clock| { + sequences + .iter() + .filter(|sequence| clock.failures.contains(sequence)) + .min_by_key(|sequence| clock.boundaries.get(sequence)) + .copied() + }); + self.advance_cancellation_sequences(&sequences, failure); + } +} diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index 25c3169..2129254 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -905,14 +905,21 @@ impl WorkflowState { Ok(()) } - pub(super) fn cancellation_error(&self) -> Error { + pub(super) fn cancellation_error(&mut self) -> Error { + if self.cancellation_consumed { + self.start_cancellation_clock(); + } match ( &self.cancellation_history.request, &self.cancellation_history.delivery, ) { (Some(request), Some(delivery)) => { + let mut request = request.clone(); + request.context = request.context.map(|context| { + context.with_replay(cancellation_replay_clock::active_binding()) + }); Error::CooperativeCancellationRequested(CooperativeCancellationRequested { - request: request.clone(), + request, delivery: delivery.clone(), }) } @@ -1034,6 +1041,7 @@ impl WorkflowContext { .request .as_ref() .and_then(|request| request.context.clone()) + .map(|context| context.with_replay(Some(Arc::downgrade(&self.state)))) } else { None }) diff --git a/src/lib.rs b/src/lib.rs index 59d59ed..39d80b2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,7 @@ #![doc = include_str!("../README.md")] mod cancellation_context; +mod cancellation_replay_clock; mod cooperative_cancellation; mod runtime_payloads; mod runtime_uploads; @@ -7744,7 +7745,18 @@ impl Worker { }; let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone()); let mut cx = TaskContext::from_waker(noop_waker_ref()); - match invocation.future.as_mut().poll(&mut cx) { + let outcome = { + let _replay = cancellation_replay_clock::ReplayGuard::enter(&workflow_context) + .map_err(|error| { + QueryTaskExecutionFailure::new( + "query_workflow_state_unavailable", + error.to_string(), + "QueryWorkflowStateUnavailable", + ) + })?; + invocation.future.as_mut().poll(&mut cx) + }; + match outcome { Poll::Ready(Ok(_)) => { workflow_context .ensure_history_consumed() @@ -7900,7 +7912,10 @@ impl Worker { let mut future = (workflow.execute)(ctx.clone(), input); let mut cx = TaskContext::from_waker(noop_waker_ref()); - let outcome = future.as_mut().poll(&mut cx); + let outcome = { + let _replay = cancellation_replay_clock::ReplayGuard::enter(&ctx)?; + future.as_mut().poll(&mut cx) + }; if ctx .state .lock() @@ -8856,7 +8871,7 @@ impl WorkflowContext { /// Passing this result to [`Saga::finish`] compensates already registered /// forward steps before the cancellation remains the initiating outcome. pub fn throw_if_cancellation_requested(&self) -> Result<()> { - let state = self + let mut state = self .state .lock() .map_err(|_| Error::WorkflowStatePoisoned)?; @@ -9670,6 +9685,7 @@ struct WorkflowState { cancel_requested: bool, cancellation_history: CancellationHistory, cancellation_consumed: bool, + cancellation_clock: Option, cancellation_shield_depth: u64, cancellation_delivery_enabled: bool, cancellation_delivery_intent: Option, @@ -9808,6 +9824,17 @@ impl WorkflowState { "WorkflowCancellationRequested" | "WorkflowCancelRequested" ) }); + let cancellation_clock = cancellation_history + .request + .as_ref() + .and_then(|request| request.context.as_ref()) + .map(|_| { + cancellation_replay_clock::ReplayClock::new( + &history, + &recorded_commands, + cancellation_history.delivery_index, + ) + }); Ok(Self { workflow_command_identity: String::new(), workflow_stream_command_counter: 0, @@ -9823,6 +9850,7 @@ impl WorkflowState { cancel_requested, cancellation_history, cancellation_consumed: false, + cancellation_clock, cancellation_shield_depth: 0, cancellation_delivery_enabled: false, cancellation_delivery_intent: None, @@ -11106,6 +11134,7 @@ impl ParallelLeafCall { struct ParallelLeaf { call: ParallelLeafCall, + sequence: u64, member_path: Vec, group_path: Vec, result: Option, @@ -11183,6 +11212,7 @@ impl ParallelCall { ); ParallelLeaf { call, + sequence: base_sequence + descriptor.offset as u64, member_path: descriptor.member_path, group_path: descriptor.group_path, result: None, @@ -11262,6 +11292,16 @@ impl ParallelCall { .first() .map(|entry| entry.parallel_group_id.clone()) .unwrap_or_default(); + let sequences = self + .leaves + .iter() + .map(|leaf| leaf.sequence) + .collect::>(); + self.ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)? + .advance_cancellation_sequences(&sequences, Some(failed.sequence)); return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure { group_id, member_path: failed.member_path.clone(), @@ -11292,6 +11332,16 @@ impl ParallelCall { self.shape.as_ref().expect("initialized parallel shape"), &mut flat_results, ); + let sequences = self + .leaves + .iter() + .map(|leaf| leaf.sequence) + .collect::>(); + self.ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)? + .advance_cancellation_sequences(&sequences, None); Poll::Ready(Ok(match results { ParallelAvroResult::Group(results) => results, ParallelAvroResult::Activity(_) @@ -11937,6 +11987,11 @@ impl Future for SelectCall { None }; let winner = handles[member_position].clone(); + self.ctx + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)? + .advance_cancellation_selection(&marker.selection_group_id); Poll::Ready(Ok(SelectionResult { key: winner.key.clone(), index: winner.index, @@ -12363,6 +12418,10 @@ impl Future for DurableOperationAwaitCall { Err(error) => return Poll::Ready(Err(error)), Ok(false) => {} Ok(true) => { + state.advance_cancellation_receipt( + &self.handle.selection_group_id, + self.handle.base_sequence, + ); return Poll::Ready(Err(Error::DurableOperationCancelled( DurableOperationCancelled { selection_group_id: self.handle.selection_group_id.clone(), @@ -12375,9 +12434,17 @@ impl Future for DurableOperationAwaitCall { } } match recorded_selection_member_outcome(&state, &self.handle) { - Ok(Some(result)) => Poll::Ready(Ok(result)), + Ok(Some(result)) => { + state.advance_cancellation_handle(&self.handle, false); + Poll::Ready(Ok(result)) + } Ok(None) => Poll::Pending, - Err(error) => Poll::Ready(Err(error)), + Err(error) => { + if !workflow_task_integrity_error(&error) { + state.advance_cancellation_handle(&self.handle, true); + } + Poll::Ready(Err(error)) + } } } } @@ -12455,7 +12522,13 @@ impl Future for CancelDurableOperationCall { }; match selection_cancellation_for_handle(&state, &self.handle) { Err(error) => return Poll::Ready(Err(error)), - Ok(true) => return Poll::Ready(Ok(())), + Ok(true) => { + state.advance_cancellation_receipt( + &self.handle.selection_group_id, + self.handle.base_sequence, + ); + return Poll::Ready(Ok(())); + } Ok(false) => {} } if recorded_selection_member_is_terminal(&state, &self.handle) { @@ -12751,6 +12824,7 @@ impl ActivityCall { } state.command_cursor += 1; if let Some(outcome) = outcome { + state.advance_cancellation_sequence(sequence, &self.parallel_group_path); return Poll::Ready(outcome.map_err(Error::ActivityFailed)); } state.matched_recorded_pending = true; @@ -12920,6 +12994,7 @@ impl Future for TimerCall { } state.command_cursor += 1; if fired { + state.advance_cancellation_sequence(sequence, &self.parallel_group_path); return Poll::Ready(Ok(())); } state.matched_recorded_pending = true; @@ -13100,6 +13175,9 @@ impl Future for ConditionWaitCall { Err(error) => return Poll::Ready(Err(error)), } result = recorded_result; + if result.is_some() { + state.advance_cancellation_sequence(sequence, &self.parallel_group_path); + } cursor += 1; } state.command_cursor = cursor; @@ -13329,6 +13407,7 @@ impl ChildWorkflowCall { } state.command_cursor += 1; if let Some(outcome) = outcome { + state.advance_cancellation_sequence(sequence, &self.parallel_group_path); return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed)); } state.matched_recorded_pending = true; @@ -13559,6 +13638,7 @@ impl SignalCall { } state.command_cursor += 1; if let Some(value) = value { + state.advance_cancellation_sequence(sequence, &self.parallel_group_path); return Poll::Ready(Ok(value)); } if state @@ -13570,6 +13650,7 @@ impl SignalCall { .resume_signal .take() .expect("matching resume signal is present"); + state.unavailable_cancellation_boundary(&self.parallel_group_path); return Poll::Ready(Ok(signal.arguments)); } @@ -13610,6 +13691,7 @@ impl SignalCall { .resume_signal .take() .expect("matching resume signal is present"); + state.unavailable_cancellation_boundary(&self.parallel_group_path); return Poll::Ready(Ok(signal.arguments)); } diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs index 36299ab..fd7d061 100644 --- a/src/tests/cooperative_cancellation.rs +++ b/src/tests/cooperative_cancellation.rs @@ -43,6 +43,464 @@ fn context_task(events: Vec) -> WorkflowTask { task } +fn remaining_delivery() -> HistoryEvent { + let mut delivery = context_delivery(); + delivery + .raw + .insert("timestamp".into(), json!("2026-10-01T00:00:08Z")); + delivery +} + +fn remaining_event(kind: &str, payload: Value, time: &str) -> HistoryEvent { + let mut result = event(kind, payload); + result.raw.insert("timestamp".into(), json!(time)); + result +} + +#[test] +fn cancellation_remaining_preserves_memo_decision_and_consumes_the_timer_on_cold_replay() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + let Err(Error::CooperativeCancellationRequested(cancelled)) = + ctx.sleep(Duration::from_secs(10)).await + else { + panic!("expected delivery"); + }; + let context = cancelled.request.context.unwrap(); + assert_eq!(context.remaining()?, Duration::new(22, 123456000)); + assert_eq!( + ctx.cancellation_context()?.unwrap().remaining()?, + context.remaining()? + ); + assert_eq!( + CancellationContext::from_value(&context.to_value())?, + context + ); + let _shield = ctx.cancellation_shield()?; + ctx.upsert_memo(json!({"phase": "cleanup"}))?; + let delay = if context.remaining()? == Duration::new(22, 123456000) { + 1 + } else { + 2 + }; + ctx.sleep(Duration::from_secs(delay)).await?; + Ok(json!(context.remaining()?.as_secs_f64())) + }); + let initial = vec![context_request(), remaining_delivery()]; + let commands = worker + .execute_workflow_task(context_task(initial.clone())) + .unwrap(); + assert_eq!(commands[0]["type"], "upsert_memo"); + assert_eq!(commands[1]["delay_seconds"], 1); + let mut history = initial; + history.push(remaining_event( + "MemoUpserted", + json!({"sequence":2,"entries":commands[0]["entries"],"merged":commands[0]["entries"]}), + "2026-10-01T00:00:12Z", + )); + history.push(event( + "TimerScheduled", + json!({"sequence":3,"timer_id":"cleanup-timer","delay_seconds":1}), + )); + // A later unrelated row must not move an earlier application decision. + history.push(remaining_event( + "WorkflowTaskScheduled", + json!({}), + "2026-10-01T00:00:29Z", + )); + assert!(worker + .execute_workflow_task(context_task(history.clone())) + .unwrap() + .is_empty()); + history.push(remaining_event( + "TimerFired", + json!({"sequence":3,"timer_id":"cleanup-timer","delay_seconds":1}), + "2026-10-01T00:00:25Z", + )); + for _restart in 0..2 { + let commands = worker + .execute_workflow_task(context_task(history.clone())) + .unwrap(); + assert_eq!( + decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).unwrap(), + json!(5.123456) + ); + } +} + +#[test] +fn cancellation_remaining_parallel_excludes_later_failure_siblings_and_keeps_skew_monotonic() { + for failed in [true, false] { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + let Err(Error::CooperativeCancellationRequested(cancelled)) = + ctx.sleep(Duration::from_secs(10)).await + else { + panic!("expected delivery"); + }; + let context = cancelled.request.context.unwrap(); + let _shield = ctx.cancellation_shield()?; + let outcome = ctx + .parallel(vec![ + ParallelOperation::activity("a", json!([])), + ParallelOperation::activity("b", json!([])), + ]) + .await; + Ok(json!({"failed":outcome.is_err(),"remaining":context.remaining()?.as_secs_f64()})) + }); + let mut history = vec![context_request(), remaining_delivery()]; + let commands = worker + .execute_workflow_task(context_task(history.clone())) + .unwrap(); + let mut payloads = Vec::new(); + for (offset, command) in commands.iter().enumerate() { + let mut payload = command.clone(); + payload.as_object_mut().unwrap().remove("type"); + payload["sequence"] = json!(2 + offset); + history.push(event("ActivityScheduled", payload.clone())); + payloads.push(payload); + } + if failed { + payloads[0]["message"] = json!("failed"); + payloads[0]["exception_type"] = json!("ExampleFailure"); + history.push(remaining_event( + "ActivityFailed", + payloads[0].clone(), + "2026-10-01T00:00:12Z", + )); + payloads[1]["result"] = fixture_envelope(Value::Null); + history.push(remaining_event( + "ActivityCompleted", + payloads[1].clone(), + "2026-10-01T00:00:25Z", + )); + } else { + for (offset, time) in ["2026-10-01T00:00:25Z", "2026-10-01T00:00:20Z"] + .into_iter() + .enumerate() + { + payloads[offset]["result"] = fixture_envelope(Value::Null); + history.push(remaining_event( + "ActivityCompleted", + payloads[offset].clone(), + time, + )); + } + } + for _restart in 0..2 { + let commands = worker + .execute_workflow_task(context_task(history.clone())) + .unwrap(); + assert_eq!( + decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).unwrap(), + json!({"failed":failed,"remaining":if failed {18.123456} else {5.123456}}) + ); + } + } +} + +#[test] +fn cancellation_remaining_refuses_detached_ended_and_missing_or_invalid_delivery_time() { + assert!(CancellationContext::from_value(&context_snapshot()) + .unwrap() + .remaining() + .is_err()); + for time in [ + Value::Null, + json!("tomorrow"), + json!("2026-02-30T00:00:08Z"), + json!("2026-10-01T00:00:08"), + ] { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + let Err(Error::CooperativeCancellationRequested(cancelled)) = + ctx.sleep(Duration::from_secs(10)).await + else { + panic!("expected delivery"); + }; + assert!(cancelled.request.context.unwrap().remaining().is_err()); + Ok(Value::Null) + }); + let mut delivery = context_delivery(); + delivery.raw.insert("timestamp".into(), time); + assert_eq!( + worker + .execute_workflow_task(context_task(vec![context_request(), delivery])) + .unwrap()[0]["type"], + "complete_workflow" + ); + } + let captured = Arc::new(Mutex::new(None)); + let capture = captured.clone(); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", move |ctx, _| { + let capture = capture.clone(); + async move { + let Err(Error::CooperativeCancellationRequested(cancelled)) = + ctx.sleep(Duration::from_secs(10)).await + else { + panic!("expected delivery"); + }; + let context = cancelled.request.context.unwrap(); + assert_eq!(context.remaining()?, Duration::new(22, 123456000)); + *capture.lock().unwrap() = Some(context); + Ok(Value::Null) + } + }); + worker + .execute_workflow_task(context_task(vec![context_request(), remaining_delivery()])) + .unwrap(); + assert!(captured + .lock() + .unwrap() + .as_ref() + .unwrap() + .remaining() + .is_err()); +} + +#[test] +fn cancellation_remaining_refuses_missing_consumed_result_and_clamps_expiry_with_timezone_offsets() +{ + for time in [None, Some("2026-10-01T08:00:32+08:00")] { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + let Err(Error::CooperativeCancellationRequested(cancelled)) = + ctx.sleep(Duration::from_secs(10)).await + else { + panic!("expected delivery"); + }; + let context = cancelled.request.context.unwrap(); + let _shield = ctx.cancellation_shield()?; + ctx.sleep(Duration::from_secs(1)).await?; + Ok(json!(context + .remaining() + .map(|remaining| remaining.as_secs_f64()) + .ok())) + }); + let mut fired = event( + "TimerFired", + json!({"sequence":2,"timer_id":"cleanup-timer","delay_seconds":1}), + ); + if let Some(time) = time { + fired.raw.insert("timestamp".into(), json!(time)); + } + let history = vec![ + context_request(), + remaining_delivery(), + event( + "TimerScheduled", + json!({"sequence":2,"timer_id":"cleanup-timer","delay_seconds":1}), + ), + fired, + ]; + let commands = worker.execute_workflow_task(context_task(history)).unwrap(); + assert_eq!( + decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).unwrap(), + if time.is_some() { + json!(0.0) + } else { + Value::Null + } + ); + } +} + +#[test] +fn cancellation_remaining_selection_uses_winner_then_handle_and_first_cancel_receipt() { + for cancel_slow in [false, true] { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + let Err(Error::CooperativeCancellationRequested(cancelled)) = ctx.sleep(Duration::from_secs(10)).await else { panic!("expected delivery"); }; + let context = cancelled.request.context.unwrap(); + let _shield = ctx.cancellation_shield()?; + let selection = ctx.select_keyed(vec![("slow", ParallelOperation::activity("slow", json!([]))), ("fast", ParallelOperation::activity("fast", json!([])))]).await?; + let selected = context.remaining()?.as_secs_f64(); + selection.winner.await_result().await?; + let winner = context.remaining()?.as_secs_f64(); + let result = selection.handle(&SelectionKey::from("slow")).unwrap().await_result().await; + Ok(json!({"selected":selected,"winner":winner,"slow":context.remaining()?.as_secs_f64(),"cancelled":matches!(result, Err(Error::DurableOperationCancelled(_)))})) + }); + let mut history = vec![context_request(), remaining_delivery()]; + let commands = worker + .execute_workflow_task(context_task(history.clone())) + .unwrap(); + assert_eq!(commands.len(), 2); + let mut payloads = Vec::new(); + for (offset, command) in commands.into_iter().enumerate() { + let mut payload = command; + payload.as_object_mut().unwrap().remove("type"); + payload["sequence"] = json!(2 + offset); + payload["activity_execution_id"] = json!(if offset == 0 { + "activity-slow" + } else { + "activity-fast" + }); + history.push(event("ActivityScheduled", payload.clone())); + payloads.push(payload); + } + payloads[1]["result"] = fixture_envelope(json!("fast")); + let mut completed = remaining_event( + "ActivityCompleted", + payloads[1].clone(), + "2026-10-01T00:00:10Z", + ); + completed.raw.insert("id".into(), json!("event-fast")); + history.push(completed); + history.push(remaining_event("SelectionResolved", json!({ + "selection_group_id":"select-calls:2:2","selection_group_base_sequence":2,"selection_group_size":2, + "member_key":"fast","member_index":1,"member_base_sequence":3,"member_size":1, + "operation_kind":"activity","operation_identity":"activity-fast","outcome":"completed", + "resolution_event_id":"event-fast","resolution_event_type":"ActivityCompleted" + }), "2026-10-01T00:00:12Z")); + if cancel_slow { + let receipt = json!({"selection_group_id":"select-calls:2:2","member_key":"slow","member_index":0,"member_base_sequence":2,"member_size":1,"operation_kind":"activity","operation_identity":"activity-slow"}); + history.push(remaining_event( + "SelectionOperationCancelled", + receipt.clone(), + "2026-10-01T00:00:15Z", + )); + history.push(remaining_event( + "SelectionOperationCancelled", + receipt, + "2026-10-01T00:00:28Z", + )); + } else { + payloads[0]["result"] = fixture_envelope(json!("slow")); + history.push(remaining_event( + "ActivityCompleted", + payloads[0].clone(), + "2026-10-01T00:00:20Z", + )); + } + for _restart in 0..2 { + let commands = worker + .execute_workflow_task(context_task(history.clone())) + .unwrap(); + assert_eq!( + decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).unwrap(), + json!({"selected":18.123456,"winner":18.123456,"slow":if cancel_slow {15.123456} else {10.123456},"cancelled":cancel_slow}) + ); + } + } +} + +#[test] +fn cancellation_remaining_parallel_consumes_the_final_physical_condition_wait() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + let Err(Error::CooperativeCancellationRequested(cancelled)) = + ctx.sleep(Duration::from_secs(10)).await + else { + panic!("expected delivery"); + }; + let context = cancelled.request.context.unwrap(); + let _shield = ctx.cancellation_shield()?; + ctx.parallel(vec![ + ParallelOperation::timer(Duration::from_secs(1)), + ParallelOperation::condition( + ConditionWaitOptions::new("ready", "sha256:ready"), + || Ok(false), + ), + ]) + .await?; + Ok(json!(context.remaining()?.as_secs_f64())) + }); + let mut history = vec![context_request(), remaining_delivery()]; + let commands = worker + .execute_workflow_task(context_task(history.clone())) + .unwrap(); + let mut timer = commands[0].clone(); + timer.as_object_mut().unwrap().remove("type"); + timer["sequence"] = json!(2); + timer["timer_id"] = json!("timer-2"); + history.push(event("TimerScheduled", timer.clone())); + let mut condition = commands[1].clone(); + condition.as_object_mut().unwrap().remove("type"); + condition["sequence"] = json!(3); + condition["condition_wait_id"] = json!("condition-3"); + history.push(event("ConditionWaitOpened", condition.clone())); + history.push(remaining_event( + "ConditionWaitSatisfied", + condition.clone(), + "2026-10-01T00:00:12Z", + )); + condition["sequence"] = json!(4); + condition["condition_wait_id"] = json!("condition-4"); + history.push(event("ConditionWaitOpened", condition.clone())); + history.push(remaining_event( + "ConditionWaitSatisfied", + condition, + "2026-10-01T00:00:25Z", + )); + history.push(remaining_event("TimerFired", timer, "2026-10-01T00:00:10Z")); + for _restart in 0..2 { + let commands = worker + .execute_workflow_task(context_task(history.clone())) + .unwrap(); + assert_eq!( + decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).unwrap(), + json!(5.123456) + ); + } +} + +#[test] +fn cancellation_remaining_cannot_escape_into_an_unrelated_nested_worker_poll() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + let Err(Error::CooperativeCancellationRequested(cancelled)) = + ctx.sleep(Duration::from_secs(10)).await + else { + panic!("expected delivery"); + }; + let context = cancelled.request.context.unwrap(); + let nested_context = context.clone(); + let mut nested = cancellation_worker(); + nested.register_workflow("cancel", move |_, _| { + let nested_context = nested_context.clone(); + async move { + assert!( + nested_context.remaining().is_err(), + "another workflow cannot borrow the outer replay binding" + ); + Ok(Value::Null) + } + }); + nested.execute_workflow_task(context_task(Vec::new()))?; + assert_eq!( + context.remaining()?, + Duration::new(22, 123456000), + "outer binding is restored after nested poll" + ); + Ok(Value::Null) + }); + worker + .execute_workflow_task(context_task(vec![context_request(), remaining_delivery()])) + .unwrap(); +} + +#[test] +fn cancellation_remaining_refuses_a_legacy_signal_without_committed_result_time() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + let _ = ctx.sleep(Duration::from_secs(10)).await; + let context = ctx.cancellation_context()?.unwrap(); + assert_eq!(context.remaining()?, Duration::new(22, 123456000)); + let _shield = ctx.cancellation_shield()?; + assert_eq!(ctx.wait_signal("resume").await?, vec![json!(7)]); + assert!( + context.remaining().is_err(), + "missing signal time cannot reuse the delivery budget" + ); + Ok(Value::Null) + }); + let mut task = context_task(vec![context_request(), remaining_delivery()]); + task.signal_name = Some("resume".into()); + task.signal_arguments = Some(fixture_envelope(json!([7]))); + worker.execute_workflow_task(task).unwrap(); +} + #[test] fn cooperative_context_retains_original_snapshot_and_detached_copies() { let mut original = context_snapshot(); diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index f958fe1..a211e93 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -9,7 +9,7 @@ use durable_workflow::{ use std::{ sync::{ atomic::{AtomicUsize, Ordering}, - Arc, + Arc, Mutex, }, time::Duration, }; @@ -785,14 +785,19 @@ async fn server_commits_earlier_side_effect_before_cancellation_delivery() { async fn server_cold_successor_runs_saga_cleanup_before_terminal_cancellation() { let client = client(); let queue = queue(); - let original = worker(&client, &queue, true, false); + let observations = Arc::new(Mutex::new(Vec::new())); + let mut original = worker(&client, &queue, true, false); + register_remaining_workflow(&mut original, observations.clone(), "original"); original.register().await.unwrap(); let handle = client .start_workflow(WORKFLOW, &queue, &queue, json!([])) .await .unwrap(); let request = handle - .request_cancellation(CooperativeCancellationOptions::default()) + .request_cancellation(CooperativeCancellationOptions { + cleanup_timeout_seconds: Some(30), + ..CooperativeCancellationOptions::default() + }) .await .unwrap(); let pending = tick_until(&original, &handle, "ActivityScheduled").await; @@ -800,6 +805,7 @@ async fn server_cold_successor_runs_saga_cleanup_before_terminal_cancellation() assert_eq!(count(&pending, "WorkflowCancelled"), 0); drop(original); let mut successor = worker(&client, &queue, true, false); + register_remaining_workflow(&mut successor, observations.clone(), "replacement"); let completed = Arc::new(AtomicUsize::new(0)); let called = Arc::clone(&completed); successor.register_activity(UNDO, move |ctx, _| { @@ -812,8 +818,82 @@ async fn server_cold_successor_runs_saga_cleanup_before_terminal_cancellation() }); successor.register().await.unwrap(); tick_until(&successor, &handle, "WorkflowCancelled").await; - assert_cancelled(&handle, &request.cancellation_request.request_id, true).await; + let snapshot = assert_cancelled(&handle, &request.cancellation_request.request_id, true).await; assert_eq!(completed.load(Ordering::SeqCst), 1); + let events = snapshot["events"].as_array().unwrap(); + let event_time = |kind| { + chrono::DateTime::parse_from_rfc3339( + events + .iter() + .find(|event| event["event_type"] == kind) + .unwrap()["timestamp"] + .as_str() + .unwrap(), + ) + .unwrap() + }; + let deadline = + chrono::DateTime::parse_from_rfc3339(&request.cancellation_request.cleanup_deadline_at) + .unwrap(); + let delivery_budget = (deadline - event_time("CooperativeCancellationDelivered")) + .to_std() + .unwrap() + .as_secs_f64(); + let completed_budget = (deadline - event_time("ActivityCompleted")) + .to_std() + .unwrap() + .as_secs_f64(); + let observed = observations.lock().unwrap(); + let deliveries = observed + .iter() + .filter(|entry| entry["phase"] == "delivery") + .collect::>(); + assert!(deliveries.iter().any(|entry| entry["worker"] == "original")); + assert!(deliveries + .iter() + .any(|entry| entry["worker"] == "replacement")); + for entry in &deliveries { + assert_eq!(entry["context"], deliveries[0]["context"]); + assert!((entry["remaining"].as_f64().unwrap() - delivery_budget).abs() < 0.000001); + } + let completed = observed + .iter() + .find(|entry| entry["phase"] == "completed") + .unwrap(); + assert_eq!(completed["context"], deliveries[0]["context"]); + assert!((completed["remaining"].as_f64().unwrap() - completed_budget).abs() < 0.000001); + assert!(0.0 < completed_budget && completed_budget < delivery_budget); + assert!(event_time("WorkflowCancelled") < deadline); + assert_eq!(count(&snapshot, "MemoUpserted"), 1); + eprintln!( + "cold cleanup remaining-time observations: {}", + json!(*observed) + ); + eprintln!("cold cleanup remaining-time history: {snapshot}"); +} + +fn register_remaining_workflow( + worker: &mut Worker, + observations: Arc>>, + label: &'static str, +) { + worker.register_workflow(WORKFLOW, move |ctx, _| { + let observations = observations.clone(); + async move { + let mut saga = ctx.saga(); + saga.add_compensation(UNDO, json!([]))?; + let result = ctx.sleep(Duration::from_secs(300)).await; + let context = ctx.cancellation_context()?.expect("committed rich delivery"); + let remaining = context.remaining()?; + observations.lock().unwrap().push(json!({"worker":label,"phase":"delivery","context":context.to_value(),"remaining":remaining.as_secs_f64()})); + ctx.upsert_memo(json!({"phase":"cleanup"}))?; + assert_eq!(context.remaining()?, remaining, "synchronous memo preserves the consumed budget"); + let finished = saga.finish(result).await; + observations.lock().unwrap().push(json!({"worker":label,"phase":"completed","context":context.to_value(),"remaining":context.remaining()?.as_secs_f64()})); + finished?; + Ok(Value::Null) + } + }); } async fn managed_remote_cancellation(user_heartbeat: bool, policy: Option) { diff --git a/tests/fixtures/replay-regressions/cancellation-remaining-final-condition.json b/tests/fixtures/replay-regressions/cancellation-remaining-final-condition.json new file mode 100644 index 0000000..7032bca --- /dev/null +++ b/tests/fixtures/replay-regressions/cancellation-remaining-final-condition.json @@ -0,0 +1,56 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-cancellation-remaining-final-condition", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": {"type": "corpus.cancellation-remaining-condition", "input": [], "payload_codec": "avro"}, + "history": [ + { + "event_type": "CooperativeCancellationRequested", "recorded_at": "2026-10-01T00:00:00.123456Z", + "payload": { + "workflow_command_id": "original-request", "workflow_run_id": "regression-corpus-run", + "cleanup_deadline_at": "2026-10-01T00:00:30.123456Z", + "cancellation": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "original-request", "root_request_id": "original-request", + "root_workflow_instance_id": "regression-corpus-rust-cancellation-remaining-final-condition", + "root_workflow_run_id": "regression-corpus-run", "parent_request_id": null, + "reason": null, "requester": {"type": "operator", "id": "fixture-operator"}, "source": "control_plane", + "requested_at": "2026-10-01T00:00:00.123456Z", "cleanup_deadline_at": "2026-10-01T00:00:30.123456Z", + "lineage": [{"request_id": "original-request", "workflow_instance_id": "regression-corpus-rust-cancellation-remaining-final-condition", "workflow_run_id": "regression-corpus-run"}] + } + } + }, + { + "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-01T00:00:08Z", + "payload": {"workflow_command_id": "original-request", "workflow_run_id": "regression-corpus-run", "sequence": 1, "call_kind": "timer"} + }, + { + "event_type": "TimerScheduled", + "payload": {"sequence": 2, "timer_id": "timer-2", "delay_seconds": 1, "parallel_group_id": "parallel-calls:2:2", "parallel_group_kind": "mixed", "parallel_group_base_sequence": 2, "parallel_group_size": 2, "parallel_group_index": 0} + }, + { + "event_type": "ConditionWaitOpened", + "payload": {"sequence": 3, "condition_wait_id": "condition-3", "condition_wait_occurrence_id": "rust:condition-wait:0", "condition_key": "ready", "condition_definition_fingerprint": "sha256:ready", "parallel_group_id": "parallel-calls:2:2", "parallel_group_kind": "mixed", "parallel_group_base_sequence": 2, "parallel_group_size": 2, "parallel_group_index": 1} + }, + { + "event_type": "ConditionWaitSatisfied", "timestamp": "2026-10-01T00:00:12Z", + "payload": {"sequence": 3, "condition_wait_id": "condition-3", "condition_wait_occurrence_id": "rust:condition-wait:0", "condition_key": "ready", "condition_definition_fingerprint": "sha256:ready", "parallel_group_id": "parallel-calls:2:2", "parallel_group_kind": "mixed", "parallel_group_base_sequence": 2, "parallel_group_size": 2, "parallel_group_index": 1} + }, + { + "event_type": "ConditionWaitOpened", + "payload": {"sequence": 4, "condition_wait_id": "condition-4", "condition_wait_occurrence_id": "rust:condition-wait:0", "condition_key": "ready", "condition_definition_fingerprint": "sha256:ready", "parallel_group_id": "parallel-calls:2:2", "parallel_group_kind": "mixed", "parallel_group_base_sequence": 2, "parallel_group_size": 2, "parallel_group_index": 1} + }, + { + "event_type": "ConditionWaitSatisfied", "timestamp": "2026-10-01T00:00:25Z", + "payload": {"sequence": 4, "condition_wait_id": "condition-4", "condition_wait_occurrence_id": "rust:condition-wait:0", "condition_key": "ready", "condition_definition_fingerprint": "sha256:ready", "parallel_group_id": "parallel-calls:2:2", "parallel_group_kind": "mixed", "parallel_group_base_sequence": 2, "parallel_group_size": 2, "parallel_group_index": 1} + }, + { + "event_type": "TimerFired", "timestamp": "2026-10-01T00:00:10Z", + "payload": {"sequence": 2, "timer_id": "timer-2", "delay_seconds": 1, "parallel_group_id": "parallel-calls:2:2", "parallel_group_kind": "mixed", "parallel_group_base_sequence": 2, "parallel_group_size": 2, "parallel_group_index": 0} + } + ], + "command_sequence": [{"type": "complete_workflow", "result": 5.123456}], + "expected": {"type": "complete_workflow", "result": 5.123456} +} diff --git a/tests/replay_regression_corpus.rs b/tests/replay_regression_corpus.rs index 3b7a912..86e7e81 100644 --- a/tests/replay_regression_corpus.rs +++ b/tests/replay_regression_corpus.rs @@ -408,6 +408,7 @@ async fn execute_fixture_delivery(fixture: &Value, delivery_id: &str) -> Result< | "corpus.condition-search" | "corpus.condition-search-adjacent" | "corpus.grouped-condition-reopen" + | "corpus.cancellation-remaining-condition" | "corpus.durable-selection" | "corpus.durable-selection-portable-affinity" | "corpus.redrive-boundary" @@ -541,6 +542,24 @@ async fn execute_fixture_delivery(fixture: &Value, delivery_id: &str) -> Result< ])) }); } + "corpus.cancellation-remaining-condition" => { + worker.register_workflow(workflow_type, |ctx, _| async move { + let _ = ctx.sleep(Duration::from_secs(10)).await; + let context = ctx + .cancellation_context()? + .expect("committed rich cancellation"); + let _shield = ctx.cancellation_shield()?; + ctx.parallel(vec![ + ParallelOperation::timer(Duration::from_secs(1)), + ParallelOperation::condition( + ConditionWaitOptions::new("ready", "sha256:ready"), + || Ok(false), + ), + ]) + .await?; + Ok(json!(context.remaining()?.as_secs_f64())) + }); + } "corpus.grouped-condition-reopen" => { worker.register_workflow(workflow_type, |ctx, _| async move { ctx.parallel(vec![ From be0cd05aa5445bfd30a80b098946f6539f140b2d Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 3 Oct 2026 07:16:52 +0000 Subject: [PATCH 38/58] Document approved Rust 3.0 migration and release gates --- docs/cooperative-cancellation-design.md | 12 ++- docs/migrating-to-v3.md | 122 ++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 5 deletions(-) create mode 100644 docs/migrating-to-v3.md diff --git a/docs/cooperative-cancellation-design.md b/docs/cooperative-cancellation-design.md index 42547dc..774ab5d 100644 --- a/docs/cooperative-cancellation-design.md +++ b/docs/cooperative-cancellation-design.md @@ -140,11 +140,13 @@ enum and fields to the public `ChildWorkflowOptions` and `ActivityOptions` structs. It also adds a total-lifetime error category to the exhaustive public `ActivityOptionsErrorKind` enum. Existing struct literals and exhaustive matches need updates. The cooperative error variants -also extend an existing exhaustive public enum. These changes require a major -Rust SDK release if retained. This draft does not authorize that release or -change the published package version. The qualified release proposal must -include its migration notes and receive the major-release decision before -publication. +also extend an existing exhaustive public enum. The maintainer approved Rust +SDK 3.0.0 in the [October 3 release decision](https://github.com/durable-workflow/sdk-rust/pull/55#issuecomment-5966048960). +The [migration and release proposal](migrating-to-v3.md) covers these source +changes, historical defaults, upgrade order and rollback. Approval resolves the +major-version decision. Source qualification, protocol freeze and the published +mixed-language cancellation acceptance scenario remain release gates. The +candidate does not change a published artifact. ## Remaining qualification diff --git a/docs/migrating-to-v3.md b/docs/migrating-to-v3.md new file mode 100644 index 0000000..28f9780 --- /dev/null +++ b/docs/migrating-to-v3.md @@ -0,0 +1,122 @@ +# Rust SDK 3.0 migration and release proposal + +Rust SDK 3.0.0 is the approved major version for cooperative cancellation. This +document describes the source candidate. Publish it after the reviewed portable +contract is frozen and the release qualifications below pass. + +## Source changes + +The minimum supported Rust version remains 1.86. Existing workflow payloads use +the same Avro encoding. Protocol 1.19 remains the default, and cooperative +execution requires explicit protocol 1.20 support on the Server and worker. + +### Activity options + +`ActivityOptions` adds `cancellation_policy: Option`. +Applications using a complete struct literal must add `cancellation_policy: +None` or use `ActivityOptions::new()` and the existing builders. + +`None` preserves historical TryCancel behavior and the old command shape. +Explicit TryCancel or WaitCancellationCompleted requires cooperative support. +Abandon also requires a finite positive `schedule_to_close_timeout`. + +```rust +use durable_workflow::{ActivityOptions, CancellationPolicy}; +use std::time::Duration; + +let historical = ActivityOptions::new(); +let wait_for_stop = ActivityOptions::new() + .cancellation_policy(CancellationPolicy::WaitCancellationCompleted); +let independently_bounded = ActivityOptions::new() + .schedule_to_close_timeout(Duration::from_secs(60)) + .cancellation_policy(CancellationPolicy::Abandon); +``` + +### Child options and parent closure + +`ChildWorkflowOptions` adds `cancellation_policy: CancellationPolicy`. Complete +struct literals must add `CancellationPolicy::Abandon`, the historical default, +or use `ChildWorkflowOptions::new(task_queue)` and its builders. + +`ParentClosePolicy` adds `RequestCancellation`. Update exhaustive matches to +handle it. `RequestCancel` keeps its existing terminal semantics. + +```rust +use durable_workflow::{CancellationPolicy, ChildWorkflowOptions, ParentClosePolicy}; + +let historical = ChildWorkflowOptions::new("child-workers"); +let cooperative = ChildWorkflowOptions::new("child-workers") + .cancellation_policy(CancellationPolicy::WaitCancellationCompleted) + .parent_close_policy(ParentClosePolicy::RequestCancellation); +``` + +Operation cancellation and parent closure are separate decisions. A child +using WaitCancellationCompleted participates in cooperative cancellation at the +awaiting operation. RequestCancellation propagates a cooperative request when +the parent closes. Both retain the original root lineage and budget. + +### Errors and request APIs + +Update exhaustive matches on `ActivityOptionsErrorKind` for +`MissingTotalTimeout`. Update exhaustive matches on `Error` for +`CooperativeCancellationRequested`, `CooperativeCancellationUnavailable`, +`InvalidCooperativeCancellation` and `ActivityExecutionAbandoned`. + +Propagate cooperative workflow cancellation through the SDK's authored cleanup +path. Do not turn it into an ordinary workflow failure or publish an abandoned +Activity result. Capability and malformed-authority errors remain errors. + +Use `WorkflowHandle::request_cancellation()` for bounded cooperative cleanup, or +`request_selected_run_cancellation()` for an explicit run. The corresponding +Client methods are `request_workflow_cancellation()` and +`request_workflow_run_cancellation()`. Existing `cancel()` and +`cancel_selected_run()` remain terminal cancellation. + +`Worker::cooperative_cancellation(true)` opts into managed cooperation. Runtime +discovery and the immutable issued claim must support it. Unsupported capability +is an explicit diagnostic, never a silent conversion to terminal cancellation. + +## Existing runs and deployment order + +Omitted historical policies retain Activity TryCancel and child Abandon. Keep +those authored defaults while replaying existing runs. Changing a policy on an +already recorded operation is a replay mismatch. Introduce changed policies +through a new workflow type or the product's documented workflow versioning. + +1. Retain the old worker artifact and a tested recovery path. Qualify the exact + Server/Native upgrade with retained ordinary histories before deployment. +2. Upgrade workers without changing their authored historical defaults. Verify + ordinary published workflow execution and replay on the selected tuple. +3. Enable cooperation only after Server/Native discovery, protocol 1.20 and the + installed capability checks agree. Route new cooperative workflows to workers + that can replay their cancellation and operation-policy histories. +4. Inspect the cascade API/UI and durable cleanup outcomes before expanding use. + +Downgrading to 2.x is safe only for histories it can replay. Older workers cannot +take over runs requiring the new cooperative contract. Keep a compatible worker +available to finish those runs or use the documented recovery procedure before +rolling back a deployment. Do not delete history to make a downgrade appear safe. + +## Qualification and publication + +Release 3.0.0 only after current-source Rust 1.86/stable, replay corpus, official +Avro, package and documentation gates pass, nested-scope work is complete and the +portable contract is frozen. Qualify supported/default lease and repair settings. + +Publish an exact compatible tuple, then run the complete PHP parent, Python child, +Rust remote Activity and PHP local Activity scenario. Verify one root identity, +the original +30-second deadline, cooperative child delivery, both managed +callbacks stopped without application heartbeats, stale publication fenced, +actual workflow-process SIGKILL during cleanup, replacement replay of the same +boundary and consumed budget, duplicate identity/deadline, both runs Cancelled +before the original deadline and one coherent API/UI cascade. + +Verify crates.io availability, source provenance, archive checksum and a fresh +Rust 1.86 consumer. Update the published examples and documentation to that exact +qualified artifact. Shared cancellation issue 136 stays open until its complete +customer outcome and competitive qualification are demonstrated. + +Managed async callback supervision does not undo external effects or stop +detached threads and processes. Cooperating downstream systems still need +idempotency and fencing. Rust local Activities and worker affinity remain outside +this release's supported cancellation surfaces. From 76318e84d8aa26b40c7add28e1fc5d46aaba68b0 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 3 Oct 2026 12:17:44 +0000 Subject: [PATCH 39/58] Refuse unqualified cancellation scope replay before Rust handlers --- src/cooperative_cancellation.rs | 26 +++ src/lib.rs | 7 + src/tests/cancellation_scope_admission.rs | 204 ++++++++++++++++++ ...ion-scope-unqualified-worker-rejected.json | 27 +++ 4 files changed, 264 insertions(+) create mode 100644 src/tests/cancellation_scope_admission.rs create mode 100644 tests/fixtures/replay-regressions/cancellation-scope-unqualified-worker-rejected.json diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index 2129254..031631c 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -4,6 +4,32 @@ use std::collections::BTreeSet; const CONTROL_BUDGET: Duration = Duration::from_secs(5); const MAX_REFRESH_PAGES: usize = 128; +pub(super) fn assert_cancellation_scope_replay_supported(events: &[HistoryEvent]) -> Result<()> { + for event in events { + let scope_marker = matches!( + event.event_type.as_str(), + "CancellationScopeOpened" + | "CancellationScopeRequested" + | "CancellationScopeRequestConflicted" + ); + let scoped_membership = std::iter::once(&event.payload) + .chain( + ["activity", "timer", "child_workflow"] + .iter() + .filter_map(|name| event.payload.get(name)), + ) + .any(|payload| { + payload + .get("cancellation_scope_id") + .is_some_and(|scope| scope.as_str() != Some("root")) + }); + if scope_marker || scoped_membership { + return Err(Error::CancellationScopeExecutionUnavailable); + } + } + Ok(()) +} + /// A delivery acknowledgment bound to one workflow task and durable run. #[derive(Clone, Debug, PartialEq, Eq)] pub struct CancellationDeliveryReceipt { diff --git a/src/lib.rs b/src/lib.rs index 101bdc8..f23b0c9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -270,6 +270,8 @@ pub enum Error { WorkflowMemoUpdatesUnavailable, #[error("cooperative cancellation is unavailable: {0}")] CooperativeCancellationUnavailable(String), + #[error("cancellation_scope_execution_not_supported: Rust worker cannot replay scoped cancellation history")] + CancellationScopeExecutionUnavailable, #[error("invalid cooperative cancellation: {0}")] InvalidCooperativeCancellation(String), #[error("activity execution no longer owns its claim: {0}")] @@ -7322,6 +7324,7 @@ impl Worker { } } } + Err(error @ Error::CancellationScopeExecutionUnavailable) => return Err(error), Err(error) => { self.client .fail_workflow_task(task_id, lease_owner, attempt, error.to_string()) @@ -7839,6 +7842,7 @@ impl Worker { observation: Option<&CancellationRequest>, ) -> Result { validate_workflow_task_payloads(&task)?; + cooperative_cancellation::assert_cancellation_scope_replay_supported(&task.history_events)?; let cancellation_history = if let Some(observation) = observation { observation.validate_observation()?; cooperative_cancellation::cancellation_claim(&task)?; @@ -9736,6 +9740,7 @@ impl WorkflowState { payload_codec: String, resume_signal: Option, ) -> Result { + cooperative_cancellation::assert_cancellation_scope_replay_supported(&history)?; let cancellation_history = CancellationHistory::from_events( &history, run_id.as_deref().unwrap_or_default(), @@ -15857,6 +15862,7 @@ fn workflow_task_integrity_error(error: &Error) -> bool { error, Error::NonDeterministicReplay(_) | Error::Protocol(_) + | Error::CancellationScopeExecutionUnavailable | Error::MissingWorkflowCommandIdentity | Error::WorkflowStatePoisoned ) @@ -16215,6 +16221,7 @@ fn value_as_u64(value: &Value) -> Option { mod tests { use super::*; mod activity_cancellation_policies; + mod cancellation_scope_admission; mod child_workflow_policies; mod cooperative_cancellation; mod runtime_payloads; diff --git a/src/tests/cancellation_scope_admission.rs b/src/tests/cancellation_scope_admission.rs new file mode 100644 index 0000000..0591c0f --- /dev/null +++ b/src/tests/cancellation_scope_admission.rs @@ -0,0 +1,204 @@ +use super::*; + +fn scope_event(kind: &str, payload: Value) -> HistoryEvent { + serde_json::from_value(json!({"event_type": kind, "payload": payload})).unwrap() +} + +fn scope_histories() -> Vec { + let mut events = [ + "CancellationScopeOpened", + "CancellationScopeRequested", + "CancellationScopeRequestConflicted", + ] + .into_iter() + .map(|kind| scope_event(kind, json!({"scope_id": "scope-one", "sequence": 1}))) + .collect::>(); + for location in [ + None, + Some("activity"), + Some("timer"), + Some("child_workflow"), + ] { + let membership = json!({"cancellation_scope_id": "scope-one"}); + let payload = location.map_or_else(|| membership.clone(), |name| json!({name: membership})); + events.push(scope_event("TimerScheduled", payload)); + } + for malformed in [ + Value::Null, + json!(true), + json!(1), + json!(""), + json!([]), + json!({}), + ] { + events.push(scope_event( + "TimerScheduled", + json!({"cancellation_scope_id": malformed}), + )); + } + events +} + +fn scope_worker() -> Worker { + Worker::new(Client::new("http://127.0.0.1:1").unwrap(), "scope-queue").worker_id("scope-worker") +} + +#[test] +fn cancellation_scope_admission_precedes_the_application_factory() { + for event in scope_histories() { + let calls = Arc::new(AtomicUsize::new(0)); + let factory_calls = Arc::clone(&calls); + let mut worker = scope_worker(); + worker.register_workflow("scope-probe", move |_ctx, _input| { + factory_calls.fetch_add(1, Ordering::SeqCst); + async move { Ok(json!("unexpected")) } + }); + let error = worker + .execute_workflow_task(workflow_task("scope-probe", vec![event], DEFAULT_CODEC)) + .expect_err("unqualified scope history must not run application code"); + assert!(error + .to_string() + .contains("cancellation_scope_execution_not_supported")); + assert!(error.to_string().contains("Rust")); + assert_eq!(calls.load(Ordering::SeqCst), 0); + } +} + +#[tokio::test] +async fn cancellation_scope_admission_precedes_replayed_query_factories_and_handlers() { + let calls = Arc::new(AtomicUsize::new(0)); + let factory_calls = Arc::clone(&calls); + let body_calls = Arc::clone(&calls); + let query_calls = Arc::clone(&calls); + let mut worker = scope_worker(); + worker.register_replayed_workflow( + "scope-probe", + move || { + factory_calls.fetch_add(1, Ordering::SeqCst); + 0_u64 + }, + move |_ctx, _input, _state| { + body_calls.fetch_add(1, Ordering::SeqCst); + async move { Ok(Value::Null) } + }, + ); + worker.register_replayed_query::("scope-probe", "state", move |_, _, _| { + query_calls.fetch_add(1, Ordering::SeqCst); + async move { Ok(Value::Null) } + }); + let task = serde_json::from_value(json!({ + "query_task_id": "scope-query", "query_task_attempt": 1, + "workflow_type": "scope-probe", "query_name": "state", "payload_codec": "avro", + "workflow_arguments": fixture_envelope(json!([])), + "query_arguments": fixture_envelope(json!([])), + "history_events": [{"event_type": "CancellationScopeOpened", "payload": {"scope_id": "scope-one"}}], + "run_status": "running" + })).unwrap(); + let error = worker + .execute_query_task(task) + .await + .expect_err("scope query must refuse replay"); + assert!(error + .message + .contains("cancellation_scope_execution_not_supported")); + assert_eq!(calls.load(Ordering::SeqCst), 0); +} + +#[test] +fn cancellation_scope_admission_preserves_omitted_and_explicit_root_membership() { + for explicit_root in [false, true] { + let mut worker = scope_worker(); + worker.register_workflow("root-probe", |ctx, _| async move { + ctx.sleep(Duration::from_secs(1)).await?; + Ok(json!("done")) + }); + let mut payload = json!({"sequence": 1, "timer_id": "timer-one", "delay_seconds": 1}); + if explicit_root { + payload["cancellation_scope_id"] = json!("root"); + } + let history = vec![ + scope_event("TimerScheduled", payload.clone()), + scope_event("TimerFired", payload), + ]; + let commands = worker + .execute_workflow_task(workflow_task("root-probe", history, DEFAULT_CODEC)) + .unwrap(); + assert_eq!(commands.len(), 1); + assert_eq!(commands[0]["type"], "complete_workflow"); + assert_eq!( + decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).unwrap(), + json!("done") + ); + } +} + +#[test] +fn cancellation_scope_admission_preserves_scope_named_application_data() { + let value = json!({"cancellation_scope_id": "application-value", "activity": {"cancellation_scope_id": null}}); + let history = vec![scope_event( + "SideEffectRecorded", + json!({"sequence": 1, "result": fixture_envelope(value.clone())}), + )]; + let mut worker = scope_worker(); + worker.register_workflow("scope-data", |ctx, _| async move { + let recorded: Value = + ctx.side_effect(|| panic!("recorded side effect must not execute"))?; + Ok(recorded) + }); + let commands = worker + .execute_workflow_task(workflow_task("scope-data", history, DEFAULT_CODEC)) + .unwrap(); + assert_eq!( + decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).unwrap(), + value + ); +} + +#[tokio::test] +async fn cancellation_scope_admission_publishes_no_completion_or_task_failure() { + let server = MockWorkerServer::start(); + let worker = Worker::new(Client::new(server.base_url()).unwrap(), "scope-queue"); + let error = worker + .settle_workflow_task_decision( + "scope-task", + "scope-worker", + 2, + Some("scope-run"), + Err(Error::CancellationScopeExecutionUnavailable), + true, + ) + .await + .expect_err("unsupported scope must retain an explicit capability refusal"); + assert!(matches!( + error, + Error::CancellationScopeExecutionUnavailable + )); + assert!(server.requests.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn cancellation_scope_admission_keeps_explicit_snapshot_inspection_available() { + let mut worker = scope_worker(); + worker.register_workflow("scope-probe", |_, _| async move { + panic!("snapshot inspection must not replay the workflow") + }); + worker.register_query("scope-probe", "inspect", |ctx, _| async move { + Ok(json!(ctx.history_events[0].event_type)) + }); + let task = serde_json::from_value(json!({ + "query_task_id": "scope-inspect", "query_task_attempt": 1, + "workflow_type": "scope-probe", "query_name": "inspect", "payload_codec": "avro", + "workflow_arguments": fixture_envelope(json!([])), "query_arguments": fixture_envelope(json!([])), + "history_events": [{"event_type": "CancellationScopeOpened", "payload": {"scope_id": "scope-one"}}], + "run_status": "running" + })).unwrap(); + assert_eq!( + worker + .execute_query_task(task) + .await + .unwrap() + .into_json() + .unwrap(), + json!("CancellationScopeOpened") + ); +} diff --git a/tests/fixtures/replay-regressions/cancellation-scope-unqualified-worker-rejected.json b/tests/fixtures/replay-regressions/cancellation-scope-unqualified-worker-rejected.json new file mode 100644 index 0000000..278fae8 --- /dev/null +++ b/tests/fixtures/replay-regressions/cancellation-scope-unqualified-worker-rejected.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-cancellation-scope-unqualified-worker-rejected", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": { + "type": "corpus.side-effect-version", + "input": [], + "payload_codec": "avro" + }, + "history": [ + { + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "regression-corpus-run", + "scope_id": "scope-one", + "parent_scope_id": "root", + "sequence": 1, + "shield_parent": false + } + } + ], + "expected": {"command_sequence": []}, + "expected_failure": "cancellation_scope_execution_not_supported: Rust worker" +} From 274bf7b30eb5b4c8409db0b550d42c9f1d9a0a0e Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 3 Oct 2026 12:49:51 +0000 Subject: [PATCH 40/58] Refuse unqualified scoped delivery replay --- src/cooperative_cancellation.rs | 1 + src/tests/cancellation_scope_admission.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index 031631c..6d6eda1 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -10,6 +10,7 @@ pub(super) fn assert_cancellation_scope_replay_supported(events: &[HistoryEvent] event.event_type.as_str(), "CancellationScopeOpened" | "CancellationScopeRequested" + | "CancellationScopeDelivered" | "CancellationScopeRequestConflicted" ); let scoped_membership = std::iter::once(&event.payload) diff --git a/src/tests/cancellation_scope_admission.rs b/src/tests/cancellation_scope_admission.rs index 0591c0f..eccb259 100644 --- a/src/tests/cancellation_scope_admission.rs +++ b/src/tests/cancellation_scope_admission.rs @@ -8,6 +8,7 @@ fn scope_histories() -> Vec { let mut events = [ "CancellationScopeOpened", "CancellationScopeRequested", + "CancellationScopeDelivered", "CancellationScopeRequestConflicted", ] .into_iter() From 08824f2950460ec590e2f16272adc6ddf9a890ed Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sun, 4 Oct 2026 01:39:02 +0000 Subject: [PATCH 41/58] Preserve scoped cancellation origins in child run contexts --- docs/cooperative-cancellation-design.md | 15 +- src/cancellation_context.rs | 308 +++++++++++++++++- src/lib.rs | 4 +- src/tests/cooperative_cancellation.rs | 272 ++++++++++++++++ .../scoped-run-cancellation-context.json | 164 ++++++++++ 5 files changed, 751 insertions(+), 12 deletions(-) create mode 100644 src/tests/fixtures/scoped-run-cancellation-context.json diff --git a/docs/cooperative-cancellation-design.md b/docs/cooperative-cancellation-design.md index 774ab5d..edca62d 100644 --- a/docs/cooperative-cancellation-design.md +++ b/docs/cooperative-cancellation-design.md @@ -13,8 +13,10 @@ after committed authored cancellation delivery. Earlier workflow code receives snapshot in `request.context`. Poll and heartbeat observations do not supply workflow-facing context, even when their transport object contains extra fields. -`CancellationContext::remaining()` returns a `Result` from the original -deadline and the recorded blocking boundary consumed by this workflow replay. +`CancellationContext::remaining()` returns a `Result` from the accepted +cleanup deadline, bounded by the original global deadline, and the recorded +blocking boundary consumed by this workflow replay. A scoped child can have an +earlier authority ceiling. Committed delivery starts the clock. Activity/child results, timer fires, signal and condition resolutions advance it. Parallel groups use their consumed members and exclude results later than the failure returned to workflow code. @@ -37,6 +39,15 @@ time, original deadline and ordered lineage. Requester fields are limited to caller type, ID and label. Timestamp helpers return immutable UTC dates. `to_value()` produces a detached portable snapshot. +A scoped child context preserves its complete immutable `scope_origin` alongside +the local run lineage. `ScopedCancellationContext::root_context()` retains the +original root deadline. Its scoped lineage records each run, instance, scope, +request identity and accepted authority ceiling, including intermediate scopes +in the same run. The child deadline can narrow that ceiling and never extend it. +Legacy version 1 snapshots remain readable. Version 2 parsing rejects altered +root metadata, substituted ancestry and widened budgets. This metadata support +does not advertise scoped execution while that source contract is unfinished. + A child keeps the root request time and cleanup budget even if its local request is accepted later. Cold replay restores the object from canonical request history. Explicit checks and shielded cleanup retain it. The parser validates local diff --git a/src/cancellation_context.rs b/src/cancellation_context.rs index 12bb91b..4bce3c6 100644 --- a/src/cancellation_context.rs +++ b/src/cancellation_context.rs @@ -49,6 +49,7 @@ pub struct CancellationContext { requested_at: DateTime, cleanup_deadline_at: DateTime, lineage: Vec, + scope_origin: Option>, replay: Option>>, } @@ -65,6 +66,7 @@ impl PartialEq for CancellationContext { && self.requested_at == other.requested_at && self.cleanup_deadline_at == other.cleanup_deadline_at && self.lineage == other.lineage + && self.scope_origin == other.scope_origin } } @@ -92,11 +94,24 @@ fn nullable_text(value: &Value, key: &str, allow_empty: bool) -> Result Result { - if value["schema"] != "durable-workflow.cancellation-context/v1" { - return Err(invalid_context("unsupported cancellation context schema")); - } + let scope_origin = match value["schema"].as_str() { + Some("durable-workflow.cancellation-context/v2") => Some(Box::new( + ScopedCancellationContext::from_value(&value["scope_origin"])?, + )), + Some("durable-workflow.cancellation-context/v1") => { + if value.get("scope_origin").is_some() + || value.get("scope_authority_deadline_at").is_some() + { + return Err(invalid_context( + "legacy cancellation context cannot discard a scoped origin", + )); + } + None + } + _ => return Err(invalid_context("unsupported cancellation context schema")), + }; let requester = value["requester"] .as_object() .filter(|requester| !requester.is_empty()) @@ -154,7 +169,7 @@ impl CancellationContext { || normalized[0].workflow_instance_id != root_workflow_instance_id || normalized[0].workflow_run_id != root_workflow_run_id || normalized.last().unwrap().request_id != request_id - || parent_request_id.as_ref() != expected_parent + || (scope_origin.is_none() && parent_request_id.as_ref() != expected_parent) { return Err(invalid_context( "cancellation lineage does not match its request identities", @@ -172,7 +187,7 @@ impl CancellationContext { "cancellation deadline must follow the original request", )); } - Ok(Self { + let context = Self { request_id, root_request_id, root_workflow_instance_id, @@ -184,8 +199,64 @@ impl CancellationContext { requested_at, cleanup_deadline_at, lineage: normalized, + scope_origin, replay: None, - }) + }; + context.assert_scope_origin(value)?; + Ok(context) + } + + fn assert_scope_origin(&self, value: &Value) -> Result<()> { + let Some(origin) = self.scope_origin.as_deref() else { + return Ok(()); + }; + let root = &origin.root_context; + let last = self.lineage.last().unwrap(); + let authority = + DateTime::parse_from_rfc3339(context_text(value, "scope_authority_deadline_at")?) + .map_err(|_| invalid_context("scoped cancellation authority timestamp is invalid"))? + .with_timezone(&Utc); + if self.parent_request_id.as_deref() != Some(origin.request_id()) + || self.root_request_id != root.root_request_id + || self.root_workflow_instance_id != root.root_workflow_instance_id + || self.root_workflow_run_id != root.root_workflow_run_id + || self.reason != root.reason + || self.requester != root.requester + || self.source != root.source + || self.requested_at != root.requested_at + || self.cleanup_deadline_at != authority + || self.cleanup_deadline_at > origin.deadline() + || origin.lineage.iter().any(|entry| { + entry.request_id == last.request_id || entry.workflow_run_id == last.workflow_run_id + }) + { + return Err(invalid_context( + "run cancellation does not preserve its original scope context", + )); + } + let mut expected = vec![root.lineage[0].clone()]; + for entry in &origin.lineage { + if entry.workflow_run_id == root.root_workflow_run_id { + continue; + } + let address = CancellationLineage { + request_id: entry.request_id.clone(), + workflow_instance_id: entry.workflow_instance_id.clone(), + workflow_run_id: entry.workflow_run_id.clone(), + }; + if expected.last().unwrap().workflow_run_id == entry.workflow_run_id { + *expected.last_mut().unwrap() = address; + } else { + expected.push(address); + } + } + expected.push(last.clone()); + if self.lineage != expected { + return Err(invalid_context( + "run cancellation lineage discards or replaces its original scope ancestry", + )); + } + Ok(()) } pub fn request_id(&self) -> &str { @@ -218,6 +289,9 @@ impl CancellationContext { pub fn deadline(&self) -> DateTime { self.cleanup_deadline_at } + pub fn scope_origin(&self) -> Option<&ScopedCancellationContext> { + self.scope_origin.as_deref() + } /// Remaining cleanup budget at the last blocking result consumed by this replay. /// @@ -251,8 +325,9 @@ impl CancellationContext { /// Detached metadata in the portable context schema. pub fn to_value(&self) -> Value { - json!({ - "schema": "durable-workflow.cancellation-context/v1", + let mut value = json!({ + "schema": if self.scope_origin.is_none() { "durable-workflow.cancellation-context/v1" } + else { "durable-workflow.cancellation-context/v2" }, "request_id": self.request_id, "root_request_id": self.root_request_id, "root_workflow_instance_id": self.root_workflow_instance_id, "root_workflow_run_id": self.root_workflow_run_id, @@ -261,6 +336,221 @@ impl CancellationContext { "requested_at": self.requested_at.to_rfc3339_opts(SecondsFormat::Micros, true), "cleanup_deadline_at": self.cleanup_deadline_at.to_rfc3339_opts(SecondsFormat::Micros, true), "lineage": self.lineage.iter().map(CancellationLineage::to_value).collect::>(), + }); + if let Some(origin) = &self.scope_origin { + value["scope_origin"] = origin.to_value(); + value["scope_authority_deadline_at"] = value["cleanup_deadline_at"].clone(); + } + value + } +} + +/// One immutable scope address and its bounded cleanup deadline. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScopedCancellationLineage { + request_id: String, + workflow_instance_id: String, + workflow_run_id: String, + scope_id: String, + cleanup_deadline_at: DateTime, +} + +impl ScopedCancellationLineage { + pub fn request_id(&self) -> &str { + &self.request_id + } + pub fn workflow_instance_id(&self) -> &str { + &self.workflow_instance_id + } + pub fn workflow_run_id(&self) -> &str { + &self.workflow_run_id + } + pub fn scope_id(&self) -> &str { + &self.scope_id + } + pub fn deadline(&self) -> DateTime { + self.cleanup_deadline_at + } + + fn to_value(&self) -> Value { + json!({ + "request_id": self.request_id, "workflow_instance_id": self.workflow_instance_id, + "workflow_run_id": self.workflow_run_id, "scope_id": self.scope_id, + "cleanup_deadline_at": self.cleanup_deadline_at.to_rfc3339_opts(SecondsFormat::Micros, true), + }) + } +} + +/// Original scope ancestry carried by a candidate cooperative child request. +/// Reading this immutable metadata does not authorize entering a scope body. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScopedCancellationContext { + root_context: CancellationContext, + lineage: Vec, +} + +fn assert_context_keys(value: &Value, keys: &[&str]) -> Result<()> { + let object = value + .as_object() + .ok_or_else(|| invalid_context("scoped context must be an object"))?; + if object.len() != keys.len() || keys.iter().any(|key| !object.contains_key(*key)) { + return Err(invalid_context( + "scoped cancellation context has missing or unsupported fields", + )); + } + Ok(()) +} + +impl ScopedCancellationContext { + pub fn from_value(value: &Value) -> Result { + assert_context_keys(value, &["schema", "root_context", "lineage"])?; + if value["schema"] != "durable-workflow.scoped-cancellation-context/v1" + || value["root_context"]["schema"] != "durable-workflow.cancellation-context/v1" + { + return Err(invalid_context( + "scoped cancellation requires its original root context", + )); + } + let root = CancellationContext::from_value(&value["root_context"])?; + let root_value = root.to_value(); + let root_keys: Vec<_> = root_value + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + assert_context_keys(&value["root_context"], &root_keys)?; + if root.request_id != root.root_request_id + || root.parent_request_id.is_some() + || root.lineage.len() != 1 + { + return Err(invalid_context( + "scoped cancellation root must contain the original root request", + )); + } + let lineage = value["lineage"] + .as_array() + .filter(|rows| !rows.is_empty()) + .ok_or_else(|| { + invalid_context("scoped cancellation lineage must contain its root address") + })?; + let mut normalized = Vec::new(); + let mut requests = BTreeSet::new(); + let mut addresses = BTreeSet::new(); + let mut instances_by_run = BTreeMap::new(); + let mut last_run = String::new(); + let mut deadline = root.deadline(); + for (index, entry) in lineage.iter().enumerate() { + assert_context_keys( + entry, + &[ + "request_id", + "workflow_instance_id", + "workflow_run_id", + "scope_id", + "cleanup_deadline_at", + ], + )?; + let address = ScopedCancellationLineage { + request_id: context_text(entry, "request_id")?.to_owned(), + workflow_instance_id: context_text(entry, "workflow_instance_id")?.to_owned(), + workflow_run_id: context_text(entry, "workflow_run_id")?.to_owned(), + scope_id: context_text(entry, "scope_id")?.to_owned(), + cleanup_deadline_at: DateTime::parse_from_rfc3339(context_text( + entry, + "cleanup_deadline_at", + )?) + .map_err(|_| invalid_context("scoped cancellation deadline timestamp is invalid"))? + .with_timezone(&Utc), + }; + if index == 0 + && (address.request_id != root.request_id + || address.workflow_instance_id != root.root_workflow_instance_id + || address.workflow_run_id != root.root_workflow_run_id + || address.cleanup_deadline_at != root.deadline()) + { + return Err(invalid_context( + "scoped cancellation root address does not match its request", + )); + } + if !requests.insert(address.request_id.clone()) + || !addresses.insert((address.workflow_run_id.clone(), address.scope_id.clone())) + { + return Err(invalid_context( + "scoped cancellation cannot repeat a request or address", + )); + } + if let Some(instance) = instances_by_run.get(&address.workflow_run_id) { + if instance != &address.workflow_instance_id || last_run != address.workflow_run_id + { + return Err(invalid_context( + "scoped cancellation cannot reenter or reassign an earlier run", + )); + } + } + if address.cleanup_deadline_at <= root.requested_at + || address.cleanup_deadline_at > deadline + { + return Err(invalid_context( + "scoped cancellation cannot extend a descendant budget", + )); + } + instances_by_run.insert( + address.workflow_run_id.clone(), + address.workflow_instance_id.clone(), + ); + last_run = address.workflow_run_id.clone(); + deadline = address.cleanup_deadline_at; + normalized.push(address); + } + Ok(Self { + root_context: root, + lineage: normalized, + }) + } + + pub fn root_context(&self) -> &CancellationContext { + &self.root_context + } + pub fn lineage(&self) -> &[ScopedCancellationLineage] { + &self.lineage + } + pub fn request_id(&self) -> &str { + &self.lineage.last().unwrap().request_id + } + pub fn parent_request_id(&self) -> Option<&str> { + self.lineage + .iter() + .rev() + .nth(1) + .map(|entry| entry.request_id.as_str()) + } + pub fn workflow_instance_id(&self) -> &str { + &self.lineage.last().unwrap().workflow_instance_id + } + pub fn workflow_run_id(&self) -> &str { + &self.lineage.last().unwrap().workflow_run_id + } + pub fn scope_id(&self) -> &str { + &self.lineage.last().unwrap().scope_id + } + pub fn root_scope_id(&self) -> &str { + &self.lineage[0].scope_id + } + pub fn requested_at(&self) -> DateTime { + self.root_context.requested_at() + } + pub fn root_deadline(&self) -> DateTime { + self.root_context.deadline() + } + pub fn deadline(&self) -> DateTime { + self.lineage.last().unwrap().cleanup_deadline_at + } + pub fn to_value(&self) -> Value { + json!({ + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": self.root_context.to_value(), + "lineage": self.lineage.iter().map(ScopedCancellationLineage::to_value).collect::>(), }) } } diff --git a/src/lib.rs b/src/lib.rs index f23b0c9..b54149c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,7 +6,9 @@ mod cooperative_cancellation; mod runtime_payloads; mod runtime_uploads; -pub use cancellation_context::{CancellationContext, CancellationLineage}; +pub use cancellation_context::{ + CancellationContext, CancellationLineage, ScopedCancellationContext, ScopedCancellationLineage, +}; pub use cooperative_cancellation::{ CancellationCallKind, CancellationDelivery, CancellationDeliveryReceipt, diff --git a/src/tests/cooperative_cancellation.rs b/src/tests/cooperative_cancellation.rs index fd7d061..bb08e99 100644 --- a/src/tests/cooperative_cancellation.rs +++ b/src/tests/cooperative_cancellation.rs @@ -7,6 +7,278 @@ fn context_snapshot() -> Value { .unwrap() } +fn scoped_run_snapshot(name: &str) -> Value { + let fixtures: Value = serde_json::from_str(include_str!( + "fixtures/scoped-run-cancellation-context.json" + )) + .unwrap(); + fixtures[name].clone() +} + +fn scoped_run_history() -> Vec { + vec![serde_json::from_value(json!({ + "event_type":"CooperativeCancellationRequested", "timestamp":"2026-10-04T00:00:05Z", + "payload": {"workflow_run_id":"child-run", "workflow_instance_id":"child-instance", + "workflow_command_id":"child-request", "reason":"maintenance", + "cleanup_deadline_at":"2026-10-04T00:00:15.123456Z", "cancellation":scoped_run_snapshot("child")} + })).unwrap(), remaining_event("CooperativeCancellationDelivered", json!({ + "workflow_run_id":"child-run", "workflow_command_id":"child-request", "sequence":1, + "call_kind":"timer", "cancellation":scoped_run_snapshot("child") + }), "2026-10-04T00:00:08Z")] +} + +fn scoped_run_task(history: Vec) -> WorkflowTask { + let mut task = cancellation_task(history); + task.run_id = Some("child-run".into()); + task +} + +#[test] +fn scoped_run_native_child_and_grandchild_preserve_every_origin_and_original_budget() { + for name in ["child", "grandchild"] { + let snapshot = scoped_run_snapshot(name); + let context = CancellationContext::from_value(&snapshot).unwrap(); + assert_eq!(context.to_value(), snapshot); + assert_eq!( + CancellationContext::from_value(&context.to_value()).unwrap(), + context + ); + assert_eq!(context.root_request_id(), "root-request"); + assert_eq!(context.reason(), Some("maintenance")); + assert_eq!(context.source(), "api"); + assert_eq!(context.requester()["id"], "operator-1"); + assert_eq!( + context + .requested_at() + .to_rfc3339_opts(chrono::SecondsFormat::Micros, true), + "2026-10-04T00:00:00.123456Z" + ); + let origin = context.scope_origin().unwrap(); + assert_eq!( + origin + .root_deadline() + .to_rfc3339_opts(chrono::SecondsFormat::Micros, true), + "2026-10-04T00:00:30.123456Z" + ); + assert_eq!(origin.requested_at(), context.requested_at()); + let (scopes, parent, deadline) = if name == "child" { + ( + vec!["outer", "inner"], + "inner-request", + "2026-10-04T00:00:15.123456Z", + ) + } else { + ( + vec!["outer", "inner", "root", "child-scope"], + "child-scope-request", + "2026-10-04T00:00:12.123456Z", + ) + }; + assert_eq!(context.parent_request_id(), Some(parent)); + assert_eq!( + origin + .lineage() + .iter() + .map(|hop| hop.scope_id()) + .collect::>(), + scopes + ); + assert_eq!( + context + .deadline() + .to_rfc3339_opts(chrono::SecondsFormat::Micros, true), + deadline + ); + assert!(matches!( + context.remaining(), + Err(Error::InvalidCooperativeCancellation(_)) + )); + } +} + +#[test] +fn scoped_run_avro_timezone_and_detached_metadata_keep_the_original_tree() { + let original = scoped_run_snapshot("grandchild"); + let mut snapshot = original.clone(); + snapshot["requested_at"] = json!("2026-10-03T20:00:00.123456-04:00"); + snapshot["cleanup_deadline_at"] = json!("2026-10-03T20:00:12.123456-04:00"); + snapshot["scope_authority_deadline_at"] = snapshot["cleanup_deadline_at"].clone(); + let decoded = decode_wire_value(&fixture_envelope(snapshot), DEFAULT_CODEC).unwrap(); + let context = CancellationContext::from_value(&decoded).unwrap(); + assert_eq!(context.to_value(), original); + let origin = context.scope_origin().unwrap(); + let mut detached = origin.to_value(); + detached["lineage"][3]["scope_id"] = json!("changed"); + assert_eq!(origin.scope_id(), "child-scope"); + assert_eq!(origin.root_scope_id(), "outer"); + assert_eq!(origin.request_id(), "child-scope-request"); + assert_eq!(origin.parent_request_id(), Some("child-request")); + assert_eq!(origin.workflow_instance_id(), "child-instance"); + assert_eq!(origin.workflow_run_id(), "child-run"); + assert_eq!(origin.to_value(), original["scope_origin"]); +} + +#[test] +fn scoped_run_cold_cleanup_replay_consumes_the_same_narrowed_original_clock() { + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + assert!(ctx.cancellation_context()?.is_none()); + let Err(Error::CooperativeCancellationRequested(cancelled)) = ctx.sleep(Duration::from_secs(10)).await else { + panic!("expected original cancellation"); + }; + let context = cancelled.request.context.unwrap(); + assert_eq!(context.to_value(), scoped_run_snapshot("child")); + assert_eq!(ctx.cancellation_context()?, Some(context.clone())); + let before = context.remaining()?.as_secs_f64(); assert_eq!(before, 7.123456); + let _shield = ctx.cancellation_shield()?; + ctx.activity("cleanup", json!([])).await?; + Ok(json!({"remaining":[before,context.remaining()?.as_secs_f64()], "cancellation":context.to_value()})) + }); + let mut history = scoped_run_history(); + let initial = worker + .execute_workflow_task(scoped_run_task(history.clone())) + .unwrap(); + assert_eq!(initial.len(), 1); + assert_eq!(initial[0]["activity_type"], "cleanup"); + let mut completion = completed_activity(2, "cleanup", json!("cleaned")); + completion + .last_mut() + .unwrap() + .raw + .insert("timestamp".into(), json!("2026-10-04T00:00:12Z")); + history.extend(completion); + history.push(remaining_event( + "WorkflowTaskScheduled", + json!({}), + "2026-10-04T00:00:29Z", + )); + for _replacement in 0..2 { + let result = worker + .execute_workflow_task(scoped_run_task(history.clone())) + .unwrap(); + assert_eq!(result[0]["type"], "complete_workflow"); + assert_eq!( + decode_wire_value(&result[0]["result"], DEFAULT_CODEC).unwrap(), + json!({ + "remaining":[7.123456,3.123456],"cancellation":scoped_run_snapshot("child") + }) + ); + } +} + +#[test] +fn scoped_run_canonical_delivery_cannot_replace_an_original_scope() { + let mut history = scoped_run_history(); + history[1].payload["cancellation"]["scope_origin"]["lineage"][1]["scope_id"] = + json!("different"); + let mut worker = cancellation_worker(); + worker.register_workflow("cancel", |ctx, _| async move { + ctx.sleep(Duration::from_secs(10)).await?; + Ok(Value::Null) + }); + assert!(worker + .execute_workflow_task(scoped_run_task(history)) + .is_err()); +} + +#[test] +fn scoped_run_invalid_origin_identity_and_budget_are_refused() { + let original = scoped_run_snapshot("grandchild"); + for (path, value) in [ + ("/schema", json!("durable-workflow.cancellation-context/v1")), + ("/root_request_id", json!("other")), + ("/root_workflow_instance_id", json!("other")), + ("/root_workflow_run_id", json!("other")), + ("/parent_request_id", json!("root-request")), + ("/reason", json!("other")), + ("/source", json!("other")), + ("/requested_at", json!("2026-10-04T00:00:01.123456Z")), + ("/scope_origin", json!([])), + ("/cleanup_deadline_at", json!("2026-10-04T00:00:15.123456Z")), + ( + "/scope_authority_deadline_at", + json!("2026-10-04T00:00:15.123456Z"), + ), + ("/requester/id", json!("other")), + ("/lineage/1/request_id", json!("child-request")), + ("/lineage/2/workflow_run_id", json!("child-run")), + ( + "/scope_origin/lineage/3/cleanup_deadline_at", + json!("2026-10-04T00:00:16.123456Z"), + ), + ( + "/scope_origin/lineage/3/workflow_instance_id", + json!("other"), + ), + ("/scope_origin/lineage/3/request_id", json!("inner-request")), + ("/scope_origin/lineage/3/scope_id", json!("root")), + ( + "/scope_origin/root_context/schema", + json!("durable-workflow.cancellation-context/v2"), + ), + ] { + let mut snapshot = original.clone(); + *snapshot.pointer_mut(path).unwrap() = value; + assert!( + matches!( + CancellationContext::from_value(&snapshot), + Err(Error::InvalidCooperativeCancellation(_)) + ), + "{path}" + ); + } + for key in [ + "scope_origin", + "scope_authority_deadline_at", + "parent_request_id", + ] { + let mut snapshot = original.clone(); + snapshot.as_object_mut().unwrap().remove(key); + assert!( + CancellationContext::from_value(&snapshot).is_err(), + "missing {key}" + ); + } + for fault in [ + "widened global budget", + "reused request", + "discarded root", + "run reentry", + "unsupported field", + ] { + let mut snapshot = original.clone(); + match fault { + "widened global budget" => { + snapshot["cleanup_deadline_at"] = json!("2026-10-04T00:00:30.123456Z"); + snapshot["scope_authority_deadline_at"] = snapshot["cleanup_deadline_at"].clone(); + } + "reused request" => { + snapshot["request_id"] = json!("inner-request"); + snapshot["lineage"][2]["request_id"] = json!("inner-request"); + } + "discarded root" => { + snapshot["scope_origin"]["lineage"] + .as_array_mut() + .unwrap() + .remove(0); + } + "run reentry" => { + snapshot["scope_origin"]["lineage"][3]["workflow_run_id"] = json!("root-run"); + snapshot["scope_origin"]["lineage"][3]["workflow_instance_id"] = + json!("root-instance"); + } + _ => snapshot["scope_origin"]["lineage"][3]["authority"] = json!("unrecorded"), + } + assert!( + matches!( + CancellationContext::from_value(&snapshot), + Err(Error::InvalidCooperativeCancellation(_)) + ), + "{fault}" + ); + } +} + fn context_request() -> HistoryEvent { serde_json::from_value(json!({ "event_type": "CooperativeCancellationRequested", "recorded_at": "2026-10-01T00:00:05Z", diff --git a/src/tests/fixtures/scoped-run-cancellation-context.json b/src/tests/fixtures/scoped-run-cancellation-context.json new file mode 100644 index 0000000..7d9d0a4 --- /dev/null +++ b/src/tests/fixtures/scoped-run-cancellation-context.json @@ -0,0 +1,164 @@ +{ + "native_commit": "a51395da59ef70945f91f00581ecb6bad8765bdd", + "child": { + "schema": "durable-workflow.cancellation-context/v2", + "request_id": "child-request", + "root_request_id": "root-request", + "root_workflow_instance_id": "root-instance", + "root_workflow_run_id": "root-run", + "parent_request_id": "inner-request", + "reason": "maintenance", + "requester": { + "type": "operator", + "id": "operator-1" + }, + "source": "api", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:15.123456Z", + "lineage": [ + { + "request_id": "root-request", + "workflow_instance_id": "root-instance", + "workflow_run_id": "root-run" + }, + { + "request_id": "child-request", + "workflow_instance_id": "child-instance", + "workflow_run_id": "child-run" + } + ], + "scope_origin": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "root-request", + "root_request_id": "root-request", + "root_workflow_instance_id": "root-instance", + "root_workflow_run_id": "root-run", + "parent_request_id": null, + "reason": "maintenance", + "requester": { + "type": "operator", + "id": "operator-1" + }, + "source": "api", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "root-request", + "workflow_instance_id": "root-instance", + "workflow_run_id": "root-run" + } + ] + }, + "lineage": [ + { + "request_id": "root-request", + "workflow_instance_id": "root-instance", + "workflow_run_id": "root-run", + "scope_id": "outer", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "inner-request", + "workflow_instance_id": "root-instance", + "workflow_run_id": "root-run", + "scope_id": "inner", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z" + } + ] + }, + "scope_authority_deadline_at": "2026-10-04T00:00:15.123456Z" + }, + "grandchild": { + "schema": "durable-workflow.cancellation-context/v2", + "request_id": "grandchild-request", + "root_request_id": "root-request", + "root_workflow_instance_id": "root-instance", + "root_workflow_run_id": "root-run", + "parent_request_id": "child-scope-request", + "reason": "maintenance", + "requester": { + "type": "operator", + "id": "operator-1" + }, + "source": "api", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:12.123456Z", + "lineage": [ + { + "request_id": "root-request", + "workflow_instance_id": "root-instance", + "workflow_run_id": "root-run" + }, + { + "request_id": "child-scope-request", + "workflow_instance_id": "child-instance", + "workflow_run_id": "child-run" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "grandchild-instance", + "workflow_run_id": "grandchild-run" + } + ], + "scope_origin": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "root-request", + "root_request_id": "root-request", + "root_workflow_instance_id": "root-instance", + "root_workflow_run_id": "root-run", + "parent_request_id": null, + "reason": "maintenance", + "requester": { + "type": "operator", + "id": "operator-1" + }, + "source": "api", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "root-request", + "workflow_instance_id": "root-instance", + "workflow_run_id": "root-run" + } + ] + }, + "lineage": [ + { + "request_id": "root-request", + "workflow_instance_id": "root-instance", + "workflow_run_id": "root-run", + "scope_id": "outer", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "inner-request", + "workflow_instance_id": "root-instance", + "workflow_run_id": "root-run", + "scope_id": "inner", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "child-instance", + "workflow_run_id": "child-run", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-04T00:00:15.123456Z" + }, + { + "request_id": "child-scope-request", + "workflow_instance_id": "child-instance", + "workflow_run_id": "child-run", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:12.123456Z" + } + ] + }, + "scope_authority_deadline_at": "2026-10-04T00:00:12.123456Z" + } +} From c69e7caaa700dcc89d5941761209da40accad67c Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Mon, 5 Oct 2026 22:32:36 +0000 Subject: [PATCH 42/58] Prove canonical cancellation scope openings on original Rust claims --- src/cancellation_scope.rs | 234 ++++++++++++++++++ src/lib.rs | 3 + src/tests/cancellation_scope_opening.rs | 303 ++++++++++++++++++++++++ 3 files changed, 540 insertions(+) create mode 100644 src/cancellation_scope.rs create mode 100644 src/tests/cancellation_scope_opening.rs diff --git a/src/cancellation_scope.rs b/src/cancellation_scope.rs new file mode 100644 index 0000000..4b91761 --- /dev/null +++ b/src/cancellation_scope.rs @@ -0,0 +1,234 @@ +use super::*; + +const OPEN_BUDGET: Duration = Duration::from_secs(5); +const MAX_SCOPE_SEQUENCE: u64 = i64::MAX as u64; +const MAX_HISTORY_PAGES: usize = 128; + +fn invalid() -> Error { + Error::InvalidCooperativeCancellation("invalid canonical cancellation scope opening".into()) +} + +fn identity(value: &str) -> bool { + !value.trim().is_empty() && value.len() <= 255 +} + +fn text<'a>(value: &'a Value, field: &str) -> Result<&'a str> { + value[field] + .as_str() + .filter(|value| identity(value)) + .ok_or_else(invalid) +} + +/// A scope opening proved against complete history on its original claim. +/// This proof does not advertise scoped workflow execution support. +#[derive(Clone, Debug)] +pub struct CancellationScopeOpenReceipt { + scope_id: String, + history_event_id: String, + sequence: u64, + parent_scope_id: String, + shield_parent: bool, + duplicate: bool, + history: Vec, +} + +impl CancellationScopeOpenReceipt { + pub fn scope_id(&self) -> &str { + &self.scope_id + } + pub fn history_event_id(&self) -> &str { + &self.history_event_id + } + pub fn sequence(&self) -> u64 { + self.sequence + } + pub fn parent_scope_id(&self) -> &str { + &self.parent_scope_id + } + pub fn shield_parent(&self) -> bool { + self.shield_parent + } + pub fn duplicate(&self) -> bool { + self.duplicate + } + pub fn history(&self) -> &[HistoryEvent] { + &self.history + } + + fn acknowledge<'a>(receipt: &'a Value, expected: &Value) -> Result<&'a str> { + if expected + .as_object() + .ok_or_else(invalid)? + .iter() + .any(|(key, value)| key != "namespace" && receipt.get(key) != Some(value)) + || receipt["opened"].as_bool() != Some(true) + || receipt["duplicate"].as_bool().is_none() + || receipt["claim_released"].as_bool() != Some(false) + || receipt.get("created_task_ids") != Some(&json!([])) + || receipt.get("reason") != Some(&Value::Null) + || text(receipt, "scope_id")? == "root" + { + return Err(invalid()); + } + text(receipt, "history_event_id")?; + receipt["history_refresh_page_token"] + .as_str() + .filter(|value| !value.trim().is_empty()) + .ok_or_else(invalid) + } + + fn from_history(receipt: &Value, events: &[Value], expected: &Value) -> Result { + Self::acknowledge(receipt, expected)?; + let kind = |index: usize| { + events + .get(index) + .and_then(|event| event.get("event_type").or_else(|| event.get("type"))) + .and_then(Value::as_str) + }; + if kind(0) != Some("WorkflowStarted") + && !(kind(0) == Some("StartAccepted") && kind(1) == Some("WorkflowStarted")) + { + return Err(invalid()); + } + let mut event_ids = BTreeSet::new(); + let mut scopes = BTreeSet::new(); + let mut last_event_sequence = 0; + let mut last_scope_sequence = 0; + let mut found = false; + let mut history = Vec::with_capacity(events.len()); + for event in events { + let event_id = text(event, "id")?; + let event_sequence = event["sequence"].as_u64().ok_or_else(invalid)?; + let payload = &event["payload"]; + let event_kind = event + .get("event_type") + .or_else(|| event.get("type")) + .and_then(Value::as_str) + .filter(|value| !value.trim().is_empty()) + .ok_or_else(invalid)?; + if !event_ids.insert(event_id) + || event_sequence <= last_event_sequence + || event_sequence > MAX_SCOPE_SEQUENCE + || event.get("namespace") != expected.get("namespace") + || !payload.is_object() + { + return Err(invalid()); + } + last_event_sequence = event_sequence; + if event_kind == "CancellationScopeOpened" { + let scope_id = text(payload, "scope_id")?; + let parent = text(payload, "parent_scope_id")?; + let sequence = payload["sequence"].as_u64().ok_or_else(invalid)?; + if payload["schema"] != "durable-workflow.cancellation-scope/v1" + || payload.get("workflow_run_id") != expected.get("workflow_run_id") + || scope_id == "root" + || scopes.contains(scope_id) + || (parent != "root" && !scopes.contains(parent)) + || payload["shield_parent"].as_bool().is_none() + || sequence <= last_scope_sequence + || sequence > MAX_SCOPE_SEQUENCE + { + return Err(invalid()); + } + scopes.insert(scope_id); + last_scope_sequence = sequence; + if Some(event_id) == receipt["history_event_id"].as_str() { + if ["scope_id", "sequence", "parent_scope_id", "shield_parent"] + .iter() + .any(|field| payload.get(*field) != receipt.get(*field)) + { + return Err(invalid()); + } + found = true; + } + } + history.push(serde_json::from_value(event.clone()).map_err(|_| invalid())?); + } + if !found { + return Err(invalid()); + } + Ok(Self { + scope_id: text(receipt, "scope_id")?.into(), + history_event_id: text(receipt, "history_event_id")?.into(), + sequence: receipt["sequence"].as_u64().ok_or_else(invalid)?, + parent_scope_id: text(receipt, "parent_scope_id")?.into(), + shield_parent: receipt["shield_parent"].as_bool().ok_or_else(invalid)?, + duplicate: receipt["duplicate"].as_bool().ok_or_else(invalid)?, + history, + }) + } +} + +impl Client { + /// Open a scope and prove its canonical identity on the original claim. + /// + /// This explicit protocol 1.20 operation keeps one five-second budget for + /// lost acknowledgement recovery and all history pages. No lease, worker + /// capability or cancellation execution authority is added by this proof. + pub async fn open_cancellation_scope_on_claim( + &self, + task: &WorkflowTask, + sequence: u64, + parent_scope_id: &str, + shield_parent: bool, + ) -> Result { + let (owner, run_id) = cooperative_cancellation::cancellation_claim(task)?; + if [ + task.task_id.as_str(), + owner, + run_id, + parent_scope_id, + self.namespace.as_str(), + ] + .iter() + .any(|value| !identity(value)) + || sequence == 0 + || sequence > MAX_SCOPE_SEQUENCE + { + return Err(invalid()); + } + let expected = json!({"task_id":task.task_id, "workflow_run_id":run_id, + "lease_owner":owner, "workflow_task_attempt":task.workflow_task_attempt, + "sequence":sequence, "parent_scope_id":parent_scope_id, + "shield_parent":shield_parent, "namespace":self.namespace}); + let path = format!( + "/worker/workflow-tasks/{}", + percent_encode_path_segment(&task.task_id) + ); + let body = json!({"lease_owner":owner, "workflow_task_attempt":task.workflow_task_attempt, + "sequence":sequence, "parent_scope_id":parent_scope_id,"shield_parent":shield_parent}); + tokio::time::timeout(OPEN_BUDGET, async { + let open_path = format!("{path}/cancellation-scopes/open"); + let request = || self.request_json::(reqwest::Method::POST, + &open_path, RequestProtocol::Worker("1.20"), Some(&body)); + let receipt = match request().await { + Err(error) if worker_operation_is_retryable(&error) => request().await?, + result => result?, + }; + let mut token = Some(CancellationScopeOpenReceipt::acknowledge(&receipt, &expected)?.to_owned()); + let mut seen = BTreeSet::new(); + let mut events = Vec::new(); + while let Some(current) = token.take() { + if seen.len() >= MAX_HISTORY_PAGES || !seen.insert(current.clone()) { return Err(invalid()); } + let page: Value = self.request_json(reqwest::Method::POST, &format!("{path}/history"), + RequestProtocol::Worker("1.20"), Some(&json!({"lease_owner":owner, + "workflow_task_attempt":task.workflow_task_attempt, "next_history_page_token":current, + "history_page_size":WORKFLOW_HISTORY_PAGE_SIZE}))).await?; + if page["task_id"].as_str() != Some(task.task_id.as_str()) + || page["workflow_task_attempt"].as_u64() != Some(task.workflow_task_attempt) + { return Err(invalid()); } + let batch = page["history_events"].as_array().ok_or_else(invalid)?; + if batch.len() > WORKFLOW_HISTORY_PAGE_SIZE as usize || batch.iter().any(|event| !event.is_object()) { + return Err(invalid()); + } + token = match page.get("next_history_page_token") { + Some(Value::Null) => None, + Some(Value::String(next)) if !next.trim().is_empty() && !batch.is_empty() => Some(next.clone()), + _ => return Err(invalid()), + }; + events.extend(batch.iter().cloned()); + } + CancellationScopeOpenReceipt::from_history(&receipt, &events, &expected) + }).await.map_err(|_| Error::Timeout)? + } +} diff --git a/src/lib.rs b/src/lib.rs index b54149c..a252d52 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,7 @@ mod cancellation_context; mod cancellation_replay_clock; +mod cancellation_scope; mod cooperative_cancellation; mod runtime_payloads; mod runtime_uploads; @@ -9,6 +10,7 @@ mod runtime_uploads; pub use cancellation_context::{ CancellationContext, CancellationLineage, ScopedCancellationContext, ScopedCancellationLineage, }; +pub use cancellation_scope::CancellationScopeOpenReceipt; pub use cooperative_cancellation::{ CancellationCallKind, CancellationDelivery, CancellationDeliveryReceipt, @@ -16224,6 +16226,7 @@ mod tests { use super::*; mod activity_cancellation_policies; mod cancellation_scope_admission; + mod cancellation_scope_opening; mod child_workflow_policies; mod cooperative_cancellation; mod runtime_payloads; diff --git a/src/tests/cancellation_scope_opening.rs b/src/tests/cancellation_scope_opening.rs new file mode 100644 index 0000000..f219896 --- /dev/null +++ b/src/tests/cancellation_scope_opening.rs @@ -0,0 +1,303 @@ +use super::*; + +fn receipt() -> Value { + json!({"task_id":"task/one", "workflow_run_id":"run-one", "lease_owner":"original", + "workflow_task_attempt":4, "sequence":2, "scope_id":"scope-two", "parent_scope_id":"scope-one", + "shield_parent":true, "opened":true, "duplicate":false, "claim_released":false, + "created_task_ids":[], "reason":null, "history_event_id":"event-scope-two", + "history_refresh_page_token":"opaque-start"}) +} + +fn history() -> Vec { + vec![ + json!({"id":"start", "sequence":1, "namespace":"tenant", "event_type":"WorkflowStarted", "payload":{}}), + json!({"id":"event-scope-one", "sequence":2, "namespace":"tenant", "event_type":"CancellationScopeOpened", + "payload":{"schema":"durable-workflow.cancellation-scope/v1", "workflow_run_id":"run-one", + "sequence":1, "scope_id":"scope-one", "parent_scope_id":"root", "shield_parent":false}}), + json!({"id":"event-scope-two", "sequence":3, "namespace":"tenant", "event_type":"CancellationScopeOpened", + "payload":{"schema":"durable-workflow.cancellation-scope/v1", "workflow_run_id":"run-one", + "sequence":2, "scope_id":"scope-two", "parent_scope_id":"scope-one", "shield_parent":true}}), + ] +} + +fn response(path: &str, body: &str, number: usize) -> Option<(&'static str, String)> { + let case = path.split('/').nth(1).unwrap_or_default(); + if case == "budget" { + thread::sleep(Duration::from_secs(3)); + } + let mut value; + if path.ends_with("/cancellation-scopes/open") { + if case == "lost-ack" && number == 1 { + return Some(("invalid-status", String::new())); + } + if case == "refused" { + return Some(( + "409 Conflict", + r#"{"reason":"lease_owner_mismatch"}"#.into(), + )); + } + value = receipt(); + if case == "lost-ack" || case == "duplicate" { + value["duplicate"] = json!(true); + } + match case { + "ack-owner" => value["lease_owner"] = json!("replacement"), + "ack-attempt" => value["workflow_task_attempt"] = json!(5), + "ack-run" => value["workflow_run_id"] = json!("foreign"), + "ack-sequence" => value["sequence"] = json!(3), + "ack-shield" => value["shield_parent"] = json!(1), + "ack-opened" => value["opened"] = json!(1), + "ack-duplicate" => value["duplicate"] = json!(0), + "ack-released" => value["claim_released"] = json!(true), + "ack-new-task" => value["created_task_ids"] = json!(["new"]), + "ack-root" => value["scope_id"] = json!("root"), + "ack-token" => value["history_refresh_page_token"] = Value::Null, + "ack-event" => value["history_event_id"] = json!(""), + "ack-reason" => { + value.as_object_mut().unwrap().remove("reason"); + } + _ => {} + } + } else if path.ends_with("/history") { + let mut events = history(); + if case == "accepted-prefix" { + events.insert( + 0, + json!({"id":"accepted", "sequence":0, "namespace":"tenant", + "event_type":"StartAccepted", "payload":{}}), + ); + for event in &mut events { + event["sequence"] = json!(event["sequence"].as_u64().unwrap() + 1); + } + } + match case { + "tree-namespace" => events[2]["namespace"] = json!("foreign"), + "tree-id" => events[2]["id"] = json!("start"), + "tree-event-sequence" => events[2]["sequence"] = json!(2), + "tree-run" => events[2]["payload"]["workflow_run_id"] = json!("foreign"), + "tree-scope" => events[2]["payload"]["scope_id"] = json!("scope-one"), + "tree-parent" => events[2]["payload"]["parent_scope_id"] = json!("unknown"), + "tree-self" => events[2]["payload"]["parent_scope_id"] = json!("scope-two"), + "tree-changed-parent" => events[2]["payload"]["parent_scope_id"] = json!("root"), + "tree-shield" => events[2]["payload"]["shield_parent"] = json!(false), + "tree-bool" => events[2]["payload"]["shield_parent"] = json!(1), + "tree-sequence" => events[2]["payload"]["sequence"] = json!(1), + "tree-schema" => events[2]["payload"]["schema"] = json!("other"), + "missing-opening" => { + events.pop(); + } + "missing-start" => { + events.remove(0); + } + "empty-history" => events.clear(), + _ => {} + } + let requested: Value = serde_json::from_str(body).unwrap(); + let (batch, token) = if requested["next_history_page_token"] == "opaque-start" { + ( + events.iter().take(1).cloned().collect::>(), + json!("opaque-next"), + ) + } else { + ( + events.iter().skip(1).cloned().collect::>(), + Value::Null, + ) + }; + value = json!({"task_id":"task/one", "workflow_task_attempt":4, + "history_events":batch,"next_history_page_token":token}); + match case { + "page-task" => value["task_id"] = json!("foreign"), + "page-attempt" => value["workflow_task_attempt"] = json!(5), + "page-cycle" => value["next_history_page_token"] = json!("opaque-start"), + "page-empty" => value["history_events"] = json!([]), + "page-events" => value["history_events"] = json!([null]), + "page-token" => { + value + .as_object_mut() + .unwrap() + .remove("next_history_page_token"); + } + _ => {} + } + } else { + return None; + } + Some(("200 OK", value.to_string())) +} + +fn server() -> MockWorkerServer { + MockWorkerServer::start_with_behavior(MockWorkerBehavior { + request_override: Some(response), + ..MockWorkerBehavior::default() + }) +} + +fn client(server: &MockWorkerServer, case: &str) -> Client { + Client::builder(format!("{}/{case}", server.base_url())) + .namespace("tenant") + .control_token(Some("control-only".into())) + .worker_token(Some("worker-only".into())) + .build() + .unwrap() +} + +fn task() -> WorkflowTask { + serde_json::from_value( + json!({"task_id":"task/one", "run_id":"run-one", "workflow_type":"scope", + "lease_owner":"original", "workflow_task_attempt":4, "payload_codec":DEFAULT_CODEC}), + ) + .unwrap() +} + +#[tokio::test] +async fn scope_opening_proves_complete_paged_tree_and_lost_ack_under_original_worker_credentials() { + for case in ["valid", "duplicate", "lost-ack", "accepted-prefix"] { + let server = server(); + let proof = client(&server, case) + .open_cancellation_scope_on_claim(&task(), 2, "scope-one", true) + .await + .unwrap(); + assert_eq!(proof.scope_id(), "scope-two"); + assert_eq!(proof.history_event_id(), "event-scope-two"); + assert_eq!(proof.sequence(), 2); + assert_eq!(proof.parent_scope_id(), "scope-one"); + assert!(proof.shield_parent()); + assert_eq!(proof.duplicate(), matches!(case, "duplicate" | "lost-ack")); + assert_eq!( + proof.history().len(), + if case == "accepted-prefix" { 4 } else { 3 } + ); + let requests = server.requests.lock().unwrap(); + let opens = requests + .iter() + .filter(|row| row.path.ends_with("/open")) + .collect::>(); + assert_eq!(opens.len(), if case == "lost-ack" { 2 } else { 1 }); + if case == "lost-ack" { + assert_eq!(opens[0].body, opens[1].body); + } + for request in requests.iter() { + assert_eq!(request.worker_protocol.as_deref(), Some("1.20")); + assert_eq!(request.authorization.as_deref(), Some("Bearer worker-only")); + assert_eq!(request.namespace.as_deref(), Some("tenant")); + assert!(request.path.contains("/task%2Fone/")); + let body: Value = serde_json::from_str(&request.body).unwrap(); + assert_eq!(body["lease_owner"], "original"); + assert_eq!(body["workflow_task_attempt"], 4); + } + } +} + +#[tokio::test] +async fn scope_opening_refuses_changed_receipts_before_history_reads() { + for case in [ + "ack-owner", + "ack-attempt", + "ack-run", + "ack-sequence", + "ack-shield", + "ack-opened", + "ack-duplicate", + "ack-released", + "ack-new-task", + "ack-root", + "ack-token", + "ack-event", + "ack-reason", + ] { + let server = server(); + assert!( + matches!( + client(&server, case) + .open_cancellation_scope_on_claim(&task(), 2, "scope-one", true) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + ), + "{case}" + ); + assert_eq!(server.requests.lock().unwrap().len(), 1, "{case}"); + } +} + +#[tokio::test] +async fn scope_opening_refuses_altered_canonical_tree_and_invalid_claim_pages() { + for case in [ + "tree-namespace", + "tree-id", + "tree-event-sequence", + "tree-run", + "tree-scope", + "tree-parent", + "tree-self", + "tree-changed-parent", + "tree-shield", + "tree-bool", + "tree-sequence", + "tree-schema", + "missing-opening", + "missing-start", + "empty-history", + "page-task", + "page-attempt", + "page-cycle", + "page-empty", + "page-events", + "page-token", + ] { + let server = server(); + assert!( + matches!( + client(&server, case) + .open_cancellation_scope_on_claim(&task(), 2, "scope-one", true) + .await, + Err(Error::InvalidCooperativeCancellation(_)) + ), + "{case}" + ); + } +} + +#[tokio::test] +async fn scope_opening_refuses_invalid_original_authority_before_io() { + let server = server(); + let client = client(&server, "valid"); + for field in ["task_id", "run_id", "lease_owner", "workflow_task_attempt"] { + let mut invalid = task(); + match field { + "task_id" => invalid.task_id.clear(), + "run_id" => invalid.run_id = None, + "lease_owner" => invalid.lease_owner = Some("x".repeat(256)), + _ => invalid.workflow_task_attempt = 0, + } + assert!(client + .open_cancellation_scope_on_claim(&invalid, 2, "scope-one", true) + .await + .is_err()); + } + for (sequence, parent) in [(0, "scope-one"), (u64::MAX, "scope-one"), (2, "")] { + assert!(client + .open_cancellation_scope_on_claim(&task(), sequence, parent, true) + .await + .is_err()); + } + assert!(server.requests.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn scope_opening_preserves_claim_refusal_and_bounds_opening_with_all_history_pages() { + let server = server(); + assert!( + matches!(client(&server,"refused").open_cancellation_scope_on_claim(&task(),2,"scope-one",true).await, + Err(Error::Http { status, .. }) if status.as_u16() == 409) + ); + assert_eq!(server.requests.lock().unwrap().len(), 1); + let started = Instant::now(); + assert!(matches!( + client(&server, "budget") + .open_cancellation_scope_on_claim(&task(), 2, "scope-one", true) + .await, + Err(Error::Timeout) + )); + assert!(started.elapsed() < Duration::from_secs(6)); +} From 0267c19360ace4215e2ea7be8f796b4482ae19e9 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Mon, 5 Oct 2026 22:36:48 +0000 Subject: [PATCH 43/58] Qualify Rust scope opening against real Native history --- .github/workflows/ci.yml | 7 +++- tests/cooperative_server.rs | 68 +++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b9d5a3..89aa0b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -257,7 +257,12 @@ jobs: echo "DURABLE_WORKFLOW_SERVER_URL=http://$endpoint" >> "$GITHUB_ENV" - name: Run actual Worker cooperative scenarios shell: bash - run: cargo test --test cooperative_server -- --ignored --nocapture --test-threads=1 2>&1 | tee cooperative-results.log + run: | + arguments=(--ignored --nocapture --test-threads=1) + if [ -z "$NATIVE_CANDIDATE_SHA" ]; then + arguments+=(--skip server_scope_opening_proves_real_native_tree_on_original_claim) + fi + cargo test --test cooperative_server -- "${arguments[@]}" 2>&1 | tee cooperative-results.log - name: Retain bounded scenario evidence if: ${{ always() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index a211e93..09b93cc 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -678,6 +678,74 @@ async fn assert_cancelled(handle: &WorkflowHandle, request_id: &str, cleanup: bo snapshot } +#[tokio::test] +#[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] +async fn server_scope_opening_proves_real_native_tree_on_original_claim() { + let client = client(); + let queue = queue(); + let owner = format!("{queue}-scope-owner"); + let worker = worker(&client, &queue, false, false).worker_id(&owner); + worker.register().await.unwrap(); + let handle = client + .start_workflow(WORKFLOW, &queue, &queue, json!([])) + .await + .unwrap(); + let claim = tokio::time::timeout(Duration::from_secs(30), async { + loop { + let reply = client + .poll_cooperative_workflow_task(&owner, &queue, Duration::from_secs(1)) + .await + .unwrap(); + if let Some(claim) = reply.task { + return claim; + } + } + }) + .await + .unwrap(); + let parent = client + .open_cancellation_scope_on_claim(claim.task(), 1, "root", false) + .await + .unwrap(); + let child = client + .open_cancellation_scope_on_claim(claim.task(), 2, parent.scope_id(), true) + .await + .unwrap(); + let duplicate = client + .open_cancellation_scope_on_claim(claim.task(), 2, parent.scope_id(), true) + .await + .unwrap(); + assert_eq!(parent.parent_scope_id(), "root"); + assert!(!parent.shield_parent() && !parent.duplicate()); + assert_eq!(child.parent_scope_id(), parent.scope_id()); + assert!(child.shield_parent() && !child.duplicate() && duplicate.duplicate()); + assert_eq!(duplicate.scope_id(), child.scope_id()); + assert_eq!(duplicate.history_event_id(), child.history_event_id()); + let openings = child + .history() + .iter() + .filter(|event| event.event_type == "CancellationScopeOpened") + .map(|event| event.payload["scope_id"].as_str().unwrap()) + .collect::>(); + assert_eq!(openings, vec![parent.scope_id(), child.scope_id()]); + let mut stale = claim.task().clone(); + stale.workflow_task_attempt += 1; + assert!( + matches!(client.open_cancellation_scope_on_claim(&stale,3,"root",false).await, + Err(Error::Http { status, .. }) if status.as_u16()==409) + ); + let snapshot = history(&handle).await; + assert_eq!(count(&snapshot, "CancellationScopeOpened"), 2); + for kind in ["TimerScheduled", "WorkflowFailed", "WorkflowCompleted"] { + assert_eq!(count(&snapshot, kind), 0); + } + eprintln!("Native scope opening proof: {:?}", child.history()); + handle + .terminate_selected_run(WorkflowCommandOptions::default()) + .await + .unwrap(); +} + #[tokio::test] #[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] async fn server_request_before_claim_replays_canonical_typed_cancellation() { From 2050e94f15f8b47e4c5989c9057e1dac62763b34 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Mon, 5 Oct 2026 23:18:05 +0000 Subject: [PATCH 44/58] feat: replay canonical scopes with stable operation contexts --- .github/workflows/ci.yml | 1 + src/cancellation_scope.rs | 352 ++++++++++++++++++++ src/cooperative_cancellation.rs | 40 ++- src/lib.rs | 159 ++++++++- src/tests/cancellation_scope_admission.rs | 1 + src/tests/cancellation_scope_authoring.rs | 377 ++++++++++++++++++++++ tests/cooperative_server.rs | 95 ++++++ 7 files changed, 1007 insertions(+), 18 deletions(-) create mode 100644 src/tests/cancellation_scope_authoring.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89aa0b4..45f98e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -261,6 +261,7 @@ jobs: arguments=(--ignored --nocapture --test-threads=1) if [ -z "$NATIVE_CANDIDATE_SHA" ]; then arguments+=(--skip server_scope_opening_proves_real_native_tree_on_original_claim) + arguments+=(--skip server_scope_authoring_replays_nested_tree_and_deferred_membership_on_replacement) fi cargo test --test cooperative_server -- "${arguments[@]}" 2>&1 | tee cooperative-results.log - name: Retain bounded scenario evidence diff --git a/src/cancellation_scope.rs b/src/cancellation_scope.rs index 4b91761..1e1f977 100644 --- a/src/cancellation_scope.rs +++ b/src/cancellation_scope.rs @@ -19,6 +19,356 @@ fn text<'a>(value: &'a Value, field: &str) -> Result<&'a str> { .ok_or_else(invalid) } +#[derive(Clone, Debug)] +pub(super) struct CanonicalScopeOpening { + pub scope_id: String, + pub parent_scope_id: String, + pub shield_parent: bool, +} + +#[derive(Default, Debug)] +pub(super) struct CancellationScopeHistory { + pub openings: BTreeMap, + pub memberships: BTreeMap, +} + +fn invalid_history(detail: &str) -> Error { + invalid_recorded_history( + "invalid_cancellation_scope_history", + 0, + "canonical scope tree and original membership", + "invalid history", + detail, + ) +} + +fn starts_with_workflow_start(events: &[HistoryEvent]) -> bool { + events + .first() + .is_some_and(|event| event.event_type == "WorkflowStarted") + || (events + .first() + .is_some_and(|event| event.event_type == "StartAccepted") + && events + .get(1) + .is_some_and(|event| event.event_type == "WorkflowStarted")) +} + +impl CancellationScopeHistory { + pub fn read(events: &[HistoryEvent], run_id: &str) -> Result { + let has_scopes = events + .iter() + .any(|event| event.event_type == "CancellationScopeOpened"); + if has_scopes && !starts_with_workflow_start(events) { + return Err(invalid_history( + "scope history lacks its original workflow start", + )); + } + let mut history = Self::default(); + let mut event_ids = BTreeSet::new(); + let mut scopes = BTreeMap::new(); + let mut namespace: Option<&str> = None; + let mut last_event_sequence = 0; + let mut last_opening = 0; + for event in events { + if has_scopes { + let event_id = event + .raw + .get("id") + .and_then(Value::as_str) + .filter(|value| identity(value)) + .ok_or_else(|| invalid_history("missing canonical event identity"))?; + let sequence = event + .raw + .get("sequence") + .and_then(Value::as_u64) + .filter(|value| *value > last_event_sequence && *value <= MAX_SCOPE_SEQUENCE) + .ok_or_else(|| invalid_history("canonical event order changed"))?; + let incoming_namespace = event + .raw + .get("namespace") + .and_then(Value::as_str) + .filter(|value| identity(value)) + .ok_or_else(|| invalid_history("missing canonical namespace"))?; + if !event_ids.insert(event_id) + || namespace.is_some_and(|previous| previous != incoming_namespace) + || !event.payload.is_object() + || event.event_type.trim().is_empty() + { + return Err(invalid_history( + "canonical event identity, namespace or payload changed", + )); + } + namespace = Some(incoming_namespace); + last_event_sequence = sequence; + } + let payload = &event.payload; + let sequence = payload["sequence"] + .as_u64() + .filter(|value| *value > 0 && *value <= MAX_SCOPE_SEQUENCE); + if event.event_type == "CancellationScopeOpened" { + let scope_id = text(payload, "scope_id") + .map_err(|_| invalid_history("invalid scope identity"))?; + let parent = text(payload, "parent_scope_id") + .map_err(|_| invalid_history("invalid parent identity"))?; + let sequence = + sequence.ok_or_else(|| invalid_history("invalid authored opening sequence"))?; + let shield = payload["shield_parent"] + .as_bool() + .ok_or_else(|| invalid_history("invalid shielding"))?; + if payload["schema"] != "durable-workflow.cancellation-scope/v1" + || run_id.is_empty() + || payload["workflow_run_id"].as_str() != Some(run_id) + || scope_id == "root" + || scopes.contains_key(scope_id) + || (parent != "root" && !scopes.contains_key(parent)) + || sequence <= last_opening + || history.memberships.contains_key(&sequence) + { + return Err(invalid_history("invalid canonical opening tree")); + } + scopes.insert(scope_id, sequence); + last_opening = sequence; + history.openings.insert( + sequence, + CanonicalScopeOpening { + scope_id: scope_id.into(), + parent_scope_id: parent.into(), + shield_parent: shield, + }, + ); + continue; + } + let admission = matches!( + event.event_type.as_str(), + "ActivityScheduled" + | "TimerScheduled" + | "ChildWorkflowScheduled" + | "ConditionWaitOpened" + | "SignalWaitOpened" + ); + let operation = admission + || matches!( + event.event_type.as_str(), + "ActivityStarted" + | "ActivityCompleted" + | "ActivityFailed" + | "ActivityTimedOut" + | "ActivityCancelled" + | "ActivityRetryScheduled" + | "TimerFired" + | "TimerCancelled" + | "ChildRunStarted" + | "ChildRunCompleted" + | "ChildRunFailed" + | "ChildRunCancelled" + | "ChildRunTerminated" + | "ConditionWaitSatisfied" + | "ConditionWaitTimedOut" + | "ConditionWaitCancelled" + | "SignalWaitReceived" + | "SignalWaitTimedOut" + | "SignalWaitCancelled" + ); + if !operation { + continue; + } + let mut membership: Option<&str> = None; + for snapshot in std::iter::once(payload).chain( + ["activity", "timer", "child_workflow"] + .iter() + .filter_map(|name| payload.get(name)), + ) { + let Some(value) = snapshot.get("cancellation_scope_id") else { + continue; + }; + let incoming = value + .as_str() + .filter(|value| identity(value)) + .ok_or_else(|| invalid_history("invalid operation scope membership"))?; + if membership.is_some_and(|previous| previous != incoming) { + return Err(invalid_history("contradictory operation scope membership")); + } + membership = Some(incoming); + } + if membership.is_none() && !admission { + continue; + } + let membership = membership.unwrap_or("root"); + if membership != "root" + && !sequence.is_some_and(|sequence| { + scopes + .get(membership) + .is_some_and(|opening| *opening < sequence) + }) + { + return Err(invalid_history( + "operation scope was not opened before original admission", + )); + } + let Some(sequence) = sequence else { + continue; + }; + if history.openings.contains_key(&sequence) + || history + .memberships + .get(&sequence) + .is_some_and(|previous| previous != membership) + { + return Err(invalid_history( + "operation changed its original scope membership", + )); + } + history.memberships.insert(sequence, membership.into()); + } + Ok(history) + } +} + +#[derive(Clone, Debug)] +pub(super) struct CancellationScopeOpening { + pub sequence: u64, + pub parent_scope_id: String, + pub shield_parent: bool, + pub command_count: usize, +} + +impl WorkflowContext { + pub(super) fn validate_scope_membership( + &self, + state: &WorkflowState, + cursor: usize, + ) -> Result<()> { + if !state.allow_cancellation_scope_authoring { + return Ok(()); + } + if let Some(recorded) = state.recorded_commands.get(cursor) { + let sequence = recorded.sequence(); + let original = state + .cancellation_scope_memberships + .get(&sequence) + .map(String::as_str) + .unwrap_or("root"); + if original != self.cancellation_scope_id { + return Err(invalid_recorded_history( + "cancellation_scope_membership_changed", + sequence, + original, + &self.cancellation_scope_id, + "operation changed the scope where it was created", + )); + } + } + Ok(()) + } + + pub(super) fn apply_scope_membership(&self, command: &mut serde_json::Map) { + if self.cancellation_scope_id != "root" { + command.insert( + "cancellation_scope_id".into(), + json!(self.cancellation_scope_id), + ); + } + } + + /// Await the original durable opening, then run a body with its own context. + /// Operations created from that context retain their scope when awaited later. + /// Candidate authoring remains disabled on ordinary workers. + pub async fn cancellation_scope(&self, shield_parent: bool, body: F) -> Result + where + F: FnOnce(WorkflowContext) -> Fut, + Fut: Future>, + { + let scope_id = ScopeOpeningCall { + ctx: self.clone(), + shield_parent, + } + .await?; + let mut scoped = self.clone(); + scoped.cancellation_scope_id = scope_id; + body(scoped).await + } +} + +struct ScopeOpeningCall { + ctx: WorkflowContext, + shield_parent: bool, +} + +impl Future for ScopeOpeningCall { + type Output = Result; + + fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll { + let mut state = match self.ctx.state.lock() { + Ok(state) => state, + Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)), + }; + if !state.allow_cancellation_scope_authoring { + return Poll::Ready(Err(Error::CancellationScopeExecutionUnavailable)); + } + if state + .cancellation_scope_opening + .as_ref() + .is_some_and(|opening| opening.command_count != state.commands.len()) + { + return Poll::Ready(Err(invalid_history( + "workflow authored commands after an uncommitted opening", + ))); + } + let sequence = (state.command_cursor as u64) + .saturating_add(state.commands.len() as u64) + .saturating_add(1); + if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() { + return match recorded { + RecordedCommand::CancellationScope { + sequence: original, + scope_id, + parent_scope_id, + shield_parent, + } if original == sequence + && parent_scope_id == self.ctx.cancellation_scope_id + && shield_parent == self.shield_parent => + { + state.command_cursor += 1; + Poll::Ready(Ok(scope_id)) + } + recorded => Poll::Ready(Err(invalid_recorded_history( + "cancellation_scope_opening_changed", + sequence, + "original scope opening, parent and shielding", + recorded.shape(), + "authored cancellation scope differs from committed history", + ))), + }; + } + if state.history_events.iter().any(|event| { + matches!( + event.event_type.as_str(), + "WorkflowCompleted" + | "WorkflowFailed" + | "WorkflowCancelled" + | "WorkflowTerminated" + | "WorkflowContinuedAsNew" + ) + }) { + return Poll::Ready(Err(invalid_recorded_history( + "cancellation_scope_opening_changed", + sequence, + "original scope opening", + "closed history", + "closed history cannot admit an unrecorded scope", + ))); + } + state.cancellation_scope_opening = Some(CancellationScopeOpening { + sequence, + parent_scope_id: self.ctx.cancellation_scope_id.clone(), + shield_parent: self.shield_parent, + command_count: state.commands.len(), + }); + Poll::Pending + } +} + /// A scope opening proved against complete history on its original claim. /// This proof does not advertise scoped workflow execution support. #[derive(Clone, Debug)] @@ -147,6 +497,8 @@ impl CancellationScopeOpenReceipt { if !found { return Err(invalid()); } + CancellationScopeHistory::read(&history, text(expected, "workflow_run_id")?) + .map_err(|_| invalid())?; Ok(Self { scope_id: text(receipt, "scope_id")?.into(), history_event_id: text(receipt, "history_event_id")?.into(), diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index 6d6eda1..89ed9d0 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -4,12 +4,15 @@ use std::collections::BTreeSet; const CONTROL_BUDGET: Duration = Duration::from_secs(5); const MAX_REFRESH_PAGES: usize = 128; -pub(super) fn assert_cancellation_scope_replay_supported(events: &[HistoryEvent]) -> Result<()> { +pub(super) fn assert_cancellation_scope_replay_supported_with_authoring( + events: &[HistoryEvent], + allow_authoring: bool, +) -> Result<()> { for event in events { let scope_marker = matches!( event.event_type.as_str(), - "CancellationScopeOpened" - | "CancellationScopeRequested" + "CancellationScopeRequested" + | "CancellationScopeDeliveryPrepared" | "CancellationScopeDelivered" | "CancellationScopeRequestConflicted" ); @@ -24,7 +27,10 @@ pub(super) fn assert_cancellation_scope_replay_supported(events: &[HistoryEvent] .get("cancellation_scope_id") .is_some_and(|scope| scope.as_str() != Some("root")) }); - if scope_marker || scoped_membership { + if scope_marker + || (!allow_authoring + && (event.event_type == "CancellationScopeOpened" || scoped_membership)) + { return Err(Error::CancellationScopeExecutionUnavailable); } } @@ -1898,7 +1904,31 @@ impl Worker { "cooperative replay omitted its original pending observation", )); } - return self.execute_workflow_task_decision(task).map(Some); + for _ in 0..1000 { + let mut decision = self.execute_workflow_task_decision(task.clone())?; + let Some(opening) = decision.cancellation_scope_opening.as_ref() else { + return Ok(Some(decision)); + }; + if !decision.commands.is_empty() { + decision.cancellation_scope_opening = None; + return Ok(Some(decision)); + } + let receipt = self + .client + .open_cancellation_scope_on_claim( + &task, + opening.sequence, + &opening.parent_scope_id, + opening.shield_parent, + ) + .await?; + task.history_events = receipt.history().to_vec(); + task.total_history_events = None; + task.history_size_bytes = None; + } + return Err(invalid( + "workflow exceeded the canonical scope opening replay limit", + )); }; task.history_events = self .client diff --git a/src/lib.rs b/src/lib.rs index a252d52..6c8e429 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6183,6 +6183,7 @@ struct WorkflowTaskDecision { message_stream_cursors: Vec, message_stream_waits: Vec, cancellation_delivery: Option, + cancellation_scope_opening: Option, } impl WorkflowTaskDecision { @@ -6192,6 +6193,7 @@ impl WorkflowTaskDecision { message_stream_cursors: Vec::new(), message_stream_waits: Vec::new(), cancellation_delivery: None, + cancellation_scope_opening: None, } } } @@ -6287,6 +6289,7 @@ pub struct Worker { retry_policy: WorkerRetryPolicy, heartbeat_observer: Option, cooperative_cancellation_enabled: bool, + allow_cancellation_scope_authoring: bool, cooperative_registration_confirmed: Arc, } @@ -6307,6 +6310,7 @@ impl Worker { retry_policy: WorkerRetryPolicy::default(), heartbeat_observer: None, cooperative_cancellation_enabled: false, + allow_cancellation_scope_authoring: false, cooperative_registration_confirmed: Arc::new(AtomicBool::new(false)), } } @@ -6343,6 +6347,13 @@ impl Worker { self } + /// Enable candidate scope authoring without advertising scoped execution. + #[doc(hidden)] + pub fn candidate_cancellation_scope_authoring(mut self, enabled: bool) -> Self { + self.allow_cancellation_scope_authoring = enabled; + self + } + /// Configure bounded retries for task-poll acquisition and worker heartbeats. pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self { self.retry_policy = policy; @@ -7274,7 +7285,10 @@ impl Worker { memo_updates_supported: bool, ) -> Result { match decision { - Ok(decision) if decision.cancellation_delivery.is_some() => { + Ok(decision) + if decision.cancellation_delivery.is_some() + || decision.cancellation_scope_opening.is_some() => + { return Err(Error::CooperativeCancellationUnavailable( "worker cancellation delivery has not been negotiated".into(), )); @@ -7749,6 +7763,7 @@ impl Worker { )); let workflow_context = WorkflowContext { state: workflow_state, + cancellation_scope_id: "root".into(), }; let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone()); let mut cx = TaskContext::from_waker(noop_waker_ref()); @@ -7845,8 +7860,14 @@ impl Worker { task: WorkflowTask, observation: Option<&CancellationRequest>, ) -> Result { + if self.allow_cancellation_scope_authoring && !self.cooperative_cancellation_enabled { + return Err(Error::CancellationScopeExecutionUnavailable); + } validate_workflow_task_payloads(&task)?; - cooperative_cancellation::assert_cancellation_scope_replay_supported(&task.history_events)?; + cooperative_cancellation::assert_cancellation_scope_replay_supported_with_authoring( + &task.history_events, + self.allow_cancellation_scope_authoring, + )?; let cancellation_history = if let Some(observation) = observation { observation.validate_observation()?; cooperative_cancellation::cancellation_claim(&task)?; @@ -7899,13 +7920,14 @@ impl Worker { .filter(|identity| !identity.is_empty()) .or_else(|| (!task.task_id.is_empty()).then(|| task.task_id.clone())) .unwrap_or_default(); - let mut workflow_state = WorkflowState::new_with_identity( + let mut workflow_state = WorkflowState::new_with_identity_and_scopes( task.history_events, task.workflow_id, task.run_id, self.task_queue.clone(), task.payload_codec.clone(), resume_signal, + self.allow_cancellation_scope_authoring, )?; workflow_state.history_budget = history_budget; workflow_state.workflow_command_identity = workflow_command_identity; @@ -7917,7 +7939,10 @@ impl Worker { workflow_state.cancel_requested = task.cancel_requested; } let state = Arc::new(Mutex::new(workflow_state)); - let ctx = WorkflowContext { state }; + let ctx = WorkflowContext { + state, + cancellation_scope_id: "root".into(), + }; let mut future = (workflow.execute)(ctx.clone(), input); let mut cx = TaskContext::from_waker(noop_waker_ref()); @@ -7925,13 +7950,12 @@ impl Worker { let _replay = cancellation_replay_clock::ReplayGuard::enter(&ctx)?; future.as_mut().poll(&mut cx) }; - if ctx - .state - .lock() - .map_err(|_| Error::WorkflowStatePoisoned)? - .cancellation_delivery_intent - .is_some() - { + let awaiting_canonical_boundary = { + let state = ctx.state.lock().map_err(|_| Error::WorkflowStatePoisoned)?; + state.cancellation_delivery_intent.is_some() + || state.cancellation_scope_opening.is_some() + }; + if awaiting_canonical_boundary { // A private delivery intent never becomes an application error. // Even a custom future which ignores a pending durable call cannot // publish terminal commands before canonical delivery and replay. @@ -8024,6 +8048,17 @@ impl Worker { } let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?; let state = ctx.state.lock().map_err(|_| Error::WorkflowStatePoisoned)?; + if let Some(opening) = &state.cancellation_scope_opening { + if commands.len() != opening.command_count { + return Err(invalid_recorded_history( + "cancellation_scope_pending_call_escaped", + opening.sequence, + "commands before original scope opening", + "commands after suspended opening", + "workflow cannot publish beyond an uncommitted scope opening", + )); + } + } if state.cancellation_delivery_intent.is_some() && commands.len() != state.cancellation_delivery_command_count { @@ -8044,6 +8079,7 @@ impl Worker { message_stream_cursors, message_stream_waits, cancellation_delivery: state.cancellation_delivery_intent.clone(), + cancellation_scope_opening: state.cancellation_scope_opening.clone(), }) } @@ -8344,6 +8380,7 @@ impl MessageStream { #[derive(Clone, Debug)] pub struct WorkflowContext { state: Arc>, + cancellation_scope_id: String, } fn valid_memo_key(key: &str) -> bool { @@ -9699,6 +9736,9 @@ struct WorkflowState { cancellation_delivery_enabled: bool, cancellation_delivery_intent: Option, cancellation_delivery_command_count: usize, + allow_cancellation_scope_authoring: bool, + cancellation_scope_opening: Option, + cancellation_scope_memberships: BTreeMap, resume_signal: Option, recorded_commands: Vec, selection_markers: Vec, @@ -9744,13 +9784,47 @@ impl WorkflowState { payload_codec: String, resume_signal: Option, ) -> Result { - cooperative_cancellation::assert_cancellation_scope_replay_supported(&history)?; + Self::new_with_identity_and_scopes( + history, + workflow_id, + run_id, + task_queue, + payload_codec, + resume_signal, + false, + ) + } + + fn new_with_identity_and_scopes( + history: Vec, + workflow_id: Option, + run_id: Option, + task_queue: String, + payload_codec: String, + resume_signal: Option, + allow_cancellation_scope_authoring: bool, + ) -> Result { + cooperative_cancellation::assert_cancellation_scope_replay_supported_with_authoring( + &history, + allow_cancellation_scope_authoring, + )?; + let scopes = if allow_cancellation_scope_authoring { + cancellation_scope::CancellationScopeHistory::read( + &history, + run_id.as_deref().unwrap_or_default(), + )? + } else { + cancellation_scope::CancellationScopeHistory::default() + }; let cancellation_history = CancellationHistory::from_events( &history, run_id.as_deref().unwrap_or_default(), None, )?; - let recorded_commands = cancellation_history.bind_commands(recorded_commands( + if allow_cancellation_scope_authoring && cancellation_history.request.is_some() { + return Err(Error::CancellationScopeExecutionUnavailable); + } + let mut recorded_commands = cancellation_history.bind_commands(recorded_commands( &history, &payload_codec, WorkflowIdentity { @@ -9758,6 +9832,27 @@ impl WorkflowState { run_id: run_id.clone(), }, )?)?; + for (sequence, opening) in scopes.openings { + if recorded_commands + .iter() + .any(|command| command.sequence() == sequence) + { + return Err(invalid_recorded_history( + "invalid_cancellation_scope_history", + sequence, + "distinct authored scope opening", + "operation collision", + "scope opening collides with a command", + )); + } + recorded_commands.push(RecordedCommand::CancellationScope { + sequence, + scope_id: opening.scope_id, + parent_scope_id: opening.parent_scope_id, + shield_parent: opening.shield_parent, + }); + } + recorded_commands.sort_by_key(RecordedCommand::sequence); let selection_markers = recorded_selection_markers(&history)?; let cancelled_selection_members = recorded_selection_cancellations(&history)?; let recorded_continue_as_new = history @@ -9865,6 +9960,9 @@ impl WorkflowState { cancellation_delivery_enabled: false, cancellation_delivery_intent: None, cancellation_delivery_command_count: 0, + allow_cancellation_scope_authoring, + cancellation_scope_opening: None, + cancellation_scope_memberships: scopes.memberships, resume_signal, recorded_commands, selection_markers, @@ -9950,6 +10048,12 @@ fn decode_message_stream_delivery(arguments: Vec) -> Result u64 { match self { Self::CancellationGroup { sequence, .. } + | Self::CancellationScope { sequence, .. } | Self::CancellationBoundary { sequence, .. } | Self::Activity { sequence, .. } | Self::Timer { sequence, .. } @@ -10646,6 +10751,7 @@ impl RecordedCommand { fn shape(&self) -> &'static str { match self { + Self::CancellationScope { .. } => "cancellation scope", Self::CancellationBoundary { .. } | Self::CancellationGroup { .. } => { "cooperative cancellation" } @@ -12319,6 +12425,7 @@ fn recorded_selection_member_is_terminal( RecordedCommand::SignalWait { value, .. } => value.is_some(), RecordedCommand::ConditionWait { result, .. } => result.is_some(), RecordedCommand::SearchAttributes { .. } + | RecordedCommand::CancellationScope { .. } | RecordedCommand::SideEffect { .. } | RecordedCommand::VersionMarker { .. } | RecordedCommand::Memo { .. } @@ -12720,6 +12827,9 @@ impl ActivityCall { }; let cursor = state.command_cursor; + if let Err(error) = ctx.validate_scope_membership(&state, cursor) { + return Poll::Ready(Err(error)); + } let recorded = match state.cancellation_replay_command(cursor, CancellationCallKind::Activity) { Ok(recorded) => recorded, @@ -12893,6 +13003,7 @@ impl ActivityCall { command.insert("cancellation_policy".to_string(), json!(policy.as_str())); } apply_parallel_group_path(&mut command, &self.parallel_group_path); + ctx.apply_scope_membership(&mut command); match state.prepare_scalar_cancellation( cursor, CancellationCallKind::Activity, @@ -12952,6 +13063,9 @@ impl Future for TimerCall { }; let cursor = state.command_cursor; + if let Err(error) = ctx.validate_scope_membership(&state, cursor) { + return Poll::Ready(Err(error)); + } let recorded = match state.cancellation_replay_command(cursor, CancellationCallKind::Timer) { Ok(recorded) => recorded, @@ -13022,6 +13136,7 @@ impl Future for TimerCall { ("delay_seconds".to_string(), json!(requested_delay)), ]); apply_parallel_group_path(&mut command, &self.parallel_group_path); + ctx.apply_scope_membership(&mut command); match state.prepare_scalar_cancellation( cursor, CancellationCallKind::Timer, @@ -13095,6 +13210,9 @@ impl Future for ConditionWaitCall { Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)), }; let initial_cursor = state.command_cursor; + if let Err(error) = ctx.validate_scope_membership(&state, initial_cursor) { + return Poll::Ready(Err(error)); + } let recorded = match state .cancellation_replay_command(initial_cursor, CancellationCallKind::Condition) { @@ -13275,6 +13393,7 @@ impl ConditionWaitCall { command.insert("timeout_seconds".to_string(), json!(timeout_seconds)); } apply_parallel_group_path(&mut command, &self.parallel_group_path); + ctx.apply_scope_membership(&mut command); state.commands.push(Value::Object(command)); drop(state); self.opened_wait = true; @@ -13357,6 +13476,9 @@ impl ChildWorkflowCall { }; let cursor = state.command_cursor; + if let Err(error) = ctx.validate_scope_membership(&state, cursor) { + return Poll::Ready(Err(error)); + } let recorded = match state.cancellation_replay_command(cursor, CancellationCallKind::Child) { Ok(recorded) => recorded, @@ -13515,6 +13637,7 @@ impl ChildWorkflowCall { object.insert("run_timeout_seconds".to_string(), json!(seconds)); } apply_parallel_group_path(object, &self.parallel_group_path); + ctx.apply_scope_membership(object); match state.prepare_scalar_cancellation( cursor, CancellationCallKind::Child, @@ -13596,6 +13719,9 @@ impl SignalCall { }; let cursor = state.command_cursor; + if let Err(error) = ctx.validate_scope_membership(&state, cursor) { + return Poll::Ready(Err(error)); + } let recorded = match state.cancellation_replay_command(cursor, CancellationCallKind::Signal) { Ok(recorded) => recorded, @@ -13711,6 +13837,7 @@ impl SignalCall { ("signal_name".to_string(), json!(self.signal_name)), ]); apply_parallel_group_path(&mut command, &self.parallel_group_path); + ctx.apply_scope_membership(&mut command); state.commands.push(Value::Object(command)); self.opened_wait = true; } @@ -16226,6 +16353,7 @@ mod tests { use super::*; mod activity_cancellation_policies; mod cancellation_scope_admission; + mod cancellation_scope_authoring; mod cancellation_scope_opening; mod child_workflow_policies; mod cooperative_cancellation; @@ -16588,6 +16716,7 @@ mod tests { payload_codec: &str, ) -> WorkflowContext { WorkflowContext { + cancellation_scope_id: "root".into(), state: Arc::new(Mutex::new( WorkflowState::new_with_identity( history, @@ -18227,6 +18356,7 @@ mod tests { state.workflow_command_identity = "command-7".to_string(); let context = WorkflowContext { state: Arc::new(Mutex::new(state)), + cancellation_scope_id: "root".into(), }; let item = WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt"); @@ -18265,6 +18395,7 @@ mod tests { replay_state.workflow_command_identity = "command-7".to_string(); let replay_context = WorkflowContext { state: Arc::new(Mutex::new(replay_state)), + cancellation_scope_id: "root".into(), }; replay_context .append_workflow_stream( @@ -19594,6 +19725,7 @@ mod tests { #[test] fn workflow_context_schedules_activity_until_completion_is_in_history() { let ctx = WorkflowContext { + cancellation_scope_id: "root".into(), state: Arc::new(Mutex::new( WorkflowState::new_with_identity( Vec::new(), @@ -23014,6 +23146,7 @@ mod tests { #[test] fn workflow_context_emits_explicit_child_workflow_contract() { let ctx = WorkflowContext { + cancellation_scope_id: "root".into(), state: Arc::new(Mutex::new( WorkflowState::new_with_identity( Vec::new(), diff --git a/src/tests/cancellation_scope_admission.rs b/src/tests/cancellation_scope_admission.rs index eccb259..526b352 100644 --- a/src/tests/cancellation_scope_admission.rs +++ b/src/tests/cancellation_scope_admission.rs @@ -8,6 +8,7 @@ fn scope_histories() -> Vec { let mut events = [ "CancellationScopeOpened", "CancellationScopeRequested", + "CancellationScopeDeliveryPrepared", "CancellationScopeDelivered", "CancellationScopeRequestConflicted", ] diff --git a/src/tests/cancellation_scope_authoring.rs b/src/tests/cancellation_scope_authoring.rs new file mode 100644 index 0000000..1a29886 --- /dev/null +++ b/src/tests/cancellation_scope_authoring.rs @@ -0,0 +1,377 @@ +use super::*; + +fn event(kind: &str, payload: Value, index: u64) -> HistoryEvent { + serde_json::from_value(json!({"id": format!("event-{index}"), "sequence": index, + "namespace":"tenant", "event_type":kind, "payload":payload})) + .unwrap() +} + +fn started() -> HistoryEvent { + event("WorkflowStarted", json!({}), 1) +} + +fn opening(sequence: u64, id: &str, parent: &str, shield: bool, index: u64) -> HistoryEvent { + event( + "CancellationScopeOpened", + json!({"schema":"durable-workflow.cancellation-scope/v1", + "workflow_run_id":"run-one", "sequence":sequence, "scope_id":id, "parent_scope_id":parent, + "shield_parent":shield}), + index, + ) +} + +fn context(history: Vec) -> WorkflowContext { + WorkflowContext { + cancellation_scope_id: "root".into(), + state: Arc::new(Mutex::new( + WorkflowState::new_with_identity_and_scopes( + history, + Some("workflow-one".into()), + Some("run-one".into()), + "queue".into(), + DEFAULT_CODEC.into(), + None, + true, + ) + .unwrap(), + )), + } +} + +fn poll(future: &mut F) -> Poll { + let mut cx = TaskContext::from_waker(noop_waker_ref()); + Pin::new(future).poll(&mut cx) +} + +#[test] +fn cancellation_scope_authoring_body_waits_for_canonical_opening() { + let ctx = context(vec![started()]); + let calls = Arc::new(AtomicUsize::new(0)); + let body_calls = Arc::clone(&calls); + let mut scope = Box::pin(ctx.cancellation_scope(false, move |_| async move { + body_calls.fetch_add(1, Ordering::SeqCst); + Ok("done") + })); + assert!(poll(&mut scope).is_pending()); + assert_eq!(calls.load(Ordering::SeqCst), 0); + let state = ctx.state.lock().unwrap(); + let intent = state.cancellation_scope_opening.as_ref().unwrap(); + assert_eq!(intent.sequence, 1); + assert_eq!(intent.parent_scope_id, "root"); + assert!(!intent.shield_parent); + assert_eq!(intent.command_count, 0); + assert!(state.commands.is_empty()); +} + +#[test] +fn cancellation_scope_authoring_default_context_refuses_before_body() { + let ctx = workflow_context(Vec::new()); + let mut scope = Box::pin(ctx.cancellation_scope(false, |_| async move { + panic!("unqualified scope cannot enter body"); + #[allow(unreachable_code)] + Ok(()) + })); + assert!(matches!( + poll(&mut scope), + Poll::Ready(Err(Error::CancellationScopeExecutionUnavailable)) + )); +} + +#[test] +fn cancellation_scope_authoring_nested_context_and_deferred_timers_retain_membership() { + let ctx = context(vec![ + started(), + opening(1, "outer", "root", false, 2), + opening(2, "inner", "outer", true, 3), + ]); + let mut scope = Box::pin(ctx.cancellation_scope(false, |outer| async move { + let deferred = outer + .cancellation_scope(true, |inner| async move { + Ok(inner.sleep(Duration::from_secs(1))) + }) + .await?; + Ok((deferred, outer.sleep(Duration::from_secs(2)))) + })); + let Poll::Ready(Ok((mut inner, mut outer))) = poll(&mut scope) else { + panic!("canonical body must resume"); + }; + let mut root = ctx.sleep(Duration::from_secs(3)); + assert!(poll(&mut inner).is_pending()); + assert!(poll(&mut outer).is_pending()); + assert!(poll(&mut root).is_pending()); + let wire = ctx.take_commands().unwrap(); + assert_eq!(wire.len(), 3); + assert_eq!(wire[0]["cancellation_scope_id"], "inner"); + assert_eq!(wire[1]["cancellation_scope_id"], "outer"); + assert!(wire[2].get("cancellation_scope_id").is_none()); + assert_eq!(ctx.cancellation_scope_id, "root"); +} + +#[test] +fn cancellation_scope_authoring_prefix_does_not_enter_body_and_keeps_sequence() { + let ctx = context(vec![started()]); + assert_eq!(ctx.side_effect(|| "prefix".to_string()).unwrap(), "prefix"); + let mut scope = Box::pin(ctx.cancellation_scope(false, |_| async move { + panic!("opening must commit after the prefix"); + #[allow(unreachable_code)] + Ok(()) + })); + assert!(poll(&mut scope).is_pending()); + let state = ctx.state.lock().unwrap(); + assert_eq!( + state.cancellation_scope_opening.as_ref().unwrap().sequence, + 2 + ); + assert_eq!( + state + .cancellation_scope_opening + .as_ref() + .unwrap() + .command_count, + 1 + ); +} + +#[test] +fn cancellation_scope_authoring_changed_parent_shield_or_position_cannot_enter_body() { + for (parent, shield, sequence) in [("root", true, 1), ("root", false, 2)] { + let ctx = context(vec![ + started(), + opening(sequence, "outer", parent, shield, 2), + ]); + let mut scope = Box::pin(ctx.cancellation_scope(false, |_| async move { + panic!("changed opening cannot enter body"); + #[allow(unreachable_code)] + Ok(()) + })); + assert!( + matches!(poll(&mut scope), Poll::Ready(Err(Error::NonDeterministicReplay(f))) + if f.reason == "cancellation_scope_opening_changed") + ); + } + let ctx = context(vec![ + started(), + opening(1, "outer", "root", false, 2), + opening(2, "inner", "root", false, 3), + ]); + let mut scope = Box::pin(ctx.cancellation_scope(false, |outer| async move { + outer + .cancellation_scope(false, |_| async move { + panic!("changed nested parent cannot enter body"); + #[allow(unreachable_code)] + Ok(()) + }) + .await + })); + assert!( + matches!(poll(&mut scope), Poll::Ready(Err(Error::NonDeterministicReplay(f))) + if f.reason == "cancellation_scope_opening_changed") + ); +} + +#[test] +fn cancellation_scope_authoring_invalid_tree_and_operation_membership_refuse_before_factory() { + let base = opening(1, "outer", "root", false, 2); + let mutations = [ + ("scope_id", json!("root")), + ("scope_id", json!("")), + ("scope_id", json!(true)), + ("scope_id", json!("é".repeat(128))), + ("parent_scope_id", json!("missing")), + ("parent_scope_id", json!("outer")), + ("workflow_run_id", json!("foreign")), + ("schema", json!("foreign")), + ("sequence", json!(true)), + ("sequence", json!(0)), + ("sequence", json!(u64::MAX)), + ("shield_parent", json!(1)), + ]; + for (field, value) in mutations { + let mut row = base.clone(); + row.payload[field] = value; + let calls = Arc::new(AtomicUsize::new(0)); + let factory_calls = Arc::clone(&calls); + let mut worker = Worker::new(Client::new("http://127.0.0.1:1").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true); + worker.register_workflow("scope-probe", move |_, _| { + factory_calls.fetch_add(1, Ordering::SeqCst); + async move { Ok(json!("unexpected")) } + }); + let mut task = workflow_task("scope-probe", vec![started(), row], DEFAULT_CODEC); + task.run_id = Some("run-one".into()); + assert!(matches!( + worker.execute_workflow_task(task), + Err(Error::NonDeterministicReplay(_)) + )); + assert_eq!(calls.load(Ordering::SeqCst), 0); + } + for payload in [ + json!({"sequence":2,"cancellation_scope_id":"missing"}), + json!({"sequence":1,"cancellation_scope_id":"outer"}), + json!({"sequence":true,"cancellation_scope_id":"outer"}), + json!({"sequence":2,"cancellation_scope_id":null}), + json!({"sequence":2,"cancellation_scope_id":"outer","timer":{"cancellation_scope_id":"root"}}), + ] { + assert!(cancellation_scope::CancellationScopeHistory::read( + &[started(), base.clone(), event("TimerScheduled", payload, 3)], + "run-one" + ) + .is_err()); + } +} + +#[test] +fn cancellation_scope_authoring_timer_replay_checks_membership_and_consumes_opening() { + let history = vec![ + started(), + opening(1, "outer", "root", false, 2), + event( + "TimerScheduled", + json!({"sequence":2,"timer_id":"timer-one","delay_seconds":1,"cancellation_scope_id":"outer"}), + 3, + ), + event( + "TimerFired", + json!({"sequence":2,"timer_id":"timer-one","delay_seconds":1}), + 4, + ), + ]; + for root_command in [false, true] { + let ctx = context(history.clone()); + let mut scope = Box::pin(ctx.cancellation_scope(false, |scoped| async move { + Ok(scoped.sleep(Duration::from_secs(1))) + })); + let Poll::Ready(Ok(mut timer)) = poll(&mut scope) else { + panic!("scope must replay"); + }; + if root_command { + timer = ctx.sleep(Duration::from_secs(1)); + } + let actual = poll(&mut timer); + if root_command { + assert!( + matches!(actual,Poll::Ready(Err(Error::NonDeterministicReplay(f))) + if f.reason == "cancellation_scope_membership_changed") + ); + } else { + assert!(matches!(actual, Poll::Ready(Ok(())))); + ctx.ensure_history_consumed().unwrap(); + } + } +} + +#[test] +fn cancellation_scope_authoring_unimplemented_delivery_and_closed_history_refuse() { + for kind in [ + "CancellationScopeRequested", + "CancellationScopeDeliveryPrepared", + "CancellationScopeDelivered", + "CancellationScopeRequestConflicted", + ] { + assert!(matches!( + WorkflowState::new_with_identity_and_scopes( + vec![ + started(), + opening(1, "outer", "root", false, 2), + event(kind, json!({}), 3) + ], + None, + Some("run-one".into()), + "queue".into(), + DEFAULT_CODEC.into(), + None, + true + ), + Err(Error::CancellationScopeExecutionUnavailable) + )); + } + for kind in [ + "WorkflowCompleted", + "WorkflowFailed", + "WorkflowCancelled", + "WorkflowTerminated", + ] { + let ctx = context(vec![started(), event(kind, json!({}), 2)]); + let mut scope = Box::pin(ctx.cancellation_scope(false, |_| async move { + panic!("closed history cannot enter new scope"); + #[allow(unreachable_code)] + Ok(()) + })); + assert!( + matches!(poll(&mut scope),Poll::Ready(Err(Error::NonDeterministicReplay(f))) + if f.reason == "cancellation_scope_opening_changed") + ); + } +} + +#[test] +fn cancellation_scope_authoring_all_supported_operation_futures_keep_creation_context() { + let ctx = context(vec![started(), opening(1, "outer", "root", false, 2)]); + let mut scope = Box::pin(ctx.cancellation_scope(false, |scoped| async move { Ok(scoped) })); + let Poll::Ready(Ok(scoped)) = poll(&mut scope) else { + panic!("scope must replay"); + }; + let mut activity = scoped.activity("remote", json!([])); + let mut child = + scoped.start_child_workflow("child", ChildWorkflowOptions::new("queue"), json!([])); + let mut signal = scoped.wait_signal("resume"); + let mut condition = scoped + .wait_condition(ConditionWaitOptions::new("ready", "sha256:ready"), || { + Ok(false) + }); + assert!(poll(&mut activity).is_pending()); + assert!(poll(&mut child).is_pending()); + assert!(poll(&mut signal).is_pending()); + assert!(poll(&mut condition).is_pending()); + let wire = ctx.take_commands().unwrap(); + assert_eq!(wire.len(), 4); + for command in wire { + assert_eq!(command["cancellation_scope_id"], "outer"); + } + assert_eq!(ctx.cancellation_scope_id, "root"); +} + +#[test] +fn cancellation_scope_authoring_worker_cannot_publish_after_ignoring_pending_opening() { + let mut worker = Worker::new(Client::new("http://127.0.0.1:1").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true); + worker.register_workflow("scope-probe", |ctx, _| async move { + let mut opening = Box::pin(ctx.cancellation_scope(false, |_| async move { + panic!("uncommitted scope cannot enter body"); + #[allow(unreachable_code)] + Ok(()) + })); + std::future::poll_fn(|cx| { + assert!(opening.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + ctx.side_effect(|| "escaped".to_string())?; + Ok(json!("must not publish")) + }); + let mut task = workflow_task("scope-probe", vec![started()], DEFAULT_CODEC); + task.run_id = Some("run-one".into()); + assert!( + matches!(worker.execute_workflow_task(task),Err(Error::NonDeterministicReplay(f)) + if f.reason == "cancellation_scope_pending_call_escaped") + ); +} + +#[test] +fn cancellation_scope_authoring_cannot_enable_protocol_or_delivery_implicitly() { + let calls = Arc::new(AtomicUsize::new(0)); + let factory_calls = Arc::clone(&calls); + let mut worker = Worker::new(Client::new("http://127.0.0.1:1").unwrap(), "queue") + .candidate_cancellation_scope_authoring(true); + worker.register_workflow("scope-probe", move |_, _| { + factory_calls.fetch_add(1, Ordering::SeqCst); + async move { Ok(json!("unexpected")) } + }); + assert!(matches!( + worker.execute_workflow_task(workflow_task("scope-probe", vec![started()], DEFAULT_CODEC)), + Err(Error::CancellationScopeExecutionUnavailable) + )); + assert_eq!(calls.load(Ordering::SeqCst), 0); +} diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index 09b93cc..50db5c6 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -746,6 +746,101 @@ async fn server_scope_opening_proves_real_native_tree_on_original_claim() { .unwrap(); } +fn scope_authoring_worker(client: &Client, queue: &str, owner: &str) -> Worker { + let mut worker = Worker::new(client.clone(), queue) + .worker_id(owner) + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .poll_timeout(Duration::from_millis(100)); + worker.register_workflow("tests.rust-candidate-scope-replay", |ctx, _| async move { + let _: String = ctx.side_effect(|| "original prefix".to_string())?; + ctx.cancellation_scope(false, |outer| async move { + let deferred = outer + .cancellation_scope(true, |inner| async move { + Ok(inner.sleep(Duration::from_secs(1))) + }) + .await?; + deferred.await?; + outer.sleep(Duration::from_secs(1)).await + }) + .await?; + ctx.sleep(Duration::from_secs(1)).await?; + Ok(json!("replayed original scopes")) + }); + worker +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with the Native scope candidate"] +async fn server_scope_authoring_replays_nested_tree_and_deferred_membership_on_replacement() { + let client = client(); + let queue = queue(); + let original = scope_authoring_worker(&client, &queue, &format!("{queue}-original")); + original.register().await.unwrap(); + let handle = client + .start_workflow( + "tests.rust-candidate-scope-replay", + &queue, + &queue, + json!([]), + ) + .await + .unwrap(); + let first = tick_until(&original, &handle, "TimerScheduled").await; + assert_eq!(count(&first, "SideEffectRecorded"), 1); + assert_eq!(count(&first, "CancellationScopeOpened"), 2); + assert_eq!(count(&first, "TimerScheduled"), 1); + + let replacement = scope_authoring_worker(&client, &queue, &format!("{queue}-replacement")); + replacement.register().await.unwrap(); + let final_history = tick_until(&replacement, &handle, "WorkflowCompleted").await; + assert_eq!(count(&final_history, "SideEffectRecorded"), 1); + assert_eq!(count(&final_history, "CancellationScopeOpened"), 2); + assert_eq!(count(&final_history, "TimerScheduled"), 3); + assert_eq!(count(&final_history, "TimerFired"), 3); + assert_eq!(count(&final_history, "WorkflowFailed"), 0); + let rows = final_history["events"].as_array().unwrap(); + let openings: Vec<_> = rows + .iter() + .filter(|row| row["event_type"] == "CancellationScopeOpened") + .map(|row| &row["payload"]) + .collect(); + assert_eq!(openings[0]["sequence"], 2); + assert_eq!(openings[0]["parent_scope_id"], "root"); + assert_eq!(openings[0]["shield_parent"], false); + assert_eq!(openings[1]["sequence"], 3); + assert_eq!(openings[1]["parent_scope_id"], openings[0]["scope_id"]); + assert_eq!(openings[1]["shield_parent"], true); + let timers: Vec<_> = rows + .iter() + .filter(|row| row["event_type"] == "TimerScheduled") + .map(|row| &row["payload"]) + .collect(); + assert_eq!( + timers + .iter() + .map(|row| row["sequence"].as_u64().unwrap()) + .collect::>(), + vec![4, 5, 6] + ); + assert_eq!(timers[0]["cancellation_scope_id"], openings[1]["scope_id"]); + assert_eq!(timers[1]["cancellation_scope_id"], openings[0]["scope_id"]); + assert!(timers[2] + .get("cancellation_scope_id") + .is_none_or(|id| id == "root")); + assert_eq!( + handle + .result_selected_run(WorkflowResultOptions { + timeout: Duration::from_secs(10), + ..WorkflowResultOptions::default() + }) + .await + .unwrap(), + json!("replayed original scopes") + ); + println!("Native scope authoring and replacement replay: {final_history}"); +} + #[tokio::test] #[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] async fn server_request_before_claim_replays_canonical_typed_cancellation() { From 41add890f7420825ec96aca5a849b0cda7e3cfc1 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 00:14:05 +0000 Subject: [PATCH 45/58] feat: prove original scope cancellation preparation and delivery --- src/cancellation_context.rs | 31 + src/cancellation_scope_history.rs | 1454 +++++++ src/lib.rs | 6 + src/tests/cancellation_scope_history.rs | 744 ++++ tests/fixtures/committed-scope-delivery.json | 695 +++ ...committed-scope-operation-projections.json | 3285 ++++++++++++++ .../populated-scope-single-calls.json | 3756 +++++++++++++++++ .../run-inherited-scope-delivery.json | 937 ++++ 8 files changed, 10908 insertions(+) create mode 100644 src/cancellation_scope_history.rs create mode 100644 src/tests/cancellation_scope_history.rs create mode 100644 tests/fixtures/committed-scope-delivery.json create mode 100644 tests/fixtures/committed-scope-operation-projections.json create mode 100644 tests/fixtures/populated-scope-single-calls.json create mode 100644 tests/fixtures/run-inherited-scope-delivery.json diff --git a/src/cancellation_context.rs b/src/cancellation_context.rs index 4bce3c6..45160de 100644 --- a/src/cancellation_context.rs +++ b/src/cancellation_context.rs @@ -402,6 +402,37 @@ fn assert_context_keys(value: &Value, keys: &[&str]) -> Result<()> { } impl ScopedCancellationContext { + pub(super) fn from_run_context(context: &CancellationContext) -> Result { + let mut root = context.to_value(); + let mut lineage = Vec::new(); + if let Some(origin) = context.scope_origin() { + root = origin.root_context().to_value(); + lineage.extend( + origin + .lineage() + .iter() + .map(ScopedCancellationLineage::to_value), + ); + let local = context.lineage().last().unwrap(); + lineage.push(json!({"request_id":context.request_id(), + "workflow_instance_id":local.workflow_instance_id(), + "workflow_run_id":local.workflow_run_id(), "scope_id":"root", + "cleanup_deadline_at":context.deadline().to_rfc3339_opts(SecondsFormat::Micros, true)})); + } else { + root["request_id"] = json!(context.root_request_id()); + root["parent_request_id"] = Value::Null; + root["lineage"] = json!([context.lineage()[0].to_value()]); + lineage.extend(context.lineage().iter().map(|entry| json!({ + "request_id":entry.request_id(), "workflow_instance_id":entry.workflow_instance_id(), + "workflow_run_id":entry.workflow_run_id(), "scope_id":"root", + "cleanup_deadline_at":context.deadline().to_rfc3339_opts(SecondsFormat::Micros, true)}))); + } + Self::from_value( + &json!({"schema":"durable-workflow.scoped-cancellation-context/v1", + "root_context":root, "lineage":lineage}), + ) + } + pub fn from_value(value: &Value) -> Result { assert_context_keys(value, &["schema", "root_context", "lineage"])?; if value["schema"] != "durable-workflow.scoped-cancellation-context/v1" diff --git a/src/cancellation_scope_history.rs b/src/cancellation_scope_history.rs new file mode 100644 index 0000000..493f6f7 --- /dev/null +++ b/src/cancellation_scope_history.rs @@ -0,0 +1,1454 @@ +//! Original accepted scope requests and frozen v5 delivery projections. +//! These facts alone grant no callback, stop, or publication authority. + +use super::*; +use chrono::{SecondsFormat, Utc}; +use serde::ser::{SerializeMap, SerializeSeq}; + +const FIELDS: [&str; 4] = [ + "activity_members", + "timer_members", + "wait_members", + "child_members", +]; +const GROUP_KEYS: [&str; 11] = [ + "parallel_group_id", + "parallel_group_kind", + "parallel_group_mode", + "parallel_group_base_sequence", + "parallel_group_size", + "parallel_group_index", + "selection_member_key", + "selection_member_index", + "selection_member_base_sequence", + "selection_member_size", + "selection_member_kind", +]; + +fn invalid(detail: &str) -> Error { + invalid_recorded_history( + "invalid_cancellation_scope_history", + 0, + "original accepted scope and frozen operation projection", + "invalid history", + detail, + ) +} + +fn identity(value: &Value) -> bool { + value + .as_str() + .is_some_and(|text| !text.trim().is_empty() && text.len() <= 255) +} + +fn text<'a>(value: &'a Value, key: &str) -> Result<&'a str> { + value + .get(key) + .filter(|value| identity(value)) + .and_then(Value::as_str) + .ok_or_else(|| invalid("scope proof requires its original identity")) +} + +fn positive(value: &Value) -> bool { + value + .as_u64() + .is_some_and(|value| value > 0 && value <= i64::MAX as u64) +} + +pub(super) fn timestamp(value: &Value) -> Result> { + value + .as_str() + .and_then(|value| DateTime::parse_from_rfc3339(value).ok()) + .map(|value| value.with_timezone(&Utc)) + .ok_or_else(|| invalid("scope authority requires an original timestamp with timezone")) +} + +fn event_time(event: &HistoryEvent) -> Result> { + timestamp( + event + .raw + .get("timestamp") + .or_else(|| event.raw.get("recorded_at")) + .unwrap_or(&Value::Null), + ) +} + +fn event_id(event: &HistoryEvent) -> Result<&str> { + event + .raw + .get("id") + .filter(|value| identity(value)) + .and_then(Value::as_str) + .ok_or_else(|| invalid("scope proof lacks its original event identity")) +} +fn event_sequence(event: &HistoryEvent) -> Result { + event + .raw + .get("sequence") + .filter(|value| positive(value)) + .and_then(Value::as_u64) + .ok_or_else(|| invalid("scope proof lacks canonical event order")) +} +fn canonical_time(time: DateTime) -> String { + time.to_rfc3339_opts(SecondsFormat::Micros, true) +} +fn policy(value: &Value) -> bool { + matches!( + value.as_str(), + Some("try_cancel" | "wait_cancellation_completed" | "abandon") + ) +} + +// Only descriptor group objects use this key order. Do not change serde_json's +// global map behavior or any payload codec to reproduce Native's PHP hashes. +struct Descriptor<'a>(&'a Value); +impl Serialize for Descriptor<'_> { + fn serialize( + &self, + serializer: S, + ) -> std::result::Result { + match self.0 { + Value::Array(values) => { + let mut seq = serializer.serialize_seq(Some(values.len()))?; + for value in values { + seq.serialize_element(&Descriptor(value))?; + } + seq.end() + } + Value::Object(object) => { + let mut map = serializer.serialize_map(Some(object.len()))?; + for key in GROUP_KEYS { + if let Some(value) = object.get(key) { + map.serialize_entry(key, &Descriptor(value))?; + } + } + if object.keys().any(|key| !GROUP_KEYS.contains(&key.as_str())) { + return Err(serde::ser::Error::custom( + "unexpected scope descriptor object", + )); + } + map.end() + } + value => value.serialize(serializer), + } + } +} + +fn descriptor_hash(values: Value) -> Result { + let encoded = serde_json::to_string(&Descriptor(&values))?; + let mut php = String::new(); + for character in encoded.chars() { + if character == '/' { + php.push_str("\\/"); + } else if character as u32 > 0x7f { + for unit in character.encode_utf16(&mut [0; 2]) { + use std::fmt::Write; + write!(&mut php, "\\u{unit:04x}").unwrap(); + } + } else { + php.push(character); + } + } + Ok(format!("{:x}", Sha256::digest(php.as_bytes()))) +} + +#[cfg(test)] +mod descriptor_interop { + use super::*; + + #[test] + fn scope_history_descriptor_hash_matches_php_unicode_slashes_del_and_ordered_groups() { + assert_eq!( + descriptor_hash(json!([ + "scope/é😀", + "event/a", + "\u{7f}", + "x\u{2028}y", + true, + null + ])) + .unwrap(), + "afde5922dded878e86ff3e376d9ebd20804265c8df877ac50bcd6d3ca0b22aa8" + ); + assert_eq!( + descriptor_hash(json!([{"parallel_group_id":"parallel-calls:scope/é😀", + "parallel_group_kind":"mixed", "parallel_group_base_sequence":3, + "parallel_group_size":2, "parallel_group_index":0}])) + .unwrap(), + "c31d38786755cb4d32824eb8b765e305757bb627eaa7433eb6f82926928a970a" + ); + } +} + +pub(super) fn normalize_members(field: &str, value: &Value) -> Result { + let (keys, id_key): (&[&str], &str) = match field { + "activity_members" => ( + &["sequence", "activity_execution_id", "descriptor_hash"], + "activity_execution_id", + ), + "timer_members" => (&["sequence", "timer_id", "descriptor_hash"], "timer_id"), + "wait_members" => ( + &["kind", "sequence", "wait_id", "timer_id", "descriptor_hash"], + "wait_id", + ), + "child_members" => ( + &[ + "sequence", + "child_call_id", + "child_workflow_instance_id", + "child_workflow_run_id", + "cancellation_policy", + "descriptor_hash", + ], + "child_call_id", + ), + _ => return Err(invalid("unknown scope projection")), + }; + let members = value + .as_array() + .ok_or_else(|| invalid("scope projection must be a list"))?; + let mut ids = BTreeSet::new(); + let mut sequences = BTreeSet::new(); + for member in members { + let object = member + .as_object() + .ok_or_else(|| invalid("scope member must be an object"))?; + let hash = member["descriptor_hash"].as_str().unwrap_or_default(); + if object.len() != keys.len() + || keys.iter().any(|key| !object.contains_key(*key)) + || !positive(&member["sequence"]) + || !identity(&member[id_key]) + || !ids.insert(member[id_key].as_str().unwrap().to_owned()) + || hash.len() != 64 + || !hash + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + || (matches!(field, "activity_members" | "child_members") + && !sequences.insert(member["sequence"].as_u64().unwrap())) + { + return Err(invalid( + "scope projection changes its original member address", + )); + } + if field == "wait_members" + && (!matches!(member["kind"].as_str(), Some("signal" | "condition")) + || (!member["timer_id"].is_null() && !identity(&member["timer_id"]))) + { + return Err(invalid("scope wait changes its kind or timeout address")); + } + if field == "child_members" + && (!identity(&member["child_workflow_instance_id"]) + || !identity(&member["child_workflow_run_id"]) + || !policy(&member["cancellation_policy"])) + { + return Err(invalid("scope child changes its target or policy")); + } + } + Ok(value.clone()) +} + +fn address<'a>(payload: &'a Value, descriptor_key: &str) -> Result<&'a str> { + let descriptor = payload.get(descriptor_key); + if descriptor.is_some_and(|value| !value.is_object()) { + return Err(invalid("scope operation lacks its original descriptor")); + } + let nested = descriptor.and_then(|value| value.get("cancellation_scope_id")); + let address = payload.get("cancellation_scope_id").or(nested); + if address.is_some_and(|value| !identity(value)) || (nested.is_some() && nested != address) { + return Err(invalid("scope operation changes its immediate membership")); + } + Ok(address.and_then(Value::as_str).unwrap_or("root")) +} + +fn group_entry(payload: &Value) -> Option { + let id = payload["parallel_group_id"].as_str()?; + let kind = payload["parallel_group_kind"].as_str().or_else(|| { + [ + ("parallel-activities:", "activity"), + ("parallel-calls:", "mixed"), + ("select-calls:", "mixed"), + ("parallel-timers:", "timer"), + ("parallel-children:", "child"), + ] + .into_iter() + .find_map(|(prefix, kind)| id.starts_with(prefix).then_some(kind)) + })?; + let mode = payload["parallel_group_mode"] + .as_str() + .filter(|value| !value.is_empty()) + .or_else(|| id.starts_with("select-calls:").then_some("select")); + let select = mode == Some("select"); + if [ + "parallel_group_base_sequence", + "parallel_group_size", + "parallel_group_index", + ] + .iter() + .any(|key| payload[*key].as_i64().is_none()) + || payload["parallel_group_size"].as_i64()? < 1 + || (select + && !(payload["selection_member_key"] + .as_i64() + .is_some_and(|key| key >= 0) + || payload["selection_member_key"] + .as_str() + .is_some_and(|key| !key.is_empty()))) + { + return None; + } + let mut result = json!({"parallel_group_id":id, "parallel_group_kind":kind, + "parallel_group_base_sequence":payload["parallel_group_base_sequence"], + "parallel_group_size":payload["parallel_group_size"], "parallel_group_index":payload["parallel_group_index"]}); + if select { + result["parallel_group_mode"] = json!("select"); + result["selection_member_key"] = payload["selection_member_key"].clone(); + } + for key in [ + "selection_member_index", + "selection_member_base_sequence", + "selection_member_size", + ] { + if payload[key].as_i64().is_some() { + result[key] = payload[key].clone(); + } + } + if payload["selection_member_kind"] + .as_str() + .is_some_and(|value| !value.is_empty()) + { + result["selection_member_kind"] = payload["selection_member_kind"].clone(); + } + Some(result) +} + +fn group_path(payload: &Value) -> Result { + let raw = match payload.get("parallel_group_path") { + None => &[][..], + Some(Value::Array(path)) => path.as_slice(), + _ => { + return Err(invalid( + "scope operation has a malformed original group path", + )) + } + }; + let path: Vec<_> = raw.iter().filter_map(group_entry).collect(); + Ok(json!(if path.is_empty() { + group_entry(payload).into_iter().collect::>() + } else { + path + })) +} + +pub(super) fn members_from_prefix( + field: &str, + prefix: &[HistoryEvent], + scope: &str, + run: &str, +) -> Result { + let mut members = Vec::new(); + let mut activity_ids = BTreeSet::new(); + let mut activity_sequences = BTreeSet::new(); + let mut timer_sequences = BTreeMap::new(); + for event in prefix { + let p = &event.payload; + let sequence = &p["sequence"]; + match (field, event.event_type.as_str()) { + ("activity_members", "ActivityScheduled") => { + let activity = &p["activity"]; + let id = &p["activity_execution_id"]; + if !activity.is_object() + || !identity(id) + || activity["id"] != *id + || !positive(sequence) + || !activity_ids.insert(id.as_str().unwrap().to_owned()) + || !activity_sequences.insert(sequence.as_u64().unwrap()) + || p.get("cancellation_scope_id").unwrap_or( + activity + .get("cancellation_scope_id") + .unwrap_or(&json!("root")), + ) != activity + .get("cancellation_scope_id") + .unwrap_or(&json!("root")) + { + return Err(invalid( + "scope Activity admission changes identity or membership", + )); + } + if address(p, "activity")? != scope { + continue; + } + for key in ["local_preparation", "local_group_admission"] { + if p[key]["cancellation_cleanup"].get("scope_id").is_some() { + return Err(invalid("nested cleanup needs its original cleanup proof")); + } + } + let cancellation_policy = activity + .get("cancellation_policy") + .cloned() + .unwrap_or(json!("try_cancel")); + if !policy(&cancellation_policy) + || p.get("local_activity") + .is_some_and(|value| !value.is_boolean()) + || p.get("execution_mode") + .is_some_and(|value| !value.is_null() && !value.is_string()) + || activity + .get("schedule_to_close_deadline_at") + .is_some_and(|value| !value.is_null() && !value.is_string()) + { + return Err(invalid( + "scope Activity changes its cancellation descriptor", + )); + } + members.push(json!({"sequence":sequence, "activity_execution_id":id, + "descriptor_hash":descriptor_hash(json!([scope, event_id(event)?, cancellation_policy, + p.get("local_activity").unwrap_or(&Value::Bool(false)), p["execution_mode"], + activity["schedule_to_close_deadline_at"]]))?})); + } + ("timer_members", "TimerScheduled") if address(p, "timer")? == scope => { + let id = &p["timer_id"]; + let kind = &p["timer_kind"]; + if !positive(sequence) + || !identity(id) + || p["delay_seconds"].as_i64().is_none_or(|delay| delay < 0) + || timer_sequences + .get(&sequence.as_u64().unwrap_or_default()) + .is_some_and(|previous| { + previous != kind + || !matches!( + kind.as_str(), + Some("signal_timeout" | "condition_timeout") + ) + }) + || p["fire_at"].as_str() + != Some(canonical_time(timestamp(&p["fire_at"])?).as_str()) + { + return Err(invalid("scope timer changes its original descriptor")); + } + timer_sequences.insert(sequence.as_u64().unwrap(), kind.clone()); + members.push(json!({"sequence":sequence, "timer_id":id, + "descriptor_hash":descriptor_hash(json!([scope, event_id(event)?, sequence, id, + p["delay_seconds"], p["fire_at"], kind, p["condition_wait_id"], + p["condition_wait_occurrence_id"], p["signal_wait_id"], group_path(p)?]))?})); + } + ("wait_members", "SignalWaitOpened" | "ConditionWaitOpened") + if p["cancellation_scope_id"].as_str().unwrap_or("root") == scope => + { + let kind = if event.event_type == "SignalWaitOpened" { + "signal" + } else { + "condition" + }; + let id = &p[format!("{kind}_wait_id")]; + if !positive(sequence) || !identity(id) { + return Err(invalid("scope wait changes its original address")); + } + let timers: Vec<_> = prefix + .iter() + .filter(|row| { + row.event_type == "TimerScheduled" + && row.payload[format!("{kind}_wait_id")] == *id + }) + .collect(); + if timers.len() > 1 { + return Err(invalid("scope wait duplicates its timeout")); + } + let timer = timers.first().copied(); + let empty = Value::Null; + let timeout = timer.map_or(&empty, |row| &row.payload); + if let Some(timer) = timer { + if timeout["timer_kind"] != format!("{kind}_timeout") + || timeout["sequence"] != *sequence + || event_sequence(timer)? <= event_sequence(event)? + || timeout["cancellation_scope_id"].as_str().unwrap_or("root") != scope + || !identity(&timeout["timer_id"]) + { + return Err(invalid("scope wait changes its original timeout")); + } + } + members.push(json!({"kind":kind, "sequence":sequence, "wait_id":id, "timer_id":timeout["timer_id"], + "descriptor_hash":descriptor_hash(json!([scope, event_id(event)?, kind, sequence, id, + timer.map(event_id).transpose()?, timeout["timer_id"], p["timeout_seconds"], p["signal_name"], + p["condition_wait_occurrence_id"], p["condition_key"], p["condition_definition_fingerprint"], + group_path(p)?]))?})); + } + ("child_members", "ChildWorkflowScheduled") + if address(p, "child_workflow")? == scope => + { + let id = &p["child_call_id"]; + let instance = &p["child_workflow_instance_id"]; + let mut target = p["child_workflow_run_id"].clone(); + let cancellation_policy = p + .get("cancellation_policy") + .cloned() + .unwrap_or(json!("abandon")); + if !positive(sequence) + || !identity(id) + || !identity(instance) + || !identity(&target) + || target == run + || !policy(&cancellation_policy) + { + return Err(invalid("scope child changes its original target or policy")); + } + let mut last_started = None; + for started in prefix.iter().filter(|row| { + row.event_type == "ChildRunStarted" && row.payload["sequence"] == *sequence + }) { + let next = &started.payload; + if event_sequence(started)? <= event_sequence(event)? + || next["child_call_id"] != *id + || next["child_workflow_instance_id"] != *instance + || !identity(&next["child_workflow_run_id"]) + || next["child_workflow_run_id"] == run + || next + .get("cancellation_scope_id") + .is_some_and(|value| value != scope) + || next + .get("cancellation_policy") + .is_some_and(|value| value != &cancellation_policy) + { + return Err(invalid("scope child changes its original continuation")); + } + target = next["child_workflow_run_id"].clone(); + last_started = Some(event_id(started)?); + } + members.push(json!({"sequence":sequence, "child_call_id":id, "child_workflow_instance_id":instance, + "child_workflow_run_id":target, "cancellation_policy":cancellation_policy, + "descriptor_hash":descriptor_hash(json!([scope, event_id(event)?, sequence, id, instance, + target, cancellation_policy, p["parent_close_policy"], p["child_workflow_type"], + last_started, group_path(p)?]))?})); + } + _ => {} + } + } + normalize_members(field, &json!(members)) +} + +fn descendants_from_prefix( + prefix: &[HistoryEvent], + scope: &str, + run: &str, + authority: DateTime, +) -> Result { + let openings: BTreeMap<_, _> = prefix + .iter() + .filter(|row| row.event_type == "CancellationScopeOpened") + .map(|row| Ok((text(&row.payload, "scope_id")?, row))) + .collect::>()?; + let requests: BTreeMap<_, _> = prefix + .iter() + .filter(|row| row.event_type == "CancellationScopeRequested") + .map(|row| Ok((text(&row.payload, "scope_id")?, row))) + .collect::>()?; + let mut included = BTreeSet::from([scope]); + let mut members = Vec::new(); + // Use canonical opening order, rather than a map's lexical identity order. + for opening in prefix + .iter() + .filter(|row| row.event_type == "CancellationScopeOpened") + { + let id = text(&opening.payload, "scope_id")?; + let parent_id = text(&opening.payload, "parent_scope_id")?; + if id == scope || opening.payload["shield_parent"] == true || !included.contains(parent_id) + { + continue; + } + included.insert(id); + let request = requests + .get(id) + .ok_or_else(|| invalid("descendant lacks its accepted propagation"))?; + let parent_request = requests + .get(parent_id) + .ok_or_else(|| invalid("descendant lacks its accepted parent"))?; + let context = ScopedCancellationContext::from_value(&request.payload["cancellation"])?; + let parent = + ScopedCancellationContext::from_value(&parent_request.payload["cancellation"])?; + let propagation = if context.root_context() == parent.root_context() { + if request.payload["parent_scope_id"] != parent_id { + return Err(invalid("descendant changes its original parent")); + } + *request + } else { + let mut conflict = None; + for event in prefix.iter().filter(|row| { + row.event_type == "CancellationScopeRequestConflicted" + && row.payload["scope_id"] == id + && row.payload["parent_scope_id"] == parent_id + }) { + let p = &event.payload; + if p["schema"] != "durable-workflow.cancellation-scope-request/v1" + || p["workflow_run_id"] != run + || p["reason"] != "cancellation_root_conflict" + || event_sequence(event)? + <= event_sequence(request)?.max(event_sequence(parent_request)?) + { + return Err(invalid("descendant changes its original conflict boundary")); + } + let incoming = ScopedCancellationContext::from_value(&p["incoming_cancellation"])?; + let accepted = ScopedCancellationContext::from_value(&p["accepted_cancellation"])?; + if incoming.root_context() == parent.root_context() + && incoming.lineage()[..incoming.lineage().len() - 1] == *parent.lineage() + && incoming.deadline() == parent.deadline() + && incoming.scope_id() == id + && incoming.workflow_run_id() == run + && incoming.workflow_instance_id() == context.workflow_instance_id() + && accepted == context + { + conflict = Some(event); + break; + } + } + conflict + .ok_or_else(|| invalid("descendant lacks its original competing-root conflict"))? + }; + let mut deadline = authority; + let mut ancestor = id; + while ancestor != "root" { + if let Some(request) = requests.get(ancestor) { + deadline = deadline.min( + ScopedCancellationContext::from_value(&request.payload["cancellation"])? + .deadline(), + ); + } + ancestor = text( + &openings + .get(ancestor) + .ok_or_else(|| invalid("descendant lost its original ancestry"))? + .payload, + "parent_scope_id", + )?; + } + let mut member = json!({"scope_id":id, "parent_scope_id":parent_id, "scope_history_event_id":event_id(opening)?, + "request_history_event_id":event_id(request)?, "request_id":context.request_id(), + "propagation_history_event_id":event_id(propagation)?, "authority_deadline_at":canonical_time(deadline), + "cancellation":context.to_value()}); + for field in FIELDS { + member[field] = members_from_prefix(field, prefix, id, run)?; + } + members.push(member); + } + Ok(json!(members)) +} + +#[derive(Clone, Debug)] +pub(super) struct ScopeRequest { + pub context: ScopedCancellationContext, + pub timestamp: DateTime, + pub history_index: usize, +} + +#[derive(Clone, Debug)] +pub(super) struct ScopeBoundary { + pub context: ScopedCancellationContext, + pub boundary: CancellationDelivery, + pub authority_deadline: DateTime, + pub event: HistoryEvent, +} + +#[derive(Default, Debug)] +pub(super) struct CommittedCancellationScopeHistory { + pub preparations: BTreeMap, + pub deliveries: BTreeMap, + pub pending_requests: BTreeMap, +} + +fn admission(kind: &str) -> Option<(&str, CancellationCallKind)> { + match kind { + "ActivityScheduled" => Some(("activity_members", CancellationCallKind::Activity)), + "TimerScheduled" => Some(("timer_members", CancellationCallKind::Timer)), + "ChildWorkflowScheduled" => Some(("child_members", CancellationCallKind::Child)), + "ConditionWaitOpened" => Some(("wait_members", CancellationCallKind::Condition)), + "SignalWaitOpened" => Some(("wait_members", CancellationCallKind::Signal)), + _ => None, + } +} + +fn assert_complete_group( + boundary: &CancellationDelivery, + scope: &str, + prefix: &[HistoryEvent], + scopes: &cancellation_scope::CancellationScopeHistory, +) -> Result<()> { + let mut members = BTreeSet::new(); + for event in prefix { + let p = &event.payload; + let Some(sequence) = p["sequence"] + .as_u64() + .filter(|value| *value > 0 && boundary.interrupts(*value)) + else { + continue; + }; + if admission(&event.event_type).is_none() + || (event.event_type == "TimerScheduled" + && matches!( + p["timer_kind"].as_str(), + Some("condition_timeout" | "signal_timeout") + )) + { + continue; + } + if scopes + .memberships + .get(&sequence) + .map(String::as_str) + .unwrap_or("root") + != scope + || !members.insert(sequence) + { + return Err(invalid("scope group changes its original member address")); + } + let fallback = vec![p.clone()]; + let path = match p.get("parallel_group_path") { + None => &fallback, + Some(Value::Array(path)) if !path.is_empty() && path.iter().all(Value::is_object) => { + path + } + _ => return Err(invalid("scope group requires its original admitted path")), + }; + if path.iter().any(|entry| { + entry["parallel_group_mode"].as_str().unwrap_or("all") != "all" + || entry["parallel_group_id"] + .as_str() + .unwrap_or_default() + .starts_with("select-calls:") + }) { + return Err(invalid( + "scoped selection needs its original selection proof", + )); + } + if path[0]["parallel_group_base_sequence"].as_u64() != Some(boundary.sequence) + || path[0]["parallel_group_size"].as_u64() != Some(boundary.sequence_span) + || path[0]["parallel_group_index"].as_u64() != Some(sequence - boundary.sequence) + { + return Err(invalid( + "scope group changes its original range or member index", + )); + } + } + if members.len() as u64 != boundary.sequence_span { + return Err(invalid("scope group omits an admitted member")); + } + Ok(()) +} + +impl CommittedCancellationScopeHistory { + pub fn read(history: &[HistoryEvent], run: &str, workflow: &str) -> Result { + let scopes = cancellation_scope::CancellationScopeHistory::read(history, run)?; + let addresses: BTreeMap<_, _> = scopes + .openings + .iter() + .map(|(sequence, opening)| (opening.scope_id.as_str(), (*sequence, opening))) + .collect(); + let run_state = CancellationHistory::from_events(history, run, None)?; + let run_root = run_state + .request + .as_ref() + .and_then(|request| request.context.as_ref()) + .map(ScopedCancellationContext::from_run_context) + .transpose()?; + let mut requests: BTreeMap = BTreeMap::new(); + let mut request_ids = BTreeSet::new(); + let mut committed = Self::default(); + let mut delivered = BTreeSet::new(); + let mut opened = BTreeSet::new(); + let mut admissions: BTreeMap<&str, BTreeMap> = BTreeMap::new(); + for (index, event) in history.iter().enumerate() { + let kind = event.event_type.as_str(); + let p = &event.payload; + if kind == "CancellationScopeOpened" { + opened.insert(text(p, "scope_id")?); + } + if admission(kind).is_some() && positive(&p["sequence"]) { + let sequence = p["sequence"].as_u64().unwrap(); + admissions.entry(kind).or_default().insert( + sequence, + scopes + .memberships + .get(&sequence) + .map(String::as_str) + .unwrap_or("root"), + ); + } + if !matches!( + kind, + "CancellationScopeRequested" + | "CancellationScopeDeliveryPrepared" + | "CancellationScopeDelivered" + ) { + continue; + } + let context = ScopedCancellationContext::from_value(&p["cancellation"])?; + let id = context.scope_id(); + let (opening_sequence, opening) = addresses + .get(id) + .ok_or_else(|| invalid("scope request lacks its original opening"))?; + if !opened.contains(id) + || run.is_empty() + || workflow.is_empty() + || context.workflow_run_id() != run + || context.workflow_instance_id() != workflow + || p["workflow_run_id"] != run + || p["scope_id"] != id + || p["request_id"] != context.request_id() + { + return Err(invalid( + "scope cancellation changes its original run, request or address", + )); + } + let time = event_time(event)?; + if time < context.requested_at() { + return Err(invalid("scope boundary predates its original request")); + } + if kind == "CancellationScopeRequested" { + if p["schema"] != "durable-workflow.cancellation-scope-request/v1" + || p.get("parent_scope_id").is_none() + || requests.contains_key(id) + || !request_ids.insert(context.request_id().to_owned()) + { + return Err(invalid( + "scope cancellation requires one accepted original request", + )); + } + if p["parent_scope_id"].is_null() { + if context.lineage().len() != 1 + || context.request_id() != context.root_context().request_id() + { + return Err(invalid( + "direct scope request substitutes an inherited lineage", + )); + } + } else { + let parent_id = text(p, "parent_scope_id")?; + let parent = if parent_id == "root" && run_state.request_index < index { + run_root.as_ref() + } else { + requests + .get(parent_id) + .filter(|request| request.history_index < index) + .map(|request| &request.context) + } + .ok_or_else(|| invalid("inherited scope request lacks its earlier parent"))?; + if parent.workflow_run_id() != run + || parent.workflow_instance_id() != workflow + || opening.shield_parent + || parent_id != opening.parent_scope_id + || context.root_context() != parent.root_context() + || context.lineage()[..context.lineage().len() - 1] != *parent.lineage() + || context.deadline() != parent.deadline() + { + return Err(invalid("inherited scope request changes its accepted parent or crosses a shield")); + } + } + requests.insert( + id.to_owned(), + ScopeRequest { + context, + timestamp: time, + history_index: index, + }, + ); + continue; + } + let request = requests + .get(id) + .ok_or_else(|| invalid("scope boundary lacks its accepted request"))?; + if context != request.context || time < request.timestamp { + return Err(invalid( + "scope boundary changes its immutable accepted request", + )); + } + if [ + "sequence_span", + "operation_sequence", + "operation_sequence_span", + ] + .iter() + .any(|key| p.get(*key).is_none()) + { + return Err(invalid("scope boundary omits its original operation range")); + } + let mut payload = p.clone(); + payload["workflow_command_id"] = json!(context.request_id()); + let boundary = CancellationDelivery::from_payload(&payload)?; + let deadline = timestamp(&p["authority_deadline_at"])?; + if boundary.sequence <= *opening_sequence + || deadline < context.requested_at() + || deadline > context.deadline() + || time > deadline + { + return Err(invalid( + "scope boundary changes or exceeds its original authority ceiling", + )); + } + if kind == "CancellationScopeDeliveryPrepared" { + if p["schema"] != "durable-workflow.cancellation-scope-preparation/v5" + || committed.preparations.contains_key(id) + || delivered.contains(id) + { + return Err(invalid( + "scope delivery requires one original v5 preparation", + )); + } + let prefix = &history[..index]; + for field in FIELDS { + if normalize_members(field, &p[field])? + != members_from_prefix(field, prefix, id, run)? + { + return Err(invalid( + "scope preparation changes its original member projection", + )); + } + } + if p["descendant_members"] != descendants_from_prefix(prefix, id, run, deadline)? { + return Err(invalid( + "scope preparation changes its original descendant projection", + )); + } + let mut by_scope = BTreeMap::from([(id, p)]); + for member in p["descendant_members"].as_array().unwrap() { + by_scope.insert(text(member, "scope_id")?, member); + } + let operation_scope = scopes + .memberships + .get(&boundary.sequence) + .map(String::as_str) + .unwrap_or(id); + if !by_scope.contains_key(operation_scope) { + return Err(invalid( + "scope boundary consumes an operation outside its frozen subtree", + )); + } + if boundary.call_kind == CancellationCallKind::Parallel { + assert_complete_group(&boundary, operation_scope, prefix, &scopes)?; + } else if !matches!( + boundary.call_kind, + CancellationCallKind::Activity + | CancellationCallKind::LocalActivity + | CancellationCallKind::Timer + | CancellationCallKind::Condition + | CancellationCallKind::Child + ) { + return Err(invalid("scope boundary uses an unsupported operation kind")); + } + for (member_scope, projection) in by_scope { + for (kind, positions) in &admissions { + let (field, mut call_kind) = admission(kind).unwrap(); + let sequences: BTreeSet<_> = projection[field] + .as_array() + .unwrap() + .iter() + .filter(|entry| { + !matches!(*kind, "ConditionWaitOpened" | "SignalWaitOpened") + || entry["kind"] + == if *kind == "ConditionWaitOpened" { + "condition" + } else { + "signal" + } + }) + .map(|entry| entry["sequence"].as_u64().unwrap()) + .collect(); + if positions.iter().any(|(sequence, owner)| { + *owner == member_scope && !sequences.contains(sequence) + }) { + return Err(invalid("scope preparation omits an admitted operation")); + } + if positions.get(&boundary.sequence).copied() != Some(member_scope) { + continue; + } + if call_kind == CancellationCallKind::Activity + && prefix.iter().any(|row| { + row.event_type == *kind + && row.payload["sequence"].as_u64() == Some(boundary.sequence) + && row.payload["local_activity"] == true + }) + { + call_kind = CancellationCallKind::LocalActivity; + } + let timeout = + *kind == "TimerScheduled" + && projection["wait_members"].as_array().unwrap().iter().any( + |entry| entry["sequence"].as_u64() == Some(boundary.sequence), + ); + if !sequences.contains(&boundary.sequence) + || (!timeout + && boundary.call_kind != CancellationCallKind::Parallel + && boundary.call_kind != call_kind) + { + return Err(invalid("scope boundary replaces an admitted operation")); + } + } + } + committed.preparations.insert( + id.to_owned(), + ScopeBoundary { + context, + boundary, + authority_deadline: deadline, + event: event.clone(), + }, + ); + continue; + } + let preparation = committed + .preparations + .get(id) + .ok_or_else(|| invalid("scope delivery lacks its original preparation"))?; + if p["schema"] != "durable-workflow.cancellation-scope-delivery/v1" + || delivered.contains(id) + || committed.deliveries.contains_key(&boundary.sequence) + || p["preparation_history_event_id"].as_str() != Some(event_id(&preparation.event)?) + || boundary != preparation.boundary + || deadline != preparation.authority_deadline + || time < event_time(&preparation.event)? + { + return Err(invalid( + "scope delivery changes its original prepared boundary", + )); + } + delivered.insert(id.to_owned()); + committed.deliveries.insert( + boundary.sequence, + ScopeBoundary { + context, + boundary, + authority_deadline: deadline, + event: event.clone(), + }, + ); + } + let mut covered = delivered; + for delivery in committed.deliveries.values() { + for member in committed.preparations[delivery.context.scope_id()] + .event + .payload["descendant_members"] + .as_array() + .unwrap() + { + covered.insert(text(member, "scope_id")?.to_owned()); + } + } + committed.pending_requests = requests + .into_iter() + .filter(|(scope, _)| !covered.contains(scope)) + .collect(); + Ok(committed) + } + + pub fn pending_request_for_scope<'a>( + &'a self, + scope: &str, + scopes: &cancellation_scope::CancellationScopeHistory, + ) -> Result> { + let addresses: BTreeMap<_, _> = scopes + .openings + .values() + .map(|opening| (opening.scope_id.as_str(), opening)) + .collect(); + let mut id = scope; + let active = self.pending_requests.get(scope); + let mut request = active; + while let Some(opening) = addresses.get(id).filter(|opening| !opening.shield_parent) { + id = &opening.parent_scope_id; + let Some(ancestor) = self.pending_requests.get(id) else { + continue; + }; + if active.is_none_or(|active| { + ancestor.context.root_context() != active.context.root_context() + || !active + .context + .lineage() + .starts_with(ancestor.context.lineage()) + }) { + return Err(invalid( + "pending scope selection changes its original ancestor lineage", + )); + } + request = Some(ancestor); + } + Ok(request) + } +} + +/// One original monotonic budget shared by scope preparation, delivery and proof. +#[doc(hidden)] +pub struct CancellationScopeDeliveryBudget { + expires_at: tokio::time::Instant, +} + +impl Default for CancellationScopeDeliveryBudget { + fn default() -> Self { + Self::new() + } +} + +impl CancellationScopeDeliveryBudget { + pub fn new() -> Self { + Self { + expires_at: tokio::time::Instant::now() + Duration::from_secs(5), + } + } + + /// Narrow the original budget, without granting a new request or lease. + pub fn restrict(&mut self, authority_deadline: DateTime) -> Result<()> { + let unix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| Error::Timeout)?; + let now = DateTime::::from_timestamp( + i64::try_from(unix.as_secs()).map_err(|_| Error::Timeout)?, + unix.subsec_nanos(), + ) + .ok_or(Error::Timeout)?; + let remaining = (authority_deadline - now) + .to_std() + .unwrap_or(Duration::ZERO); + self.expires_at = self.expires_at.min(tokio::time::Instant::now() + remaining); + self.remaining() + } + + fn remaining(&self) -> Result<()> { + if tokio::time::Instant::now() >= self.expires_at { + Err(Error::Timeout) + } else { + Ok(()) + } + } +} + +/// Proof of the original scope preparation and optional committed delivery. +/// This object does not grant callback execution or result publication authority. +#[doc(hidden)] +#[derive(Clone, Debug)] +pub struct CancellationScopeDeliveryReceipt { + pub(super) preparation: ScopeBoundary, + pub(super) delivery: Option, + history: Vec, +} + +impl CancellationScopeDeliveryReceipt { + pub fn context(&self) -> &ScopedCancellationContext { + &self.preparation.context + } + pub fn boundary(&self) -> &CancellationDelivery { + &self.preparation.boundary + } + pub fn authority_deadline(&self) -> DateTime { + self.preparation.authority_deadline + } + pub fn history(&self) -> &[HistoryEvent] { + &self.history + } + pub fn preparation_history_event_id(&self) -> &str { + event_id(&self.preparation.event).unwrap() + } + pub fn delivery_history_event_id(&self) -> Option<&str> { + self.delivery + .as_ref() + .map(|delivery| event_id(&delivery.event).unwrap()) + } + + fn acknowledge<'a>(receipt: &'a Value, expected: &Value, delivering: bool) -> Result<&'a str> { + if expected + .as_object() + .ok_or_else(|| invalid("missing original scope claim"))? + .iter() + .any(|(key, value)| { + !matches!(key.as_str(), "namespace" | "workflow_instance_id") + && receipt.get(key) != Some(value) + }) + || receipt["prepared"].as_bool() != Some(true) + || receipt["delivered"].as_bool() != Some(delivering) + || receipt["claim_released"].as_bool() != Some(false) + || receipt.get("created_task_ids") != Some(&json!([])) + || receipt.get("reason") != Some(&Value::Null) + || (text(receipt, "history_event_id")? + == text(receipt, "preparation_history_event_id")?) + == delivering + { + return Err(invalid( + "scope acknowledgement changes its original claim or retained preparation", + )); + } + for field in FIELDS { + normalize_members(field, &receipt[field])?; + } + let context = ScopedCancellationContext::from_value(&receipt["cancellation"])?; + let authority = timestamp(&receipt["authority_deadline_at"])?; + if context.workflow_run_id() != text(expected, "workflow_run_id")? + || context.workflow_instance_id() != text(expected, "workflow_instance_id")? + || context.scope_id() != text(expected, "scope_id")? + || context.request_id() != text(expected, "request_id")? + || authority < context.requested_at() + || authority > context.deadline() + { + return Err(invalid( + "scope acknowledgement changes its original cancellation authority", + )); + } + receipt["history_refresh_page_token"] + .as_str() + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| { + invalid("scope acknowledgement lacks an opaque canonical history cursor") + }) + } + + fn from_history( + receipt: &Value, + history: Vec, + expected: &Value, + delivering: bool, + ) -> Result { + Self::acknowledge(receipt, expected, delivering)?; + let mut starts = BTreeSet::new(); + for event in &history { + if event.raw.get("namespace") != expected.get("namespace") { + return Err(invalid( + "scope receipt history changes its original namespace", + )); + } + if matches!( + event.event_type.as_str(), + "StartAccepted" | "WorkflowStarted" + ) && (!starts.insert(event.event_type.as_str()) + || event.payload.get("workflow_run_id") != expected.get("workflow_run_id") + || event.payload.get("workflow_instance_id") + != expected.get("workflow_instance_id")) + { + return Err(invalid( + "scope receipt history changes its original workflow start", + )); + } + } + if !starts.contains("WorkflowStarted") { + return Err(invalid("scope receipt history omits its workflow start")); + } + let committed = CommittedCancellationScopeHistory::read( + &history, + text(expected, "workflow_run_id")?, + text(expected, "workflow_instance_id")?, + )?; + let preparation = committed + .preparations + .get(text(expected, "scope_id")?) + .ok_or_else(|| invalid("scope receipt lacks its original committed preparation"))? + .clone(); + let context = ScopedCancellationContext::from_value(&receipt["cancellation"])?; + let mut payload = receipt.clone(); + payload["workflow_command_id"] = expected["request_id"].clone(); + let boundary = CancellationDelivery::from_payload(&payload)?; + if event_id(&preparation.event)? != text(receipt, "preparation_history_event_id")? + || preparation.context != context + || preparation.boundary != boundary + || preparation.authority_deadline != timestamp(&receipt["authority_deadline_at"])? + { + return Err(invalid( + "scope receipt differs from its original committed preparation", + )); + } + for field in FIELDS { + if normalize_members(field, &preparation.event.payload[field])? + != normalize_members(field, &receipt[field])? + { + return Err(invalid( + "scope receipt changes its original frozen projection", + )); + } + } + let delivery = if delivering { + let delivery = committed + .deliveries + .get(&boundary.sequence) + .ok_or_else(|| invalid("scope receipt lacks its original committed delivery"))? + .clone(); + if event_id(&delivery.event)? != text(receipt, "history_event_id")? + || delivery.context != context + || delivery.boundary != boundary + { + return Err(invalid( + "scope receipt changes its original committed delivery", + )); + } + Some(delivery) + } else { + None + }; + Ok(Self { + preparation, + delivery, + history, + }) + } + + fn assert_original_preparation(&self, original: &Self) -> Result<()> { + let before = &original.preparation; + let after = &self.preparation; + if event_id(&before.event)? != event_id(&after.event)? + || before.context != after.context + || before.boundary != after.boundary + || before.authority_deadline != after.authority_deadline + || FIELDS + .into_iter() + .chain(["descendant_members"]) + .any(|field| before.event.payload[field] != after.event.payload[field]) + { + return Err(invalid( + "scope delivery substitutes its previously proved original preparation", + )); + } + Ok(()) + } +} + +impl Client { + /// Candidate scalar preparation proof. All operations share the supplied budget. + #[doc(hidden)] + pub async fn prepare_cancellation_scope_on_claim( + &self, + task: &WorkflowTask, + context: &ScopedCancellationContext, + boundary: &CancellationDelivery, + budget: &CancellationScopeDeliveryBudget, + ) -> Result { + budget.remaining()?; + let run = task.run_id.as_deref().unwrap_or_default(); + let committed = CommittedCancellationScopeHistory::read( + &task.history_events, + run, + task.workflow_id.as_deref().unwrap_or_default(), + )?; + let scopes = cancellation_scope::CancellationScopeHistory::read(&task.history_events, run)?; + let operation_scope = scopes + .memberships + .get(&boundary.sequence) + .map(String::as_str) + .unwrap_or(context.scope_id()); + if committed + .pending_request_for_scope(operation_scope, &scopes)? + .is_none_or(|request| &request.context != context) + { + return Err(invalid( + "scope preparation lacks its original pending request on this claim", + )); + } + self.cancellation_scope_boundary_on_claim(task, context, boundary, None, budget) + .await + } + + /// Candidate scalar delivery requires the earlier original preparation proof. + #[doc(hidden)] + pub async fn deliver_cancellation_scope_on_claim( + &self, + task: &WorkflowTask, + original: &CancellationScopeDeliveryReceipt, + budget: &CancellationScopeDeliveryBudget, + ) -> Result { + self.cancellation_scope_boundary_on_claim( + task, + original.context(), + original.boundary(), + Some(original), + budget, + ) + .await + } + + async fn cancellation_scope_boundary_on_claim( + &self, + task: &WorkflowTask, + context: &ScopedCancellationContext, + boundary: &CancellationDelivery, + original: Option<&CancellationScopeDeliveryReceipt>, + budget: &CancellationScopeDeliveryBudget, + ) -> Result { + let (owner, run) = cooperative_cancellation::cancellation_claim(task)?; + let workflow = task.workflow_id.as_deref().unwrap_or_default(); + if [ + task.task_id.as_str(), + owner, + run, + workflow, + context.scope_id(), + context.request_id(), + self.namespace.as_str(), + ] + .iter() + .any(|value| !identity(&json!(value))) + || context.scope_id() == "root" + || context.workflow_run_id() != run + || context.workflow_instance_id() != workflow + || boundary.request_id != context.request_id() + || boundary.sequence_span != 1 + || boundary.operation_sequence.is_some() + || boundary.operation_sequence_span != 1 + || !matches!( + boundary.call_kind, + CancellationCallKind::Activity + | CancellationCallKind::LocalActivity + | CancellationCallKind::Timer + | CancellationCallKind::Condition + | CancellationCallKind::Child + ) + { + return Err(invalid( + "scope boundary requires its original claim and scalar durable call", + )); + } + budget.remaining()?; + let body = json!({"scope_id":context.scope_id(), "request_id":context.request_id(), + "sequence":boundary.sequence, "call_kind":boundary.call_kind, "sequence_span":boundary.sequence_span, + "operation_sequence":boundary.operation_sequence, "operation_sequence_span":boundary.operation_sequence_span, + "lease_owner":owner, "workflow_task_attempt":task.workflow_task_attempt}); + let mut checked = body.clone(); + checked["workflow_command_id"] = json!(boundary.request_id); + CancellationDelivery::from_payload(&checked)?; + let mut expected = body.clone(); + expected["task_id"] = json!(task.task_id); + expected["workflow_run_id"] = json!(run); + expected["workflow_instance_id"] = json!(workflow); + expected["namespace"] = json!(self.namespace); + let path = format!( + "/worker/workflow-tasks/{}", + percent_encode_path_segment(&task.task_id) + ); + let delivering = original.is_some(); + tokio::time::timeout_at(budget.expires_at, async { + let boundary_path = format!("{path}/cancellation-scopes/{}", if delivering { "deliver" } else { "prepare" }); + let request = || self.request_json::(reqwest::Method::POST, &boundary_path, + RequestProtocol::Worker("1.20"), Some(&body)); + let receipt = match request().await { + Err(error) if worker_operation_is_retryable(&error) => request().await?, + result => result?, + }; + let mut token = Some(CancellationScopeDeliveryReceipt::acknowledge(&receipt, &expected, delivering)?.to_owned()); + let mut seen = BTreeSet::new(); + let mut history = Vec::new(); + while let Some(current) = token.take() { + budget.remaining()?; + if seen.len() >= 128 || !seen.insert(current.clone()) { return Err(invalid("scope history repeated or exceeded its opaque cursors")); } + let page: Value = self.request_json(reqwest::Method::POST, &format!("{path}/history"), + RequestProtocol::Worker("1.20"), Some(&json!({"lease_owner":owner, + "workflow_task_attempt":task.workflow_task_attempt, "next_history_page_token":current, + "history_page_size":WORKFLOW_HISTORY_PAGE_SIZE}))).await?; + if page["task_id"].as_str() != Some(task.task_id.as_str()) + || page["workflow_task_attempt"].as_u64() != Some(task.workflow_task_attempt) { + return Err(invalid("scope history page changes its original claim")); + } + let batch = page["history_events"].as_array().ok_or_else(|| invalid("scope history page lacks canonical events"))?; + if batch.len() > WORKFLOW_HISTORY_PAGE_SIZE as usize || batch.iter().any(|event| !event.is_object()) { + return Err(invalid("scope history page exceeds its bounded complete shape")); + } + token = match page.get("next_history_page_token") { + Some(Value::Null) => None, + Some(Value::String(next)) if !next.trim().is_empty() && !batch.is_empty() => Some(next.clone()), + _ => return Err(invalid("scope history page omits its terminal cursor")), + }; + for event in batch { history.push(serde_json::from_value(event.clone()).map_err(|_| invalid("scope history event is malformed"))?); } + } + let proved = CancellationScopeDeliveryReceipt::from_history(&receipt, history, &expected, delivering)?; + if let Some(original) = original { proved.assert_original_preparation(original)?; } + budget.remaining()?; + Ok(proved) + }).await.map_err(|_| Error::Timeout)? + } +} diff --git a/src/lib.rs b/src/lib.rs index 6c8e429..eef574c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,7 @@ mod cancellation_context; mod cancellation_replay_clock; mod cancellation_scope; +mod cancellation_scope_history; mod cooperative_cancellation; mod runtime_payloads; mod runtime_uploads; @@ -11,6 +12,10 @@ pub use cancellation_context::{ CancellationContext, CancellationLineage, ScopedCancellationContext, ScopedCancellationLineage, }; pub use cancellation_scope::CancellationScopeOpenReceipt; +#[doc(hidden)] +pub use cancellation_scope_history::{ + CancellationScopeDeliveryBudget, CancellationScopeDeliveryReceipt, +}; pub use cooperative_cancellation::{ CancellationCallKind, CancellationDelivery, CancellationDeliveryReceipt, @@ -16354,6 +16359,7 @@ mod tests { mod activity_cancellation_policies; mod cancellation_scope_admission; mod cancellation_scope_authoring; + mod cancellation_scope_history; mod cancellation_scope_opening; mod child_workflow_policies; mod cooperative_cancellation; diff --git a/src/tests/cancellation_scope_history.rs b/src/tests/cancellation_scope_history.rs new file mode 100644 index 0000000..77f15d7 --- /dev/null +++ b/src/tests/cancellation_scope_history.rs @@ -0,0 +1,744 @@ +use super::*; +use crate::cancellation_scope_history::CommittedCancellationScopeHistory; + +fn scalar_fixture() -> Value { + fixture("empty", "unshielded") +} + +fn boundary(value: &Value) -> (ScopedCancellationContext, CancellationDelivery) { + let committed = read(value).unwrap(); + let prepared = committed.preparations.values().next().unwrap(); + (prepared.context.clone(), prepared.boundary.clone()) +} + +fn claim(value: &Value) -> WorkflowTask { + let mut events = value["history"].as_array().unwrap().clone(); + let position = events + .iter() + .position(|row| row["event_type"] == "CancellationScopeDeliveryPrepared") + .unwrap(); + events.truncate(position); + serde_json::from_value(json!({"task_id":"task/one", "run_id":value["task"]["run_id"], + "workflow_id":value["task"]["workflow_id"], "workflow_type":"scope", "lease_owner":"original", + "workflow_task_attempt":4, "payload_codec":DEFAULT_CODEC, "history_events":events})).unwrap() +} + +fn scope_response(path: &str, body: &str, number: usize) -> Option<(&'static str, String)> { + let case = path.split('/').nth(1).unwrap_or_default(); + if case == "budget" { + thread::sleep(Duration::from_millis(900)); + } + if case == "lost-ack" && path.ends_with("/prepare") && number == 1 { + return Some(("invalid-status", String::new())); + } + let mut source = scalar_fixture(); + let delivering = + path.ends_with("/deliver") || (path.ends_with("/history") && body.contains("deliver-")); + if case == "substituted" && delivering { + event_mut(&mut source, "CancellationScopeDeliveryPrepared")["id"] = + json!("borrowed-preparation"); + event_mut(&mut source, "CancellationScopeDelivered")["payload"] + ["preparation_history_event_id"] = json!("borrowed-preparation"); + } + let mut prepared = event_mut(&mut source, "CancellationScopeDeliveryPrepared").clone(); + let delivered = event_mut(&mut source, "CancellationScopeDelivered").clone(); + let mut result; + if path.ends_with("/prepare") || path.ends_with("/deliver") { + result = prepared["payload"].clone(); + result["task_id"] = json!("task/one"); + result["lease_owner"] = json!("original"); + result["workflow_task_attempt"] = json!(4); + result["prepared"] = json!(true); + result["delivered"] = json!(delivering); + result["claim_released"] = json!(false); + result["created_task_ids"] = json!([]); + result["reason"] = Value::Null; + result["preparation_history_event_id"] = prepared["id"].clone(); + result["history_event_id"] = if delivering { + delivered["id"].clone() + } else { + prepared["id"].clone() + }; + result["history_refresh_page_token"] = json!(if delivering { + "deliver-start" + } else { + "prepare-start" + }); + match case { + "ack-owner" => result["lease_owner"] = json!("replacement"), + "ack-attempt" => result["workflow_task_attempt"] = json!(true), + "ack-run" => result["workflow_run_id"] = json!("foreign"), + "ack-boundary" => result["sequence"] = json!(99), + "ack-range" => result["sequence_span"] = json!(2), + "ack-prepared" => result["prepared"] = json!(1), + "ack-delivered" => result["delivered"] = json!(!delivering), + "ack-released" => result["claim_released"] = json!(true), + "ack-new-task" => result["created_task_ids"] = json!(["new"]), + "ack-deadline" => { + result["authority_deadline_at"] = json!("2026-10-04T00:00:31.123456Z") + } + "ack-token" => result["history_refresh_page_token"] = Value::Null, + "ack-event" => result["history_event_id"] = json!(""), + "ack-reason" => { + result.as_object_mut().unwrap().remove("reason"); + } + _ => {} + } + } else if path.ends_with("/history") { + let events = source["history"].as_array_mut().unwrap(); + if !delivering { + events.pop(); + } + match case { + "history-namespace" => events[0]["namespace"] = json!("foreign"), + "history-start" => events[0]["payload"]["workflow_instance_id"] = json!("foreign"), + "history-context" => { + events[4]["payload"]["cancellation"]["root_context"]["reason"] = json!("changed") + } + "history-range" => { + prepared["payload"] + .as_object_mut() + .unwrap() + .remove("operation_sequence_span"); + *events.last_mut().unwrap() = prepared; + } + "history-preparation" => { + events.pop(); + } + _ => {} + } + let request: Value = serde_json::from_str(body).unwrap(); + let first = request["next_history_page_token"] + .as_str() + .unwrap() + .ends_with("start"); + let prefix = if delivering { "deliver" } else { "prepare" }; + result = json!({"task_id":"task/one", "workflow_task_attempt":4, + "history_events": if first { events.iter().take(2).cloned().collect::>() } + else { events.iter().skip(2).cloned().collect::>() }, + "next_history_page_token": if first { json!(format!("{prefix}-next")) } else { Value::Null }}); + match case { + "page-task" => result["task_id"] = json!("foreign"), + "page-attempt" => result["workflow_task_attempt"] = json!(5), + "page-cycle" => { + result["next_history_page_token"] = request["next_history_page_token"].clone() + } + "page-terminal" => { + result + .as_object_mut() + .unwrap() + .remove("next_history_page_token"); + } + _ => {} + } + } else { + return None; + } + Some(("200 OK", result.to_string())) +} + +fn scope_server() -> MockWorkerServer { + MockWorkerServer::start_with_behavior(MockWorkerBehavior { + request_override: Some(scope_response), + ..MockWorkerBehavior::default() + }) +} + +fn scope_client(server: &MockWorkerServer, case: &str) -> Client { + Client::builder(format!("{}/{case}", server.base_url())) + .namespace("sdk-scope-fixture") + .control_token(Some("control-only".into())) + .worker_token(Some("worker-only".into())) + .build() + .unwrap() +} + +#[tokio::test] +async fn scope_history_prepare_and_delivery_prove_original_claim_and_all_pages_after_lost_ack() { + for case in ["valid", "lost-ack"] { + let server = scope_server(); + let client = scope_client(&server, case); + let value = scalar_fixture(); + let task = claim(&value); + let (context, boundary) = boundary(&value); + let budget = CancellationScopeDeliveryBudget::new(); + let prepared = client + .prepare_cancellation_scope_on_claim(&task, &context, &boundary, &budget) + .await + .unwrap(); + assert!(prepared.delivery_history_event_id().is_none()); + let delivered = client + .deliver_cancellation_scope_on_claim(&task, &prepared, &budget) + .await + .unwrap(); + assert_eq!( + prepared.preparation_history_event_id(), + delivered.preparation_history_event_id() + ); + assert_ne!( + delivered.delivery_history_event_id().unwrap(), + delivered.preparation_history_event_id() + ); + assert_eq!(delivered.context(), &context); + assert_eq!(delivered.boundary(), &boundary); + assert_eq!( + delivered.history().len(), + value["history"].as_array().unwrap().len() + ); + let requests = server.requests.lock().unwrap(); + let preparations: Vec<_> = requests + .iter() + .filter(|row| row.path.ends_with("/prepare")) + .collect(); + assert_eq!(preparations.len(), if case == "lost-ack" { 2 } else { 1 }); + if case == "lost-ack" { + assert_eq!(preparations[0].body, preparations[1].body); + } + for request in requests.iter() { + assert_eq!(request.worker_protocol.as_deref(), Some("1.20")); + assert_eq!(request.authorization.as_deref(), Some("Bearer worker-only")); + assert_eq!(request.namespace.as_deref(), Some("sdk-scope-fixture")); + assert!(request.path.contains("/task%2Fone/")); + let body: Value = serde_json::from_str(&request.body).unwrap(); + assert_eq!(body["lease_owner"], "original"); + assert_eq!(body["workflow_task_attempt"], 4); + } + } +} + +#[tokio::test] +async fn scope_history_changed_receipts_refuse_before_history_reads() { + for case in [ + "ack-owner", + "ack-attempt", + "ack-run", + "ack-boundary", + "ack-range", + "ack-prepared", + "ack-delivered", + "ack-released", + "ack-new-task", + "ack-deadline", + "ack-token", + "ack-event", + "ack-reason", + ] { + let server = scope_server(); + let value = scalar_fixture(); + let (context, boundary) = boundary(&value); + assert!( + scope_client(&server, case) + .prepare_cancellation_scope_on_claim( + &claim(&value), + &context, + &boundary, + &CancellationScopeDeliveryBudget::new() + ) + .await + .is_err(), + "{case}" + ); + assert_eq!(server.requests.lock().unwrap().len(), 1, "{case}"); + } +} + +#[tokio::test] +async fn scope_history_incomplete_or_changed_canonical_pages_refuse_delivery_authority() { + for case in [ + "history-namespace", + "history-start", + "history-context", + "history-range", + "history-preparation", + "page-task", + "page-attempt", + "page-cycle", + "page-terminal", + ] { + let server = scope_server(); + let value = scalar_fixture(); + let (context, boundary) = boundary(&value); + assert!( + scope_client(&server, case) + .prepare_cancellation_scope_on_claim( + &claim(&value), + &context, + &boundary, + &CancellationScopeDeliveryBudget::new() + ) + .await + .is_err(), + "{case}" + ); + assert!(server + .requests + .lock() + .unwrap() + .iter() + .all(|request| !request.path.ends_with("/deliver"))); + } +} + +#[tokio::test] +async fn scope_history_delivery_cannot_substitute_a_different_valid_preparation() { + let server = scope_server(); + let client = scope_client(&server, "substituted"); + let value = scalar_fixture(); + let task = claim(&value); + let (context, boundary) = boundary(&value); + let budget = CancellationScopeDeliveryBudget::new(); + let prepared = client + .prepare_cancellation_scope_on_claim(&task, &context, &boundary, &budget) + .await + .unwrap(); + assert!(client + .deliver_cancellation_scope_on_claim(&task, &prepared, &budget) + .await + .is_err()); +} + +#[tokio::test] +async fn scope_history_expired_budget_or_borrowed_claim_perform_no_io() { + let server = scope_server(); + let client = scope_client(&server, "valid"); + let value = scalar_fixture(); + let (context, boundary) = boundary(&value); + let mut budget = CancellationScopeDeliveryBudget::new(); + assert!(budget + .restrict(DateTime::::from_timestamp(0, 0).unwrap()) + .is_err()); + assert!(matches!( + client + .prepare_cancellation_scope_on_claim(&claim(&value), &context, &boundary, &budget) + .await, + Err(Error::Timeout) + )); + let mut foreign = claim(&value); + foreign.run_id = Some("borrowed-run".into()); + assert!(client + .prepare_cancellation_scope_on_claim( + &foreign, + &context, + &boundary, + &CancellationScopeDeliveryBudget::new() + ) + .await + .is_err()); + assert!(server.requests.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn scope_history_preparation_delivery_and_history_share_one_original_budget() { + let server = scope_server(); + let client = scope_client(&server, "budget"); + let value = scalar_fixture(); + let task = claim(&value); + let (context, boundary) = boundary(&value); + let budget = CancellationScopeDeliveryBudget::new(); + let started = Instant::now(); + let prepared = client + .prepare_cancellation_scope_on_claim(&task, &context, &boundary, &budget) + .await + .unwrap(); + assert!(matches!( + client + .deliver_cancellation_scope_on_claim(&task, &prepared, &budget) + .await, + Err(Error::Timeout) + )); + assert!(started.elapsed() < Duration::from_millis(5600)); +} + +fn fixture(name: &str, variant: &str) -> Value { + let source = match name { + "operations" => { + include_str!("../../tests/fixtures/committed-scope-operation-projections.json") + } + "empty" => include_str!("../../tests/fixtures/committed-scope-delivery.json"), + "single" => include_str!("../../tests/fixtures/populated-scope-single-calls.json"), + "root" => include_str!("../../tests/fixtures/run-inherited-scope-delivery.json"), + _ => panic!("unknown fixture"), + }; + let value: Value = serde_json::from_str(source).unwrap(); + if variant.is_empty() { + value + } else { + value[variant].clone() + } +} + +fn history(value: &Value) -> Vec { + serde_json::from_value(value["history"].clone()).unwrap() +} + +fn read(value: &Value) -> Result { + CommittedCancellationScopeHistory::read( + &history(value), + value["task"]["run_id"].as_str().unwrap(), + value["task"]["workflow_id"].as_str().unwrap(), + ) +} + +fn event_mut<'a>(value: &'a mut Value, kind: &str) -> &'a mut Value { + value["history"] + .as_array_mut() + .unwrap() + .iter_mut() + .find(|event| event["event_type"] == kind) + .unwrap() +} + +fn renumber(value: &mut Value) { + for (index, event) in value["history"] + .as_array_mut() + .unwrap() + .iter_mut() + .enumerate() + { + event["sequence"] = json!(index + 1); + } +} + +#[test] +fn scope_history_native_v5_operations_groups_descendants_and_competing_roots() { + for variant in ["operations", "groups", "descendants", "competing"] { + let value = fixture("operations", variant); + let committed = read(&value).unwrap_or_else(|error| panic!("{variant}: {error:?}")); + let preparation = &committed.preparations[value["scope_id"].as_str().unwrap()]; + let delivery = &committed.deliveries[&preparation.boundary.sequence]; + assert_eq!(preparation.context, delivery.context); + assert_eq!(preparation.boundary, delivery.boundary); + assert_eq!(preparation.authority_deadline, delivery.authority_deadline); + assert!(committed.pending_requests.is_empty()); + let projection = &preparation.event.payload; + assert_eq!( + projection["timer_members"] + .as_array() + .unwrap() + .iter() + .map(|row| row["timer_id"].as_str().unwrap()) + .collect::>(), + ["timer-plain", "signal-timer-id", "condition-timer-id"] + ); + assert_eq!( + projection["child_members"][1]["child_workflow_run_id"], + "child-continued-run" + ); + assert_eq!( + projection["child_members"] + .as_array() + .unwrap() + .iter() + .map(|row| row["cancellation_policy"].as_str().unwrap()) + .collect::>(), + ["try_cancel", "wait_cancellation_completed", "abandon"] + ); + if matches!(variant, "descendants" | "competing") { + assert_eq!( + projection["descendant_members"][0]["scope_id"], + "desc-child" + ); + assert_eq!( + projection["descendant_members"][1]["scope_id"], + "desc-grandchild" + ); + assert_eq!( + projection["descendant_members"][0]["propagation_history_event_id"], + if variant == "competing" { + "original-child-conflict" + } else { + "inherited-child-accepted" + } + ); + assert_eq!( + projection["descendant_members"][0]["authority_deadline_at"], + if variant == "competing" { + "2026-10-04T00:00:20.123456Z" + } else { + "2026-10-04T00:00:30.123456Z" + } + ); + } else { + assert_eq!(projection["descendant_members"], json!([])); + } + } +} + +#[test] +fn scope_history_empty_delivery_and_populated_scalar_policies() { + for variant in ["shielded", "unshielded"] { + let committed = read(&fixture("empty", variant)).unwrap(); + assert_eq!(committed.deliveries.len(), 1); + let delivery = committed.deliveries.values().next().unwrap(); + assert_eq!(delivery.boundary.call_kind, CancellationCallKind::Timer); + assert_eq!( + delivery.context.request_id(), + delivery.context.root_context().request_id() + ); + assert_eq!(delivery.authority_deadline, delivery.context.deadline()); + } + for variant in [ + "activity-try_cancel", + "activity-wait_cancellation_completed", + "activity-abandon", + "child-try_cancel", + "child-wait_cancellation_completed", + "child-abandon", + "timer", + "condition", + "condition-untimed", + ] { + let committed = read(&fixture("single", variant)) + .unwrap_or_else(|error| panic!("{variant}: {error:?}")); + let preparation = committed.preparations.values().next().unwrap(); + let delivery = &committed.deliveries[&preparation.boundary.sequence]; + assert_eq!(delivery.boundary.sequence, 4); + assert_eq!(delivery.context, preparation.context); + assert!(committed.pending_requests.is_empty()); + assert!(!preparation.event.payload["activity_members"] + .as_array() + .unwrap() + .is_empty()); + } +} + +#[test] +fn scope_history_changed_frozen_members_and_descriptors_are_refused() { + for variant in ["operations", "groups", "descendants", "competing"] { + for field in [ + "activity_members", + "timer_members", + "wait_members", + "child_members", + ] { + for failure in ["hash", "omitted", "boolean", "extra"] { + let mut value = fixture("operations", variant); + let members = event_mut(&mut value, "CancellationScopeDeliveryPrepared")["payload"] + [field] + .as_array_mut() + .unwrap(); + if members.is_empty() { + members.push(json!({})); + assert!(read(&value).is_err(), "{variant}/{field}/invented"); + continue; + } + match failure { + "hash" => members[0]["descriptor_hash"] = json!("0".repeat(64)), + "omitted" => { + members.remove(0); + } + "boolean" => members[0]["sequence"] = json!(true), + "extra" => members[0]["unexpected"] = json!("borrowed"), + _ => unreachable!(), + } + assert!(read(&value).is_err(), "{variant}/{field}/{failure}"); + } + } + } +} + +#[test] +fn scope_history_changed_request_boundary_deadline_ancestry_and_continuation_are_refused() { + for failure in [ + "request", + "run", + "instance", + "deadline", + "boundary", + "preparation", + "before_preparation", + "missing_request", + "duplicate_request", + "duplicate_delivery", + "future_member", + "continuation", + "descendant_deadline", + "descendant_propagation", + "cross_shield", + "inherited_deadline", + ] { + let mut value = fixture("operations", "descendants"); + match failure { + "request" => { + event_mut(&mut value, "CancellationScopeDelivered")["payload"]["request_id"] = + json!("new-request") + } + "run" => { + event_mut(&mut value, "CancellationScopeDeliveryPrepared")["payload"] + ["workflow_run_id"] = json!("new-run") + } + "instance" => { + event_mut(&mut value, "CancellationScopeDelivered")["payload"]["cancellation"] + ["lineage"][0]["workflow_instance_id"] = json!("new-instance") + } + "deadline" => { + event_mut(&mut value, "CancellationScopeDelivered")["payload"] + ["authority_deadline_at"] = json!("2026-10-04T00:00:31.123456Z") + } + "boundary" => { + event_mut(&mut value, "CancellationScopeDelivered")["payload"]["sequence"] = + json!(999) + } + "preparation" => { + event_mut(&mut value, "CancellationScopeDelivered")["payload"] + ["preparation_history_event_id"] = json!("new-preparation") + } + "descendant_deadline" => { + event_mut(&mut value, "CancellationScopeDeliveryPrepared")["payload"] + ["descendant_members"][0]["authority_deadline_at"] = + json!("2026-10-04T00:00:31.123456Z") + } + "descendant_propagation" => { + event_mut(&mut value, "CancellationScopeDeliveryPrepared")["payload"] + ["descendant_members"][0]["propagation_history_event_id"] = + json!("new-propagation") + } + "continuation" | "cross_shield" | "inherited_deadline" => { + let id = match failure { + "continuation" => "child-continued-before-preparation", + "cross_shield" => "desc-child-opened", + _ => "inherited-child-accepted", + }; + let event = value["history"] + .as_array_mut() + .unwrap() + .iter_mut() + .find(|row| row["id"] == id) + .unwrap(); + match failure { + "continuation" => { + event["payload"]["child_workflow_run_id"] = json!("new-child-run") + } + "cross_shield" => event["payload"]["shield_parent"] = json!(true), + _ => { + event["payload"]["cancellation"]["lineage"][1]["cleanup_deadline_at"] = + json!("2026-10-04T00:00:29.123456Z") + } + } + } + _ => { + let events = value["history"].as_array_mut().unwrap(); + let target = match failure { + "missing_request" | "duplicate_request" => "CancellationScopeRequested", + "future_member" => "TimerScheduled", + _ => "CancellationScopeDelivered", + }; + let index = events + .iter() + .position(|row| row["event_type"] == target) + .unwrap(); + if failure.starts_with("duplicate") { + let mut duplicate = events[index].clone(); + duplicate["id"] = + json!(format!("{}-duplicate", duplicate["id"].as_str().unwrap())); + events.insert(index + 1, duplicate); + } else { + let event = events.remove(index); + if failure == "before_preparation" { + let index = events + .iter() + .position(|row| { + row["event_type"] == "CancellationScopeDeliveryPrepared" + }) + .unwrap(); + events.insert(index, event); + } else if failure == "future_member" { + events.push(event); + } + } + } + } + renumber(&mut value); + assert!(read(&value).is_err(), "{failure}"); + } +} + +#[test] +fn scope_history_pending_ancestor_and_original_preparation_survive_replacement() { + let mut value = fixture("operations", "descendants"); + value["history"].as_array_mut().unwrap().pop(); + let committed = read(&value).unwrap(); + let scopes = cancellation_scope::CancellationScopeHistory::read( + &history(&value), + value["task"]["run_id"].as_str().unwrap(), + ) + .unwrap(); + let request = committed + .pending_request_for_scope("desc-grandchild", &scopes) + .unwrap() + .unwrap(); + assert_eq!( + request.context.scope_id(), + value["scope_id"].as_str().unwrap() + ); + let preparation = &committed.preparations[request.context.scope_id()]; + value["task"]["lease_owner"] = json!("replacement"); + value["task"]["workflow_task_attempt"] = json!(17); + let replayed = read(&value).unwrap(); + assert_eq!( + replayed.preparations[request.context.scope_id()].context, + preparation.context + ); + assert_eq!( + replayed.preparations[request.context.scope_id()].boundary, + preparation.boundary + ); + assert_eq!( + replayed.preparations[request.context.scope_id()].authority_deadline, + preparation.authority_deadline + ); +} + +#[test] +fn scope_history_run_inheritance_preserves_original_parent_and_shielding() { + for failure in [ + "valid", "missing", "later", "shield", "metadata", "deadline", + ] { + let mut value = fixture("root", ""); + value["history"].as_array_mut().unwrap().truncate(5); + match failure { + "missing" => { + value["history"].as_array_mut().unwrap().remove(3); + } + "later" => value["history"].as_array_mut().unwrap().swap(3, 4), + "shield" => value["history"][2]["payload"]["shield_parent"] = json!(true), + "metadata" => { + value["history"][4]["payload"]["cancellation"]["root_context"]["reason"] = + json!("changed") + } + "deadline" => { + value["history"][4]["payload"]["cancellation"]["lineage"][1] + ["cleanup_deadline_at"] = json!("2026-10-05T00:00:25.000000Z") + } + _ => {} + } + renumber(&mut value); + let result = read(&value); + if failure != "valid" { + assert!(result.is_err(), "{failure}"); + continue; + } + let committed = result.unwrap(); + let id = value["history"][2]["payload"]["scope_id"].as_str().unwrap(); + let request = &committed.pending_requests[id]; + assert_eq!( + request.context.request_id(), + value["history"][4]["payload"]["request_id"] + .as_str() + .unwrap() + ); + assert_eq!( + request.context.parent_request_id(), + value["history"][3]["payload"]["workflow_command_id"].as_str() + ); + assert_eq!( + request + .context + .lineage() + .iter() + .map(ScopedCancellationLineage::scope_id) + .collect::>(), + ["root", id] + ); + } +} diff --git a/tests/fixtures/committed-scope-delivery.json b/tests/fixtures/committed-scope-delivery.json new file mode 100644 index 0000000..b58f412 --- /dev/null +++ b/tests/fixtures/committed-scope-delivery.json @@ -0,0 +1,695 @@ +{ + "native_commit": "6d484654c7a966770cd6bbce90d9007a1c75e0ba", + "unshielded": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "outer_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 5, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 6, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 3, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:05.250000Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 7, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 3, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:07.500000Z" + } + ] + }, + "shielded": { + "task": { + "workflow_id": "scope-fixture-shielded", + "run_id": "01m43xf6c15h003132br5fcabx" + }, + "outer_scope_id": "01M43XF6CN7YF8HKCNJ5RQX0CN", + "scope_id": "01M43XF6CSSPQNT6VMA6WKG7Z6", + "history": [ + { + "id": "01m43xf6c4f0n5d26xy0j05w1s", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf6c2avkvv3aybv47qtvp", + "workflow_instance_id": "scope-fixture-shielded", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf6c2avkvv3aybv47qtvp", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf6c15h003132br5fcabx", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6c5gcfdww4t1z91nzhx", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-shielded", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "workflow_command_id": "01m43xf6c2avkvv3aybv47qtvp", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf6c2avkvv3aybv47qtvp", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf6c15h003132br5fcabx", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6cpajmgwqsrxxcvsv7s", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "sequence": 1, + "scope_id": "01M43XF6CN7YF8HKCNJ5RQX0CN", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf6c6ctyp5qgqscpch981", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6ctdkeyg4bxp8shzkc4", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "sequence": 2, + "scope_id": "01M43XF6CSSPQNT6VMA6WKG7Z6", + "parent_scope_id": "01M43XF6CN7YF8HKCNJ5RQX0CN", + "shield_parent": true, + "task": { + "id": "01m43xf6c6ctyp5qgqscpch981", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6czvbndcpec0kab3nwb", + "namespace": "sdk-scope-fixture", + "sequence": 5, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "scope_id": "01M43XF6CSSPQNT6VMA6WKG7Z6", + "parent_scope_id": null, + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "root_request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "root_workflow_instance_id": "scope-fixture-shielded", + "root_workflow_run_id": "01m43xf6c15h003132br5fcabx", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "workflow_instance_id": "scope-fixture-shielded", + "workflow_run_id": "01m43xf6c15h003132br5fcabx" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "workflow_instance_id": "scope-fixture-shielded", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "scope_id": "01M43XF6CSSPQNT6VMA6WKG7Z6", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6dg25ypwscmh8dfft10", + "namespace": "sdk-scope-fixture", + "sequence": 6, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "scope_id": "01M43XF6CSSPQNT6VMA6WKG7Z6", + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "root_request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "root_workflow_instance_id": "scope-fixture-shielded", + "root_workflow_run_id": "01m43xf6c15h003132br5fcabx", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "workflow_instance_id": "scope-fixture-shielded", + "workflow_run_id": "01m43xf6c15h003132br5fcabx" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "workflow_instance_id": "scope-fixture-shielded", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "scope_id": "01M43XF6CSSPQNT6VMA6WKG7Z6", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 3, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m43xf6c6ctyp5qgqscpch981", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:05.250000Z" + }, + { + "id": "01m43xf6eb0t411jspfy8yv3n1", + "namespace": "sdk-scope-fixture", + "sequence": 7, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "scope_id": "01M43XF6CSSPQNT6VMA6WKG7Z6", + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "root_request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "root_workflow_instance_id": "scope-fixture-shielded", + "root_workflow_run_id": "01m43xf6c15h003132br5fcabx", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "workflow_instance_id": "scope-fixture-shielded", + "workflow_run_id": "01m43xf6c15h003132br5fcabx" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6CWQ3HHM4QRJVQ0CNKJ", + "workflow_instance_id": "scope-fixture-shielded", + "workflow_run_id": "01m43xf6c15h003132br5fcabx", + "scope_id": "01M43XF6CSSPQNT6VMA6WKG7Z6", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 3, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6dg25ypwscmh8dfft10", + "task": { + "id": "01m43xf6c6ctyp5qgqscpch981", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:07.500000Z" + } + ] + } +} diff --git a/tests/fixtures/committed-scope-operation-projections.json b/tests/fixtures/committed-scope-operation-projections.json new file mode 100644 index 0000000..86bda4b --- /dev/null +++ b/tests/fixtures/committed-scope-operation-projections.json @@ -0,0 +1,3285 @@ +{ + "_provenance": { + "native_commit": "0471f1ebbb98c61fe53663086d62dc2db74db9e3", + "input_fixture_sha256": "85320e25c7c19521bca4a1cff138911df1f19f76471ea033e961695e8ed761be", + "producers": [ + "CancellationScopeDelivery::activityMembers", + "ScopedTimerCancellation::members", + "ScopedWaitCancellation::members", + "ScopedChildCancellation::members", + "CancellationScopeDescendants::members" + ], + "history_store": "isolated SQLite read-only projection inputs", + "effect_authority": false + }, + "operations": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "plain-timer", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 10, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "timer-plain", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:00.123456Z" + } + }, + { + "id": "signal-wait", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "SignalWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "signal_wait_id": "signal-wait-id", + "timeout_seconds": 30, + "signal_name": "inbox", + "condition_key": null, + "condition_definition_fingerprint": null, + "condition_wait_occurrence_id": null + } + }, + { + "id": "signal-timer", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "signal-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z", + "timer_kind": "signal_timeout", + "signal_wait_id": "signal-wait-id", + "condition_wait_occurrence_id": null + } + }, + { + "id": "condition-wait", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 12, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "condition-wait-id", + "timeout_seconds": 30, + "signal_name": null, + "condition_key": "ready", + "condition_definition_fingerprint": "predicate-v1", + "condition_wait_occurrence_id": "occurrence-1" + } + }, + { + "id": "condition-timer", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 12, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "condition-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z", + "timer_kind": "condition_timeout", + "condition_wait_id": "condition-wait-id", + "condition_wait_occurrence_id": "occurrence-1" + } + }, + { + "id": "wait-no-timeout", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 13, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "wait-no-timeout-id", + "condition_key": "done" + } + }, + { + "id": "child-0", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 14, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-0", + "child_workflow_instance_id": "child-instance-0", + "child_workflow_run_id": "child-run-0", + "cancellation_policy": "try_cancel", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child" + } + }, + { + "id": "child-1", + "sequence": 12, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 15, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-run-1", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child" + } + }, + { + "id": "child-2", + "sequence": 13, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 16, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-2", + "child_workflow_instance_id": "child-instance-2", + "child_workflow_run_id": "child-run-2", + "cancellation_policy": "abandon", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child" + } + }, + { + "id": "child-continued-before-preparation", + "sequence": 14, + "namespace": "sdk-scope-fixture", + "event_type": "ChildRunStarted", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 15, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-continued-run", + "cancellation_policy": "wait_cancellation_completed" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 15, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 16, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 20, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [], + "timer_members": [ + { + "sequence": 10, + "timer_id": "timer-plain", + "descriptor_hash": "d78a4b0d2c2f326857a31358e3ddfa20bd567e2364315ca30cef3cb11eae6c0f" + }, + { + "sequence": 11, + "timer_id": "signal-timer-id", + "descriptor_hash": "cb7fd119a14082cae54f6f418beb5769a3506cc5769df7ac10318017b1360b70" + }, + { + "sequence": 12, + "timer_id": "condition-timer-id", + "descriptor_hash": "167249f4220f18c56597a3a14165aa464443b4e71e7e28ffd708f271b6f0460e" + } + ], + "wait_members": [ + { + "kind": "signal", + "sequence": 11, + "wait_id": "signal-wait-id", + "timer_id": "signal-timer-id", + "descriptor_hash": "68a60d453bbc004c3c17ca4e33f95bbad92e24508e680152c1140be5f247a63f" + }, + { + "kind": "condition", + "sequence": 12, + "wait_id": "condition-wait-id", + "timer_id": "condition-timer-id", + "descriptor_hash": "e8f0ce4c1242168f2a97286c5ceea4bb9458822aedc8301734226b139510e960" + }, + { + "kind": "condition", + "sequence": 13, + "wait_id": "wait-no-timeout-id", + "timer_id": null, + "descriptor_hash": "b462fc9baefdd0f5b792ed8f41177d3a97a3d4bce08a58f04782a31215d4f6f6" + } + ], + "child_members": [ + { + "sequence": 14, + "child_call_id": "child-call-0", + "child_workflow_instance_id": "child-instance-0", + "child_workflow_run_id": "child-run-0", + "cancellation_policy": "try_cancel", + "descriptor_hash": "19c48c0ad10c781d4c08900a51fa1d98e4606bc7b65556b12d705d3f49367d07" + }, + { + "sequence": 15, + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-continued-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "0d65b5082c06e33b7db952b1d61085a344a4b97b60d982998a4fba6b6e6748ef" + }, + { + "sequence": 16, + "child_call_id": "child-call-2", + "child_workflow_instance_id": "child-instance-2", + "child_workflow_run_id": "child-run-2", + "cancellation_policy": "abandon", + "descriptor_hash": "a6a5e39e958de9cf34a656dc01f417209dae2a5787ab4ee0e169c9e3f08421bd" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 17, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 20, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "groups": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "plain-timer", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 10, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "timer-plain", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:00.123456Z", + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:10:10", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 10, + "parallel_group_size": 10, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ] + } + }, + { + "id": "signal-wait", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "SignalWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "signal_wait_id": "signal-wait-id", + "timeout_seconds": 30, + "signal_name": "inbox", + "condition_key": null, + "condition_definition_fingerprint": null, + "condition_wait_occurrence_id": null, + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:10:10", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 10, + "parallel_group_size": 10, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ] + } + }, + { + "id": "signal-timer", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "signal-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z", + "timer_kind": "signal_timeout", + "signal_wait_id": "signal-wait-id", + "condition_wait_occurrence_id": null, + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:10:10", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 10, + "parallel_group_size": 10, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ] + } + }, + { + "id": "condition-wait", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 12, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "condition-wait-id", + "timeout_seconds": 30, + "signal_name": null, + "condition_key": "ready", + "condition_definition_fingerprint": "predicate-v1", + "condition_wait_occurrence_id": "occurrence-1", + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:10:10", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 10, + "parallel_group_size": 10, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ] + } + }, + { + "id": "condition-timer", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 12, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "condition-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z", + "timer_kind": "condition_timeout", + "condition_wait_id": "condition-wait-id", + "condition_wait_occurrence_id": "occurrence-1", + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:10:10", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 10, + "parallel_group_size": 10, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ] + } + }, + { + "id": "wait-no-timeout", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 13, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "wait-no-timeout-id", + "condition_key": "done", + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:10:10", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 10, + "parallel_group_size": 10, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ] + } + }, + { + "id": "child-0", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 14, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-0", + "child_workflow_instance_id": "child-instance-0", + "child_workflow_run_id": "child-run-0", + "cancellation_policy": "try_cancel", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child", + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:10:10", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 10, + "parallel_group_size": 10, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ] + } + }, + { + "id": "child-1", + "sequence": 12, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 15, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-run-1", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child", + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:10:10", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 10, + "parallel_group_size": 10, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ] + } + }, + { + "id": "child-2", + "sequence": 13, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 16, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-2", + "child_workflow_instance_id": "child-instance-2", + "child_workflow_run_id": "child-run-2", + "cancellation_policy": "abandon", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child", + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:10:10", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 10, + "parallel_group_size": 10, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "select-calls:11:6", + "parallel_group_kind": "mixed", + "parallel_group_mode": "select", + "parallel_group_base_sequence": 11, + "parallel_group_size": 6, + "parallel_group_index": 2, + "selection_member_key": "choice", + "selection_member_index": 2, + "selection_member_base_sequence": 14, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ] + } + }, + { + "id": "child-continued-before-preparation", + "sequence": 14, + "namespace": "sdk-scope-fixture", + "event_type": "ChildRunStarted", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 15, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-continued-run", + "cancellation_policy": "wait_cancellation_completed" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 15, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 16, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 20, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [], + "timer_members": [ + { + "sequence": 10, + "timer_id": "timer-plain", + "descriptor_hash": "d3012a07431c6e6fccdadde3185bdea12d2c5bae3c83c631704bab9e84c2badc" + }, + { + "sequence": 11, + "timer_id": "signal-timer-id", + "descriptor_hash": "3cc49b05da15311eac9d5791fb2099a5ade355ba4ea9ae90f86b597bfdd1ecb8" + }, + { + "sequence": 12, + "timer_id": "condition-timer-id", + "descriptor_hash": "4793559947b722bcc9f8d3a5d983f0c496b1d3da62127b14d677d241be3862f9" + } + ], + "wait_members": [ + { + "kind": "signal", + "sequence": 11, + "wait_id": "signal-wait-id", + "timer_id": "signal-timer-id", + "descriptor_hash": "0c73d277e69cb9a54a442fe8c16b461aca8218d80db5780ed57f41a4a85da3b3" + }, + { + "kind": "condition", + "sequence": 12, + "wait_id": "condition-wait-id", + "timer_id": "condition-timer-id", + "descriptor_hash": "dbf2614f9457f7e94ec5e6858923b8381beaaefe3d322dbc4b1acb33d360eb86" + }, + { + "kind": "condition", + "sequence": 13, + "wait_id": "wait-no-timeout-id", + "timer_id": null, + "descriptor_hash": "c7c84bd9d2dc024450a9c520deba77ac10239e019a587d42201ad7dd03fe2236" + } + ], + "child_members": [ + { + "sequence": 14, + "child_call_id": "child-call-0", + "child_workflow_instance_id": "child-instance-0", + "child_workflow_run_id": "child-run-0", + "cancellation_policy": "try_cancel", + "descriptor_hash": "d4c2dd2ddf569b6b6b156b4d82f30a59262f30fff94a5d579c737266cfa228a2" + }, + { + "sequence": 15, + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-continued-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "e3eec098ec70e85f527e9bb6cf14faaed9092540e32638b022e23f4c9ef50763" + }, + { + "sequence": 16, + "child_call_id": "child-call-2", + "child_workflow_instance_id": "child-instance-2", + "child_workflow_run_id": "child-run-2", + "cancellation_policy": "abandon", + "descriptor_hash": "ec6572e50f0c0eb8738ec8d523f93991476d5e501428d03c6e9af0824e53c022" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 17, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 20, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "descendants": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "desc-child-opened", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "parent_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "shield_parent": false, + "sequence": 3 + } + }, + { + "id": "desc-grandchild-opened", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-grandchild", + "parent_scope_id": "desc-child", + "shield_parent": false, + "sequence": 4 + } + }, + { + "id": "desc-shield-opened", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-shield", + "parent_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "shield_parent": true, + "sequence": 5 + } + }, + { + "id": "desc-shield-child-opened", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-shield-child", + "parent_scope_id": "desc-shield", + "shield_parent": false, + "sequence": 6 + } + }, + { + "id": "plain-timer", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 10, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "timer-plain", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:00.123456Z" + } + }, + { + "id": "signal-wait", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "SignalWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "signal_wait_id": "signal-wait-id", + "timeout_seconds": 30, + "signal_name": "inbox", + "condition_key": null, + "condition_definition_fingerprint": null, + "condition_wait_occurrence_id": null + } + }, + { + "id": "signal-timer", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "signal-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z", + "timer_kind": "signal_timeout", + "signal_wait_id": "signal-wait-id", + "condition_wait_occurrence_id": null + } + }, + { + "id": "condition-wait", + "sequence": 12, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 12, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "condition-wait-id", + "timeout_seconds": 30, + "signal_name": null, + "condition_key": "ready", + "condition_definition_fingerprint": "predicate-v1", + "condition_wait_occurrence_id": "occurrence-1" + } + }, + { + "id": "condition-timer", + "sequence": 13, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 12, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "condition-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z", + "timer_kind": "condition_timeout", + "condition_wait_id": "condition-wait-id", + "condition_wait_occurrence_id": "occurrence-1" + } + }, + { + "id": "wait-no-timeout", + "sequence": 14, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 13, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "wait-no-timeout-id", + "condition_key": "done" + } + }, + { + "id": "child-0", + "sequence": 15, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 14, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-0", + "child_workflow_instance_id": "child-instance-0", + "child_workflow_run_id": "child-run-0", + "cancellation_policy": "try_cancel", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child" + } + }, + { + "id": "child-1", + "sequence": 16, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 15, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-run-1", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child" + } + }, + { + "id": "child-2", + "sequence": 17, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 16, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-2", + "child_workflow_instance_id": "child-instance-2", + "child_workflow_run_id": "child-run-2", + "cancellation_policy": "abandon", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child" + } + }, + { + "id": "child-continued-before-preparation", + "sequence": 18, + "namespace": "sdk-scope-fixture", + "event_type": "ChildRunStarted", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 15, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-continued-run", + "cancellation_policy": "wait_cancellation_completed" + } + }, + { + "id": "descendant-timer", + "sequence": 19, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 17, + "cancellation_scope_id": "desc-child", + "timer_id": "descendant-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 20, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "inherited-child-accepted", + "sequence": 21, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "parent_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "inherited-child-request", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "inherited-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "inherited-grandchild-accepted", + "sequence": 22, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-grandchild", + "parent_scope_id": "desc-child", + "request_id": "inherited-grandchild-request", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "inherited-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "inherited-grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-grandchild", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 23, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 20, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [], + "timer_members": [ + { + "sequence": 10, + "timer_id": "timer-plain", + "descriptor_hash": "d78a4b0d2c2f326857a31358e3ddfa20bd567e2364315ca30cef3cb11eae6c0f" + }, + { + "sequence": 11, + "timer_id": "signal-timer-id", + "descriptor_hash": "cb7fd119a14082cae54f6f418beb5769a3506cc5769df7ac10318017b1360b70" + }, + { + "sequence": 12, + "timer_id": "condition-timer-id", + "descriptor_hash": "167249f4220f18c56597a3a14165aa464443b4e71e7e28ffd708f271b6f0460e" + } + ], + "wait_members": [ + { + "kind": "signal", + "sequence": 11, + "wait_id": "signal-wait-id", + "timer_id": "signal-timer-id", + "descriptor_hash": "68a60d453bbc004c3c17ca4e33f95bbad92e24508e680152c1140be5f247a63f" + }, + { + "kind": "condition", + "sequence": 12, + "wait_id": "condition-wait-id", + "timer_id": "condition-timer-id", + "descriptor_hash": "e8f0ce4c1242168f2a97286c5ceea4bb9458822aedc8301734226b139510e960" + }, + { + "kind": "condition", + "sequence": 13, + "wait_id": "wait-no-timeout-id", + "timer_id": null, + "descriptor_hash": "b462fc9baefdd0f5b792ed8f41177d3a97a3d4bce08a58f04782a31215d4f6f6" + } + ], + "child_members": [ + { + "sequence": 14, + "child_call_id": "child-call-0", + "child_workflow_instance_id": "child-instance-0", + "child_workflow_run_id": "child-run-0", + "cancellation_policy": "try_cancel", + "descriptor_hash": "19c48c0ad10c781d4c08900a51fa1d98e4606bc7b65556b12d705d3f49367d07" + }, + { + "sequence": 15, + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-continued-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "0d65b5082c06e33b7db952b1d61085a344a4b97b60d982998a4fba6b6e6748ef" + }, + { + "sequence": 16, + "child_call_id": "child-call-2", + "child_workflow_instance_id": "child-instance-2", + "child_workflow_run_id": "child-run-2", + "cancellation_policy": "abandon", + "descriptor_hash": "a6a5e39e958de9cf34a656dc01f417209dae2a5787ab4ee0e169c9e3f08421bd" + } + ], + "descendant_members": [ + { + "scope_id": "desc-child", + "parent_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "scope_history_event_id": "desc-child-opened", + "request_history_event_id": "inherited-child-accepted", + "request_id": "inherited-child-request", + "propagation_history_event_id": "inherited-child-accepted", + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "inherited-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [], + "timer_members": [ + { + "sequence": 17, + "timer_id": "descendant-timer-id", + "descriptor_hash": "b37b71a960ac7b3edf51dbf31e3a0d22c863c10d05abf2ac362f74d6ac131bf6" + } + ], + "wait_members": [], + "child_members": [] + }, + { + "scope_id": "desc-grandchild", + "parent_scope_id": "desc-child", + "scope_history_event_id": "desc-grandchild-opened", + "request_history_event_id": "inherited-grandchild-accepted", + "request_id": "inherited-grandchild-request", + "propagation_history_event_id": "inherited-grandchild-accepted", + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "inherited-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "inherited-grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-grandchild", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [] + } + ], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 24, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 20, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "competing": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "desc-child-opened", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "parent_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "shield_parent": false, + "sequence": 3 + } + }, + { + "id": "desc-grandchild-opened", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-grandchild", + "parent_scope_id": "desc-child", + "shield_parent": false, + "sequence": 4 + } + }, + { + "id": "desc-shield-opened", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-shield", + "parent_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "shield_parent": true, + "sequence": 5 + } + }, + { + "id": "desc-shield-child-opened", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-shield-child", + "parent_scope_id": "desc-shield", + "shield_parent": false, + "sequence": 6 + } + }, + { + "id": "plain-timer", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 10, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "timer-plain", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:00.123456Z" + } + }, + { + "id": "signal-wait", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "SignalWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "signal_wait_id": "signal-wait-id", + "timeout_seconds": 30, + "signal_name": "inbox", + "condition_key": null, + "condition_definition_fingerprint": null, + "condition_wait_occurrence_id": null + } + }, + { + "id": "signal-timer", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "signal-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z", + "timer_kind": "signal_timeout", + "signal_wait_id": "signal-wait-id", + "condition_wait_occurrence_id": null + } + }, + { + "id": "condition-wait", + "sequence": 12, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 12, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "condition-wait-id", + "timeout_seconds": 30, + "signal_name": null, + "condition_key": "ready", + "condition_definition_fingerprint": "predicate-v1", + "condition_wait_occurrence_id": "occurrence-1" + } + }, + { + "id": "condition-timer", + "sequence": 13, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 12, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "condition-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z", + "timer_kind": "condition_timeout", + "condition_wait_id": "condition-wait-id", + "condition_wait_occurrence_id": "occurrence-1" + } + }, + { + "id": "wait-no-timeout", + "sequence": 14, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 13, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "wait-no-timeout-id", + "condition_key": "done" + } + }, + { + "id": "child-0", + "sequence": 15, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 14, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-0", + "child_workflow_instance_id": "child-instance-0", + "child_workflow_run_id": "child-run-0", + "cancellation_policy": "try_cancel", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child" + } + }, + { + "id": "child-1", + "sequence": 16, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 15, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-run-1", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child" + } + }, + { + "id": "child-2", + "sequence": 17, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 16, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-2", + "child_workflow_instance_id": "child-instance-2", + "child_workflow_run_id": "child-run-2", + "cancellation_policy": "abandon", + "parent_close_policy": "request_cancel", + "child_workflow_type": "python-child" + } + }, + { + "id": "child-continued-before-preparation", + "sequence": 18, + "namespace": "sdk-scope-fixture", + "event_type": "ChildRunStarted", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 15, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-continued-run", + "cancellation_policy": "wait_cancellation_completed" + } + }, + { + "id": "descendant-timer", + "sequence": 19, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "sequence": 17, + "cancellation_scope_id": "desc-child", + "timer_id": "descendant-timer-id", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:30.123456Z" + } + }, + { + "id": "independent-child-accepted", + "sequence": 20, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "parent_scope_id": null, + "request_id": "independent-child-request", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "independent-child-request", + "root_request_id": "independent-child-request", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z", + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z" + } + ] + } + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 21, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "original-child-conflict", + "sequence": 22, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequestConflicted", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "scope_id": "desc-child", + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "reason": "cancellation_root_conflict", + "parent_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "incoming_cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "inherited-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "accepted_cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "independent-child-request", + "root_request_id": "independent-child-request", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z", + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z" + } + ] + } + } + }, + { + "id": "inherited-grandchild-accepted", + "sequence": 23, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:09.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-grandchild", + "parent_scope_id": "desc-child", + "request_id": "inherited-grandchild-request", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "independent-child-request", + "root_request_id": "independent-child-request", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z", + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z" + }, + { + "request_id": "inherited-grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-grandchild", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z" + } + ] + } + } + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 24, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 20, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [], + "timer_members": [ + { + "sequence": 10, + "timer_id": "timer-plain", + "descriptor_hash": "d78a4b0d2c2f326857a31358e3ddfa20bd567e2364315ca30cef3cb11eae6c0f" + }, + { + "sequence": 11, + "timer_id": "signal-timer-id", + "descriptor_hash": "cb7fd119a14082cae54f6f418beb5769a3506cc5769df7ac10318017b1360b70" + }, + { + "sequence": 12, + "timer_id": "condition-timer-id", + "descriptor_hash": "167249f4220f18c56597a3a14165aa464443b4e71e7e28ffd708f271b6f0460e" + } + ], + "wait_members": [ + { + "kind": "signal", + "sequence": 11, + "wait_id": "signal-wait-id", + "timer_id": "signal-timer-id", + "descriptor_hash": "68a60d453bbc004c3c17ca4e33f95bbad92e24508e680152c1140be5f247a63f" + }, + { + "kind": "condition", + "sequence": 12, + "wait_id": "condition-wait-id", + "timer_id": "condition-timer-id", + "descriptor_hash": "e8f0ce4c1242168f2a97286c5ceea4bb9458822aedc8301734226b139510e960" + }, + { + "kind": "condition", + "sequence": 13, + "wait_id": "wait-no-timeout-id", + "timer_id": null, + "descriptor_hash": "b462fc9baefdd0f5b792ed8f41177d3a97a3d4bce08a58f04782a31215d4f6f6" + } + ], + "child_members": [ + { + "sequence": 14, + "child_call_id": "child-call-0", + "child_workflow_instance_id": "child-instance-0", + "child_workflow_run_id": "child-run-0", + "cancellation_policy": "try_cancel", + "descriptor_hash": "19c48c0ad10c781d4c08900a51fa1d98e4606bc7b65556b12d705d3f49367d07" + }, + { + "sequence": 15, + "child_call_id": "child-call-1", + "child_workflow_instance_id": "child-instance-1", + "child_workflow_run_id": "child-continued-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "0d65b5082c06e33b7db952b1d61085a344a4b97b60d982998a4fba6b6e6748ef" + }, + { + "sequence": 16, + "child_call_id": "child-call-2", + "child_workflow_instance_id": "child-instance-2", + "child_workflow_run_id": "child-run-2", + "cancellation_policy": "abandon", + "descriptor_hash": "a6a5e39e958de9cf34a656dc01f417209dae2a5787ab4ee0e169c9e3f08421bd" + } + ], + "descendant_members": [ + { + "scope_id": "desc-child", + "parent_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "scope_history_event_id": "desc-child-opened", + "request_history_event_id": "independent-child-accepted", + "request_id": "independent-child-request", + "propagation_history_event_id": "original-child-conflict", + "authority_deadline_at": "2026-10-04T00:00:20.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "independent-child-request", + "root_request_id": "independent-child-request", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z", + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z" + } + ] + }, + "activity_members": [], + "timer_members": [ + { + "sequence": 17, + "timer_id": "descendant-timer-id", + "descriptor_hash": "b37b71a960ac7b3edf51dbf31e3a0d22c863c10d05abf2ac362f74d6ac131bf6" + } + ], + "wait_members": [], + "child_members": [] + }, + { + "scope_id": "desc-grandchild", + "parent_scope_id": "desc-child", + "scope_history_event_id": "desc-grandchild-opened", + "request_history_event_id": "inherited-grandchild-accepted", + "request_id": "inherited-grandchild-request", + "propagation_history_event_id": "inherited-grandchild-accepted", + "authority_deadline_at": "2026-10-04T00:00:20.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "independent-child-request", + "root_request_id": "independent-child-request", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z", + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "independent-child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-child", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z" + }, + { + "request_id": "inherited-grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "desc-grandchild", + "cleanup_deadline_at": "2026-10-04T00:00:20.123456Z" + } + ] + }, + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [] + } + ], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 25, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 20, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + } +} diff --git a/tests/fixtures/populated-scope-single-calls.json b/tests/fixtures/populated-scope-single-calls.json new file mode 100644 index 0000000..d083851 --- /dev/null +++ b/tests/fixtures/populated-scope-single-calls.json @@ -0,0 +1,3756 @@ +{ + "_provenance": { + "native_commit": "0471f1ebbb98c61fe53663086d62dc2db74db9e3", + "input_fixture_sha256": "85320e25c7c19521bca4a1cff138911df1f19f76471ea033e961695e8ed761be", + "history_store": "isolated SQLite projection inputs", + "effect_authority": false, + "producers": [ + "CancellationScopeDelivery::activityMembers", + "ScopedTimerCancellation::members", + "ScopedWaitCancellation::members", + "ScopedChildCancellation::members" + ] + }, + "activity-try_cancel": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "kind": "activity", + "policy": "try_cancel", + "timeout": 30, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-admission", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 8, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 9, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "activity", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "a26f8a0ff7beb14634f3846cf6f6998166bedd9a468920cf2ccf911762ca9f03" + } + ], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "activity", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "activity-wait_cancellation_completed": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "kind": "activity", + "policy": "wait_cancellation_completed", + "timeout": 30, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-admission", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "wait_cancellation_completed", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 8, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 9, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "activity", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "70125d7921b68151c2e28d6a383487f75b6adab62decd19418ccf636ac540e03" + } + ], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "activity", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "activity-abandon": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "kind": "activity", + "policy": "abandon", + "timeout": 30, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-admission", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "abandon", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 8, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 9, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "activity", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "271cff6e0f86da5383c36fed6e1e3b291a29dbc4cd1ce4105910ddad5c4cb318" + } + ], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "activity", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "child-try_cancel": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "kind": "child", + "policy": "try_cancel", + "timeout": 30, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-admission", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "try_cancel", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 8, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 9, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "child", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + } + ], + "timer_members": [], + "wait_members": [], + "child_members": [ + { + "sequence": 4, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "try_cancel", + "descriptor_hash": "a958cabe4cf43b14934fed381fcd5ea2fe2ad1b83788dfc3de16c442231a1b23" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "child", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "child-wait_cancellation_completed": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "kind": "child", + "policy": "wait_cancellation_completed", + "timeout": 30, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-admission", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 8, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 9, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "child", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + } + ], + "timer_members": [], + "wait_members": [], + "child_members": [ + { + "sequence": 4, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "ef3b5c2dea5496740fceb297f0715a49d8ed675e9a529df754a19d1aacc60a54" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "child", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "child-abandon": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "kind": "child", + "policy": "abandon", + "timeout": 30, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-admission", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "abandon", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 8, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 9, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "child", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + } + ], + "timer_members": [], + "wait_members": [], + "child_members": [ + { + "sequence": 4, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "abandon", + "descriptor_hash": "2798ee25796bf6132d9b8020055ce55b4a46dbf2ea7ad6a2a9261b875247f3b6" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "child", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "timer": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "kind": "timer", + "policy": null, + "timeout": 30, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-admission", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 8, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 9, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + } + ], + "timer_members": [ + { + "sequence": 4, + "timer_id": "original-timer", + "descriptor_hash": "0120b0c0164724ed0fd3b589206d8e67a6177689a25a77f66add6dd7b9aea8f7" + } + ], + "wait_members": [], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "condition": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "kind": "condition", + "policy": null, + "timeout": 30, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-admission", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "original-wait", + "condition_key": "ready", + "timeout_seconds": 30, + "condition_wait_occurrence_id": "original-occurrence", + "condition_definition_fingerprint": "sha256:a1afc3faf0d393d76f4c4f2dfa825e189d8b8294ed8ac63b8a1d260b3076fdc5" + } + }, + { + "id": "condition-timeout", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "original-timeout", + "timer_kind": "condition_timeout", + "condition_wait_id": "original-wait", + "condition_wait_occurrence_id": "original-occurrence", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:32.123456Z" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 9, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "condition", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + } + ], + "timer_members": [ + { + "sequence": 4, + "timer_id": "original-timeout", + "descriptor_hash": "4e574cb5fe9e89df7c963b3d473e1d5ec232af084af565707e3405936b35a6be" + } + ], + "wait_members": [ + { + "kind": "condition", + "sequence": 4, + "wait_id": "original-wait", + "timer_id": "original-timeout", + "descriptor_hash": "3b1c0ac20ee7b4b205a4c8d10959c5ef76721bef02e4fc7cf2ae8e1f35590ebe" + } + ], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 11, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "condition", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "condition-untimed": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "kind": "condition", + "policy": "untimed", + "timeout": null, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-admission", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "condition_wait_id": "original-wait", + "condition_key": "ready", + "timeout_seconds": null, + "condition_wait_occurrence_id": "original-occurrence", + "condition_definition_fingerprint": "sha256:a1afc3faf0d393d76f4c4f2dfa825e189d8b8294ed8ac63b8a1d260b3076fdc5" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 8, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 9, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "condition", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + } + ], + "timer_members": [], + "wait_members": [ + { + "kind": "condition", + "sequence": 4, + "wait_id": "original-wait", + "timer_id": null, + "descriptor_hash": "cf5a40353f288f5d9a4eb75f7a922fc32224e665386b2f61db40e75c77d4cb4c" + } + ], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "condition", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + } +} diff --git a/tests/fixtures/run-inherited-scope-delivery.json b/tests/fixtures/run-inherited-scope-delivery.json new file mode 100644 index 0000000..14ef74c --- /dev/null +++ b/tests/fixtures/run-inherited-scope-delivery.json @@ -0,0 +1,937 @@ +{ + "native_commit": "afce2bcff8eab9a945545521a7b57695d52b6736", + "task": { + "workflow_id": "sdk-run-scope-fixture", + "run_id": "01m46rrz72hsttdx8y9m2df4nz" + }, + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "history_ranges": { + "requested": 5, + "prepared": 6, + "scope_delivered": 7, + "scope_cleaned": 10, + "root_delivered": 11, + "root_cleaned": 14 + }, + "history": [ + { + "id": "01m46rrz7bef6cws9ebad38zjy", + "namespace": "sdk-root-scopes", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m46rrz77fw3g5bmn3h6cep9m", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m46rrz77fw3g5bmn3h6cep9m", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-05T00:00:00.000000Z", + "applied_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:00.000000Z" + }, + { + "id": "01m46rrz7ftv911jxc3qgkk92a", + "namespace": "sdk-root-scopes", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "workflow_command_id": "01m46rrz77fw3g5bmn3h6cep9m", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m46rrz77fw3g5bmn3h6cep9m", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-05T00:00:00.000000Z", + "applied_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:00.000000Z" + }, + { + "id": "01m46rrzdj99vk7az33x50ndz7", + "namespace": "sdk-root-scopes", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "sequence": 1, + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:01:00.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:00.000000Z" + }, + { + "id": "01m46rrzep84mhpck4txrhwmt3", + "namespace": "sdk-root-scopes", + "sequence": 4, + "event_type": "CooperativeCancellationRequested", + "payload": { + "workflow_command_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "command_type": "request_cancellation", + "reason": "finish the entire run", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z", + "cancellation": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_workflow_instance_id": "sdk-run-scope-fixture", + "root_workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "parent_request_id": null, + "reason": "finish the entire run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-05T00:00:00.000000Z", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z", + "lineage": [ + { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz" + } + ] + }, + "command": { + "id": "01M46RRZECZJ27TVWWFQRF3YXD", + "sequence": 2, + "type": "request_cancellation", + "target_scope": "instance", + "resolved_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNw4GDHJlYXNvbgoqZmluaXNoIHRoZSBlbnRpcmUgcnVuJmNsZWFudXBfZGVhZGxpbmVfYXQKNjIwMjYtMTAtMDVUMDA6MDA6MzAuMDAwMDAwWhhjYW5jZWxsYXRpb24OGAxzY2hlbWEKUGR1cmFibGUtd29ya2Zsb3cuY2FuY2VsbGF0aW9uLWNvbnRleHQvdjEUcmVxdWVzdF9pZAo0MDFNNDZSUlpFQ1pKMjdUVldXRlFSRjNZWEQecm9vdF9yZXF1ZXN0X2lkCjQwMU00NlJSWkVDWkoyN1RWV1dGUVJGM1lYRDJyb290X3dvcmtmbG93X2luc3RhbmNlX2lkCipzZGstcnVuLXNjb3BlLWZpeHR1cmUocm9vdF93b3JrZmxvd19ydW5faWQKNDAxbTQ2cnJ6NzJoc3R0ZHg4eTltMmRmNG56InBhcmVudF9yZXF1ZXN0X2lkAAxyZWFzb24KKmZpbmlzaCB0aGUgZW50aXJlIHJ1bhJyZXF1ZXN0ZXIOBAh0eXBlCgZwaHAKbGFiZWwKDlBIUCBBUEkADHNvdXJjZQoGcGhwGHJlcXVlc3RlZF9hdAo2MjAyNi0xMC0wNVQwMDowMDowMC4wMDAwMDBaJmNsZWFudXBfZGVhZGxpbmVfYXQKNjIwMjYtMTAtMDVUMDA6MDA6MzAuMDAwMDAwWg5saW5lYWdlDAIOBhRyZXF1ZXN0X2lkCjQwMU00NlJSWkVDWkoyN1RWV1dGUVJGM1lYRCh3b3JrZmxvd19pbnN0YW5jZV9pZAoqc2RrLXJ1bi1zY29wZS1maXh0dXJlHndvcmtmbG93X3J1bl9pZAo0MDFtNDZycno3MmhzdHRkeDh5OW0yZGY0bnoAAAAA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "cancellation_requested", + "accepted_at": "2026-10-05T00:00:00.000000Z", + "applied_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:00.000000Z" + }, + { + "id": "01m46rrzf8cdgxx95rzzdrx2hm", + "namespace": "sdk-root-scopes", + "sequence": 5, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "parent_scope_id": "root", + "request_id": "01M46RRZF2NQZNW5BH8TP04RZ7", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_workflow_instance_id": "sdk-run-scope-fixture", + "root_workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "parent_request_id": null, + "reason": "finish the entire run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-05T00:00:00.000000Z", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z", + "lineage": [ + { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz" + } + ] + }, + "lineage": [ + { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z" + }, + { + "request_id": "01M46RRZF2NQZNW5BH8TP04RZ7", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z" + } + ] + } + }, + "timestamp": "2026-10-05T00:00:00.000000Z" + }, + { + "id": "01m46rrzk55z6pg33e55n885er", + "namespace": "sdk-root-scopes", + "sequence": 6, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "request_id": "01M46RRZF2NQZNW5BH8TP04RZ7", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_workflow_instance_id": "sdk-run-scope-fixture", + "root_workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "parent_request_id": null, + "reason": "finish the entire run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-05T00:00:00.000000Z", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z", + "lineage": [ + { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz" + } + ] + }, + "lineage": [ + { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z" + }, + { + "request_id": "01M46RRZF2NQZNW5BH8TP04RZ7", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z" + } + ] + }, + "sequence": 2, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-05T00:00:30.000000Z", + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:00:10.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:05.000000Z" + }, + { + "id": "01m46rrzqan7q27ajs2dtte082", + "namespace": "sdk-root-scopes", + "sequence": 7, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "request_id": "01M46RRZF2NQZNW5BH8TP04RZ7", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_workflow_instance_id": "sdk-run-scope-fixture", + "root_workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "parent_request_id": null, + "reason": "finish the entire run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-05T00:00:00.000000Z", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z", + "lineage": [ + { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz" + } + ] + }, + "lineage": [ + { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z" + }, + { + "request_id": "01M46RRZF2NQZNW5BH8TP04RZ7", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z" + } + ] + }, + "sequence": 2, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-05T00:00:30.000000Z", + "preparation_history_event_id": "01m46rrzk55z6pg33e55n885er", + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:00:15.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:05.000000Z" + }, + { + "id": "01m46rrzw27psk234pnmdv6fvj", + "namespace": "sdk-root-scopes", + "sequence": 8, + "event_type": "ActivityScheduled", + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "01m46rrzw12cmnyxcqq3g3ysjp", + "activity_class": "tests.scoped-cleanup", + "activity_type": "tests.scoped-cleanup", + "sequence": 3, + "workflow_task_id": "01m46rrz7gd3k22qas007tjrt7", + "activity": { + "id": "01m46rrzw12cmnyxcqq3g3ysjp", + "idempotency_key": "01m46rrzw12cmnyxcqq3g3ysjp", + "sequence": 3, + "type": "tests.scoped-cleanup", + "class": "tests.scoped-cleanup", + "execution_mode": "local", + "local_activity": true, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "start_to_close_timeout": null, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null, + "non_retryable_error_types": [] + }, + "cancellation_scope_id": "01M46RRZDG58VBRDV14REK37H3", + "queue": "default", + "created_at": "2026-10-05T00:00:05.000000Z", + "arguments": "wwHioz3/VYAiNwwA" + }, + "local_preparation": { + "version": 1, + "descriptor_fingerprint": "644cb65803c9776806936fa612567447325318b32ce65d7c6096fb6d793d1b2c", + "worker_attempt_id": "scope-cleanup-attempt", + "workflow_task_attempt": 1, + "cancellation_cleanup": { + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "operation_scope_id": "01M46RRZDG58VBRDV14REK37H3", + "request_id": "01M46RRZF2NQZNW5BH8TP04RZ7", + "root_request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "delivery_history_event_id": "01m46rrzqan7q27ajs2dtte082", + "preparation_history_event_id": "01m46rrzk55z6pg33e55n885er", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z", + "authority_deadline_at": "2026-10-05T00:00:30.000000Z" + } + }, + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:00:15.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:05.000000Z" + }, + { + "id": "01m46rrzwr8pnjmh25t0267099", + "namespace": "sdk-root-scopes", + "sequence": 9, + "event_type": "ActivityStarted", + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "01m46rrzw12cmnyxcqq3g3ysjp", + "activity_attempt_id": "01M46RRZW5PH9WA5VWN52TV9DK", + "activity_class": "tests.scoped-cleanup", + "activity_type": "tests.scoped-cleanup", + "sequence": 3, + "attempt_number": 1, + "workflow_task_id": "01m46rrz7gd3k22qas007tjrt7", + "lease_expires_at": "2026-10-05T00:00:15.000000Z", + "activity": { + "id": "01m46rrzw12cmnyxcqq3g3ysjp", + "idempotency_key": "01m46rrzw12cmnyxcqq3g3ysjp", + "sequence": 3, + "type": "tests.scoped-cleanup", + "class": "tests.scoped-cleanup", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "01M46RRZW5PH9WA5VWN52TV9DK", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "start_to_close_timeout": null, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null, + "non_retryable_error_types": [] + }, + "cancellation_scope_id": "01M46RRZDG58VBRDV14REK37H3", + "queue": "default", + "last_heartbeat_at": "2026-10-05T00:00:05.000000Z", + "created_at": "2026-10-05T00:00:05.000000Z", + "started_at": "2026-10-05T00:00:05.000000Z", + "arguments": "wwHioz3/VYAiNwwA" + }, + "activity_attempt": { + "id": "01M46RRZW5PH9WA5VWN52TV9DK", + "worker_attempt_id": "scope-cleanup-attempt", + "activity_execution_id": "01m46rrzw12cmnyxcqq3g3ysjp", + "task_id": "01m46rrz7gd3k22qas007tjrt7", + "attempt_number": 1, + "status": "running", + "lease_owner": "original", + "started_at": "2026-10-05T00:00:05.000000Z", + "last_heartbeat_at": "2026-10-05T00:00:05.000000Z", + "lease_expires_at": "2026-10-05T00:00:15.000000Z" + }, + "worker_attempt_id": "scope-cleanup-attempt", + "local_preparation": { + "version": 1, + "descriptor_fingerprint": "644cb65803c9776806936fa612567447325318b32ce65d7c6096fb6d793d1b2c", + "worker_attempt_id": "scope-cleanup-attempt", + "workflow_task_attempt": 1, + "cancellation_cleanup": { + "scope_id": "01M46RRZDG58VBRDV14REK37H3", + "operation_scope_id": "01M46RRZDG58VBRDV14REK37H3", + "request_id": "01M46RRZF2NQZNW5BH8TP04RZ7", + "root_request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "delivery_history_event_id": "01m46rrzqan7q27ajs2dtte082", + "preparation_history_event_id": "01m46rrzk55z6pg33e55n885er", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z", + "authority_deadline_at": "2026-10-05T00:00:30.000000Z" + }, + "schedule_to_close_deadline_at": "2026-10-05T00:00:30.000000Z" + }, + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:00:15.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:05.000000Z" + }, + { + "id": "01m46rs0217fay9tdxg92rf9hb", + "namespace": "sdk-root-scopes", + "sequence": 10, + "event_type": "ActivityCompleted", + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "01m46rrzw12cmnyxcqq3g3ysjp", + "activity_attempt_id": "01M46RRZW5PH9WA5VWN52TV9DK", + "activity_class": "tests.scoped-cleanup", + "activity_type": "tests.scoped-cleanup", + "sequence": 3, + "attempt_number": 1, + "result": "wwHioz3/VYAiNwoac2NvcGUtY2xlYW5lZA==", + "payload_codec": "avro", + "workflow_task_id": "01m46rrz7gd3k22qas007tjrt7", + "activity": { + "id": "01m46rrzw12cmnyxcqq3g3ysjp", + "idempotency_key": "01m46rrzw12cmnyxcqq3g3ysjp", + "sequence": 3, + "type": "tests.scoped-cleanup", + "class": "tests.scoped-cleanup", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "01M46RRZW5PH9WA5VWN52TV9DK", + "status": "completed", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "start_to_close_timeout": null, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null, + "non_retryable_error_types": [] + }, + "cancellation_scope_id": "01M46RRZDG58VBRDV14REK37H3", + "queue": "default", + "last_heartbeat_at": "2026-10-05T00:00:05.000000Z", + "created_at": "2026-10-05T00:00:05.000000Z", + "started_at": "2026-10-05T00:00:05.000000Z", + "closed_at": "2026-10-05T00:00:06.000000Z", + "arguments": "wwHioz3/VYAiNwwA", + "result": "wwHioz3/VYAiNwoac2NvcGUtY2xlYW5lZA==" + }, + "activity_attempt": { + "id": "01M46RRZW5PH9WA5VWN52TV9DK", + "worker_attempt_id": "scope-cleanup-attempt", + "activity_execution_id": "01m46rrzw12cmnyxcqq3g3ysjp", + "task_id": "01m46rrz7gd3k22qas007tjrt7", + "attempt_number": 1, + "status": "completed", + "lease_owner": "original", + "started_at": "2026-10-05T00:00:05.000000Z", + "last_heartbeat_at": "2026-10-05T00:00:05.000000Z", + "closed_at": "2026-10-05T00:00:06.000000Z" + }, + "worker_attempt_id": "scope-cleanup-attempt", + "local_outcome": { + "version": 1, + "report_fingerprint": "c32cb4bcc458921215e509d0b9d301da67c823423a013ea8013f40fe53e03184", + "workflow_task_attempt": 1, + "submitted_outcome": "completed" + }, + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:00:15.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:06.000000Z" + }, + { + "id": "01m46rs04d8jqrcamnx2mk6sk0", + "namespace": "sdk-root-scopes", + "sequence": 11, + "event_type": "CooperativeCancellationDelivered", + "payload": { + "workflow_command_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "sequence": 4, + "call_kind": "timer", + "cancellation": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_workflow_instance_id": "sdk-run-scope-fixture", + "root_workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz", + "parent_request_id": null, + "reason": "finish the entire run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-05T00:00:00.000000Z", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z", + "lineage": [ + { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "workflow_instance_id": "sdk-run-scope-fixture", + "workflow_run_id": "01m46rrz72hsttdx8y9m2df4nz" + } + ] + }, + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:00:15.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:07.000000Z" + }, + { + "id": "01m46rs06q8qxmmzy16h3y1dx5", + "namespace": "sdk-root-scopes", + "sequence": 12, + "event_type": "ActivityScheduled", + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "01m46rs06ny40t1ne7js0b8nac", + "activity_class": "tests.root-cleanup", + "activity_type": "tests.root-cleanup", + "sequence": 5, + "workflow_task_id": "01m46rrz7gd3k22qas007tjrt7", + "activity": { + "id": "01m46rs06ny40t1ne7js0b8nac", + "idempotency_key": "01m46rs06ny40t1ne7js0b8nac", + "sequence": 5, + "type": "tests.root-cleanup", + "class": "tests.root-cleanup", + "execution_mode": "local", + "local_activity": true, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "start_to_close_timeout": null, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null, + "non_retryable_error_types": [] + }, + "queue": "default", + "created_at": "2026-10-05T00:00:07.000000Z", + "arguments": "wwHioz3/VYAiNwwA" + }, + "local_preparation": { + "version": 1, + "descriptor_fingerprint": "49c9f59482e82cc4a310fd1878bd1677d3a6af46a14e94395ef953bb5db10c9a", + "worker_attempt_id": "root-cleanup-attempt", + "workflow_task_attempt": 1, + "cancellation_cleanup": { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "delivery_history_event_id": "01m46rs04d8jqrcamnx2mk6sk0", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z" + } + }, + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:00:17.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:07.000000Z" + }, + { + "id": "01m46rs06v4hr2nbq37cc217j9", + "namespace": "sdk-root-scopes", + "sequence": 13, + "event_type": "ActivityStarted", + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "01m46rs06ny40t1ne7js0b8nac", + "activity_attempt_id": "01M46RS06R879KJC0T0V7RBSCJ", + "activity_class": "tests.root-cleanup", + "activity_type": "tests.root-cleanup", + "sequence": 5, + "attempt_number": 1, + "workflow_task_id": "01m46rrz7gd3k22qas007tjrt7", + "lease_expires_at": "2026-10-05T00:00:17.000000Z", + "activity": { + "id": "01m46rs06ny40t1ne7js0b8nac", + "idempotency_key": "01m46rs06ny40t1ne7js0b8nac", + "sequence": 5, + "type": "tests.root-cleanup", + "class": "tests.root-cleanup", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "01M46RS06R879KJC0T0V7RBSCJ", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "start_to_close_timeout": null, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null, + "non_retryable_error_types": [] + }, + "queue": "default", + "last_heartbeat_at": "2026-10-05T00:00:07.000000Z", + "created_at": "2026-10-05T00:00:07.000000Z", + "started_at": "2026-10-05T00:00:07.000000Z", + "arguments": "wwHioz3/VYAiNwwA" + }, + "activity_attempt": { + "id": "01M46RS06R879KJC0T0V7RBSCJ", + "worker_attempt_id": "root-cleanup-attempt", + "activity_execution_id": "01m46rs06ny40t1ne7js0b8nac", + "task_id": "01m46rrz7gd3k22qas007tjrt7", + "attempt_number": 1, + "status": "running", + "lease_owner": "original", + "started_at": "2026-10-05T00:00:07.000000Z", + "last_heartbeat_at": "2026-10-05T00:00:07.000000Z", + "lease_expires_at": "2026-10-05T00:00:17.000000Z" + }, + "worker_attempt_id": "root-cleanup-attempt", + "local_preparation": { + "version": 1, + "descriptor_fingerprint": "49c9f59482e82cc4a310fd1878bd1677d3a6af46a14e94395ef953bb5db10c9a", + "worker_attempt_id": "root-cleanup-attempt", + "workflow_task_attempt": 1, + "cancellation_cleanup": { + "request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "root_request_id": "01M46RRZECZJ27TVWWFQRF3YXD", + "delivery_history_event_id": "01m46rs04d8jqrcamnx2mk6sk0", + "cleanup_deadline_at": "2026-10-05T00:00:30.000000Z" + }, + "schedule_to_close_deadline_at": "2026-10-05T00:00:30.000000Z" + }, + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:00:17.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:07.000000Z" + }, + { + "id": "01m46rs090gqkd6hzzs8nx9t8m", + "namespace": "sdk-root-scopes", + "sequence": 14, + "event_type": "ActivityCompleted", + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "01m46rs06ny40t1ne7js0b8nac", + "activity_attempt_id": "01M46RS06R879KJC0T0V7RBSCJ", + "activity_class": "tests.root-cleanup", + "activity_type": "tests.root-cleanup", + "sequence": 5, + "attempt_number": 1, + "result": "wwHioz3/VYAiNwoYcm9vdC1jbGVhbmVk", + "payload_codec": "avro", + "workflow_task_id": "01m46rrz7gd3k22qas007tjrt7", + "activity": { + "id": "01m46rs06ny40t1ne7js0b8nac", + "idempotency_key": "01m46rs06ny40t1ne7js0b8nac", + "sequence": 5, + "type": "tests.root-cleanup", + "class": "tests.root-cleanup", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "01M46RS06R879KJC0T0V7RBSCJ", + "status": "completed", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "snapshot_version": 1, + "max_attempts": 1, + "backoff_seconds": [], + "start_to_close_timeout": null, + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": null, + "heartbeat_timeout": null, + "non_retryable_error_types": [] + }, + "queue": "default", + "last_heartbeat_at": "2026-10-05T00:00:07.000000Z", + "created_at": "2026-10-05T00:00:07.000000Z", + "started_at": "2026-10-05T00:00:07.000000Z", + "closed_at": "2026-10-05T00:00:08.000000Z", + "arguments": "wwHioz3/VYAiNwwA", + "result": "wwHioz3/VYAiNwoYcm9vdC1jbGVhbmVk" + }, + "activity_attempt": { + "id": "01M46RS06R879KJC0T0V7RBSCJ", + "worker_attempt_id": "root-cleanup-attempt", + "activity_execution_id": "01m46rs06ny40t1ne7js0b8nac", + "task_id": "01m46rrz7gd3k22qas007tjrt7", + "attempt_number": 1, + "status": "completed", + "lease_owner": "original", + "started_at": "2026-10-05T00:00:07.000000Z", + "last_heartbeat_at": "2026-10-05T00:00:07.000000Z", + "closed_at": "2026-10-05T00:00:08.000000Z" + }, + "worker_attempt_id": "root-cleanup-attempt", + "local_outcome": { + "version": 1, + "report_fingerprint": "8cfab1f8c3d2b9f046c4652ac1fe520c8714ed495593cb07be048f9f5f1424ce", + "workflow_task_attempt": 1, + "submitted_outcome": "completed" + }, + "task": { + "id": "01m46rrz7gd3k22qas007tjrt7", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-05T00:00:00.000000Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-05T00:00:17.000000Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-05T00:00:00.000000Z", + "last_dispatched_at": "2026-10-05T00:00:00.000000Z" + } + }, + "timestamp": "2026-10-05T00:00:08.000000Z" + } + ], + "native_terminal_status": "cancelled" +} From 6fa3f8b83d40e41d9e37e14fc4a512179196ba1e Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 00:41:40 +0000 Subject: [PATCH 46/58] feat: coordinate original scalar scope delivery and cleanup replay --- src/cancellation_context.rs | 31 +++ src/cancellation_replay_clock.rs | 7 + src/cancellation_scope.rs | 14 +- src/cancellation_scope_replay.rs | 279 ++++++++++++++++++++++++ src/cooperative_cancellation.rs | 39 +++- src/lib.rs | 181 +++++++++++++-- src/tests/cancellation_scope_history.rs | 104 ++++++++- src/tests/cancellation_scope_replay.rs | 211 ++++++++++++++++++ 8 files changed, 839 insertions(+), 27 deletions(-) create mode 100644 src/cancellation_scope_replay.rs create mode 100644 src/tests/cancellation_scope_replay.rs diff --git a/src/cancellation_context.rs b/src/cancellation_context.rs index 45160de..1917bce 100644 --- a/src/cancellation_context.rs +++ b/src/cancellation_context.rs @@ -402,6 +402,37 @@ fn assert_context_keys(value: &Value, keys: &[&str]) -> Result<()> { } impl ScopedCancellationContext { + /// Remaining original scope authority at the last consumed durable result. + #[doc(hidden)] + pub fn remaining(&self) -> Result { + let state = self + .root_context + .replay + .as_ref() + .and_then(Weak::upgrade) + .filter(cancellation_replay_clock::is_active) + .ok_or_else(|| { + invalid_context("remaining() is available only in its active scope replay") + })?; + let state = state.lock().map_err(|_| Error::WorkflowStatePoisoned)?; + let (_, ceiling) = state + .scope_delivery + .as_ref() + .and_then(|replay| replay.contexts.get(self.scope_id())) + .filter(|(original, _)| original == self) + .ok_or_else(|| { + invalid_context("scope remaining() requires its original consumed delivery") + })?; + Ok((self.deadline().min(*ceiling) - state.cancellation_time()?) + .to_std() + .unwrap_or(Duration::ZERO)) + } + + pub(super) fn with_replay(mut self, replay: Option>>) -> Self { + self.root_context = self.root_context.with_replay(replay); + self + } + pub(super) fn from_run_context(context: &CancellationContext) -> Result { let mut root = context.to_value(); let mut lineage = Vec::new(); diff --git a/src/cancellation_replay_clock.rs b/src/cancellation_replay_clock.rs index 795e321..bb0506d 100644 --- a/src/cancellation_replay_clock.rs +++ b/src/cancellation_replay_clock.rs @@ -170,6 +170,13 @@ impl ReplayClock { } impl WorkflowState { + pub(super) fn observe_scope_cancellation_delivery(&mut self, event: &HistoryEvent) { + if let Some(clock) = &mut self.cancellation_clock { + clock.started = true; + clock.observe(Some(event)); + } + } + pub(super) fn start_cancellation_clock(&mut self) { if let Some(clock) = &mut self.cancellation_clock { if !clock.started { diff --git a/src/cancellation_scope.rs b/src/cancellation_scope.rs index 1e1f977..96a627f 100644 --- a/src/cancellation_scope.rs +++ b/src/cancellation_scope.rs @@ -26,7 +26,7 @@ pub(super) struct CanonicalScopeOpening { pub shield_parent: bool, } -#[derive(Default, Debug)] +#[derive(Clone, Default, Debug)] pub(super) struct CancellationScopeHistory { pub openings: BTreeMap, pub memberships: BTreeMap, @@ -236,18 +236,28 @@ pub(super) struct CancellationScopeOpening { impl WorkflowContext { pub(super) fn validate_scope_membership( &self, - state: &WorkflowState, + state: &mut WorkflowState, cursor: usize, ) -> Result<()> { if !state.allow_cancellation_scope_authoring { return Ok(()); } + if let Some(replay) = &mut state.scope_delivery { + replay.active_scope = self.cancellation_scope_id.clone(); + } if let Some(recorded) = state.recorded_commands.get(cursor) { let sequence = recorded.sequence(); let original = state .cancellation_scope_memberships .get(&sequence) .map(String::as_str) + .or_else(|| { + state + .scope_delivery + .as_ref() + .and_then(|replay| replay.canonical.deliveries.get(&sequence)) + .map(|delivered| delivered.context.scope_id()) + }) .unwrap_or("root"); if original != self.cancellation_scope_id { return Err(invalid_recorded_history( diff --git a/src/cancellation_scope_replay.rs b/src/cancellation_scope_replay.rs new file mode 100644 index 0000000..8d719ad --- /dev/null +++ b/src/cancellation_scope_replay.rs @@ -0,0 +1,279 @@ +//! Candidate scalar scope delivery. Admission remains private and disabled by default. + +use super::*; +use crate::cancellation_scope_history::CommittedCancellationScopeHistory; +use chrono::Utc; + +#[doc(hidden)] +#[derive(Clone, Debug, Error, PartialEq, Eq)] +#[error("scope cancellation {request_id} was requested", request_id = .context.request_id())] +pub struct CancellationScopeRequested { + pub context: ScopedCancellationContext, + pub delivery: CancellationDelivery, +} + +#[derive(Clone, Debug)] +pub(super) struct ScopeDeliveryIntent { + pub context: ScopedCancellationContext, + pub boundary: CancellationDelivery, + pub command_count: usize, +} + +#[derive(Debug)] +pub(super) struct ScopeReplay { + pub canonical: CommittedCancellationScopeHistory, + pub tree: cancellation_scope::CancellationScopeHistory, + pub active_scope: String, + pub consumed: BTreeSet, + pub contexts: BTreeMap)>, + pub intent: Option, +} + +fn invalid(sequence: u64, detail: &str) -> Error { + invalid_recorded_history( + "cancellation_scope_call_mismatch", + sequence, + "original scalar scoped call", + "changed call or authority", + detail, + ) +} + +fn scalar(kind: CancellationCallKind) -> bool { + matches!( + kind, + CancellationCallKind::Activity + | CancellationCallKind::Timer + | CancellationCallKind::Condition + | CancellationCallKind::Child + ) +} + +impl ScopeReplay { + pub fn read( + history: &[HistoryEvent], + tree: &cancellation_scope::CancellationScopeHistory, + run: &str, + workflow: &str, + ) -> Result { + let canonical = CommittedCancellationScopeHistory::read(history, run, workflow)?; + for preparation in canonical.preparations.values() { + if !scalar(preparation.boundary.call_kind) + || preparation.boundary.sequence_span != 1 + || preparation.boundary.operation_sequence.is_some() + || preparation.event.payload["descendant_members"] + .as_array() + .is_none_or(|members| !members.is_empty()) + { + return Err(Error::CancellationScopeExecutionUnavailable); + } + } + Ok(Self { + canonical, + tree: tree.clone(), + active_scope: "root".into(), + consumed: BTreeSet::new(), + contexts: BTreeMap::new(), + intent: None, + }) + } + + pub fn bind_commands(&self, commands: &mut Vec) -> Result<()> { + for (&sequence, delivered) in &self.canonical.deliveries { + let index = commands.partition_point(|command| command.sequence() < sequence); + let original = if commands + .get(index) + .is_some_and(|command| command.sequence() == sequence) + { + Some(Box::new(commands.remove(index))) + } else { + let prior = index + .checked_sub(1) + .map_or(0, |index| commands[index].sequence()); + if prior.checked_add(1) != Some(sequence) { + return Err(invalid( + sequence, + "scope delivery skips an earlier authored call", + )); + } + None + }; + commands.insert( + index, + RecordedCommand::CancellationBoundary { + sequence, + call_kind: delivered.boundary.call_kind, + original, + }, + ); + } + Ok(()) + } +} + +impl WorkflowState { + pub(super) fn prepare_scalar_scope_cancellation( + &mut self, + index: usize, + kind: CancellationCallKind, + path: &[ParallelGroupMetadata], + ) -> Result { + let Some(replay) = &self.scope_delivery else { + return Ok(false); + }; + if self.cancellation_shield_depth > 0 { + return Ok(false); + } + if replay.intent.is_some() { + self.matched_recorded_pending = true; + return Ok(true); + } + let Some(request) = replay + .canonical + .pending_request_for_scope(&replay.active_scope, &replay.tree)? + else { + return Ok(false); + }; + if replay.contexts.contains_key(request.context.scope_id()) { + return Ok(false); + } + if !scalar(kind) || !path.is_empty() || request.context.scope_id() != replay.active_scope { + return Err(Error::CancellationScopeExecutionUnavailable); + } + let sequence = self.recorded_commands.get(index).map_or_else( + || self.next_scope_cancellation_sequence(), + |command| Ok(command.sequence()), + )?; + if self + .history_events + .iter() + .take(request.history_index) + .any(|event| { + durable_event_sequence(event) == Some(sequence) + && matches!( + event.event_type.as_str(), + "ActivityCompleted" + | "ActivityFailed" + | "ActivityCancelled" + | "ActivityTimedOut" + | "TimerFired" + | "TimerCancelled" + | "ConditionWaitSatisfied" + | "ConditionWaitTimedOut" + | "ChildRunCompleted" + | "ChildRunFailed" + | "ChildRunCancelled" + | "ChildRunTerminated" + ) + }) + { + return Ok(false); + } + let boundary = CancellationDelivery { + request_id: request.context.request_id().into(), + sequence, + call_kind: kind, + sequence_span: 1, + operation_sequence: None, + operation_sequence_span: 1, + }; + if replay + .canonical + .preparations + .get(request.context.scope_id()) + .is_some_and(|prepared| prepared.boundary != boundary) + { + return Err(invalid( + sequence, + "prepared scope changed its original authored boundary", + )); + } + let intent = ScopeDeliveryIntent { + context: request.context.clone(), + boundary, + command_count: self.commands.len(), + }; + self.scope_delivery.as_mut().unwrap().intent = Some(intent); + self.matched_recorded_pending = true; + if index < self.recorded_commands.len() { + self.command_cursor = index + 1; + } + Ok(true) + } + + fn next_scope_cancellation_sequence(&self) -> Result { + self.recorded_commands + .last() + .map_or(0, RecordedCommand::sequence) + .checked_add(self.commands.len() as u64) + .and_then(|sequence| sequence.checked_add(1)) + .filter(|sequence| *sequence < i64::MAX as u64) + .ok_or_else(|| invalid(0, "scope cancellation authored sequence overflowed")) + } + + pub(super) fn replay_scope_cancellation_at( + &mut self, + index: usize, + kind: CancellationCallKind, + ) -> Result<()> { + let Some(sequence) = self + .recorded_commands + .get(index) + .map(RecordedCommand::sequence) + else { + return Ok(()); + }; + let Some(replay) = &self.scope_delivery else { + return Ok(()); + }; + let Some(delivered) = replay.canonical.deliveries.get(&sequence).cloned() else { + return Ok(()); + }; + if replay.consumed.contains(&sequence) + || delivered.boundary.call_kind != kind + || replay.active_scope != delivered.context.scope_id() + || self.cancellation_shield_depth > 0 + { + return Err(invalid( + sequence, + "committed scope delivery changed its original call, membership or shielding", + )); + } + self.observe_scope_cancellation_delivery(&delivered.event); + let replay = self.scope_delivery.as_mut().unwrap(); + replay.consumed.insert(sequence); + replay.contexts.insert( + delivered.context.scope_id().into(), + (delivered.context.clone(), delivered.authority_deadline), + ); + self.command_cursor = index + 1; + Err(Error::CancellationScopeRequested( + CancellationScopeRequested { + context: delivered + .context + .with_replay(cancellation_replay_clock::active_binding()), + delivery: delivered.boundary, + }, + )) + } +} + +impl WorkflowContext { + /// Original metadata only after consuming this context's committed scope delivery. + #[doc(hidden)] + pub fn scoped_cancellation_context(&self) -> Result> { + let state = self + .state + .lock() + .map_err(|_| Error::WorkflowStatePoisoned)?; + Ok(state + .scope_delivery + .as_ref() + .and_then(|replay| replay.contexts.get(&self.cancellation_scope_id)) + .map(|(context, _)| { + context + .clone() + .with_replay(Some(Arc::downgrade(&self.state))) + })) + } +} diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index 89ed9d0..bec57a7 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -4,10 +4,14 @@ use std::collections::BTreeSet; const CONTROL_BUDGET: Duration = Duration::from_secs(5); const MAX_REFRESH_PAGES: usize = 128; -pub(super) fn assert_cancellation_scope_replay_supported_with_authoring( +pub(super) fn assert_cancellation_scope_replay_supported( events: &[HistoryEvent], allow_authoring: bool, + allow_delivery: bool, ) -> Result<()> { + if allow_delivery && !allow_authoring { + return Err(Error::CancellationScopeExecutionUnavailable); + } for event in events { let scope_marker = matches!( event.event_type.as_str(), @@ -27,7 +31,7 @@ pub(super) fn assert_cancellation_scope_replay_supported_with_authoring( .get("cancellation_scope_id") .is_some_and(|scope| scope.as_str() != Some("root")) }); - if scope_marker + if (scope_marker && !allow_delivery) || (!allow_authoring && (event.event_type == "CancellationScopeOpened" || scoped_membership)) { @@ -786,6 +790,9 @@ impl WorkflowState { kind: CancellationCallKind, group_path: &[ParallelGroupMetadata], ) -> Result { + if self.prepare_scalar_scope_cancellation(index, kind, group_path)? { + return Ok(true); + } if !self.cancellation_delivery_enabled || self.cancellation_shield_depth > 0 { return Ok(false); } @@ -992,6 +999,7 @@ impl WorkflowState { if let Some(original) = original { return Ok(Some(*original)); } + self.replay_scope_cancellation_at(index, kind)?; self.validate_cancellation_call(sequence, kind, call_kind)?; self.cancellation_consumed = true; self.cancel_requested = true; @@ -1007,6 +1015,7 @@ impl WorkflowState { index: usize, kind: CancellationCallKind, ) -> Result<()> { + self.replay_scope_cancellation_at(index, kind)?; if let Some(RecordedCommand::CancellationBoundary { sequence, call_kind, @@ -1906,6 +1915,32 @@ impl Worker { } for _ in 0..1000 { let mut decision = self.execute_workflow_task_decision(task.clone())?; + if let Some(intent) = decision.cancellation_scope_delivery.as_ref() { + if !decision.commands.is_empty() { + decision.cancellation_scope_delivery = None; + return Ok(Some(decision)); + } + let mut budget = CancellationScopeDeliveryBudget::new(); + budget.restrict(intent.context.deadline())?; + let prepared = self + .client + .prepare_cancellation_scope_on_claim( + &task, + &intent.context, + &intent.boundary, + &budget, + ) + .await?; + budget.restrict(prepared.authority_deadline())?; + let delivered = self + .client + .deliver_cancellation_scope_on_claim(&task, &prepared, &budget) + .await?; + task.history_events = delivered.history().to_vec(); + task.total_history_events = None; + task.history_size_bytes = None; + continue; + } let Some(opening) = decision.cancellation_scope_opening.as_ref() else { return Ok(Some(decision)); }; diff --git a/src/lib.rs b/src/lib.rs index eef574c..2801f2a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ mod cancellation_context; mod cancellation_replay_clock; mod cancellation_scope; mod cancellation_scope_history; +mod cancellation_scope_replay; mod cooperative_cancellation; mod runtime_payloads; mod runtime_uploads; @@ -16,6 +17,8 @@ pub use cancellation_scope::CancellationScopeOpenReceipt; pub use cancellation_scope_history::{ CancellationScopeDeliveryBudget, CancellationScopeDeliveryReceipt, }; +#[doc(hidden)] +pub use cancellation_scope_replay::CancellationScopeRequested; pub use cooperative_cancellation::{ CancellationCallKind, CancellationDelivery, CancellationDeliveryReceipt, @@ -210,6 +213,8 @@ pub enum Error { #[error(transparent)] CooperativeCancellationRequested(CooperativeCancellationRequested), #[error(transparent)] + CancellationScopeRequested(CancellationScopeRequested), + #[error(transparent)] WorkflowCommandRejected(WorkflowCommandRejection), #[error(transparent)] WorkflowFailed(WorkflowTerminalOutcome), @@ -6189,6 +6194,7 @@ struct WorkflowTaskDecision { message_stream_waits: Vec, cancellation_delivery: Option, cancellation_scope_opening: Option, + cancellation_scope_delivery: Option, } impl WorkflowTaskDecision { @@ -6199,6 +6205,7 @@ impl WorkflowTaskDecision { message_stream_waits: Vec::new(), cancellation_delivery: None, cancellation_scope_opening: None, + cancellation_scope_delivery: None, } } } @@ -6295,6 +6302,7 @@ pub struct Worker { heartbeat_observer: Option, cooperative_cancellation_enabled: bool, allow_cancellation_scope_authoring: bool, + allow_cancellation_scope_delivery: bool, cooperative_registration_confirmed: Arc, } @@ -6316,6 +6324,7 @@ impl Worker { heartbeat_observer: None, cooperative_cancellation_enabled: false, allow_cancellation_scope_authoring: false, + allow_cancellation_scope_delivery: false, cooperative_registration_confirmed: Arc::new(AtomicBool::new(false)), } } @@ -6359,6 +6368,13 @@ impl Worker { self } + /// Enable the candidate scalar scope consumer with canonical authoring. + #[doc(hidden)] + pub fn candidate_cancellation_scope_delivery(mut self, enabled: bool) -> Self { + self.allow_cancellation_scope_delivery = enabled; + self + } + /// Configure bounded retries for task-poll acquisition and worker heartbeats. pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self { self.retry_policy = policy; @@ -7292,6 +7308,7 @@ impl Worker { match decision { Ok(decision) if decision.cancellation_delivery.is_some() + || decision.cancellation_scope_delivery.is_some() || decision.cancellation_scope_opening.is_some() => { return Err(Error::CooperativeCancellationUnavailable( @@ -7865,13 +7882,16 @@ impl Worker { task: WorkflowTask, observation: Option<&CancellationRequest>, ) -> Result { - if self.allow_cancellation_scope_authoring && !self.cooperative_cancellation_enabled { + if (self.allow_cancellation_scope_authoring || self.allow_cancellation_scope_delivery) + && !self.cooperative_cancellation_enabled + { return Err(Error::CancellationScopeExecutionUnavailable); } validate_workflow_task_payloads(&task)?; - cooperative_cancellation::assert_cancellation_scope_replay_supported_with_authoring( + cooperative_cancellation::assert_cancellation_scope_replay_supported( &task.history_events, self.allow_cancellation_scope_authoring, + self.allow_cancellation_scope_delivery, )?; let cancellation_history = if let Some(observation) = observation { observation.validate_observation()?; @@ -7895,6 +7915,9 @@ impl Worker { .as_deref() .filter(|update_id| !update_id.is_empty()) { + if self.allow_cancellation_scope_delivery { + return Err(Error::CancellationScopeExecutionUnavailable); + } CancellationHistory::from_events( &task.history_events, task.run_id.as_deref().unwrap_or_default(), @@ -7925,7 +7948,7 @@ impl Worker { .filter(|identity| !identity.is_empty()) .or_else(|| (!task.task_id.is_empty()).then(|| task.task_id.clone())) .unwrap_or_default(); - let mut workflow_state = WorkflowState::new_with_identity_and_scopes( + let mut workflow_state = WorkflowState::new_with_identity_and_scope_delivery( task.history_events, task.workflow_id, task.run_id, @@ -7933,6 +7956,7 @@ impl Worker { task.payload_codec.clone(), resume_signal, self.allow_cancellation_scope_authoring, + self.allow_cancellation_scope_delivery, )?; workflow_state.history_budget = history_budget; workflow_state.workflow_command_identity = workflow_command_identity; @@ -7959,6 +7983,10 @@ impl Worker { let state = ctx.state.lock().map_err(|_| Error::WorkflowStatePoisoned)?; state.cancellation_delivery_intent.is_some() || state.cancellation_scope_opening.is_some() + || state + .scope_delivery + .as_ref() + .is_some_and(|replay| replay.intent.is_some()) }; if awaiting_canonical_boundary { // A private delivery intent never becomes an application error. @@ -8053,6 +8081,22 @@ impl Worker { } let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?; let state = ctx.state.lock().map_err(|_| Error::WorkflowStatePoisoned)?; + let scope_delivery = state + .scope_delivery + .as_ref() + .and_then(|replay| replay.intent.clone()); + if scope_delivery + .as_ref() + .is_some_and(|intent| commands.len() != intent.command_count) + { + return Err(invalid_recorded_history( + "cancellation_scope_pending_call_escaped", + scope_delivery.as_ref().unwrap().boundary.sequence, + "commands preceding the pending scoped call", + "commands after suspension", + "workflow cannot publish beyond an uncommitted scope delivery", + )); + } if let Some(opening) = &state.cancellation_scope_opening { if commands.len() != opening.command_count { return Err(invalid_recorded_history( @@ -8085,6 +8129,7 @@ impl Worker { message_stream_waits, cancellation_delivery: state.cancellation_delivery_intent.clone(), cancellation_scope_opening: state.cancellation_scope_opening.clone(), + cancellation_scope_delivery: scope_delivery, }) } @@ -8914,7 +8959,11 @@ impl WorkflowContext { .state .lock() .map_err(|_| Error::WorkflowStatePoisoned)?; - Ok(state.cancel_requested) + Ok(state.cancel_requested + || state + .scope_delivery + .as_ref() + .is_some_and(|replay| replay.contexts.contains_key(&self.cancellation_scope_id))) } /// Raise a typed cooperative cancellation at an author-controlled point. @@ -8926,6 +8975,26 @@ impl WorkflowContext { .state .lock() .map_err(|_| Error::WorkflowStatePoisoned)?; + if state.cancellation_shield_depth == 0 { + if let Some(replay) = &state.scope_delivery { + if let Some((context, _)) = replay.contexts.get(&self.cancellation_scope_id) { + let delivered = replay + .canonical + .deliveries + .values() + .find(|delivered| delivered.context.request_id() == context.request_id()) + .unwrap(); + return Err(Error::CancellationScopeRequested( + CancellationScopeRequested { + context: context + .clone() + .with_replay(Some(Arc::downgrade(&self.state))), + delivery: delivered.boundary.clone(), + }, + )); + } + } + } if state.cancel_requested && state.cancellation_shield_depth == 0 { return Err(state.cancellation_error()); } @@ -9652,6 +9721,22 @@ impl WorkflowContext { .state .lock() .map_err(|_| Error::WorkflowStatePoisoned)?; + if let Some(replay) = &state.scope_delivery { + if let Some((&sequence, _)) = replay + .canonical + .deliveries + .iter() + .find(|(sequence, _)| !replay.consumed.contains(sequence)) + { + return Err(invalid_recorded_history( + "cancellation_scope_delivery_unconsumed", + sequence, + "original scoped delivery", + "workflow completion", + "workflow completed without consuming its committed scope delivery", + )); + } + } if let Some(command) = state.recorded_commands.get(state.command_cursor) { return Err(Error::NonDeterministicReplay(ReplayFailure::new( "recorded_commands_unconsumed", @@ -9744,6 +9829,7 @@ struct WorkflowState { allow_cancellation_scope_authoring: bool, cancellation_scope_opening: Option, cancellation_scope_memberships: BTreeMap, + scope_delivery: Option, resume_signal: Option, recorded_commands: Vec, selection_markers: Vec, @@ -9809,9 +9895,32 @@ impl WorkflowState { resume_signal: Option, allow_cancellation_scope_authoring: bool, ) -> Result { - cooperative_cancellation::assert_cancellation_scope_replay_supported_with_authoring( + Self::new_with_identity_and_scope_delivery( + history, + workflow_id, + run_id, + task_queue, + payload_codec, + resume_signal, + allow_cancellation_scope_authoring, + false, + ) + } + + fn new_with_identity_and_scope_delivery( + history: Vec, + workflow_id: Option, + run_id: Option, + task_queue: String, + payload_codec: String, + resume_signal: Option, + allow_cancellation_scope_authoring: bool, + allow_cancellation_scope_delivery: bool, + ) -> Result { + cooperative_cancellation::assert_cancellation_scope_replay_supported( &history, allow_cancellation_scope_authoring, + allow_cancellation_scope_delivery, )?; let scopes = if allow_cancellation_scope_authoring { cancellation_scope::CancellationScopeHistory::read( @@ -9829,6 +9938,16 @@ impl WorkflowState { if allow_cancellation_scope_authoring && cancellation_history.request.is_some() { return Err(Error::CancellationScopeExecutionUnavailable); } + let scope_delivery = if allow_cancellation_scope_delivery { + Some(cancellation_scope_replay::ScopeReplay::read( + &history, + &scopes, + run_id.as_deref().unwrap_or_default(), + workflow_id.as_deref().unwrap_or_default(), + )?) + } else { + None + }; let mut recorded_commands = cancellation_history.bind_commands(recorded_commands( &history, &payload_codec, @@ -9837,27 +9956,30 @@ impl WorkflowState { run_id: run_id.clone(), }, )?)?; - for (sequence, opening) in scopes.openings { + for (sequence, opening) in &scopes.openings { if recorded_commands .iter() - .any(|command| command.sequence() == sequence) + .any(|command| command.sequence() == *sequence) { return Err(invalid_recorded_history( "invalid_cancellation_scope_history", - sequence, + *sequence, "distinct authored scope opening", "operation collision", "scope opening collides with a command", )); } recorded_commands.push(RecordedCommand::CancellationScope { - sequence, - scope_id: opening.scope_id, - parent_scope_id: opening.parent_scope_id, + sequence: *sequence, + scope_id: opening.scope_id.clone(), + parent_scope_id: opening.parent_scope_id.clone(), shield_parent: opening.shield_parent, }); } recorded_commands.sort_by_key(RecordedCommand::sequence); + if let Some(replay) = &scope_delivery { + replay.bind_commands(&mut recorded_commands)?; + } let selection_markers = recorded_selection_markers(&history)?; let cancelled_selection_members = recorded_selection_cancellations(&history)?; let recorded_continue_as_new = history @@ -9944,6 +10066,11 @@ impl WorkflowState { &recorded_commands, cancellation_history.delivery_index, ) + }) + .or_else(|| { + scope_delivery.as_ref().map(|_| { + cancellation_replay_clock::ReplayClock::new(&history, &recorded_commands, None) + }) }); Ok(Self { workflow_command_identity: String::new(), @@ -9968,6 +10095,7 @@ impl WorkflowState { allow_cancellation_scope_authoring, cancellation_scope_opening: None, cancellation_scope_memberships: scopes.memberships, + scope_delivery, resume_signal, recorded_commands, selection_markers, @@ -12832,7 +12960,7 @@ impl ActivityCall { }; let cursor = state.command_cursor; - if let Err(error) = ctx.validate_scope_membership(&state, cursor) { + if let Err(error) = ctx.validate_scope_membership(&mut state, cursor) { return Poll::Ready(Err(error)); } let recorded = @@ -13068,7 +13196,7 @@ impl Future for TimerCall { }; let cursor = state.command_cursor; - if let Err(error) = ctx.validate_scope_membership(&state, cursor) { + if let Err(error) = ctx.validate_scope_membership(&mut state, cursor) { return Poll::Ready(Err(error)); } let recorded = match state.cancellation_replay_command(cursor, CancellationCallKind::Timer) @@ -13215,7 +13343,7 @@ impl Future for ConditionWaitCall { Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)), }; let initial_cursor = state.command_cursor; - if let Err(error) = ctx.validate_scope_membership(&state, initial_cursor) { + if let Err(error) = ctx.validate_scope_membership(&mut state, initial_cursor) { return Poll::Ready(Err(error)); } let recorded = match state @@ -13481,7 +13609,7 @@ impl ChildWorkflowCall { }; let cursor = state.command_cursor; - if let Err(error) = ctx.validate_scope_membership(&state, cursor) { + if let Err(error) = ctx.validate_scope_membership(&mut state, cursor) { return Poll::Ready(Err(error)); } let recorded = match state.cancellation_replay_command(cursor, CancellationCallKind::Child) @@ -13724,7 +13852,7 @@ impl SignalCall { }; let cursor = state.command_cursor; - if let Err(error) = ctx.validate_scope_membership(&state, cursor) { + if let Err(error) = ctx.validate_scope_membership(&mut state, cursor) { return Poll::Ready(Err(error)); } let recorded = match state.cancellation_replay_command(cursor, CancellationCallKind::Signal) @@ -15913,6 +16041,13 @@ fn workflow_failure_command( json!({"reason": "cancelled", "request_id": cancellation.request.request_id, "cleanup_deadline_at": cancellation.request.cleanup_deadline_at}), ), + Error::CancellationScopeRequested(cancellation) => ( + "WorkflowCancellationRequested", + "durable_workflow::CancellationScopeRequested", + json!({"reason":"scope_cancelled", "request_id":cancellation.context.request_id(), + "cleanup_deadline_at":cancellation.context.deadline().to_rfc3339(), + "cancellation":cancellation.context.to_value()}), + ), Error::NonDeterministicReplay(_) => ( "NonDeterministicReplay", "durable_workflow::Error", @@ -15927,9 +16062,9 @@ fn workflow_failure_command( Error::SagaCompensationFailed(failure) => { workflow_error_non_retryable(&failure.compensation_failure) } - Error::WorkflowCancellationRequested(_) | Error::CooperativeCancellationRequested(_) => { - true - } + Error::WorkflowCancellationRequested(_) + | Error::CooperativeCancellationRequested(_) + | Error::CancellationScopeRequested(_) => true, Error::NonDeterministicReplay(_) => true, _ => false, }; @@ -15970,9 +16105,9 @@ fn workflow_error_type(error: &Error) -> &'static str { }, Error::ParallelFailed(_) => "ParallelFailed", Error::SagaCompensationFailed(_) => "SagaCompensationFailed", - Error::WorkflowCancellationRequested(_) | Error::CooperativeCancellationRequested(_) => { - "WorkflowCancellationRequested" - } + Error::WorkflowCancellationRequested(_) + | Error::CooperativeCancellationRequested(_) + | Error::CancellationScopeRequested(_) => "WorkflowCancellationRequested", Error::NonDeterministicReplay(_) => "NonDeterministicReplay", _ => "RustWorkflowError", } @@ -15988,6 +16123,7 @@ fn workflow_error_non_retryable(error: &Error) -> bool { } Error::WorkflowCancellationRequested(_) | Error::CooperativeCancellationRequested(_) + | Error::CancellationScopeRequested(_) | Error::NonDeterministicReplay(_) => true, _ => false, } @@ -16361,6 +16497,7 @@ mod tests { mod cancellation_scope_authoring; mod cancellation_scope_history; mod cancellation_scope_opening; + mod cancellation_scope_replay; mod child_workflow_policies; mod cooperative_cancellation; mod runtime_payloads; diff --git a/src/tests/cancellation_scope_history.rs b/src/tests/cancellation_scope_history.rs index 77f15d7..80fc200 100644 --- a/src/tests/cancellation_scope_history.rs +++ b/src/tests/cancellation_scope_history.rs @@ -28,10 +28,30 @@ fn scope_response(path: &str, body: &str, number: usize) -> Option<(&'static str if case == "budget" { thread::sleep(Duration::from_millis(900)); } - if case == "lost-ack" && path.ends_with("/prepare") && number == 1 { + if matches!(case, "lost-ack" | "worker-lost-ack") && path.ends_with("/prepare") && number == 1 { return Some(("invalid-status", String::new())); } let mut source = scalar_fixture(); + if case.starts_with("worker") { + // Synthetic transport time is safely ahead of the actual request budget. + // The fixture has no admitted members whose descriptor contains dates. + source = + serde_json::from_str(&source.to_string().replace("2026-10-04", "2029-10-04")).unwrap(); + if path.ends_with("/poll") { + let task = claim(&source); + return Some(("200 OK", json!({"protocol_version":"1.20", "task":{ + "task_id":task.task_id, "run_id":task.run_id, "workflow_id":task.workflow_id, + "workflow_type":task.workflow_type, "lease_owner":task.lease_owner, + "workflow_task_attempt":task.workflow_task_attempt, "payload_codec":DEFAULT_CODEC, + "history_events":source["history"].as_array().unwrap().iter() + .take_while(|event| event["event_type"] != "CancellationScopeDeliveryPrepared") + .cloned().collect::>() + }}).to_string())); + } + if path.ends_with("/complete") { + return Some(("200 OK", "{}".into())); + } + } let delivering = path.ends_with("/deliver") || (path.ends_with("/history") && body.contains("deliver-")); if case == "substituted" && delivering { @@ -153,6 +173,88 @@ fn scope_client(server: &MockWorkerServer, case: &str) -> Client { .unwrap() } +#[tokio::test] +async fn cancellation_scope_replay_worker_coordinates_original_claim_and_lost_ack() { + for case in ["worker", "worker-lost-ack"] { + let server = scope_server(); + let cleanup = Arc::new(AtomicUsize::new(0)); + let observed = Arc::clone(&cleanup); + let mut worker = Worker::new(scope_client(&server, case), "queue") + .worker_id("original") + .poll_timeout(Duration::ZERO) + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + worker.register_workflow("scope", move |ctx, _| { + let observed = Arc::clone(&observed); + async move { + ctx.cancellation_scope(false, move |outer| async move { + outer + .cancellation_scope(false, move |inner| async move { + match inner.sleep(Duration::from_secs(3600)).await { + Err(Error::CancellationScopeRequested(cancellation)) => { + observed.fetch_add(1, Ordering::SeqCst); + assert_eq!( + cancellation.context.remaining()?, + Duration::from_micros(22_623_456) + ); + let _shield = inner.cancellation_shield()?; + inner.sleep(Duration::from_secs(1)).await?; + Ok(Value::Null) + } + Err(error) => Err(error), + Ok(()) => panic!("cancelled timer cannot complete ordinarily"), + } + }) + .await + }) + .await + } + }); + assert_eq!( + worker.poll_workflow_once().await.unwrap(), + ManagedPollOutcome::Handled + ); + assert_eq!(cleanup.load(Ordering::SeqCst), 1); + let requests = server.requests.lock().unwrap(); + let preparations: Vec<_> = requests + .iter() + .filter(|request| request.path.ends_with("/prepare")) + .collect(); + assert_eq!( + preparations.len(), + if case == "worker-lost-ack" { 2 } else { 1 } + ); + if preparations.len() == 2 { + assert_eq!(preparations[0].body, preparations[1].body); + } + assert_eq!( + requests + .iter() + .filter(|request| request.path.ends_with("/deliver")) + .count(), + 1 + ); + assert_eq!( + requests + .iter() + .filter(|request| request.path.ends_with("/history")) + .count(), + 4 + ); + let completion = requests.last().unwrap(); + assert_eq!( + completion.path, + format!("/{case}/api/worker/workflow-tasks/task/one/complete") + ); + let body: Value = serde_json::from_str(&completion.body).unwrap(); + assert_eq!(body["workflow_task_attempt"], 4); + assert_eq!(body["lease_owner"], "original"); + assert_eq!(body["commands"][0]["type"], "start_timer"); + assert_eq!(body["commands"][0]["delay_seconds"], 1); + } +} + #[tokio::test] async fn scope_history_prepare_and_delivery_prove_original_claim_and_all_pages_after_lost_ack() { for case in ["valid", "lost-ack"] { diff --git a/src/tests/cancellation_scope_replay.rs b/src/tests/cancellation_scope_replay.rs new file mode 100644 index 0000000..585cfc7 --- /dev/null +++ b/src/tests/cancellation_scope_replay.rs @@ -0,0 +1,211 @@ +use super::*; + +fn fixture() -> Value { + let source: Value = serde_json::from_str(include_str!( + "../../tests/fixtures/populated-scope-single-calls.json" + )) + .unwrap(); + source["timer"].clone() +} + +fn task(value: &Value) -> WorkflowTask { + serde_json::from_value(json!({"task_id":"task-one", "workflow_id":value["task"]["workflow_id"], + "run_id":value["task"]["run_id"], "workflow_type":"scope-replay", "payload_codec":DEFAULT_CODEC, + "lease_owner":"original", "workflow_task_attempt":1, "history_events":value["history"]})).unwrap() +} + +fn worker(cleanup: Arc, changed_delay: bool) -> Worker { + let mut worker = Worker::new(Client::new("http://unused.invalid").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + worker.register_workflow("scope-replay", move |ctx, _| { + let cleanup = Arc::clone(&cleanup); + async move { + let parent = ctx.clone(); + let result = ctx.cancellation_scope(false, move |outer| async move { + outer.cancellation_scope(false, move |inner| async move { + assert_eq!(inner.activity("prior-step", json!([])).await?, json!("prior-value")); + match inner.sleep(Duration::from_secs(if changed_delay { 3599 } else { 3600 })).await { + Ok(()) => Ok(json!("ordinary-result")), + Err(Error::CancellationScopeRequested(cancellation)) => { + cleanup.fetch_add(1, Ordering::SeqCst); + assert!(inner.is_cancellation_requested()?); + assert_eq!(inner.scoped_cancellation_context()?.unwrap(), cancellation.context); + let before = cancellation.context.remaining()?; + assert_eq!(before, Duration::from_secs(21)); + let _shield = inner.cancellation_shield()?; + inner.sleep(Duration::from_secs(2)).await?; + let after = cancellation.context.remaining()?; + assert_eq!(after, Duration::from_secs(19)); + Ok(json!({"cleaned":true, "request_id":cancellation.context.request_id()})) + } + Err(error) => Err(error), + } + }).await + }).await?; + assert!(!parent.is_cancellation_requested()?); + assert!(parent.scoped_cancellation_context()?.is_none()); + parent.sleep(Duration::from_secs(1)).await?; + Ok(result) + } + }); + worker +} + +fn prefix(value: &Value, before: &str) -> Value { + let mut value = value.clone(); + let events = value["history"].as_array_mut().unwrap(); + let index = events + .iter() + .position(|event| event["event_type"] == before) + .unwrap(); + events.truncate(index); + value +} + +fn append(value: &mut Value, kind: &str, payload: Value, timestamp: &str) { + let events = value["history"].as_array_mut().unwrap(); + let sequence = events.last().unwrap()["sequence"].as_u64().unwrap() + 1; + let namespace = events[0]["namespace"].clone(); + events.push( + json!({"id":format!("cleanup-event-{sequence}"), "sequence":sequence, + "namespace":namespace, "event_type":kind, "payload":payload, "timestamp":timestamp}), + ); +} + +#[test] +fn cancellation_scope_replay_waits_for_original_prepare_and_committed_delivery() { + let original = fixture(); + let cleanup = Arc::new(AtomicUsize::new(0)); + let worker = worker(Arc::clone(&cleanup), false); + for before in [ + "CancellationScopeDeliveryPrepared", + "CancellationScopeDelivered", + ] { + let decision = worker + .execute_workflow_task_decision(task(&prefix(&original, before))) + .unwrap(); + let intent = decision.cancellation_scope_delivery.unwrap(); + assert_eq!(intent.boundary.sequence, 4); + assert_eq!(intent.boundary.call_kind, CancellationCallKind::Timer); + assert!(decision.commands.is_empty()); + assert_eq!(cleanup.load(Ordering::SeqCst), 0); + } + let decision = worker + .execute_workflow_task_decision(task(&original)) + .unwrap(); + assert!(decision.cancellation_scope_delivery.is_none()); + assert_eq!(decision.commands.len(), 1); + assert_eq!(decision.commands[0]["type"], "start_timer"); + assert_eq!(decision.commands[0]["delay_seconds"], 2); + assert!(decision.commands[0]["cancellation_scope_id"].is_string()); + assert_eq!(cleanup.load(Ordering::SeqCst), 1); +} + +#[test] +fn cancellation_scope_replay_replacement_keeps_context_clock_and_parent_unaffected() { + let mut source = fixture(); + let scope = source["history"] + .as_array() + .unwrap() + .iter() + .find(|event| event["event_type"] == "CancellationScopeDelivered") + .unwrap()["payload"]["scope_id"] + .clone(); + append( + &mut source, + "TimerScheduled", + json!({"sequence":5, "timer_id":"cleanup-timer", + "delay_seconds":2, "cancellation_scope_id":scope}), + "2026-10-04T00:00:09.123456Z", + ); + append( + &mut source, + "TimerFired", + json!({"sequence":5, "timer_id":"cleanup-timer", "delay_seconds":2, + "cancellation_scope_id":scope}), + "2026-10-04T00:00:11.123456Z", + ); + let replacement = worker(Arc::new(AtomicUsize::new(0)), false).worker_id("replacement"); + let mut claim = task(&source); + claim.lease_owner = Some("replacement".into()); + claim.workflow_task_attempt = 2; + let decision = replacement.execute_workflow_task_decision(claim).unwrap(); + assert_eq!(decision.commands.len(), 1); + assert_eq!(decision.commands[0]["delay_seconds"], 1); + assert!(decision.commands[0].get("cancellation_scope_id").is_none()); + append( + &mut source, + "TimerScheduled", + json!({"sequence":6, "timer_id":"parent-timer", "delay_seconds":1}), + "2026-10-04T00:00:11.123456Z", + ); + append( + &mut source, + "TimerFired", + json!({"sequence":6, "timer_id":"parent-timer", "delay_seconds":1}), + "2026-10-04T00:00:12.123456Z", + ); + let decision = replacement + .execute_workflow_task_decision(task(&source)) + .unwrap(); + assert_eq!(decision.commands.len(), 1); + assert_eq!(decision.commands[0]["type"], "complete_workflow"); +} + +#[test] +fn cancellation_scope_replay_descriptor_change_cannot_enter_cleanup() { + let cleanup = Arc::new(AtomicUsize::new(0)); + let result = + worker(Arc::clone(&cleanup), true).execute_workflow_task_decision(task(&fixture())); + assert!( + matches!(result, Err(Error::NonDeterministicReplay(failure)) if failure.reason == "timer_delay_mismatch") + ); + assert_eq!(cleanup.load(Ordering::SeqCst), 0); +} + +#[test] +fn cancellation_scope_replay_default_and_incomplete_opt_in_refuse_before_factory() { + for (authoring, delivery) in [(false, false), (true, false), (false, true)] { + let calls = Arc::new(AtomicUsize::new(0)); + let observed = Arc::clone(&calls); + let mut worker = Worker::new(Client::new("http://unused.invalid").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(authoring) + .candidate_cancellation_scope_delivery(delivery); + worker.register_workflow("scope-replay", move |_, _| { + observed.fetch_add(1, Ordering::SeqCst); + async { Ok(json!("must-not-enter")) } + }); + assert!(matches!( + worker.execute_workflow_task_decision(task(&fixture())), + Err(Error::CancellationScopeExecutionUnavailable) + )); + assert_eq!(calls.load(Ordering::SeqCst), 0); + } +} + +#[test] +fn cancellation_scope_replay_bad_projection_refuses_before_factory() { + let mut source = fixture(); + for event in source["history"].as_array_mut().unwrap() { + if event["event_type"] == "CancellationScopeDeliveryPrepared" { + event["payload"]["timer_members"][0]["descriptor_hash"] = json!("0".repeat(64)); + } + } + let calls = Arc::new(AtomicUsize::new(0)); + let observed = Arc::clone(&calls); + let mut worker = Worker::new(Client::new("http://unused.invalid").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + worker.register_workflow("scope-replay", move |_, _| { + observed.fetch_add(1, Ordering::SeqCst); + async { Ok(json!("must-not-enter")) } + }); + assert!(worker + .execute_workflow_task_decision(task(&source)) + .is_err()); + assert_eq!(calls.load(Ordering::SeqCst), 0); +} From d311e46125b4fc48a330c9f2d5eaef9ce0858e06 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 01:07:48 +0000 Subject: [PATCH 47/58] feat: bind scope cleanup timers to original delivery and qualify replacement --- .github/workflows/ci.yml | 1 + docker-compose.native-cancellation.yml | 1 + src/cancellation_scope_history.rs | 59 +++++++ src/cancellation_scope_replay.rs | 48 ++++++ src/lib.rs | 12 ++ src/tests/cancellation_scope_replay.rs | 61 +++++++- tests/cooperative_server.rs | 146 ++++++++++++++++++ .../fixtures/source/native-scope-request.php | 31 ++++ 8 files changed, 358 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/source/native-scope-request.php diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45f98e6..d2b6177 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -262,6 +262,7 @@ jobs: if [ -z "$NATIVE_CANDIDATE_SHA" ]; then arguments+=(--skip server_scope_opening_proves_real_native_tree_on_original_claim) arguments+=(--skip server_scope_authoring_replays_nested_tree_and_deferred_membership_on_replacement) + arguments+=(--skip server_scope_cleanup_preserves_original_delivery_and_budget_after_replacement) fi cargo test --test cooperative_server -- "${arguments[@]}" 2>&1 | tee cooperative-results.log - name: Retain bounded scenario evidence diff --git a/docker-compose.native-cancellation.yml b/docker-compose.native-cancellation.yml index 4393085..8ca3cdd 100644 --- a/docker-compose.native-cancellation.yml +++ b/docker-compose.native-cancellation.yml @@ -6,6 +6,7 @@ services: server: volumes: - ${DURABLE_WORKFLOW_NATIVE_SOURCE:?exact Native checkout}:/app/vendor/durable-workflow/workflow:ro + - ./tests/fixtures/source/native-scope-request.php:/app/sdk-source-fixtures/native-scope-request.php:ro worker: volumes: - ${DURABLE_WORKFLOW_NATIVE_SOURCE:?exact Native checkout}:/app/vendor/durable-workflow/workflow:ro diff --git a/src/cancellation_scope_history.rs b/src/cancellation_scope_history.rs index 493f6f7..448151a 100644 --- a/src/cancellation_scope_history.rs +++ b/src/cancellation_scope_history.rs @@ -63,6 +63,59 @@ pub(super) fn timestamp(value: &Value) -> Result> { .ok_or_else(|| invalid("scope authority requires an original timestamp with timezone")) } +fn cleanup_timer(event: &HistoryEvent, prefix: &[HistoryEvent]) -> Result { + let p = &event.payload; + let Some(snapshot) = p.get("cancellation_cleanup") else { + return Ok(false); + }; + let matches: Vec<_> = prefix + .iter() + .filter(|row| { + row.event_type == "CancellationScopeDelivered" + && row.raw.get("id") == snapshot.get("delivery_history_event_id") + }) + .collect(); + if matches.len() != 1 { + return Err(invalid( + "cleanup timer lacks its earlier canonical delivery", + )); + } + let delivery = matches[0]; + let original = &delivery.payload; + let context = ScopedCancellationContext::from_value(&original["cancellation"])?; + let expected = json!({"scope_id":context.scope_id(), "operation_scope_id":context.scope_id(), + "request_id":context.request_id(), "root_request_id":context.root_context().root_request_id(), + "delivery_history_event_id":event_id(delivery)?, + "preparation_history_event_id":original["preparation_history_event_id"], + "cleanup_deadline_at":canonical_time(context.deadline()), + "authority_deadline_at":original["authority_deadline_at"]}); + let ceiling = timestamp(&original["authority_deadline_at"])?; + let boundary = CancellationDelivery::from_payload(&json!({ + "workflow_command_id":context.request_id(), "sequence":original["sequence"], + "call_kind":original["call_kind"], "sequence_span":original["sequence_span"], + "operation_sequence":original["operation_sequence"], "operation_sequence_span":original["operation_sequence_span"]}))?; + if snapshot != &expected + || p["cancellation_scope_id"] != context.scope_id() + || p["sequence"] + .as_u64() + .is_none_or(|sequence| sequence < boundary.sequence + boundary.sequence_span) + || event.raw["sequence"].as_u64().is_none_or(|sequence| { + delivery.raw["sequence"] + .as_u64() + .is_none_or(|prior| prior >= sequence) + }) + || event_time(event)? < event_time(delivery)? + || event_time(event)? >= ceiling + || timestamp(&p["fire_at"])? >= ceiling + || !p["timer_kind"].is_null() + { + return Err(invalid( + "cleanup timer changes its original delivery or authority ceiling", + )); + } + Ok(true) +} + fn event_time(event: &HistoryEvent) -> Result> { timestamp( event @@ -425,6 +478,9 @@ pub(super) fn members_from_prefix( return Err(invalid("scope timer changes its original descriptor")); } timer_sequences.insert(sequence.as_u64().unwrap(), kind.clone()); + if cleanup_timer(event, prefix)? { + continue; + } members.push(json!({"sequence":sequence, "timer_id":id, "descriptor_hash":descriptor_hash(json!([scope, event_id(event)?, sequence, id, p["delay_seconds"], p["fire_at"], kind, p["condition_wait_id"], @@ -755,6 +811,9 @@ impl CommittedCancellationScopeHistory { for (index, event) in history.iter().enumerate() { let kind = event.event_type.as_str(); let p = &event.payload; + if kind == "TimerScheduled" { + cleanup_timer(event, &history[..index])?; + } if kind == "CancellationScopeOpened" { opened.insert(text(p, "scope_id")?); } diff --git a/src/cancellation_scope_replay.rs b/src/cancellation_scope_replay.rs index 8d719ad..0f13c61 100644 --- a/src/cancellation_scope_replay.rs +++ b/src/cancellation_scope_replay.rs @@ -26,6 +26,7 @@ pub(super) struct ScopeReplay { pub active_scope: String, pub consumed: BTreeSet, pub contexts: BTreeMap)>, + pub cleanup_timers: BTreeMap, pub intent: Option, } @@ -74,6 +75,10 @@ impl ScopeReplay { active_scope: "root".into(), consumed: BTreeSet::new(), contexts: BTreeMap::new(), + cleanup_timers: history.iter().filter(|row| row.event_type == "TimerScheduled") + .filter_map(|row| row.payload.get("cancellation_cleanup").map(|snapshot| + (row.payload["sequence"].as_u64().unwrap(), json!({"scope_id":snapshot["scope_id"], + "request_id":snapshot["request_id"], "delivery_history_event_id":snapshot["delivery_history_event_id"]})))).collect(), intent: None, }) } @@ -112,6 +117,49 @@ impl ScopeReplay { } impl WorkflowState { + pub(super) fn scope_cleanup_timer_proof(&self, scope: &str) -> Result> { + if self.cancellation_shield_depth == 0 { + return Ok(None); + } + let Some(replay) = &self.scope_delivery else { + return Ok(None); + }; + let Some((context, _)) = replay.contexts.get(scope) else { + return Ok(None); + }; + let delivered = replay + .canonical + .deliveries + .values() + .find(|delivery| { + delivery.context == *context + && replay.consumed.contains(&delivery.boundary.sequence) + }) + .ok_or_else(|| { + Error::InvalidCooperativeCancellation( + "scoped cleanup lacks its consumed original delivery".into(), + ) + })?; + Ok(Some( + json!({"scope_id":scope, "request_id":context.request_id(), + "delivery_history_event_id":delivered.event.raw["id"]}), + )) + } + + pub(super) fn validate_scope_cleanup_timer(&self, scope: &str, sequence: u64) -> Result<()> { + let original = self + .scope_delivery + .as_ref() + .and_then(|replay| replay.cleanup_timers.get(&sequence)); + if original != self.scope_cleanup_timer_proof(scope)?.as_ref() { + return Err(invalid( + sequence, + "cleanup timer changed its original delivery or shielding", + )); + } + Ok(()) + } + pub(super) fn prepare_scalar_scope_cancellation( &mut self, index: usize, diff --git a/src/lib.rs b/src/lib.rs index 2801f2a..b44dc57 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13213,6 +13213,11 @@ impl Future for TimerCall { parallel_group_path, .. } => { + if let Err(error) = + state.validate_scope_cleanup_timer(&ctx.cancellation_scope_id, sequence) + { + return Poll::Ready(Err(error)); + } if let Err(error) = ensure_parallel_path_matches( sequence, parallel_group_path.as_deref(), @@ -13268,6 +13273,13 @@ impl Future for TimerCall { ("type".to_string(), json!("start_timer")), ("delay_seconds".to_string(), json!(requested_delay)), ]); + match state.scope_cleanup_timer_proof(&ctx.cancellation_scope_id) { + Ok(Some(proof)) => { + command.insert("cancellation_cleanup".into(), proof); + } + Ok(None) => {} + Err(error) => return Poll::Ready(Err(error)), + } apply_parallel_group_path(&mut command, &self.parallel_group_path); ctx.apply_scope_membership(&mut command); match state.prepare_scalar_cancellation( diff --git a/src/tests/cancellation_scope_replay.rs b/src/tests/cancellation_scope_replay.rs index 585cfc7..376cab6 100644 --- a/src/tests/cancellation_scope_replay.rs +++ b/src/tests/cancellation_scope_replay.rs @@ -74,6 +74,23 @@ fn append(value: &mut Value, kind: &str, payload: Value, timestamp: &str) { ); } +fn cleanup_snapshot(value: &Value) -> Value { + let delivery = value["history"] + .as_array() + .unwrap() + .iter() + .find(|row| row["event_type"] == "CancellationScopeDelivered") + .unwrap(); + let context = + ScopedCancellationContext::from_value(&delivery["payload"]["cancellation"]).unwrap(); + json!({"scope_id":context.scope_id(), "operation_scope_id":context.scope_id(), + "request_id":context.request_id(), "root_request_id":context.root_context().root_request_id(), + "delivery_history_event_id":delivery["id"], + "preparation_history_event_id":delivery["payload"]["preparation_history_event_id"], + "cleanup_deadline_at":context.deadline().to_rfc3339_opts(chrono::SecondsFormat::Micros, true), + "authority_deadline_at":delivery["payload"]["authority_deadline_at"]}) +} + #[test] fn cancellation_scope_replay_waits_for_original_prepare_and_committed_delivery() { let original = fixture(); @@ -100,12 +117,20 @@ fn cancellation_scope_replay_waits_for_original_prepare_and_committed_delivery() assert_eq!(decision.commands[0]["type"], "start_timer"); assert_eq!(decision.commands[0]["delay_seconds"], 2); assert!(decision.commands[0]["cancellation_scope_id"].is_string()); + let snapshot = cleanup_snapshot(&original); + assert_eq!( + decision.commands[0]["cancellation_cleanup"], + json!({ + "scope_id":snapshot["scope_id"], "request_id":snapshot["request_id"], + "delivery_history_event_id":snapshot["delivery_history_event_id"]}) + ); assert_eq!(cleanup.load(Ordering::SeqCst), 1); } #[test] fn cancellation_scope_replay_replacement_keeps_context_clock_and_parent_unaffected() { let mut source = fixture(); + let snapshot = cleanup_snapshot(&source); let scope = source["history"] .as_array() .unwrap() @@ -117,7 +142,8 @@ fn cancellation_scope_replay_replacement_keeps_context_clock_and_parent_unaffect &mut source, "TimerScheduled", json!({"sequence":5, "timer_id":"cleanup-timer", - "delay_seconds":2, "cancellation_scope_id":scope}), + "delay_seconds":2, "cancellation_scope_id":scope, + "fire_at":"2026-10-04T00:00:11.123456Z", "cancellation_cleanup":snapshot}), "2026-10-04T00:00:09.123456Z", ); append( @@ -154,6 +180,39 @@ fn cancellation_scope_replay_replacement_keeps_context_clock_and_parent_unaffect assert_eq!(decision.commands[0]["type"], "complete_workflow"); } +#[test] +fn cancellation_scope_replay_cleanup_timer_refuses_changed_authority_before_factory() { + for field in [ + "request_id", + "delivery_history_event_id", + "authority_deadline_at", + "cleanup_deadline_at", + "operation_scope_id", + ] { + let mut value = fixture(); + let mut snapshot = cleanup_snapshot(&value); + let scope = snapshot["scope_id"].clone(); + snapshot[field] = json!("changed"); + append( + &mut value, + "TimerScheduled", + json!({"sequence":5, "timer_id":"cleanup-timer", + "delay_seconds":2, "cancellation_scope_id":scope, + "fire_at":"2026-10-04T00:00:11.123456Z", "cancellation_cleanup":snapshot}), + "2026-10-04T00:00:09.123456Z", + ); + let factories = Arc::new(AtomicUsize::new(0)); + let called = Arc::clone(&factories); + let mut worker = worker(Arc::new(AtomicUsize::new(0)), false); + worker.register_workflow("scope-replay", move |_, _| { + called.fetch_add(1, Ordering::SeqCst); + async { Ok(Value::Null) } + }); + assert!(worker.execute_workflow_task_decision(task(&value)).is_err()); + assert_eq!(factories.load(Ordering::SeqCst), 0); + } +} + #[test] fn cancellation_scope_replay_descriptor_change_cannot_enter_cleanup() { let cleanup = Arc::new(AtomicUsize::new(0)); diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index 50db5c6..42d6992 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -841,6 +841,152 @@ async fn server_scope_authoring_replays_nested_tree_and_deferred_membership_on_r println!("Native scope authoring and replacement replay: {final_history}"); } +fn request_native_scope_fixture(handle: &WorkflowHandle, scope: &str) -> Value { + use std::io::Write; + use std::process::{Command, Stdio}; + let mut process = Command::new("timeout") + .args([ + "--kill-after=1", + "10", + "docker", + "compose", + "exec", + "-T", + "--user", + "1000:1000", + "--env", + "DURABLE_WORKFLOW_NATIVE_SCOPE_FIXTURE=1", + "server", + "timeout", + "--kill-after=1", + "5", + "php", + "/app/sdk-source-fixtures/native-scope-request.php", + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let input = serde_json::to_vec(&json!({"run_id":handle.run_id.as_deref().unwrap(), + "workflow_id":handle.workflow_id, "scope_id":scope})) + .unwrap(); + process.stdin.take().unwrap().write_all(&input).unwrap(); + let result = process.wait_with_output().unwrap(); + assert!( + result.status.success(), + "Native scope fixture: {}", + String::from_utf8_lossy(&result.stderr) + ); + serde_json::from_slice(&result.stdout).unwrap() +} + +fn scope_cleanup_worker(client: &Client, queue: &str, owner: &str) -> Worker { + let mut worker = Worker::new(client.clone(), queue) + .worker_id(owner) + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true) + .poll_timeout(Duration::from_millis(100)); + worker.register_workflow("tests.rust-candidate-scope-cleanup", |ctx, _| async move { + let _: String = ctx.side_effect(|| "original prefix".to_owned())?; + let (cancellation, metadata) = ctx + .cancellation_scope(false, |scope| async move { + let cancellation = match scope.sleep(Duration::from_secs(300)).await { + Err(Error::CancellationScopeRequested(request)) => request.context, + Err(error) => return Err(error), + Ok(()) => { + return Err(Error::InvalidCooperativeCancellation( + "original timer was not interrupted".into(), + )) + } + }; + assert!(scope.is_cancellation_requested()?); + let _shield = scope.cancellation_shield()?; + let metadata = scope.side_effect(|| cancellation.to_value())?; + scope.sleep(Duration::from_secs(1)).await?; + Ok((cancellation, metadata)) + }) + .await?; + assert!(!ctx.is_cancellation_requested()?); + assert!(ctx.scoped_cancellation_context()?.is_none()); + ctx.sleep(Duration::from_secs(1)).await?; + Ok(json!({"context":metadata, "remaining":cancellation.remaining()?.as_secs_f64()})) + }); + worker +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with the Native scope candidate"] +async fn server_scope_cleanup_preserves_original_delivery_and_budget_after_replacement() { + let client = client(); + let queue = format!( + "rust-cooperative-scope-boundary-{}", + durable_workflow::Uuid::new_v4().simple() + ); + let original = scope_cleanup_worker(&client, &queue, &format!("{queue}-original")); + original.register().await.unwrap(); + let handle = client + .start_workflow( + "tests.rust-candidate-scope-cleanup", + &queue, + &queue, + json!([]), + ) + .await + .unwrap(); + let first = tick_until(&original, &handle, "TimerScheduled").await; + assert_eq!(count(&first, "SideEffectRecorded"), 1); + let scope = first["events"] + .as_array() + .unwrap() + .iter() + .find(|row| row["event_type"] == "CancellationScopeOpened") + .unwrap()["payload"]["scope_id"] + .as_str() + .unwrap(); + let accepted = request_native_scope_fixture(&handle, scope); + assert_eq!(accepted, request_native_scope_fixture(&handle, scope)); + let cleaning = tick_until_count(&original, &handle, "SideEffectRecorded", 2).await; + assert_eq!(count(&cleaning, "TimerScheduled"), 2); + assert_eq!(count(&cleaning, "CancellationScopeDelivered"), 1); + let replacement = scope_cleanup_worker(&client, &queue, &format!("{queue}-replacement")); + replacement.register().await.unwrap(); + let final_history = tick_until(&replacement, &handle, "WorkflowCompleted").await; + for kind in [ + "CancellationScopeOpened", + "CancellationScopeRequested", + "CancellationScopeDeliveryPrepared", + "CancellationScopeDelivered", + "TimerCancelled", + "WorkflowCompleted", + ] { + assert_eq!(count(&final_history, kind), 1, "{kind}"); + } + assert_eq!(count(&final_history, "SideEffectRecorded"), 2); + assert_eq!(count(&final_history, "TimerScheduled"), 3); + assert_eq!(count(&final_history, "TimerFired"), 2); + for kind in [ + "CooperativeCancellationRequested", + "WorkflowCancelled", + "WorkflowFailed", + ] { + assert_eq!(count(&final_history, kind), 0, "{kind}"); + } + let result = handle + .result_selected_run(WorkflowResultOptions { + timeout: Duration::from_secs(5), + ..WorkflowResultOptions::default() + }) + .await + .unwrap(); + assert_eq!(result["context"], accepted["payload"]["cancellation"]); + assert!(result["remaining"] + .as_f64() + .is_some_and(|remaining| remaining > 0.0 && remaining < 30.0)); + println!("Native scope cleanup and replacement replay: {final_history}"); +} + #[tokio::test] #[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] async fn server_request_before_claim_replays_canonical_typed_cancellation() { diff --git a/tests/fixtures/source/native-scope-request.php b/tests/fixtures/source/native-scope-request.php new file mode 100644 index 0000000..457a751 --- /dev/null +++ b/tests/fixtures/source/native-scope-request.php @@ -0,0 +1,31 @@ +make(Kernel::class)->bootstrap(); +if (!$app->environment('testing')) { + throw new RuntimeException('Native scope admission requires the disposable testing application.'); +} +$input = json_decode(stream_get_contents(STDIN), true, flags: JSON_THROW_ON_ERROR); +if (!is_array($input) || count($input) !== 3 || !is_string($input['run_id'] ?? null) + || !is_string($input['workflow_id'] ?? null) || !is_string($input['scope_id'] ?? null) + || preg_match('/\Arust-cooperative-scope-boundary-[a-f0-9]{32}\z/', $input['workflow_id']) !== 1 + || $input['scope_id'] === '' || $input['scope_id'] === 'root') { + throw new RuntimeException('Native scope fixture requires its original synthetic workflow and scope.'); +} +$run = WorkflowRun::query()->where('namespace', 'default')->findOrFail($input['run_id']); +if ($run->workflow_instance_id !== $input['workflow_id']) { + throw new RuntimeException('Native scope fixture cannot cross its original workflow.'); +} +$accepted = CancellationScopeRequests::request($run, $input['scope_id'], '1.20', 30, 'connected scope receipt fixture'); +echo json_encode(['history_event_id' => $accepted->id, 'payload' => $accepted->payload], JSON_THROW_ON_ERROR), "\n"; From 5d23aea011e842c9f6002ec29fb4bc870ceb0acc Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 01:49:55 +0000 Subject: [PATCH 48/58] feat: replay scoped cancellation across complete parallel groups --- .github/workflows/ci.yml | 1 + docs/cooperative-cancellation-design.md | 22 +- src/cancellation_scope_replay.rs | 371 +- src/lib.rs | 11 + src/tests/cancellation_scope_replay.rs | 390 ++ tests/cooperative_server.rs | 98 +- tests/fixtures/populated-scope-groups.json | 3919 ++++++++++++++++++++ 7 files changed, 4770 insertions(+), 42 deletions(-) create mode 100644 tests/fixtures/populated-scope-groups.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2b6177..91aeb99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -263,6 +263,7 @@ jobs: arguments+=(--skip server_scope_opening_proves_real_native_tree_on_original_claim) arguments+=(--skip server_scope_authoring_replays_nested_tree_and_deferred_membership_on_replacement) arguments+=(--skip server_scope_cleanup_preserves_original_delivery_and_budget_after_replacement) + arguments+=(--skip server_scope_group_cleanup_preserves_original_delivery_and_budget_after_replacement) fi cargo test --test cooperative_server -- "${arguments[@]}" 2>&1 | tee cooperative-results.log - name: Retain bounded scenario evidence diff --git a/docs/cooperative-cancellation-design.md b/docs/cooperative-cancellation-design.md index edca62d..fe45dc0 100644 --- a/docs/cooperative-cancellation-design.md +++ b/docs/cooperative-cancellation-design.md @@ -159,11 +159,27 @@ major-version decision. Source qualification, protocol freeze and the published mixed-language cancellation acceptance scenario remain release gates. The candidate does not change a published artifact. +## Candidate scoped delivery + +The private scope opt-ins consume scalar calls and flat or nested `parallel` +groups of remote activities, timers, children and conditions. The complete group +is checked against its original definitions, policies, member addresses and paths +before workflow code receives `CancellationScopeRequested`. Earlier completed +members do not hide pending siblings. A committed condition is interrupted +without evaluating its predicate. Replacement replay retains the original group +span, cleanup sequence, context and deadline. + +Shielded cleanup timers carry only the original scope, request and delivery +references. Server derives their immutable authority and rejects a timer that +would reach its ceiling. Unsupported local, signal, selection, incomplete or +mixed-scope groups are refused before the workflow factory runs. Descendant +delivery and root/scope composition still require qualification. These opt-ins +remain disabled by default and do not advertise general scoped execution. + ## Remaining qualification -Nested scopes, competitive qualification and exact published artifacts still -need completion. The remaining-time helper needs connected exact-head -qualification. Do not subtract the host clock from the deadline in workflow code. +Complete the remaining scoped consumers, competitive qualification and exact +published artifacts. Do not subtract the host clock from the deadline in workflow code. The runtime continues enforcing the original deadline and fencing task and activity ownership. Rust local activities and worker affinity remain unsupported. diff --git a/src/cancellation_scope_replay.rs b/src/cancellation_scope_replay.rs index 0f13c61..4d699c4 100644 --- a/src/cancellation_scope_replay.rs +++ b/src/cancellation_scope_replay.rs @@ -1,4 +1,4 @@ -//! Candidate scalar scope delivery. Admission remains private and disabled by default. +//! Candidate scope delivery. Admission remains private and disabled by default. use super::*; use crate::cancellation_scope_history::CommittedCancellationScopeHistory; @@ -34,7 +34,7 @@ fn invalid(sequence: u64, detail: &str) -> Error { invalid_recorded_history( "cancellation_scope_call_mismatch", sequence, - "original scalar scoped call", + "original scoped call", "changed call or authority", detail, ) @@ -50,6 +50,157 @@ fn scalar(kind: CancellationCallKind) -> bool { ) } +fn scope_operation_terminal(kind: &str) -> bool { + matches!( + kind, + "ActivityCompleted" + | "ActivityFailed" + | "ActivityCancelled" + | "ActivityTimedOut" + | "TimerFired" + | "TimerCancelled" + | "ConditionWaitSatisfied" + | "ConditionWaitTimedOut" + | "ChildRunCompleted" + | "ChildRunFailed" + | "ChildRunCancelled" + | "ChildRunTerminated" + ) +} + +fn validate_scoped_group_definition( + state: &WorkflowState, + descriptor: &ParallelDescriptor, + recorded: &RecordedCommand, +) -> Result<()> { + let sequence = recorded.sequence(); + let path = match (&descriptor.operation, recorded) { + ( + ParallelOperation::Activity { + activity_type, + options, + .. + }, + RecordedCommand::Activity { + activity_type: original_type, + cancellation_policy, + options: original_options, + parallel_group_path, + .. + }, + ) => { + let options = options.validate().map_err(Error::InvalidActivityOptions)?; + if original_type + .as_ref() + .is_some_and(|original| original != activity_type) + || cancellation_policy + != options + .cancellation_policy + .unwrap_or(CancellationPolicy::TryCancel) + .as_str() + { + return Err(invalid( + sequence, + "scoped activity changed its original type or cancellation policy", + )); + } + if let Some(original) = original_options { + let queue = RecordedSnapshotValue::Known(Some( + options + .task_queue + .clone() + .unwrap_or_else(|| state.task_queue.clone()), + )); + if !original.task_queue.matches_current(&queue) + || !original + .execution_mode + .matches_current(&RecordedSnapshotValue::Known(None)) + || !original + .retry_policy + .matches_current(¤t_activity_retry_snapshot(&options)) + { + return Err(invalid(sequence, "scoped activity changed its original queue, execution mode or retry policy")); + } + } + parallel_group_path + } + ( + ParallelOperation::Timer(delay), + RecordedCommand::Timer { + delay_seconds, + parallel_group_path, + .. + }, + ) => { + if delay + .as_secs() + .checked_add(u64::from(delay.subsec_nanos() > 0)) + != Some(*delay_seconds) + { + return Err(invalid(sequence, "scoped timer changed its original delay")); + } + parallel_group_path + } + ( + ParallelOperation::ChildWorkflow { + workflow_type, + options, + .. + }, + RecordedCommand::ChildWorkflow { + workflow_type: original_type, + policies, + parallel_group_path, + .. + }, + ) => { + if original_type + .as_ref() + .is_some_and(|original| original != workflow_type) + { + return Err(invalid( + sequence, + "scoped child changed its original workflow type", + )); + } + ensure_child_policies_match(sequence, policies, options)?; + parallel_group_path + } + ( + ParallelOperation::Condition { options, .. }, + RecordedCommand::ConditionWait { + condition_key, + predicate_identity, + timeout_seconds, + parallel_group_path, + .. + }, + ) => { + let options = options + .validate() + .map_err(Error::InvalidConditionWaitOptions)?; + if condition_key.as_deref() != Some(options.condition_key.as_str()) + || predicate_identity != &options.predicate_identity + || timeout_seconds != &options.timeout_seconds + { + return Err(invalid( + sequence, + "scoped condition changed its original key, predicate identity or timeout", + )); + } + // Delivery interrupts the original occurrence without evaluating the predicate. + parallel_group_path + } + _ => { + return Err(invalid( + sequence, + "scoped group changed its original member kind", + )) + } + }; + ensure_parallel_path_matches(sequence, path.as_deref(), &descriptor.group_path) +} + impl ScopeReplay { pub fn read( history: &[HistoryEvent], @@ -59,8 +210,28 @@ impl ScopeReplay { ) -> Result { let canonical = CommittedCancellationScopeHistory::read(history, run, workflow)?; for preparation in canonical.preparations.values() { - if !scalar(preparation.boundary.call_kind) - || preparation.boundary.sequence_span != 1 + let boundary = &preparation.boundary; + let supported = if boundary.call_kind == CancellationCallKind::Parallel { + history + .iter() + .filter(|event| { + durable_event_sequence(event) + .is_some_and(|sequence| boundary.interrupts(sequence)) + }) + .all(|event| match event.event_type.as_str() { + "SignalWaitOpened" => false, + "ActivityScheduled" => { + event.payload["local_activity"] != true + && event.payload["execution_mode"] != "local" + && event.payload["activity"]["local_activity"] != true + && event.payload["activity"]["execution_mode"] != "local" + } + _ => true, + }) + } else { + scalar(boundary.call_kind) && boundary.sequence_span == 1 + }; + if !supported || preparation.boundary.operation_sequence.is_some() || preparation.event.payload["descendant_members"] .as_array() @@ -86,6 +257,33 @@ impl ScopeReplay { pub fn bind_commands(&self, commands: &mut Vec) -> Result<()> { for (&sequence, delivered) in &self.canonical.deliveries { let index = commands.partition_point(|command| command.sequence() < sequence); + if delivered.boundary.call_kind == CancellationCallKind::Parallel { + let end = commands.partition_point(|command| { + delivered.boundary.interrupts(command.sequence()) + || command.sequence() < sequence + }); + let original: Vec<_> = commands.drain(index..end).collect(); + if original.len() as u64 != delivered.boundary.sequence_span + || original + .iter() + .enumerate() + .any(|(offset, command)| command.sequence() != sequence + offset as u64) + { + return Err(invalid( + sequence, + "scope delivery omits an original group member", + )); + } + commands.insert( + index, + RecordedCommand::CancellationGroup { + sequence, + span: delivered.boundary.sequence_span, + original, + }, + ); + continue; + } let original = if commands .get(index) .is_some_and(|command| command.sequence() == sequence) @@ -117,6 +315,158 @@ impl ScopeReplay { } impl WorkflowState { + /// Validate the whole group before exposing cancellation to application code. + pub(super) fn prepare_parallel_scope_cancellation( + &mut self, + descriptors: &[ParallelDescriptor], + ) -> Result { + let Some(replay) = &self.scope_delivery else { + return Ok(false); + }; + let index = self.command_cursor; + let sequence = descriptors[0].group_path[0].parallel_group_base_sequence; + if let Some(delivered) = replay.canonical.deliveries.get(&sequence) { + let delivered = delivered.clone(); + if replay.active_scope != delivered.context.scope_id() + || self.cancellation_shield_depth > 0 + || delivered.boundary.call_kind != CancellationCallKind::Parallel + { + return Err(invalid( + sequence, + "scoped group changed its original membership, kind or shielding", + )); + } + let Some(RecordedCommand::CancellationGroup { span, original, .. }) = + self.recorded_commands.get(index).cloned() + else { + return Err(invalid( + sequence, + "scoped group crossed its original delivery boundary", + )); + }; + if span != descriptors.len() as u64 { + return Err(invalid(sequence, "scoped group changed its original span")); + } + self.validate_scoped_group_members(descriptors, &original)?; + let conditions = descriptors + .iter() + .filter(|descriptor| { + matches!(descriptor.operation, ParallelOperation::Condition { .. }) + }) + .count() as u64; + self.condition_wait_occurrence_counter = self + .condition_wait_occurrence_counter + .checked_add(conditions) + .ok_or_else(|| { + invalid(sequence, "scoped condition occurrence counter overflowed") + })?; + self.recorded_commands.splice(index..=index, original); + return self + .replay_scope_cancellation_at(index, CancellationCallKind::Parallel) + .map(|()| false); + } + if self.cancellation_shield_depth > 0 { + return Ok(false); + } + if replay.intent.is_some() { + return Ok(true); + } + let Some(request) = replay + .canonical + .pending_request_for_scope(&replay.active_scope, &replay.tree)? + .cloned() + else { + return Ok(false); + }; + if replay.contexts.contains_key(request.context.scope_id()) { + return Ok(false); + } + if request.context.scope_id() != replay.active_scope + || descriptors + .iter() + .any(|descriptor| matches!(descriptor.operation, ParallelOperation::Signal(_))) + { + return Err(Error::CancellationScopeExecutionUnavailable); + } + let span = descriptors.len() as u64; + let resolved = |sequence| { + self.history_events + .iter() + .take(request.history_index) + .any(|event| { + durable_event_sequence(event) == Some(sequence) + && scope_operation_terminal(&event.event_type) + }) + }; + if (sequence..sequence + span).all(resolved) { + return Ok(false); + } + let original: Vec<_> = self + .recorded_commands + .iter() + .skip(index) + .take(descriptors.len()) + .cloned() + .collect(); + self.validate_scoped_group_members(descriptors, &original)?; + let boundary = CancellationDelivery { + request_id: request.context.request_id().into(), + sequence, + call_kind: CancellationCallKind::Parallel, + sequence_span: span, + operation_sequence: None, + operation_sequence_span: 1, + }; + if replay + .canonical + .preparations + .get(request.context.scope_id()) + .is_some_and(|prepared| prepared.boundary != boundary) + { + return Err(invalid( + sequence, + "prepared scope changed its original authored group", + )); + } + self.scope_delivery.as_mut().unwrap().intent = Some(ScopeDeliveryIntent { + context: request.context, + boundary, + command_count: self.commands.len(), + }); + self.matched_recorded_pending = true; + self.command_cursor = index + descriptors.len(); + Ok(true) + } + + fn validate_scoped_group_members( + &self, + descriptors: &[ParallelDescriptor], + original: &[RecordedCommand], + ) -> Result<()> { + if descriptors.len() != original.len() { + return Err(invalid( + descriptors[0].group_path[0].parallel_group_base_sequence, + "scoped group omits an original admitted member", + )); + } + let replay = self.scope_delivery.as_ref().unwrap(); + for (descriptor, recorded) in descriptors.iter().zip(original) { + let sequence = + descriptor.group_path[0].parallel_group_base_sequence + descriptor.offset as u64; + if recorded.sequence() != sequence + || replay.tree.memberships.get(&sequence).map(String::as_str) + != Some(replay.active_scope.as_str()) + { + return Err(invalid( + sequence, + "scoped group changed its original member address", + )); + } + validate_scoped_group_definition(self, descriptor, recorded)?; + } + Ok(()) + } + pub(super) fn scope_cleanup_timer_proof(&self, scope: &str) -> Result> { if self.cancellation_shield_depth == 0 { return Ok(None); @@ -185,9 +535,6 @@ impl WorkflowState { if replay.contexts.contains_key(request.context.scope_id()) { return Ok(false); } - if !scalar(kind) || !path.is_empty() || request.context.scope_id() != replay.active_scope { - return Err(Error::CancellationScopeExecutionUnavailable); - } let sequence = self.recorded_commands.get(index).map_or_else( || self.next_scope_cancellation_sequence(), |command| Ok(command.sequence()), @@ -217,6 +564,9 @@ impl WorkflowState { { return Ok(false); } + if !scalar(kind) || !path.is_empty() || request.context.scope_id() != replay.active_scope { + return Err(Error::CancellationScopeExecutionUnavailable); + } let boundary = CancellationDelivery { request_id: request.context.request_id().into(), sequence, @@ -294,7 +644,12 @@ impl WorkflowState { delivered.context.scope_id().into(), (delivered.context.clone(), delivered.authority_deadline), ); - self.command_cursor = index + 1; + self.command_cursor = index + + if kind == CancellationCallKind::Parallel { + delivered.boundary.sequence_span as usize + } else { + 1 + }; Err(Error::CancellationScopeRequested( CancellationScopeRequested { context: delivered diff --git a/src/lib.rs b/src/lib.rs index b44dc57..60db3af 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11395,6 +11395,7 @@ pub struct ParallelCall { operations: Option>, shape: Option, leaves: Vec, + pending_scope_delivery: bool, } impl ParallelCall { @@ -11404,6 +11405,7 @@ impl ParallelCall { operations: Some(operations), shape: None, leaves: Vec::new(), + pending_scope_delivery: false, } } @@ -11448,6 +11450,12 @@ impl ParallelCall { .state .lock() .map_err(|_| Error::WorkflowStatePoisoned)?; + let cursor = state.command_cursor; + self.ctx.validate_scope_membership(&mut state, cursor)?; + if state.prepare_parallel_scope_cancellation(&descriptors)? { + self.pending_scope_delivery = true; + return Ok(()); + } state.expand_cancellation_group(&descriptors)?; state.prepare_group_cancellation(&descriptors)?; } @@ -11480,6 +11488,9 @@ impl ParallelCall { return Poll::Ready(Err(error)); } } + if self.pending_scope_delivery { + return Poll::Pending; + } if self.leaves.is_empty() { return Poll::Ready(Ok(Vec::new())); } diff --git a/src/tests/cancellation_scope_replay.rs b/src/tests/cancellation_scope_replay.rs index 376cab6..d932795 100644 --- a/src/tests/cancellation_scope_replay.rs +++ b/src/tests/cancellation_scope_replay.rs @@ -1,5 +1,395 @@ use super::*; +fn group_fixture(name: &str) -> Value { + let source: Value = serde_json::from_str(include_str!( + "../../tests/fixtures/populated-scope-groups.json" + )) + .unwrap(); + source[name].clone() +} + +fn group_worker(cleanup: Arc, nested: bool, changed: &'static str) -> Worker { + let mut worker = Worker::new(Client::new("http://unused.invalid").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + worker.register_workflow("scope-replay", move |ctx, _| { + let cleanup = Arc::clone(&cleanup); + async move { + let parent = ctx.clone(); + let result = ctx + .cancellation_scope(false, move |outer| async move { + outer + .cancellation_scope(false, move |inner| async move { + if changed != "prefix" { + inner.activity("prior-step", json!([])).await?; + } + let activity = ParallelOperation::activity_with_options( + if changed == "activity" { + "changed-activity" + } else { + "original-activity" + }, + ActivityOptions::new().cancellation_policy(if changed == "activity_policy" { + CancellationPolicy::WaitCancellationCompleted + } else { CancellationPolicy::TryCancel }), json!([]), + ); + let timer = ParallelOperation::timer(Duration::from_secs( + if changed == "timer" { 3599 } else { 3600 }, + )); + let child = ParallelOperation::child_workflow( + if changed == "child" { "changed-child" } else { "original-child" }, + ChildWorkflowOptions::new("queue") + .parent_close_policy(if changed == "parent_policy" { + ParentClosePolicy::Abandon + } else { ParentClosePolicy::RequestCancel }) + .cancellation_policy(if changed == "child_policy" { + CancellationPolicy::Abandon + } else { + CancellationPolicy::WaitCancellationCompleted + }), + json!([]), + ); + let condition = ParallelOperation::condition(ConditionWaitOptions::new( + if changed == "condition" { "changed-key" } else { "ready" }, + if changed == "predicate" { "sha256:changed" } else { "sha256:3b2b1ad635030d842ab378e8bf021f4613ee6d221dafb4743565467800def433" }) + .timeout(Duration::from_secs(if changed == "condition_timeout" { 31 } else { 30 })), || panic!("committed cancellation must not evaluate the pending predicate")); + let mut group = if nested { + vec![ + activity, + ParallelOperation::group(vec![timer, child]), + condition, + ] + } else { + vec![activity, timer, child, condition] + }; + if changed == "size" { group.pop(); } + if changed == "kind" { group[0] = ParallelOperation::timer(Duration::from_secs(3600)); } + let _shield = if changed == "shield" { + Some(inner.cancellation_shield()?) + } else { + None + }; + match inner.parallel(group).await { + Err(Error::CancellationScopeRequested(cancellation)) => { + cleanup.fetch_add(1, Ordering::SeqCst); + assert_eq!( + cancellation.delivery.call_kind, + CancellationCallKind::Parallel + ); + assert_eq!(cancellation.delivery.sequence_span, 4); + assert_eq!( + cancellation.context.remaining()?, + Duration::from_secs(21) + ); + let _shield = inner.cancellation_shield()?; + inner.sleep(Duration::from_secs(2)).await?; + assert_eq!(cancellation.context.remaining()?, Duration::from_secs(19)); + Ok(json!({"cleaned":true})) + } + Err(error) => Err(error), + Ok(_) => Ok(json!("ordinary")), + } + }) + .await + }) + .await?; + assert!(!parent.is_cancellation_requested()?); + parent.sleep(Duration::from_secs(1)).await?; + Ok(result) + } + }); + worker +} + +#[test] +fn cancellation_scope_replay_flat_and_nested_groups_wait_for_committed_delivery() { + for (name, nested) in [("flat", false), ("nested", true)] { + let source = group_fixture(name); + let cleanup = Arc::new(AtomicUsize::new(0)); + let worker = group_worker(Arc::clone(&cleanup), nested, ""); + for before in [ + "CancellationScopeDeliveryPrepared", + "CancellationScopeDelivered", + ] { + let decision = worker + .execute_workflow_task_decision(task(&prefix(&source, before))) + .unwrap(); + assert!(decision.commands.is_empty()); + let intent = decision.cancellation_scope_delivery.unwrap(); + assert_eq!(intent.boundary.call_kind, CancellationCallKind::Parallel); + assert_eq!(intent.boundary.sequence, 4); + assert_eq!(intent.boundary.sequence_span, 4); + assert_eq!(cleanup.load(Ordering::SeqCst), 0); + } + let decision = worker + .execute_workflow_task_decision(task(&source)) + .unwrap(); + assert!(decision.cancellation_scope_delivery.is_none()); + assert_eq!(cleanup.load(Ordering::SeqCst), 1); + assert_eq!(decision.commands.len(), 1); + assert_eq!(decision.commands[0]["type"], "start_timer"); + assert_eq!(decision.commands[0]["delay_seconds"], 2); + assert_eq!( + decision.commands[0]["cancellation_cleanup"]["request_id"], + cleanup_snapshot(&source)["request_id"] + ); + } +} + +#[test] +fn cancellation_scope_replay_group_rejects_changed_members_before_cleanup() { + for changed in [ + "activity", + "activity_policy", + "timer", + "child", + "child_policy", + "parent_policy", + "condition", + "condition_timeout", + "predicate", + "shield", + "shape", + "prefix", + "size", + "kind", + ] { + let cleanup = Arc::new(AtomicUsize::new(0)); + let worker = group_worker(Arc::clone(&cleanup), changed == "shape", changed); + assert!( + worker + .execute_workflow_task_decision(task(&group_fixture("flat"))) + .is_err(), + "{changed}" + ); + assert_eq!(cleanup.load(Ordering::SeqCst), 0, "{changed}"); + } +} + +#[test] +fn cancellation_scope_replay_unqualified_groups_stop_before_factory() { + for name in [ + "flat-local", + "flat-other-scope", + "flat-selection", + "flat-signal", + "flat-incomplete", + ] { + let invoked = Arc::new(AtomicUsize::new(0)); + let mut worker = Worker::new(Client::new("http://unused.invalid").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + let calls = Arc::clone(&invoked); + worker.register_workflow("scope-replay", move |_, _| { + calls.fetch_add(1, Ordering::SeqCst); + async { Ok(json!(null)) } + }); + assert!( + worker + .execute_workflow_task_decision(task(&group_fixture(name))) + .is_err(), + "{name}" + ); + assert_eq!(invoked.load(Ordering::SeqCst), 0, "{name}"); + } +} + +#[test] +fn cancellation_scope_replay_group_replacement_preserves_cleanup_sequence_and_authority() { + for (name, nested) in [("flat", false), ("nested", true)] { + let mut source = group_fixture(name); + let snapshot = cleanup_snapshot(&source); + let scope = snapshot["scope_id"].clone(); + append( + &mut source, + "TimerScheduled", + json!({"sequence":8, "timer_id":"group-cleanup", + "delay_seconds":2, "cancellation_scope_id":scope, "cancellation_cleanup":snapshot, + "fire_at":"2026-10-04T00:00:11.123456Z"}), + "2026-10-04T00:00:09.123456Z", + ); + append( + &mut source, + "TimerFired", + json!({"sequence":8, "timer_id":"group-cleanup", + "delay_seconds":2, "cancellation_scope_id":scope}), + "2026-10-04T00:00:11.123456Z", + ); + let worker = + group_worker(Arc::new(AtomicUsize::new(0)), nested, "").worker_id("replacement"); + let mut claim = task(&source); + claim.lease_owner = Some("replacement".into()); + claim.workflow_task_attempt = 19; + let decision = worker.execute_workflow_task_decision(claim).unwrap(); + assert_eq!(decision.commands.len(), 1); + assert_eq!(decision.commands[0]["delay_seconds"], 1); + assert!(decision.commands[0].get("cancellation_scope_id").is_none()); + assert!(decision.commands[0].get("cancellation_cleanup").is_none()); + append( + &mut source, + "TimerScheduled", + json!({"sequence":9, "timer_id":"parent-timer", + "delay_seconds":1}), + "2026-10-04T00:00:11.123456Z", + ); + append( + &mut source, + "TimerFired", + json!({"sequence":9, "timer_id":"parent-timer", + "delay_seconds":1}), + "2026-10-04T00:00:12.123456Z", + ); + let decision = worker + .execute_workflow_task_decision(task(&source)) + .unwrap(); + assert_eq!(decision.commands.len(), 1); + assert_eq!(decision.commands[0]["type"], "complete_workflow"); + } +} + +#[test] +fn cancellation_scope_replay_completed_member_does_not_hide_pending_siblings() { + for (name, nested) in [("flat", false), ("nested", true)] { + for prepared in [false, true] { + let source = group_fixture(name); + let mut value = prefix( + &source, + if prepared { + "CancellationScopeDelivered" + } else { + "CancellationScopeDeliveryPrepared" + }, + ); + let events = value["history"].as_array_mut().unwrap(); + let request = events + .iter() + .position(|row| row["event_type"] == "CancellationScopeRequested") + .unwrap(); + let namespace = events[0]["namespace"].clone(); + events.insert( + request, + json!({"id":"completed-first-member", "sequence":1, + "namespace":namespace, "timestamp":"2026-10-04T00:00:02.123456Z", + "event_type":"ActivityCompleted", "payload":{"sequence":4, + "activity_execution_id":"original-id", "activity_type":"original-activity", + "result":{"codec":"avro", "blob":"wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU="}}}), + ); + for (index, row) in events.iter_mut().enumerate() { + row["sequence"] = json!(index + 1); + } + let cleanup = Arc::new(AtomicUsize::new(0)); + let decision = group_worker(Arc::clone(&cleanup), nested, "") + .execute_workflow_task_decision(task(&value)) + .unwrap(); + assert!(decision.commands.is_empty()); + assert_eq!( + decision + .cancellation_scope_delivery + .unwrap() + .boundary + .sequence_span, + 4 + ); + assert_eq!(cleanup.load(Ordering::SeqCst), 0); + } + } +} + +#[test] +fn cancellation_scope_replay_completed_group_preserves_results_and_prepares_next_call() { + let mut source = group_fixture("flat"); + let request = source["history"] + .as_array() + .unwrap() + .iter() + .find(|row| row["event_type"] == "CancellationScopeRequested") + .unwrap()["payload"] + .clone(); + let scope = request["scope_id"].clone(); + source["history"].as_array_mut().unwrap().retain(|row| { + !row["event_type"] + .as_str() + .unwrap() + .starts_with("CancellationScopeRequest") + && !row["event_type"] + .as_str() + .unwrap() + .starts_with("CancellationScopeDeliver") + && row["payload"]["sequence"] + .as_u64() + .is_none_or(|sequence| sequence < 4) + }); + let descriptors = parallel_descriptors( + vec![ + ParallelOperation::timer(Duration::from_secs(1)), + ParallelOperation::timer(Duration::from_secs(1)), + ], + 4, + ) + .unwrap(); + for descriptor in descriptors { + let sequence = 4 + descriptor.offset as u64; + let delay = 1; + let mut payload = serde_json::Map::from_iter([ + ("sequence".into(), json!(sequence)), + ("timer_id".into(), json!(format!("timer-{sequence}"))), + ("delay_seconds".into(), json!(delay)), + ("cancellation_scope_id".into(), scope.clone()), + ]); + apply_parallel_group_path(&mut payload, &descriptor.group_path); + append( + &mut source, + "TimerScheduled", + Value::Object(payload.clone()), + "2026-10-04T00:00:01.123456Z", + ); + append( + &mut source, + "TimerFired", + Value::Object(payload), + "2026-10-04T00:00:02.123456Z", + ); + } + append( + &mut source, + "CancellationScopeRequested", + request, + "2026-10-04T00:00:03.123456Z", + ); + let mut worker = Worker::new(Client::new("http://unused.invalid").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + worker.register_workflow("scope-replay", |ctx, _| async move { + ctx.cancellation_scope(false, |outer| async move { + outer + .cancellation_scope(false, |inner| async move { + inner.activity("prior-step", json!([])).await?; + let results = inner + .parallel(vec![ + ParallelOperation::timer(Duration::from_secs(1)), + ParallelOperation::timer(Duration::from_secs(1)), + ]) + .await?; + assert_eq!(results.len(), 2); + inner.sleep(Duration::from_secs(1)).await?; + Ok(json!(null)) + }) + .await + }) + .await + }); + let decision = worker + .execute_workflow_task_decision(task(&source)) + .unwrap(); + assert!(decision.commands.is_empty()); + let intent = decision.cancellation_scope_delivery.unwrap(); + assert_eq!(intent.boundary.sequence, 6); + assert_eq!(intent.boundary.call_kind, CancellationCallKind::Timer); +} + fn fixture() -> Value { let source: Value = serde_json::from_str(include_str!( "../../tests/fixtures/populated-scope-single-calls.json" diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index 42d6992..a79b04f 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -881,50 +881,76 @@ fn request_native_scope_fixture(handle: &WorkflowHandle, scope: &str) -> Value { serde_json::from_slice(&result.stdout).unwrap() } -fn scope_cleanup_worker(client: &Client, queue: &str, owner: &str) -> Worker { +fn scope_cleanup_worker(client: &Client, queue: &str, owner: &str, grouped: bool) -> Worker { let mut worker = Worker::new(client.clone(), queue) .worker_id(owner) .cooperative_cancellation(true) .candidate_cancellation_scope_authoring(true) .candidate_cancellation_scope_delivery(true) .poll_timeout(Duration::from_millis(100)); - worker.register_workflow("tests.rust-candidate-scope-cleanup", |ctx, _| async move { - let _: String = ctx.side_effect(|| "original prefix".to_owned())?; - let (cancellation, metadata) = ctx - .cancellation_scope(false, |scope| async move { - let cancellation = match scope.sleep(Duration::from_secs(300)).await { - Err(Error::CancellationScopeRequested(request)) => request.context, - Err(error) => return Err(error), - Ok(()) => { - return Err(Error::InvalidCooperativeCancellation( - "original timer was not interrupted".into(), - )) - } - }; - assert!(scope.is_cancellation_requested()?); - let _shield = scope.cancellation_shield()?; - let metadata = scope.side_effect(|| cancellation.to_value())?; - scope.sleep(Duration::from_secs(1)).await?; - Ok((cancellation, metadata)) - }) - .await?; - assert!(!ctx.is_cancellation_requested()?); - assert!(ctx.scoped_cancellation_context()?.is_none()); - ctx.sleep(Duration::from_secs(1)).await?; - Ok(json!({"context":metadata, "remaining":cancellation.remaining()?.as_secs_f64()})) - }); + worker.register_workflow( + "tests.rust-candidate-scope-cleanup", + move |ctx, _| async move { + let _: String = ctx.side_effect(|| "original prefix".to_owned())?; + let (cancellation, metadata) = ctx + .cancellation_scope(false, move |scope| async move { + let pending = if grouped { + scope + .parallel(vec![ + ParallelOperation::timer(Duration::from_secs(300)), + ParallelOperation::group(vec![ParallelOperation::timer( + Duration::from_secs(600), + )]), + ]) + .await + .map(|_| ()) + } else { + scope.sleep(Duration::from_secs(300)).await + }; + let cancellation = match pending { + Err(Error::CancellationScopeRequested(request)) => request.context, + Err(error) => return Err(error), + Ok(()) => { + return Err(Error::InvalidCooperativeCancellation( + "original timer was not interrupted".into(), + )) + } + }; + assert!(scope.is_cancellation_requested()?); + let _shield = scope.cancellation_shield()?; + let metadata = scope.side_effect(|| cancellation.to_value())?; + scope.sleep(Duration::from_secs(1)).await?; + Ok((cancellation, metadata)) + }) + .await?; + assert!(!ctx.is_cancellation_requested()?); + assert!(ctx.scoped_cancellation_context()?.is_none()); + ctx.sleep(Duration::from_secs(1)).await?; + Ok(json!({"context":metadata, "remaining":cancellation.remaining()?.as_secs_f64()})) + }, + ); worker } #[tokio::test] #[ignore = "requires an isolated cooperative Server with the Native scope candidate"] async fn server_scope_cleanup_preserves_original_delivery_and_budget_after_replacement() { + qualify_scope_cleanup_replacement(false).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with the Native scope candidate"] +async fn server_scope_group_cleanup_preserves_original_delivery_and_budget_after_replacement() { + qualify_scope_cleanup_replacement(true).await; +} + +async fn qualify_scope_cleanup_replacement(grouped: bool) { let client = client(); let queue = format!( "rust-cooperative-scope-boundary-{}", durable_workflow::Uuid::new_v4().simple() ); - let original = scope_cleanup_worker(&client, &queue, &format!("{queue}-original")); + let original = scope_cleanup_worker(&client, &queue, &format!("{queue}-original"), grouped); original.register().await.unwrap(); let handle = client .start_workflow( @@ -948,9 +974,13 @@ async fn server_scope_cleanup_preserves_original_delivery_and_budget_after_repla let accepted = request_native_scope_fixture(&handle, scope); assert_eq!(accepted, request_native_scope_fixture(&handle, scope)); let cleaning = tick_until_count(&original, &handle, "SideEffectRecorded", 2).await; - assert_eq!(count(&cleaning, "TimerScheduled"), 2); + assert_eq!( + count(&cleaning, "TimerScheduled"), + if grouped { 3 } else { 2 } + ); assert_eq!(count(&cleaning, "CancellationScopeDelivered"), 1); - let replacement = scope_cleanup_worker(&client, &queue, &format!("{queue}-replacement")); + let replacement = + scope_cleanup_worker(&client, &queue, &format!("{queue}-replacement"), grouped); replacement.register().await.unwrap(); let final_history = tick_until(&replacement, &handle, "WorkflowCompleted").await; for kind in [ @@ -958,13 +988,19 @@ async fn server_scope_cleanup_preserves_original_delivery_and_budget_after_repla "CancellationScopeRequested", "CancellationScopeDeliveryPrepared", "CancellationScopeDelivered", - "TimerCancelled", "WorkflowCompleted", ] { assert_eq!(count(&final_history, kind), 1, "{kind}"); } assert_eq!(count(&final_history, "SideEffectRecorded"), 2); - assert_eq!(count(&final_history, "TimerScheduled"), 3); + assert_eq!( + count(&final_history, "TimerCancelled"), + if grouped { 2 } else { 1 } + ); + assert_eq!( + count(&final_history, "TimerScheduled"), + if grouped { 4 } else { 3 } + ); assert_eq!(count(&final_history, "TimerFired"), 2); for kind in [ "CooperativeCancellationRequested", diff --git a/tests/fixtures/populated-scope-groups.json b/tests/fixtures/populated-scope-groups.json new file mode 100644 index 0000000..d93139f --- /dev/null +++ b/tests/fixtures/populated-scope-groups.json @@ -0,0 +1,3919 @@ +{ + "_provenance": { + "native_commit": "0471f1ebbb98c61fe53663086d62dc2db74db9e3", + "input_fixture_sha256": "85320e25c7c19521bca4a1cff138911df1f19f76471ea033e961695e8ed761be", + "history_store": "isolated SQLite projection inputs", + "effect_authority": false, + "producers": [ + "CancellationScopeDelivery::activityMembers", + "ScopedTimerCancellation::members", + "ScopedWaitCancellation::members", + "ScopedChildCancellation::members" + ] + }, + "flat": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "layout": "flat", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "group-activity", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "group-timer", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 5, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "group-child", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 6, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "group-condition", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "condition_wait_id": "original-wait", + "condition_key": "ready", + "timeout_seconds": 30, + "condition_wait_occurrence_id": "original-occurrence", + "condition_definition_fingerprint": "sha256:3b2b1ad635030d842ab378e8bf021f4613ee6d221dafb4743565467800def433" + } + }, + { + "id": "condition-timeout", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "original-timeout", + "timer_kind": "condition_timeout", + "condition_wait_id": "original-wait", + "condition_wait_occurrence_id": "original-occurrence", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:32.123456Z" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 12, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 13, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "c3a3d20bd8882dfc59673847c3f8ed67c66cdaee74e989810002e2b622764699" + } + ], + "timer_members": [ + { + "sequence": 5, + "timer_id": "original-timer", + "descriptor_hash": "bae0466390116ff60cbd3597bcdaf8e0097fae4c7f8b789db6f31ee9553b62c1" + }, + { + "sequence": 7, + "timer_id": "original-timeout", + "descriptor_hash": "f5518182dcbfc2737f54b9a1754e941e8efa416815a2aad9c8e4cec7f266d423" + } + ], + "wait_members": [ + { + "kind": "condition", + "sequence": 7, + "wait_id": "original-wait", + "timer_id": "original-timeout", + "descriptor_hash": "0bd2f375d595a9433320f9b48a03831232d40aac2258f1193d2026248f822d3f" + } + ], + "child_members": [ + { + "sequence": 6, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "ecced7a15fa4e651c44eb0b9f2c0962d94f73a3caae0798d375ec65d3618ee1f" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 14, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "nested": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "layout": "nested", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "group-activity", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "group-timer", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 5, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:5:2", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 5, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "parallel-calls:5:2", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 5, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ], + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "group-child", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 6, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:5:2", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 5, + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2 + }, + { + "parallel_group_id": "parallel-calls:5:2", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 5, + "parallel_group_size": 2, + "parallel_group_index": 1 + } + ], + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "group-condition", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "condition_wait_id": "original-wait", + "condition_key": "ready", + "timeout_seconds": 30, + "condition_wait_occurrence_id": "original-occurrence", + "condition_definition_fingerprint": "sha256:3b2b1ad635030d842ab378e8bf021f4613ee6d221dafb4743565467800def433" + } + }, + { + "id": "condition-timeout", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "original-timeout", + "timer_kind": "condition_timeout", + "condition_wait_id": "original-wait", + "condition_wait_occurrence_id": "original-occurrence", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:32.123456Z" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 12, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 13, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "c3a3d20bd8882dfc59673847c3f8ed67c66cdaee74e989810002e2b622764699" + } + ], + "timer_members": [ + { + "sequence": 5, + "timer_id": "original-timer", + "descriptor_hash": "63a2bd9bc43403237583a4025f6dd94fd3af405b54ec9d8dc52d6f786924bcf8" + }, + { + "sequence": 7, + "timer_id": "original-timeout", + "descriptor_hash": "f5518182dcbfc2737f54b9a1754e941e8efa416815a2aad9c8e4cec7f266d423" + } + ], + "wait_members": [ + { + "kind": "condition", + "sequence": 7, + "wait_id": "original-wait", + "timer_id": "original-timeout", + "descriptor_hash": "0bd2f375d595a9433320f9b48a03831232d40aac2258f1193d2026248f822d3f" + } + ], + "child_members": [ + { + "sequence": 6, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "83bc0b6e32d165958c0009a10112a72c3ed757a8c4c398138fe324c5613445ac" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 14, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "flat-local": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "layout": "flat-local", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "group-activity", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + }, + "local_activity": true, + "execution_mode": "local" + } + }, + { + "id": "group-timer", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 5, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "group-child", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 6, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "group-condition", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "condition_wait_id": "original-wait", + "condition_key": "ready", + "timeout_seconds": 30, + "condition_wait_occurrence_id": "original-occurrence", + "condition_definition_fingerprint": "sha256:3b2b1ad635030d842ab378e8bf021f4613ee6d221dafb4743565467800def433" + } + }, + { + "id": "condition-timeout", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "original-timeout", + "timer_kind": "condition_timeout", + "condition_wait_id": "original-wait", + "condition_wait_occurrence_id": "original-occurrence", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:32.123456Z" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 12, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 13, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "938c445dfa05b9573b85e49ab0839f77d706dc694ac7801a13c621eff301e727" + } + ], + "timer_members": [ + { + "sequence": 5, + "timer_id": "original-timer", + "descriptor_hash": "bae0466390116ff60cbd3597bcdaf8e0097fae4c7f8b789db6f31ee9553b62c1" + }, + { + "sequence": 7, + "timer_id": "original-timeout", + "descriptor_hash": "f5518182dcbfc2737f54b9a1754e941e8efa416815a2aad9c8e4cec7f266d423" + } + ], + "wait_members": [ + { + "kind": "condition", + "sequence": 7, + "wait_id": "original-wait", + "timer_id": "original-timeout", + "descriptor_hash": "0bd2f375d595a9433320f9b48a03831232d40aac2258f1193d2026248f822d3f" + } + ], + "child_members": [ + { + "sequence": 6, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "ecced7a15fa4e651c44eb0b9f2c0962d94f73a3caae0798d375ec65d3618ee1f" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 14, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "flat-selection": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "layout": "flat-selection", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "group-activity", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "select-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_mode": "select", + "selection_member_key": 0, + "selection_member_index": 0, + "selection_member_base_sequence": 4, + "selection_member_size": 1, + "selection_member_kind": "activity", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_mode": "select", + "selection_member_key": 0, + "selection_member_index": 0, + "selection_member_base_sequence": 4, + "selection_member_size": 1, + "selection_member_kind": "activity" + } + ], + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "group-timer", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 5, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "select-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_mode": "select", + "selection_member_key": 1, + "selection_member_index": 1, + "selection_member_base_sequence": 5, + "selection_member_size": 1, + "selection_member_kind": "timer", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_mode": "select", + "selection_member_key": 1, + "selection_member_index": 1, + "selection_member_base_sequence": 5, + "selection_member_size": 1, + "selection_member_kind": "timer" + } + ], + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "group-child", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 6, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "select-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_mode": "select", + "selection_member_key": 2, + "selection_member_index": 2, + "selection_member_base_sequence": 6, + "selection_member_size": 1, + "selection_member_kind": "child", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_mode": "select", + "selection_member_key": 2, + "selection_member_index": 2, + "selection_member_base_sequence": 6, + "selection_member_size": 1, + "selection_member_kind": "child" + } + ], + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "group-condition", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "select-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_mode": "select", + "selection_member_key": 3, + "selection_member_index": 3, + "selection_member_base_sequence": 7, + "selection_member_size": 1, + "selection_member_kind": "condition", + "parallel_group_path": [ + { + "parallel_group_id": "select-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_mode": "select", + "selection_member_key": 3, + "selection_member_index": 3, + "selection_member_base_sequence": 7, + "selection_member_size": 1, + "selection_member_kind": "condition" + } + ], + "condition_wait_id": "original-wait", + "condition_key": "ready", + "timeout_seconds": 30, + "condition_wait_occurrence_id": "original-occurrence", + "condition_definition_fingerprint": "sha256:3b2b1ad635030d842ab378e8bf021f4613ee6d221dafb4743565467800def433" + } + }, + { + "id": "condition-timeout", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "original-timeout", + "timer_kind": "condition_timeout", + "condition_wait_id": "original-wait", + "condition_wait_occurrence_id": "original-occurrence", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:32.123456Z" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 12, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 13, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "c3a3d20bd8882dfc59673847c3f8ed67c66cdaee74e989810002e2b622764699" + } + ], + "timer_members": [ + { + "sequence": 5, + "timer_id": "original-timer", + "descriptor_hash": "72554083760b2fa13f5a8050ca408c127e31d3d31406e5a15c36aa58ccad6fc2" + }, + { + "sequence": 7, + "timer_id": "original-timeout", + "descriptor_hash": "f5518182dcbfc2737f54b9a1754e941e8efa416815a2aad9c8e4cec7f266d423" + } + ], + "wait_members": [ + { + "kind": "condition", + "sequence": 7, + "wait_id": "original-wait", + "timer_id": "original-timeout", + "descriptor_hash": "2226681f0dfa1f9cbd32dde39187b68a7bc2c8271849babe6e09dd799f79de0d" + } + ], + "child_members": [ + { + "sequence": 6, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "cf830fe095b233d4d675edb3ab40d03ef634b68b4ed8c17b58cea04b00e3bb44" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 14, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "flat-incomplete": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "layout": "flat-incomplete", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "group-activity", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "group-timer", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 5, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "group-child", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 6, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 10, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 11, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "c3a3d20bd8882dfc59673847c3f8ed67c66cdaee74e989810002e2b622764699" + } + ], + "timer_members": [ + { + "sequence": 5, + "timer_id": "original-timer", + "descriptor_hash": "bae0466390116ff60cbd3597bcdaf8e0097fae4c7f8b789db6f31ee9553b62c1" + } + ], + "wait_members": [], + "child_members": [ + { + "sequence": 6, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "ecced7a15fa4e651c44eb0b9f2c0962d94f73a3caae0798d375ec65d3618ee1f" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 12, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "flat-signal": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "layout": "flat-signal", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "group-activity", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "group-timer", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 5, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "group-child", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 6, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "group-condition", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "SignalWaitOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "timeout_seconds": 30, + "signal_wait_id": "original-wait", + "signal_name": "ready" + } + }, + { + "id": "condition-timeout", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "original-timeout", + "timer_kind": "signal_timeout", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:32.123456Z", + "signal_wait_id": "original-wait" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 12, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 13, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "c3a3d20bd8882dfc59673847c3f8ed67c66cdaee74e989810002e2b622764699" + } + ], + "timer_members": [ + { + "sequence": 5, + "timer_id": "original-timer", + "descriptor_hash": "bae0466390116ff60cbd3597bcdaf8e0097fae4c7f8b789db6f31ee9553b62c1" + }, + { + "sequence": 7, + "timer_id": "original-timeout", + "descriptor_hash": "c02f931bce4a2a8528bfd5ec9eb522c5402f88b96c77a0b211d43948e1484244" + } + ], + "wait_members": [ + { + "kind": "signal", + "sequence": 7, + "wait_id": "original-wait", + "timer_id": "original-timeout", + "descriptor_hash": "4386073479943cef92b70c14b427ad2bb9cbad16b2481af8e6406b98db9c780d" + } + ], + "child_members": [ + { + "sequence": 6, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "ecced7a15fa4e651c44eb0b9f2c0962d94f73a3caae0798d375ec65d3618ee1f" + } + ], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 14, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "flat-other-scope": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "layout": "flat-other-scope", + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a29mr7ywhvs5mrahrt", + "namespace": "sdk-scope-fixture", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf6a5edjad2cnw74ktca8", + "namespace": "sdk-scope-fixture", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": "01M43XF6A1JYQH2ZMZ0XE0M00C", + "shield_parent": false, + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "prior-scheduled", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 3, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "activity_type": "prior-step", + "activity_execution_id": "prior-id", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "group-activity", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 4, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "group-timer", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 5, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "group-child", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 6, + "cancellation_scope_id": "root", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "group-condition", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:4:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 4, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "condition_wait_id": "original-wait", + "condition_key": "ready", + "timeout_seconds": 30, + "condition_wait_occurrence_id": "original-occurrence", + "condition_definition_fingerprint": "sha256:3b2b1ad635030d842ab378e8bf021f4613ee6d221dafb4743565467800def433" + } + }, + { + "id": "condition-timeout", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "timer_id": "original-timeout", + "timer_kind": "condition_timeout", + "condition_wait_id": "original-wait", + "condition_wait_occurrence_id": "original-occurrence", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:32.123456Z" + } + }, + { + "id": "01m43xf6acd9a7a5pepm5pjnb0", + "namespace": "sdk-scope-fixture", + "sequence": 12, + "event_type": "CancellationScopeRequested", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "parent_scope_id": null, + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "timestamp": "2026-10-04T00:00:03.123456Z" + }, + { + "id": "01m43xf6az7v2abqsr0tbz27a1", + "namespace": "sdk-scope-fixture", + "sequence": 13, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "activity_members": [ + { + "sequence": 3, + "activity_execution_id": "prior-id", + "descriptor_hash": "51d402b88de4404211114f85b23592d22a41e5a2e9ea7d827178e3ad62683ac4" + }, + { + "sequence": 4, + "activity_execution_id": "original-id", + "descriptor_hash": "c3a3d20bd8882dfc59673847c3f8ed67c66cdaee74e989810002e2b622764699" + } + ], + "timer_members": [ + { + "sequence": 5, + "timer_id": "original-timer", + "descriptor_hash": "bae0466390116ff60cbd3597bcdaf8e0097fae4c7f8b789db6f31ee9553b62c1" + }, + { + "sequence": 7, + "timer_id": "original-timeout", + "descriptor_hash": "f5518182dcbfc2737f54b9a1754e941e8efa416815a2aad9c8e4cec7f266d423" + } + ], + "wait_members": [ + { + "kind": "condition", + "sequence": 7, + "wait_id": "original-wait", + "timer_id": "original-timeout", + "descriptor_hash": "0bd2f375d595a9433320f9b48a03831232d40aac2258f1193d2026248f822d3f" + } + ], + "child_members": [], + "descendant_members": [], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m43xf6bwxny6v9ys141sszkp", + "namespace": "sdk-scope-fixture", + "sequence": 14, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "01M43XF6A4PXETWBHA5DZH9ZE5", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 4, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:30.123456Z", + "preparation_history_event_id": "01m43xf6az7v2abqsr0tbz27a1", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + } +} From 6a3eac2915e927fb5357a781754bbf08f1528be3 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 02:02:42 +0000 Subject: [PATCH 49/58] Allow proved cancellation groups through scope transport --- src/cancellation_scope_history.rs | 9 +++++---- src/lib.rs | 2 +- src/tests/cancellation_scope_history.rs | 15 ++++++++++++--- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/cancellation_scope_history.rs b/src/cancellation_scope_history.rs index 448151a..c5d8ebc 100644 --- a/src/cancellation_scope_history.rs +++ b/src/cancellation_scope_history.rs @@ -1360,7 +1360,7 @@ impl CancellationScopeDeliveryReceipt { } impl Client { - /// Candidate scalar preparation proof. All operations share the supplied budget. + /// Candidate preparation proof. All operations share the supplied budget. #[doc(hidden)] pub async fn prepare_cancellation_scope_on_claim( &self, @@ -1394,7 +1394,7 @@ impl Client { .await } - /// Candidate scalar delivery requires the earlier original preparation proof. + /// Candidate delivery requires the earlier original preparation proof. #[doc(hidden)] pub async fn deliver_cancellation_scope_on_claim( &self, @@ -1437,7 +1437,7 @@ impl Client { || context.workflow_run_id() != run || context.workflow_instance_id() != workflow || boundary.request_id != context.request_id() - || boundary.sequence_span != 1 + || (boundary.call_kind != CancellationCallKind::Parallel && boundary.sequence_span != 1) || boundary.operation_sequence.is_some() || boundary.operation_sequence_span != 1 || !matches!( @@ -1447,10 +1447,11 @@ impl Client { | CancellationCallKind::Timer | CancellationCallKind::Condition | CancellationCallKind::Child + | CancellationCallKind::Parallel ) { return Err(invalid( - "scope boundary requires its original claim and scalar durable call", + "scope boundary requires its original claim and supported durable call", )); } budget.remaining()?; diff --git a/src/lib.rs b/src/lib.rs index 60db3af..458bc55 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6368,7 +6368,7 @@ impl Worker { self } - /// Enable the candidate scalar scope consumer with canonical authoring. + /// Enable the candidate scoped call consumer with canonical authoring. #[doc(hidden)] pub fn candidate_cancellation_scope_delivery(mut self, enabled: bool) -> Self { self.allow_cancellation_scope_delivery = enabled; diff --git a/src/tests/cancellation_scope_history.rs b/src/tests/cancellation_scope_history.rs index 80fc200..c5ed1bd 100644 --- a/src/tests/cancellation_scope_history.rs +++ b/src/tests/cancellation_scope_history.rs @@ -31,7 +31,11 @@ fn scope_response(path: &str, body: &str, number: usize) -> Option<(&'static str if matches!(case, "lost-ack" | "worker-lost-ack") && path.ends_with("/prepare") && number == 1 { return Some(("invalid-status", String::new())); } - let mut source = scalar_fixture(); + let mut source = match case { + "group-flat" => fixture("groups", "flat"), + "group-nested" => fixture("groups", "nested"), + _ => scalar_fixture(), + }; if case.starts_with("worker") { // Synthetic transport time is safely ahead of the actual request budget. // The fixture has no admitted members whose descriptor contains dates. @@ -257,10 +261,14 @@ async fn cancellation_scope_replay_worker_coordinates_original_claim_and_lost_ac #[tokio::test] async fn scope_history_prepare_and_delivery_prove_original_claim_and_all_pages_after_lost_ack() { - for case in ["valid", "lost-ack"] { + for case in ["valid", "lost-ack", "group-flat", "group-nested"] { let server = scope_server(); let client = scope_client(&server, case); - let value = scalar_fixture(); + let value = match case { + "group-flat" => fixture("groups", "flat"), + "group-nested" => fixture("groups", "nested"), + _ => scalar_fixture(), + }; let task = claim(&value); let (context, boundary) = boundary(&value); let budget = CancellationScopeDeliveryBudget::new(); @@ -458,6 +466,7 @@ fn fixture(name: &str, variant: &str) -> Value { } "empty" => include_str!("../../tests/fixtures/committed-scope-delivery.json"), "single" => include_str!("../../tests/fixtures/populated-scope-single-calls.json"), + "groups" => include_str!("../../tests/fixtures/populated-scope-groups.json"), "root" => include_str!("../../tests/fixtures/run-inherited-scope-delivery.json"), _ => panic!("unknown fixture"), }; From 9fb00bb03e9eb8e75938b37a11e2379e5fb3ddce Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 02:53:35 +0000 Subject: [PATCH 50/58] Replay original cancellation contexts through included scope descendants --- docs/cooperative-cancellation-design.md | 13 +- src/cancellation_scope_history.rs | 101 +- src/cancellation_scope_replay.rs | 93 +- src/lib.rs | 11 +- src/tests/cancellation_scope_replay.rs | 2 + .../cancellation_scope_replay/descendants.rs | 496 +++ tests/cooperative_server.rs | 150 +- .../fixtures/committed-scope-descendants.json | 3512 +++++++++++++++++ ...ion-scope-descendants-default-refused.json | 38 + 9 files changed, 4331 insertions(+), 85 deletions(-) create mode 100644 src/tests/cancellation_scope_replay/descendants.rs create mode 100644 tests/fixtures/committed-scope-descendants.json create mode 100644 tests/fixtures/replay-regressions/cancellation-scope-descendants-default-refused.json diff --git a/docs/cooperative-cancellation-design.md b/docs/cooperative-cancellation-design.md index fe45dc0..a414565 100644 --- a/docs/cooperative-cancellation-design.md +++ b/docs/cooperative-cancellation-design.md @@ -172,9 +172,16 @@ span, cleanup sequence, context and deadline. Shielded cleanup timers carry only the original scope, request and delivery references. Server derives their immutable authority and rejects a timer that would reach its ceiling. Unsupported local, signal, selection, incomplete or -mixed-scope groups are refused before the workflow factory runs. Descendant -delivery and root/scope composition still require qualification. These opt-ins -remain disabled by default and do not advertise general scoped execution. +mixed-scope groups are refused before the workflow factory runs. An ancestor +delivery restores each included descendant's original accepted context as code +unwinds. Shielded branches remain unaffected. Cleanup timers in each included +scope retain its own request, the ancestor receipt and its narrower authority +ceiling. Replacement replay preserves the recorded clock and immutable metadata. +Pending ancestor requests keep the original authored boundary without entering +cleanup. Competing roots, overlapping deliveries, subtree local callbacks and +root/scope composition remain gated. These opt-ins remain disabled by default +and do not advertise general scoped execution. Exact connected qualification +and the published acceptance scenario remain separate gates. ## Remaining qualification diff --git a/src/cancellation_scope_history.rs b/src/cancellation_scope_history.rs index c5d8ebc..bfffeea 100644 --- a/src/cancellation_scope_history.rs +++ b/src/cancellation_scope_history.rs @@ -63,9 +63,88 @@ pub(super) fn timestamp(value: &Value) -> Result> { .ok_or_else(|| invalid("scope authority requires an original timestamp with timezone")) } +pub(super) fn boundary_scope_states( + context: &ScopedCancellationContext, + preparation: &Value, +) -> Result)>> { + let mut states = BTreeMap::from([( + context.scope_id().to_owned(), + ( + context.clone(), + timestamp(&preparation["authority_deadline_at"])?, + ), + )]); + for member in preparation["descendant_members"] + .as_array() + .ok_or_else(|| invalid("scope preparation lacks its frozen subtree"))? + { + let descendant = ScopedCancellationContext::from_value(&member["cancellation"])?; + if descendant.root_context() != context.root_context() + || !descendant.lineage().starts_with(context.lineage()) + || states.contains_key(descendant.scope_id()) + { + return Err(invalid( + "scope descendant changes its original root or accepted lineage", + )); + } + states.insert( + descendant.scope_id().to_owned(), + (descendant, timestamp(&member["authority_deadline_at"])?), + ); + } + Ok(states) +} + +fn delivery_scope_states( + delivery: &HistoryEvent, + prefix: &[HistoryEvent], +) -> Result)>> { + let preparations: Vec<_> = prefix + .iter() + .filter(|row| { + row.event_type == "CancellationScopeDeliveryPrepared" + && row.raw["id"] == delivery.payload["preparation_history_event_id"] + }) + .collect(); + if preparations.len() != 1 + || preparations[0].raw["sequence"] + .as_u64() + .zip(delivery.raw["sequence"].as_u64()) + .is_none_or(|(before, after)| before >= after) + { + return Err(invalid( + "cleanup timer lacks its original canonical preparation", + )); + } + boundary_scope_states( + &ScopedCancellationContext::from_value(&delivery.payload["cancellation"])?, + &preparations[0].payload, + ) +} + fn cleanup_timer(event: &HistoryEvent, prefix: &[HistoryEvent]) -> Result { let p = &event.payload; let Some(snapshot) = p.get("cancellation_cleanup") else { + if p["cancellation_scope_id"] + .as_str() + .is_some_and(|scope| scope != "root") + { + for delivery in prefix.iter().filter(|row| { + row.event_type == "CancellationScopeDelivered" + && row.raw["sequence"] + .as_u64() + .zip(event.raw["sequence"].as_u64()) + .is_some_and(|(delivery, timer)| delivery < timer) + }) { + if delivery_scope_states(delivery, prefix)? + .contains_key(text(p, "cancellation_scope_id")?) + { + return Err(invalid( + "cleanup timer omits its original delivery snapshot", + )); + } + } + } return Ok(false); }; let matches: Vec<_> = prefix @@ -82,16 +161,18 @@ fn cleanup_timer(event: &HistoryEvent, prefix: &[HistoryEvent]) -> Result } let delivery = matches[0]; let original = &delivery.payload; - let context = ScopedCancellationContext::from_value(&original["cancellation"])?; + let states = delivery_scope_states(delivery, prefix)?; + let (context, ceiling) = states + .get(text(p, "cancellation_scope_id")?) + .ok_or_else(|| invalid("cleanup timer changes its original frozen subtree membership"))?; let expected = json!({"scope_id":context.scope_id(), "operation_scope_id":context.scope_id(), "request_id":context.request_id(), "root_request_id":context.root_context().root_request_id(), "delivery_history_event_id":event_id(delivery)?, "preparation_history_event_id":original["preparation_history_event_id"], "cleanup_deadline_at":canonical_time(context.deadline()), - "authority_deadline_at":original["authority_deadline_at"]}); - let ceiling = timestamp(&original["authority_deadline_at"])?; + "authority_deadline_at":canonical_time(*ceiling)}); let boundary = CancellationDelivery::from_payload(&json!({ - "workflow_command_id":context.request_id(), "sequence":original["sequence"], + "workflow_command_id":original["request_id"], "sequence":original["sequence"], "call_kind":original["call_kind"], "sequence_span":original["sequence_span"], "operation_sequence":original["operation_sequence"], "operation_sequence_span":original["operation_sequence_span"]}))?; if snapshot != &expected @@ -105,8 +186,9 @@ fn cleanup_timer(event: &HistoryEvent, prefix: &[HistoryEvent]) -> Result .is_none_or(|prior| prior >= sequence) }) || event_time(event)? < event_time(delivery)? - || event_time(event)? >= ceiling - || timestamp(&p["fire_at"])? >= ceiling + || event_time(event)? >= *ceiling + || timestamp(&p["fire_at"])? < event_time(event)? + || timestamp(&p["fire_at"])? >= *ceiling || !p["timer_kind"].is_null() { return Err(invalid( @@ -811,13 +893,12 @@ impl CommittedCancellationScopeHistory { for (index, event) in history.iter().enumerate() { let kind = event.event_type.as_str(); let p = &event.payload; - if kind == "TimerScheduled" { - cleanup_timer(event, &history[..index])?; - } + let timer_cleanup = + kind == "TimerScheduled" && cleanup_timer(event, &history[..index])?; if kind == "CancellationScopeOpened" { opened.insert(text(p, "scope_id")?); } - if admission(kind).is_some() && positive(&p["sequence"]) { + if admission(kind).is_some() && positive(&p["sequence"]) && !timer_cleanup { let sequence = p["sequence"].as_u64().unwrap(); admissions.entry(kind).or_default().insert( sequence, diff --git a/src/cancellation_scope_replay.rs b/src/cancellation_scope_replay.rs index 4d699c4..cd39642 100644 --- a/src/cancellation_scope_replay.rs +++ b/src/cancellation_scope_replay.rs @@ -1,7 +1,9 @@ //! Candidate scope delivery. Admission remains private and disabled by default. use super::*; -use crate::cancellation_scope_history::CommittedCancellationScopeHistory; +use crate::cancellation_scope_history::{ + boundary_scope_states, CommittedCancellationScopeHistory, ScopeBoundary, +}; use chrono::Utc; #[doc(hidden)] @@ -26,6 +28,8 @@ pub(super) struct ScopeReplay { pub active_scope: String, pub consumed: BTreeSet, pub contexts: BTreeMap)>, + pub boundary_states: + BTreeMap)>>, pub cleanup_timers: BTreeMap, pub intent: Option, } @@ -209,6 +213,7 @@ impl ScopeReplay { workflow: &str, ) -> Result { let canonical = CommittedCancellationScopeHistory::read(history, run, workflow)?; + let mut boundary_states = BTreeMap::new(); for preparation in canonical.preparations.values() { let boundary = &preparation.boundary; let supported = if boundary.call_kind == CancellationCallKind::Parallel { @@ -231,14 +236,35 @@ impl ScopeReplay { } else { scalar(boundary.call_kind) && boundary.sequence_span == 1 }; - if !supported - || preparation.boundary.operation_sequence.is_some() - || preparation.event.payload["descendant_members"] - .as_array() - .is_none_or(|members| !members.is_empty()) - { + if !supported || preparation.boundary.operation_sequence.is_some() { return Err(Error::CancellationScopeExecutionUnavailable); } + let states = boundary_scope_states(&preparation.context, &preparation.event.payload)?; + for event in history + .iter() + .take_while(|row| row.raw["id"] != preparation.event.raw["id"]) + { + if event.event_type == "ActivityScheduled" + && durable_event_sequence(event) + .and_then(|sequence| tree.memberships.get(&sequence)) + .is_some_and(|scope| states.contains_key(scope)) + && (event.payload["local_activity"] == true + || event.payload["execution_mode"] == "local" + || event.payload["activity"]["local_activity"] == true + || event.payload["activity"]["execution_mode"] == "local") + { + return Err(Error::CancellationScopeExecutionUnavailable); + } + } + boundary_states.insert(preparation.context.scope_id().to_owned(), states); + } + let mut delivered_scopes = BTreeSet::new(); + for delivery in canonical.deliveries.values() { + for scope in boundary_states[delivery.context.scope_id()].keys() { + if !delivered_scopes.insert(scope.clone()) { + return Err(Error::CancellationScopeExecutionUnavailable); + } + } } Ok(Self { canonical, @@ -246,6 +272,7 @@ impl ScopeReplay { active_scope: "root".into(), consumed: BTreeSet::new(), contexts: BTreeMap::new(), + boundary_states, cleanup_timers: history.iter().filter(|row| row.event_type == "TimerScheduled") .filter_map(|row| row.payload.get("cancellation_cleanup").map(|snapshot| (row.payload["sequence"].as_u64().unwrap(), json!({"scope_id":snapshot["scope_id"], @@ -254,6 +281,13 @@ impl ScopeReplay { }) } + pub(super) fn consumed_delivery_for_scope(&self, scope: &str) -> Option<&ScopeBoundary> { + self.canonical.deliveries.values().find(|delivery| { + self.consumed.contains(&delivery.boundary.sequence) + && self.boundary_states[delivery.context.scope_id()].contains_key(scope) + }) + } + pub fn bind_commands(&self, commands: &mut Vec) -> Result<()> { for (&sequence, delivered) in &self.canonical.deliveries { let index = commands.partition_point(|command| command.sequence() < sequence); @@ -327,7 +361,8 @@ impl WorkflowState { let sequence = descriptors[0].group_path[0].parallel_group_base_sequence; if let Some(delivered) = replay.canonical.deliveries.get(&sequence) { let delivered = delivered.clone(); - if replay.active_scope != delivered.context.scope_id() + if !replay.boundary_states[delivered.context.scope_id()] + .contains_key(&replay.active_scope) || self.cancellation_shield_depth > 0 || delivered.boundary.call_kind != CancellationCallKind::Parallel { @@ -381,10 +416,9 @@ impl WorkflowState { if replay.contexts.contains_key(request.context.scope_id()) { return Ok(false); } - if request.context.scope_id() != replay.active_scope - || descriptors - .iter() - .any(|descriptor| matches!(descriptor.operation, ParallelOperation::Signal(_))) + if descriptors + .iter() + .any(|descriptor| matches!(descriptor.operation, ParallelOperation::Signal(_))) { return Err(Error::CancellationScopeExecutionUnavailable); } @@ -477,19 +511,11 @@ impl WorkflowState { let Some((context, _)) = replay.contexts.get(scope) else { return Ok(None); }; - let delivered = replay - .canonical - .deliveries - .values() - .find(|delivery| { - delivery.context == *context - && replay.consumed.contains(&delivery.boundary.sequence) - }) - .ok_or_else(|| { - Error::InvalidCooperativeCancellation( - "scoped cleanup lacks its consumed original delivery".into(), - ) - })?; + let delivered = replay.consumed_delivery_for_scope(scope).ok_or_else(|| { + Error::InvalidCooperativeCancellation( + "scoped cleanup lacks its consumed original delivery".into(), + ) + })?; Ok(Some( json!({"scope_id":scope, "request_id":context.request_id(), "delivery_history_event_id":delivered.event.raw["id"]}), @@ -564,7 +590,7 @@ impl WorkflowState { { return Ok(false); } - if !scalar(kind) || !path.is_empty() || request.context.scope_id() != replay.active_scope { + if !scalar(kind) || !path.is_empty() { return Err(Error::CancellationScopeExecutionUnavailable); } let boundary = CancellationDelivery { @@ -629,7 +655,8 @@ impl WorkflowState { }; if replay.consumed.contains(&sequence) || delivered.boundary.call_kind != kind - || replay.active_scope != delivered.context.scope_id() + || !replay.boundary_states[delivered.context.scope_id()] + .contains_key(&replay.active_scope) || self.cancellation_shield_depth > 0 { return Err(invalid( @@ -640,10 +667,10 @@ impl WorkflowState { self.observe_scope_cancellation_delivery(&delivered.event); let replay = self.scope_delivery.as_mut().unwrap(); replay.consumed.insert(sequence); - replay.contexts.insert( - delivered.context.scope_id().into(), - (delivered.context.clone(), delivered.authority_deadline), - ); + replay + .contexts + .extend(replay.boundary_states[delivered.context.scope_id()].clone()); + let active_context = replay.contexts[&replay.active_scope].0.clone(); self.command_cursor = index + if kind == CancellationCallKind::Parallel { delivered.boundary.sequence_span as usize @@ -652,9 +679,7 @@ impl WorkflowState { }; Err(Error::CancellationScopeRequested( CancellationScopeRequested { - context: delivered - .context - .with_replay(cancellation_replay_clock::active_binding()), + context: active_context.with_replay(cancellation_replay_clock::active_binding()), delivery: delivered.boundary, }, )) diff --git a/src/lib.rs b/src/lib.rs index 458bc55..c7f5c93 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8979,11 +8979,12 @@ impl WorkflowContext { if let Some(replay) = &state.scope_delivery { if let Some((context, _)) = replay.contexts.get(&self.cancellation_scope_id) { let delivered = replay - .canonical - .deliveries - .values() - .find(|delivered| delivered.context.request_id() == context.request_id()) - .unwrap(); + .consumed_delivery_for_scope(&self.cancellation_scope_id) + .ok_or_else(|| { + Error::InvalidCooperativeCancellation( + "scoped request lacks its consumed original delivery".into(), + ) + })?; return Err(Error::CancellationScopeRequested( CancellationScopeRequested { context: context diff --git a/src/tests/cancellation_scope_replay.rs b/src/tests/cancellation_scope_replay.rs index d932795..779c7df 100644 --- a/src/tests/cancellation_scope_replay.rs +++ b/src/tests/cancellation_scope_replay.rs @@ -1,5 +1,7 @@ use super::*; +mod descendants; + fn group_fixture(name: &str) -> Value { let source: Value = serde_json::from_str(include_str!( "../../tests/fixtures/populated-scope-groups.json" diff --git a/src/tests/cancellation_scope_replay/descendants.rs b/src/tests/cancellation_scope_replay/descendants.rs new file mode 100644 index 0000000..9ad42d7 --- /dev/null +++ b/src/tests/cancellation_scope_replay/descendants.rs @@ -0,0 +1,496 @@ +use super::*; + +type Observed = Arc>>; + +fn descendant_fixture(layout: &str) -> Value { + let fixtures: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/committed-scope-descendants.json" + )) + .unwrap(); + fixtures[layout].clone() +} + +async fn capture( + ctx: &WorkflowContext, + result: Result<()>, + name: &str, + source: &Value, + seen: &Observed, + cleanup: &str, +) -> Result { + let context = match result { + Err(Error::CancellationScopeRequested(request)) => { + assert_eq!( + request.delivery.request_id, + source["contexts"]["parent"]["lineage"] + .as_array() + .unwrap() + .last() + .unwrap()["request_id"] + ); + request.context + } + Err(error) => return Err(error), + Ok(()) => panic!("{name} must retain its original accepted request"), + }; + assert!(ctx.is_cancellation_requested()?); + assert_eq!(ctx.scoped_cancellation_context()?.unwrap(), context); + assert_eq!(context.to_value(), source["contexts"][name]); + let rank = |scope| match scope { + "grandchild" => 0, + "child" => 1, + _ => 2, + }; + let fired = source["history"].as_array().unwrap().iter().any(|row| { + row["event_type"] == "TimerFired" && row["payload"]["timer_id"] == "descendant-cleanup" + }); + assert_eq!( + context.remaining()?, + Duration::from_secs( + if fired && !cleanup.is_empty() && rank(name) > rank(cleanup) { + 15 + } else { + 17 + } + ) + ); + seen.lock().unwrap().insert(name.into(), context); + if cleanup == name { + let _shield = ctx.cancellation_shield()?; + ctx.sleep(Duration::from_secs(1)).await?; + } + Ok(Value::Null) +} + +async fn grandchild_body( + ctx: WorkflowContext, + source: Value, + seen: Observed, + cleanup: &'static str, + changed: &'static str, +) -> Result { + if changed != "prefix" { + assert_eq!( + ctx.activity("prior-step", json!([])).await?, + json!("prior-value") + ); + } + let interrupted = if source["layout"] == "timer" { + ctx.sleep(Duration::from_secs(if changed == "timer" { + 42 + } else { + 3600 + })) + .await + } else { + let predicate = source["history"] + .as_array() + .unwrap() + .iter() + .find(|row| row["event_type"] == "ConditionWaitOpened") + .unwrap()["payload"]["condition_definition_fingerprint"] + .as_str() + .unwrap(); + let activity = ParallelOperation::activity_with_options( + if changed == "activity" { + "changed" + } else { + "original-activity" + }, + ActivityOptions::new().cancellation_policy(if changed == "activity-policy" { + CancellationPolicy::Abandon + } else { + CancellationPolicy::TryCancel + }), + json!([]), + ); + let timer = ParallelOperation::timer(Duration::from_secs(if changed == "timer" { + 42 + } else { + 3600 + })); + let child = ParallelOperation::child_workflow( + if changed == "child" { + "changed" + } else { + "original-child" + }, + ChildWorkflowOptions::new("queue") + .parent_close_policy(ParentClosePolicy::RequestCancel) + .cancellation_policy(if changed == "child-policy" { + CancellationPolicy::Abandon + } else { + CancellationPolicy::WaitCancellationCompleted + }), + json!([]), + ); + let condition = ParallelOperation::condition( + ConditionWaitOptions::new("ready", predicate).timeout(Duration::from_secs(30)), + || panic!("committed cancellation must not evaluate the interrupted predicate"), + ); + let mut group = if changed == "layout" { + vec![activity, timer, child, condition] + } else { + vec![ + activity, + ParallelOperation::group(vec![timer, child]), + condition, + ] + }; + if changed == "size" { + group.pop(); + } + ctx.parallel(group).await.map(|_| ()) + }; + capture(&ctx, interrupted, "grandchild", &source, &seen, cleanup).await +} + +async fn child_body( + ctx: WorkflowContext, + source: Value, + seen: Observed, + cleanup: &'static str, + changed: &'static str, +) -> Result { + let value = source.clone(); + let observed = Arc::clone(&seen); + ctx.cancellation_scope(changed == "shield", move |inner| { + grandchild_body(inner, value, observed, cleanup, changed) + }) + .await?; + capture( + &ctx, + ctx.throw_if_cancellation_requested(), + "child", + &source, + &seen, + cleanup, + ) + .await +} + +async fn parent_body( + ctx: WorkflowContext, + source: Value, + seen: Observed, + cleanup: &'static str, + changed: &'static str, +) -> Result { + ctx.cancellation_scope(true, move |shield| async move { + assert!(!shield.is_cancellation_requested()?); + assert!(shield.scoped_cancellation_context()?.is_none()); + shield + .cancellation_scope(false, move |inner| async move { + assert!(!inner.is_cancellation_requested()?); + assert!(inner.scoped_cancellation_context()?.is_none()); + Ok(Value::Null) + }) + .await + }) + .await?; + let value = source.clone(); + let observed = Arc::clone(&seen); + ctx.cancellation_scope(false, move |inner| { + child_body(inner, value, observed, cleanup, changed) + }) + .await?; + capture( + &ctx, + ctx.throw_if_cancellation_requested(), + "parent", + &source, + &seen, + cleanup, + ) + .await +} + +fn descendant_worker( + source: Value, + seen: Observed, + cleanup: &'static str, + changed: &'static str, +) -> Worker { + let mut worker = Worker::new(Client::new("http://unused.invalid").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + worker.register_workflow("scope-replay", move |ctx, _| { + let source = source.clone(); + let observed = Arc::clone(&seen); + async move { + let value = source.clone(); + let inside = Arc::clone(&observed); + ctx.cancellation_scope(false, move |outer| async move { + outer + .cancellation_scope(false, move |parent| { + parent_body(parent, value, inside, cleanup, changed) + }) + .await?; + assert!(!outer.is_cancellation_requested()?); + assert!(outer.scoped_cancellation_context()?.is_none()); + Ok(Value::Null) + }) + .await?; + assert!(!ctx.is_cancellation_requested()?); + assert!(ctx.scoped_cancellation_context()?.is_none()); + ctx.sleep(Duration::from_secs(1)).await?; + Ok(if cleanup.is_empty() { + json!("survivor") + } else { + json!({"remaining":observed.lock().unwrap()[cleanup].remaining()?.as_secs()}) + }) + } + }); + worker +} + +fn next_sequence(source: &Value) -> u64 { + if source["layout"] == "timer" { + 9 + } else { + 12 + } +} + +fn snapshot_for(source: &Value, target: &str) -> Value { + let context = ScopedCancellationContext::from_value(&source["contexts"][target]).unwrap(); + json!({"scope_id":context.scope_id(), "operation_scope_id":context.scope_id(), + "request_id":context.request_id(), "root_request_id":context.root_context().root_request_id(), + "delivery_history_event_id":"ancestor-delivered", "preparation_history_event_id":"ancestor-prepared", + "cleanup_deadline_at":"2026-10-04T00:00:30.123456Z", "authority_deadline_at":"2026-10-04T00:00:26.123456Z"}) +} + +#[test] +fn descendant_original_contexts_restore_and_leave_outer_and_root_unaffected() { + for layout in ["timer", "group"] { + let source = descendant_fixture(layout); + let seen = Arc::new(Mutex::new(BTreeMap::new())); + let worker = descendant_worker(source.clone(), Arc::clone(&seen), "", ""); + let decision = worker + .execute_workflow_task_decision(task(&source)) + .unwrap(); + assert_eq!(seen.lock().unwrap().len(), 3); + assert_eq!( + decision.commands, + vec![json!({"type":"start_timer", "delay_seconds":1})] + ); + } +} + +#[test] +fn descendant_pending_ancestor_preserves_original_identity_and_range_without_cleanup() { + for layout in ["timer", "group"] { + for before in [ + "CancellationScopeDeliveryPrepared", + "CancellationScopeDelivered", + ] { + let source = prefix(&descendant_fixture(layout), before); + let seen = Arc::new(Mutex::new(BTreeMap::new())); + let decision = descendant_worker(source.clone(), Arc::clone(&seen), "", "") + .execute_workflow_task_decision(task(&source)) + .unwrap(); + assert!(decision.commands.is_empty()); + assert!(seen.lock().unwrap().is_empty()); + let intent = decision.cancellation_scope_delivery.unwrap(); + assert_eq!(intent.context.to_value(), source["contexts"]["parent"]); + assert_eq!(intent.boundary.sequence, 8); + assert_eq!( + intent.boundary.sequence_span, + if layout == "timer" { 1 } else { 4 } + ); + } + } +} + +#[test] +fn descendant_cleanup_retains_ancestor_proof_narrower_authority_and_replacement_clock() { + for layout in ["timer", "group"] { + for target in ["grandchild", "child", "parent"] { + let mut source = descendant_fixture(layout); + let seen = Arc::new(Mutex::new(BTreeMap::new())); + let snapshot = snapshot_for(&source, target); + let sequence = next_sequence(&source); + let wire = json!({"type":"start_timer", "delay_seconds":1, + "cancellation_scope_id":snapshot["scope_id"], "cancellation_cleanup":{ + "scope_id":snapshot["scope_id"], "request_id":snapshot["request_id"], + "delivery_history_event_id":snapshot["delivery_history_event_id"]}}); + assert_eq!( + descendant_worker(source.clone(), Arc::clone(&seen), target, "") + .execute_workflow_task_decision(task(&source)) + .unwrap() + .commands, + vec![wire.clone()] + ); + append( + &mut source, + "TimerScheduled", + json!({"sequence":sequence, + "timer_id":"descendant-cleanup", "delay_seconds":1, + "cancellation_scope_id":snapshot["scope_id"], "cancellation_cleanup":snapshot, + "fire_at":"2026-10-04T00:00:11.123456Z"}), + "2026-10-04T00:00:10.123456Z", + ); + let mut claim = task(&source); + claim.lease_owner = Some("replacement".into()); + claim.workflow_task_attempt = 17; + let blocked = descendant_worker(source.clone(), Arc::clone(&seen), target, "") + .execute_workflow_task_decision(claim) + .unwrap(); + assert!( + blocked.commands.is_empty(), + "an admitted cleanup timer must remain pending" + ); + assert!(blocked.cancellation_scope_delivery.is_none()); + append( + &mut source, + "TimerFired", + json!({"sequence":sequence, + "timer_id":"descendant-cleanup", "delay_seconds":1, + "cancellation_scope_id":snapshot["scope_id"]}), + "2026-10-04T00:00:11.123456Z", + ); + assert_eq!( + descendant_worker(source.clone(), Arc::clone(&seen), target, "") + .execute_workflow_task_decision(task(&source)) + .unwrap() + .commands, + vec![json!({"type":"start_timer", "delay_seconds":1})] + ); + append( + &mut source, + "TimerScheduled", + json!({"sequence":sequence+1, "timer_id":"root-timer", + "delay_seconds":1, "fire_at":"2026-10-04T00:00:12.123456Z"}), + "2026-10-04T00:00:11.123456Z", + ); + append( + &mut source, + "TimerFired", + json!({"sequence":sequence+1, "timer_id":"root-timer", + "delay_seconds":1}), + "2026-10-04T00:00:20.123456Z", + ); + let result = descendant_worker(source.clone(), Arc::clone(&seen), target, "") + .execute_workflow_task_decision(task(&source)) + .unwrap(); + assert_eq!(result.commands[0]["type"], "complete_workflow"); + assert_eq!( + decode_payload::( + &serde_json::from_value(result.commands[0]["result"].clone()).unwrap() + ) + .unwrap(), + json!({"remaining":6}) + ); + } + } +} + +#[test] +fn descendant_changed_subtree_cannot_enter_cleanup() { + for (layout, change) in [ + ("timer", "shield"), + ("timer", "timer"), + ("timer", "prefix"), + ("group", "activity"), + ("group", "activity-policy"), + ("group", "child"), + ("group", "child-policy"), + ("group", "layout"), + ("group", "size"), + ] { + let source = descendant_fixture(layout); + let seen = Arc::new(Mutex::new(BTreeMap::new())); + assert!( + descendant_worker(source.clone(), Arc::clone(&seen), "", change) + .execute_workflow_task_decision(task(&source)) + .is_err(), + "{layout}/{change}" + ); + assert!(seen.lock().unwrap().is_empty(), "{layout}/{change}"); + } +} + +#[test] +fn descendant_changed_cleanup_snapshot_refuses_before_factory() { + for field in [ + "scope_id", + "operation_scope_id", + "request_id", + "root_request_id", + "delivery_history_event_id", + "preparation_history_event_id", + "cleanup_deadline_at", + "authority_deadline_at", + "omitted", + "null", + "retrograde", + ] { + let mut source = descendant_fixture("timer"); + let mut snapshot = snapshot_for(&source, "grandchild"); + if field == "null" { + snapshot = Value::Null; + } else if field != "omitted" && field != "retrograde" { + snapshot[field] = json!("changed"); + } + let mut payload = json!({"sequence":9, "timer_id":"descendant-cleanup", "delay_seconds":1, + "cancellation_scope_id":"grandchild-scope", "cancellation_cleanup":snapshot, + "fire_at":if field=="retrograde" { "2026-10-04T00:00:09.123456Z" } else { "2026-10-04T00:00:11.123456Z" }}); + if field == "omitted" { + payload + .as_object_mut() + .unwrap() + .remove("cancellation_cleanup"); + } + append( + &mut source, + "TimerScheduled", + payload, + "2026-10-04T00:00:10.123456Z", + ); + let invoked = Arc::new(AtomicUsize::new(0)); + let calls = Arc::clone(&invoked); + let mut worker = descendant_worker( + source.clone(), + Arc::new(Mutex::new(BTreeMap::new())), + "", + "", + ); + worker.register_workflow("scope-replay", move |_, _| { + calls.fetch_add(1, Ordering::SeqCst); + async { Ok(Value::Null) } + }); + assert!( + worker + .execute_workflow_task_decision(task(&source)) + .is_err(), + "{field}" + ); + assert_eq!(invoked.load(Ordering::SeqCst), 0, "{field}"); + } +} + +#[test] +fn descendant_competing_root_refuses_before_factory() { + let fixtures: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/committed-scope-operation-projections.json" + )) + .unwrap(); + let source = fixtures["competing"].clone(); + let invoked = Arc::new(AtomicUsize::new(0)); + let calls = Arc::clone(&invoked); + let mut worker = descendant_worker( + source.clone(), + Arc::new(Mutex::new(BTreeMap::new())), + "", + "", + ); + worker.register_workflow("scope-replay", move |_, _| { + calls.fetch_add(1, Ordering::SeqCst); + async { Ok(Value::Null) } + }); + assert!(worker + .execute_workflow_task_decision(task(&source)) + .is_err()); + assert_eq!(invoked.load(Ordering::SeqCst), 0); +} diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index a79b04f..cff5566 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -3,8 +3,9 @@ use durable_workflow::{ json, ActivityContext, ActivityOptions, CancellationPolicy, Client, ConditionWaitOptions, - CooperativeCancellationOptions, Error, ParallelOperation, ParallelResult, SelectionKey, Value, - Worker, WorkflowCommandOptions, WorkflowHandle, WorkflowResultOptions, + CooperativeCancellationOptions, Error, ParallelOperation, ParallelResult, + ScopedCancellationContext, SelectionKey, Value, Worker, WorkflowCommandOptions, + WorkflowContext, WorkflowHandle, WorkflowResultOptions, }; use std::{ sync::{ @@ -881,7 +882,44 @@ fn request_native_scope_fixture(handle: &WorkflowHandle, scope: &str) -> Value { serde_json::from_slice(&result.stdout).unwrap() } -fn scope_cleanup_worker(client: &Client, queue: &str, owner: &str, grouped: bool) -> Worker { +async fn scope_cleanup_body( + scope: WorkflowContext, + grouped: bool, +) -> durable_workflow::Result<(ScopedCancellationContext, Value)> { + let pending = if grouped { + scope + .parallel(vec![ + ParallelOperation::timer(Duration::from_secs(300)), + ParallelOperation::group(vec![ParallelOperation::timer(Duration::from_secs(600))]), + ]) + .await + .map(|_| ()) + } else { + scope.sleep(Duration::from_secs(300)).await + }; + let cancellation = match pending { + Err(Error::CancellationScopeRequested(request)) => request.context, + Err(error) => return Err(error), + Ok(()) => { + return Err(Error::InvalidCooperativeCancellation( + "original timer was not interrupted".into(), + )) + } + }; + assert!(scope.is_cancellation_requested()?); + let _shield = scope.cancellation_shield()?; + let metadata = scope.side_effect(|| cancellation.to_value())?; + scope.sleep(Duration::from_secs(1)).await?; + Ok((cancellation, metadata)) +} + +fn scope_cleanup_worker( + client: &Client, + queue: &str, + owner: &str, + grouped: bool, + descendants: bool, +) -> Worker { let mut worker = Worker::new(client.clone(), queue) .worker_id(owner) .cooperative_cancellation(true) @@ -892,41 +930,39 @@ fn scope_cleanup_worker(client: &Client, queue: &str, owner: &str, grouped: bool "tests.rust-candidate-scope-cleanup", move |ctx, _| async move { let _: String = ctx.side_effect(|| "original prefix".to_owned())?; - let (cancellation, metadata) = ctx + let (cancellation, metadata, descendant_metadata) = ctx .cancellation_scope(false, move |scope| async move { - let pending = if grouped { - scope - .parallel(vec![ - ParallelOperation::timer(Duration::from_secs(300)), - ParallelOperation::group(vec![ParallelOperation::timer( - Duration::from_secs(600), - )]), - ]) - .await - .map(|_| ()) - } else { - scope.sleep(Duration::from_secs(300)).await - }; - let cancellation = match pending { + if !descendants { + let (cancellation, metadata) = scope_cleanup_body(scope, grouped).await?; + return Ok((cancellation, metadata, None)); + } + let (_, child_metadata) = scope + .cancellation_scope(false, move |child| scope_cleanup_body(child, grouped)) + .await?; + let cancellation = match scope.throw_if_cancellation_requested() { Err(Error::CancellationScopeRequested(request)) => request.context, Err(error) => return Err(error), Ok(()) => { return Err(Error::InvalidCooperativeCancellation( - "original timer was not interrupted".into(), + "parent lost its original accepted request".into(), )) } }; - assert!(scope.is_cancellation_requested()?); + let metadata = cancellation.to_value(); + assert_eq!(metadata["root_context"], child_metadata["root_context"]); + assert_eq!(scope.scoped_cancellation_context()?.unwrap(), cancellation); let _shield = scope.cancellation_shield()?; - let metadata = scope.side_effect(|| cancellation.to_value())?; scope.sleep(Duration::from_secs(1)).await?; - Ok((cancellation, metadata)) + Ok((cancellation, metadata, Some(child_metadata))) }) .await?; assert!(!ctx.is_cancellation_requested()?); assert!(ctx.scoped_cancellation_context()?.is_none()); ctx.sleep(Duration::from_secs(1)).await?; - Ok(json!({"context":metadata, "remaining":cancellation.remaining()?.as_secs_f64()})) + Ok( + json!({"context":metadata, "remaining":cancellation.remaining()?.as_secs_f64(), + "descendant_context":descendant_metadata}), + ) }, ); worker @@ -935,22 +971,40 @@ fn scope_cleanup_worker(client: &Client, queue: &str, owner: &str, grouped: bool #[tokio::test] #[ignore = "requires an isolated cooperative Server with the Native scope candidate"] async fn server_scope_cleanup_preserves_original_delivery_and_budget_after_replacement() { - qualify_scope_cleanup_replacement(false).await; + qualify_scope_cleanup_replacement(false, false).await; } #[tokio::test] #[ignore = "requires an isolated cooperative Server with the Native scope candidate"] async fn server_scope_group_cleanup_preserves_original_delivery_and_budget_after_replacement() { - qualify_scope_cleanup_replacement(true).await; + qualify_scope_cleanup_replacement(true, false).await; } -async fn qualify_scope_cleanup_replacement(grouped: bool) { +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with the Native scope candidate"] +async fn server_scope_descendant_cleanup_preserves_original_delivery_after_replacement() { + qualify_scope_cleanup_replacement(false, true).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with the Native scope candidate"] +async fn server_scope_descendant_group_cleanup_preserves_original_delivery_after_replacement() { + qualify_scope_cleanup_replacement(true, true).await; +} + +async fn qualify_scope_cleanup_replacement(grouped: bool, descendants: bool) { let client = client(); let queue = format!( "rust-cooperative-scope-boundary-{}", durable_workflow::Uuid::new_v4().simple() ); - let original = scope_cleanup_worker(&client, &queue, &format!("{queue}-original"), grouped); + let original = scope_cleanup_worker( + &client, + &queue, + &format!("{queue}-original"), + grouped, + descendants, + ); original.register().await.unwrap(); let handle = client .start_workflow( @@ -979,19 +1033,29 @@ async fn qualify_scope_cleanup_replacement(grouped: bool) { if grouped { 3 } else { 2 } ); assert_eq!(count(&cleaning, "CancellationScopeDelivered"), 1); - let replacement = - scope_cleanup_worker(&client, &queue, &format!("{queue}-replacement"), grouped); + let replacement = scope_cleanup_worker( + &client, + &queue, + &format!("{queue}-replacement"), + grouped, + descendants, + ); replacement.register().await.unwrap(); let final_history = tick_until(&replacement, &handle, "WorkflowCompleted").await; for kind in [ - "CancellationScopeOpened", - "CancellationScopeRequested", "CancellationScopeDeliveryPrepared", "CancellationScopeDelivered", "WorkflowCompleted", ] { assert_eq!(count(&final_history, kind), 1, "{kind}"); } + for kind in ["CancellationScopeOpened", "CancellationScopeRequested"] { + assert_eq!( + count(&final_history, kind), + if descendants { 2 } else { 1 }, + "{kind}" + ); + } assert_eq!(count(&final_history, "SideEffectRecorded"), 2); assert_eq!( count(&final_history, "TimerCancelled"), @@ -999,9 +1063,12 @@ async fn qualify_scope_cleanup_replacement(grouped: bool) { ); assert_eq!( count(&final_history, "TimerScheduled"), - if grouped { 4 } else { 3 } + (if grouped { 4 } else { 3 }) + usize::from(descendants) + ); + assert_eq!( + count(&final_history, "TimerFired"), + 2 + usize::from(descendants) ); - assert_eq!(count(&final_history, "TimerFired"), 2); for kind in [ "CooperativeCancellationRequested", "WorkflowCancelled", @@ -1017,6 +1084,23 @@ async fn qualify_scope_cleanup_replacement(grouped: bool) { .await .unwrap(); assert_eq!(result["context"], accepted["payload"]["cancellation"]); + if descendants { + let child = final_history["events"] + .as_array() + .unwrap() + .iter() + .find(|row| { + row["event_type"] == "CancellationScopeRequested" + && row["payload"]["scope_id"] != scope + }) + .unwrap()["payload"]["cancellation"] + .clone(); + assert_eq!(result["descendant_context"], child); + assert_eq!(child["root_context"], result["context"]["root_context"]); + let mut lineage = child["lineage"].as_array().unwrap().clone(); + lineage.pop(); + assert_eq!(json!(lineage), result["context"]["lineage"]); + } assert!(result["remaining"] .as_f64() .is_some_and(|remaining| remaining > 0.0 && remaining < 30.0)); diff --git a/tests/fixtures/committed-scope-descendants.json b/tests/fixtures/committed-scope-descendants.json new file mode 100644 index 0000000..c96b3b5 --- /dev/null +++ b/tests/fixtures/committed-scope-descendants.json @@ -0,0 +1,3512 @@ +{ + "_provenance": { + "native_commit": "0471f1ebbb98c61fe53663086d62dc2db74db9e3", + "history_store": "isolated SQLite synthetic accepted-history projection inputs", + "effect_authority": false, + "producers": [ + "ScopedCancellationContext::forDescendant", + "CancellationScopeDescendants::members", + "CancellationScopeDelivery::activityMembers", + "ScopedTimerCancellation::members", + "ScopedWaitCancellation::members", + "ScopedChildCancellation::members" + ], + "input_fixture_sha256": "85320e25c7c19521bca4a1cff138911df1f19f76471ea033e961695e8ed761be" + }, + "timer": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scopes": { + "outer": "outer-scope", + "parent": "parent-scope", + "shield": "shield-scope", + "shield-child": "shield-child-scope", + "child": "child-scope", + "grandchild": "grandchild-scope" + }, + "layout": "timer", + "contexts": { + "parent": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "child": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "grandchild": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "opened-outer", + "sequence": 3, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "outer-scope", + "parent_scope_id": "root", + "shield_parent": false + } + }, + { + "id": "opened-parent", + "sequence": 4, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "parent-scope", + "parent_scope_id": "outer-scope", + "shield_parent": false + } + }, + { + "id": "opened-shield", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 3, + "scope_id": "shield-scope", + "parent_scope_id": "parent-scope", + "shield_parent": true + } + }, + { + "id": "opened-shield-child", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 4, + "scope_id": "shield-child-scope", + "parent_scope_id": "shield-scope", + "shield_parent": false + } + }, + { + "id": "opened-child", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 5, + "scope_id": "child-scope", + "parent_scope_id": "parent-scope", + "shield_parent": false + } + }, + { + "id": "opened-grandchild", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 6, + "scope_id": "grandchild-scope", + "parent_scope_id": "child-scope", + "shield_parent": false + } + }, + { + "id": "prior-scheduled", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "grandchild-scope", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "grandchild-scope", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "grandchild-scope", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-timer", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 8, + "cancellation_scope_id": "grandchild-scope", + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "requested-parent", + "sequence": 12, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "parent_scope_id": null, + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "requested-child", + "sequence": 13, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "request_id": "child-request", + "parent_scope_id": "parent-scope", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "requested-grandchild", + "sequence": 14, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "request_id": "grandchild-request", + "parent_scope_id": "child-scope", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "ancestor-prepared", + "namespace": "sdk-scope-fixture", + "sequence": 15, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 8, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [ + { + "scope_id": "child-scope", + "parent_scope_id": "parent-scope", + "scope_history_event_id": "opened-child", + "request_history_event_id": "requested-child", + "request_id": "child-request", + "propagation_history_event_id": "requested-child", + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [] + }, + { + "scope_id": "grandchild-scope", + "parent_scope_id": "child-scope", + "scope_history_event_id": "opened-grandchild", + "request_history_event_id": "requested-grandchild", + "request_id": "grandchild-request", + "propagation_history_event_id": "requested-grandchild", + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [ + { + "sequence": 7, + "activity_execution_id": "prior-id", + "descriptor_hash": "f60e9f5c60924a14222285670b881c3e456b61bc18e79addfd3d3308ef5ccbae" + } + ], + "timer_members": [ + { + "sequence": 8, + "timer_id": "original-timer", + "descriptor_hash": "4ceb714bc8fed33200f9b15a44469e5d70db95e41357293fa6a210eeb31d53c8" + } + ], + "wait_members": [], + "child_members": [] + } + ], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "ancestor-delivered", + "namespace": "sdk-scope-fixture", + "sequence": 16, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 8, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "preparation_history_event_id": "ancestor-prepared", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "group": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scopes": { + "outer": "outer-scope", + "parent": "parent-scope", + "shield": "shield-scope", + "shield-child": "shield-child-scope", + "child": "child-scope", + "grandchild": "grandchild-scope" + }, + "layout": "group", + "contexts": { + "parent": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "child": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "grandchild": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "opened-outer", + "sequence": 3, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "outer-scope", + "parent_scope_id": "root", + "shield_parent": false + } + }, + { + "id": "opened-parent", + "sequence": 4, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "parent-scope", + "parent_scope_id": "outer-scope", + "shield_parent": false + } + }, + { + "id": "opened-shield", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 3, + "scope_id": "shield-scope", + "parent_scope_id": "parent-scope", + "shield_parent": true + } + }, + { + "id": "opened-shield-child", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 4, + "scope_id": "shield-child-scope", + "parent_scope_id": "shield-scope", + "shield_parent": false + } + }, + { + "id": "opened-child", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 5, + "scope_id": "child-scope", + "parent_scope_id": "parent-scope", + "shield_parent": false + } + }, + { + "id": "opened-grandchild", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 6, + "scope_id": "grandchild-scope", + "parent_scope_id": "child-scope", + "shield_parent": false + } + }, + { + "id": "prior-scheduled", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "grandchild-scope", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "grandchild-scope", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "grandchild-scope", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "group-activity", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 8, + "cancellation_scope_id": "grandchild-scope", + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "grandchild-scope", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "group-timer", + "sequence": 12, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 9, + "cancellation_scope_id": "grandchild-scope", + "parallel_group_id": "parallel-calls:9:2", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 9, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 1 + }, + { + "parallel_group_id": "parallel-calls:9:2", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 9, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ], + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "group-child", + "sequence": 13, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 10, + "cancellation_scope_id": "grandchild-scope", + "parallel_group_id": "parallel-calls:9:2", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 9, + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 2 + }, + { + "parallel_group_id": "parallel-calls:9:2", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 9, + "parallel_group_size": 2, + "parallel_group_index": 1 + } + ], + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "group-condition", + "sequence": 14, + "namespace": "sdk-scope-fixture", + "event_type": "ConditionWaitOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "grandchild-scope", + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 3, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 3 + } + ], + "condition_wait_id": "original-wait", + "condition_key": "ready", + "timeout_seconds": 30, + "condition_wait_occurrence_id": "original-occurrence", + "condition_definition_fingerprint": "sha256:3b2b1ad635030d842ab378e8bf021f4613ee6d221dafb4743565467800def433" + } + }, + { + "id": "condition-timeout", + "sequence": 15, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 11, + "cancellation_scope_id": "grandchild-scope", + "timer_id": "original-timeout", + "timer_kind": "condition_timeout", + "condition_wait_id": "original-wait", + "condition_wait_occurrence_id": "original-occurrence", + "delay_seconds": 30, + "fire_at": "2026-10-04T00:00:32.123456Z" + } + }, + { + "id": "requested-parent", + "sequence": 16, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "parent_scope_id": null, + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "requested-child", + "sequence": 17, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "request_id": "child-request", + "parent_scope_id": "parent-scope", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "requested-grandchild", + "sequence": 18, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "request_id": "grandchild-request", + "parent_scope_id": "child-scope", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "ancestor-prepared", + "namespace": "sdk-scope-fixture", + "sequence": 19, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 8, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [ + { + "scope_id": "child-scope", + "parent_scope_id": "parent-scope", + "scope_history_event_id": "opened-child", + "request_history_event_id": "requested-child", + "request_id": "child-request", + "propagation_history_event_id": "requested-child", + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [] + }, + { + "scope_id": "grandchild-scope", + "parent_scope_id": "child-scope", + "scope_history_event_id": "opened-grandchild", + "request_history_event_id": "requested-grandchild", + "request_id": "grandchild-request", + "propagation_history_event_id": "requested-grandchild", + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [ + { + "sequence": 7, + "activity_execution_id": "prior-id", + "descriptor_hash": "f60e9f5c60924a14222285670b881c3e456b61bc18e79addfd3d3308ef5ccbae" + }, + { + "sequence": 8, + "activity_execution_id": "original-id", + "descriptor_hash": "1f575cec410a08e47f0bd32b2a95b40f317eacb774f17ec542f4a85dd244503c" + } + ], + "timer_members": [ + { + "sequence": 9, + "timer_id": "original-timer", + "descriptor_hash": "bb99cb1a1ff6ac1a6923dd29afbc845055f997356283f437b56dbd0f4d42383a" + }, + { + "sequence": 11, + "timer_id": "original-timeout", + "descriptor_hash": "72ff6f4ce79da9fa15a173cd173c2ffc79f791f1b3af7a44f2cdad0cdbf9bc0c" + } + ], + "wait_members": [ + { + "kind": "condition", + "sequence": 11, + "wait_id": "original-wait", + "timer_id": "original-timeout", + "descriptor_hash": "c51fd1d0e85fb66892f7b623278b7c89da4bccb287ff4397494ac596adf8da84" + } + ], + "child_members": [ + { + "sequence": 10, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "a482bfd51e2c03fe2a285880a24e614586e621aa3ad21fb67db151503514834e" + } + ] + } + ], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "ancestor-delivered", + "namespace": "sdk-scope-fixture", + "sequence": 20, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 8, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "preparation_history_event_id": "ancestor-prepared", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "partial-group": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scopes": { + "outer": "outer-scope", + "parent": "parent-scope", + "shield": "shield-scope", + "shield-child": "shield-child-scope", + "child": "child-scope", + "grandchild": "grandchild-scope" + }, + "layout": "partial-group", + "contexts": { + "parent": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "child": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "grandchild": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "opened-outer", + "sequence": 3, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "outer-scope", + "parent_scope_id": "root", + "shield_parent": false + } + }, + { + "id": "opened-parent", + "sequence": 4, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "parent-scope", + "parent_scope_id": "outer-scope", + "shield_parent": false + } + }, + { + "id": "opened-shield", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 3, + "scope_id": "shield-scope", + "parent_scope_id": "parent-scope", + "shield_parent": true + } + }, + { + "id": "opened-shield-child", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 4, + "scope_id": "shield-child-scope", + "parent_scope_id": "shield-scope", + "shield_parent": false + } + }, + { + "id": "opened-child", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 5, + "scope_id": "child-scope", + "parent_scope_id": "parent-scope", + "shield_parent": false + } + }, + { + "id": "opened-grandchild", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 6, + "scope_id": "grandchild-scope", + "parent_scope_id": "child-scope", + "shield_parent": false + } + }, + { + "id": "prior-scheduled", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "grandchild-scope", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "grandchild-scope", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "grandchild-scope", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "group-activity", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 8, + "cancellation_scope_id": "grandchild-scope", + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 0 + } + ], + "activity_execution_id": "original-id", + "activity_type": "original-activity", + "activity": { + "id": "original-id", + "cancellation_scope_id": "grandchild-scope", + "cancellation_policy": "try_cancel", + "schedule_to_close_deadline_at": "2026-10-04T00:01:02.123456Z" + } + } + }, + { + "id": "group-timer", + "sequence": 12, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 9, + "cancellation_scope_id": "grandchild-scope", + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 1 + } + ], + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "group-child", + "sequence": 13, + "namespace": "sdk-scope-fixture", + "event_type": "ChildWorkflowScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 10, + "cancellation_scope_id": "grandchild-scope", + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-calls:8:4", + "parallel_group_kind": "mixed", + "parallel_group_base_sequence": 8, + "parallel_group_size": 4, + "parallel_group_index": 2 + } + ], + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "child_workflow_type": "original-child", + "cancellation_policy": "wait_cancellation_completed", + "parent_close_policy": "request_cancel" + } + }, + { + "id": "requested-parent", + "sequence": 14, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "parent_scope_id": null, + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "requested-child", + "sequence": 15, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "request_id": "child-request", + "parent_scope_id": "parent-scope", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "requested-grandchild", + "sequence": 16, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "request_id": "grandchild-request", + "parent_scope_id": "child-scope", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "ancestor-prepared", + "namespace": "sdk-scope-fixture", + "sequence": 17, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 8, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [ + { + "scope_id": "child-scope", + "parent_scope_id": "parent-scope", + "scope_history_event_id": "opened-child", + "request_history_event_id": "requested-child", + "request_id": "child-request", + "propagation_history_event_id": "requested-child", + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [] + }, + { + "scope_id": "grandchild-scope", + "parent_scope_id": "child-scope", + "scope_history_event_id": "opened-grandchild", + "request_history_event_id": "requested-grandchild", + "request_id": "grandchild-request", + "propagation_history_event_id": "requested-grandchild", + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [ + { + "sequence": 7, + "activity_execution_id": "prior-id", + "descriptor_hash": "f60e9f5c60924a14222285670b881c3e456b61bc18e79addfd3d3308ef5ccbae" + }, + { + "sequence": 8, + "activity_execution_id": "original-id", + "descriptor_hash": "1f575cec410a08e47f0bd32b2a95b40f317eacb774f17ec542f4a85dd244503c" + } + ], + "timer_members": [ + { + "sequence": 9, + "timer_id": "original-timer", + "descriptor_hash": "6881ed38e69f1e10fd153d33d725657e4cf25cfe5df525fc1780324a29136aff" + } + ], + "wait_members": [], + "child_members": [ + { + "sequence": 10, + "child_call_id": "original-call", + "child_workflow_instance_id": "original-child-instance", + "child_workflow_run_id": "original-child-run", + "cancellation_policy": "wait_cancellation_completed", + "descriptor_hash": "30bd3db70d54bc9c67e4f961bccbca06ee1f3749d2842022610e0e2d4f11bd66" + } + ] + } + ], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "ancestor-delivered", + "namespace": "sdk-scope-fixture", + "sequence": 18, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 8, + "call_kind": "parallel", + "sequence_span": 4, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "preparation_history_event_id": "ancestor-prepared", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + }, + "shielded-operation": { + "task": { + "workflow_id": "scope-fixture-plain", + "run_id": "01m43xf68q7341x3zjapav5pse" + }, + "scopes": { + "outer": "outer-scope", + "parent": "parent-scope", + "shield": "shield-scope", + "shield-child": "shield-child-scope", + "child": "child-scope", + "grandchild": "grandchild-scope" + }, + "layout": "shielded-operation", + "contexts": { + "parent": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "child": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "grandchild": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "history": [ + { + "id": "01m43xf68yxpw4t2kv4w8gga08", + "namespace": "sdk-scope-fixture", + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m43xf691v1srvsccbmkmmc4t", + "namespace": "sdk-scope-fixture", + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "workflow_command_id": "01m43xf68v3zv390headffbdgz", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m43xf68v3zv390headffbdgz", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m43xf68q7341x3zjapav5pse", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "opened-outer", + "sequence": 3, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 1, + "scope_id": "outer-scope", + "parent_scope_id": "root", + "shield_parent": false + } + }, + { + "id": "opened-parent", + "sequence": 4, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 2, + "scope_id": "parent-scope", + "parent_scope_id": "outer-scope", + "shield_parent": false + } + }, + { + "id": "opened-shield", + "sequence": 5, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 3, + "scope_id": "shield-scope", + "parent_scope_id": "parent-scope", + "shield_parent": true + } + }, + { + "id": "opened-shield-child", + "sequence": 6, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 4, + "scope_id": "shield-child-scope", + "parent_scope_id": "shield-scope", + "shield_parent": false + } + }, + { + "id": "opened-child", + "sequence": 7, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 5, + "scope_id": "child-scope", + "parent_scope_id": "parent-scope", + "shield_parent": false + } + }, + { + "id": "opened-grandchild", + "sequence": 8, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeOpened", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "sequence": 6, + "scope_id": "grandchild-scope", + "parent_scope_id": "child-scope", + "shield_parent": false + } + }, + { + "id": "prior-scheduled", + "sequence": 9, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "grandchild-scope", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "activity": { + "id": "prior-id", + "cancellation_scope_id": "grandchild-scope", + "cancellation_policy": "try_cancel" + } + } + }, + { + "id": "prior-completed", + "sequence": 10, + "namespace": "sdk-scope-fixture", + "event_type": "ActivityCompleted", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 7, + "cancellation_scope_id": "grandchild-scope", + "activity_execution_id": "prior-id", + "activity_type": "prior-step", + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoWcHJpb3ItdmFsdWU=" + } + } + }, + { + "id": "original-timer", + "sequence": 11, + "namespace": "sdk-scope-fixture", + "event_type": "TimerScheduled", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "sequence": 8, + "cancellation_scope_id": "shield-scope", + "timer_id": "original-timer", + "delay_seconds": 3600, + "fire_at": "2026-10-04T01:00:02.123456Z" + } + }, + { + "id": "requested-parent", + "sequence": 12, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "parent_scope_id": null, + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "requested-child", + "sequence": 13, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "request_id": "child-request", + "parent_scope_id": "parent-scope", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "requested-grandchild", + "sequence": 14, + "namespace": "sdk-scope-fixture", + "event_type": "CancellationScopeRequested", + "timestamp": "2026-10-04T00:00:02.123456Z", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "request_id": "grandchild-request", + "parent_scope_id": "child-scope", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + } + }, + { + "id": "ancestor-prepared", + "namespace": "sdk-scope-fixture", + "sequence": 15, + "event_type": "CancellationScopeDeliveryPrepared", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 8, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [ + { + "scope_id": "child-scope", + "parent_scope_id": "parent-scope", + "scope_history_event_id": "opened-child", + "request_history_event_id": "requested-child", + "request_id": "child-request", + "propagation_history_event_id": "requested-child", + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [] + }, + { + "scope_id": "grandchild-scope", + "parent_scope_id": "child-scope", + "scope_history_event_id": "opened-grandchild", + "request_history_event_id": "requested-grandchild", + "request_id": "grandchild-request", + "propagation_history_event_id": "requested-grandchild", + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "child-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "child-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "grandchild-request", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "grandchild-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [ + { + "sequence": 7, + "activity_execution_id": "prior-id", + "descriptor_hash": "f60e9f5c60924a14222285670b881c3e456b61bc18e79addfd3d3308ef5ccbae" + } + ], + "timer_members": [], + "wait_members": [], + "child_members": [] + } + ], + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "ancestor-delivered", + "namespace": "sdk-scope-fixture", + "sequence": 16, + "event_type": "CancellationScopeDelivered", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "root_workflow_instance_id": "scope-fixture-plain", + "root_workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "parent_request_id": null, + "reason": "release one scope", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse" + } + ] + }, + "lineage": [ + { + "request_id": "01M43XF6A8D0VH3FG2M21RRPPZ", + "workflow_instance_id": "scope-fixture-plain", + "workflow_run_id": "01m43xf68q7341x3zjapav5pse", + "scope_id": "parent-scope", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 8, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:26.123456Z", + "preparation_history_event_id": "ancestor-prepared", + "task": { + "id": "01m43xf692g8zz7d38k45m9g0z", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:01:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "timestamp": "2026-10-04T00:00:09.123456Z" + } + ] + } +} diff --git a/tests/fixtures/replay-regressions/cancellation-scope-descendants-default-refused.json b/tests/fixtures/replay-regressions/cancellation-scope-descendants-default-refused.json new file mode 100644 index 0000000..6d406c7 --- /dev/null +++ b/tests/fixtures/replay-regressions/cancellation-scope-descendants-default-refused.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "rust-cancellation-scope-descendants-default-refused", + "protocol_version": "1.20", + "bindings": ["rust"], + "workflow": { + "type": "corpus.side-effect-version", + "input": [], + "payload_codec": "avro" + }, + "history": [ + { + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "regression-corpus-run", + "scope_id": "parent-scope", + "parent_scope_id": "root", + "sequence": 1, + "shield_parent": false + } + }, + { + "event_type": "CancellationScopeOpened", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "regression-corpus-run", + "scope_id": "child-scope", + "parent_scope_id": "parent-scope", + "sequence": 2, + "shield_parent": false + } + } + ], + "expected": {"command_sequence": []}, + "expected_failure": "cancellation_scope_execution_not_supported: Rust worker" +} From 1529a2c04ddedcd315523ca7737f180238f5a15b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 03:05:31 +0000 Subject: [PATCH 51/58] Prepare pending ancestor cancellation before descendant propagation --- src/cancellation_scope_history.rs | 9 ++- .../cancellation_scope_replay/descendants.rs | 56 +++++++++++++++++-- ...ion-scope-descendants-default-refused.json | 38 ------------- 3 files changed, 56 insertions(+), 47 deletions(-) delete mode 100644 tests/fixtures/replay-regressions/cancellation-scope-descendants-default-refused.json diff --git a/src/cancellation_scope_history.rs b/src/cancellation_scope_history.rs index bfffeea..64516fe 100644 --- a/src/cancellation_scope_history.rs +++ b/src/cancellation_scope_history.rs @@ -1186,16 +1186,15 @@ impl CommittedCancellationScopeHistory { .map(|opening| (opening.scope_id.as_str(), opening)) .collect(); let mut id = scope; - let active = self.pending_requests.get(scope); - let mut request = active; + let mut request = self.pending_requests.get(scope); while let Some(opening) = addresses.get(id).filter(|opening| !opening.shield_parent) { id = &opening.parent_scope_id; let Some(ancestor) = self.pending_requests.get(id) else { continue; }; - if active.is_none_or(|active| { - ancestor.context.root_context() != active.context.root_context() - || !active + if request.is_some_and(|selected| { + ancestor.context.root_context() != selected.context.root_context() + || !selected .context .lineage() .starts_with(ancestor.context.lineage()) diff --git a/src/tests/cancellation_scope_replay/descendants.rs b/src/tests/cancellation_scope_replay/descendants.rs index 9ad42d7..2d6a122 100644 --- a/src/tests/cancellation_scope_replay/descendants.rs +++ b/src/tests/cancellation_scope_replay/descendants.rs @@ -281,11 +281,19 @@ fn descendant_original_contexts_restore_and_leave_outer_and_root_unaffected() { #[test] fn descendant_pending_ancestor_preserves_original_identity_and_range_without_cleanup() { for layout in ["timer", "group"] { - for before in [ - "CancellationScopeDeliveryPrepared", - "CancellationScopeDelivered", + for (before, inherited) in [ + ("CancellationScopeDeliveryPrepared", false), + ("CancellationScopeDeliveryPrepared", true), + ("CancellationScopeDelivered", true), ] { - let source = prefix(&descendant_fixture(layout), before); + let mut source = prefix(&descendant_fixture(layout), before); + if !inherited { + let parent = source["scopes"]["parent"].clone(); + source["history"].as_array_mut().unwrap().retain(|row| { + row["event_type"] != "CancellationScopeRequested" + || row["payload"]["scope_id"] == parent + }); + } let seen = Arc::new(Mutex::new(BTreeMap::new())); let decision = descendant_worker(source.clone(), Arc::clone(&seen), "", "") .execute_workflow_task_decision(task(&source)) @@ -299,10 +307,50 @@ fn descendant_pending_ancestor_preserves_original_identity_and_range_without_cle intent.boundary.sequence_span, if layout == "timer" { 1 } else { 4 } ); + let mut replacement = task(&source); + replacement.lease_owner = Some("replacement".into()); + replacement.workflow_task_attempt = 17; + let repeated = descendant_worker(source, Arc::clone(&seen), "", "") + .execute_workflow_task_decision(replacement) + .unwrap() + .cancellation_scope_delivery + .unwrap(); + assert_eq!(repeated.context, intent.context); + assert_eq!(repeated.boundary, intent.boundary); + assert!(seen.lock().unwrap().is_empty()); } } } +#[test] +fn descendant_pending_ancestor_cannot_bypass_competing_intermediate_request() { + let fixtures: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/committed-scope-operation-projections.json" + )) + .unwrap(); + let mut source = prefix(&fixtures["competing"], "CancellationScopeDeliveryPrepared"); + source["history"].as_array_mut().unwrap().retain(|row| { + row["event_type"] != "CancellationScopeRequested" + || row["payload"]["scope_id"] != "desc-grandchild" + }); + let claim = task(&source); + let scopes = cancellation_scope::CancellationScopeHistory::read( + &claim.history_events, + claim.run_id.as_deref().unwrap(), + ) + .unwrap(); + let committed = crate::cancellation_scope_history::CommittedCancellationScopeHistory::read( + &claim.history_events, + claim.run_id.as_deref().unwrap(), + claim.workflow_id.as_deref().unwrap(), + ) + .unwrap(); + let error = committed + .pending_request_for_scope("desc-grandchild", &scopes) + .unwrap_err(); + assert!(error.to_string().contains("original ancestor lineage")); +} + #[test] fn descendant_cleanup_retains_ancestor_proof_narrower_authority_and_replacement_clock() { for layout in ["timer", "group"] { diff --git a/tests/fixtures/replay-regressions/cancellation-scope-descendants-default-refused.json b/tests/fixtures/replay-regressions/cancellation-scope-descendants-default-refused.json deleted file mode 100644 index 6d406c7..0000000 --- a/tests/fixtures/replay-regressions/cancellation-scope-descendants-default-refused.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", - "fixture_schema": "durable-workflow.replay-regression/v1", - "id": "rust-cancellation-scope-descendants-default-refused", - "protocol_version": "1.20", - "bindings": ["rust"], - "workflow": { - "type": "corpus.side-effect-version", - "input": [], - "payload_codec": "avro" - }, - "history": [ - { - "event_type": "CancellationScopeOpened", - "payload": { - "schema": "durable-workflow.cancellation-scope/v1", - "workflow_run_id": "regression-corpus-run", - "scope_id": "parent-scope", - "parent_scope_id": "root", - "sequence": 1, - "shield_parent": false - } - }, - { - "event_type": "CancellationScopeOpened", - "payload": { - "schema": "durable-workflow.cancellation-scope/v1", - "workflow_run_id": "regression-corpus-run", - "scope_id": "child-scope", - "parent_scope_id": "parent-scope", - "sequence": 2, - "shield_parent": false - } - } - ], - "expected": {"command_sequence": []}, - "expected_failure": "cancellation_scope_execution_not_supported: Rust worker" -} From 84a9283446744a879e03b9394c30583b3b47ab8b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 03:28:03 +0000 Subject: [PATCH 52/58] Authorize descendant cleanup with its original ancestor delivery --- src/cancellation_scope_history.rs | 7 ++++--- src/cancellation_scope_replay.rs | 4 ++-- src/tests/cancellation_scope_replay/descendants.rs | 11 ++++++----- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/cancellation_scope_history.rs b/src/cancellation_scope_history.rs index 64516fe..1278027 100644 --- a/src/cancellation_scope_history.rs +++ b/src/cancellation_scope_history.rs @@ -165,11 +165,12 @@ fn cleanup_timer(event: &HistoryEvent, prefix: &[HistoryEvent]) -> Result let (context, ceiling) = states .get(text(p, "cancellation_scope_id")?) .ok_or_else(|| invalid("cleanup timer changes its original frozen subtree membership"))?; - let expected = json!({"scope_id":context.scope_id(), "operation_scope_id":context.scope_id(), - "request_id":context.request_id(), "root_request_id":context.root_context().root_request_id(), + let ancestor = ScopedCancellationContext::from_value(&original["cancellation"])?; + let expected = json!({"scope_id":ancestor.scope_id(), "operation_scope_id":context.scope_id(), + "request_id":ancestor.request_id(), "root_request_id":ancestor.root_context().root_request_id(), "delivery_history_event_id":event_id(delivery)?, "preparation_history_event_id":original["preparation_history_event_id"], - "cleanup_deadline_at":canonical_time(context.deadline()), + "cleanup_deadline_at":canonical_time(ancestor.deadline()), "authority_deadline_at":canonical_time(*ceiling)}); let boundary = CancellationDelivery::from_payload(&json!({ "workflow_command_id":original["request_id"], "sequence":original["sequence"], diff --git a/src/cancellation_scope_replay.rs b/src/cancellation_scope_replay.rs index cd39642..83fc345 100644 --- a/src/cancellation_scope_replay.rs +++ b/src/cancellation_scope_replay.rs @@ -508,7 +508,7 @@ impl WorkflowState { let Some(replay) = &self.scope_delivery else { return Ok(None); }; - let Some((context, _)) = replay.contexts.get(scope) else { + let Some(_) = replay.contexts.get(scope) else { return Ok(None); }; let delivered = replay.consumed_delivery_for_scope(scope).ok_or_else(|| { @@ -517,7 +517,7 @@ impl WorkflowState { ) })?; Ok(Some( - json!({"scope_id":scope, "request_id":context.request_id(), + json!({"scope_id":delivered.context.scope_id(), "request_id":delivered.context.request_id(), "delivery_history_event_id":delivered.event.raw["id"]}), )) } diff --git a/src/tests/cancellation_scope_replay/descendants.rs b/src/tests/cancellation_scope_replay/descendants.rs index 2d6a122..a18cbf7 100644 --- a/src/tests/cancellation_scope_replay/descendants.rs +++ b/src/tests/cancellation_scope_replay/descendants.rs @@ -255,8 +255,9 @@ fn next_sequence(source: &Value) -> u64 { fn snapshot_for(source: &Value, target: &str) -> Value { let context = ScopedCancellationContext::from_value(&source["contexts"][target]).unwrap(); - json!({"scope_id":context.scope_id(), "operation_scope_id":context.scope_id(), - "request_id":context.request_id(), "root_request_id":context.root_context().root_request_id(), + let ancestor = ScopedCancellationContext::from_value(&source["contexts"]["parent"]).unwrap(); + json!({"scope_id":ancestor.scope_id(), "operation_scope_id":context.scope_id(), + "request_id":ancestor.request_id(), "root_request_id":ancestor.root_context().root_request_id(), "delivery_history_event_id":"ancestor-delivered", "preparation_history_event_id":"ancestor-prepared", "cleanup_deadline_at":"2026-10-04T00:00:30.123456Z", "authority_deadline_at":"2026-10-04T00:00:26.123456Z"}) } @@ -360,7 +361,7 @@ fn descendant_cleanup_retains_ancestor_proof_narrower_authority_and_replacement_ let snapshot = snapshot_for(&source, target); let sequence = next_sequence(&source); let wire = json!({"type":"start_timer", "delay_seconds":1, - "cancellation_scope_id":snapshot["scope_id"], "cancellation_cleanup":{ + "cancellation_scope_id":snapshot["operation_scope_id"], "cancellation_cleanup":{ "scope_id":snapshot["scope_id"], "request_id":snapshot["request_id"], "delivery_history_event_id":snapshot["delivery_history_event_id"]}}); assert_eq!( @@ -375,7 +376,7 @@ fn descendant_cleanup_retains_ancestor_proof_narrower_authority_and_replacement_ "TimerScheduled", json!({"sequence":sequence, "timer_id":"descendant-cleanup", "delay_seconds":1, - "cancellation_scope_id":snapshot["scope_id"], "cancellation_cleanup":snapshot, + "cancellation_scope_id":snapshot["operation_scope_id"], "cancellation_cleanup":snapshot, "fire_at":"2026-10-04T00:00:11.123456Z"}), "2026-10-04T00:00:10.123456Z", ); @@ -395,7 +396,7 @@ fn descendant_cleanup_retains_ancestor_proof_narrower_authority_and_replacement_ "TimerFired", json!({"sequence":sequence, "timer_id":"descendant-cleanup", "delay_seconds":1, - "cancellation_scope_id":snapshot["scope_id"]}), + "cancellation_scope_id":snapshot["operation_scope_id"]}), "2026-10-04T00:00:11.123456Z", ); assert_eq!( From 1596f0deb4875ac046f54fbf0e27bd7366297220 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 03:51:22 +0000 Subject: [PATCH 53/58] Preserve scoped cleanup before root cancellation delivery --- .github/workflows/ci.yml | 1 + src/cancellation_replay_clock.rs | 14 +- src/cancellation_scope.rs | 3 + src/cooperative_cancellation.rs | 62 +- src/lib.rs | 16 +- src/tests/cancellation_scope_replay.rs | 1 + .../cancellation_scope_replay/run_scopes.rs | 311 ++++++ tests/cooperative_server.rs | 221 ++++- .../fixtures/run-inherited-scope-timers.json | 892 ++++++++++++++++++ 9 files changed, 1496 insertions(+), 25 deletions(-) create mode 100644 src/tests/cancellation_scope_replay/run_scopes.rs create mode 100644 tests/fixtures/run-inherited-scope-timers.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91aeb99..0f8476f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -264,6 +264,7 @@ jobs: arguments+=(--skip server_scope_authoring_replays_nested_tree_and_deferred_membership_on_replacement) arguments+=(--skip server_scope_cleanup_preserves_original_delivery_and_budget_after_replacement) arguments+=(--skip server_scope_group_cleanup_preserves_original_delivery_and_budget_after_replacement) + arguments+=(--skip server_sigkill_scoped_cleanup_replays_before_root_with_original_30_second_deadline) fi cargo test --test cooperative_server -- "${arguments[@]}" 2>&1 | tee cooperative-results.log - name: Retain bounded scenario evidence diff --git a/src/cancellation_replay_clock.rs b/src/cancellation_replay_clock.rs index bb0506d..dd9e9ea 100644 --- a/src/cancellation_replay_clock.rs +++ b/src/cancellation_replay_clock.rs @@ -179,14 +179,12 @@ impl WorkflowState { pub(super) fn start_cancellation_clock(&mut self) { if let Some(clock) = &mut self.cancellation_clock { - if !clock.started { - clock.started = true; - clock.observe( - clock - .delivery_index - .and_then(|index| self.history_events.get(index)), - ); - } + clock.started = true; + clock.observe( + clock + .delivery_index + .and_then(|index| self.history_events.get(index)), + ); } } diff --git a/src/cancellation_scope.rs b/src/cancellation_scope.rs index 96a627f..28600ce 100644 --- a/src/cancellation_scope.rs +++ b/src/cancellation_scope.rs @@ -316,6 +316,9 @@ impl Future for ScopeOpeningCall { if !state.allow_cancellation_scope_authoring { return Poll::Ready(Err(Error::CancellationScopeExecutionUnavailable)); } + if state.cancellation_consumed { + return Poll::Ready(Err(Error::CancellationScopeExecutionUnavailable)); + } if state .cancellation_scope_opening .as_ref() diff --git a/src/cooperative_cancellation.rs b/src/cooperative_cancellation.rs index bec57a7..3170523 100644 --- a/src/cooperative_cancellation.rs +++ b/src/cooperative_cancellation.rs @@ -763,6 +763,7 @@ impl WorkflowState { if !self.cancellation_delivery_enabled || self.cancellation_shield_depth > 0 { return Ok(false); } + self.assert_root_cancellation_membership()?; if self.cancellation_delivery_intent.is_some() { self.matched_recorded_pending = true; return Ok(true); @@ -784,6 +785,17 @@ impl WorkflowState { Ok(true) } + fn assert_root_cancellation_membership(&self) -> Result<()> { + if self + .scope_delivery + .as_ref() + .is_some_and(|replay| replay.active_scope != "root") + { + return Err(Error::CancellationScopeExecutionUnavailable); + } + Ok(()) + } + pub(super) fn prepare_scalar_cancellation( &mut self, index: usize, @@ -1000,6 +1012,7 @@ impl WorkflowState { return Ok(Some(*original)); } self.replay_scope_cancellation_at(index, kind)?; + self.assert_root_cancellation_membership()?; self.validate_cancellation_call(sequence, kind, call_kind)?; self.cancellation_consumed = true; self.cancel_requested = true; @@ -1022,6 +1035,7 @@ impl WorkflowState { .. }) = self.recorded_commands.get(index) { + self.assert_root_cancellation_membership()?; self.validate_cancellation_call(*sequence, kind, *call_kind)?; self.cancellation_consumed = true; self.cancel_requested = true; @@ -1077,16 +1091,18 @@ impl WorkflowContext { .state .lock() .map_err(|_| Error::WorkflowStatePoisoned)?; - Ok(if state.cancellation_consumed { - state - .cancellation_history - .request - .as_ref() - .and_then(|request| request.context.clone()) - .map(|context| context.with_replay(Some(Arc::downgrade(&self.state)))) - } else { - None - }) + Ok( + if state.cancellation_consumed && self.cancellation_scope_id == "root" { + state + .cancellation_history + .request + .as_ref() + .and_then(|request| request.context.clone()) + .map(|context| context.with_replay(Some(Arc::downgrade(&self.state)))) + } else { + None + }, + ) } /// Shield explicit cleanup from repeated cancellation checks. @@ -1969,7 +1985,7 @@ impl Worker { .client .refresh_workflow_cancellation_history(&claim.task, observation) .await?; - for _ in 0..3 { + for _ in 0..1000 { // Count the actual refreshed snapshot. Its byte budget was not // remeasured, so do not expose the old snapshot's size as current. task.total_history_events = None; @@ -1978,6 +1994,30 @@ impl Worker { task.clone(), Some(observation), )?; + if let Some(intent) = decision.cancellation_scope_delivery.as_ref() { + if !decision.commands.is_empty() { + decision.cancellation_scope_delivery = None; + return Ok(Some(decision)); + } + let mut budget = CancellationScopeDeliveryBudget::new(); + budget.restrict(intent.context.deadline())?; + let prepared = self + .client + .prepare_cancellation_scope_on_claim( + &task, + &intent.context, + &intent.boundary, + &budget, + ) + .await?; + budget.restrict(prepared.authority_deadline())?; + let delivered = self + .client + .deliver_cancellation_scope_on_claim(&task, &prepared, &budget) + .await?; + task.history_events = delivered.history().to_vec(); + continue; + } let Some(intent) = decision.cancellation_delivery.as_ref() else { return Ok(Some(decision)); }; diff --git a/src/lib.rs b/src/lib.rs index c7f5c93..387fef7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8959,6 +8959,14 @@ impl WorkflowContext { .state .lock() .map_err(|_| Error::WorkflowStatePoisoned)?; + if self.cancellation_scope_id != "root" && state.scope_delivery.is_some() { + return Ok(state + .scope_delivery + .as_ref() + .unwrap() + .contexts + .contains_key(&self.cancellation_scope_id)); + } Ok(state.cancel_requested || state .scope_delivery @@ -8997,6 +9005,9 @@ impl WorkflowContext { } } if state.cancel_requested && state.cancellation_shield_depth == 0 { + if self.cancellation_scope_id != "root" && state.scope_delivery.is_some() { + return Ok(()); + } return Err(state.cancellation_error()); } Ok(()) @@ -9936,7 +9947,10 @@ impl WorkflowState { run_id.as_deref().unwrap_or_default(), None, )?; - if allow_cancellation_scope_authoring && cancellation_history.request.is_some() { + if allow_cancellation_scope_authoring + && cancellation_history.request.is_some() + && !allow_cancellation_scope_delivery + { return Err(Error::CancellationScopeExecutionUnavailable); } let scope_delivery = if allow_cancellation_scope_delivery { diff --git a/src/tests/cancellation_scope_replay.rs b/src/tests/cancellation_scope_replay.rs index 779c7df..9105fd0 100644 --- a/src/tests/cancellation_scope_replay.rs +++ b/src/tests/cancellation_scope_replay.rs @@ -1,6 +1,7 @@ use super::*; mod descendants; +mod run_scopes; fn group_fixture(name: &str) -> Value { let source: Value = serde_json::from_str(include_str!( diff --git a/src/tests/cancellation_scope_replay/run_scopes.rs b/src/tests/cancellation_scope_replay/run_scopes.rs new file mode 100644 index 0000000..fde630b --- /dev/null +++ b/src/tests/cancellation_scope_replay/run_scopes.rs @@ -0,0 +1,311 @@ +use super::*; +use std::collections::BTreeMap; + +fn source() -> Value { + serde_json::from_str(include_str!( + "../../../tests/fixtures/run-inherited-scope-timers.json" + )) + .unwrap() +} + +fn at_phase(source: &Value, phase: &str) -> Value { + let mut value = source.clone(); + let end = value["history_ranges"][phase].as_u64().unwrap() as usize; + value["history"].as_array_mut().unwrap().truncate(end); + value +} + +fn probe(seen: Arc>>, changed: &'static str) -> Worker { + let mut worker = Worker::new(Client::new("http://unused.invalid").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + worker.register_workflow("scope-replay", move |ctx, _| { + let seen = Arc::clone(&seen); + async move { + let root = ctx.clone(); + let scoped_seen = Arc::clone(&seen); + let captured = Arc::new(Mutex::new(None::)); + let captured_scope = Arc::clone(&captured); + ctx.cancellation_scope(false, move |outer| async move { + outer + .cancellation_scope(false, move |inner| async move { + *captured_scope.lock().unwrap() = Some(inner.clone()); + let cancelled = match inner.sleep(Duration::from_secs(300)).await { + Err(Error::CancellationScopeRequested(cancelled)) => cancelled, + other => panic!("Expected the original scoped delivery, got {other:?}"), + }; + assert!(inner.cancellation_context()?.is_none()); + assert_eq!( + inner.scoped_cancellation_context()?.unwrap().to_value(), + cancelled.context.to_value() + ); + scoped_seen + .lock() + .unwrap() + .insert("scoped".into(), cancelled.context.to_value()); + scoped_seen.lock().unwrap().insert( + "scoped_remaining".into(), + json!(cancelled.context.remaining()?.as_secs_f64()), + ); + let _shield = inner.cancellation_shield()?; + if changed == "unshielded" { + drop(_shield); + } + inner.sleep(Duration::from_secs(2)).await?; + scoped_seen.lock().unwrap().insert( + "scoped_after".into(), + json!(cancelled.context.remaining()?.as_secs_f64()), + ); + Ok(Value::Null) + }) + .await + }) + .await?; + if matches!( + changed, + "captured_scalar" | "captured_parallel" | "captured_selection" + ) { + let captured = captured.lock().unwrap().as_ref().unwrap().clone(); + match changed { + "captured_scalar" => captured.sleep(Duration::from_secs(300)).await?, + "captured_parallel" => { + captured + .parallel(vec![ParallelOperation::timer(Duration::from_secs(300))]) + .await?; + } + _ => { + captured + .select_keyed(vec![( + "timer", + ParallelOperation::timer(Duration::from_secs(300)), + )]) + .await?; + } + } + } + if changed == "new_scope_before_root" { + return root + .cancellation_scope(false, |scope| async move { + scope.sleep(Duration::from_secs(300)).await?; + Ok(Value::Null) + }) + .await; + } + let Err(Error::CooperativeCancellationRequested(cancelled)) = + root.sleep(Duration::from_secs(300)).await + else { + return Err(Error::CancellationScopeExecutionUnavailable); + }; + let context = root.cancellation_context()?.unwrap(); + seen.lock() + .unwrap() + .insert("root".into(), context.to_value()); + seen.lock().unwrap().insert( + "root_remaining".into(), + json!(context.remaining()?.as_secs_f64()), + ); + { + let _shield = root.cancellation_shield()?; + if changed == "new_scope" { + root.cancellation_scope(false, |scope| async move { + scope.sleep(Duration::from_secs(1)).await?; + Ok(Value::Null) + }) + .await?; + } + root.sleep(Duration::from_secs(2)).await?; + } + seen.lock().unwrap().insert( + "root_after".into(), + json!(context.remaining()?.as_secs_f64()), + ); + Err(Error::CooperativeCancellationRequested(cancelled)) + } + }); + worker +} + +#[test] +fn run_scope_replay_preserves_ancestor_delivery_and_descendant_cleanup_before_root() { + let source = source(); + for phase in [ + "requested", + "prepared", + "scope_delivered", + "scope_cleanup_scheduled", + "scope_cleaned", + "root_delivered", + "root_cleanup_scheduled", + "root_cleaned", + ] { + let seen = Arc::new(Mutex::new(BTreeMap::new())); + let claim = task(&at_phase(&source, phase)); + let mut pending = CancellationHistory::from_events( + &claim.history_events, + claim.run_id.as_deref().unwrap(), + None, + ) + .unwrap() + .request + .unwrap(); + pending.history_refresh_page_token = Some("MA==".into()); + let decision = probe(Arc::clone(&seen), "") + .execute_workflow_task_decision_with_cancellation(claim, Some(&pending)) + .unwrap_or_else(|error| panic!("{phase}: {error:?}")); + let observed = seen.lock().unwrap(); + match phase { + "requested" | "prepared" => { + assert!(observed.is_empty()); + let intent = decision.cancellation_scope_delivery.unwrap(); + assert_eq!(intent.context.scope_id(), source["scopes"]["parent"]); + assert_eq!(intent.boundary.sequence, 3); + assert!(decision.cancellation_delivery.is_none()); + assert!(decision.commands.is_empty()); + } + "scope_delivered" => { + assert_eq!(observed["scoped_remaining"].as_f64(), Some(16.0)); + assert_eq!( + observed["scoped"]["lineage"] + .as_array() + .unwrap() + .last() + .unwrap()["scope_id"], + source["scopes"]["child"] + ); + assert_eq!( + decision.commands[0]["cancellation_scope_id"], + source["scopes"]["child"] + ); + assert_eq!( + decision.commands[0]["cancellation_cleanup"]["scope_id"], + source["scopes"]["parent"] + ); + assert!(decision.cancellation_delivery.is_none()); + } + "scope_cleanup_scheduled" => { + assert!(decision.commands.is_empty()); + assert!(decision.cancellation_delivery.is_none()); + assert!(!observed.contains_key("root")); + } + "scope_cleaned" => { + assert_eq!(observed["scoped_after"].as_f64(), Some(14.0)); + assert!(decision.commands.is_empty()); + assert_eq!(decision.cancellation_delivery.unwrap().sequence, 5); + assert!(!observed.contains_key("root")); + } + "root_delivered" => { + assert_eq!(observed["scoped_after"].as_f64(), Some(14.0)); + assert_eq!(observed["root"], observed["scoped"]["root_context"]); + assert_eq!(observed["root_remaining"].as_f64(), Some(11.0)); + assert_eq!( + decision.commands, + vec![json!({"type":"start_timer", "delay_seconds":2})] + ); + } + "root_cleanup_scheduled" => { + assert!(decision.commands.is_empty()); + assert!(!observed.contains_key("root_after")); + } + "root_cleaned" => { + assert_eq!(observed["root_after"].as_f64(), Some(9.0)); + assert_eq!(decision.commands.len(), 1); + assert_eq!(decision.commands[0]["type"], "fail_workflow"); + assert_eq!( + decision.commands[0]["exception_type"], + "WorkflowCancellationRequested" + ); + } + _ => unreachable!(), + } + } +} + +#[test] +fn run_scope_replay_refuses_changed_shielding_and_new_scope_during_root_cleanup() { + let source = source(); + for changed in [ + "unshielded", + "captured_scalar", + "captured_parallel", + "captured_selection", + "new_scope_before_root", + "new_scope", + ] { + let seen = Arc::new(Mutex::new(BTreeMap::new())); + assert!( + probe(Arc::clone(&seen), changed) + .execute_workflow_task_decision(task(&source)) + .is_err(), + "{changed}" + ); + let observed = seen.lock().unwrap(); + assert!( + observed.contains_key("scoped"), + "{changed}: valid scoped delivery must be reached" + ); + if changed == "new_scope" { + assert!(observed.contains_key("root")); + } else { + assert!(!observed.contains_key("root")); + } + } +} + +#[test] +fn run_scope_replay_refuses_changed_native_cleanup_authority_before_factory() { + let original = source(); + for field in [ + "scope_id", + "operation_scope_id", + "request_id", + "root_request_id", + "delivery_history_event_id", + "preparation_history_event_id", + "cleanup_deadline_at", + "authority_deadline_at", + "omitted", + "null", + "extra", + ] { + let mut changed = original.clone(); + let index = changed["history_ranges"]["scope_cleanup_scheduled"] + .as_u64() + .unwrap() as usize + - 1; + let payload = changed["history"][index]["payload"] + .as_object_mut() + .unwrap(); + match field { + "omitted" => { + payload.remove("cancellation_cleanup"); + } + "null" => { + payload.insert("cancellation_cleanup".into(), Value::Null); + } + "extra" => { + payload.get_mut("cancellation_cleanup").unwrap()["extra"] = json!("borrowed"); + } + _ => { + payload.get_mut("cancellation_cleanup").unwrap()[field] = json!("changed"); + } + } + let invoked = Arc::new(AtomicUsize::new(0)); + let calls = Arc::clone(&invoked); + let mut worker = Worker::new(Client::new("http://unused.invalid").unwrap(), "queue") + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + worker.register_workflow("scope-replay", move |_, _| { + calls.fetch_add(1, Ordering::SeqCst); + async { Ok(Value::Null) } + }); + assert!( + worker + .execute_workflow_task_decision(task(&changed)) + .is_err(), + "{field}" + ); + assert_eq!(invoked.load(Ordering::SeqCst), 0, "{field}"); + } +} diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index cff5566..eab4a4b 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -20,6 +20,7 @@ const UNDO: &str = "tests.rust-cooperative-undo"; const BLOCKED: &str = "tests.rust-cooperative-blocked"; const REPLAY: &str = "tests.rust-cooperative-replay"; const PROCESS: &str = "tests.rust-cooperative-process-reclaim"; +const RUN_SCOPE: &str = "tests.rust-run-scope-cleanup"; #[derive(Debug, serde::Serialize, serde::Deserialize)] struct ActivityGrant { @@ -75,11 +76,18 @@ async fn cooperative_process_worker_child() { }; let ready = std::env::var("DURABLE_WORKFLOW_PROCESS_CHILD_READY").unwrap(); let grant = std::env::var("DURABLE_WORKFLOW_PROCESS_CHILD_GRANT").unwrap(); - let mut worker = Worker::new(client(), &queue) - .worker_id(format!("{queue}-killed")) - .cooperative_cancellation(true) - .poll_timeout(Duration::from_secs(1)); - register_process_workflow(&mut worker); + let scoped = + std::env::var("DURABLE_WORKFLOW_PROCESS_CHILD_PROFILE").as_deref() == Ok("scope-root"); + let mut worker = if scoped { + run_scope_worker(&client(), &queue, &format!("{queue}-killed")) + } else { + let mut worker = Worker::new(client(), &queue) + .worker_id(format!("{queue}-killed")) + .cooperative_cancellation(true) + .poll_timeout(Duration::from_secs(1)); + register_process_workflow(&mut worker); + worker + }; worker.register_activity(BLOCKED, move |ctx, _| { let grant = grant.clone(); async move { @@ -1107,6 +1115,209 @@ async fn qualify_scope_cleanup_replacement(grouped: bool, descendants: bool) { println!("Native scope cleanup and replacement replay: {final_history}"); } +fn run_scope_worker(client: &Client, queue: &str, owner: &str) -> Worker { + let mut worker = Worker::new(client.clone(), queue) + .worker_id(owner) + .cooperative_cancellation(true) + .poll_timeout(Duration::from_secs(1)) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true); + worker.register_workflow(RUN_SCOPE, |root, _| async move { + let scoped = root + .cancellation_scope(false, |parent| async move { + parent + .cancellation_scope(false, |child| async move { + match child.sleep(Duration::from_secs(300)).await { + Err(Error::CancellationScopeRequested(cancelled)) => { + assert!(child.cancellation_context()?.is_none()); + let original = cancelled.context.to_value(); + { + let _shield = child.cancellation_shield()?; + child.sleep(Duration::from_secs(5)).await?; + } + Ok(original) + } + Err(error) => Err(error), + Ok(()) => { + panic!("original scoped timer completed without cancellation") + } + } + }) + .await + }) + .await?; + match root.sleep(Duration::from_secs(300)).await { + Err(Error::CooperativeCancellationRequested(cancelled)) => { + assert_eq!( + scoped["root_context"], + root.cancellation_context()?.unwrap().to_value() + ); + { + let _shield = root.cancellation_shield()?; + root.sleep(Duration::from_secs(1)).await?; + } + Err(Error::CooperativeCancellationRequested(cancelled)) + } + Err(error) => Err(error), + Ok(()) => panic!("root timer completed without canonical root delivery"), + } + }); + worker +} + +#[cfg(unix)] +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with the Native scope candidate"] +async fn server_sigkill_scoped_cleanup_replays_before_root_with_original_30_second_deadline() { + use std::os::unix::process::ExitStatusExt; + + let client = client(); + let queue = queue(); + let scratch = ProcessScratch(std::env::temp_dir().join(format!("{queue}-run-scope"))); + std::fs::create_dir(&scratch.0).unwrap(); + let ready = scratch.0.join("ready"); + let mut original = WorkerProcess( + std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "cooperative_process_worker_child", "--nocapture"]) + .env("DURABLE_WORKFLOW_PROCESS_CHILD_QUEUE", &queue) + .env("DURABLE_WORKFLOW_PROCESS_CHILD_PROFILE", "scope-root") + .env("DURABLE_WORKFLOW_PROCESS_CHILD_READY", &ready) + .env( + "DURABLE_WORKFLOW_PROCESS_CHILD_GRANT", + scratch.0.join("unused-grant"), + ) + .spawn() + .unwrap(), + ); + await_child_file(&mut original, &ready).await; + let handle = client + .start_workflow(RUN_SCOPE, &queue, &queue, json!([])) + .await + .unwrap(); + let observe = |target| { + let handle = handle.clone(); + async move { + tokio::time::timeout(Duration::from_secs(15), async { + loop { + let snapshot = history(&handle).await; + if count(&snapshot, "TimerScheduled") >= target { + return snapshot; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("actual process must admit its original timer") + } + }; + observe(1).await; + let accepted = handle + .request_cancellation(CooperativeCancellationOptions { + reason: Some("bounded scope cleanup".into()), + cleanup_timeout_seconds: Some(30), + }) + .await + .unwrap(); + let cleaning = observe(2).await; + assert_eq!(count(&cleaning, "CancellationScopeDelivered"), 1); + assert_eq!(count(&cleaning, "CooperativeCancellationDelivered"), 0); + let snapshot = cleaning["events"] + .as_array() + .unwrap() + .iter() + .find(|row| { + row["event_type"] == "TimerScheduled" + && row["payload"]["cancellation_cleanup"].is_object() + }) + .unwrap()["payload"]["cancellation_cleanup"] + .clone(); + assert_eq!(snapshot.as_object().unwrap().len(), 8); + assert_ne!(snapshot["scope_id"], snapshot["operation_scope_id"]); + assert_eq!( + snapshot["root_request_id"], + accepted.cancellation_request.request_id + ); + assert_eq!( + snapshot["cleanup_deadline_at"], + accepted.cancellation_request.cleanup_deadline_at + ); + original.0.kill().unwrap(); + assert_eq!(original.0.wait().unwrap().signal(), Some(9)); + let duplicate = handle + .request_cancellation(CooperativeCancellationOptions { + reason: Some("duplicate request".into()), + cleanup_timeout_seconds: Some(90), + }) + .await + .unwrap(); + assert!(duplicate.duplicate); + assert_eq!( + duplicate.cancellation_request.request_id, + accepted.cancellation_request.request_id + ); + assert_eq!( + duplicate.cancellation_request.requested_at, + accepted.cancellation_request.requested_at + ); + assert_eq!( + duplicate.cancellation_request.cleanup_deadline_at, + accepted.cancellation_request.cleanup_deadline_at + ); + let replacement = run_scope_worker(&client, &queue, &format!("{queue}-replacement")); + replacement.register().await.unwrap(); + let deadline = + chrono::DateTime::parse_from_rfc3339(&accepted.cancellation_request.cleanup_deadline_at) + .unwrap(); + let wall = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap(); + let now = + chrono::DateTime::::from_timestamp(wall.as_secs() as i64, wall.subsec_nanos()) + .unwrap(); + let remaining = (deadline.with_timezone(&chrono::Utc) - now) + .to_std() + .unwrap(); + let final_history = tokio::time::timeout( + remaining, + tick_until(&replacement, &handle, "WorkflowCancelled"), + ) + .await + .expect("replacement must converge before the original deadline"); + for kind in [ + "CooperativeCancellationRequested", + "CancellationScopeDeliveryPrepared", + "CancellationScopeDelivered", + "CooperativeCancellationDelivered", + "WorkflowCancelled", + ] { + assert_eq!(count(&final_history, kind), 1, "{kind}"); + } + assert_eq!(count(&final_history, "CancellationScopeOpened"), 2); + assert_eq!(count(&final_history, "CancellationScopeRequested"), 2); + assert_eq!(count(&final_history, "TimerScheduled"), 3); + assert_eq!(count(&final_history, "TimerFired"), 2); + assert_eq!(count(&final_history, "WorkflowCompleted"), 0); + assert_eq!(count(&final_history, "WorkflowFailed"), 0); + let events = final_history["events"].as_array().unwrap(); + let recovered = events + .iter() + .find(|row| { + row["event_type"] == "TimerScheduled" + && row["payload"]["cancellation_cleanup"].is_object() + }) + .unwrap(); + assert_eq!(recovered["payload"]["cancellation_cleanup"], snapshot); + let terminal = events + .iter() + .find(|row| row["event_type"] == "WorkflowCancelled") + .unwrap(); + assert!( + chrono::DateTime::parse_from_rfc3339(terminal["timestamp"].as_str().unwrap()).unwrap() + < deadline + ); + println!("Root/scope SIGKILL and original budget: {final_history}"); +} + #[tokio::test] #[ignore = "requires an isolated cooperative Server protocol 1.20 candidate"] async fn server_request_before_claim_replays_canonical_typed_cancellation() { diff --git a/tests/fixtures/run-inherited-scope-timers.json b/tests/fixtures/run-inherited-scope-timers.json new file mode 100644 index 0000000..1b48585 --- /dev/null +++ b/tests/fixtures/run-inherited-scope-timers.json @@ -0,0 +1,892 @@ +{ + "native_commit": "bdb6e005154f27fb48eef231beaff507eed3b4c6", + "task": { + "workflow_id": "run-scope-timers", + "run_id": "01m47mzvj3fzggw9shywzxs59p" + }, + "scopes": { + "parent": "01M47MZVM91EK5APNK3F6MQFPR", + "child": "01M47MZVMDP91A23HCY5YXK9V2" + }, + "history_ranges": { + "requested": 8, + "prepared": 9, + "scope_delivered": 11, + "scope_cleanup_scheduled": 12, + "scope_cleaned": 13, + "root_delivered": 14, + "root_cleanup_scheduled": 15, + "root_cleaned": 16 + }, + "history": [ + { + "id": "01m47mzvjaqqsx2wt3pbzvrbqp", + "sequence": 1, + "event_type": "StartAccepted", + "namespace": "sdk-root-scopes", + "payload": { + "workflow_command_id": "01m47mzvj722a7s0djfqp6xceb", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "outcome": "started_new", + "command": { + "id": "01m47mzvj722a7s0djfqp6xceb", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m47mzvj3fzggw9shywzxs59p", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": null, + "workflow_command_id": "01m47mzvj722a7s0djfqp6xceb", + "recorded_at": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m47mzvjd0b5ee9bdh2nj3ys2", + "sequence": 2, + "event_type": "WorkflowStarted", + "namespace": "sdk-root-scopes", + "payload": { + "workflow_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "workflow_type": "test-signal-workflow", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "workflow_command_id": "01m47mzvj722a7s0djfqp6xceb", + "business_key": null, + "visibility_labels": null, + "memo": null, + "search_attributes": null, + "execution_timeout_seconds": null, + "run_timeout_seconds": null, + "execution_deadline_at": null, + "run_deadline_at": null, + "workflow_definition_fingerprint": "sha256:5617b572b0eb270f91fe65ee11da3d428b3f6b570d381a87213671a587b20475", + "declared_queries": [], + "declared_query_contracts": [], + "declared_signals": [ + "name-provided" + ], + "declared_signal_contracts": [], + "declared_updates": [], + "declared_update_contracts": [], + "declared_entry_method": "handle", + "declared_entry_mode": "canonical", + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestSignalWorkflow", + "command": { + "id": "01m47mzvj722a7s0djfqp6xceb", + "sequence": 1, + "type": "start", + "target_scope": "instance", + "resolved_run_id": "01m47mzvj3fzggw9shywzxs59p", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNwwA", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "started_new", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": null, + "workflow_command_id": "01m47mzvj722a7s0djfqp6xceb", + "recorded_at": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m47mzvmaz45g5fg429em4pdf", + "sequence": 3, + "event_type": "CancellationScopeOpened", + "namespace": "sdk-root-scopes", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "sequence": 1, + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "parent_scope_id": "root", + "shield_parent": false, + "task": { + "id": "01m47mzvje3sws2b0q7c4c6rdb", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "leased_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:05:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": "01m47mzvje3sws2b0q7c4c6rdb", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m47mzvmeearb1rhh0sje8192", + "sequence": 4, + "event_type": "CancellationScopeOpened", + "namespace": "sdk-root-scopes", + "payload": { + "schema": "durable-workflow.cancellation-scope/v1", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "sequence": 2, + "scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "parent_scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "shield_parent": false, + "task": { + "id": "01m47mzvje3sws2b0q7c4c6rdb", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "leased_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:05:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": "01m47mzvje3sws2b0q7c4c6rdb", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m47mzvmnsm9zxfas2zn0sg85", + "sequence": 5, + "event_type": "TimerScheduled", + "namespace": "sdk-root-scopes", + "payload": { + "timer_id": "01m47mzvmmr7hq3m5kzqq9hdwb", + "sequence": 3, + "delay_seconds": 300, + "fire_at": "2026-10-04T00:05:00.123456Z", + "cancellation_scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "task": { + "id": "01m47mzvje3sws2b0q7c4c6rdb", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "leased_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:05:00.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": "01m47mzvje3sws2b0q7c4c6rdb", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m47mzvp0cj76bdwk49593d34", + "sequence": 6, + "event_type": "CooperativeCancellationRequested", + "namespace": "sdk-root-scopes", + "payload": { + "workflow_command_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "command_type": "request_cancellation", + "reason": "clean up the run", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "cancellation": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_workflow_instance_id": "run-scope-timers", + "root_workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "parent_request_id": null, + "reason": "clean up the run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p" + } + ] + }, + "command": { + "id": "01M47MZVNV78SG2539WCVZGP7Z", + "sequence": 2, + "type": "request_cancellation", + "target_scope": "run", + "requested_run_id": "01m47mzvj3fzggw9shywzxs59p", + "resolved_run_id": "01m47mzvj3fzggw9shywzxs59p", + "payload_codec": "avro", + "payload": "wwHioz3/VYAiNw4GDHJlYXNvbgogY2xlYW4gdXAgdGhlIHJ1biZjbGVhbnVwX2RlYWRsaW5lX2F0CjYyMDI2LTEwLTA0VDAwOjAwOjMwLjEyMzQ1NloYY2FuY2VsbGF0aW9uDhgMc2NoZW1hClBkdXJhYmxlLXdvcmtmbG93LmNhbmNlbGxhdGlvbi1jb250ZXh0L3YxFHJlcXVlc3RfaWQKNDAxTTQ3TVpWTlY3OFNHMjUzOVdDVlpHUDdaHnJvb3RfcmVxdWVzdF9pZAo0MDFNNDdNWlZOVjc4U0cyNTM5V0NWWkdQN1oycm9vdF93b3JrZmxvd19pbnN0YW5jZV9pZAogcnVuLXNjb3BlLXRpbWVycyhyb290X3dvcmtmbG93X3J1bl9pZAo0MDFtNDdtenZqM2Z6Z2d3OXNoeXd6eHM1OXAicGFyZW50X3JlcXVlc3RfaWQADHJlYXNvbgogY2xlYW4gdXAgdGhlIHJ1bhJyZXF1ZXN0ZXIOBAh0eXBlCgZwaHAKbGFiZWwKDlBIUCBBUEkADHNvdXJjZQoGcGhwGHJlcXVlc3RlZF9hdAo2MjAyNi0xMC0wNFQwMDowMDowMC4xMjM0NTZaJmNsZWFudXBfZGVhZGxpbmVfYXQKNjIwMjYtMTAtMDRUMDA6MDA6MzAuMTIzNDU2Wg5saW5lYWdlDAIOBhRyZXF1ZXN0X2lkCjQwMU00N01aVk5WNzhTRzI1MzlXQ1ZaR1A3Wih3b3JrZmxvd19pbnN0YW5jZV9pZAogcnVuLXNjb3BlLXRpbWVycx53b3JrZmxvd19ydW5faWQKNDAxbTQ3bXp2ajNmemdndzlzaHl3enhzNTlwAAAAAA==", + "source": "php", + "caller_label": "PHP API", + "auth_status": "not_applicable", + "auth_method": "none", + "status": "accepted", + "outcome": "cancellation_requested", + "accepted_at": "2026-10-04T00:00:00.123456Z", + "applied_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": null, + "workflow_command_id": "01M47MZVNV78SG2539WCVZGP7Z", + "recorded_at": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m47mzvp92r71hw38e435wxr7", + "sequence": 7, + "event_type": "CancellationScopeRequested", + "namespace": "sdk-root-scopes", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "parent_scope_id": "root", + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_workflow_instance_id": "run-scope-timers", + "root_workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "parent_request_id": null, + "reason": "clean up the run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p" + } + ] + }, + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "workflow_task_id": null, + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m47mzvq9dr7mc2qz7s4a94z5", + "sequence": 8, + "event_type": "CancellationScopeRequested", + "namespace": "sdk-root-scopes", + "payload": { + "schema": "durable-workflow.cancellation-scope-request/v1", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "parent_scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "request_id": "01M47MZVQ283NWZQW43SM4NCZA", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_workflow_instance_id": "run-scope-timers", + "root_workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "parent_request_id": null, + "reason": "clean up the run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p" + } + ] + }, + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "01M47MZVQ283NWZQW43SM4NCZA", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + } + }, + "workflow_task_id": null, + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:00.123456Z" + }, + { + "id": "01m47mzvv4fev6k1h72hawz9ec", + "sequence": 9, + "event_type": "CancellationScopeDeliveryPrepared", + "namespace": "sdk-root-scopes", + "payload": { + "schema": "durable-workflow.cancellation-scope-preparation/v5", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_workflow_instance_id": "run-scope-timers", + "root_workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "parent_request_id": null, + "reason": "clean up the run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p" + } + ] + }, + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 3, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:25.123456Z", + "activity_members": [], + "timer_members": [], + "wait_members": [], + "child_members": [], + "descendant_members": [ + { + "scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "parent_scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "scope_history_event_id": "01m47mzvmeearb1rhh0sje8192", + "request_history_event_id": "01m47mzvq9dr7mc2qz7s4a94z5", + "request_id": "01M47MZVQ283NWZQW43SM4NCZA", + "propagation_history_event_id": "01m47mzvq9dr7mc2qz7s4a94z5", + "authority_deadline_at": "2026-10-04T00:00:25.123456Z", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_workflow_instance_id": "run-scope-timers", + "root_workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "parent_request_id": null, + "reason": "clean up the run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p" + } + ] + }, + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "01M47MZVQ283NWZQW43SM4NCZA", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "activity_members": [], + "timer_members": [ + { + "sequence": 3, + "timer_id": "01m47mzvmmr7hq3m5kzqq9hdwb", + "descriptor_hash": "17f35c9f66c474e8dc05aee8884967bca0d73e7d19b304248fc600c7e400fcd3" + } + ], + "wait_members": [], + "child_members": [] + } + ], + "task": { + "id": "01m47mzvpabc0pd223wam7wb3t", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "leased_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:00:10.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": "01m47mzvpabc0pd223wam7wb3t", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m47mzw11e8n4es101s5gn6dw", + "sequence": 10, + "event_type": "TimerCancelled", + "namespace": "sdk-root-scopes", + "payload": { + "timer_id": "01m47mzvmmr7hq3m5kzqq9hdwb", + "sequence": 3, + "delay_seconds": 300, + "fire_at": "2026-10-04T00:05:00.123456Z", + "cancelled_at": "2026-10-04T00:00:09.123456Z", + "cancellation_scope": { + "schema": "durable-workflow.scoped-timer-cancellation/v1", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "request_id": "01M47MZVQ283NWZQW43SM4NCZA", + "request_history_event_id": "01m47mzvq9dr7mc2qz7s4a94z5", + "preparation_history_event_id": "01m47mzvv4fev6k1h72hawz9ec", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_workflow_instance_id": "run-scope-timers", + "root_workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "parent_request_id": null, + "reason": "clean up the run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p" + } + ] + }, + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "01M47MZVQ283NWZQW43SM4NCZA", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "authority_deadline_at": "2026-10-04T00:00:25.123456Z" + }, + "task": { + "id": "01m47mzvpabc0pd223wam7wb3t", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "leased_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:00:19.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": "01m47mzvpabc0pd223wam7wb3t", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m47mzw4xxwj9rznrtcserxsh", + "sequence": 11, + "event_type": "CancellationScopeDelivered", + "namespace": "sdk-root-scopes", + "payload": { + "schema": "durable-workflow.cancellation-scope-delivery/v1", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "cancellation": { + "schema": "durable-workflow.scoped-cancellation-context/v1", + "root_context": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_workflow_instance_id": "run-scope-timers", + "root_workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "parent_request_id": null, + "reason": "clean up the run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p" + } + ] + }, + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "root", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + }, + { + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z" + } + ] + }, + "sequence": 3, + "call_kind": "timer", + "sequence_span": 1, + "operation_sequence": null, + "operation_sequence_span": 1, + "authority_deadline_at": "2026-10-04T00:00:25.123456Z", + "preparation_history_event_id": "01m47mzvv4fev6k1h72hawz9ec", + "task": { + "id": "01m47mzvpabc0pd223wam7wb3t", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "leased_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:00:19.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": "01m47mzvpabc0pd223wam7wb3t", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m47mzwkbp380vd6tx5dxgjf1", + "sequence": 12, + "event_type": "TimerScheduled", + "namespace": "sdk-root-scopes", + "payload": { + "timer_id": "01m47mzwf7fzstazjgsb7g7wwn", + "sequence": 4, + "delay_seconds": 2, + "fire_at": "2026-10-04T00:00:11.123456Z", + "cancellation_cleanup": { + "scope_id": "01M47MZVM91EK5APNK3F6MQFPR", + "operation_scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "request_id": "01M47MZVP57EBSZ2EFQTWJMGE0", + "root_request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "delivery_history_event_id": "01m47mzw4xxwj9rznrtcserxsh", + "preparation_history_event_id": "01m47mzvv4fev6k1h72hawz9ec", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "authority_deadline_at": "2026-10-04T00:00:25.123456Z" + }, + "cancellation_scope_id": "01M47MZVMDP91A23HCY5YXK9V2", + "task": { + "id": "01m47mzvpabc0pd223wam7wb3t", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:00.123456Z", + "leased_at": "2026-10-04T00:00:00.123456Z", + "lease_owner": "original", + "lease_expires_at": "2026-10-04T00:00:19.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:00.123456Z", + "last_dispatched_at": "2026-10-04T00:00:00.123456Z" + } + }, + "workflow_task_id": "01m47mzvpabc0pd223wam7wb3t", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:09.123456Z" + }, + { + "id": "01m47mzwnbzg488ypk71skz0nn", + "sequence": 13, + "event_type": "TimerFired", + "namespace": "sdk-root-scopes", + "payload": { + "timer_id": "01m47mzwf7fzstazjgsb7g7wwn", + "sequence": 4, + "delay_seconds": 2, + "fire_at": "2026-10-04T00:00:11.123456Z", + "fired_at": "2026-10-04T00:00:11.123456Z", + "task": { + "id": "01m47mzwkcmek99kv4vh0fs1pe", + "type": "timer", + "status": "leased", + "available_at": "2026-10-04T00:00:11.123456Z", + "leased_at": "2026-10-04T00:00:11.123456Z", + "lease_owner": "01m47mzwkcmek99kv4vh0fs1pe", + "lease_expires_at": "2026-10-04T00:05:11.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default" + } + }, + "workflow_task_id": "01m47mzwkcmek99kv4vh0fs1pe", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:11.123456Z" + }, + { + "id": "01m47mzwqhqnc2kk3k59sc0a6g", + "sequence": 14, + "event_type": "CooperativeCancellationDelivered", + "namespace": "sdk-root-scopes", + "payload": { + "workflow_command_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "sequence": 5, + "call_kind": "timer", + "cancellation": { + "schema": "durable-workflow.cancellation-context/v1", + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "root_workflow_instance_id": "run-scope-timers", + "root_workflow_run_id": "01m47mzvj3fzggw9shywzxs59p", + "parent_request_id": null, + "reason": "clean up the run", + "requester": { + "type": "php", + "label": "PHP API" + }, + "source": "php", + "requested_at": "2026-10-04T00:00:00.123456Z", + "cleanup_deadline_at": "2026-10-04T00:00:30.123456Z", + "lineage": [ + { + "request_id": "01M47MZVNV78SG2539WCVZGP7Z", + "workflow_instance_id": "run-scope-timers", + "workflow_run_id": "01m47mzvj3fzggw9shywzxs59p" + } + ] + }, + "task": { + "id": "01m47mzwndpke4wqztb0y5x2xg", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:11.123456Z", + "leased_at": "2026-10-04T00:00:19.123456Z", + "lease_owner": "replacement", + "lease_expires_at": "2026-10-04T00:00:29.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:11.123456Z", + "last_dispatched_at": "2026-10-04T00:00:11.123456Z" + } + }, + "workflow_task_id": "01m47mzwndpke4wqztb0y5x2xg", + "workflow_command_id": "01M47MZVNV78SG2539WCVZGP7Z", + "recorded_at": "2026-10-04T00:00:19.123456Z" + }, + { + "id": "01m47mzwsb92kb88fhpsqrvnjy", + "sequence": 15, + "event_type": "TimerScheduled", + "namespace": "sdk-root-scopes", + "payload": { + "timer_id": "01m47mzws9w9qz1vrxrqj0jsvn", + "sequence": 6, + "delay_seconds": 2, + "fire_at": "2026-10-04T00:00:21.123456Z", + "task": { + "id": "01m47mzwndpke4wqztb0y5x2xg", + "type": "workflow", + "status": "leased", + "available_at": "2026-10-04T00:00:11.123456Z", + "leased_at": "2026-10-04T00:00:19.123456Z", + "lease_owner": "replacement", + "lease_expires_at": "2026-10-04T00:00:29.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default", + "last_dispatch_attempt_at": "2026-10-04T00:00:11.123456Z", + "last_dispatched_at": "2026-10-04T00:00:11.123456Z" + } + }, + "workflow_task_id": "01m47mzwndpke4wqztb0y5x2xg", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:19.123456Z" + }, + { + "id": "01m47mzwvgn43gp86tbtjfs51p", + "sequence": 16, + "event_type": "TimerFired", + "namespace": "sdk-root-scopes", + "payload": { + "timer_id": "01m47mzws9w9qz1vrxrqj0jsvn", + "sequence": 6, + "delay_seconds": 2, + "fire_at": "2026-10-04T00:00:21.123456Z", + "fired_at": "2026-10-04T00:00:21.123456Z", + "task": { + "id": "01m47mzwsb92kb88fhpsqrvnk0", + "type": "timer", + "status": "leased", + "available_at": "2026-10-04T00:00:21.123456Z", + "leased_at": "2026-10-04T00:00:21.123456Z", + "lease_owner": "01m47mzwsb92kb88fhpsqrvnk0", + "lease_expires_at": "2026-10-04T00:05:21.123456Z", + "attempt_count": 1, + "repair_count": 0, + "connection": "testing", + "queue": "default" + } + }, + "workflow_task_id": "01m47mzwsb92kb88fhpsqrvnk0", + "workflow_command_id": null, + "recorded_at": "2026-10-04T00:00:21.123456Z" + } + ] +} From afe56b280f42551993575de9ba444f75e8e2353e Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 03:57:33 +0000 Subject: [PATCH 54/58] Qualify actual scoped remote callback stop and publication fencing --- .github/workflows/ci.yml | 3 + tests/cooperative_server.rs | 254 ++++++++++++++++++++++++++++++++++++ 2 files changed, 257 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f8476f..80582b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -265,6 +265,9 @@ jobs: arguments+=(--skip server_scope_cleanup_preserves_original_delivery_and_budget_after_replacement) arguments+=(--skip server_scope_group_cleanup_preserves_original_delivery_and_budget_after_replacement) arguments+=(--skip server_sigkill_scoped_cleanup_replays_before_root_with_original_30_second_deadline) + arguments+=(--skip server_scope_try_stops_actual_remote_callback_without_application_heartbeats) + arguments+=(--skip server_scope_wait_stops_actual_remote_callback_before_cleanup) + arguments+=(--skip server_scope_group_wait_stops_actual_remote_callback_before_cleanup) fi cargo test --test cooperative_server -- "${arguments[@]}" 2>&1 | tee cooperative-results.log - name: Retain bounded scenario evidence diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index eab4a4b..65537af 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -21,6 +21,7 @@ const BLOCKED: &str = "tests.rust-cooperative-blocked"; const REPLAY: &str = "tests.rust-cooperative-replay"; const PROCESS: &str = "tests.rust-cooperative-process-reclaim"; const RUN_SCOPE: &str = "tests.rust-run-scope-cleanup"; +const REMOTE_SCOPE: &str = "tests.rust-scope-remote"; #[derive(Debug, serde::Serialize, serde::Deserialize)] struct ActivityGrant { @@ -1165,6 +1166,259 @@ fn run_scope_worker(client: &Client, queue: &str, owner: &str) -> Worker { worker } +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with the Native scope candidate"] +async fn server_scope_try_stops_actual_remote_callback_without_application_heartbeats() { + qualify_scoped_remote_stop(false, CancellationPolicy::TryCancel).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with the Native scope candidate"] +async fn server_scope_wait_stops_actual_remote_callback_before_cleanup() { + qualify_scoped_remote_stop(false, CancellationPolicy::WaitCancellationCompleted).await; +} + +#[tokio::test] +#[ignore = "requires an isolated cooperative Server with the Native scope candidate"] +async fn server_scope_group_wait_stops_actual_remote_callback_before_cleanup() { + qualify_scoped_remote_stop(true, CancellationPolicy::WaitCancellationCompleted).await; +} + +async fn qualify_scoped_remote_stop(grouped: bool, policy: CancellationPolicy) { + let client = client(); + let queue = format!( + "rust-cooperative-scope-boundary-{}", + durable_workflow::Uuid::new_v4().simple() + ); + let dropped = Arc::new(AtomicUsize::new(0)); + let cleanup_dropped = Arc::clone(&dropped); + let (entered_tx, mut entered_rx) = tokio::sync::mpsc::unbounded_channel::(); + let (heartbeat_tx, mut heartbeat_rx) = tokio::sync::mpsc::unbounded_channel(); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let mut worker = Worker::new(client.clone(), &queue) + .worker_id(format!("{queue}-managed")) + .cooperative_cancellation(true) + .candidate_cancellation_scope_authoring(true) + .candidate_cancellation_scope_delivery(true) + .max_concurrent_workflow_tasks(1) + .max_concurrent_activity_tasks(1) + .poll_timeout(Duration::from_secs(1)) + .on_worker_heartbeat(move |observation| { + if observation.acknowledgement["acknowledged"] == true { + let _ = heartbeat_tx.send(()); + } + }); + worker.register_workflow(REMOTE_SCOPE, move |root, _| { + let dropped = Arc::clone(&cleanup_dropped); + async move { + let result = root + .cancellation_scope(false, move |scope| async move { + let options = ActivityOptions::new().cancellation_policy(policy); + let pending = if grouped { + scope + .parallel(vec![ + ParallelOperation::activity_with_options( + BLOCKED, + options, + json!([]), + ), + ParallelOperation::group(vec![ParallelOperation::timer( + Duration::from_secs(300), + )]), + ]) + .await + .map(|_| ()) + } else { + scope + .activity_with_options(BLOCKED, options, json!([])) + .await + .map(|_| ()) + }; + let cancelled = match pending { + Err(Error::CancellationScopeRequested(cancelled)) => cancelled, + Err(error) => return Err(error), + Ok(()) => panic!("blocked remote callback completed without cancellation"), + }; + if policy == CancellationPolicy::WaitCancellationCompleted { + assert_eq!( + dropped.load(Ordering::SeqCst), + 1, + "Wait cleanup preceded callback drop" + ); + } + let _shield = scope.cancellation_shield()?; + scope.sleep(Duration::from_secs(1)).await?; + Ok(cancelled.context.to_value()) + }) + .await?; + assert!(!root.is_cancellation_requested()?); + root.sleep(Duration::from_secs(1)).await?; + Ok(result) + } + }); + let callback_dropped = Arc::clone(&dropped); + worker.register_activity(BLOCKED, move |ctx, _| { + let entered_tx = entered_tx.clone(); + let dropped = Arc::clone(&callback_dropped); + async move { + let _drop = CallbackDrop(dropped); + entered_tx.send(ctx).unwrap(); + std::future::pending::>().await + } + }); + let run = tokio::spawn(async move { + worker + .run_until(async { + let _ = shutdown_rx.await; + }) + .await + }); + let _abort_on_failure = AbortWorkerOnDrop(run.abort_handle()); + tokio::time::timeout(Duration::from_secs(10), heartbeat_rx.recv()) + .await + .unwrap() + .expect("actual accepted worker heartbeat"); + let handle = client + .start_workflow(REMOTE_SCOPE, &queue, &queue, json!([])) + .await + .unwrap(); + let original = tokio::time::timeout(Duration::from_secs(10), entered_rx.recv()) + .await + .unwrap() + .expect("actual scoped activity callback entered"); + let initial = history(&handle).await; + let scope = initial["events"] + .as_array() + .unwrap() + .iter() + .find(|row| row["event_type"] == "CancellationScopeOpened") + .unwrap()["payload"]["scope_id"] + .as_str() + .unwrap(); + let accepted = request_native_scope_fixture(&handle, scope); + assert_eq!(accepted, request_native_scope_fixture(&handle, scope)); + let context = + ScopedCancellationContext::from_value(&accepted["payload"]["cancellation"]).unwrap(); + let result = handle + .result_selected_run(WorkflowResultOptions { + timeout: Duration::from_secs(20), + poll_interval: Duration::from_millis(50), + }) + .await + .unwrap(); + assert_eq!(result, context.to_value()); + assert_eq!(dropped.load(Ordering::SeqCst), 1); + let status = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let status = client + .activity_task_status( + &original.task_id, + &original.activity_attempt_id, + &original.lease_owner, + ) + .await + .unwrap(); + if status["cancellation_acknowledgement"]["callback_state"] == "stopped" { + return status; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("actual callback drop must leave its original stop receipt"); + let receipt = &status["cancellation_acknowledgement"]; + assert_eq!(receipt["callback_state"], "stopped"); + assert_eq!(receipt["request_id"], context.request_id()); + assert_eq!( + receipt["root_request_id"], + context.root_context().request_id() + ); + assert_eq!( + receipt["cleanup_deadline_at"], + accepted["payload"]["cancellation"]["root_context"]["cleanup_deadline_at"] + ); + assert_eq!(receipt["received_after_deadline"], false); + assert_eq!(status["can_continue"], false); + let duplicate = client + .acknowledge_activity_cancellation( + &original.task_id, + &original.activity_attempt_id, + &original.lease_owner, + context.request_id(), + ) + .await + .unwrap(); + assert_eq!(duplicate["duplicate"], true); + assert_eq!(duplicate["history_event_id"], receipt["history_event_id"]); + let snapshot = history(&handle).await; + for kind in [ + "CancellationScopeRequested", + "CancellationScopeDeliveryPrepared", + "CancellationScopeDelivered", + "ActivityCancelled", + "ActivityCancellationAcknowledged", + "WorkflowCompleted", + ] { + assert_eq!(count(&snapshot, kind), 1, "{kind}"); + } + for kind in [ + "CooperativeCancellationRequested", + "CooperativeCancellationDelivered", + "WorkflowCancelled", + "WorkflowFailed", + "ActivityHeartbeatRecorded", + "ActivityCompleted", + ] { + assert_eq!(count(&snapshot, kind), 0, "{kind}"); + } + let events = snapshot["events"].as_array().unwrap(); + assert_eq!(count(&snapshot, "TimerCancelled"), usize::from(grouped)); + assert_eq!(count(&snapshot, "TimerScheduled"), 2 + usize::from(grouped)); + assert_eq!(count(&snapshot, "TimerFired"), 2); + let terminal = events + .iter() + .find(|row| row["event_type"] == "WorkflowCompleted") + .unwrap(); + assert!( + chrono::DateTime::parse_from_rfc3339(terminal["timestamp"].as_str().unwrap()) + .unwrap() + .with_timezone(&chrono::Utc) + < context.deadline() + ); + if policy == CancellationPolicy::WaitCancellationCompleted { + let stopped = events + .iter() + .position(|row| row["event_type"] == "ActivityCancellationAcknowledged") + .unwrap(); + let delivered = events + .iter() + .position(|row| row["event_type"] == "CancellationScopeDelivered") + .unwrap(); + assert!(stopped < delivered); + } + assert!(matches!( + original.heartbeat(json!({"late":true})).await, + Err(Error::ActivityExecutionAbandoned(_)) + )); + assert!(matches!(client.complete_activity_task(&original.task_id, + &original.activity_attempt_id, &original.lease_owner, json!("late"), "avro").await, + Err(Error::ActivityTaskRejected(rejection)) if rejection.status == 409)); + assert!(matches!(client.fail_activity_task(&original.task_id, + &original.activity_attempt_id, &original.lease_owner, "late failure", true).await, + Err(Error::ActivityTaskRejected(rejection)) if rejection.status == 409)); + assert_eq!(snapshot, history(&handle).await); + println!( + "Actual scoped callback stop and unchanged parent: {}", + json!({"status":status,"history":snapshot}) + ); + shutdown_tx.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(10), run) + .await + .unwrap() + .unwrap() + .unwrap(); +} + #[cfg(unix)] #[tokio::test] #[ignore = "requires an isolated cooperative Server with the Native scope candidate"] From 9aeb22198c8a1b7c10f42a31ea8d43ecc6c2c801 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 04:22:14 +0000 Subject: [PATCH 55/58] Wait for scoped callback stop on the original preparation budget --- src/cancellation_scope_history.rs | 26 ++++- src/tests/cancellation_scope_history.rs | 140 ++++++++++++++++++++++++ tests/cooperative_server.rs | 8 +- 3 files changed, 168 insertions(+), 6 deletions(-) diff --git a/src/cancellation_scope_history.rs b/src/cancellation_scope_history.rs index 1278027..f34870e 100644 --- a/src/cancellation_scope_history.rs +++ b/src/cancellation_scope_history.rs @@ -1557,11 +1557,25 @@ impl Client { let boundary_path = format!("{path}/cancellation-scopes/{}", if delivering { "deliver" } else { "prepare" }); let request = || self.request_json::(reqwest::Method::POST, &boundary_path, RequestProtocol::Worker("1.20"), Some(&body)); - let receipt = match request().await { + loop { + budget.remaining()?; + let mut receipt = match request().await { Err(error) if worker_operation_is_retryable(&error) => request().await?, result => result?, }; - let mut token = Some(CancellationScopeDeliveryReceipt::acknowledge(&receipt, &expected, delivering)?.to_owned()); + let pending = delivering && receipt["reason"].as_str() + == Some("cancellation_scope_activity_stop_not_acknowledged"); + if pending { + if receipt.get("history_event_id").is_some() { + return Err(invalid("pending scope stop cannot claim a committed delivery")); + } + // A pending stop proves only the retained preparation. Never expose + // it to workflow cleanup as a delivery or renew the original budget. + receipt["reason"] = Value::Null; + receipt["history_event_id"] = json!(original.unwrap().preparation_history_event_id()); + } + let committed = delivering && !pending; + let mut token = Some(CancellationScopeDeliveryReceipt::acknowledge(&receipt, &expected, committed)?.to_owned()); let mut seen = BTreeSet::new(); let mut history = Vec::new(); while let Some(current) = token.take() { @@ -1586,10 +1600,14 @@ impl Client { }; for event in batch { history.push(serde_json::from_value(event.clone()).map_err(|_| invalid("scope history event is malformed"))?); } } - let proved = CancellationScopeDeliveryReceipt::from_history(&receipt, history, &expected, delivering)?; + let proved = CancellationScopeDeliveryReceipt::from_history(&receipt, history, &expected, committed)?; if let Some(original) = original { proved.assert_original_preparation(original)?; } budget.remaining()?; - Ok(proved) + if !pending { return Ok(proved); } + // Yield so the activity supervisor can stop and acknowledge the + // original callback while this workflow retains its hosting claim. + tokio::time::sleep(Duration::from_millis(100)).await; + } }).await.map_err(|_| Error::Timeout)? } } diff --git a/src/tests/cancellation_scope_history.rs b/src/tests/cancellation_scope_history.rs index c5ed1bd..6157c29 100644 --- a/src/tests/cancellation_scope_history.rs +++ b/src/tests/cancellation_scope_history.rs @@ -88,6 +88,24 @@ fn scope_response(path: &str, body: &str, number: usize) -> Option<(&'static str } else { "prepare-start" }); + if case.starts_with("pending-") + && path.ends_with("/deliver") + && (number == 1 || case == "pending-budget") + { + result["delivered"] = json!(false); + result["reason"] = json!("cancellation_scope_activity_stop_not_acknowledged"); + result.as_object_mut().unwrap().remove("history_event_id"); + result["history_refresh_page_token"] = json!("prepare-start"); + match case { + "pending-owner" => result["lease_owner"] = json!("replacement"), + "pending-preparation" => result["preparation_history_event_id"] = json!("borrowed"), + "pending-deadline" => { + result["authority_deadline_at"] = json!("2026-10-04T00:00:31.123456Z") + } + "pending-delivery" => result["history_event_id"] = delivered["id"].clone(), + _ => {} + } + } match case { "ack-owner" => result["lease_owner"] = json!("replacement"), "ack-attempt" => result["workflow_task_attempt"] = json!(true), @@ -177,6 +195,128 @@ fn scope_client(server: &MockWorkerServer, case: &str) -> Client { .unwrap() } +#[tokio::test] +async fn scope_history_waits_for_callback_stop_without_changing_claim_or_preparation() { + let server = scope_server(); + let client = scope_client(&server, "pending-stop"); + let value = scalar_fixture(); + let task = claim(&value); + let (context, boundary) = boundary(&value); + let budget = CancellationScopeDeliveryBudget::new(); + let prepared = client + .prepare_cancellation_scope_on_claim(&task, &context, &boundary, &budget) + .await + .unwrap(); + let delivered = client + .deliver_cancellation_scope_on_claim(&task, &prepared, &budget) + .await + .unwrap(); + assert_eq!( + delivered.preparation_history_event_id(), + prepared.preparation_history_event_id() + ); + assert_eq!(delivered.context(), prepared.context()); + assert_eq!(delivered.boundary(), prepared.boundary()); + assert!(delivered.delivery_history_event_id().is_some()); + let requests = server.requests.lock().unwrap(); + let mutations: Vec<_> = requests + .iter() + .filter(|r| r.path.ends_with("/deliver")) + .collect(); + assert_eq!(mutations.len(), 2); + assert_eq!(mutations[0].body, mutations[1].body); + assert_eq!( + requests + .iter() + .filter(|r| r.path.ends_with("/history")) + .count(), + 6 + ); +} + +#[tokio::test] +async fn scope_history_pending_stop_cannot_change_authority_or_invent_delivery() { + for case in [ + "pending-owner", + "pending-preparation", + "pending-deadline", + "pending-delivery", + ] { + let server = scope_server(); + let client = scope_client(&server, case); + let value = scalar_fixture(); + let task = claim(&value); + let (context, boundary) = boundary(&value); + let budget = CancellationScopeDeliveryBudget::new(); + let prepared = client + .prepare_cancellation_scope_on_claim(&task, &context, &boundary, &budget) + .await + .unwrap(); + assert!( + client + .deliver_cancellation_scope_on_claim(&task, &prepared, &budget) + .await + .is_err(), + "{case}" + ); + assert_eq!( + server + .requests + .lock() + .unwrap() + .iter() + .filter(|r| r.path.ends_with("/deliver")) + .count(), + 1, + "{case}" + ); + } +} + +#[tokio::test] +async fn scope_history_pending_stop_keeps_original_budget() { + let server = scope_server(); + let client = scope_client(&server, "pending-budget"); + let value = scalar_fixture(); + let task = claim(&value); + let (context, boundary) = boundary(&value); + let mut budget = CancellationScopeDeliveryBudget::new(); + let prepared = client + .prepare_cancellation_scope_on_claim(&task, &context, &boundary, &budget) + .await + .unwrap(); + budget + .restrict( + DateTime::::from_timestamp_millis( + i64::try_from( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap() + + 250, + ) + .unwrap(), + ) + .unwrap(); + let started = Instant::now(); + assert!(matches!( + client + .deliver_cancellation_scope_on_claim(&task, &prepared, &budget) + .await, + Err(Error::Timeout) + )); + assert!(started.elapsed() < Duration::from_secs(1)); + let requests = server.requests.lock().unwrap(); + let mutations: Vec<_> = requests + .iter() + .filter(|r| r.path.ends_with("/deliver")) + .collect(); + assert!(mutations.len() >= 2); + assert!(mutations.iter().all(|r| r.body == mutations[0].body)); +} + #[tokio::test] async fn cancellation_scope_replay_worker_coordinates_original_claim_and_lost_ack() { for case in ["worker", "worker-lost-ack"] { diff --git a/tests/cooperative_server.rs b/tests/cooperative_server.rs index 65537af..6d450e9 100644 --- a/tests/cooperative_server.rs +++ b/tests/cooperative_server.rs @@ -1267,11 +1267,15 @@ async fn qualify_scoped_remote_stop(grouped: bool, policy: CancellationPolicy) { } }); let run = tokio::spawn(async move { - worker + let result = worker .run_until(async { let _ = shutdown_rx.await; }) - .await + .await; + if let Err(error) = &result { + eprintln!("Scoped remote worker stopped before convergence: {error:?}"); + } + result }); let _abort_on_failure = AbortWorkerOnDrop(run.abort_handle()); tokio::time::timeout(Duration::from_secs(10), heartbeat_rx.recv()) From 63f2fcaab7e542853ceed5c4967df52059391e96 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 04:52:43 +0000 Subject: [PATCH 56/58] Prepare approved Rust 3 cancellation release candidate --- Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4d56eca..1a62985 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "durable-workflow" -version = "2.1.5" +version = "3.0.0-rc.1" edition = "2021" description = "Rust client and worker SDK for Durable Workflow Cloud and self-hosted Server" license = "MIT" @@ -14,10 +14,10 @@ categories = ["api-bindings", "asynchronous"] include = ["/src/**", "/schema/**", "/examples/**", "/Cargo.toml", "/README.md", "/CHANGELOG.md", "/LICENSE"] [package.metadata.durable-workflow] -product-train = "2.1.5" +product-train = "3.0.0-rc.1" compatibility-authority = "protocol-manifests" supported-server-versions = "2.0.0" -qualified-server-version = "2.4.0" +qualified-server-version = "2.5.0-rc.1" worker-protocol-version = "1.19" server-worker-protocol-versions = ">=1.19,<2.0" portable-worker-affinity-minimum-worker-protocol-version = "1.18" From 2c99a3e6a85d10c838518eccd53f33b1244799a5 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 04:57:04 +0000 Subject: [PATCH 57/58] Describe cancellation release candidate and Rust 3 migration --- CHANGELOG.md | 17 +++++++++++++++++ README.md | 8 ++++++++ 2 files changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a09945..f0078d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## 3.0.0-rc.1 + +- Add cooperative whole-run cancellation with immutable request metadata, + lineage, deterministic cleanup time helpers and the original bounded deadline. +- Add explicit Activity and Child TryCancel, WaitCancellationCompleted and + Abandon policies, and cooperative RequestCancellation on parent closure. +- Supervise opted-in async Activity callbacks independently of application + heartbeats, acknowledge their stop and reject stale publication. Resume + shielded durable cleanup from the original delivery boundary after worker loss. +- Keep cooperation opt-in and ordinary worker protocol 1.19. Independently + cancellable scopes remain a disabled source preview. Portable local activities, + worker sessions and sticky execution remain unsupported by this Rust profile. +- This major release adds public cancellation fields and enum variants. Follow + the [v3 migration guide](https://github.com/durable-workflow/sdk-rust/blob/main/docs/migrating-to-v3.md) + when updating struct literals and exhaustive matches. The wire protocol remains + compatible with older same-major workers on a supporting Server. + ## 2.1.5 - Allow the ordinary Worker to append and close workflow streams when Server diff --git a/README.md b/README.md index 49820ce..58c94fe 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,14 @@ payload fetches never follow redirects. ## Compatibility +The cancellation release candidate adds cooperative requests and bounded, +replayable cleanup. Enable `Worker::cooperative_cancellation(true)` against a +Server that advertises protocol 1.20 and the required capabilities. This profile +supervises async Activity futures independently of application heartbeats. +Independently cancellable scopes remain disabled. See the +[cancellation guide](https://github.com/durable-workflow/sdk-rust/blob/main/docs/cooperative-cancellation-design.md) +and [v3 migration guide](https://github.com/durable-workflow/sdk-rust/blob/main/docs/migrating-to-v3.md). + The crate publishes its supported Server and worker-protocol ranges in `[package.metadata.durable-workflow]` in [`Cargo.toml`](https://github.com/durable-workflow/sdk-rust/blob/main/Cargo.toml). Runtime capability manifests, not matching package version strings, determine protocol From eb4e4f4980521c50e28f9c0db789912b1a088e8a Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 6 Oct 2026 05:04:42 +0000 Subject: [PATCH 58/58] Validate release candidates against release-owned package identities --- scripts/ci/publish-rust-sdk.sh | 2 +- scripts/ci/test-publish-rust-sdk.py | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/scripts/ci/publish-rust-sdk.sh b/scripts/ci/publish-rust-sdk.sh index b8550d7..61e1a0a 100755 --- a/scripts/ci/publish-rust-sdk.sh +++ b/scripts/ci/publish-rust-sdk.sh @@ -116,7 +116,7 @@ if [[ "$package_version" != "$product_train" ]]; then printf 'Rust SDK package version must match its product train metadata: %s, %s\n' "$package_version" "$product_train" >&2 exit 1 fi -if [[ "$compatibility_authority" != "protocol-manifests" || "$server_compatibility" != "2.0.0" || "$qualified_server_version" != "2.4.0" || "$server_worker_protocols" != ">=1.19,<2.0" ]]; then +if [[ "$compatibility_authority" != "protocol-manifests" || "$server_compatibility" != "2.0.0" || ! "$qualified_server_version" =~ ^2\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[1-9][0-9]*)?$ || "$server_worker_protocols" != ">=1.19,<2.0" ]]; then printf 'unexpected Rust SDK supported Server contract: %s, %s, %s, %s\n' "$compatibility_authority" "$server_compatibility" "$qualified_server_version" "$server_worker_protocols" >&2 exit 1 fi diff --git a/scripts/ci/test-publish-rust-sdk.py b/scripts/ci/test-publish-rust-sdk.py index 8f0f832..25d864a 100644 --- a/scripts/ci/test-publish-rust-sdk.py +++ b/scripts/ci/test-publish-rust-sdk.py @@ -18,10 +18,11 @@ PUBLISH = ROOT / "scripts" / "ci" / "publish-rust-sdk.sh" RELEASE_WORKFLOW = ROOT / ".github" / "workflows" / "release.yml" RELEASE_TOOLING_INSTALLER = ROOT / "scripts" / "ci" / "install-release-tooling.sh" -PACKAGE_VERSION = "2.1.5" +SOURCE_PACKAGE = tomllib.loads(MANIFEST.read_text(encoding="utf-8"))["package"] +PACKAGE_VERSION = SOURCE_PACKAGE["version"] PRODUCT_TRAIN = PACKAGE_VERSION SERVER_VERSIONS = "2.0.0" -QUALIFIED_SERVER_VERSION = "2.4.0" +QUALIFIED_SERVER_VERSION = SOURCE_PACKAGE["metadata"]["durable-workflow"]["qualified-server-version"] WORKER_PROTOCOL_VERSION = "1.19" SERVER_WORKER_PROTOCOLS = ">=1.19,<2.0" RELEASE_COMMIT = "0123456789abcdef0123456789abcdef01234567" @@ -429,6 +430,17 @@ def test_release_path_rejects_missing_condition_capability(self) -> None: result = self._publish(manifest) self.assertNotEqual(0, result.returncode) + def test_release_path_rejects_an_invalid_qualified_server_identity(self) -> None: + for version in ("3.0.0", "2.5", "2.5.0-rc.0", "2.5.0-preview.1"): + with self.subTest(version=version): + manifest = self._manifest_with( + f'qualified-server-version = "{QUALIFIED_SERVER_VERSION}"', + f'qualified-server-version = "{version}"', + ) + result = self._publish(manifest) + self.assertNotEqual(0, result.returncode) + self.assertIn("supported Server contract", result.stderr) + def test_release_path_rejects_package_without_current_release_notes(self) -> None: changelog = self.temp / "CHANGELOG.md" changelog.write_text(