From 82fc9f62efbd63973d9b7cc958b129d5569a53c2 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 17:43:59 +0000 Subject: [PATCH 01/30] Refuse malformed input envelopes like published PHP Published PHP 2.5.14 refuses a malformed input envelope on start, signal, update and query as Laravel validation of input.codec or input.blob, with the Avro serializer's diagnosis. Native used its own reasons. In the harness, only 4 of 60 malformed-input responses matched. Native now applies PHP's transport, codec and envelope checks on all four routes. Signal and update refuse an undecodable or scalar blob with PHP's messages, and take a non-empty map's values as the arguments. Avro decoding uses PHP's strict base64 rules, so whitespace is skipped and padding may be omitted. Update refusals carry the control-plane envelope, and command validation failures include workflow_id. Start also refuses an undecodable blob with the signal message, where PHP 2.5.14 answers 500 (server#601). Co-Authored-By: Claude Opus 5.5 --- .../2026-10-21-input-envelopes.md | 65 ++++++++++ docs/rust-server-port.md | 14 +-- rust/src/codec.rs | 115 +++++++++++++++++- rust/src/runtime/control_plane.rs | 24 +++- rust/src/runtime/execution.rs | 31 ++++- rust/src/runtime/http.rs | 8 +- rust/src/runtime/input_envelope.rs | 105 ++++++++++++++++ rust/src/runtime/memo_json.rs | 8 +- rust/src/runtime/mod.rs | 1 + rust/src/runtime/start_request.rs | 25 +--- rust/src/runtime/store.rs | 23 +++- rust/src/runtime/updates.rs | 3 +- rust/tests/execution.rs | 5 +- rust/tests/execution/input_envelopes.rs | 55 +++++++++ rust/tests/execution/query_arguments.rs | 5 +- 15 files changed, 436 insertions(+), 51 deletions(-) create mode 100644 docs/rust-server-port-checkpoints/2026-10-21-input-envelopes.md create mode 100644 rust/src/runtime/input_envelope.rs create mode 100644 rust/tests/execution/input_envelopes.rs diff --git a/docs/rust-server-port-checkpoints/2026-10-21-input-envelopes.md b/docs/rust-server-port-checkpoints/2026-10-21-input-envelopes.md new file mode 100644 index 000000000..3dcf6c292 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-21-input-envelopes.md @@ -0,0 +1,65 @@ +# October 21, input envelope refusals + +- **Found by.** While porting plain query input, the Waterline harness sent the + same 15 malformed `input` values to start, signal, update and query on + published PHP 2.5.14 and on native. In 4 of the 60 cases the responses + matched. +- **PHP's rules.** These come from `RuntimeExternalPayloadTransport`, + `AvroPayloadEnvelopeResolver`, `PayloadEnvelopeResolver` and the + workflow package's Avro serializer: + - **Transport.** Before validation, the transport refuses any input object + with an `external_storage` key: `422 external_payload_unsupported`. + - **Codec.** Any input object with a `codec` key must name `avro`, or have + an empty codec. Otherwise PHP refuses it on `input.codec` with + `unsupported_payload_codec: workflow payload codec "X" is not supported …`. + For a codec that is not a string, X is the PHP type name, such as `int`. + - **Envelope shape.** An exact envelope also needs a non-empty codec + string ("The payload envelope codec must be a non-empty string.") and a + string blob ("The payload envelope blob must be a string."). Laravel's + middleware turns an empty string into null first. + - **Signal and update decode their blob.** If decoding fails, PHP answers + "The input envelope blob could not be decoded with codec "avro": …" with + the serializer's diagnosis and remediation. A blob that decodes to a + scalar gets "The input envelope blob must decode to an array." A + non-empty map supplies its values as the arguments. + - **Base64.** PHP's strict base64 skips ASCII whitespace. Padding must be + absent or exactly what the final quantum needs, and unused trailing bits + may be set. + - **Envelope.** Every refusal is `422 validation_failed` with the + operation's control-plane envelope and `workflow_id`. +- **Source.** + - **Input checks.** `input_envelope` applies the transport, codec and + shape rules on all four routes. + - Signal and update refuse undecodable and scalar blobs with PHP's + messages, and take a map's values. + - Start parses non-envelope objects in request order, but keeps an object + whose codec would be refused, so the codec check still sees it. + - Start also refuses an undecodable blob with the signal message. + Published PHP 2.5.14 answers that case with a 500 (server#601). + - **Codec.** `codec::ingress_diagnosis` gives PHP's diagnosis for each + decode failure: JSON-looking text, bad base64, missing magic, unknown + fingerprint, trailing bytes, truncation, or a malformed datum. Native's + Avro decoding now uses PHP's base64 rules everywhere. + - **Envelopes.** Update refusals now carry the control-plane envelope, as + signal and query refusals do. A command validation failure always + includes `workflow_id`. + - **Stored signals.** A signal already stored and applied during a worker's + task completion keeps its `invalid_signal_arguments` refusal. +- **Remaining differences.** + - **Start with an undecodable blob.** Native refuses it with 422; PHP 2.5.14 + answers 500 (server#601). + - **Query with an undecodable blob.** Native refuses it at admission with + `invalid_query_arguments`. PHP first checks for a query worker, so + without one it answers `409 query_worker_unavailable`. + - **External payload references.** `{codec, external_payload}` input on + signal, update and query is still unsupported in native. PHP resolves it. +- **Verification.** + - **Harness.** The 60 malformed-input cases now match PHP's status, reason + and errors in 56 cases; the 4 left are the start and query differences + above. For seven refusal cases covering signal, update, query and an + unknown workflow, the bodies are identical, envelope included. + - **Tests.** `input_envelopes` covers the refusals on start and signal, plus + a map blob and an unpadded blob, on SQLite, PostgreSQL and MySQL. Unit + tests cover the diagnoses and the base64 rules. The query and update + tests now expect PHP's `validation_failed`. The full native suite passes + on SQLite. diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 0a7394960..d48e1daf9 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -102,13 +102,13 @@ contract fixture before its correction. each priority tier across fairness keys by their weights (`TaskFairnessScheduler`); native stores the keys and weights but does not rebalance yet. -- **Malformed input envelopes.** Published PHP refuses a malformed `input` - envelope on start, signal, update and query with `422 validation_failed`, - using Laravel messages on `input.codec` or `input.blob`. The cases are a - codec that is not `avro`, a missing codec, a blob that is empty or not a - string, and a blob Avro cannot decode. Native answers with its own reasons - instead: `unsupported_payload_codec`, `invalid_payload_envelope`, or an - `invalid_*_arguments` reason. +- **Undecodable query input before routing.** Native decodes query + arguments at admission and refuses an undecodable blob with + `invalid_query_arguments`. Published PHP first looks for a query worker, + and answers `409 query_worker_unavailable` when there is none. +- **External payload references on signal, update and query.** Published + PHP resolves `{codec, external_payload}` input on these routes; native + supports it only on start. - **Key order inside plain JSON input.** For input that is not an envelope, PHP encodes nested objects as Avro maps in request order; native encodes them with sorted keys. The decoded values are equal, but the blobs differ. diff --git a/rust/src/codec.rs b/rust/src/codec.rs index c51680190..e4480854f 100644 --- a/rust/src/codec.rs +++ b/rust/src/codec.rs @@ -10,7 +10,31 @@ mod validation; use apache_avro::{ GenericSingleObjectReader, GenericSingleObjectWriter, Schema, types::Value as Datum, }; -use base64::{Engine, engine::general_purpose::STANDARD}; +use base64::{ + Engine, alphabet, + engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig, general_purpose::STANDARD}, +}; + +/// PHP's strict `base64_decode`: padding is absent or canonical, and unused +/// trailing bits may be set. +const PHP_BASE64: GeneralPurpose = GeneralPurpose::new( + &alphabet::STANDARD, + GeneralPurposeConfig::new() + .with_decode_padding_mode(DecodePaddingMode::Indifferent) + .with_decode_allow_trailing_bits(true), +); + +/// Decodes a blob as published PHP does: ASCII whitespace is skipped, and +/// any padding must be exactly what the final quantum needs. +fn php_base64(blob: &str) -> Result, base64::DecodeError> { + let compact: String = blob.chars().filter(|c| !matches!(c, ' ' | '\t' | '\n' | '\r')).collect(); + let unpadded = compact.trim_end_matches('='); + let padding = compact.len() - unpadded.len(); + if padding > 0 && padding != (4 - unpadded.len() % 4) % 4 { + return Err(base64::DecodeError::InvalidPadding); + } + PHP_BASE64.decode(unpadded) +} pub const VALUE_SCHEMA: &str = include_str!("../schema/durable_workflow.protocol.Value.v1.avsc"); @@ -60,6 +84,60 @@ pub enum CodecError { NonFiniteDouble, } +/// The writer schema fingerprint of a Value single-object frame. +const VALUE_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37]; + +/// Published PHP's `CodecDecodeException` message for a blob that +/// [`ValueCodec::decode`] refused: what the decoder saw, then a remediation. +pub fn ingress_diagnosis(blob: &str, error: &CodecError) -> String { + const SCHEMA: &str = "Provide a schema-conformant Avro Value datum."; + let (detail, remediation) = match php_base64(blob) { + Err(_) if looks_like_json(blob) => ( + "invalid_payload_framing: payload bytes look like JSON, not base64-encoded Avro.".to_owned(), + "Encode the value with the fixed Avro Value codec; JSON is only the HTTP document transport.", + ), + Err(_) => ( + "invalid_payload_framing: failed to base64-decode Avro payload bytes.".to_owned(), + "Avro payloads must be strict base64 containing a c301 single-object frame.", + ), + Ok(bytes) if bytes.len() < 10 || !bytes.starts_with(&[0xc3, 0x01]) => ( + "invalid_payload_framing: expected Avro single-object magic c301.".to_owned(), + "Encode the value with the Durable Workflow 2.0 fixed Avro Value schema and single-object framing; JSON is only the HTTP document transport.", + ), + Ok(bytes) if bytes[2..10] != VALUE_FINGERPRINT => ( + format!( + "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}.", + bytes[2..10].iter().map(|byte| format!("{byte:02x}")).collect::() + ), + "Upgrade to a runtime that bundles this writer schema; never guess or fall back to JSON.", + ), + Ok(_) => match error { + CodecError::TrailingBytes => ( + "invalid_payload_framing: trailing bytes after Avro Value datum.".to_owned(), + "Provide exactly one Avro Value datum in each single-object frame.", + ), + CodecError::Truncated => ( + "invalid_payload_framing: truncated Avro Value datum.".to_owned(), + "Provide one complete Avro Value datum after the single-object fingerprint.", + ), + CodecError::InvalidValue => ( + "invalid_payload_framing: datum is not a durable_workflow.protocol.Value record.".to_owned(), + SCHEMA, + ), + _ => ( + "invalid_payload_framing: malformed Avro Value datum.".to_owned(), + "Provide one complete datum encoded with the writer schema selected by the single-object fingerprint.", + ), + }, + }; + format!("{detail} Remediation: {remediation}") +} + +/// PHP's `looksLikeJson`: the first byte could start a JSON document. +fn looks_like_json(blob: &str) -> bool { + blob.bytes().next().is_some_and(|first| b"{[\"-tfn".contains(&first) || first.is_ascii_digit()) +} + pub struct ValueCodec { schema: Schema, reader: GenericSingleObjectReader, @@ -87,7 +165,7 @@ impl ValueCodec { } pub fn decode(&self, blob: &str) -> Result { - let bytes = STANDARD.decode(blob)?; + let bytes = php_base64(blob)?; let mut input = CompleteInput { remaining: bytes.as_slice(), truncated: false, @@ -247,6 +325,39 @@ mod tests { ); } + #[test] + fn base64_follows_php_strict_decoding() { + for accepted in ["QUJD", "QUI", "QUI=", "QQ", "QQ==", "QR==", "QU JD", "QUJD\nQUJD", ""] { + assert!(php_base64(accepted).is_ok(), "{accepted:?}"); + } + for refused in ["QUJD=", "QUI==", "QQ=", "QQ===", "QUJD====", "Q", "QUJD\tQ", "QU-D", "QU_D", "Q=UJD"] { + assert!(php_base64(refused).is_err(), "{refused:?}"); + } + let codec = ValueCodec::new().unwrap(); + let padded = codec.encode(&Value::Long(7)).unwrap(); + assert_eq!(codec.decode(padded.trim_end_matches('=')).unwrap(), Value::Long(7)); + } + + #[test] + fn ingress_diagnosis_names_what_php_saw() { + let codec = ValueCodec::new().unwrap(); + let good = codec.encode(&Value::Array(vec![Value::Long(1)])).unwrap(); + let diagnose = |blob: &str| ingress_diagnosis(blob, &codec.decode(blob).unwrap_err()); + assert!(diagnose("not-avro").starts_with("invalid_payload_framing: payload bytes look like JSON, not base64-encoded Avro. Remediation: Encode")); + assert!(diagnose("@@@@").starts_with("invalid_payload_framing: failed to base64-decode Avro payload bytes.")); + assert!(diagnose("QUJD").starts_with("invalid_payload_framing: expected Avro single-object magic c301.")); + let mut bytes = STANDARD.decode(&good).unwrap(); + bytes[2..10].fill(0); + assert!(diagnose(&STANDARD.encode(&bytes)).starts_with( + "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint 0000000000000000. Remediation: Upgrade" + )); + let mut trailing = STANDARD.decode(&good).unwrap(); + trailing.push(0); + assert!(diagnose(&STANDARD.encode(&trailing)).starts_with("invalid_payload_framing: trailing bytes after Avro Value datum.")); + let truncated = STANDARD.decode(&good).unwrap(); + assert!(diagnose(&STANDARD.encode(&truncated[..truncated.len() - 1])).starts_with("invalid_payload_framing: truncated Avro Value datum.")); + } + #[test] fn rejects_bad_base64_header_fingerprint_truncation_and_trailing_bytes() { let codec = ValueCodec::new().unwrap(); diff --git a/rust/src/runtime/control_plane.rs b/rust/src/runtime/control_plane.rs index e69da2480..56d5a37b4 100644 --- a/rust/src/runtime/control_plane.rs +++ b/rust/src/runtime/control_plane.rs @@ -15,7 +15,7 @@ pub(super) enum Operation<'a> { Terminate, Repair, Debug, - Update, + Update(&'a str), Signal(&'a str), Query(&'a str), } @@ -49,6 +49,16 @@ impl Operation<'_> { "message":"Commands cannot target historical runs. Use the instance-level endpoint to command the current run, or omit the run ID."}), ) } + // PHP names the workflow on every command validation failure. + super::RuntimeError::Protocol { status, mut response } + if matches!(self, Self::Signal(_) | Self::Query(_) | Self::Update(_)) + && response["reason"] == "validation_failed" => + { + if response.get("workflow_id").is_none() { + response["workflow_id"] = json!(workflow_id); + } + (status, response) + } super::RuntimeError::Protocol { status, response } if matches!(self, Self::Signal(_) | Self::Query(_)) => { @@ -100,7 +110,7 @@ impl Operation<'_> { Self::Terminate => ("terminate", &["workflow_id"], &["outcome"]), Self::Repair => ("repair", &["workflow_id"], &["outcome"]), Self::Debug => ("debug_workflow", &["workflow_id"], &["run_id", "diagnostic_status"]), - Self::Update => ( + Self::Update(_) => ( "update", &["workflow_id", "operation_name", "operation_name_field"], &["outcome"], @@ -126,8 +136,8 @@ impl Operation<'_> { metadata["contract"]["rejection_reasons"] = words(reasons); } match self { - Self::Update => { - metadata["operation_name"] = body["update_name"].clone(); + Self::Update(name) => { + metadata["operation_name"] = body.get("update_name").cloned().unwrap_or_else(|| json!(name)); metadata["operation_name_field"] = json!("update_name"); for field in [ "command_id", @@ -140,7 +150,9 @@ impl Operation<'_> { "wait_timed_out", "wait_timeout_seconds", ] { - metadata[field] = body[field].clone(); + if let Some(value) = body.get(field) { + metadata[field] = value.clone(); + } } } Self::Query(name) => { @@ -244,7 +256,7 @@ impl Operation<'_> { Self::Cancel | Self::Terminate | Self::Repair => { ("workflow_id run_id reason message remediation", "v1_projection_read_only") } - Self::Update => ( + Self::Update(_) => ( "workflow_id run_id update_id update_name update_status outcome reason \ rejection_reason validation_errors retryable message remediation", "update_validator_rejected update_validator_contract_missing \ diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index 021692ed0..cdbf1da51 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -356,6 +356,7 @@ impl Runtime { /// empty input is no arguments and any list or object other than an /// envelope is its values, both encoded here. An envelope is left as sent. async fn plain_arguments(&self, body: &Value) -> Result> { + super::input_envelope::check(body)?; let arguments = match super::start_request::input(body) { super::start_request::Input::Envelope => return Ok(None), super::start_request::Input::Empty => Vec::new(), @@ -378,17 +379,38 @@ impl Runtime { } } + /// Refuses an envelope blob the codec cannot decode, with PHP's message. + async fn decodable(&self, blob: &Value) -> Result<()> { + let Some(blob) = blob.as_str().map(str::to_owned) else { + return Ok(()); + }; + let permit = self.signal_codec.clone().acquire_owned().await + .map_err(|_| refuse(StatusCode::SERVICE_UNAVAILABLE, "signal_codec_unavailable"))?; + tokio::task::spawn_blocking(move || { + let _permit = permit; + let codec = crate::codec::ValueCodec::new() + .map_err(|_| refuse(StatusCode::SERVICE_UNAVAILABLE, "signal_codec_unavailable"))?; + codec.decode(&blob).map(drop).map_err(|error| super::input_envelope::undecodable(&blob, &error)) + }) + .await + .map_err(|_| refuse(StatusCode::SERVICE_UNAVAILABLE, "signal_codec_unavailable"))? + } + pub(crate) async fn start(&self, body: Value) -> Result { // PHP validates the request before it reads the memo or the input. let mut body = super::start_request::normalized(body); + super::input_envelope::transport(&body)?; super::start_request::validate(&body)?; if body["search_attributes"].as_object().is_some_and(|fields| !fields.is_empty()) || body["search_attributes"].as_array().is_some_and(|items| !items.is_empty()) { return Err(refuse(StatusCode::NOT_IMPLEMENTED, "start_search_attributes_unavailable")); } - if let Some(blob) = self.plain_arguments(&body).await? { - body["input"] = json!({"codec": "avro", "blob": blob}); + match self.plain_arguments(&body).await? { + Some(blob) => body["input"] = json!({"codec": "avro", "blob": blob}), + // PHP 2.5.14 answers an undecodable start blob with a 500 + // (server#601); native refuses it as PHP's signal does. + None => self.decodable(&body["input"]["blob"]).await?, } let memo = super::memos::prepare_initial(self.signal_codec.clone(), body["memo"].clone()).await?; @@ -448,6 +470,7 @@ impl Runtime { body: Value, ) -> Result { let body = super::start_request::normalized(body); + super::input_envelope::transport(&body)?; super::command_requests::signal(&body)?; let arguments = self.command_arguments(&body).await?; let prepared = super::store::prepare_control_arguments( @@ -455,6 +478,7 @@ impl Runtime { name.to_owned(), arguments, "invalid_signal_arguments", + true, ) .await?; match &*self.storage { @@ -587,6 +611,7 @@ impl Runtime { body: Value, ) -> Result<(StatusCode, Value)> { let body = super::start_request::normalized(body); + super::input_envelope::transport(&body)?; super::command_requests::update(&body)?; let arguments = self.command_arguments(&body).await?; if arguments.len() > super::queries::ARGUMENT_BYTES { @@ -600,6 +625,7 @@ impl Runtime { name.to_owned(), arguments, "invalid_update_arguments", + true, ) .await?; match &*self.storage { @@ -632,6 +658,7 @@ impl Runtime { // nullable|array; other request fields are ignored, as in PHP. // Laravel trims strings and turns empty ones into null first. let body = super::start_request::normalized(body); + super::input_envelope::transport(&body)?; let invalid = |field: &'static str, message: &'static str| { let mut error = super::laravel_validation::failure(vec![(field, vec![message])]).expect("one message"); if let super::RuntimeError::Protocol { response, .. } = &mut error { diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index f307045bb..8f4d19079 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -986,7 +986,8 @@ async fn update_current( ) -> Result<(StatusCode, Json)> { let (status, result) = runtime .update_workflow(&workflow_id, None, &name, without_trace_carrier(body)) - .await?; + .await + .map_err(|error| super::control_plane::Operation::Update(&name).refusal(error, &workflow_id, None))?; Ok((status, Json(result))) } @@ -997,7 +998,10 @@ async fn update_run( ) -> Result<(StatusCode, Json)> { let (status, result) = runtime .update_workflow(&workflow_id, Some(&run_id), &name, without_trace_carrier(body)) - .await?; + .await + .map_err(|error| { + super::control_plane::Operation::Update(&name).refusal(error, &workflow_id, Some(&run_id)) + })?; Ok((status, Json(result))) } diff --git a/rust/src/runtime/input_envelope.rs b/rust/src/runtime/input_envelope.rs new file mode 100644 index 000000000..4086c7947 --- /dev/null +++ b/rust/src/runtime/input_envelope.rs @@ -0,0 +1,105 @@ +//! Published PHP's checks on a control-plane `input` before it resolves it: +//! the runtime external payload transport, then `AvroPayloadEnvelopeResolver`. +//! Its refusals are Laravel validation failures on `input.codec` and +//! `input.blob`. +use super::laravel_validation::failure_messages; +use super::{Result, RuntimeError, refuse}; +use axum::http::StatusCode; +use serde_json::Value; + +/// `PayloadCodecContract::canonicalize`: null, empty or `avro` names the +/// codec; anything else is refused with PHP's message. +pub(super) fn codec_refusal(codec: &Value) -> Option { + let rendered = match codec { + Value::Null => return None, + Value::String(codec) if codec.is_empty() || codec == "avro" => return None, + Value::String(codec) => codec.as_str(), + Value::Bool(_) => "bool", + Value::Number(number) if number.is_i64() || number.is_u64() => "int", + Value::Number(_) => "float", + Value::Array(_) | Value::Object(_) => "array", + }; + Some(format!( + "unsupported_payload_codec: workflow payload codec \"{rendered}\" is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec." + )) +} + +fn invalid(field: &str, message: String) -> RuntimeError { + failure_messages(vec![(field.to_owned(), vec![message])]).expect("one message") +} + +/// `RuntimeExternalPayloadTransport` refuses a provider-specific reference +/// before the request reaches validation. +pub(super) fn transport(body: &Value) -> Result<()> { + if body["input"].get("external_storage").is_some() { + return Err(refuse(StatusCode::UNPROCESSABLE_ENTITY, "external_payload_unsupported")); + } + Ok(()) +} + +/// `assertEnvelope`, then `resolveExplicitEnvelope`, on the trimmed `input`. +/// Any object with a codec names an allowed codec; an exact envelope also +/// has a non-empty codec and a string blob. +pub(super) fn check(body: &Value) -> Result<()> { + let Some(fields) = body["input"].as_object() else { + return Ok(()); + }; + if let Some(message) = fields.get("codec").and_then(codec_refusal) { + return Err(invalid("input.codec", message)); + } + let mut keys: Vec<&str> = fields.keys().map(String::as_str).collect(); + keys.sort_unstable(); + if keys != ["blob", "codec"] { + return Ok(()); + } + if !fields["codec"].as_str().is_some_and(|codec| !codec.is_empty()) { + return Err(invalid("input.codec", "The payload envelope codec must be a non-empty string.".into())); + } + if !fields["blob"].is_string() { + return Err(invalid("input.blob", "The payload envelope blob must be a string.".into())); + } + Ok(()) +} + +/// `resolveToArray`'s refusal of a blob the codec cannot decode. +pub(super) fn undecodable(blob: &str, error: &crate::codec::CodecError) -> RuntimeError { + invalid( + "input.blob", + format!( + "The input envelope blob could not be decoded with codec \"avro\": {}", + crate::codec::ingress_diagnosis(blob, error) + ), + ) +} + +/// `resolveToArray`'s refusal of a blob that is not a list or a map. +pub(super) fn not_a_list() -> RuntimeError { + invalid("input.blob", "The input envelope blob must decode to an array.".into()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn errors(result: Result<()>) -> Value { + match result { + Err(RuntimeError::Protocol { response, .. }) => response["errors"].clone(), + other => panic!("{:?}", other.err()), + } + } + + #[test] + fn input_checks_follow_php() { + let codec = |value: Value| errors(check(&json!({"input": value}))); + assert!(codec(json!({"codec":"json"}))["input.codec"][0].as_str().unwrap().contains("codec \"json\" is not supported")); + assert!(codec(json!({"codec":1,"blob":"x"}))["input.codec"][0].as_str().unwrap().contains("codec \"int\" is not supported")); + assert_eq!(codec(json!({"codec":null,"blob":"x"}))["input.codec"], json!(["The payload envelope codec must be a non-empty string."])); + assert_eq!(codec(json!({"codec":"avro","blob":null}))["input.blob"], json!(["The payload envelope blob must be a string."])); + for accepted in [json!([1]), json!({"codec":"avro"}), json!({"codec":null}), json!({"codec":"avro","blob":"x"}), json!(null)] { + assert!(check(&json!({"input": accepted})).is_ok(), "{accepted}"); + } + assert!(transport(&json!({"input":{"codec":"avro","external_storage":"x"}})).is_err()); + assert!(transport(&json!({"input":[{"external_storage":1}]})).is_ok()); + } +} diff --git a/rust/src/runtime/memo_json.rs b/rust/src/runtime/memo_json.rs index 0f59a3f12..6e7c12df2 100644 --- a/rust/src/runtime/memo_json.rs +++ b/rust/src/runtime/memo_json.rs @@ -162,7 +162,13 @@ impl<'de> Deserialize<'de> for InputJson { } let mut keys: Vec<&str> = entries.iter().map(|(key, _)| key.as_str()).collect(); keys.sort_unstable(); - Ok(InputJson(if super::start_request::envelope_keys(&keys) { + // PHP checks the codec of any input object, so one it would + // refuse stays an object for that check. + let refused_codec = entries.iter().any(|(key, value)| { + key == "codec" + && super::input_envelope::codec_refusal(&super::start_request::normalized(value.clone())).is_some() + }); + Ok(InputJson(if refused_codec || super::start_request::envelope_keys(&keys) { Value::Object(entries.into_iter().collect()) } else { Value::Array(entries.into_iter().map(|(_, value)| value).collect()) diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index d0a7938e8..9e213bd67 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -30,6 +30,7 @@ mod external_wait_summary; mod failure_commands; mod history_budget; mod history_export; +mod input_envelope; mod history_presentation; mod http; mod laravel_validation; diff --git a/rust/src/runtime/start_request.rs b/rust/src/runtime/start_request.rs index 144c23b89..cdd5ddaa9 100644 --- a/rust/src/runtime/start_request.rs +++ b/rust/src/runtime/start_request.rs @@ -85,28 +85,9 @@ pub(super) fn normalized(value: Value) -> Value { /// `PayloadCodecContract::canonicalize`. fn payload_codec(value: &Value) -> Result<()> { - match value { - Value::Null => Ok(()), - Value::String(codec) if codec == "avro" => Ok(()), - other => { - let rendered = other.as_str().map_or_else(|| debug_type(other).to_owned(), str::to_owned); - Err(failure_messages(vec![("payload_codec".into(), vec![format!( - "unsupported_payload_codec: workflow payload codec \"{rendered}\" is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec." - )])]) - .expect("one message")) - } - } -} - -/// PHP's `get_debug_type`. -fn debug_type(value: &Value) -> &'static str { - match value { - Value::Null => "null", - Value::Bool(_) => "bool", - Value::Number(number) if number.is_i64() || number.is_u64() => "int", - Value::Number(_) => "float", - Value::String(_) => "string", - Value::Array(_) | Value::Object(_) => "array", + match super::input_envelope::codec_refusal(value) { + None => Ok(()), + Some(message) => Err(failure_messages(vec![("payload_codec".into(), vec![message])]).expect("one message")), } } diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index e86052ed9..5b4fb2e5b 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -840,6 +840,7 @@ where name.clone(), DB::string(&signal, "arguments")?, "invalid_signal_arguments", + false, ) .await?; let command_id = DB::string(&signal, "workflow_command_id")?; @@ -2913,11 +2914,14 @@ where /// A start's optional `nullable|integer|min:1` timeout. Absent or null means /// no timeout, as in published PHP. +/// `request` arguments come from a control-plane `input`, which PHP refuses +/// as validation of `input.blob`; stored arguments keep `invalid`. pub(super) async fn prepare_control_arguments( semaphore: Arc, name: String, arguments: String, invalid: &'static str, + request: bool, ) -> Result { let permit = semaphore .acquire_owned() @@ -2929,11 +2933,20 @@ pub(super) async fn prepare_control_arguments( let _permit = permit; let codec = crate::codec::ValueCodec::new() .map_err(|_| refuse(StatusCode::UNPROCESSABLE_ENTITY, invalid))?; - let crate::codec::Value::Array(values) = codec - .decode(&arguments) - .map_err(|_| refuse(StatusCode::UNPROCESSABLE_ENTITY, invalid))? - else { - return Err(refuse(StatusCode::UNPROCESSABLE_ENTITY, invalid)); + // PHP's `resolveToArray`: the blob decodes to a list, or to a + // non-empty map whose values are the arguments. + let (values, arguments) = match codec.decode(&arguments) { + Ok(crate::codec::Value::Array(values)) => (values, arguments), + Ok(crate::codec::Value::Map(entries)) if request && !entries.is_empty() => { + let values: Vec<_> = entries.into_values().collect(); + let encoded = codec + .encode(&crate::codec::Value::Array(values.clone())) + .map_err(|_| refuse(StatusCode::UNPROCESSABLE_ENTITY, invalid))?; + (values, encoded) + } + Ok(_) if request => return Err(super::input_envelope::not_a_list()), + Err(error) if request => return Err(super::input_envelope::undecodable(&arguments, &error)), + _ => return Err(refuse(StatusCode::UNPROCESSABLE_ENTITY, invalid)), }; let value = match values.as_slice() { [] => crate::codec::Value::Null, diff --git a/rust/src/runtime/updates.rs b/rust/src/runtime/updates.rs index 3a0be2fdc..a81ca14cc 100644 --- a/rust/src/runtime/updates.rs +++ b/rust/src/runtime/updates.rs @@ -465,7 +465,8 @@ where "closed_at":DB::optional_instant(update,"closed_at")?,"reason":null,"rejection_reason":null,"validation_errors":[], "wait_for":wait_for,"wait_timed_out":timed_out,"wait_timeout_seconds":timeout }); - Ok(super::control_plane::Operation::Update.response(response)) + let name = response["update_name"].as_str().unwrap_or_default().to_owned(); + Ok(super::control_plane::Operation::Update(&name).response(response)) } pub(super) fn update_worker_supported( diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 89fa3a9cd..fc44bb768 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -85,6 +85,8 @@ mod read_snapshots; mod query_responses; #[path = "execution/history_export.rs"] mod history_export; +#[path = "execution/input_envelopes.rs"] +mod input_envelopes; #[path = "execution/history_retention.rs"] mod history_retention; #[path = "execution/activity_timeout_pass.rs"] @@ -1729,10 +1731,11 @@ async fn updates_preserve_original_receipts_and_fence_recovered_tasks_across_poo json!({"input":envelope(Payload::Array(vec![Payload::String("wrong".into())]))}), "invalid_update_arguments", ), + // PHP refuses an undecodable blob as validation of `input.blob`. ( path.clone(), json!({"input":{"codec":"avro","blob":"bad"}}), - "invalid_update_arguments", + "validation_failed", ), ( path.clone(), diff --git a/rust/tests/execution/input_envelopes.rs b/rust/tests/execution/input_envelopes.rs new file mode 100644 index 000000000..c47d0bbba --- /dev/null +++ b/rust/tests/execution/input_envelopes.rs @@ -0,0 +1,55 @@ +use super::*; + +const MAGIC: &str = "The input envelope blob could not be decoded with codec \"avro\": invalid_payload_framing: expected Avro single-object magic c301. Remediation: Encode the value with the Durable Workflow 2.0 fixed Avro Value schema and single-object framing; JSON is only the HTTP document transport."; + +async fn started(app: &Router, workflow: &str, input: Value) -> (StatusCode, Value) { + request(app, "POST", "/api/workflows", json!({"workflow_id":workflow,"workflow_type":"echo","task_queue":"test","input":input})).await +} + +async fn signalled(app: &Router, input: Value) -> (StatusCode, Value) { + request(app, "POST", "/api/workflows/envelopes/signal/note", json!({"input":input})).await +} + +// Published PHP refuses a malformed input envelope as Laravel validation of +// `input.codec` or `input.blob`, on start and signal alike. +#[tokio::test] +async fn malformed_input_envelopes_are_refused_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register(&app, "worker", json!(["echo"]), json!([])).await; + start(&app, "envelopes").await; + let codec = |name: &str| format!("unsupported_payload_codec: workflow payload codec \"{name}\" is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec."); + let cases = [ + (json!({"codec":"json","blob":"[]"}), "input.codec", codec("json")), + (json!({"codec":"json"}), "input.codec", codec("json")), + (json!({"codec":1,"blob":"x"}), "input.codec", codec("int")), + (json!({"codec":null,"blob":"x"}), "input.codec", "The payload envelope codec must be a non-empty string.".to_owned()), + (json!({"codec":"avro","blob":""}), "input.blob", "The payload envelope blob must be a string.".to_owned()), + (json!({"codec":"avro","blob":"QUJD"}), "input.blob", MAGIC.to_owned()), + ]; + for (index, (input, field, message)) in cases.into_iter().enumerate() { + for (status, body) in [started(&app, &format!("refused-{index}"), input.clone()).await, signalled(&app, input.clone()).await] { + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{input}: {body}"); + assert_eq!(body["reason"], "validation_failed", "{input}: {body}"); + assert_eq!(body["errors"][field], json!([message]), "{input}: {body}"); + } + } + // The transport refuses provider-specific references first. + for (status, body) in [started(&app, "provider", json!({"codec":"avro","external_storage":{}})).await, + signalled(&app, json!({"codec":"avro","external_storage":{}})).await] { + assert_eq!((status, body["reason"].clone()), (StatusCode::UNPROCESSABLE_ENTITY, json!("external_payload_unsupported")), "{body}"); + } + + // A signal's blob must decode to a list or a map; a start's may be any value. + let scalar = envelope(Payload::Long(7)); + let (status, body) = signalled(&app, scalar.clone()).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert_eq!(body["errors"]["input.blob"], json!(["The input envelope blob must decode to an array."])); + assert_eq!(started(&app, "scalar", scalar).await.0, StatusCode::CREATED); + let map = envelope(Payload::Map([("a".to_owned(), Payload::Long(1))].into_iter().collect())); + assert_eq!(signalled(&app, map).await.0, StatusCode::ACCEPTED); + // PHP's strict base64 takes a blob without its padding. + let list = envelope(Payload::Array(vec![Payload::Long(1)])); + let unpadded = json!({"codec":"avro","blob":list["blob"].as_str().unwrap().trim_end_matches('=')}); + assert_eq!(signalled(&app, unpadded).await.0, StatusCode::ACCEPTED); +} diff --git a/rust/tests/execution/query_arguments.rs b/rust/tests/execution/query_arguments.rs index aa7816dfb..b2479c2cf 100644 --- a/rust/tests/execution/query_arguments.rs +++ b/rust/tests/execution/query_arguments.rs @@ -78,11 +78,12 @@ async fn omitted_query_input_uses_official_empty_arguments_without_mutating_the_ assert_eq!(query_entries(&database).await, 0); for (path, input, status, reason) in [ + // PHP refuses a codec other than Avro as validation of `input.codec`. ( instance_query.clone(), json!({"input":{"codec":"json","blob":"[]"}}), StatusCode::UNPROCESSABLE_ENTITY, - "unsupported_payload_codec", + "validation_failed", ), // As in PHP, an object that is not exactly an envelope is plain // values, here one argument the query does not take. @@ -102,7 +103,7 @@ async fn omitted_query_input_uses_official_empty_arguments_without_mutating_the_ instance_query.clone(), json!({"input":{"codec":"avro","blob":""}}), StatusCode::UNPROCESSABLE_ENTITY, - "invalid_payload_envelope", + "validation_failed", ), ( instance_query.clone(), From 5a9136cbc29ea9adbcafe439fd37836d0489b352 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 18:10:56 +0000 Subject: [PATCH 02/30] Report repair candidates like published PHP Published dw 2.3.0's system:repair-status reached native's 501. Native now serves admin-only GET /api/system/repair. Its candidates are TaskRepairCandidates::snapshot across every namespace: overdue, failed or expired tasks, and open runs without a next task, grouped by connection, queue and compatibility with PHP's scope-fair round-robin selection. Its policy is native's own claim-time lease recovery, as operator metrics already report. Co-Authored-By: Claude Opus 5.5 --- .../2026-10-21-repair-status.md | 43 +++ rust/src/runtime/execution.rs | 1 + rust/src/runtime/http.rs | 6 + rust/src/runtime/mod.rs | 1 + rust/src/runtime/operator_metrics.rs | 41 +++ rust/src/runtime/repair_candidates.rs | 267 ++++++++++++++++++ rust/tests/execution.rs | 2 + rust/tests/execution/repair_status.rs | 55 ++++ 8 files changed, 416 insertions(+) create mode 100644 docs/rust-server-port-checkpoints/2026-10-21-repair-status.md create mode 100644 rust/src/runtime/repair_candidates.rs create mode 100644 rust/tests/execution/repair_status.rs diff --git a/docs/rust-server-port-checkpoints/2026-10-21-repair-status.md b/docs/rust-server-port-checkpoints/2026-10-21-repair-status.md new file mode 100644 index 000000000..c866cdd00 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-21-repair-status.md @@ -0,0 +1,43 @@ +# October 21, repair status + +- **Found by.** The published dw 2.3.0 CLI sweep. `system:repair-status` + (`GET /api/system/repair`) got native's 501. +- **Source.** Native serves the admin-only `GET /api/system/repair` in the + default or a named namespace. Like published PHP 2.5.14, the response has + two parts: `policy` and `candidates`. + - **Candidates.** This is `TaskRepairCandidates::snapshot`, scanned across + every namespace, as in PHP: + - **Ready tasks** that are due and whose dispatch failed, whose claim + failed, or whose dispatch is overdue past the 3-second redispatch + cutoff, each with PHP's backoff rules. + - **Leased tasks** whose lease expired. + - **Open runs without a next task** that need repair, or that wait on + children that have all closed. + - **Grouping and selection.** Candidates are grouped by connection, + queue and compatibility. Null, empty and public-default values share a + scope, and each scope's selection uses its representative's exact + values, as PHP's `applyScope` does. Selection takes up to 25 per scope + and goes round-robin, oldest scope first, up to 25 in all. + - **Order.** Scopes are sorted busiest first, then oldest, then by key. + - **Time comparisons** bind "now" in whole seconds, as Laravel does. + - **Policy.** `policy` is native's own `repair_policy`, the same one + operator metrics reports: claim-time lease recovery, with no repair loop, + loop throttle or failure backoff. PHP reports its maintenance loop's + policy instead. + - **Operator metrics.** These keep their empty `repair` section, which is + what PHP returns when the request names a namespace. +- **Still open.** + - `POST /api/system/repair/pass`. + - Repairing a run whose task is ready (`workflow:repair` gets + `development_workflow_task_repair_unqualified`). +- **Verification.** + - **Integration test.** `repair_status` passes on SQLite, PostgreSQL and + MySQL. It covers two overdue ready tasks in one queue, an expired lease + in another, and a run in a third queue that lost its task. Unit tests + cover the round-robin selection and the merging of default scopes. + - **Harness, empty.** On the seeded data, both servers reported no + candidates. + - **Harness, candidates.** With three runs started on two queues and no + worker, the candidates matched published PHP 2.5.14 field for field, + apart from times. dw 2.3.0's `system:repair-status` succeeds against + both servers. diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index 7692f867e..c8e815c62 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -299,6 +299,7 @@ impl Runtime { delegate!(operator_metrics(config: &super::operator_metrics::MetricsConfig) -> Result); delegate!(system_health(config: &super::operator_metrics::MetricsConfig) -> Result<(bool, Value)>); delegate!(operator_dashboard(config: &super::operator_metrics::MetricsConfig, bounded: bool) -> Result); + delegate!(repair_status() -> Result); delegate!(retention_status(limit: i64) -> Result); delegate!(retention_pass(limit: i64, run_ids: Vec) -> Result<(Value, bool)>); delegate!(activity_timeout_status(limit: i64) -> Result); diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index f307045bb..0f854b621 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -198,6 +198,7 @@ pub fn router(runtime: Runtime) -> Router { "/api/system/operator-dashboard/bounded/workflow-types", get(workflow_type_operator_dashboard), ) + .route("/api/system/repair", get(repair_status)) .route("/api/system/retention", get(retention_status)) .route("/api/system/retention/pass", post(retention_pass)) .route("/api/system/activity-timeouts", get(activity_timeout_status)) @@ -540,6 +541,7 @@ fn named_namespace_operation(method: &str, path: &str) -> bool { // timeout scan spans namespaces in PHP too. || path == "/api/system/retention" || path == "/api/system/activity-timeouts" + || path == "/api/system/repair" { return method == "GET"; } @@ -1168,6 +1170,10 @@ async fn workflow_type_operator_dashboard( type QueryMap = Query>; +async fn repair_status(Extension(runtime): Extension) -> Result> { + Ok(Json(runtime.repair_status().await?)) +} + async fn retention_status(Extension(runtime): Extension, Query(query): QueryMap) -> Result> { let limit = super::retention::status_limit(query.get("limit").map(String::as_str))?; Ok(Json(runtime.retention_status(limit).await?)) diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index d0a7938e8..b1e4037f6 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -45,6 +45,7 @@ pub mod postgres; mod previews; mod queries; mod query_authority; +mod repair_candidates; mod retention; mod run_observation; mod run_observation_store; diff --git a/rust/src/runtime/operator_metrics.rs b/rust/src/runtime/operator_metrics.rs index 6d0926405..120ae2d60 100644 --- a/rust/src/runtime/operator_metrics.rs +++ b/rust/src/runtime/operator_metrics.rs @@ -12,6 +12,7 @@ use super::{ Result, backend::Backend, + repair_candidates::{self, ScopeRow}, retention, store::{Store, now}, }; @@ -1079,6 +1080,27 @@ where })) } + /// `GET /api/system/repair`: native's repair policy, and the candidates + /// published PHP's repair pass would select across every namespace. + pub(super) async fn repair_status(&self) -> Result { + let at = now(); + let bound = whole_second(at); + let cutoff = whole_second(at - Duration::seconds(REDISPATCH_AFTER_SECONDS)); + let mut tasks = Self::query(repair_candidates::TASK_SCOPES); + for time in [bound, bound, bound, cutoff, cutoff, cutoff, bound] { + tasks = tasks.bind(DB::bind_time(time)); + } + let tasks = scope_rows::(tasks.fetch_all(&self.pool).await?)?; + let runs = scope_rows::(Self::query(repair_candidates::RUN_SCOPES).fetch_all(&self.pool).await?)?; + let oldest_run = match Self::query(repair_candidates::OLDEST_RUN).fetch_optional(&self.pool).await? { + Some(row) => DB::optional_instant(&row, "wait_started_at")? + .or(DB::optional_instant(&row, "started_at")?) + .or(DB::optional_instant(&row, "created_at")?), + None => None, + }; + Ok(json!({"policy": repair_policy(), "candidates": repair_candidates::snapshot(&tasks, &runs, oldest_run, at)})) + } + /// `HistoryRetentionEnforcer::retentionPolicy` for this namespace. async fn retention_policy(&self) -> Result { let row = Self::query("SELECT retention_mode,retention_days FROM workflow_namespaces WHERE name=$1") @@ -1380,6 +1402,25 @@ fn backend_capabilities(now: DateTime, connection: &str) -> Value { } /// Native claims ready tasks on each poll; it runs no repair loop. +/// The grouped candidate rows of one repair candidate query. +fn scope_rows(rows: Vec) -> Result> +where + usize: ColumnIndex, + for<'r> i64: Decode<'r, DB>, +{ + rows.iter() + .map(|row| { + Ok(ScopeRow { + connection: DB::optional_string(row, "connection")?, + queue: DB::optional_string(row, "queue")?, + compatibility: DB::optional_string(row, "compatibility")?, + count: usize::try_from(DB::number(row, "candidate_count")?).unwrap_or(0), + oldest: DB::optional_instant(row, "oldest_candidate_at")?, + }) + }) + .collect() +} + fn repair_policy() -> Value { json!({ "redispatch_after_seconds": REDISPATCH_AFTER_SECONDS, diff --git a/rust/src/runtime/repair_candidates.rs b/rust/src/runtime/repair_candidates.rs new file mode 100644 index 000000000..323951192 --- /dev/null +++ b/rust/src/runtime/repair_candidates.rs @@ -0,0 +1,267 @@ +//! `TaskRepairCandidates::snapshot`: the tasks and runs that published PHP's +//! repair pass would select, across every namespace, grouped by connection, +//! queue and compatibility, with its scope-fair round-robin selection. +use chrono::{DateTime, Utc}; +use serde_json::{Value, json}; + +/// `TaskRepairPolicy::SCAN_LIMIT`. +pub(super) const SCAN_LIMIT: usize = 25; + +/// `existingTaskQuery`: ready tasks that are due and whose dispatch failed, +/// whose claim failed, or whose dispatch is overdue, and leased tasks whose +/// lease expired. `$1`–`$3` and `$7` are "now"; `$4`–`$6` are the redispatch +/// cutoff, both in whole seconds as Laravel binds them. +macro_rules! repairable_tasks { + () => { + "((status='ready' AND (available_at IS NULL OR available_at<=$1) AND (\ + (last_dispatch_attempt_at IS NOT NULL AND last_dispatch_error IS NOT NULL AND last_dispatch_error<>'' \ + AND (last_dispatched_at IS NULL OR last_dispatch_attempt_at>last_dispatched_at) \ + AND (repair_available_at IS NULL OR repair_available_at<=$2)) \ + OR (last_claim_failed_at IS NOT NULL AND last_claim_error IS NOT NULL AND last_claim_error<>'' \ + AND ((repair_available_at IS NOT NULL AND repair_available_at<=$3) \ + OR (repair_available_at IS NULL AND last_claim_failed_at<=$4))) \ + OR ((last_dispatch_attempt_at IS NULL OR last_dispatch_error IS NULL OR last_dispatch_error='' \ + OR (last_dispatched_at IS NOT NULL AND last_dispatch_attempt_at<=last_dispatched_at)) \ + AND (last_claim_failed_at IS NULL OR last_claim_error IS NULL OR last_claim_error='') \ + AND (last_dispatched_at<=$5 OR (last_dispatched_at IS NULL AND created_at<=$6))))) \ + OR (status='leased' AND lease_expires_at IS NOT NULL AND lease_expires_at<=$7))" + }; +} + +/// `missingTaskRunQuery`: open runs without a next task that need repair, or +/// that wait for children which have all closed while no workflow task is open. +macro_rules! missing_task_runs { + () => { + "(liveness_state='repair_needed' OR (liveness_state='waiting_for_child' \ + AND NOT EXISTS (SELECT 1 FROM workflow_tasks t WHERE t.workflow_run_id=workflow_run_summaries.id \ + AND t.task_type='workflow' AND t.status IN ('ready','leased')) \ + AND EXISTS (SELECT 1 FROM workflow_links l WHERE l.parent_workflow_run_id=workflow_run_summaries.id \ + AND l.link_type='child_workflow') \ + AND NOT EXISTS (SELECT 1 FROM workflow_links l WHERE l.parent_workflow_run_id=workflow_run_summaries.id \ + AND l.link_type='child_workflow' AND (NOT EXISTS (SELECT 1 FROM workflow_runs c WHERE c.id=l.child_workflow_run_id) \ + OR EXISTS (SELECT 1 FROM workflow_runs c WHERE c.id=l.child_workflow_run_id \ + AND c.status IN ('pending','running','waiting')))))) \ + AND next_task_id IS NULL AND status IN ('pending','running','waiting')" + }; +} + +pub(super) const TASK_SCOPES: &str = concat!( + "SELECT connection,queue,compatibility,COUNT(*) AS candidate_count,MIN(created_at) AS oldest_candidate_at \ + FROM workflow_tasks WHERE ", + repairable_tasks!(), + " GROUP BY connection,queue,compatibility" +); +pub(super) const RUN_SCOPES: &str = concat!( + "SELECT connection,queue,compatibility,COUNT(*) AS candidate_count,\ + MIN(COALESCE(wait_started_at,started_at,created_at)) AS oldest_candidate_at FROM workflow_run_summaries WHERE ", + missing_task_runs!(), + " GROUP BY connection,queue,compatibility" +); +/// `oldestMissingRunCandidate`, in the backend's own null ordering, as PHP's. +pub(super) const OLDEST_RUN: &str = concat!( + "SELECT wait_started_at,started_at,created_at FROM workflow_run_summaries WHERE ", + missing_task_runs!(), + " ORDER BY wait_started_at,started_at,id LIMIT 1" +); + +/// One `GROUP BY` row of candidates. +pub(super) struct ScopeRow { + pub(super) connection: Option, + pub(super) queue: Option, + pub(super) compatibility: Option, + pub(super) count: usize, + pub(super) oldest: Option>, +} + +impl ScopeRow { + /// `nullableString` of each scope column. + fn values(&self) -> [Option<&str>; 3] { + [&self.connection, &self.queue, &self.compatibility] + .map(|value| value.as_deref().filter(|value| !value.is_empty())) + } + + fn key(&self) -> String { + let [connection, queue, compatibility] = self.values(); + format!("{}:{}:{}", connection.unwrap_or("default"), queue.unwrap_or("default"), compatibility.unwrap_or("any")) + } + + /// `applyScope`: an absent scope value matches null, empty or the public + /// default; a present one matches exactly. + fn within(&self, scope: &ScopeRow) -> bool { + let defaults = ["default", "default", "any"]; + let raw = [&self.connection, &self.queue, &self.compatibility]; + scope.values().into_iter().zip(raw).zip(defaults).all(|((scope, raw), default)| match scope { + None => raw.as_deref().is_none_or(|raw| raw.is_empty() || raw == default), + Some(scope) => raw.as_deref() == Some(scope), + }) + } +} + +/// `normalizedScopes` with each scope's selected ids counted: oldest scope +/// first, each holding at most `limit` candidates. +fn buckets(rows: &[ScopeRow], limit: usize) -> Vec<(String, usize)> { + let mut chosen: Vec<&ScopeRow> = Vec::new(); + for row in rows { + match chosen.iter().position(|scope| scope.key() == row.key()) { + None => chosen.push(row), + Some(index) => { + if row.oldest.is_some_and(|oldest| chosen[index].oldest.is_none_or(|current| oldest < current)) { + chosen[index] = row; + } + } + } + } + chosen.sort_by(|left, right| { + let order = |scope: &ScopeRow| scope.oldest.map_or(i64::MAX, |oldest| oldest.timestamp_millis()); + order(left).cmp(&order(right)).then_with(|| left.key().cmp(&right.key())) + }); + chosen + .into_iter() + .map(|scope| { + let count: usize = rows.iter().filter(|row| row.within(scope)).map(|row| row.count).sum(); + (scope.key(), count.min(limit)) + }) + .filter(|(_, count)| *count > 0) + .collect() +} + +/// `selectedScopeCounts`: one candidate from each scope in turn, up to +/// `limit` in all. +fn selected(buckets: &[(String, usize)], limit: usize) -> Vec<(String, usize)> { + let mut remaining: Vec = buckets.iter().map(|(_, count)| *count).collect(); + let mut counts: Vec<(String, usize)> = buckets.iter().map(|(key, _)| (key.clone(), 0)).collect(); + let mut total = 0; + while total < limit && remaining.iter().any(|count| *count > 0) { + for (index, left) in remaining.iter_mut().enumerate() { + if *left == 0 { + continue; + } + *left -= 1; + counts[index].1 += 1; + total += 1; + if total >= limit { + break; + } + } + } + counts +} + +fn count_for(counts: &[(String, usize)], key: &str) -> usize { + counts.iter().find(|(scope, _)| scope == key).map_or(0, |(_, count)| *count) +} + +fn age(oldest: Option>, now: DateTime) -> i64 { + oldest.map_or(0, |oldest| (now - oldest).num_milliseconds()) +} + +/// One scope of `scopeSnapshot`. +struct Scope<'a> { + row: &'a ScopeRow, + tasks: usize, + runs: usize, + oldest_task: Option>, + oldest_run: Option>, +} + +/// `scopeSnapshot`: every scope with candidates, the busiest and oldest +/// first. A scope keeps the columns of the first row seen with its key. +fn scopes(tasks: &[ScopeRow], runs: &[ScopeRow], selected_tasks: &[(String, usize)], selected_runs: &[(String, usize)], now: DateTime) -> Vec { + let mut scopes: Vec = Vec::new(); + for (row, is_task) in tasks.iter().map(|row| (row, true)).chain(runs.iter().map(|row| (row, false))) { + let index = match scopes.iter().position(|scope| scope.row.key() == row.key()) { + Some(index) => index, + None => { + scopes.push(Scope { row, tasks: 0, runs: 0, oldest_task: None, oldest_run: None }); + scopes.len() - 1 + } + }; + let scope = &mut scopes[index]; + let (count, oldest) = if is_task { (&mut scope.tasks, &mut scope.oldest_task) } else { (&mut scope.runs, &mut scope.oldest_run) }; + *count += row.count; + if row.oldest.is_some_and(|at| oldest.is_none_or(|current| at < current)) { + *oldest = row.oldest; + } + } + let mut documents: Vec<(usize, i64, String, Value)> = scopes + .into_iter() + .map(|scope| { + let key = scope.row.key(); + let [connection, queue, compatibility] = scope.row.values(); + let (picked_tasks, picked_runs) = (count_for(selected_tasks, &key), count_for(selected_runs, &key)); + let total = scope.tasks + scope.runs; + let (task_age, run_age) = (age(scope.oldest_task, now), age(scope.oldest_run, now)); + let document = json!({"scope_key": key, "connection": connection, "queue": queue, "compatibility": compatibility, + "existing_task_candidates": scope.tasks, "missing_task_candidates": scope.runs, "total_candidates": total, + "selected_existing_task_candidates": picked_tasks, "selected_missing_task_candidates": picked_runs, + "selected_total_candidates": picked_tasks + picked_runs, + "oldest_task_candidate_created_at": super::task_queues::json_time(scope.oldest_task), + "oldest_missing_run_started_at": super::task_queues::json_time(scope.oldest_run), + "max_task_candidate_age_ms": task_age, "max_missing_run_age_ms": run_age, + "scan_limited_by_global_policy": total > picked_tasks + picked_runs}); + (total, task_age.max(run_age), key, document) + }) + .collect(); + documents.sort_by(|left, right| right.0.cmp(&left.0).then(right.1.cmp(&left.1)).then_with(|| left.2.cmp(&right.2))); + documents.into_iter().map(|(_, _, _, document)| document).collect() +} + +/// The snapshot from the grouped task and run candidates and the oldest +/// missing run's time. +pub(super) fn snapshot(tasks: &[ScopeRow], runs: &[ScopeRow], oldest_run: Option>, now: DateTime) -> Value { + let task_count: usize = tasks.iter().map(|row| row.count).sum(); + let run_count: usize = runs.iter().map(|row| row.count).sum(); + let selected_tasks = selected(&buckets(tasks, SCAN_LIMIT), SCAN_LIMIT); + let selected_runs = selected(&buckets(runs, SCAN_LIMIT), SCAN_LIMIT); + let picked_tasks: usize = selected_tasks.iter().map(|(_, count)| count).sum(); + let picked_runs: usize = selected_runs.iter().map(|(_, count)| count).sum(); + let oldest_task = tasks.iter().filter_map(|row| row.oldest).min(); + json!({ + "existing_task_candidates": task_count, "missing_task_candidates": run_count, + "total_candidates": task_count + run_count, "scan_limit": SCAN_LIMIT, "scan_strategy": "scope_fair_round_robin", + "selected_existing_task_candidates": picked_tasks, "selected_missing_task_candidates": picked_runs, + "selected_total_candidates": picked_tasks + picked_runs, + "existing_task_scan_limit_reached": task_count >= SCAN_LIMIT, + "missing_task_scan_limit_reached": run_count >= SCAN_LIMIT, + "scan_pressure": task_count >= SCAN_LIMIT || run_count >= SCAN_LIMIT, + "oldest_task_candidate_created_at": super::task_queues::json_time(oldest_task), + "oldest_missing_run_started_at": super::task_queues::json_time(oldest_run), + "max_task_candidate_age_ms": age(oldest_task, now), "max_missing_run_age_ms": age(oldest_run, now), + "scopes": scopes(tasks, runs, &selected_tasks, &selected_runs, now), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn row(queue: Option<&str>, count: usize, minute: u32) -> ScopeRow { + let oldest = DateTime::parse_from_rfc3339(&format!("2026-10-21T12:{minute:02}:00Z")).unwrap().with_timezone(&Utc); + ScopeRow { connection: None, queue: queue.map(str::to_owned), compatibility: None, count, oldest: Some(oldest) } + } + + #[test] + fn selection_takes_one_candidate_from_each_scope_in_turn() { + // The oldest scope goes first; each bucket holds at most the limit. + let rows = [row(Some("busy"), 40, 5), row(Some("quiet"), 2, 1), row(None, 3, 9)]; + let buckets = buckets(&rows, SCAN_LIMIT); + assert_eq!(buckets, [("default:quiet:any".to_owned(), 2), ("default:busy:any".to_owned(), 25), ("default:default:any".to_owned(), 3)]); + let picked = selected(&buckets, SCAN_LIMIT); + assert_eq!(picked, [("default:quiet:any".to_owned(), 2), ("default:busy:any".to_owned(), 20), ("default:default:any".to_owned(), 3)]); + } + + #[test] + fn empty_and_default_values_share_a_scope_like_php() { + let rows = [row(None, 1, 3), row(Some(""), 2, 2), row(Some("default"), 4, 1)]; + // The literal `default` row is oldest, so it represents the scope and + // its exact value selects only its own rows. + assert_eq!(buckets(&rows, SCAN_LIMIT), [("default:default:any".to_owned(), 4)]); + let now = DateTime::parse_from_rfc3339("2026-10-21T12:10:00Z").unwrap().with_timezone(&Utc); + let document = snapshot(&rows, &[], None, now); + assert_eq!(document["existing_task_candidates"], 7); + assert_eq!(document["scopes"][0]["existing_task_candidates"], 7); + assert_eq!(document["scopes"][0]["scan_limited_by_global_policy"], true); + assert_eq!(document["oldest_task_candidate_created_at"], "2026-10-21T12:01:00.000000Z"); + assert_eq!(document["max_task_candidate_age_ms"], 540_000); + } +} diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 89fa3a9cd..58ad874c9 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -87,6 +87,8 @@ mod query_responses; mod history_export; #[path = "execution/history_retention.rs"] mod history_retention; +#[path = "execution/repair_status.rs"] +mod repair_status; #[path = "execution/activity_timeout_pass.rs"] mod activity_timeout_pass; #[path = "execution/start_request.rs"] diff --git a/rust/tests/execution/repair_status.rs b/rust/tests/execution/repair_status.rs new file mode 100644 index 000000000..cd0a2c76b --- /dev/null +++ b/rust/tests/execution/repair_status.rs @@ -0,0 +1,55 @@ +use super::*; + +async fn started_on(app: &Router, workflow: &str, queue: &str) -> Value { + let (status, body) = request(app, "POST", "/api/workflows", json!({"workflow_id":workflow,"workflow_type":"echo","task_queue":queue,"input":[]})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); + body +} + +// Published PHP's repair status reports its policy and the tasks and runs its +// repair pass would select, grouped by scope and picked round-robin. +#[tokio::test] +async fn repair_status_reports_candidates_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let (status, empty) = request(&app, "GET", "/api/system/repair", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{empty}"); + assert_eq!(empty["policy"]["redispatch_after_seconds"], 3); + assert_eq!((empty["candidates"]["total_candidates"].clone(), empty["candidates"]["scopes"].clone()), (json!(0), json!([]))); + + started_on(&app, "overdue-a1", "repair-a").await; + started_on(&app, "overdue-a2", "repair-a").await; + started_on(&app, "overdue-b", "repair-b").await; + let stalled = started_on(&app, "stalled", "repair-c").await; + // The ready tasks were made available long before the redispatch cutoff. + // (Laravel compares "now" in whole seconds, so a task made available in + // the current second is not yet due.) + database.execute("UPDATE workflow_tasks SET created_at='2020-01-02 03:04:05.000000',available_at='2020-01-02 03:04:05.000000',last_dispatched_at='2020-01-02 03:04:05.000000'").await; + // The task in queue repair-b is leased, but its lease expired. + database.execute("UPDATE workflow_tasks SET status='leased',lease_owner='gone',lease_expires_at='2020-01-02 03:04:05.000000' WHERE queue='repair-b'").await; + // The run in queue repair-c lost its task and needs repair. + let run = stalled["run_id"].as_str().unwrap(); + database.execute(sqlx::AssertSqlSafe(format!("DELETE FROM workflow_tasks WHERE workflow_run_id='{run}'"))).await; + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_run_summaries SET liveness_state='repair_needed',next_task_id=NULL,wait_started_at=NULL WHERE id='{run}'" + ))).await; + + let (status, body) = request(&app, "GET", "/api/system/repair", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let candidates = &body["candidates"]; + assert_eq!( + [&candidates["existing_task_candidates"], &candidates["missing_task_candidates"], &candidates["total_candidates"], &candidates["selected_total_candidates"]], + [&json!(3), &json!(1), &json!(4), &json!(4)], + "{candidates}" + ); + assert_eq!(candidates["oldest_task_candidate_created_at"], "2020-01-02T03:04:05.000000Z"); + assert!(candidates["max_task_candidate_age_ms"].as_i64().unwrap() > 0); + assert!(candidates["oldest_missing_run_started_at"].is_string(), "{candidates}"); + let scopes = candidates["scopes"].as_array().unwrap(); + // The busiest scope first. + assert_eq!(scopes[0]["scope_key"], "default:repair-a:any"); + assert_eq!((scopes[0]["existing_task_candidates"].clone(), scopes[0]["selected_total_candidates"].clone()), (json!(2), json!(2))); + let missing: Vec<_> = scopes.iter().filter(|scope| scope["missing_task_candidates"] == 1).collect(); + assert_eq!(missing.len(), 1, "{candidates}"); + assert_eq!(missing[0]["queue"], "repair-c"); +} From 2bc38eba0866997028bac3541f84cf5086fe8461 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 18:48:14 +0000 Subject: [PATCH 03/30] Repair workflow runs like published PHP Published dw 2.3.0's workflow:repair on a run whose ready task was overdue got native's 422 development_workflow_task_repair_unqualified, where PHP answered repair_dispatched. Native repaired only a worker-reported version-range blockage. Repair now follows WorkflowStub::attemptRepair. A closed run's repair is a durable rejected command with 409. Otherwise native projects the run's summary in the same transaction. When it says repair_needed, the recoverable task is redispatched: a ready task due for redispatch counts one more repair, and an expired workflow-task lease returns to ready. Any other active run records repair_not_needed. Both outcomes record PHP's command payload, RepairRequested event and response. The version-range repair keeps its existing qualified path. Creating a missing task and recovering an expired activity lease are still refused, and are recorded as known gaps. Co-Authored-By: Claude Opus 5.5 --- .../2026-10-21-workflow-repair.md | 55 ++++ docs/rust-server-port.md | 4 + rust/src/runtime/execution.rs | 41 +-- rust/src/runtime/http.rs | 20 +- rust/src/runtime/task_repair.rs | 295 +++++++++++++++--- rust/src/runtime/worker_incarnations.rs | 38 ++- rust/tests/execution.rs | 2 + rust/tests/execution/task_retry.rs | 27 +- rust/tests/execution/workflow_repair.rs | 68 ++++ 9 files changed, 426 insertions(+), 124 deletions(-) create mode 100644 docs/rust-server-port-checkpoints/2026-10-21-workflow-repair.md create mode 100644 rust/tests/execution/workflow_repair.rs diff --git a/docs/rust-server-port-checkpoints/2026-10-21-workflow-repair.md b/docs/rust-server-port-checkpoints/2026-10-21-workflow-repair.md new file mode 100644 index 000000000..51b0137eb --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-21-workflow-repair.md @@ -0,0 +1,55 @@ +# October 21, workflow repair + +- **Found by.** The published dw 2.3.0 CLI sweep. `workflow:repair` on a run + whose task was ready but overdue got native's + `422 development_workflow_task_repair_unqualified`; published PHP 2.5.14 + answered `repair_dispatched`. Native repaired only a worker-reported + version-range blockage. +- **Source.** `POST /api/workflows/{id}/repair` and its run-scoped route now + follow PHP's `WorkflowStub::attemptRepair`: + - **Closed run.** A repair of a closed run is a durable rejection: a + rejected command (`rejected_not_active`, `run_not_active`) and `409`, + with no history event. + - **Version-range blockage.** A run with a worker-reported version-range + blockage keeps native's existing qualified repair. + - **Any other active run.** Native projects the run's summary inside the + same transaction and decides from its liveness: + - **`repair_needed`:** the run's recoverable task is repaired, chosen as + PHP's `recoverableTask` chooses it. + - A ready task due for redispatch counts one more repair; this uses + PHP's dispatch-failure, claim-failure and overdue rules, with their + backoff. + - A workflow task whose lease expired returns to ready. + - The task is then marked dispatched, as PHP's poll-mode + `TaskDispatcher` does after commit, and the outcome is + `repair_dispatched`. + - **Otherwise:** the command records `repair_not_needed`. + - **Record.** Both accepted outcomes record PHP's command payload + (`liveness_state`, `wait_kind`, `task_id`, `task_type`) and a + `RepairRequested` event, which has a task snapshot only when a task was + dispatched. + - **Response.** The body is PHP's, with `command_sequence`, + `rejection_reason` and `validation_errors`, and without native's old + `namespace` field. +- **Still refused.** These keep + `development_workflow_task_repair_unqualified`: + - **A run with no task to recover.** PHP creates the missing task + (`TaskRepair::createMissingTask`). + - **An activity task whose lease expired.** PHP also closes the expired + attempt; native recovers that lease when a worker next claims it. + - **A named namespace.** Repairs outside the default namespace keep their + earlier refusal. +- **Tests changed.** Two version-range tests expected a second repair to be + refused. As in PHP, it is now accepted as `repair_not_needed` and records + a second command. +- **Verification.** + - **Integration test.** `workflow_repair` covers an overdue ready task, a + second repair, a healthy leased task, an expired workflow-task lease + reclaimed by the same worker, and a completed run's rejection. It passes + on SQLite, PostgreSQL and MySQL, and the full native suite passes on + SQLite. + - **Harness.** Three repairs were compared with published PHP 2.5.14: an + overdue run, the same run again, and a completed run. The responses were + identical apart from identifiers, control-plane envelope included, and + so were the `RepairRequested` events. + - **CLI.** dw 2.3.0's `workflow:repair` succeeds against both servers. diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 845c562bf..1d98f9d6c 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -102,6 +102,10 @@ contract fixture before its correction. each priority tier across fairness keys by their weights (`TaskFairnessScheduler`); native stores the keys and weights but does not rebalance yet. +- **Repair of a run without a recoverable task.** Published PHP's repair + creates a run's missing task, and recovers an expired activity-task lease + with its attempt. Native refuses both with + `development_workflow_task_repair_unqualified`. - **Key order inside plain JSON input.** For input that is not an envelope, PHP encodes nested objects as Avro maps in request order; native encodes them with sorted keys. The decoded values are equal, but the blobs differ. diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index 7692f867e..37f0e090c 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -496,47 +496,18 @@ impl Runtime { } } } - pub(crate) async fn repair_range_blocked( + pub(crate) async fn repair_workflow( &self, workflow_id: &str, run_id: Option<&str>, body: Value, context: Value, - ) -> Result { + ) -> Result<(StatusCode, Value)> { + let codec = self.transition_codec.clone(); match &*self.storage { - Storage::Sqlite(store) => { - store - .repair_range_blocked( - workflow_id, - run_id, - body, - context, - self.transition_codec.clone(), - ) - .await - } - Storage::Postgres(store) => { - store - .repair_range_blocked( - workflow_id, - run_id, - body, - context, - self.transition_codec.clone(), - ) - .await - } - Storage::MySql(store) => { - store - .repair_range_blocked( - workflow_id, - run_id, - body, - context, - self.transition_codec.clone(), - ) - .await - } + Storage::Sqlite(store) => store.repair_workflow(workflow_id, run_id, body, context, codec).await, + Storage::Postgres(store) => store.repair_workflow(workflow_id, run_id, body, context, codec).await, + Storage::MySql(store) => store.repair_workflow(workflow_id, run_id, body, context, codec).await, } } diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index f307045bb..a7befe7dd 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -841,17 +841,15 @@ async fn repair_current( Extension(runtime): Extension, Path(workflow_id): Path, request: Request, -) -> Result> { +) -> Result<(StatusCode, Json)> { let (body, context) = super::task_repair_http::request(request, &workflow_id).await?; - let response = runtime - .repair_range_blocked(&workflow_id, None, body, context) + let (status, response) = runtime + .repair_workflow(&workflow_id, None, body, context) .await .map_err(|error| { super::control_plane::Operation::Repair.refusal(error, &workflow_id, None) })?; - Ok(Json( - super::control_plane::Operation::Repair.response(response), - )) + Ok((status, Json(super::control_plane::Operation::Repair.response(response)))) } /// Laravel's `$request->boolean('include_last_event_payload')`. @@ -897,17 +895,15 @@ async fn repair_run( Extension(runtime): Extension, Path((workflow_id, run_id)): Path<(String, String)>, request: Request, -) -> Result> { +) -> Result<(StatusCode, Json)> { let (body, context) = super::task_repair_http::request(request, &workflow_id).await?; - let response = runtime - .repair_range_blocked(&workflow_id, Some(&run_id), body, context) + let (status, response) = runtime + .repair_workflow(&workflow_id, Some(&run_id), body, context) .await .map_err(|error| { super::control_plane::Operation::Repair.refusal(error, &workflow_id, Some(&run_id)) })?; - Ok(Json( - super::control_plane::Operation::Repair.response(response), - )) + Ok((status, Json(super::control_plane::Operation::Repair.response(response)))) } async fn cancel_run( diff --git a/rust/src/runtime/task_repair.rs b/rust/src/runtime/task_repair.rs index fd054ed42..5e09532cb 100644 --- a/rust/src/runtime/task_repair.rs +++ b/rust/src/runtime/task_repair.rs @@ -1,17 +1,77 @@ -//! Explicit recovery of a worker-reported version-range blockage. +//! `WorkflowStub::attemptRepair` for control-plane repair commands: a run +//! whose summary needs repair gets its task redispatched; any other active run +//! records that no repair was needed; a closed run's repair is a durable +//! rejection. use super::{ Result, RuntimeError, backend::Backend, refuse, store::{Store, id, now}, + transaction::Transaction, wire, }; use axum::http::StatusCode; +use chrono::{DateTime, Duration, Utc}; use serde_json::{Value, json}; use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; use std::sync::Arc; use tokio::sync::Semaphore; +/// `TaskRepairPolicy::readyTaskNeedsRedispatch` or `leaseExpired` for one +/// task row, as of `at`. +fn needs_recovery(task: &DB::Row, at: DateTime) -> Result +where + usize: ColumnIndex, + for<'r> i64: Decode<'r, DB>, +{ + let status = DB::string(task, "status")?; + if status == "leased" { + return Ok(DB::optional_instant(task, "lease_expires_at")?.is_some_and(|expires| expires <= at)); + } + if status != "ready" { + return Ok(false); + } + let error = |field: &str| -> Result { + Ok(DB::optional_string(task, field)?.is_some_and(|error| !error.trim().is_empty())) + }; + let attempted = DB::optional_instant(task, "last_dispatch_attempt_at")?; + let dispatched = DB::optional_instant(task, "last_dispatched_at")?; + let claim_failed_at = DB::optional_instant(task, "last_claim_failed_at")?; + let backoff = DB::optional_instant(task, "repair_available_at")?; + let cutoff = at - Duration::seconds(super::operator_metrics::REDISPATCH_AFTER_SECONDS); + let dispatch_failed = attempted.is_some_and(|attempted| dispatched.is_none_or(|dispatched| attempted > dispatched)) + && error("last_dispatch_error")?; + let claim_failed = claim_failed_at.is_some() && error("last_claim_error")?; + if dispatch_failed { + return Ok(backoff.is_none_or(|backoff| backoff <= at)); + } + if claim_failed { + return Ok(match backoff { + Some(backoff) => backoff <= at, + None => claim_failed_at.is_some_and(|failed| failed <= cutoff), + }); + } + let available = DB::optional_instant(task, "available_at")?; + let reference = dispatched.or(DB::optional_instant(task, "created_at")?); + Ok(available.is_none_or(|available| available <= at) && reference.is_some_and(|reference| reference <= cutoff)) +} + +/// `recoverableTask`'s order: earliest available, then created, then id. +fn recovery_order(task: &DB::Row) -> Result<(i64, i64, String)> +where + usize: ColumnIndex, + for<'r> i64: Decode<'r, DB>, +{ + let millis = |field: &str| -> Result { + Ok(DB::optional_instant(task, field)?.map_or(i64::MAX, |at| at.timestamp_millis())) + }; + Ok((millis("available_at")?, millis("created_at")?, DB::string(task, "id")?)) +} + +fn unqualified() -> RuntimeError { + refuse(StatusCode::UNPROCESSABLE_ENTITY, "development_workflow_task_repair_unqualified") +} + impl Store where for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, @@ -24,14 +84,14 @@ where for<'r> i64: Decode<'r, DB>, usize: ColumnIndex, { - pub(crate) async fn repair_range_blocked( + pub(crate) async fn repair_workflow( &self, workflow_id: &str, requested_run_id: Option<&str>, body: Value, context: Value, codec: Arc, - ) -> Result { + ) -> Result<(StatusCode, Value)> { if !body.is_object() || body.get("request_id").is_some_and(|value| { !value.is_null() && value.as_str().is_none_or(|id| id.chars().count() > 255) @@ -63,98 +123,241 @@ where }), }); } + let current = Self::query("SELECT * FROM workflow_runs WHERE id=$1") + .bind(&run_id) + .fetch_one(&mut *tx) + .await?; + if !matches!(DB::string(¤t, "status")?.as_str(), "pending" | "running" | "waiting") { + return Self::reject_repair(tx, workflow_id, ¤t, &context).await; + } let run = Self::active_run(&mut tx, &run_id).await?; - let task = Self::query(DB::RANGE_REPAIR_TASK_SQL) + if let Some(task) = Self::query(DB::RANGE_REPAIR_TASK_SQL) .bind(&run_id) .bind(self.namespace.as_ref()) .fetch_optional(&mut *tx) .await? - .ok_or_else(|| { - refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "development_workflow_task_repair_unqualified", - ) - })?; + { + let response = Self::repair_replay_blocked(&mut tx, workflow_id, &run, &task, &context, codec).await?; + tx.commit().await?; + self.wake.notify_waiters(); + return Ok((StatusCode::OK, response)); + } + // PHP projects the run, then repairs only what its summary says needs it. + Self::project_run_summary(&mut tx, &run_id).await?; + let summary = Self::query("SELECT liveness_state,wait_kind FROM workflow_run_summaries WHERE id=$1") + .bind(&run_id) + .fetch_optional(&mut *tx) + .await?; + let field = |name: &str| -> Result> { + summary.as_ref().map(|row| DB::optional_string(row, name)).transpose().map(Option::flatten) + }; + let (liveness, wait_kind) = (field("liveness_state")?, field("wait_kind")?); + let task = if liveness.as_deref() == Some("repair_needed") { + Some(Self::recover_task(&mut tx, &run_id).await?) + } else { + None + }; + let task_type = task.as_ref().map(|task| DB::string(task, "task_type")).transpose()?; + let identity = task.as_ref().map(|task| DB::string(task, "id")).transpose()?.zip(task_type); + let payload = super::worker_incarnations::repair_command_payload(codec, liveness.clone(), wait_kind.clone(), identity).await?; + let recorded = Repair { liveness: liveness.as_deref(), wait_kind: wait_kind.as_deref(), payload: &payload, task: task.as_ref() }; + let response = Self::record_repair(&mut tx, workflow_id, &run, &context, recorded).await?; + if let Some(task) = &task { + // `TaskDispatcher::markDispatched`, which PHP runs after the commit. + let at = now(); + Self::query("UPDATE workflow_tasks SET last_dispatch_attempt_at=$1,last_dispatched_at=$2,last_dispatch_error=NULL,repair_available_at=NULL WHERE id=$3 AND status='ready'") + .bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(DB::string(task, "id")?).execute(&mut *tx).await?; + } + tx.commit().await?; + self.wake.notify_waiters(); + Ok((StatusCode::OK, response)) + } + + /// `TaskRepair::repairRun` for an existing task: a ready task due for + /// redispatch counts one more repair, and a workflow task whose lease + /// expired returns to ready. Creating a missing task, and recovering an + /// expired activity lease, are not ported. + async fn recover_task(tx: &mut Transaction<'_, DB>, run_id: &str) -> Result { + let at = now(); + let tasks = Self::query("SELECT * FROM workflow_tasks WHERE workflow_run_id=$1 AND status IN ('ready','leased')") + .bind(run_id) + .fetch_all(&mut **tx) + .await?; + let mut recoverable = Vec::new(); + for task in tasks { + if needs_recovery::(&task, at)? { + recoverable.push((recovery_order::(&task)?, task)); + } + } + recoverable.sort_by(|left, right| left.0.cmp(&right.0)); + let (_, task) = recoverable.into_iter().next().ok_or_else(unqualified)?; + let task_id = DB::string(&task, "id")?; + let repair_count = DB::number(&task, "repair_count")? + .checked_add(1) + .filter(|count| *count <= DB::REPAIR_COUNTER_MAX) + .ok_or_else(|| refuse(StatusCode::CONFLICT, "repair_count_exhausted"))?; + if DB::string(&task, "status")? == "leased" { + if DB::string(&task, "task_type")? != "workflow" { + return Err(unqualified()); + } + Self::query("UPDATE workflow_tasks SET status='ready',leased_at=NULL,lease_owner=NULL,lease_expires_at=NULL,repair_count=$1,repair_available_at=NULL,last_error=NULL,updated_at=$2 WHERE id=$3") + .bind(repair_count).bind(DB::bind_time(at)).bind(&task_id).execute(&mut **tx).await?; + } else { + Self::query("UPDATE workflow_tasks SET repair_count=$1,repair_available_at=NULL,last_error=NULL,updated_at=$2 WHERE id=$3") + .bind(repair_count).bind(DB::bind_time(at)).bind(&task_id).execute(&mut **tx).await?; + } + Ok(Self::query("SELECT * FROM workflow_tasks WHERE id=$1").bind(&task_id).fetch_one(&mut **tx).await?) + } + + /// Explicit recovery of a worker-reported version-range blockage. + async fn repair_replay_blocked( + tx: &mut Transaction<'_, DB>, + workflow_id: &str, + run: &DB::Row, + task: &DB::Row, + context: &Value, + codec: Arc, + ) -> Result { + let run_id = DB::string(run, "id")?; let signals = Self::scalar("SELECT COUNT(*) FROM workflow_signal_records WHERE workflow_run_id=$1 AND status='received'") - .bind(&run_id).fetch_one(&mut *tx).await?; + .bind(&run_id).fetch_one(&mut **tx).await?; let updates = Self::scalar( "SELECT COUNT(*) FROM workflow_updates WHERE workflow_run_id=$1 AND status='accepted'", ) .bind(&run_id) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await?; - if signals != 0 || updates != 0 || Self::cancellation_request(&run)?.is_some() { + if signals != 0 || updates != 0 || Self::cancellation_request(run)?.is_some() { // Message/cancellation repair needs its own shared contract. return Err(refuse( StatusCode::UNPROCESSABLE_ENTITY, "development_workflow_task_repair_active_work_unqualified", )); } - let wait_kind = Self::range_repair_wait(&mut tx, &run_id).await?; - let task_id = DB::string(&task, "id")?; - let repair_count = DB::number(&task, "repair_count")? + let wait_kind = Self::range_repair_wait(tx, &run_id).await?; + let task_id = DB::string(task, "id")?; + let repair_count = DB::number(task, "repair_count")? .checked_add(1) .filter(|count| *count <= DB::REPAIR_COUNTER_MAX) .ok_or_else(|| refuse(StatusCode::CONFLICT, "repair_count_exhausted"))?; - let sequence = DB::number(&run, "last_command_sequence")? - .checked_add(1) - .filter(|count| *count <= DB::REPAIR_COUNTER_MAX) - .ok_or_else(|| refuse(StatusCode::CONFLICT, "command_sequence_exhausted"))?; - let command_payload = super::worker_incarnations::repair_payload( + let payload = super::worker_incarnations::repair_payload( codec, task_id.clone(), "workflow_replay_blocked", wait_kind, ) .await?; - let mut task_payload = DB::document_row(&task, "payload")?; + let mut task_payload = DB::document_row(task, "payload")?; task_payload .as_object_mut() .ok_or_else(|| refuse(StatusCode::CONFLICT, "invalid_workflow_task_payload"))? .retain(|key, _| !key.starts_with("replay_blocked")); + Self::query("UPDATE workflow_tasks SET status='ready',payload=$1,leased_at=NULL,lease_owner=NULL,lease_expires_at=NULL,repair_count=$2,repair_available_at=NULL,last_error=NULL,updated_at=$3 WHERE id=$4") + .bind(DB::document(&task_payload)).bind(repair_count).bind(DB::bind_time(now())).bind(&task_id).execute(&mut **tx).await?; + let repaired = Self::query("SELECT * FROM workflow_tasks WHERE id=$1") + .bind(&task_id) + .fetch_one(&mut **tx) + .await?; + let recorded = Repair { liveness: Some("workflow_replay_blocked"), wait_kind, payload: &payload, task: Some(&repaired) }; + Self::record_repair(tx, workflow_id, run, context, recorded).await + } + + /// The accepted repair command, its `RepairRequested` event and PHP's + /// response. + async fn record_repair( + tx: &mut Transaction<'_, DB>, + workflow_id: &str, + run: &DB::Row, + context: &Value, + repair: Repair<'_, DB::Row>, + ) -> Result { + let run_id = DB::string(run, "id")?; + let sequence = DB::number(run, "last_command_sequence")? + .checked_add(1) + .filter(|count| *count <= DB::REPAIR_COUNTER_MAX) + .ok_or_else(|| refuse(StatusCode::CONFLICT, "command_sequence_exhausted"))?; + let outcome = if repair.task.is_some() { "repair_dispatched" } else { "repair_not_needed" }; let timestamp = now(); let command_id = id(); - Self::query("UPDATE workflow_tasks SET status='ready',payload=$1,leased_at=NULL,lease_owner=NULL,lease_expires_at=NULL,repair_count=$2,repair_available_at=NULL,last_error=NULL,updated_at=$3 WHERE id=$4") - .bind(DB::document(&task_payload)).bind(repair_count).bind(DB::bind_time(timestamp)).bind(&task_id).execute(&mut *tx).await?; // PHP guards a selected run then records an instance-targeted repair. // Preserve that command identity rather than treating the URL as scope. - Self::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,command_type,source,target_scope,status,outcome,workflow_type,payload_codec,payload,command_sequence,accepted_at,applied_at,context) VALUES ($1,$2,$3,$4,'repair','control_plane','instance','accepted','repair_dispatched',$5,'avro',$6,$7,$8,$9,$10)") - .bind(&command_id).bind(workflow_id).bind(&run_id).bind(&run_id).bind(DB::string(&run, "workflow_type")?) - .bind(&command_payload).bind(sequence).bind(DB::bind_time(timestamp)).bind(DB::bind_time(timestamp)) - .bind(DB::document(&context)).execute(&mut *tx).await?; + Self::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,command_type,source,target_scope,status,outcome,workflow_type,payload_codec,payload,command_sequence,accepted_at,applied_at,context) VALUES ($1,$2,$3,$4,'repair','control_plane','instance','accepted',$5,$6,'avro',$7,$8,$9,$10,$11)") + .bind(&command_id).bind(workflow_id).bind(&run_id).bind(&run_id).bind(outcome).bind(DB::string(run, "workflow_type")?) + .bind(repair.payload).bind(sequence).bind(DB::bind_time(timestamp)).bind(DB::bind_time(timestamp)) + .bind(DB::document(context)).execute(&mut **tx).await?; Self::query( "UPDATE workflow_runs SET last_command_sequence=$1,last_progress_at=$2 WHERE id=$3", ) .bind(sequence) .bind(DB::bind_time(timestamp)) .bind(&run_id) - .execute(&mut *tx) + .execute(&mut **tx) .await?; - let repaired = Self::query("SELECT * FROM workflow_tasks WHERE id=$1") - .bind(&task_id) - .fetch_one(&mut *tx) - .await?; - Self::append(&mut tx, &run_id, "RepairRequested", json!({ + let task_id = repair.task.map(|task| DB::string(task, "id")).transpose()?; + let task_type = repair.task.map(|task| DB::string(task, "task_type")).transpose()?; + let mut event = json!({ "workflow_command_id":command_id,"workflow_instance_id":workflow_id,"workflow_run_id":run_id, - "command_type":"repair","outcome":"repair_dispatched","liveness_state":"workflow_replay_blocked", - "wait_kind":wait_kind,"task_id":task_id,"task_type":"workflow","task":Self::task_snapshot(&repaired)?, + "command_type":"repair","outcome":outcome,"liveness_state":repair.liveness, + "wait_kind":repair.wait_kind,"task_id":task_id,"task_type":task_type, "command":{"id":command_id,"sequence":sequence,"type":"repair","target_scope":"instance", - "resolved_run_id":run_id,"status":"accepted","outcome":"repair_dispatched", - "source":"control_plane","payload_codec":"avro","payload":wire(&command_payload), + "resolved_run_id":run_id,"status":"accepted","outcome":outcome, + "source":"control_plane","payload_codec":"avro","payload":wire(repair.payload), "context":{"principal":context["principal"]}, "caller_label":context["caller"]["label"],"auth_status":context["auth"]["status"], "auth_method":context["auth"]["method"],"principal_type":context["principal"]["type"], "principal_id":context["principal"]["id"],"principal_label":context["principal"]["label"], "request_method":context["request"]["method"],"request_path":context["request"]["path"], "request_fingerprint":context["request"]["fingerprint"],"accepted_at":timestamp,"applied_at":timestamp} - }), Some(&task_id), Some(&command_id)).await?; + }); + if let Some(task) = repair.task { + event["task"] = Self::task_snapshot(task)?; + } + Self::append(tx, &run_id, "RepairRequested", event, task_id.as_deref(), Some(&command_id)).await?; + Ok(json!({"workflow_id":workflow_id,"run_id":run_id,"workflow_type":DB::string(run, "workflow_type")?, + "command_id":command_id,"command_sequence":sequence,"command_status":"accepted", + "command_source":"control_plane","outcome":outcome,"requested_run_id":null, + "resolved_run_id":run_id,"target_scope":"instance","reason":null,"rejection_reason":null, + "validation_errors":[]})) + } + + /// PHP's `rejectCommand` for a closed run: a durable rejected command and + /// a 409, without a history event. + async fn reject_repair( + mut tx: Transaction<'_, DB>, + workflow_id: &str, + run: &DB::Row, + context: &Value, + ) -> Result<(StatusCode, Value)> { + let run_id = DB::string(run, "id")?; + let sequence = DB::number(run, "last_command_sequence")? + .checked_add(1) + .filter(|count| *count <= DB::REPAIR_COUNTER_MAX) + .ok_or_else(|| refuse(StatusCode::CONFLICT, "command_sequence_exhausted"))?; + let command_id = id(); + Self::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,command_type,source,target_scope,status,outcome,rejection_reason,workflow_type,payload_codec,command_sequence,rejected_at,context) VALUES ($1,$2,$3,$4,'repair','control_plane','instance','rejected','rejected_not_active','run_not_active',$5,'avro',$6,$7,$8)") + .bind(&command_id).bind(workflow_id).bind(&run_id).bind(&run_id).bind(DB::string(run, "workflow_type")?) + .bind(sequence).bind(DB::bind_time(now())).bind(DB::document(context)).execute(&mut *tx).await?; + Self::query("UPDATE workflow_runs SET last_command_sequence=$1 WHERE id=$2") + .bind(sequence) + .bind(&run_id) + .execute(&mut *tx) + .await?; tx.commit().await?; - self.wake.notify_waiters(); - Ok( - json!({"workflow_id":workflow_id,"run_id":run_id,"workflow_type":DB::string(&run, "workflow_type")?, - "namespace":self.namespace.as_ref(),"command_id":command_id,"command_status":"accepted", - "command_source":"control_plane","outcome":"repair_dispatched","requested_run_id":null, - "resolved_run_id":run_id,"target_scope":"instance","reason":null}), - ) + Ok(( + StatusCode::CONFLICT, + json!({"accepted":false,"workflow_id":workflow_id,"run_id":run_id,"workflow_type":DB::string(run, "workflow_type")?, + "command_id":command_id,"command_sequence":sequence,"command_status":"rejected", + "command_source":"control_plane","outcome":"rejected_not_active","requested_run_id":null, + "resolved_run_id":run_id,"target_scope":"instance","reason":"run_not_active", + "rejection_reason":"run_not_active","validation_errors":[]}), + )) } } + +/// What one accepted repair records. +struct Repair<'a, Row> { + liveness: Option<&'a str>, + wait_kind: Option<&'a str>, + payload: &'a str, + task: Option<&'a Row>, +} diff --git a/rust/src/runtime/worker_incarnations.rs b/rust/src/runtime/worker_incarnations.rs index 056d31517..5f567689b 100644 --- a/rust/src/runtime/worker_incarnations.rs +++ b/rust/src/runtime/worker_incarnations.rs @@ -63,6 +63,23 @@ pub(super) async fn repair_payload( task_id: String, liveness_state: &'static str, wait_kind: Option<&'static str>, +) -> Result { + repair_command_payload( + codec, + Some(liveness_state.to_owned()), + wait_kind.map(str::to_owned), + Some((task_id, "workflow".to_owned())), + ) + .await +} + +/// A repair command's payload: the run's liveness and wait kind, and the +/// task the repair dispatched, each null when absent. +pub(super) async fn repair_command_payload( + codec: Arc, + liveness_state: Option, + wait_kind: Option, + task: Option<(String, String)>, ) -> Result { let permit = codec .acquire_owned() @@ -72,22 +89,13 @@ pub(super) async fn repair_payload( let _permit = permit; let codec = crate::codec::ValueCodec::new() .map_err(|_| refuse(StatusCode::SERVICE_UNAVAILABLE, "repair_codec_unavailable"))?; + let text = |value: Option| value.map_or(crate::codec::Value::Null, crate::codec::Value::String); + let (task_id, task_type) = task.unzip(); let value = crate::codec::Value::Map(std::collections::BTreeMap::from([ - ( - "liveness_state".into(), - crate::codec::Value::String(liveness_state.into()), - ), - ( - "wait_kind".into(), - wait_kind.map_or(crate::codec::Value::Null, |kind| { - crate::codec::Value::String(kind.into()) - }), - ), - ("task_id".into(), crate::codec::Value::String(task_id)), - ( - "task_type".into(), - crate::codec::Value::String("workflow".into()), - ), + ("liveness_state".into(), text(liveness_state)), + ("wait_kind".into(), text(wait_kind)), + ("task_id".into(), text(task_id)), + ("task_type".into(), text(task_type)), ])); codec .encode(&value) diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 89fa3a9cd..56d1be34d 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -85,6 +85,8 @@ mod read_snapshots; mod query_responses; #[path = "execution/history_export.rs"] mod history_export; +#[path = "execution/workflow_repair.rs"] +mod workflow_repair; #[path = "execution/history_retention.rs"] mod history_retention; #[path = "execution/activity_timeout_pass.rs"] diff --git a/rust/tests/execution/task_retry.rs b/rust/tests/execution/task_retry.rs index 17e714569..ee50b7866 100644 --- a/rust/tests/execution/task_retry.rs +++ b/rust/tests/execution/task_retry.rs @@ -553,10 +553,10 @@ async fn worker_reported_range_failure_requires_explicit_original_task_repair_ac ("task_type".into(), Payload::String("workflow".into())), ])) ); - assert_eq!( - request(&app, "POST", &repair_path, json!({})).await.0, - StatusCode::UNPROCESSABLE_ENTITY - ); + // As in PHP, repairing again records that no repair was needed. + let again = request(&app, "POST", &repair_path, json!({})).await; + assert_eq!(again.0, StatusCode::OK, "{}", again.1); + assert_eq!(again.1["outcome"], "repair_not_needed"); assert_eq!( report(&app, &task, body.clone()).await.0, StatusCode::CONFLICT @@ -621,24 +621,19 @@ async fn two_nodes_commit_one_range_repair_with_server_derived_audit_identity() request(&a, "POST", &path, spoof), request(&b, "POST", &path, json!({})) ); - let (accepted, refused) = if left.0 == StatusCode::OK { - (left, right) - } else { - (right, left) - }; - assert_eq!(accepted.0, StatusCode::OK, "{}", accepted.1); - assert_eq!(refused.0, StatusCode::UNPROCESSABLE_ENTITY, "{}", refused.1); - assert_eq!( - refused.1["reason"], - "development_workflow_task_repair_unqualified" - ); + // One repair dispatches the task. As in PHP, the other then finds + // nothing to repair and records that. + assert_eq!((left.0, right.0), (StatusCode::OK, StatusCode::OK), "{} {}", left.1, right.1); + let mut outcomes = [left.1["outcome"].clone(), right.1["outcome"].clone()]; + outcomes.sort_by_key(|outcome| outcome.to_string()); + assert_eq!(outcomes, [json!("repair_dispatched"), json!("repair_not_needed")]); assert_eq!( rows( &database, "SELECT COUNT(*) FROM workflow_commands WHERE command_type='repair'" ) .await, - 1 + 2 ); assert_eq!( rows( diff --git a/rust/tests/execution/workflow_repair.rs b/rust/tests/execution/workflow_repair.rs new file mode 100644 index 000000000..53ec3ae10 --- /dev/null +++ b/rust/tests/execution/workflow_repair.rs @@ -0,0 +1,68 @@ +use super::*; + +const OLD: &str = "2020-01-02 03:04:05.000000"; + +async fn repair(app: &Router, workflow: &str) -> (StatusCode, Value) { + request(app, "POST", &format!("/api/workflows/{workflow}/repair"), json!([])).await +} + +async fn repairs(app: &Router, started: &Value) -> Vec { + let history = run_history(app, &started["workflow_id"], &started["run_id"]).await; + history.as_array().unwrap().iter().filter(|event| event["event_type"] == "RepairRequested").cloned().collect() +} + +// Published PHP repairs only what the run's summary says needs repair: +// an overdue ready task is redispatched, an expired workflow-task lease +// returns to ready, a healthy run records that no repair was needed, and a +// closed run's repair is a durable rejection. +#[tokio::test] +async fn repair_follows_the_run_summary_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let overdue = start(&app, "repair-overdue").await; + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_tasks SET available_at='{OLD}',created_at='{OLD}',last_dispatched_at='{OLD}',last_dispatch_attempt_at='{OLD}'" + ))).await; + let (status, dispatched) = repair(&app, "repair-overdue").await; + assert_eq!(status, StatusCode::OK, "{dispatched}"); + assert_eq!((dispatched["outcome"].clone(), dispatched["command_status"].clone()), (json!("repair_dispatched"), json!("accepted"))); + assert_eq!(dispatched["control_plane"]["operation"], "repair"); + let event = &repairs(&app, &overdue).await[0]["payload"]; + assert_eq!((event["liveness_state"].clone(), event["wait_kind"].clone()), (json!("repair_needed"), json!("workflow-task")), "{event}"); + assert_eq!((event["task_type"].clone(), event["task"]["repair_count"].clone()), (json!("workflow"), json!(1)), "{event}"); + // The repair dispatched the task, so it is no longer overdue. + let (status, again) = repair(&app, "repair-overdue").await; + assert_eq!((status, again["outcome"].clone()), (StatusCode::OK, json!("repair_not_needed")), "{again}"); + let second = &repairs(&app, &overdue).await[1]["payload"]; + assert!(second["task_id"].is_null() && second.get("task").is_none(), "{second}"); + + // The overdue run completes. + register(&app, "worker", json!(["echo"]), json!([])).await; + let task = poll(&app, "worker", "workflow").await; + assert_eq!(task["workflow_id"], "repair-overdue", "{task}"); + let complete = json!({"type":"complete_workflow","result":envelope(Payload::Long(1))}); + assert_eq!(finish_task(&app, &task, json!([complete])).await.0, StatusCode::OK); + + // A worker claims the next run's task, then its lease expires. + let leased = start(&app, "repair-leased").await; + let task = poll(&app, "worker", "workflow").await; + assert_eq!(task["workflow_id"], "repair-leased", "{task}"); + let (status, healthy) = repair(&app, "repair-leased").await; + assert_eq!((status, healthy["outcome"].clone()), (StatusCode::OK, json!("repair_not_needed")), "{healthy}"); + let run = leased["run_id"].as_str().unwrap(); + database.execute(sqlx::AssertSqlSafe(format!("UPDATE workflow_tasks SET lease_expires_at='{OLD}' WHERE workflow_run_id='{run}'"))).await; + let (status, recovered) = repair(&app, "repair-leased").await; + assert_eq!((status, recovered["outcome"].clone()), (StatusCode::OK, json!("repair_dispatched")), "{recovered}"); + let reclaimed = poll(&app, "worker", "workflow").await; + assert_eq!(reclaimed["task_id"], task["task_id"], "{reclaimed}"); + + // A closed run's repair is recorded as rejected, without a history event. + let (status, rejected) = repair(&app, "repair-overdue").await; + assert_eq!(status, StatusCode::CONFLICT, "{rejected}"); + assert_eq!( + [&rejected["accepted"], &rejected["command_status"], &rejected["outcome"], &rejected["reason"]], + [&json!(false), &json!("rejected"), &json!("rejected_not_active"), &json!("run_not_active")] + ); + assert!(rejected["command_id"].is_string()); + assert_eq!(repairs(&app, &overdue).await.len(), 2, "two accepted repairs, no event for the rejection"); +} From 0497b64b62ae6ad0efe3a286ffe4638f4628a0fc Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 18:59:15 +0000 Subject: [PATCH 04/30] Read PHP's legacy authentication settings The Sample App's unchanged server environment sets WORKFLOW_SERVER_AUTH_TOKEN. Native required DW_AUTH_TOKEN, and refused to start when one of the legacy role-token names WORKFLOW_SERVER_WORKER_TOKEN, WORKFLOW_SERVER_OPERATOR_TOKEN or WORKFLOW_SERVER_ADMIN_TOKEN was set. Published PHP reads each through EnvAuditor::env: the DW_* name when it is set, otherwise the legacy name. Native now does the same. The checkpoint compares all 49 settings PHP's config reads with native's, and records the ones native ignores but that change behavior as a known gap. Co-Authored-By: Claude Opus 5.5 --- .../2026-10-21-legacy-auth-env.md | 40 ++++++++++++++ docs/rust-server-port.md | 7 +++ rust/src/main.rs | 52 ++++++++++++++++--- 3 files changed, 92 insertions(+), 7 deletions(-) create mode 100644 docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md diff --git a/docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md b/docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md new file mode 100644 index 000000000..d7e39d898 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md @@ -0,0 +1,40 @@ +# October 21, legacy authentication settings + +- **Found by.** Preparing to run the Sample App's polyglot stack against + native. Its unchanged server environment sets `WORKFLOW_SERVER_AUTH_TOKEN`. + Native required `DW_AUTH_TOKEN`, and refused to start when one of the + legacy role-token names `WORKFLOW_SERVER_WORKER_TOKEN`, + `WORKFLOW_SERVER_OPERATOR_TOKEN` or `WORKFLOW_SERVER_ADMIN_TOKEN` was set. + Published PHP reads each of these through `EnvAuditor::env`: the `DW_*` + name when it is set, otherwise the legacy name. +- **Source.** Native now reads the auth token and the three role tokens the + same way. A set primary wins even when it is empty, as PHP's `env()` + returns it. The provider, principal-token and runtime-credential settings + keep their existing refusals. +- **Environment comparison.** Published PHP 2.5.14's `config/*.php` read 49 + settings through `EnvAuditor::env`. Native reads 16 of them, together with + their legacy names. The rest fall into four groups: + - **Refused when set.** Authentication drivers other than `token` are + refused at start, so their signature keys and headers cannot take effect + silently. + - **Settings native ignores but that change behavior:** + - `DW_DEFAULT_NAMESPACE`; + - `DW_MAX_HISTORY_EVENTS`, `DW_MAX_PENDING_ACTIVITIES`, + `DW_MAX_PENDING_CHILDREN` and `DW_MAX_TASKS_PER_POLL`; + - `DW_ACTIVITY_TASK_TIMEOUT` and `DW_WORKER_PROTOCOL_VERSION`. + + Native uses PHP's defaults for these. A deployment that changed one + would get different behavior from native without notice. This is + recorded as a known gap; native should refuse a value it cannot honor. + - **PHP process roles and topology.** `DW_MODE`, `DW_SERVER_ID`, + `DW_SERVER_PROCESS_CLASS`, `DW_SERVER_TOPOLOGY_SHAPE`, + `DW_TASK_DISPATCH_MODE`, the poll interval settings and the + update-validation settings describe PHP's separate processes. Native + runs one process type. + - **Features native does not have.** Response compression, package + provenance, the external executor configuration and storage admission + files. +- **Verification.** A unit test covers the precedence: the primary wins, the + legacy name fills in, a set but empty primary still wins, and neither + gives nothing. The Sample App's polyglot run against native uses the + legacy token name unchanged. diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 845c562bf..2338ac261 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -102,6 +102,13 @@ contract fixture before its correction. each priority tier across fairness keys by their weights (`TaskFairnessScheduler`); native stores the keys and weights but does not rebalance yet. +- **PHP settings native ignores.** Native uses PHP's defaults for + `DW_DEFAULT_NAMESPACE`, `DW_MAX_HISTORY_EVENTS`, + `DW_MAX_PENDING_ACTIVITIES`, `DW_MAX_PENDING_CHILDREN`, + `DW_MAX_TASKS_PER_POLL`, `DW_ACTIVITY_TASK_TIMEOUT` and + `DW_WORKER_PROTOCOL_VERSION`, and does not read their configured values. + A deployment that changed one of them would get different behavior from + native. Native should refuse to start on a value it cannot honor. - **Key order inside plain JSON input.** For input that is not an envelope, PHP encodes nested objects as Avro maps in request order; native encodes them with sorted keys. The decoded values are equal, but the blobs differ. diff --git a/rust/src/main.rs b/rust/src/main.rs index 22003f39f..a8fe71684 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -81,6 +81,22 @@ fn auth_value(key: &str) -> Result, Box> { } } +/// `EnvAuditor::env`: the primary name when it is set, else PHP's legacy +/// `WORKFLOW_SERVER_*` name. +fn aliased(lookup: F, primary: &str, legacy: &str) -> Result, Box> +where + F: Fn(&str) -> Result, Box>, +{ + match lookup(primary)? { + Some(value) => Ok(Some(value)), + None => lookup(legacy), + } +} + +fn auth_setting(primary: &str, legacy: &str) -> Result, Box> { + aliased(auth_value, primary, legacy) +} + fn validate_auth_configuration() -> Result<(), Box> { // Unqualified credentials must not silently leave a full-access legacy // guard in place. These profiles require their own contract qualification. @@ -96,9 +112,6 @@ fn validate_auth_configuration() -> Result<(), Box> { "DW_PRINCIPAL_TOKENS", "WORKFLOW_SERVER_AUTH_PROVIDER", "WORKFLOW_SERVER_PRINCIPAL_TOKENS", - "WORKFLOW_SERVER_WORKER_TOKEN", - "WORKFLOW_SERVER_OPERATOR_TOKEN", - "WORKFLOW_SERVER_ADMIN_TOKEN", ] { if auth_value(key)?.is_some_and(|value| !value.is_empty()) { return Err("unsupported development authentication configuration".into()); @@ -406,10 +419,13 @@ async fn main() -> Result<(), Box> { return Ok(()); } validate_auth_configuration()?; - let worker_token = auth_value("DW_WORKER_TOKEN")?; - let operator_token = auth_value("DW_OPERATOR_TOKEN")?; - let admin_token = auth_value("DW_ADMIN_TOKEN")?; - let token = auth_value("DW_AUTH_TOKEN")?.ok_or("DW_AUTH_TOKEN is required")?; + // Published PHP reads each token under its `DW_*` name, or else its + // legacy `WORKFLOW_SERVER_*` name, so an unchanged deployment starts. + let worker_token = auth_setting("DW_WORKER_TOKEN", "WORKFLOW_SERVER_WORKER_TOKEN")?; + let operator_token = auth_setting("DW_OPERATOR_TOKEN", "WORKFLOW_SERVER_OPERATOR_TOKEN")?; + let admin_token = auth_setting("DW_ADMIN_TOKEN", "WORKFLOW_SERVER_ADMIN_TOKEN")?; + let token = auth_setting("DW_AUTH_TOKEN", "WORKFLOW_SERVER_AUTH_TOKEN")? + .ok_or("DW_AUTH_TOKEN (or WORKFLOW_SERVER_AUTH_TOKEN) is required")?; if token.trim().is_empty() { return Err("DW_AUTH_TOKEN must not be empty".into()); } @@ -471,3 +487,25 @@ async fn main() -> Result<(), Box> { runtime.close().await; Ok(()) } + +#[cfg(test)] +mod auth_env_tests { + use super::*; + + fn lookup(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Result, Box> { + move |key| Ok(values.iter().find(|(name, _)| *name == key).map(|(_, value)| (*value).to_owned())) + } + + #[test] + fn primary_names_win_and_legacy_names_fill_in_like_php() { + let both = lookup(&[("DW_AUTH_TOKEN", "primary"), ("WORKFLOW_SERVER_AUTH_TOKEN", "legacy")]); + assert_eq!(aliased(both, "DW_AUTH_TOKEN", "WORKFLOW_SERVER_AUTH_TOKEN").unwrap().as_deref(), Some("primary")); + let legacy = lookup(&[("WORKFLOW_SERVER_AUTH_TOKEN", "legacy")]); + assert_eq!(aliased(legacy, "DW_AUTH_TOKEN", "WORKFLOW_SERVER_AUTH_TOKEN").unwrap().as_deref(), Some("legacy")); + // A set but empty primary still wins, as PHP's env() returns it. + let empty = lookup(&[("DW_AUTH_TOKEN", ""), ("WORKFLOW_SERVER_AUTH_TOKEN", "legacy")]); + assert_eq!(aliased(empty, "DW_AUTH_TOKEN", "WORKFLOW_SERVER_AUTH_TOKEN").unwrap().as_deref(), Some("")); + let neither = lookup(&[]); + assert_eq!(aliased(neither, "DW_AUTH_TOKEN", "WORKFLOW_SERVER_AUTH_TOKEN").unwrap(), None); + } +} From 3e0b62df0cd1a164eebc02ff652a6f477d969abb Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 19:19:19 +0000 Subject: [PATCH 05/30] Refuse PHP settings native cannot honor DW_DEFAULT_NAMESPACE and DW_WORKER_PROTOCOL_VERSION (and their legacy names) change published PHP's behavior. Native serves `default` and implements protocol 1.20, so it now refuses to start when either holds another value instead of ignoring it. The protocol version is one constant shared by every response that advertises it. The other settings recorded as ignored are read by PHP into its configuration but never applied, so ignoring them matches PHP. Co-Authored-By: Claude Opus 5.5 --- .../2026-10-21-legacy-auth-env.md | 23 +++--- docs/rust-server-port.md | 7 -- rust/src/main.rs | 75 ++++++++++++++++++- rust/src/runtime/cancellation.rs | 2 +- rust/src/runtime/http.rs | 12 +-- rust/src/runtime/mod.rs | 3 + rust/src/runtime/workflow_task_guard.rs | 2 +- 7 files changed, 99 insertions(+), 25 deletions(-) diff --git a/docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md b/docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md index d7e39d898..2172ca05c 100644 --- a/docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md +++ b/docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md @@ -17,15 +17,20 @@ - **Refused when set.** Authentication drivers other than `token` are refused at start, so their signature keys and headers cannot take effect silently. - - **Settings native ignores but that change behavior:** - - `DW_DEFAULT_NAMESPACE`; - - `DW_MAX_HISTORY_EVENTS`, `DW_MAX_PENDING_ACTIVITIES`, - `DW_MAX_PENDING_CHILDREN` and `DW_MAX_TASKS_PER_POLL`; - - `DW_ACTIVITY_TASK_TIMEOUT` and `DW_WORKER_PROTOCOL_VERSION`. - - Native uses PHP's defaults for these. A deployment that changed one - would get different behavior from native without notice. This is - recorded as a known gap; native should refuse a value it cannot honor. + - **Refused unless they match native.** `DW_DEFAULT_NAMESPACE` names the + namespace PHP serves to requests that name none, after lowercasing it. + `DW_WORKER_PROTOCOL_VERSION` is the protocol version PHP advertises and + gates features on. Native serves `default` and implements protocol + `1.20`, so it now refuses to start when either setting, or its legacy + name, holds another value. The error names the setting, not its value. + - **Read by PHP but never applied.** Published PHP puts + `DW_MAX_HISTORY_EVENTS`, `DW_MAX_PENDING_ACTIVITIES`, + `DW_MAX_PENDING_CHILDREN` and `DW_ACTIVITY_TASK_TIMEOUT` into its + configuration, but no code reads those entries. The structural limits + PHP enforces come from the Workflow package's own configuration. + Ignoring these settings matches PHP. `DW_MAX_TASKS_PER_POLL` only widens + PHP's candidate scan (ten candidates for each task); a poll still claims + one task, so native's own scan width is not a contract difference. - **PHP process roles and topology.** `DW_MODE`, `DW_SERVER_ID`, `DW_SERVER_PROCESS_CLASS`, `DW_SERVER_TOPOLOGY_SHAPE`, `DW_TASK_DISPATCH_MODE`, the poll interval settings and the diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 2338ac261..845c562bf 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -102,13 +102,6 @@ contract fixture before its correction. each priority tier across fairness keys by their weights (`TaskFairnessScheduler`); native stores the keys and weights but does not rebalance yet. -- **PHP settings native ignores.** Native uses PHP's defaults for - `DW_DEFAULT_NAMESPACE`, `DW_MAX_HISTORY_EVENTS`, - `DW_MAX_PENDING_ACTIVITIES`, `DW_MAX_PENDING_CHILDREN`, - `DW_MAX_TASKS_PER_POLL`, `DW_ACTIVITY_TASK_TIMEOUT` and - `DW_WORKER_PROTOCOL_VERSION`, and does not read their configured values. - A deployment that changed one of them would get different behavior from - native. Native should refuse to start on a value it cannot honor. - **Key order inside plain JSON input.** For input that is not an envelope, PHP encodes nested objects as Avro maps in request order; native encodes them with sorted keys. The decoded values are equal, but the blobs differ. diff --git a/rust/src/main.rs b/rust/src/main.rs index a8fe71684..090725440 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -2,7 +2,8 @@ use std::{env, error::Error, net::SocketAddr, str::FromStr}; use durable_workflow_server::database_tls::{self, Driver, Settings}; use durable_workflow_server::runtime::{ - Runtime, RuntimeError, mysql::MySqlStorage, postgres::PostgresStorage, router, + Runtime, RuntimeError, WORKER_PROTOCOL_VERSION, mysql::MySqlStorage, postgres::PostgresStorage, + router, }; use sqlx::mysql::MySqlConnectOptions; use sqlx::postgres::PgConnectOptions; @@ -97,6 +98,43 @@ fn auth_setting(primary: &str, legacy: &str) -> Result, Box Result, Box> { + match env::var(key) { + Ok(value) => Ok(Some(value)), + Err(env::VarError::NotPresent) => Ok(None), + Err(env::VarError::NotUnicode(_)) => Err(format!("{key} must be valid UTF-8").into()), + } +} + +/// A PHP setting whose value changes behavior that native does not offer. +/// Native refuses to start rather than run with PHP's default silently. +fn unhonored_php_setting(lookup: F) -> Result, Box> +where + F: Fn(&str) -> Result, Box>, +{ + // PHP lowercases the namespace it falls back to; native always falls + // back to `default`. + let namespace = aliased(&lookup, "DW_DEFAULT_NAMESPACE", "WORKFLOW_SERVER_DEFAULT_NAMESPACE")?; + if namespace.is_some_and(|value| value.to_ascii_lowercase() != "default") { + return Ok(Some( + "DW_DEFAULT_NAMESPACE must be unset or `default`: native serves `default` to requests that name no namespace" + .into(), + )); + } + // PHP advertises this value and gates protocol features on it. + let protocol = aliased( + &lookup, + "DW_WORKER_PROTOCOL_VERSION", + "WORKFLOW_SERVER_WORKER_PROTOCOL_VERSION", + )?; + if protocol.is_some_and(|value| value != WORKER_PROTOCOL_VERSION) { + return Ok(Some(format!( + "DW_WORKER_PROTOCOL_VERSION must be unset or `{WORKER_PROTOCOL_VERSION}`: native implements only that worker protocol" + ))); + } + Ok(None) +} + fn validate_auth_configuration() -> Result<(), Box> { // Unqualified credentials must not silently leave a full-access legacy // guard in place. These profiles require their own contract qualification. @@ -419,6 +457,9 @@ async fn main() -> Result<(), Box> { return Ok(()); } validate_auth_configuration()?; + if let Some(refusal) = unhonored_php_setting(setting_value)? { + return Err(refusal.into()); + } // Published PHP reads each token under its `DW_*` name, or else its // legacy `WORKFLOW_SERVER_*` name, so an unchanged deployment starts. let worker_token = auth_setting("DW_WORKER_TOKEN", "WORKFLOW_SERVER_WORKER_TOKEN")?; @@ -508,4 +549,36 @@ mod auth_env_tests { let neither = lookup(&[]); assert_eq!(aliased(neither, "DW_AUTH_TOKEN", "WORKFLOW_SERVER_AUTH_TOKEN").unwrap(), None); } + + #[test] + fn php_settings_native_cannot_honor_refuse_start() { + let refused = + |values: &'static [(&'static str, &'static str)]| unhonored_php_setting(lookup(values)).unwrap(); + let accepted: [&'static [(&str, &str)]; 7] = [ + &[], + &[("DW_DEFAULT_NAMESPACE", "default")], + &[("DW_DEFAULT_NAMESPACE", "Default")], + &[("WORKFLOW_SERVER_DEFAULT_NAMESPACE", "default")], + &[("DW_WORKER_PROTOCOL_VERSION", "1.20")], + &[("WORKFLOW_SERVER_WORKER_PROTOCOL_VERSION", "1.20")], + // Settings PHP reads but never applies stay accepted. + &[("DW_MAX_PENDING_ACTIVITIES", "10"), ("DW_ACTIVITY_TASK_TIMEOUT", "30")], + ]; + for accepted in accepted { + assert_eq!(refused(accepted), None, "{accepted:?}"); + } + let refusals: [(&'static [(&str, &str)], &str); 6] = [ + (&[("DW_DEFAULT_NAMESPACE", "billing")], "DW_DEFAULT_NAMESPACE"), + (&[("DW_DEFAULT_NAMESPACE", "")], "DW_DEFAULT_NAMESPACE"), + (&[("WORKFLOW_SERVER_DEFAULT_NAMESPACE", "billing")], "DW_DEFAULT_NAMESPACE"), + // A set primary wins over a matching legacy value, as in PHP. + (&[("DW_DEFAULT_NAMESPACE", "billing"), ("WORKFLOW_SERVER_DEFAULT_NAMESPACE", "default")], "DW_DEFAULT_NAMESPACE"), + (&[("DW_WORKER_PROTOCOL_VERSION", "1.19")], "DW_WORKER_PROTOCOL_VERSION"), + (&[("WORKFLOW_SERVER_WORKER_PROTOCOL_VERSION", "1.2")], "DW_WORKER_PROTOCOL_VERSION"), + ]; + for (values, setting) in refusals { + let refusal = refused(values).unwrap_or_else(|| panic!("{values:?} was accepted")); + assert!(refusal.starts_with(setting), "{refusal}"); + } + } } diff --git a/rust/src/runtime/cancellation.rs b/rust/src/runtime/cancellation.rs index 7900e6b6c..a20bbeaa9 100644 --- a/rust/src/runtime/cancellation.rs +++ b/rust/src/runtime/cancellation.rs @@ -535,7 +535,7 @@ where tx.summary_changed(&run_id); Ok(json!({"task_id":task_id,"workflow_task_attempt":attempt,"outcome":"completed","recorded":false, "run_id":run_id,"run_status":"failed","created_task_ids":[],"reason":"run_timed_out", - "protocol_version":"1.20","server_capabilities":super::store::capabilities()})) + "protocol_version":super::WORKER_PROTOCOL_VERSION,"server_capabilities":super::store::capabilities()})) } pub(super) async fn cancelled_activity_outcome( diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index f307045bb..2cf4e42ef 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -380,7 +380,7 @@ async fn guard(State(runtime): State, mut request: Request, next: Next) } else { "x-durable-workflow-control-plane-version" }; - let version = if worker { "1.20" } else { "2" }; + let version = if worker { super::WORKER_PROTOCOL_VERSION } else { "2" }; let requested = request .headers() .get(header) @@ -667,9 +667,9 @@ async fn admission_error( mut body: Value, ) -> Response { let (header, version) = if worker { - body["protocol_version"] = json!("1.20"); + body["protocol_version"] = json!(super::WORKER_PROTOCOL_VERSION); body["server_capabilities"] = capabilities(); - ("x-durable-workflow-protocol-version", "1.20") + ("x-durable-workflow-protocol-version", super::WORKER_PROTOCOL_VERSION) } else { body = super::control_plane::admission_response(request, body).await; ("x-durable-workflow-control-plane-version", "2") @@ -714,7 +714,7 @@ async fn cluster() -> Json { .expect("valid bundled carrier auth composition contract"); Json( json!({"version": env!("CARGO_PKG_VERSION"), "implementation": "rust", "development": true, - "control_plane": {"version": "2", "request_contract": super::visibility::request_contract()}, "worker_protocol": {"version": "1.20", "server_capabilities": capabilities()}, + "control_plane": {"version": "2", "request_contract": super::visibility::request_contract()}, "worker_protocol": {"version": super::WORKER_PROTOCOL_VERSION, "server_capabilities": capabilities()}, "auth_composition_contract": auth_composition, "payload_codec": "avro", "qualified_for_php_takeover": false}), ) @@ -1120,7 +1120,7 @@ async fn register( } fn worker_response(mut body: Value) -> Value { - body["protocol_version"] = json!("1.20"); + body["protocol_version"] = json!(super::WORKER_PROTOCOL_VERSION); body["server_capabilities"] = capabilities(); body } @@ -1438,7 +1438,7 @@ fn completion_refusal( reason: "task_not_found", } if status == StatusCode::NOT_FOUND => { let mut response = json!({"task_id":task_id,"error":message,"reason":"task_not_found", - "protocol_version":"1.20","server_capabilities":capabilities()}); + "protocol_version":super::WORKER_PROTOCOL_VERSION,"server_capabilities":capabilities()}); response[attempt_field] = attempt; super::RuntimeError::Protocol { status, response } } diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index d0a7938e8..c005ec09d 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -95,6 +95,9 @@ mod workflow_task_guard; pub use execution::Runtime; pub use external_config::ExternalPayloadConfig; pub use http::router; + +/// The worker protocol version native implements and advertises. +pub const WORKER_PROTOCOL_VERSION: &str = "1.20"; pub use query_authority::QueryTaskConfig; use axum::{ diff --git a/rust/src/runtime/workflow_task_guard.rs b/rust/src/runtime/workflow_task_guard.rs index 595b6754a..b1b24b9f3 100644 --- a/rust/src/runtime/workflow_task_guard.rs +++ b/rust/src/runtime/workflow_task_guard.rs @@ -15,7 +15,7 @@ pub(super) fn refusal( fields: Value, ) -> RuntimeError { let mut response = json!({"task_id":task_id,"workflow_task_attempt":attempt,"error":error,"reason":reason, - "protocol_version":"1.20","server_capabilities":super::store::capabilities()}); + "protocol_version":super::WORKER_PROTOCOL_VERSION,"server_capabilities":super::store::capabilities()}); if let (Some(response), Value::Object(fields)) = (response.as_object_mut(), fields) { response.extend(fields); } From d0f7dae85221cf2b8b19aa537bf4f21e42a94175 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 19:24:23 +0000 Subject: [PATCH 06/30] Report activity deadlines on claims and status reads like PHP Published PHP returns the activity execution's deadlines (schedule to start, start to close, schedule to close and heartbeat) on activity claims and on the attempt status read. Native returned null on status and omitted the claim field, so the Sample App's activity recovery scenario could not see the original total deadline. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/activity_deadlines.rs | 20 ++++++++++++++++++++ rust/src/runtime/activity_heartbeats.rs | 2 +- rust/src/runtime/store.rs | 5 +++++ rust/tests/execution.rs | 4 ++++ rust/tests/execution/activity_deadlines.rs | 19 +++++++++++++------ 5 files changed, 43 insertions(+), 7 deletions(-) diff --git a/rust/src/runtime/activity_deadlines.rs b/rust/src/runtime/activity_deadlines.rs index 237875c27..9e3182808 100644 --- a/rust/src/runtime/activity_deadlines.rs +++ b/rust/src/runtime/activity_deadlines.rs @@ -96,6 +96,26 @@ pub(super) fn expired_kind( Ok(None) } +/// PHP's `deadlines` map on activity claims and status reads: each deadline +/// the execution has, in UTC with microseconds, or null when it has none. +pub(super) fn deadlines_view(activity: &DB::Row) -> Result { + let mut deadlines = serde_json::Map::new(); + for (kind, field) in [ + ("schedule_to_start", "schedule_deadline_at"), + ("start_to_close", "close_deadline_at"), + ("schedule_to_close", "schedule_to_close_deadline_at"), + ("heartbeat", "heartbeat_deadline_at"), + ] { + if let Some(at) = DB::optional_instant(activity, field)? { + deadlines.insert( + kind.into(), + json!(at.to_rfc3339_opts(chrono::SecondsFormat::Micros, true)), + ); + } + } + Ok(if deadlines.is_empty() { Value::Null } else { Value::Object(deadlines) }) +} + /// PHP's message for an expired deadline. fn timeout_message(activity: &DB::Row, kind: &str) -> Result where diff --git a/rust/src/runtime/activity_heartbeats.rs b/rust/src/runtime/activity_heartbeats.rs index d323b560d..cfb540c20 100644 --- a/rust/src/runtime/activity_heartbeats.rs +++ b/rust/src/runtime/activity_heartbeats.rs @@ -133,7 +133,7 @@ where "lease_expires_at":expires,"last_heartbeat_at":DB::optional_instant(&attempt,"last_heartbeat_at")?.or(DB::optional_instant(&activity,"last_heartbeat_at")?), "run_status":run_status,"run_closed_reason":DB::optional_string(&run,"closed_reason")?,"run_closed_at":DB::optional_instant(&run,"closed_at")?, "activity_status":activity_status,"attempt_status":attempt_status,"task_status":task_status, - "deadlines":null,"worker_session":null,"protocol_version":protocol,"server_capabilities":super::store::capabilities()}); + "deadlines":super::activity_deadlines::deadlines_view::(&activity)?,"worker_session":null,"protocol_version":protocol,"server_capabilities":super::store::capabilities()}); Ok((response, Self::cancellation_request(&run)?.is_none())) } diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index e86052ed9..a1b2ba147 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -1360,6 +1360,11 @@ where claim["arguments"] = wire(&DB::string(&activity, "arguments")?); claim["retry_policy"] = DB::optional_document_row(&activity, "retry_policy")?.unwrap_or(Value::Null); + // PHP omits the field when the activity has no deadline. + let deadlines = super::activity_deadlines::deadlines_view::(&activity)?; + if !deadlines.is_null() { + claim["deadlines"] = deadlines; + } } else { claim["workflow_task_attempt"] = json!(attempt); claim["arguments"] = wire(&DB::string(&run, "arguments")?); diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 89fa3a9cd..d4c814155 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -3700,6 +3700,10 @@ async fn cooperative_delivery_cleanup_and_original_deadline_survive_fresh_runtim assert_eq!(status.1["heartbeat_recorded"], false); assert_eq!(status.1["lease_expires_at"], activity["lease_expires_at"]); assert_eq!(status.1["last_heartbeat_at"], Value::Null); + // An activity without deadlines: PHP omits the claim field and + // reports null on status. + assert!(activity.get("deadlines").is_none(), "{activity}"); + assert_eq!(status.1["deadlines"], Value::Null); let mut wrong_owner = status_body.clone(); wrong_owner["lease_owner"] = json!("other-owner"); assert_eq!( diff --git a/rust/tests/execution/activity_deadlines.rs b/rust/tests/execution/activity_deadlines.rs index 6aae8d781..4b07e960c 100644 --- a/rust/tests/execution/activity_deadlines.rs +++ b/rust/tests/execution/activity_deadlines.rs @@ -423,12 +423,19 @@ async fn heartbeat_renews_only_progress_deadline_and_cannot_resurrect_elapsed_au "/api/worker/activity-tasks/{}/status", activity["task_id"].as_str().unwrap() ); - assert_eq!( - request_protocol(&app, "POST", &status_path, base.clone(), "1.20") - .await - .1["can_continue"], - true - ); + // Claims and status reads carry PHP's deadlines map in UTC microseconds. + let micros = |at: chrono::DateTime| { + json!(at.with_timezone(&chrono::Utc).to_rfc3339_opts(chrono::SecondsFormat::Micros, true)) + }; + let deadlines = &activity["deadlines"]; + assert_eq!(deadlines["start_to_close"], micros(close), "{activity}"); + assert_eq!(deadlines["heartbeat"], micros(at + chrono::Duration::seconds(10))); + let lifecycle = deadlines["schedule_to_close"].as_str().unwrap(); + assert!(lifecycle.ends_with('Z') && lifecycle.len() == 27, "{lifecycle}"); + assert!(deadlines.get("schedule_to_start").is_none(), "{deadlines}"); + let status = request_protocol(&app, "POST", &status_path, base.clone(), "1.20").await; + assert_eq!(status.1["can_continue"], true, "{}", status.1); + assert_eq!(&status.1["deadlines"], deadlines); assert_eq!( running_deadlines(&database).await, initial, From f6ff81aaef62cb592df9f1c672717bef4aef3b1a Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 19:26:09 +0000 Subject: [PATCH 07/30] Record the structural limits native does not enforce yet Co-Authored-By: Claude Opus 5.5 --- .../2026-10-21-legacy-auth-env.md | 5 +++-- docs/rust-server-port.md | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md b/docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md index 2172ca05c..24e5d9c6d 100644 --- a/docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md +++ b/docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md @@ -27,8 +27,9 @@ `DW_MAX_HISTORY_EVENTS`, `DW_MAX_PENDING_ACTIVITIES`, `DW_MAX_PENDING_CHILDREN` and `DW_ACTIVITY_TASK_TIMEOUT` into its configuration, but no code reads those entries. The structural limits - PHP enforces come from the Workflow package's own configuration. - Ignoring these settings matches PHP. `DW_MAX_TASKS_PER_POLL` only widens + PHP enforces come from the Workflow package's own `DW_V2_LIMIT_*` + settings. Ignoring these four settings matches PHP. Of the package + limits, native enforces only the memo size; #612 tracks the rest. `DW_MAX_TASKS_PER_POLL` only widens PHP's candidate scan (ten candidates for each task); a poll still claims one task, so native's own scan width is not a contract difference. - **PHP process roles and topology.** `DW_MODE`, `DW_SERVER_ID`, diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 845c562bf..8d3911f68 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -102,6 +102,11 @@ contract fixture before its correction. each priority tier across fairness keys by their weights (`TaskFairnessScheduler`); native stores the keys and weights but does not rebalance yet. +- **Structural limits.** Published PHP enforces the Workflow package's + structural limits (pending activities, children, timers, signals and + updates, command batch size, payload size, search attribute size and + history transaction size), configured by `DW_V2_LIMIT_*`. Native enforces + only the memo size limit. #612 tracks the rest. - **Key order inside plain JSON input.** For input that is not an envelope, PHP encodes nested objects as Avro maps in request order; native encodes them with sorted keys. The decoded values are equal, but the blobs differ. From c2c0eebbd026ffd6c997abc320b1e8974222313d Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 19:26:18 +0000 Subject: [PATCH 08/30] Record the settings checks' verification Co-Authored-By: Claude Opus 5.5 --- .../2026-10-21-legacy-auth-env.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md b/docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md index 24e5d9c6d..7aba64ff0 100644 --- a/docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md +++ b/docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md @@ -42,5 +42,12 @@ files. - **Verification.** A unit test covers the precedence: the primary wins, the legacy name fills in, a set but empty primary still wins, and neither - gives nothing. The Sample App's polyglot run against native uses the - legacy token name unchanged. + gives nothing. A second unit test covers the refusals: the namespace and + protocol settings refuse other values under either name, accept + `default` in any case and `1.20`, and settings PHP never applies stay + accepted. Starting the binary with `DW_DEFAULT_NAMESPACE=billing` or + `WORKFLOW_SERVER_WORKER_PROTOCOL_VERSION=1.19` exits with the setting's + name; `DW_DEFAULT_NAMESPACE=Default` starts. On a disposable host, the + Sample App's PolyglotWorkflow and its timers scenario passed against + native with the sample's unchanged server environment, which sets only + `WORKFLOW_SERVER_AUTH_TOKEN`. From 94edbf4a986bc2a0265c4f691cef66249097e252 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 19:30:57 +0000 Subject: [PATCH 09/30] Judge a late activity result by its attempt first, like PHP Published PHP checks the reporting attempt before the task's lease or recorded outcome: a result from an attempt that is no longer current is answered with 409 stale_attempt, even after another attempt finished the task. Native compared the task's completion receipt first and answered conflicting_completion, or task_not_leased for a failure. The Sample App's activity recovery scenario requires PHP's answer. A repeated result from the attempt that finished the task keeps its idempotent receipt. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/activity_failures.rs | 10 +++++++++- rust/src/runtime/store.rs | 12 +++++++++++- rust/tests/execution/worker_listing.rs | 27 ++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/rust/src/runtime/activity_failures.rs b/rust/src/runtime/activity_failures.rs index bae901b29..c0764db94 100644 --- a/rust/src/runtime/activity_failures.rs +++ b/rust/src/runtime/activity_failures.rs @@ -221,7 +221,6 @@ where json!({"task_id":task_id,"activity_attempt_id":attempt_id,"outcome":"failed","recorded":false,"reason":"already_completed"}), ); } - Self::fence(&task, &body)?; let attempt = Self::query("SELECT * FROM activity_attempts WHERE id=$1 AND workflow_task_id=$2") .bind(attempt_id) @@ -230,6 +229,15 @@ where .await? .ok_or_else(|| refuse(StatusCode::CONFLICT, "stale_attempt"))?; let count = DB::number(&attempt, "attempt_number")?; + // As for completions, PHP judges the attempt before the task's lease. + if DB::string(&attempt, "status")? != "running" || count != DB::number(&task, "attempt_count")? { + return Err(super::RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"task_id":task_id,"activity_attempt_id":attempt_id,"outcome":"failed", + "recorded":false,"reason":"stale_attempt","next_task_id":null}), + }); + } + Self::fence(&task, &body)?; let run_id = DB::string(&task, "workflow_run_id")?; Self::active_run(&mut tx, &run_id).await?; let activity_id = DB::string(&attempt, "activity_execution_id")?; diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index a1b2ba147..78b56829a 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -2344,7 +2344,6 @@ where json!({"task_id": task_id, "activity_attempt_id": attempt_id, "outcome": "completed", "recorded": false, "reason": "already_completed"}), ); } - Self::fence(&task, &body)?; let attempt = Self::query("SELECT * FROM activity_attempts WHERE id=$1 AND workflow_task_id=$2") .bind(attempt_id) @@ -2352,6 +2351,9 @@ where .fetch_optional(&mut *tx) .await? .ok_or_else(|| refuse(StatusCode::CONFLICT, "stale_attempt"))?; + // PHP judges the attempt before the task's lease: a late result from + // an attempt that is no longer current is stale even after another + // attempt finished the task. if DB::string(&attempt, "status")? != "running" || DB::number(&attempt, "attempt_number")? != DB::number(&task, "attempt_count")? { @@ -2378,6 +2380,7 @@ where response: receipt, }); } + Self::fence(&task, &body)?; let run_id = DB::string(&task, "workflow_run_id")?; let run = Self::active_run(&mut tx, &run_id).await?; let activity_id = DB::string(&attempt, "activity_execution_id")?; @@ -2611,6 +2614,13 @@ where // receipt. A later completion must refuse it, not decode SQL NULL. let receipt = DB::optional_document_row(task, "receipt")? .ok_or_else(|| refuse(StatusCode::CONFLICT, "task_not_leased"))?; + // A result from another activity attempt is not this one's duplicate; + // its attempt is stale, which the caller reports as PHP does. + if DB::string(task, "task_type")? == "activity" + && receipt["activity_attempt_id"] != body["activity_attempt_id"] + { + return Ok(false); + } if receipt == *body { return Ok(true); } diff --git a/rust/tests/execution/worker_listing.rs b/rust/tests/execution/worker_listing.rs index c706abf64..1abcbc1e9 100644 --- a/rust/tests/execution/worker_listing.rs +++ b/rust/tests/execution/worker_listing.rs @@ -339,6 +339,33 @@ async fn a_replacement_registration_releases_the_activity_lease_at_once() { let (status, done) = request(&app, "POST", &path, complete(&replacement)).await; assert_eq!(status, StatusCode::OK, "{done}"); assert_eq!(done["recorded"], true); + // After the replacement finished the task, the original attempt's late + // result is still stale, as in PHP, not a conflicting duplicate. + let (status, late) = request(&app, "POST", &path, complete(&original)).await; + assert_eq!(status, StatusCode::CONFLICT, "{late}"); + assert_eq!( + late, + json!({"task_id":original["task_id"],"activity_attempt_id":original["activity_attempt_id"], + "outcome":"completed","recorded":false,"reason":"stale_attempt","next_task_id":null, + "lease_owner":"replaced","activity_status":"completed","attempt_status":"expired","task_status":"completed"}) + ); + let fail_path = format!( + "/api/worker/activity-tasks/{}/fail", + original["task_id"].as_str().unwrap() + ); + let (status, late_failure) = request(&app, "POST", &fail_path, + json!({"activity_attempt_id":original["activity_attempt_id"],"lease_owner":"replaced", + "failure":{"message":"late","type":"RuntimeException"}})).await; + assert_eq!(status, StatusCode::CONFLICT, "{late_failure}"); + assert_eq!( + late_failure, + json!({"task_id":original["task_id"],"activity_attempt_id":original["activity_attempt_id"], + "outcome":"failed","recorded":false,"reason":"stale_attempt","next_task_id":null}) + ); + // The replacement's own repeated result keeps its idempotent receipt. + let (status, again) = request(&app, "POST", &path, complete(&replacement)).await; + assert_eq!(status, StatusCode::OK, "{again}"); + assert_eq!(again["reason"], "already_completed"); let history = run_history(&app, &json!("replaced-peer"), &started["run_id"]).await; let started_attempts: Vec<_> = history .as_array() From 9f8321fc29134e67afe356d8271bfaa3e0a59ad6 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 19:38:05 +0000 Subject: [PATCH 10/30] Answer any other late result for a closed activity task as stale A retried activity closes its failed attempt's task with that failure as the receipt. A later completion from the same attempt differed from the receipt and was refused as conflicting_completion. Published PHP answers it with 409 stale_attempt, because the task is no longer leased. Only an identical repeat keeps native's idempotent receipt. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/store.rs | 13 +++++----- rust/tests/execution.rs | 29 ++++++++++++++++------ rust/tests/execution/activity_deadlines.rs | 9 ++++--- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 78b56829a..fd239251f 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -2614,16 +2614,15 @@ where // receipt. A later completion must refuse it, not decode SQL NULL. let receipt = DB::optional_document_row(task, "receipt")? .ok_or_else(|| refuse(StatusCode::CONFLICT, "task_not_leased"))?; - // A result from another activity attempt is not this one's duplicate; - // its attempt is stale, which the caller reports as PHP does. - if DB::string(task, "task_type")? == "activity" - && receipt["activity_attempt_id"] != body["activity_attempt_id"] - { - return Ok(false); - } if receipt == *body { return Ok(true); } + // Any other result for a closed activity task comes from an attempt + // that is no longer current; the caller reports it as stale, as PHP + // does, rather than as a conflicting duplicate. + if DB::string(task, "task_type")? == "activity" { + return Ok(false); + } Err(refuse(StatusCode::CONFLICT, "conflicting_completion")) } diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index d4c814155..1fdd379c5 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -4421,23 +4421,26 @@ async fn reported_activity_failure_commits_one_persistent_retry_and_fences_old_o poll(&app_b, "worker", "activity").await.is_null(), "retry waits the actual one-second backoff" ); + // A different late outcome from the failed attempt is stale, as in PHP. let mut conflicting = fail.clone(); conflicting["failure"]["message"] = json!("different"); + let refused = request(&app_b, "POST", &fail_path, conflicting).await; + assert_eq!(refused.0, StatusCode::CONFLICT, "{}", refused.1); assert_eq!( - request(&app_b, "POST", &fail_path, conflicting).await.0, - StatusCode::CONFLICT + refused.1, + json!({"task_id":first["task_id"],"activity_attempt_id":first["activity_attempt_id"], + "outcome":"failed","recorded":false,"reason":"stale_attempt","next_task_id":null}) ); let old_complete_path = format!( "/api/worker/activity-tasks/{}/complete", first["task_id"].as_str().unwrap() ); let old_complete = json!({"lease_owner":"worker","activity_attempt_id":first["activity_attempt_id"],"result":envelope(Payload::Long(-1))}); - assert_eq!( - request(&app_b, "POST", &old_complete_path, old_complete.clone()) - .await - .0, - StatusCode::CONFLICT - ); + let refused = request(&app_b, "POST", &old_complete_path, old_complete.clone()).await; + assert_eq!(refused.0, StatusCode::CONFLICT, "{}", refused.1); + assert_eq!(refused.1["reason"], "stale_attempt"); + assert_eq!(refused.1["attempt_status"], "failed"); + assert_eq!(refused.1["task_status"], "completed"); node_a.close().await; let recovered = database.open().await.unwrap(); let app_c = router(recovered.clone()); @@ -4468,6 +4471,16 @@ async fn reported_activity_failure_commits_one_persistent_retry_and_fences_old_o assert_eq!(second["arguments"], arguments); assert_eq!(second["attempt_number"], 2); assert_ne!(second["activity_attempt_id"], first["activity_attempt_id"]); + // While the retry runs, the first attempt's late result is still the + // stale receipt PHP returns to that attempt's owner. + let refused = request(&app_b, "POST", &old_complete_path, old_complete.clone()).await; + assert_eq!(refused.0, StatusCode::CONFLICT, "{}", refused.1); + assert_eq!( + refused.1, + json!({"task_id":first["task_id"],"activity_attempt_id":first["activity_attempt_id"], + "outcome":"completed","recorded":false,"reason":"stale_attempt","next_task_id":null, + "lease_owner":"worker","activity_status":"running","attempt_status":"failed","task_status":"completed"}) + ); assert_eq!( request(&app_b, "POST", &old_complete_path, old_complete) .await diff --git a/rust/tests/execution/activity_deadlines.rs b/rust/tests/execution/activity_deadlines.rs index 4b07e960c..2d2d52d38 100644 --- a/rust/tests/execution/activity_deadlines.rs +++ b/rust/tests/execution/activity_deadlines.rs @@ -541,10 +541,11 @@ async fn committed_activity_receipt_survives_elapsed_deadlines_and_conflicting_o ); let mut conflicting = completion; conflicting["result"] = envelope(Payload::Long(-1)); - assert_eq!( - request(&app, "POST", &path, conflicting).await.0, - StatusCode::CONFLICT - ); + // A different result for the finished attempt is stale, as in PHP. + let refused = request(&app, "POST", &path, conflicting).await; + assert_eq!(refused.0, StatusCode::CONFLICT, "{}", refused.1); + assert_eq!(refused.1["reason"], "stale_attempt"); + assert_eq!(refused.1["recorded"], false); assert_eq!( run_history(&app, &started["workflow_id"], &started["run_id"]).await, prefix From 6abc63074f3e323e2b2efef8e98c714ba4e0627b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 19:48:37 +0000 Subject: [PATCH 11/30] Answer late results after an activity timeout with PHP's stale receipt A timeout closes the attempt and cancels its task. A later completion or failure for that task got a bare 409 stale_attempt without the task, attempt and status fields, so the PHP SDK could not recognize the refusal, tried to fail the task, and stopped its worker. Published PHP answers with its full stale attempt receipt, which the SDKs discard. Heartbeats keep their existing answer. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/activity_heartbeats.rs | 3 +++ rust/src/runtime/cancellation.rs | 4 ++- rust/tests/execution/activity_deadlines.rs | 29 ++++++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/rust/src/runtime/activity_heartbeats.rs b/rust/src/runtime/activity_heartbeats.rs index cfb540c20..ffa0d0676 100644 --- a/rust/src/runtime/activity_heartbeats.rs +++ b/rust/src/runtime/activity_heartbeats.rs @@ -210,6 +210,9 @@ where return Err(refuse(StatusCode::NOT_FOUND, "task_not_found")); } Self::cancelled_activity_outcome(&mut tx, &task, &body).await?; + if DB::string(&task, "status")? == "cancelled" { + return Err(refuse(StatusCode::CONFLICT, "stale_attempt")); + } Self::fence(&task, &body)?; let attempt = Self::query("SELECT * FROM activity_attempts WHERE id=$1 AND workflow_task_id=$2") diff --git a/rust/src/runtime/cancellation.rs b/rust/src/runtime/cancellation.rs index 7900e6b6c..5a70fccda 100644 --- a/rust/src/runtime/cancellation.rs +++ b/rust/src/runtime/cancellation.rs @@ -560,7 +560,9 @@ where let reason = match run_status.as_str() { "cancelled" => "run_cancelled", "terminated" => "run_terminated", - _ => return Err(refuse(StatusCode::CONFLICT, "stale_attempt")), + // A deadline, not the run, cancelled this task. The caller + // answers the late outcome with PHP's stale attempt receipt. + _ => return Ok(()), }; if let Some(attempt) = attempt { if DB::string(&attempt, "status")? == "cancelled" diff --git a/rust/tests/execution/activity_deadlines.rs b/rust/tests/execution/activity_deadlines.rs index 2d2d52d38..b5dd52bc3 100644 --- a/rust/tests/execution/activity_deadlines.rs +++ b/rust/tests/execution/activity_deadlines.rs @@ -224,6 +224,35 @@ async fn every_authored_deadline_fences_outcomes_and_resolves_once() { } } let history = resolution(&app, &started, "ActivityTimedOut").await; + if !activity.is_null() { + // Once the timeout closed the attempt, a late result gets PHP's + // stale attempt receipt, which the SDKs recognize and discard. + let path = |outcome: &str| { + format!("/api/worker/activity-tasks/{}/{outcome}", activity["task_id"].as_str().unwrap()) + }; + let late = request(&app, "POST", &path("complete"), + json!({"lease_owner":activity["lease_owner"],"activity_attempt_id":activity["activity_attempt_id"], + "result":envelope(Payload::Long(-1))})).await; + assert_eq!(late.0, StatusCode::CONFLICT, "{field}: {}", late.1); + assert_eq!( + late.1, + json!({"task_id":activity["task_id"],"activity_attempt_id":activity["activity_attempt_id"], + "outcome":"completed","recorded":false,"reason":"stale_attempt","next_task_id":null, + "lease_owner":activity["lease_owner"],"activity_status":"failed","attempt_status":"failed", + "task_status":"cancelled"}), + "{field}" + ); + let late = request(&app, "POST", &path("fail"), + json!({"lease_owner":activity["lease_owner"],"activity_attempt_id":activity["activity_attempt_id"], + "failure":{"type":"RuntimeException","message":"late","code":0}})).await; + assert_eq!(late.0, StatusCode::CONFLICT, "{field}: {}", late.1); + assert_eq!( + late.1, + json!({"task_id":activity["task_id"],"activity_attempt_id":activity["activity_attempt_id"], + "outcome":"failed","recorded":false,"reason":"stale_attempt","next_task_id":null}), + "{field}" + ); + } let timed = history .as_array() .unwrap() From 9f7e063c9aaf89f8cd73d32a34db1a1210f78d1b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 19:58:20 +0000 Subject: [PATCH 12/30] Record the activity recovery checkpoint and two activity result gaps Co-Authored-By: Claude Opus 5.5 --- .../2026-10-21-activity-recovery-receipts.md | 39 +++++++++++++++++++ docs/rust-server-port.md | 8 ++++ 2 files changed, 47 insertions(+) create mode 100644 docs/rust-server-port-checkpoints/2026-10-21-activity-recovery-receipts.md diff --git a/docs/rust-server-port-checkpoints/2026-10-21-activity-recovery-receipts.md b/docs/rust-server-port-checkpoints/2026-10-21-activity-recovery-receipts.md new file mode 100644 index 000000000..c53f22794 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-21-activity-recovery-receipts.md @@ -0,0 +1,39 @@ +# October 21, activity deadlines and late results + +- **Found by.** Running the Sample App's activity recovery scenario + (`scripts/sdk-activity-recovery.sh`) against native, unchanged except that + its server, and its separate PHP `activity:timeout-enforce` loop, were + native processes. The scenario failed four times in turn, once per gap below. +- **Deadlines on claims and status.** Published PHP returns the activity + execution's deadlines on an activity claim and on the attempt status read: + `schedule_to_start`, `start_to_close`, `schedule_to_close` and `heartbeat`, + each in UTC with microseconds, leaving out the ones that are not set. PHP + leaves the claim field out when there are none and reports `null` on + status. Native returned `null` on status and had no claim field, so the + scenario could not find the original total deadline. +- **Late results are judged by their attempt first.** PHP checks the + reporting attempt before the task's lease or recorded outcome. A result + from an attempt that is no longer current gets `409 stale_attempt`, even + after another attempt finished the task, or after the attempt's own + failure closed its task. The completion receipt carries the attempt + owner, `activity_status`, `attempt_status` and `task_status`; the failure + receipt does not. Native compared the task's recorded receipt first and + answered `conflicting_completion`, or `task_not_leased` for a failure. An + identical repeat of the result that finished the task keeps native's + idempotent receipt. +- **Results after a timeout.** A timeout closes the attempt and cancels its + task. Native answered a later completion or failure for that task with a + bare `409 stale_attempt`, without the task, attempt and status fields. The + PHP SDK could not recognize that refusal, tried to fail the task, received + the same refusal and stopped its worker. Every later case of the scenario + then had no PHP activity worker. Native now answers with PHP's full stale + attempt receipt, which the PHP SDK discards. Heartbeats keep their answer. +- **Verification.** + - Tests cover the deadlines map on a claim and a status read with and + without deadlines, a late result from a replaced attempt and from a + failed and retried attempt, and late results after each authored + timeout. They pass on SQLite, MySQL and PostgreSQL. + - On a disposable host, the scenario's five PHP-to-PHP cases and two + PHP-to-Python cases then passed. The next case stopped on a separate + gap: native refuses the Python SDK's activity failure reports, which + carry `class` and `stack_trace`. diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 0a7394960..14df70476 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -198,6 +198,14 @@ contract fixture before its correction. a signal therefore waits on PHP and is refused on native. The port takes the snapshot at claim time and must keep query lease replay stable, so it needs its own fixture. +- **Activity results after a deadline, before its timeout runs.** Published + PHP records a completion that arrives after an activity deadline has + passed when its separate timeout command has not yet closed the attempt. + Native refuses it with the deadline's timeout reason, because its own + timeout pass runs in every server process. +- **A repeated identical activity result.** Published PHP answers a repeat + of the result that finished an activity task with `409 stale_attempt`. + Native returns its earlier idempotent receipt (`200`, `already_completed`). - **Terminating a child run.** Published Workflow resolves the parent with `ChildRunTerminated` and the `terminated` child call status. Native refuses this with `422 termination_propagation_not_available` until the parent-side From 37a7702d81add499eccf11ff12a8842f05a3ef79 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 20:03:11 +0000 Subject: [PATCH 13/30] Admit queries while a workflow task is open, like PHP Published PHP admits a query whose run has an open workflow task and lets it wait for its claim. A workflow task leased at admission fixes the query's history cutoff at the run's sequence then. The claim waits while a workflow task holds a live lease, or while a due resume task (signal, update, activity, child or timer) is ready that the query cannot see yet: not one created after a cutoff query was admitted, and not a signal whose SignalReceived the query already observed. The claim then delivers the history up to the cutoff, or all of it without one. A poll that finds only held queries ends with workflow_task_pending at once, or with workflow_task_leased when its wait times out. Native refused such queries with 409 query_snapshot_busy, which stopped the Sample App's update scenario. A held poll re-checks without the transition lock until the workflow task ends. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/backend.rs | 5 + rust/src/runtime/queries.rs | 204 +++++++++++++++++++++-- rust/tests/execution.rs | 12 +- rust/tests/execution/query_open_tasks.rs | 186 +++++++++++++++++++++ 4 files changed, 381 insertions(+), 26 deletions(-) create mode 100644 rust/tests/execution/query_open_tasks.rs diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 9d5df7249..d307389f9 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -30,6 +30,8 @@ pub(super) trait Backend: Database { const POLL_RECEIPT_CLEANUP_SQL: &'static str = super::store::POLL_RECEIPT_CLEANUP_SQL; const TIMER_TASK_SQL: &'static str; const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='TimerScheduled' AND payload->>'timer_id'=$2 ORDER BY sequence DESC LIMIT 1"; + /// The history sequence of a signal's `SignalReceived` event. + const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SignalReceived' AND payload->>'signal_id'=$2 ORDER BY sequence LIMIT 1"; const SELECTION_RESOLUTION_SQL: &'static str = "SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SelectionResolved' AND payload->>'selection_group_id'=$2 ORDER BY sequence LIMIT 1"; // Frozen run command/task repair counters use each backend's actual width. const REPAIR_COUNTER_MAX: i64 = i32::MAX as i64; @@ -159,6 +161,7 @@ ORDER BY t.available_order,t.id LIMIT 100 type EncodedDocument = String; const TIMER_TASK_SQL: &'static str = "SELECT id,payload FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='timer' AND status='ready' AND json_extract(payload,'$.timer_id')=$2 AND namespace=$3"; const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='TimerScheduled' AND json_extract(payload,'$.timer_id')=$2 ORDER BY sequence DESC LIMIT 1"; + const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SignalReceived' AND json_extract(payload,'$.signal_id')=$2 ORDER BY sequence LIMIT 1"; const HISTORY_PAGE: &'static str = "SELECT * FROM (SELECT *,SUM(length(CAST(payload AS BLOB))) OVER (ORDER BY sequence) AS page_bytes FROM workflow_history_events WHERE workflow_run_id=$1 AND sequence>$2 ORDER BY sequence LIMIT $3) AS page WHERE page_bytes<=8388608 ORDER BY sequence"; // Normalize only the two supported UTC storage shapes. Removing trailing // fractional zeros preserves exact decimal ordering, including earlier @@ -400,6 +403,7 @@ impl Backend for MySql { const QUERY_POLL_EXPIRED: &'static str = "SELECT `key` FROM dw_query_cache WHERE `key` LIKE 'dw-native-query-poll:%' AND expiration<=$1 ORDER BY `key` LIMIT 32"; const TIMER_TASK_SQL: &'static str = "SELECT id,payload FROM workflow_tasks WHERE workflow_run_id=? AND task_type='timer' AND status='ready' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.timer_id'))=? AND BINARY namespace=BINARY ?"; const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=? AND event_type='TimerScheduled' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.timer_id'))=? ORDER BY sequence DESC LIMIT 1"; + const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=? AND event_type='SignalReceived' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.signal_id'))=? ORDER BY sequence LIMIT 1"; // First use the preserved run/sequence index to materialize only IDs and // byte counts for this bounded page. Sorting JSON payloads in the window // exhausts MySQL's default sort buffer before the size rejection can run. @@ -411,6 +415,7 @@ impl Backend for MySql { if !sql.contains('$') || sql == Self::TIMER_TASK_SQL || sql == Self::TIMER_SCHEDULE_SQL + || sql == Self::SIGNAL_RECEIVED_SEQUENCE_SQL || sql == Self::RANGE_REPAIR_TASK_SQL || sql == Self::SELECTION_RESOLUTION_SQL || sql == Self::ACTIVITY_OPEN_TASK_SQL diff --git a/rust/src/runtime/queries.rs b/rust/src/runtime/queries.rs index 8888af0c5..f7b97a3e1 100644 --- a/rust/src/runtime/queries.rs +++ b/rust/src/runtime/queries.rs @@ -32,6 +32,33 @@ enum QueryClaim { StaleWorker, // A newer poll request from the same worker replaced this one. Superseded, + // `pendingTaskWorkflowBlockStatus`: every matching query waits on its run. + Blocked(&'static str), +} + +const LEASED_WORKFLOW_TASKS: &str = "SELECT COUNT(*) FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='workflow' AND status='leased' AND (lease_expires_at IS NULL OR lease_expires_at>$2)"; + +/// Laravel binds "now" in whole seconds when PHP compares task leases. +fn lease_now() -> chrono::DateTime { + use chrono::Timelike; + let at = super::store::now(); + at.with_nanosecond(0).unwrap_or(at) +} + +/// The resume kinds PHP's `isWorkflowResumePayload` recognizes. Native's +/// activity and ordinary timer resumes name their source without a kind. +fn resume_kind(payload: &Value) -> Option<&'static str> { + match payload["resume_source_kind"].as_str() { + Some("workflow_signal") => Some("workflow_signal"), + Some("workflow_update") => Some("workflow_update"), + Some("activity_execution") => Some("activity_execution"), + Some("child_workflow_run") => Some("child_workflow_run"), + Some("timer") => Some("timer"), + Some(_) => None, + None if payload["activity_execution_id"].is_string() => Some("activity_execution"), + None if payload["timer_id"].is_string() => Some("timer"), + None => None, + } } fn poll_result(task: Option, status: &str) -> Value { @@ -338,15 +365,16 @@ where if !admits_queries(&status) { return Err(run_not_active(workflow_id, &run_id, name, &status)); } - let busy = Self::scalar("SELECT COUNT(*) FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='workflow' AND status IN ('ready','leased')") - .bind(&run_id).fetch_one(&mut *tx).await?; - // Native answers from a snapshot taken here, so it refuses while a - // workflow task is open. PHP admits the query and holds its claim - // instead; see the port log's known gaps. - if busy != 0 { - return Err(query_refusal(StatusCode::CONFLICT, workflow_id, &run_id, name, "query_snapshot_busy", - &format!("Workflow query [{name}] cannot run while run [{run_id}] has an open workflow task. Retry after the task completes."))); - } + // `enqueue`: a workflow task leased now fixes the query's history + // cutoff at the run's current sequence. Without one, the claim reads + // the history it finds then. Either way the query waits for its claim. + let observed = DB::number(&run, "last_history_sequence")?; + let leased = Self::scalar(LEASED_WORKFLOW_TASKS) + .bind(&run_id) + .bind(DB::bind_time(lease_now())) + .fetch_one(&mut *tx) + .await?; + let cutoff = (leased != 0).then_some(observed); let entries = self.query_entries(&mut tx).await?; if entries.len() >= MAX_ENTRIES || entries @@ -360,6 +388,8 @@ where "query_task_queue_full", )); } + // Native delivers one bounded history page; a run already past it + // cannot be queried. let (history, next) = Self::history_page_connection(&mut tx, &run_id, 0, 1000).await?; if next.is_some() || serde_json::to_vec(&history)?.len() > SNAPSHOT_BYTES { return Err(refuse( @@ -367,6 +397,7 @@ where "query_snapshot_exceeds_budget", )); } + drop(history); let task_id = super::store::id(); let key = key(&task_id)?; let admitted = authority::now(); @@ -376,8 +407,8 @@ where "workflow_type":DB::string(&run,"workflow_type")?,"workflow_class":DB::string(&run,"workflow_class")?, "task_queue":DB::string(&run,"queue")?,"payload_codec":"avro","query_name":name, "workflow_arguments":wire(&DB::string(&run,"arguments")?),"query_arguments":wire(&arguments), - "run_status":status,"history_events":history,"last_history_sequence":DB::number(&run,"last_history_sequence")?, - "history_cutoff_sequence":DB::number(&run,"last_history_sequence")?, + "run_status":status,"last_history_sequence":observed, + "history_observed_sequence":observed,"history_cutoff_sequence":cutoff, "query_admitted_at":authority::wire(admitted),"query_deadline_at":authority::wire(deadline), "receipt_expires_at":authority::wire(retained_until)}); authority::expire(&mut task, authority::now())?; @@ -426,6 +457,7 @@ where let status = match task["failure"]["reason"].as_str() { Some("rejected_unknown_query") => StatusCode::NOT_FOUND, Some("invalid_query_arguments") => StatusCode::UNPROCESSABLE_ENTITY, + Some("query_snapshot_exceeds_budget") => StatusCode::PAYLOAD_TOO_LARGE, _ => StatusCode::CONFLICT, }; let mut body = task["failure"].clone(); @@ -561,6 +593,7 @@ where // Only the first probe claims under the lock unconditionally; later // probes claim after a lock-free read finds something to claim or refuse. let mut first = true; + let mut held = false; let mut pace = super::poll_readiness::ReadPace::new(); loop { let wake = self.wake.notified(); @@ -571,11 +604,17 @@ where return Ok(poll_result(None, "stale_worker_registration")); } QueryClaim::Superseded => return Ok(poll_result(None, "empty")), - QueryClaim::Empty => {} + // `performPoll`: a query held by a ready resume task ends + // the poll at once; one held by a leased task waits. + QueryClaim::Blocked("workflow_task_pending") => { + return Ok(poll_result(None, "workflow_task_pending")); + } + QueryClaim::Blocked(_) => held = true, + QueryClaim::Empty => held = false, } } if tokio::time::Instant::now() >= deadline { - return Ok(poll_result(None, "empty")); + return Ok(poll_result(None, if held { "workflow_task_leased" } else { "empty" })); } if self .poll_queue_changed(&body, "query", &mut cross_kind) @@ -669,15 +708,37 @@ where } let at = super::store::now(); let query_now = authority::wire(at); - Ok(Self::query(DB::POLL_WAKE_QUERIES_SQL) + let candidates = Self::query(DB::POLL_WAKE_QUERIES_SQL) .bind(at.timestamp()) .bind(self.namespace.as_ref()) .bind(text(body, "task_queue")?) .bind(query_now.clone()) .bind(query_now) - .fetch_optional(&self.pool) - .await? - .is_some()) + .fetch_all(&self.pool) + .await?; + if candidates.is_empty() { + return Ok(false); + } + // A query held by a leased workflow task stays held until that task + // ends, which wakes this poll; reading that needs no write lock. + let mut read = self.begin_read().await?; + let mut ready = false; + for candidate in candidates { + let Some(row) = Self::query(DB::QUERY_CACHE_GET) + .bind(DB::string(&candidate, "id")?) + .fetch_optional(&mut *read) + .await? + else { + continue; + }; + let task: Value = serde_json::from_str(&DB::string(&row, "value")?)?; + if Self::query_hold(&mut read, &task).await? != Some("workflow_task_leased") { + ready = true; + break; + } + } + read.commit().await?; + Ok(ready) } async fn claim_query( @@ -723,6 +784,7 @@ where { return Ok(QueryClaim::Empty); } + let mut held = None; for (key, mut task) in self.query_entries(&mut tx).await? { if !matches!(task["status"].as_str(), Some("pending" | "leased")) || task["namespace"] != self.namespace.as_ref() @@ -768,6 +830,16 @@ where continue; } } + if let Some(hold) = Self::query_hold(&mut tx, &task).await? { + if held != Some("workflow_task_leased") { + held = Some(hold); + } + continue; + } + if !Self::query_delivery_history(&mut tx, &mut task).await? { + Self::put_query(&mut tx, &key, &task, authority::expiration(&task)?).await?; + continue; + } let next_attempt = task["query_task_attempt"] .as_i64() .and_then(|attempt| attempt.checked_add(1)) @@ -790,7 +862,103 @@ where } // Commit cleanup even when no compatible task is found. tx.commit().await?; - Ok(QueryClaim::Empty) + Ok(held.map_or(QueryClaim::Empty, QueryClaim::Blocked)) + } + + /// `canClaimPendingTaskForPoller`'s run checks, in PHP's order. A live + /// workflow task lease holds the claim. So does a due resume task (signal, + /// update, activity, child or timer) whose effect the query cannot see, + /// except one created after a query with a cutoff was admitted, or a + /// signal whose `SignalReceived` the query already observed. + async fn query_hold(conn: &mut DB::Connection, task: &Value) -> Result> { + let run_id = task["run_id"].as_str().unwrap_or_default(); + let leased = Self::scalar(LEASED_WORKFLOW_TASKS) + .bind(run_id) + .bind(DB::bind_time(lease_now())) + .fetch_one(&mut *conn) + .await?; + if leased != 0 { + return Ok(Some("workflow_task_leased")); + } + let ready = Self::query("SELECT payload,available_at,created_at FROM workflow_tasks WHERE workflow_run_id=$1 AND queue=$2 AND task_type='workflow' AND status='ready' ORDER BY id") + .bind(run_id) + .bind(task["task_queue"].as_str().unwrap_or_default()) + .fetch_all(&mut *conn) + .await?; + let at = super::store::now(); + let cutoff = !task["history_cutoff_sequence"].is_null(); + let admitted = authority::timestamp(&task["query_admitted_at"]); + let observed = task["history_observed_sequence"].as_i64().unwrap_or(0); + for row in ready { + if DB::optional_instant(&row, "available_at")?.is_some_and(|available| available > at) { + continue; + } + if cutoff + && let (Some(admitted), Some(created)) = + (admitted, DB::optional_instant(&row, "created_at")?) + && created > admitted + { + continue; + } + let payload = DB::document_row(&row, "payload")?; + let Some(kind) = resume_kind(&payload) else { + continue; + }; + if kind == "workflow_signal" + && observed >= 1 + && let Some(signal) = payload["workflow_signal_id"] + .as_str() + .or(payload["resume_source_id"].as_str()) + { + let received = Self::query(DB::SIGNAL_RECEIVED_SEQUENCE_SQL) + .bind(run_id) + .bind(signal) + .fetch_optional(&mut *conn) + .await?; + if let Some(received) = received + && DB::number(&received, "sequence")? <= observed + { + continue; + } + } + return Ok(Some("workflow_task_pending")); + } + Ok(None) + } + + /// `queryTaskPayload`: the history a claim delivers, read at claim time + /// and never past the admitted cutoff or the run's current history. A run + /// that grew past native's one bounded page fails the query instead. + async fn query_delivery_history(conn: &mut DB::Connection, task: &mut Value) -> Result { + let run_id = task["run_id"].as_str().unwrap_or_default().to_owned(); + let run = Self::query("SELECT status,last_history_sequence FROM workflow_runs WHERE id=$1") + .bind(&run_id) + .fetch_one(&mut *conn) + .await?; + let last = DB::number(&run, "last_history_sequence")?; + let cutoff = task["history_cutoff_sequence"] + .as_i64() + .map(|cutoff| cutoff.min(last).max(0)); + let (mut history, next) = Self::history_page_connection(conn, &run_id, 0, 1000).await?; + let covered = history + .last() + .and_then(|event| event["sequence"].as_i64()) + .unwrap_or(0); + if let Some(cutoff) = cutoff { + history.retain(|event| event["sequence"].as_i64().is_some_and(|sequence| sequence <= cutoff)); + } + let complete = next.is_none() || cutoff.is_some_and(|cutoff| covered >= cutoff); + if !complete || serde_json::to_vec(&history)?.len() > SNAPSHOT_BYTES { + task["status"] = json!("failed"); + task["failure"] = json!({"reason":"query_snapshot_exceeds_budget", + "message":"The run's history is larger than one query snapshot."}); + return Ok(false); + } + task["run_status"] = json!(DB::string(&run, "status")?); + task["history_events"] = json!(history); + task["last_history_sequence"] = json!(cutoff.unwrap_or(last)); + task["history_cutoff_sequence"] = json!(cutoff); + Ok(true) } fn cached_query(row: &DB::Row, id: &str, namespace: &str) -> Result { diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 89fa3a9cd..30adcdca0 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -67,6 +67,8 @@ mod worker_deregistration; #[path = "execution/worker_listing.rs"] mod worker_listing; +#[path = "execution/query_open_tasks.rs"] +mod query_open_tasks; #[path = "execution/worker_pressure.rs"] mod worker_pressure; @@ -2349,15 +2351,9 @@ async fn queries_fence_recovered_leases_across_pools_without_mutating_the_run() started["run_id"].as_str().unwrap() ); let input = json!({"input":envelope(Payload::Array(vec![Payload::Long(9007199254740993)]))}); - assert_eq!( - request(&a, "POST", &query_path, input.clone()).await.1["reason"], - "query_snapshot_busy" - ); + // Queries during an open workflow task wait for their claim; see + // `query_open_tasks`. let workflow_task = poll(&a, "query-a", "workflow").await; - assert_eq!( - request(&a, "POST", &query_path, input.clone()).await.1["reason"], - "query_snapshot_busy" - ); let complete_path = format!( "/api/worker/workflow-tasks/{}/complete", workflow_task["task_id"].as_str().unwrap() diff --git a/rust/tests/execution/query_open_tasks.rs b/rust/tests/execution/query_open_tasks.rs new file mode 100644 index 000000000..3cc32bed0 --- /dev/null +++ b/rust/tests/execution/query_open_tasks.rs @@ -0,0 +1,186 @@ +//! Queries admitted while their run has an open workflow task, as in published +//! PHP's `WorkflowQueryTaskBroker`: the query waits for its claim instead of +//! being refused, and a task leased at admission fixes its history cutoff. +use super::*; + +async fn register_query_worker(app: &Router, worker: &str) { + let (status, body) = request(app, "POST", "/api/worker/register", json!({ + "worker_id":worker,"task_queue":"test","runtime":"php","supported_workflow_types":["echo"], + "supported_activity_types":[],"capabilities":["query_tasks"], + "workflow_command_contracts":{"echo":{"queries":["state"],"signals":["go"], + "signal_contracts":[{"name":"go","parameters":[]}]}}})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); +} + +async fn query_poll(app: &Router, worker: &str, timeout: u64) -> Value { + let (status, body) = request(app, "POST", "/api/worker/query-tasks/poll", + json!({"worker_id":worker,"task_queue":"test","timeout_seconds":timeout})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + body +} + +fn query_request(app: &Router, workflow: &str, run: &Value) -> tokio::task::JoinHandle<(StatusCode, Value)> { + let app = app.clone(); + let path = format!("/api/workflows/{workflow}/runs/{}/query/state", run.as_str().unwrap()); + tokio::spawn(async move { + request(&app, "POST", &path, json!({"input":envelope(Payload::Array(vec![]))})).await + }) +} + +/// Polls until the query is admitted and the poll reports something other +/// than an empty queue. +async fn admitted_poll(app: &Router, worker: &str, timeout: u64) -> Value { + tokio::time::timeout(std::time::Duration::from_secs(10), async { + loop { + let body = query_poll(app, worker, timeout).await; + if body["poll_status"] != "empty" { + break body; + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + }) + .await + .expect("the query is admitted") +} + +async fn answer(app: &Router, claim: &Value, value: i64) { + let (status, body) = request(app, "POST", + &format!("/api/worker/query-tasks/{}/complete", claim["query_task_id"].as_str().unwrap()), + json!({"lease_owner":claim["lease_owner"],"query_task_attempt":claim["query_task_attempt"], + "result_envelope":envelope(Payload::Long(value))})).await; + assert_eq!(status, StatusCode::OK, "{body}"); +} + +fn sequences(claim: &Value) -> Vec { + claim["history_events"] + .as_array() + .unwrap() + .iter() + .map(|event| event["sequence"].as_i64().unwrap()) + .collect() +} + +#[tokio::test] +async fn a_query_during_a_leased_workflow_task_waits_and_reads_history_to_its_cutoff() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + register_query_worker(&app, "query-worker").await; + let started = start(&app, "leased-query").await; + let task = poll(&app, "query-worker", "workflow").await; + let before = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let cutoff = before.as_array().unwrap().len() as i64; + let query = query_request(&app, "leased-query", &started["run_id"]); + // While the task stays leased the query waits, and a poll that finds + // only held queries ends as PHP's `workflow_task_leased`. + let held = admitted_poll(&app, "query-worker", 1).await; + assert_eq!(held["task"], Value::Null, "{held}"); + assert_eq!(held["poll_status"], "workflow_task_leased"); + assert!(!query.is_finished()); + assert_eq!( + finish_task(&app, &task, json!([{"type":"start_timer","delay_seconds":60}])).await.0, + StatusCode::OK + ); + let claim = query_poll(&app, "query-worker", 5).await; + assert_eq!(claim["poll_status"], "leased", "{claim}"); + let claim = &claim["task"]; + // The task leased at admission fixed the history the query reads. + assert_eq!(claim["history_cutoff_sequence"], cutoff); + assert_eq!(claim["last_history_sequence"], cutoff); + assert_eq!(sequences(claim), (1..=cutoff).collect::>()); + let after = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + assert!(after.as_array().unwrap().len() as i64 > cutoff); + answer(&app, claim, 7).await; + let (status, body) = query.await.unwrap(); + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["result_envelope"], envelope(Payload::Long(7))); + runtime.close().await; + database.remove().await; +} + +#[tokio::test] +async fn a_ready_resume_task_holds_the_query_until_the_worker_applies_it() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + register_query_worker(&app, "query-worker").await; + let started = start(&app, "resumed-query").await; + let task = poll(&app, "query-worker", "workflow").await; + assert_eq!( + finish_task(&app, &task, json!([{"type":"start_timer","delay_seconds":1}])).await.0, + StatusCode::OK + ); + // The fired timer's resume task is ready but its effect is not in history. + let resumed = tokio::time::timeout(std::time::Duration::from_secs(10), async { + loop { + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + if history.as_array().unwrap().iter().any(|event| event["event_type"] == "TimerFired") { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + }); + resumed.await.expect("the timer fires"); + let query = query_request(&app, "resumed-query", &started["run_id"]); + // PHP ends such a poll at once with `workflow_task_pending`. + let pending = admitted_poll(&app, "query-worker", 5).await; + assert_eq!(pending["task"], Value::Null, "{pending}"); + assert_eq!(pending["poll_status"], "workflow_task_pending"); + let resume = poll(&app, "query-worker", "workflow").await; + assert!(!resume.is_null()); + // Leased, the resume task still holds the query. + assert_eq!(query_poll(&app, "query-worker", 1).await["poll_status"], "workflow_task_leased"); + assert_eq!( + finish_task(&app, &resume, json!([{"type":"start_timer","delay_seconds":60}])).await.0, + StatusCode::OK + ); + let claim = query_poll(&app, "query-worker", 5).await; + assert_eq!(claim["poll_status"], "leased", "{claim}"); + let claim = &claim["task"]; + // Admitted without a leased task, the query reads the history at its claim. + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let last = history.as_array().unwrap().len() as i64; + assert_eq!(claim["history_cutoff_sequence"], Value::Null); + assert_eq!(claim["last_history_sequence"], last); + assert_eq!(sequences(claim), (1..=last).collect::>()); + answer(&app, claim, 8).await; + assert_eq!(query.await.unwrap().0, StatusCode::OK); + runtime.close().await; + database.remove().await; +} + +#[tokio::test] +async fn a_runs_first_task_and_an_observed_signal_do_not_hold_the_query() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + register_query_worker(&app, "query-worker").await; + let started = start(&app, "first-task-query").await; + // A ready first task is not a resume: the query is claimed at once. + let query = query_request(&app, "first-task-query", &started["run_id"]); + let claim = admitted_poll(&app, "query-worker", 5).await; + assert_eq!(claim["poll_status"], "leased", "{claim}"); + assert_eq!(claim["task"]["history_cutoff_sequence"], Value::Null); + answer(&app, &claim["task"], 1).await; + assert_eq!(query.await.unwrap().0, StatusCode::OK); + let task = poll(&app, "query-worker", "workflow").await; + assert_eq!( + finish_task(&app, &task, json!([{"type":"open_signal_wait","signal_name":"go"}])).await.0, + StatusCode::OK + ); + let signal = request(&app, "POST", + &format!("/api/workflows/first-task-query/runs/{}/signal/go", started["run_id"].as_str().unwrap()), + json!({"input":envelope(Payload::Array(vec![]))})).await; + assert_eq!(signal.0, StatusCode::ACCEPTED, "{}", signal.1); + // The signal's resume task is ready, but the query already observes its + // `SignalReceived`, so it does not wait for the worker. + let query = query_request(&app, "first-task-query", &started["run_id"]); + let claim = admitted_poll(&app, "query-worker", 5).await; + assert_eq!(claim["poll_status"], "leased", "{claim}"); + let events = &claim["task"]["history_events"]; + assert_eq!(events.as_array().unwrap().last().unwrap()["event_type"], "SignalReceived"); + answer(&app, &claim["task"], 2).await; + assert_eq!(query.await.unwrap().0, StatusCode::OK); + runtime.close().await; + database.remove().await; +} From 7faaf93aacc006714584e330cc25334e8c7078a4 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 20:04:29 +0000 Subject: [PATCH 14/30] Record the query open-task checkpoint Co-Authored-By: Claude Opus 5.5 --- .../2026-10-21-query-open-tasks.md | 47 +++++++++++++++++++ docs/rust-server-port.md | 11 ----- 2 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 docs/rust-server-port-checkpoints/2026-10-21-query-open-tasks.md diff --git a/docs/rust-server-port-checkpoints/2026-10-21-query-open-tasks.md b/docs/rust-server-port-checkpoints/2026-10-21-query-open-tasks.md new file mode 100644 index 000000000..0fcd044b5 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-21-query-open-tasks.md @@ -0,0 +1,47 @@ +# October 21, queries during open workflow tasks + +- **Found by.** The Sample App's update scenario (`scripts/sdk-updates.sh`), + run unchanged against native. After nine cross-SDK update round trips, its + snapshot query reached a Rust workflow that had an open workflow task. + Native refused the query with `409 query_snapshot_busy`, a known gap. +- **Published PHP (`WorkflowQueryTaskBroker`).** + - **Admission.** PHP admits the query and records the run's history + sequence. If a workflow task holds a live lease at that moment, the + same sequence becomes the query's history cutoff. + - **Claim.** A worker cannot claim the query while any workflow task of + the run holds a live lease. It also cannot while a due resume task + (signal, update, activity, child or timer) is ready and the query + cannot see its effect. Two resume tasks do not hold the claim: one + created after a query with a cutoff was admitted, and a signal whose + `SignalReceived` event the query already observed. A run's first + workflow task is not a resume task. + - **Poll status.** A poll that finds only held queries returns + `workflow_task_pending` at once when a ready resume task holds them. It + returns `workflow_task_leased` when its wait ends while a lease holds + them. + - **Delivery.** The claim delivers the history read at claim time, up to + the cutoff (and never past the run's current history) or all of it + without one. `history_cutoff_sequence` is `null` without a cutoff. +- **Correction.** The earlier known-gap entry said a query sent right after + a signal waits on PHP. PHP records `SignalReceived` when the signal is + accepted, and the query observes it, so that resume task does not hold + the claim. +- **Native.** It follows these rules. Native's activity and ordinary timer + resume tasks name their source without a `resume_source_kind`, so native + recognizes them by `activity_execution_id` and `timer_id`. A run whose + history is already larger than native's one bounded query page is still + refused at admission with `413 query_snapshot_exceeds_budget`. A run that + grows past it before the claim fails the query with that reason. A poll + held by a lease re-checks with a read-only snapshot instead of the + transition lock, and wakes when the task ends. +- **Verification.** + - New tests cover: + - a query admitted during a leased task: it waits, the poll reports + `workflow_task_leased`, and after completion it reads history only to + its cutoff; + - a ready timer resume: `workflow_task_pending` at once, still held + while that task is leased, then the full history at the claim; + - a run's first task and an observed signal, neither of which holds the + claim. + - The existing query lease and replay tests pass unchanged. The two + `query_snapshot_busy` assertions they made were removed. diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 0a7394960..150869d17 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -187,17 +187,6 @@ contract fixture before its correction. `upgrade_source_catalog_unqualified` until its recipe lands. The recipe must also refuse rows that carry trace context until native implements the contract. The boundary is agreed on #325. -- **Queries while a workflow task is open.** Native answers a query from a - history snapshot taken at admission, so it refuses with 409 - `query_snapshot_busy` while the run has a ready or leased workflow task. - Published PHP admits the query. A workflow task leased at admission fixes - the query's history cutoff at that point. The query's claim then waits - while a workflow task is leased, or while an earlier resume task (signal, - update, activity, child or timer) is ready and not yet in history. A - run's first workflow task does not hold the claim. A query sent right after - a signal therefore waits on PHP and is refused on native. The port takes the - snapshot at claim time and must keep query lease replay stable, so it needs - its own fixture. - **Terminating a child run.** Published Workflow resolves the parent with `ChildRunTerminated` and the `terminated` child call status. Native refuses this with `422 termination_propagation_not_available` until the parent-side From 28741de99445a70f37915d8f82902de0ea508b81 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 20:33:13 +0000 Subject: [PATCH 15/30] Narrow the structural limits gap to what PHP enforces for external workers Co-Authored-By: Claude Opus 5.5 --- .../2026-10-21-legacy-auth-env.md | 3 ++- docs/rust-server-port.md | 11 ++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md b/docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md index 7aba64ff0..b180ce19e 100644 --- a/docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md +++ b/docs/rust-server-port-checkpoints/2026-10-21-legacy-auth-env.md @@ -29,7 +29,8 @@ configuration, but no code reads those entries. The structural limits PHP enforces come from the Workflow package's own `DW_V2_LIMIT_*` settings. Ignoring these four settings matches PHP. Of the package - limits, native enforces only the memo size; #612 tracks the rest. `DW_MAX_TASKS_PER_POLL` only widens + limits PHP applies to external workers, native enforces only the memo + size; #612 tracks the rest. `DW_MAX_TASKS_PER_POLL` only widens PHP's candidate scan (ten candidates for each task); a poll still claims one task, so native's own scan width is not a contract difference. - **PHP process roles and topology.** `DW_MODE`, `DW_SERVER_ID`, diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index b20b94ef6..9ec637738 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -102,11 +102,12 @@ contract fixture before its correction. each priority tier across fairness keys by their weights (`TaskFairnessScheduler`); native stores the keys and weights but does not rebalance yet. -- **Structural limits.** Published PHP enforces the Workflow package's - structural limits (pending activities, children, timers, signals and - updates, command batch size, payload size, search attribute size and - history transaction size), configured by `DW_V2_LIMIT_*`. Native enforces - only the memo size limit. #612 tracks the rest. +- **Structural limits.** For external workers, published PHP enforces four + of the Workflow package's `DW_V2_LIMIT_*` limits: the memo size, the + command batch size as a bound on parallel group metadata, the pending + signal and update counts with their payload size at intake, and the + payload size of an activity's result. Native enforces only the memo size. + #612 tracks the rest. - **Malformed input envelopes.** Published PHP refuses a malformed `input` envelope on start, signal, update and query with `422 validation_failed`, using Laravel messages on `input.codec` or `input.blob`. The cases are a From 6eae18eadb61b9fd0088fffc6e1f028511c6cc28 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 20:43:36 +0000 Subject: [PATCH 16/30] Retry a failed workflow task that carries an update or a signal Published PHP answers a reported workflow task failure by marking the task failed and, unless the failure blocks replay, creating a ready retry task with the same payload, including any update or signal it carries. Native refused the report with development_workflow_task_retry_messages_ unqualified when the run had pending messages. The Rust SDK stops its worker on that refusal, so the run's accepted update was never applied. Native now retries such a task as PHP does. The waiting-for-history acknowledgement with pending messages keeps its refusal. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/store.rs | 20 +++--- rust/tests/execution.rs | 2 + rust/tests/execution/retry_message_tasks.rs | 75 +++++++++++++++++++++ 3 files changed, 87 insertions(+), 10 deletions(-) create mode 100644 rust/tests/execution/retry_message_tasks.rs diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index e86052ed9..2595201d9 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -1557,19 +1557,19 @@ where .bind(&run_id) .fetch_one(&mut *tx) .await?; - if signals != 0 - || updates != 0 - || payload["workflow_signal_id"].is_string() - || payload["workflow_update_id"].is_string() - || Self::cancellation_request(&run)?.is_some() + // Published PHP retries a failed task as it is, with any signal or + // update it carries; its waiting acknowledgement with pending + // messages still needs its own contract. + if waiting + && (signals != 0 + || updates != 0 + || payload["workflow_signal_id"].is_string() + || payload["workflow_update_id"].is_string() + || Self::cancellation_request(&run)?.is_some()) { return Err(refuse( StatusCode::UNPROCESSABLE_ENTITY, - if waiting { - "development_workflow_task_waiting_messages_unqualified" - } else { - "development_workflow_task_retry_messages_unqualified" - }, + "development_workflow_task_waiting_messages_unqualified", )); } if !waiting { diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 89fa3a9cd..952cf44f8 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -67,6 +67,8 @@ mod worker_deregistration; #[path = "execution/worker_listing.rs"] mod worker_listing; +#[path = "execution/retry_message_tasks.rs"] +mod retry_message_tasks; #[path = "execution/worker_pressure.rs"] mod worker_pressure; diff --git a/rust/tests/execution/retry_message_tasks.rs b/rust/tests/execution/retry_message_tasks.rs new file mode 100644 index 000000000..0ab9bc72b --- /dev/null +++ b/rust/tests/execution/retry_message_tasks.rs @@ -0,0 +1,75 @@ +//! A failed workflow task that carries an update or a signal is retried as +//! published PHP retries it: a new ready task with the same payload. +use super::*; + +async fn fail_task(app: &Router, task: &Value, message: &str) -> (StatusCode, Value) { + request(app, "POST", + &format!("/api/worker/workflow-tasks/{}/fail", task["task_id"].as_str().unwrap()), + json!({"lease_owner":task["lease_owner"],"workflow_task_attempt":task["workflow_task_attempt"], + "failure":{"message":message,"type":"RuntimeException"}})).await +} + +#[tokio::test] +async fn a_failed_update_task_is_retried_with_its_update() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + update_worker(&app, "update-worker").await; + let started = start(&app, "retried-update").await; + open_update_wait(&app, "update-worker").await; + let receipt = request(&app, "POST", "/api/workflows/retried-update/update/set", + json!({"input":envelope(Payload::Array(vec![Payload::Long(7)])),"wait_for":"accepted"})).await; + assert_eq!(receipt.0, StatusCode::ACCEPTED, "{}", receipt.1); + let task = poll(&app, "update-worker", "workflow").await; + assert_eq!(task["workflow_update_id"], receipt.1["update_id"]); + let failed = fail_task(&app, &task, "handler threw").await; + assert_eq!(failed.0, StatusCode::OK, "{}", failed.1); + assert_eq!(failed.1["recorded"], true); + let retry_id = failed.1["next_task_id"].clone(); + assert!(retry_id.is_string(), "{}", failed.1); + // The retry carries the same update, and the worker can complete it. + let retry = poll(&app, "update-worker", "workflow").await; + assert_eq!(retry["task_id"], retry_id); + assert_eq!(retry["workflow_update_id"], receipt.1["update_id"]); + let completed = request(&app, "POST", + &format!("/api/worker/workflow-tasks/{}/complete", retry["task_id"].as_str().unwrap()), + completion(&retry, json!([{"type":"complete_update","update_id":receipt.1["update_id"], + "result":envelope(Payload::Long(7))}]))).await; + assert_eq!(completed.0, StatusCode::OK, "{}", completed.1); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let completions = history.as_array().unwrap().iter() + .filter(|event| event["event_type"] == "UpdateCompleted").count(); + assert_eq!(completions, 1, "{history}"); + runtime.close().await; + database.remove().await; +} + +#[tokio::test] +async fn a_failed_signal_task_is_retried_with_its_signal() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + update_worker(&app, "update-worker").await; + let started = start(&app, "retried-signal").await; + open_update_wait(&app, "update-worker").await; + let signal = request(&app, "POST", "/api/workflows/retried-signal/signal/finish", + json!({"input":envelope(Payload::Array(vec![Payload::Long(1)]))})).await; + assert_eq!(signal.0, StatusCode::ACCEPTED, "{}", signal.1); + let task = poll(&app, "update-worker", "workflow").await; + assert_eq!(task["resume_source_kind"], "workflow_signal", "{task}"); + let failed = fail_task(&app, &task, "signal handler failed").await; + assert_eq!(failed.0, StatusCode::OK, "{}", failed.1); + let retry = poll(&app, "update-worker", "workflow").await; + assert_eq!(retry["task_id"], failed.1["next_task_id"]); + assert_eq!(retry["resume_source_kind"], "workflow_signal"); + assert_eq!(retry["signal_wait_id"], task["signal_wait_id"]); + let completed = request(&app, "POST", + &format!("/api/worker/workflow-tasks/{}/complete", retry["task_id"].as_str().unwrap()), + completion(&retry, json!([{"type":"complete_workflow","result":envelope(Payload::Long(1))}]))).await; + assert_eq!(completed.0, StatusCode::OK, "{}", completed.1); + let run = request(&app, "GET", "/api/workflows/retried-signal", Value::Null).await; + assert_eq!(run.1["status"], "completed", "{}", run.1); + let _ = started; + runtime.close().await; + database.remove().await; +} From 6e3f07db2aab834fe64a8b363487f375297ddc8f Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 21:01:49 +0000 Subject: [PATCH 17/30] Apply a signal without arguments as true, like PHP Published PHP's signalValueFromRecord applies a signal with no arguments as true, one argument as itself, and several as their list. Native recorded null for a signal without arguments in SignalApplied. The Rust SDK checks that value against the received arguments, so it refused a query over such a history with "conflicting committed signal identity". Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/store.rs | 3 +- rust/tests/execution.rs | 2 + rust/tests/execution/signal_applied_values.rs | 39 +++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 rust/tests/execution/signal_applied_values.rs diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 2595201d9..57024b772 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -2935,8 +2935,9 @@ pub(super) async fn prepare_control_arguments( else { return Err(refuse(StatusCode::UNPROCESSABLE_ENTITY, invalid)); }; + // `signalValueFromRecord`: no arguments apply as true, one as itself. let value = match values.as_slice() { - [] => crate::codec::Value::Null, + [] => crate::codec::Value::Boolean(true), [value] => value.clone(), _ => crate::codec::Value::Array(values.clone()), }; diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 952cf44f8..af13c4d7e 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -69,6 +69,8 @@ mod worker_deregistration; mod worker_listing; #[path = "execution/retry_message_tasks.rs"] mod retry_message_tasks; +#[path = "execution/signal_applied_values.rs"] +mod signal_applied_values; #[path = "execution/worker_pressure.rs"] mod worker_pressure; diff --git a/rust/tests/execution/signal_applied_values.rs b/rust/tests/execution/signal_applied_values.rs new file mode 100644 index 000000000..3d7faeb0f --- /dev/null +++ b/rust/tests/execution/signal_applied_values.rs @@ -0,0 +1,39 @@ +//! The value a `SignalApplied` event records, as published PHP's +//! `signalValueFromRecord` derives it from the signal's arguments. +use super::*; + +#[tokio::test] +async fn applied_signal_values_follow_php() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let (status, body) = request(&app, "POST", "/api/worker/register", json!({ + "worker_id":"signal-worker","task_queue":"test","runtime":"php","supported_workflow_types":["echo"], + "supported_activity_types":[],"workflow_command_contracts":{"echo":{"signals":["go"]}}})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); + let started = start(&app, "applied-values").await; + let mut task = poll(&app, "signal-worker", "workflow").await; + let cases = [ + (vec![], Payload::Boolean(true)), + (vec![Payload::Long(7)], Payload::Long(7)), + (vec![Payload::Long(1), Payload::Long(2)], Payload::Array(vec![Payload::Long(1), Payload::Long(2)])), + ]; + for (arguments, _) in &cases { + assert_eq!(finish_task(&app, &task, json!([{"type":"open_signal_wait","signal_name":"go"}])).await.0, StatusCode::OK); + let signal = request(&app, "POST", "/api/workflows/applied-values/signal/go", + json!({"input":envelope(Payload::Array(arguments.clone()))})).await; + assert_eq!(signal.0, StatusCode::ACCEPTED, "{}", signal.1); + task = poll(&app, "signal-worker", "workflow").await; + assert!(!task.is_null()); + } + assert_eq!(finish_task(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::Long(0))}])).await.0, StatusCode::OK); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let applied: Vec = history.as_array().unwrap().iter() + .filter(|event| event["event_type"] == "SignalApplied") + .map(|event| event["payload"]["value"].clone()) + .collect(); + let expected: Vec = cases.iter().map(|(_, value)| envelope(value.clone())).collect(); + assert_eq!(applied, expected, "{history}"); + runtime.close().await; + database.remove().await; +} From ffe23b0f6f11fd4e0e59564596be99d24b4eca08 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 21:03:37 +0000 Subject: [PATCH 18/30] Record the update scenario message checkpoint Co-Authored-By: Claude Opus 5.5 --- .../2026-10-21-update-scenario-messages.md | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 docs/rust-server-port-checkpoints/2026-10-21-update-scenario-messages.md diff --git a/docs/rust-server-port-checkpoints/2026-10-21-update-scenario-messages.md b/docs/rust-server-port-checkpoints/2026-10-21-update-scenario-messages.md new file mode 100644 index 000000000..bb3b768ce --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-21-update-scenario-messages.md @@ -0,0 +1,32 @@ +# October 21, signal values and retried message tasks + +- **Found by.** The Sample App's update scenario (`scripts/sdk-updates.sh`) + against native, in two runs: + - In one run, the Rust workflow failed the workflow task for its + `snapshot` update. Native refused the failure report with `422 + development_workflow_task_retry_messages_unqualified`, the Rust SDK + stopped its worker, and the accepted update was never applied. + - In another run, the Rust SDK refused the scenario's snapshot query with + "cannot decode committed workflow signals: conflicting committed signal + identity". + + Both handlers read the run's committed signals. +- **Signal values.** Published PHP's `signalValueFromRecord` applies a + signal with no arguments as `true`, one argument as itself and several as + their list, and records that value in `SignalApplied`. Native recorded + null for a signal without arguments. The Rust SDK compares each applied + value with the received arguments, so a history holding the scenario's + argument-less signal was refused. Native now records `true`. +- **Retried message tasks.** Published PHP answers a reported workflow task + failure by marking the task failed. Unless the failure blocks replay, it + creates a ready retry task with the same payload, including any update or + signal the task carries. Native refused such a report whenever the run + had pending messages. It now retries the task as PHP does. The + waiting-for-history acknowledgement with pending messages keeps its + refusal. +- **Verification.** + - New tests cover: + - the applied values of signals with zero, one and two arguments; + - a failed update task, retried with its update and then completed once; + - a failed signal task, retried with its signal. + - The signal and retry suites pass on SQLite. From 48aadb899154f19e2ab91fbc289ebfc183d26d0f Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 21:08:00 +0000 Subject: [PATCH 19/30] Record the missing search attribute upsert command Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-port.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 0a7394960..a49b8bac5 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -198,6 +198,10 @@ contract fixture before its correction. a signal therefore waits on PHP and is refused on native. The port takes the snapshot at claim time and must keep query lease replay stable, so it needs its own fixture. +- **Search attribute upserts.** Native has no `upsert_search_attributes` + workflow command and refuses it with `unsupported_workflow_command`. The + Sample App's search attribute scenario stops there, on its Rust worker's + first upsert. - **Terminating a child run.** Published Workflow resolves the parent with `ChildRunTerminated` and the `terminated` child call status. Native refuses this with `422 termination_propagation_not_available` until the parent-side From 22e70d5c79aa6ffec823d49c3eaa779c1fae2f37 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 22:01:03 +0000 Subject: [PATCH 20/30] Wait for an unreachable database at startup, like PHP Published PHP starts while its database is unavailable (server:assert-php-database --allow-unavailable). Native exited on the first refused connection, so an unchanged Compose file failed when MySQL's health check passed during its temporary init server. Native now retries a database it cannot reach yet with a capped backoff: refused or dropped connections, unresolved hosts, pool timeouts and PostgreSQL's starting-up state. A refused login or a configuration error still stops the process. SIGTERM or SIGINT while waiting exits cleanly. Co-Authored-By: Claude Opus 5.5 --- .../2026-10-21-database-wait.md | 28 ++++ docs/rust-server-port.md | 6 + rust/src/main.rs | 135 +++++++++++++++--- 3 files changed, 150 insertions(+), 19 deletions(-) create mode 100644 docs/rust-server-port-checkpoints/2026-10-21-database-wait.md diff --git a/docs/rust-server-port-checkpoints/2026-10-21-database-wait.md b/docs/rust-server-port-checkpoints/2026-10-21-database-wait.md new file mode 100644 index 000000000..d11dcadf2 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-21-database-wait.md @@ -0,0 +1,28 @@ +# October 21, waiting for the database at startup + +- **Found by.** The Sample App's update scenario against native, on a fresh + VM. MySQL's health check passed while its entrypoint ran a temporary + socket-only server. Native connected just after that server stopped and + before the real one listened, got `Connection refused` and exited, and + Compose reported `dependency failed to start`. +- **PHP's behavior.** The published image's entrypoint runs + `server:assert-php-database --allow-unavailable` and starts Apache while + the database is unavailable. Requests fail until the database answers, + but the server keeps running. +- **Native's behavior now.** When the database cannot be reached yet, + native logs one line and retries with a backoff from 250 ms up to 5 s, + then logs that the database is reachable. Cases that count as not reachable + yet: the connection is refused or drops, the host doesn't resolve yet, the + pool times out, or PostgreSQL answers SQLSTATE 57P03 (starting up). A + refused login or a configuration error still stops the process. + `SIGTERM` or `SIGINT` while waiting exits cleanly, even during a + connection attempt. +- **Stop signal.** The published image declares `STOPSIGNAL SIGWINCH`. + Native does not stop on it, so this is recorded as a known gap for a + native image. +- **Verification.** + - A unit test pins which errors are awaited. + - Locally, native started four seconds before a fresh MySQL 8.0 container. + It waited, connected, and `/api/health` answered 200. + - Native waiting on an unresolvable host exited 0 within 0.4 s of + `SIGTERM`. diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 0a7394960..835cf54a8 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -198,6 +198,12 @@ contract fixture before its correction. a signal therefore waits on PHP and is refused on native. The port takes the snapshot at claim time and must keep query lease replay stable, so it needs its own fixture. +- **Stop signal in the published image.** The published Server image declares + `STOPSIGNAL SIGWINCH`, Apache's graceful stop. Native stops gracefully on + `SIGTERM` or `SIGINT` and ignores `SIGWINCH`, so `docker stop` on the + published image with the native binary as its entrypoint waits out its + timeout and kills the process. A native image must declare + `STOPSIGNAL SIGTERM`. - **Terminating a child run.** Published Workflow resolves the parent with `ChildRunTerminated` and the `terminated` child call status. Native refuses this with `422 termination_propagation_not_available` until the parent-side diff --git a/rust/src/main.rs b/rust/src/main.rs index 22003f39f..77dc2dd37 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -14,6 +14,85 @@ fn tls_settings(driver: Driver) -> Result<(Settings, Option), Box(open: F) -> Result, RuntimeError> +where + F: Fn() -> Fut, + Fut: std::future::Future>, +{ + // One listener covers every attempt: the pool retries a refused + // connection inside each one, and a stop request must not wait for it. + let shutdown = shutdown_signal(); + tokio::pin!(shutdown); + let mut delay = std::time::Duration::from_millis(250); + let mut waiting = false; + loop { + let attempt = tokio::select! { + result = open() => result, + () = &mut shutdown => return Ok(None), + }; + match attempt { + Err(error) if unreachable(&error) => { + if !waiting { + eprintln!("database unavailable ({error}); waiting for it to accept connections"); + waiting = true; + } + tokio::select! { + () = tokio::time::sleep(delay) => {} + () = &mut shutdown => return Ok(None), + } + delay = (delay * 2).min(std::time::Duration::from_secs(5)); + } + Err(error) => return Err(error), + Ok(runtime) => { + if waiting { + eprintln!("database reachable"); + } + return Ok(Some(runtime)); + } + } + } +} + +/// A database that is not accepting connections yet: the connection is +/// refused or drops, its host does not resolve yet, the pool times out, or +/// PostgreSQL is still starting (SQLSTATE 57P03). +fn unreachable(error: &RuntimeError) -> bool { + fn retryable(error: &sqlx::Error) -> bool { + match error { + sqlx::Error::Io(_) | sqlx::Error::PoolTimedOut => true, + sqlx::Error::Database(error) => error.code().as_deref() == Some("57P03"), + _ => false, + } + } + match error { + RuntimeError::Database(error) => retryable(error), + RuntimeError::Migration(sqlx::migrate::MigrateError::Execute(error)) => retryable(error), + _ => false, + } +} + +async fn shutdown_signal() { + let ctrl_c = async { + let _ = tokio::signal::ctrl_c().await; + }; + #[cfg(unix)] + let terminate = async { + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .expect("install SIGTERM handler") + .recv() + .await; + }; + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + tokio::select! { () = ctrl_c => {}, _ = terminate => {} } +} + /// A connection that required TLS and failed is reported as PHP reports it. fn tls_connection_error(settings: &Settings, error: RuntimeError) -> Box { if settings.required() && matches!(error, RuntimeError::Database(_)) { @@ -426,17 +505,27 @@ async fn main() -> Result<(), Box> { } "pgsql" => { let (tls, _) = tls_settings(Driver::Postgres)?; - let runtime = Runtime::open_postgres(postgres_options()?, token) - .await - .map_err(|error| tls_connection_error(&tls, error))?; + let options = postgres_options()?; + let Some(runtime) = + open_when_reachable(|| Runtime::open_postgres(options.clone(), token.clone())) + .await + .map_err(|error| tls_connection_error(&tls, error))? + else { + return Ok(()); + }; database_tls::record(&tls); runtime } "mysql" | "mariadb" => { let (tls, _) = tls_settings(Driver::MySql)?; - let runtime = Runtime::open_mysql(mysql_options()?, token) - .await - .map_err(|error| tls_connection_error(&tls, error))?; + let options = mysql_options()?; + let Some(runtime) = + open_when_reachable(|| Runtime::open_mysql(options.clone(), token.clone())) + .await + .map_err(|error| tls_connection_error(&tls, error))? + else { + return Ok(()); + }; database_tls::record(&tls); runtime } @@ -454,20 +543,28 @@ async fn main() -> Result<(), Box> { listener, router(runtime.clone()).into_make_service_with_connect_info::(), ) - .with_graceful_shutdown(async { - let ctrl_c = tokio::signal::ctrl_c(); - #[cfg(unix)] - let terminate = async { - tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) - .expect("install SIGTERM handler") - .recv() - .await; - }; - #[cfg(not(unix))] - let terminate = std::future::pending::<()>(); - tokio::select! { _ = ctrl_c => {}, _ = terminate => {} } - }) + .with_graceful_shutdown(shutdown_signal()) .await?; runtime.close().await; Ok(()) } + +#[cfg(test)] +mod startup_tests { + use super::*; + + #[test] + fn only_an_unreachable_database_is_awaited() { + let refused = || std::io::Error::from(std::io::ErrorKind::ConnectionRefused); + assert!(unreachable(&RuntimeError::Database(sqlx::Error::Io(refused())))); + assert!(unreachable(&RuntimeError::Database(sqlx::Error::PoolTimedOut))); + assert!(unreachable(&RuntimeError::Migration( + sqlx::migrate::MigrateError::Execute(sqlx::Error::Io(refused())) + ))); + assert!(!unreachable(&RuntimeError::Database(sqlx::Error::Configuration( + "bad url".into() + )))); + assert!(!unreachable(&RuntimeError::Database(sqlx::Error::RowNotFound))); + assert!(!unreachable(&RuntimeError::Io(refused()))); + } +} From bf3b2a649d806e629b4b34594ff1f2f49f8f056c Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 22:35:09 +0000 Subject: [PATCH 21/30] Apply upsert_search_attributes like PHP The Sample App's search attribute scenario stopped at its Rust worker's first upsert, which native refused as an unsupported command. Native now applies the command as published PHP does: - the Server's SearchAttributeValueValidator checks each value against the namespace's registered, declared or inferred type, with PHP's messages under commands.N.attributes; - the Workflow package's UpsertSearchAttributesCall normalizes values, and its key and value errors are reported as PHP reports them; - typed rows are written and deleted, within 100 attributes and 64 KB; - SearchAttributesUpserted records attributes, attribute_types and the merged view; - attribute_types needs worker protocol 1.16 and the typed_search_attributes capability, which also gates claims of runs with typed history and appears in worker discovery; - continue-as-new inherits the rows instead of being refused. A differential probe against PHP 2.5.14 matched every refusal and the history payload. PHP stores an offset datetime as the wrong instant (workflow#898); native stores the instant written. Co-Authored-By: Claude Opus 5.5 --- .../2026-10-21-search-attribute-upserts.md | 61 ++ docs/rust-server-port.md | 26 + rust/src/runtime/backend.rs | 50 ++ rust/src/runtime/continuations.rs | 13 +- rust/src/runtime/http.rs | 37 + rust/src/runtime/mod.rs | 1 + rust/src/runtime/run_describe.rs | 30 +- rust/src/runtime/search_attribute_upserts.rs | 845 ++++++++++++++++++ rust/src/runtime/search_attributes.rs | 2 +- rust/src/runtime/store.rs | 39 +- rust/tests/execution.rs | 2 + .../execution/search_attribute_upserts.rs | 158 ++++ 12 files changed, 1235 insertions(+), 29 deletions(-) create mode 100644 docs/rust-server-port-checkpoints/2026-10-21-search-attribute-upserts.md create mode 100644 rust/src/runtime/search_attribute_upserts.rs create mode 100644 rust/tests/execution/search_attribute_upserts.rs diff --git a/docs/rust-server-port-checkpoints/2026-10-21-search-attribute-upserts.md b/docs/rust-server-port-checkpoints/2026-10-21-search-attribute-upserts.md new file mode 100644 index 000000000..4fb1a3a68 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-21-search-attribute-upserts.md @@ -0,0 +1,61 @@ +# October 21, search attribute upserts + +- **Found by.** The Sample App's search attribute scenario + (`scripts/sdk-search-attributes.sh`) against native. Its Rust workflow's + first `upsert_search_attributes` command was refused with + `unsupported_workflow_command`. +- **Validation.** Native applies the Server's `SearchAttributeValueValidator` + to each upsert: + - Keys must start with a letter and contain only letters, digits and + underscores, within `DW_MAX_SEARCH_ATTRIBUTE_KEY_LENGTH` bytes. + - A value must suit its registered type. A declared type must agree with + the registration. A namespace with custom definitions refuses an + unregistered key. + - Without definitions, a value infers its type: bool, int, float, + keyword, string (longer than 255 characters or containing whitespace) + or keyword list. + - Refusals answer `422` with PHP's messages under + `commands.N.attributes`. +- **Normalization and storage.** The Workflow package's + `UpsertSearchAttributesCall` trims strings and list entries and treats an + empty string as a deletion. `SearchAttributeUpsertService` then writes typed + rows (`string`, `keyword`, `keyword_list`, `int`, `float`, `bool`, + `datetime`) and deletes nulls. A run is limited to 100 attributes and 64 KB. +- **History.** `SearchAttributesUpserted` records `sequence`, the + normalized `attributes`, the canonical `attribute_types` and the `merged` + view. +- **Capability.** `attribute_types` needs worker protocol 1.16 + (`typed_search_attributes_unavailable`). It also needs a lease owner that + advertises `typed_search_attributes` + (`workflow_metadata_capability_not_advertised`). Registering that + capability needs protocol 1.16. A worker without it cannot claim a run + whose history has typed upserts. Worker discovery lists the command and + PHP's `typed_search_attributes` block. +- **Continue-as-new.** The continued run inherits the typed rows at sequence + 1 instead of the continuation being refused, and an upsert may precede + `continue_as_new` in the same task. +- **Describe.** Reads keyword lists through each backend's JSON decoding. + PostgreSQL stores them in a `json` column. +- **Remaining.** Workflow lists still show no attributes and refuse typed + query predicates, so the Sample App scenario is not complete yet. Other + remaining differences are recorded as known gaps: start-time search + attributes, child-run inheritance, datetime forms and the limit refusal. +- **Differential probe.** Published PHP 2.5.14 and native received the same + requests: twelve upsert completions covering registered, declared and + inferred types, bad keys, wrong types, malformed `attribute_types`, empty, + null, list and string `attributes`, an over-long key and a non-list value. + - The status, reason and every validation message match. + - A valid upsert's `SearchAttributesUpserted` payload matches: attributes, + types and merged view. + - Describe matches except for an offset datetime. There PHP shifts the + instant (workflow#898) and native keeps it. + - Native's message order differs only when one command has several key + errors, because native visits object keys in sorted order rather than + request order. +- **Verification.** + - Unit tests cover type resolution, PHP's messages, normalization and + the limits. + - New tests in `tests/execution/search_attribute_upserts.rs` cover: + - the Sample App's typed values through two upserts; + - validation, protocol and capability refusals that record nothing; + - continue-as-new inheritance. diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 0a7394960..9f96a5969 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -198,6 +198,32 @@ contract fixture before its correction. a signal therefore waits on PHP and is refused on native. The port takes the snapshot at claim time and must keep query lease replay stable, so it needs its own fixture. +- **Search attributes in workflow lists.** Native lists show + `search_attributes: []`, and the list `query` refuses any comparison with + `422 unsupported_visibility_query`. Published PHP returns each run's typed + attributes and filters with `WorkflowVisibilityQuery` over typed + predicates. The Sample App's search attribute scenario needs both. +- **Inherited memos and search attributes on child runs.** Published PHP + copies the parent's memos and search attributes to a child run and to a + retried child (`inheritTypedVisibilityMetadata`). Native copies both only + on continue-as-new. +- **Search attribute datetime forms.** `Carbon::parse` accepts relative and + free-form dates. Native accepts RFC 3339, and a date or date and time + without an offset read as UTC, and refuses other forms as PHP refuses an + unparsable one. +- **Search attribute datetimes with an offset.** Published PHP stores a + typed datetime written with an offset, such as `2026-10-08T12:34:56+02:00`, + as that wall-clock time in UTC (`12:34:56Z`), which is a different + instant ([workflow#898](https://github.com/durable-workflow/workflow/issues/898)). + Native stores the instant written (`10:34:56Z`). +- **Worker payload datums.** Published PHP refuses a worker command payload, + such as a `complete_workflow` result, that is not one complete Avro Value + (`422 validation_failed` on `commands.N.result.blob`). Native checks only + the envelope's transport shape and accepts it. +- **Search attribute limits.** Over 100 attributes or 64 KB per run, PHP + raises an uncaught `InvalidArgumentException` while applying the command. + Native refuses the completion with `422 search_attribute_limit_exceeded` + and PHP's message, and records nothing. - **Terminating a child run.** Published Workflow resolves the parent with `ChildRunTerminated` and the `terminated` child call status. Native refuses this with `422 termination_propagation_not_available` until the parent-side diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 9d5df7249..a7e757907 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -11,6 +11,9 @@ pub(super) trait Backend: Database { type EncodedTime: for<'q> Encode<'q, Self> + Type + Send + Sync + 'static; type EncodedOptionalTime: for<'q> Encode<'q, Self> + Type + Send + Sync + 'static; type EncodedDocument: for<'q> Encode<'q, Self> + Type + Send + Sync + 'static; + type EncodedInteger: for<'q> Encode<'q, Self> + Type + Send + Sync + 'static; + type EncodedFloat: for<'q> Encode<'q, Self> + Type + Send + Sync + 'static; + type EncodedFlag: for<'q> Encode<'q, Self> + Type + Send + Sync + 'static; const HISTORY_PAGE: &'static str; // Multi-statement reads that must observe one committed state. SQLite's // deferred read transaction already keeps one WAL snapshot. @@ -55,6 +58,10 @@ pub(super) trait Backend: Database { const MEMO_UPSERT: &'static str = "INSERT INTO workflow_memos(workflow_run_id,workflow_instance_id,\"key\",value,upserted_at_sequence,inherited_from_parent,created_at,updated_at,portable_value,portable_value_sequence) VALUES ($1,$2,$3,$4,$5,false,$6,$7,$8,$9) ON CONFLICT(workflow_run_id,\"key\") DO UPDATE SET value=excluded.value,upserted_at_sequence=excluded.upserted_at_sequence,inherited_from_parent=false,updated_at=excluded.updated_at,portable_value=excluded.portable_value,portable_value_sequence=excluded.portable_value_sequence"; const MEMO_DELETE: &'static str = "DELETE FROM workflow_memos WHERE workflow_run_id=$1 AND \"key\"=$2"; + const SEARCH_ATTRIBUTE_ROWS: &'static str = "SELECT \"key\" AS attribute_key,type,value_string,value_keyword,value_int,value_float,CASE WHEN value_bool IS NULL THEN NULL WHEN value_bool THEN 1 ELSE 0 END AS value_bool,value_datetime,value_keyword_list FROM workflow_search_attributes WHERE workflow_run_id=$1 ORDER BY \"key\" LIMIT 1001"; + const SEARCH_ATTRIBUTE_UPSERT: &'static str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,\"key\",type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,false,$13,$14) ON CONFLICT(workflow_run_id,\"key\") DO UPDATE SET workflow_instance_id=excluded.workflow_instance_id,type=excluded.type,value_string=excluded.value_string,value_keyword=excluded.value_keyword,value_keyword_list=excluded.value_keyword_list,value_int=excluded.value_int,value_float=excluded.value_float,value_bool=excluded.value_bool,value_datetime=excluded.value_datetime,upserted_at_sequence=excluded.upserted_at_sequence,inherited_from_parent=false,updated_at=excluded.updated_at"; + const SEARCH_ATTRIBUTE_DELETE: &'static str = "DELETE FROM workflow_search_attributes WHERE workflow_run_id=$1 AND \"key\"=$2"; + const SEARCH_ATTRIBUTE_INHERIT: &'static str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,\"key\",type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) SELECT $1,$2,\"key\",type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,1,true,$3,$4 FROM workflow_search_attributes WHERE workflow_run_id=$5"; fn statement(sql: &'static str) -> Cow<'static, str> { Cow::Borrowed(sql) } @@ -62,6 +69,9 @@ pub(super) trait Backend: Database { fn bind_time(time: DateTime) -> Self::EncodedTime; fn optional_time(time: Option>) -> Self::EncodedOptionalTime; fn document(value: &Value) -> Self::EncodedDocument; + fn optional_integer(value: Option) -> Self::EncodedInteger; + fn optional_float_value(value: Option) -> Self::EncodedFloat; + fn optional_flag(value: Option) -> Self::EncodedFlag; fn optional_document(value: &Value) -> Option { (!value.is_null()).then(|| Self::document(value)) } @@ -103,6 +113,18 @@ fn sqlite_instant(value: String) -> Result> { } impl Backend for Sqlite { + type EncodedInteger = Option; + type EncodedFloat = Option; + type EncodedFlag = Option; + fn optional_integer(value: Option) -> Option { + value + } + fn optional_float_value(value: Option) -> Option { + value + } + fn optional_flag(value: Option) -> Option { + value + } const ACTIVITY_METADATA_HISTORY: &'static str = "SELECT sequence,event_type,CASE WHEN projection_bytes<=8388608 THEN payload ELSE NULL END AS payload,workflow_task_id,recorded_at,projection_count FROM (SELECT sequence,event_type,payload,workflow_task_id,recorded_at,COUNT(*) OVER () AS projection_count,SUM(length(CAST(payload AS BLOB))) OVER (ORDER BY sequence) AS projection_bytes FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled') ORDER BY sequence LIMIT 10001) AS page ORDER BY sequence"; const PHP_CONNECTION: &str = "sqlite"; // Carbon's millisecond comparison rounds at half a millisecond. Parse only @@ -277,6 +299,18 @@ impl Encode<'_, Postgres> for PostgresDocument { } impl Backend for Postgres { + type EncodedInteger = Option; + type EncodedFloat = Option; + type EncodedFlag = Option; + fn optional_integer(value: Option) -> Option { + value + } + fn optional_float_value(value: Option) -> Option { + value + } + fn optional_flag(value: Option) -> Option { + value + } const READ_SNAPSHOT_BEGIN: &'static str = "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY"; const PHP_CONNECTION: &str = "pgsql"; fn optional_float(row: &Self::Row, field: &str) -> Result> { @@ -360,6 +394,22 @@ impl Backend for Postgres { } impl Backend for MySql { + type EncodedInteger = Option; + type EncodedFloat = Option; + type EncodedFlag = Option; + fn optional_integer(value: Option) -> Option { + value + } + fn optional_float_value(value: Option) -> Option { + value + } + fn optional_flag(value: Option) -> Option { + value + } + const SEARCH_ATTRIBUTE_ROWS: &'static str = "SELECT `key` AS attribute_key,type,value_string,value_keyword,value_int,value_float,CASE WHEN value_bool IS NULL THEN NULL WHEN value_bool THEN 1 ELSE 0 END AS value_bool,value_datetime,value_keyword_list FROM workflow_search_attributes WHERE workflow_run_id=$1 ORDER BY `key` LIMIT 1001"; + const SEARCH_ATTRIBUTE_UPSERT: &'static str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,`key`,type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,false,$13,$14) ON DUPLICATE KEY UPDATE workflow_instance_id=VALUES(workflow_instance_id),type=VALUES(type),value_string=VALUES(value_string),value_keyword=VALUES(value_keyword),value_keyword_list=VALUES(value_keyword_list),value_int=VALUES(value_int),value_float=VALUES(value_float),value_bool=VALUES(value_bool),value_datetime=VALUES(value_datetime),upserted_at_sequence=VALUES(upserted_at_sequence),inherited_from_parent=false,updated_at=VALUES(updated_at)"; + const SEARCH_ATTRIBUTE_DELETE: &'static str = "DELETE FROM workflow_search_attributes WHERE workflow_run_id=$1 AND `key`=$2"; + const SEARCH_ATTRIBUTE_INHERIT: &'static str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,`key`,type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) SELECT $1,$2,`key`,type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,1,true,$3,$4 FROM workflow_search_attributes WHERE workflow_run_id=$5"; const READ_SNAPSHOT_BEGIN: &'static str = "START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY"; const ACTIVITY_METADATA_HISTORY: &'static str = "SELECT h.sequence,h.event_type,CASE WHEN page.projection_bytes<=8388608 THEN h.payload ELSE NULL END AS payload,h.workflow_task_id,h.recorded_at,page.projection_count FROM (SELECT id,sequence,COUNT(*) OVER () AS projection_count,SUM(payload_bytes) OVER (ORDER BY sequence) AS projection_bytes FROM (SELECT id,sequence,octet_length(CAST(payload AS CHAR CHARACTER SET utf8mb4)) AS payload_bytes FROM workflow_history_events FORCE INDEX (workflow_history_events_workflow_run_id_sequence_unique) WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled')) AS sizes ORDER BY sequence LIMIT 10001) AS page JOIN workflow_history_events h ON h.id=page.id ORDER BY page.sequence"; const PHP_CONNECTION: &str = "mysql"; diff --git a/rust/src/runtime/continuations.rs b/rust/src/runtime/continuations.rs index b550d736b..7bf61c16a 100644 --- a/rust/src/runtime/continuations.rs +++ b/rust/src/runtime/continuations.rs @@ -136,10 +136,6 @@ where "SELECT COUNT(*) FROM workflow_signal_records WHERE workflow_run_id=$1 AND status='received'", old.as_str(), ), - ( - "SELECT COUNT(*) FROM workflow_search_attributes WHERE workflow_run_id=$1", - old.as_str(), - ), ( "SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_instance_id=$1", workflow.as_str(), @@ -276,6 +272,15 @@ where .execute(&mut **tx) .await?; } + // `SearchAttributeUpsertService::inheritFromParent` at sequence 1. + Self::query(DB::SEARCH_ATTRIBUTE_INHERIT) + .bind(&next) + .bind(&workflow) + .bind(DB::bind_time(started)) + .bind(DB::bind_time(started)) + .bind(&old) + .execute(&mut **tx) + .await?; Self::query("UPDATE workflow_instances SET current_run_id=$1,run_count=$2,updated_at=$3 WHERE id=$4") .bind(&next).bind(number).bind(DB::bind_time(started)).bind(&workflow).execute(&mut **tx).await?; Self::query("INSERT INTO workflow_links(id,link_type,sequence,parent_workflow_instance_id,parent_workflow_run_id,child_workflow_instance_id,child_workflow_run_id,is_primary_parent,parent_close_policy,created_at,updated_at) VALUES ($1,'continue_as_new',$2,$3,$4,$5,$6,TRUE,'abandon',$7,$8)") diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index f307045bb..fb5f20b17 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -1113,6 +1113,19 @@ async fn register( { memo_protocol(&headers)?; } + if body["capabilities"].as_array().is_some_and(|flags| { + flags + .iter() + .any(|flag| flag == super::search_attribute_upserts::CAPABILITY) + }) && !protocol_at_least(&headers, 16)? + { + return Err(super::RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"reason":"workflow_metadata_capability_protocol_mismatch", + "worker_capability":super::search_attribute_upserts::CAPABILITY, + "minimum_protocol_version":super::search_attribute_upserts::MINIMUM_PROTOCOL_VERSION,"recorded":false}), + }); + } Ok(( StatusCode::CREATED, Json(worker_response(runtime.register(body).await?)), @@ -1347,6 +1360,22 @@ async fn complete_workflow( }) { memo_protocol(&headers)?; } + // `guardTypedSearchAttributeCommandsAvailable`. + if body["commands"].as_array().is_some_and(|commands| { + commands.iter().any(|command| { + command["type"] == "upsert_search_attributes" && command.get("attribute_types").is_some() + }) + }) && !protocol_at_least(&headers, 16)? + { + let minimum = super::search_attribute_upserts::MINIMUM_PROTOCOL_VERSION; + return Err(super::RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"task_id":task_id,"workflow_task_attempt":body["workflow_task_attempt"], + "outcome":"rejected","recorded":false,"reason":"typed_search_attributes_unavailable", + "requested_version":request_protocol(&headers)?,"minimum_protocol_version":minimum, + "remediation":format!("Complete typed search-attribute updates only through server nodes advertising worker protocol {minimum} or newer.")}), + }); + } let attempt = body["workflow_task_attempt"].clone(); runtime .complete_workflow(&task_id, body) @@ -1363,6 +1392,14 @@ async fn complete_workflow( .map(Json) } +/// Whether the request's worker protocol is `1.` or newer. +fn protocol_at_least(headers: &HeaderMap, minor: u32) -> Result { + Ok(request_protocol(headers)? + .strip_prefix("1.") + .and_then(|value| value.parse::().ok()) + .is_some_and(|value| value >= minor)) +} + fn memo_protocol(headers: &HeaderMap) -> Result<()> { let version = request_protocol(headers)?; if version diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index d0a7938e8..c766d187e 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -65,6 +65,7 @@ mod schedule_list; mod schedule_storage; mod schedules; mod schema; +mod search_attribute_upserts; mod search_attributes; mod selection_summary; mod selections; diff --git a/rust/src/runtime/run_describe.rs b/rust/src/runtime/run_describe.rs index 2f8eb8b78..51c1fe383 100644 --- a/rust/src/runtime/run_describe.rs +++ b/rust/src/runtime/run_describe.rs @@ -113,29 +113,13 @@ where /// `WorkflowRun::typedSearchAttributes`, sorted by key; PHP renders an /// empty set as `[]`. async fn typed_search_attributes(&self, run_id: &str) -> Result { - // `key` needs dialect quoting, so select every column; the map sorts. - let rows = Self::query("SELECT *,CASE WHEN value_bool IS NULL THEN NULL WHEN value_bool THEN 1 ELSE 0 END AS value_bool_flag FROM workflow_search_attributes WHERE workflow_run_id=$1") - .bind(run_id) - .fetch_all(&self.pool) - .await?; - let mut attributes = Map::new(); - for row in rows { - let value = match DB::string(&row, "type")?.as_str() { - "string" => json!(DB::optional_string(&row, "value_string")?), - "keyword" => json!(DB::optional_string(&row, "value_keyword")?), - "keyword_list" => DB::optional_string(&row, "value_keyword_list")? - .and_then(|text| serde_json::from_str::(&text).ok()) - .filter(Value::is_array) - .unwrap_or_else(|| json!([])), - "int" => json!(DB::optional_number(&row, "value_int")?), - "float" => json!(DB::optional_float(&row, "value_float")?), - "bool" => json!(DB::optional_number(&row, "value_bool_flag")?.map(|flag| flag != 0)), - "datetime" => json_time(DB::optional_instant(&row, "value_datetime")?), - _ => Value::Null, - }; - attributes.insert(DB::string(&row, "key")?, value); - } - Ok(if attributes.is_empty() { json!([]) } else { Value::Object(attributes) }) + let mut connection = self.pool.acquire().await?; + let rows = Self::typed_rows(&mut connection, run_id).await?; + Ok(if rows.is_empty() { + json!([]) + } else { + Value::Object(rows.into_iter().map(|(key, value)| (key, value.json())).collect()) + }) } /// The rows `HistoryExport::forRun` reads for one run, as JSON. diff --git a/rust/src/runtime/search_attribute_upserts.rs b/rust/src/runtime/search_attribute_upserts.rs new file mode 100644 index 000000000..f1d7b891b --- /dev/null +++ b/rust/src/runtime/search_attribute_upserts.rs @@ -0,0 +1,845 @@ +//! The `upsert_search_attributes` workflow command, as published PHP applies +//! it. The Server's `SearchAttributeValueValidator` checks each value against +//! the namespace's registered types and resolves its storage type. The +//! Workflow package's `UpsertSearchAttributesCall` then normalizes the values, +//! and `SearchAttributeUpsertService` stores them as typed rows. +use std::{collections::BTreeMap, env}; + +use axum::http::StatusCode; +use chrono::{DateTime, NaiveDate, NaiveDateTime, SecondsFormat, Utc}; +use serde_json::{Map, Value, json}; +use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; + +use super::{ + Result, backend::Backend, laravel_validation::failure_messages, memo_limits, refuse, + transaction::Transaction, + store::{Store, now}, +}; + +/// `WorkflowMetadataCapabilityPolicy::TYPED_SEARCH_ATTRIBUTES`. +pub(super) const CAPABILITY: &str = "typed_search_attributes"; +pub(super) const MINIMUM_PROTOCOL_VERSION: &str = "1.16"; + +/// `WorkflowSearchAttribute` limits. +const MAX_ATTRIBUTES_PER_RUN: usize = 100; +const MAX_STRING_LENGTH: usize = 2048; +const MAX_KEYWORD_LENGTH: usize = 255; +const MAX_TOTAL_SIZE_BYTES: usize = 65536; +/// `UpsertSearchAttributesCall` keys: 1-64 URL-safe characters. +const MAX_CALL_KEY_LENGTH: usize = 64; +const VALID_TYPES: [&str; 7] = ["string", "keyword", "keyword_list", "int", "float", "bool", "datetime"]; + +/// `SearchAttributeDefinition::canonicalType`. +pub(super) fn canonical_type(kind: &str) -> Option<&'static str> { + match kind { + "text" => Some("string"), + "double" => Some("float"), + _ => VALID_TYPES.iter().copied().find(|valid| *valid == kind), + } +} + +/// `server.limits.max_search_attribute_key_length` (bytes, default 128) and +/// `max_search_attribute_value_bytes` (default 2048), read like PHP's config. +pub(super) struct Limits { + key_length: i64, + value_bytes: i64, +} + +impl Limits { + pub(super) fn from_env() -> Result { + let read = |primary: &str, legacy: &str, default: i64| -> Result { + let value = |key: &str| match env::var(key) { + Ok(value) => Ok(Some(value)), + Err(env::VarError::NotPresent) => Ok(None), + Err(env::VarError::NotUnicode(_)) => { + Err(refuse(StatusCode::BAD_REQUEST, "invalid_search_attribute_configuration")) + } + }; + let primary = value(primary)?.filter(|value| !memo_limits::typed_null(value)); + Ok(match primary.or(value(legacy)?) { + Some(value) => memo_limits::cast_integer(&value), + None => default, + }) + }; + Ok(Self { + key_length: read("DW_MAX_SEARCH_ATTRIBUTE_KEY_LENGTH", "WORKFLOW_MAX_SEARCH_ATTRIBUTE_KEY_LENGTH", 128)?, + value_bytes: read( + "DW_MAX_SEARCH_ATTRIBUTE_VALUE_BYTES", + "WORKFLOW_MAX_SEARCH_ATTRIBUTE_VALUE_BYTES", + 2048, + )?, + }) + } + + fn keyword_bytes(&self) -> i64 { + let limit = if self.value_bytes > 0 { self.value_bytes } else { MAX_KEYWORD_LENGTH as i64 }; + limit.min(MAX_KEYWORD_LENGTH as i64) + } +} + +/// A decoded JSON object key that PHP turns into an integer array key. +fn php_integer_key(key: &str) -> bool { + let digits = key.strip_prefix('-').unwrap_or(key); + let canonical = key == "0" + || (!digits.is_empty() + && !digits.starts_with('0') + && digits.bytes().all(|byte| byte.is_ascii_digit())); + canonical && key.parse::().is_ok() +} + +fn is_scalar(value: &Value) -> bool { + matches!(value, Value::Bool(_) | Value::Number(_) | Value::String(_)) +} + +fn is_int(value: &Value) -> bool { + value.as_i64().is_some() +} + +/// PHP decodes `[]` and `{}` both to an empty array, which is a list. +fn as_list(value: &Value) -> Option> { + match value { + Value::Array(items) => Some(items.clone()), + Value::Object(map) if map.is_empty() => Some(Vec::new()), + _ => None, + } +} + +/// `WorkflowSearchAttribute::inferType` for a decoded JSON value. +fn infer_type(value: &Value) -> &'static str { + match value { + Value::Bool(_) => "bool", + Value::Number(number) if number.as_i64().is_some() => "int", + Value::Number(_) => "float", + Value::String(text) => { + if text.chars().count() > MAX_KEYWORD_LENGTH || text.chars().any(char::is_whitespace) { + "string" + } else { + "keyword" + } + } + Value::Array(_) | Value::Object(_) => "keyword_list", + Value::Null => "keyword", + } +} + +/// `Carbon::parse` for the forms workers send: RFC 3339, or a date and time +/// without an offset, read as UTC like the server's timezone. +fn parse_datetime(text: &str) -> Option> { + if let Ok(time) = DateTime::parse_from_rfc3339(text) { + return Some(time.with_timezone(&Utc)); + } + for format in ["%Y-%m-%dT%H:%M:%S%.f", "%Y-%m-%d %H:%M:%S%.f", "%Y-%m-%dT%H:%M", "%Y-%m-%d %H:%M"] { + if let Ok(time) = NaiveDateTime::parse_from_str(text, format) { + return Some(time.and_utc()); + } + } + NaiveDate::parse_from_str(text, "%Y-%m-%d") + .ok() + .and_then(|date| date.and_hms_opt(0, 0, 0)) + .map(|time| time.and_utc()) +} + +/// `SearchAttributeValueValidator::validateForNamespace`: the storage type of +/// each attribute, or the validation messages PHP reports. +/// `definitions` holds the namespace's custom registered types. +pub(super) fn validate( + definitions: &BTreeMap, + attributes: &Map, + declared: &Map, + limits: &Limits, +) -> std::result::Result, Vec> { + let registered = |key: &str| { + super::search_attributes::SYSTEM_ATTRIBUTES + .iter() + .find(|(name, _)| *name == key) + .map(|(_, kind)| (*kind).to_owned()) + .or_else(|| definitions.get(key).cloned()) + }; + let schema_enforced = !definitions.is_empty(); + let mut messages = Vec::new(); + let mut types = BTreeMap::new(); + for (key, kind) in declared { + if php_integer_key(key) || !attributes.contains_key(key) { + messages.push("Declared search attribute types must name keys present in the attribute update.".to_owned()); + continue; + } + if kind.as_str().is_none_or(|kind| canonical_type(kind) != Some(kind)) { + let shown = match kind { + Value::String(text) => text.clone(), + Value::Bool(true) => "1".to_owned(), + Value::Bool(false) => String::new(), + Value::Number(number) => number.to_string(), + Value::Null => "null".to_owned(), + Value::Array(_) | Value::Object(_) => "array".to_owned(), + }; + messages.push(format!( + "Search attribute [{key}] declares unsupported type [{shown}]; allowed types are {}.", + VALID_TYPES.join(", ") + )); + } + } + for (key, value) in php_order(attributes) { + if key.is_empty() || php_integer_key(key) { + messages.push("Search attribute keys must be non-empty strings.".to_owned()); + continue; + } + if limits.key_length > 0 && key.len() as i64 > limits.key_length { + messages.push(format!( + "Search attribute key [{key}] exceeds the maximum length of {} bytes.", + limits.key_length + )); + continue; + } + let mut chars = key.chars(); + if !chars.next().is_some_and(|first| first.is_ascii_alphabetic()) + || !chars.all(|character| character.is_ascii_alphanumeric() || character == '_') + { + messages.push(format!( + "Search attribute key [{key}] must start with a letter and contain only letters, numbers, and underscores." + )); + continue; + } + let registered_type = registered(key).and_then(|kind| canonical_type(&kind)); + let declared_type = declared.get(key).and_then(Value::as_str).and_then(canonical_type); + if let (Some(registered), Some(declared)) = (registered_type, declared_type) + && registered != declared + { + messages.push(format!( + "Search attribute [{key}] declares type [{declared}] but is registered as [{registered}]." + )); + continue; + } + if let Some(effective) = registered_type.or(declared_type) { + validate_declared_value(&mut messages, key, value, effective, limits); + if !value.is_null() || declared_type.is_some() { + types.insert(key.clone(), effective); + } + continue; + } + if schema_enforced { + messages.push(format!("Search attribute [{key}] is not registered for this namespace.")); + continue; + } + if !value.is_null() && !is_scalar(value) { + messages.push(format!( + "Search attribute [{key}] must be a scalar value or null unless it is registered as keyword_list." + )); + } else if let Value::String(text) = value + && limits.value_bytes > 0 + && text.len() as i64 > limits.value_bytes + { + messages.push(format!( + "Search attribute [{key}] value exceeds the maximum of {} bytes.", + limits.value_bytes + )); + } + if !value.is_null() { + types.insert(key.clone(), infer_type(value)); + } + } + if messages.is_empty() { Ok(types) } else { Err(messages) } +} + +fn validate_declared_value(messages: &mut Vec, key: &str, value: &Value, kind: &str, limits: &Limits) { + if value.is_null() { + return; + } + let exceeds = |bytes: usize, limit: i64| limit > 0 && bytes as i64 > limit; + match kind { + "keyword" | "string" => match value { + Value::String(text) => { + let limit = if kind == "keyword" { limits.keyword_bytes() } else { limits.value_bytes }; + if exceeds(text.len(), limit) { + messages.push(format!("Search attribute [{key}] value exceeds the maximum of {limit} bytes.")); + } + } + _ => messages.push(format!("Search attribute [{key}] is registered as {kind} and must be a string.")), + }, + "int" if !is_int(value) => { + messages.push(format!("Search attribute [{key}] is registered as int and must be an integer.")); + } + "float" if !value.is_number() => { + messages.push(format!("Search attribute [{key}] is registered as float and must be a number.")); + } + "bool" if !value.is_boolean() => { + messages.push(format!("Search attribute [{key}] is registered as bool and must be a boolean.")); + } + "datetime" => { + let rfc = || format!("Search attribute [{key}] is registered as datetime and must be an RFC3339 datetime string."); + match value { + Value::String(text) if exceeds(text.len(), limits.value_bytes) => messages.push(format!( + "Search attribute [{key}] value exceeds the maximum of {} bytes.", + limits.value_bytes + )), + Value::String(text) if parse_datetime(text).is_some() => {} + _ => messages.push(rfc()), + } + } + "keyword_list" => { + let Some(items) = as_list(value) else { + messages.push(format!( + "Search attribute [{key}] is registered as keyword_list and must be a list of strings." + )); + return; + }; + let limit = limits.keyword_bytes(); + for item in items { + let Value::String(text) = item else { + messages.push(format!( + "Search attribute [{key}] is registered as keyword_list and must contain only strings." + )); + return; + }; + if exceeds(text.len(), limit) { + messages.push(format!("Search attribute [{key}] list value exceeds the maximum of {limit} bytes.")); + return; + } + } + } + _ => {} + } +} + +/// `UpsertSearchAttributesCall`: trimmed values, an empty string as a +/// deletion, sorted keys. An error is the `LogicException` message that +/// `WorkflowCommandNormalizer` reports for the command's attributes. +fn normalized_call(attributes: &Map) -> std::result::Result, String> { + if attributes.is_empty() { + return Err("Upsert search attributes commands require a non-empty attributes object.".to_owned()); + } + let mut normalized = BTreeMap::new(); + for (key, value) in php_order(attributes) { + if php_integer_key(key) + || key.is_empty() + || key.chars().count() > MAX_CALL_KEY_LENGTH + || !key.bytes().all(|byte| byte.is_ascii_alphanumeric() || b"_.:-".contains(&byte)) + { + return Err("Workflow v2 search attribute keys must be 1-64 URL-safe characters using letters, numbers, \".\", \"_\", \"-\", and \":\".".to_owned()); + } + let value = match value { + Value::Null => Value::Null, + Value::String(text) => { + let trimmed = php_trim(text); + if trimmed.chars().count() > MAX_STRING_LENGTH { + return Err(format!("Workflow v2 search attribute [{key}] must be up to {MAX_STRING_LENGTH} characters.")); + } + if trimmed.is_empty() { Value::Null } else { json!(trimmed) } + } + Value::Bool(_) | Value::Number(_) => value.clone(), + Value::Array(_) | Value::Object(_) => { + let items = as_list(value) + .ok_or_else(|| format!("Workflow v2 search attribute [{key}] list value must be a JSON array."))?; + let mut list = Vec::with_capacity(items.len()); + for item in items { + let Value::String(text) = item else { + return Err(format!("Workflow v2 search attribute [{key}] list values must contain only strings.")); + }; + let trimmed = php_trim(&text); + if trimmed.chars().count() > MAX_KEYWORD_LENGTH { + return Err(format!( + "Workflow v2 search attribute [{key}] list values must be up to {MAX_KEYWORD_LENGTH} characters." + )); + } + list.push(json!(trimmed)); + } + Value::Array(list) + } + }; + normalized.insert(key.clone(), value); + } + Ok(normalized) +} + +/// The order PHP visits a decoded object's keys in: integer-like keys +/// first, as PHP reports them. Native compares the remaining keys sorted, +/// not in request order. +fn php_order(map: &Map) -> impl Iterator { + map.iter() + .filter(|(key, _)| php_integer_key(key)) + .chain(map.iter().filter(|(key, _)| !php_integer_key(key))) +} + +/// PHP's `trim`: spaces, tabs, newlines, carriage returns, NUL and vertical tabs. +fn php_trim(text: &str) -> &str { + text.trim_matches([' ', '\t', '\n', '\r', '\0', '\x0b']) +} + +/// One typed row: its type and the value for that type's column. +#[derive(Clone, Debug, PartialEq)] +pub(super) enum Typed { + String(String), + Keyword(String), + KeywordList(Vec), + Int(i64), + Float(f64), + Bool(bool), + Datetime(DateTime), +} + +impl Typed { + pub(super) fn kind(&self) -> &'static str { + match self { + Self::String(_) => "string", + Self::Keyword(_) => "keyword", + Self::KeywordList(_) => "keyword_list", + Self::Int(_) => "int", + Self::Float(_) => "float", + Self::Bool(_) => "bool", + Self::Datetime(_) => "datetime", + } + } + + /// `WorkflowSearchAttribute::getValue`, as PHP serializes it. + pub(super) fn json(&self) -> Value { + match self { + Self::String(text) | Self::Keyword(text) => json!(text), + Self::KeywordList(items) => json!(items), + Self::Int(number) => json!(number), + Self::Float(number) => json!(number), + Self::Bool(flag) => json!(flag), + Self::Datetime(time) => json!(time.to_rfc3339_opts(SecondsFormat::Micros, true)), + } + } + + /// `validateAttributeSet`'s size of one stored value. + fn size(&self) -> usize { + match self { + Self::String(text) | Self::Keyword(text) => text.len(), + Self::KeywordList(items) => items.iter().map(String::len).sum(), + Self::Int(_) | Self::Float(_) | Self::Datetime(_) => 8, + Self::Bool(_) => 1, + } + } +} + +/// `WorkflowSearchAttribute::setTypedValue` for a validated value. +fn typed(value: &Value, kind: &str) -> Option { + let text = || match value { + Value::String(text) => Some(text.clone()), + Value::Number(number) => Some(number.to_string()), + Value::Bool(true) => Some("1".to_owned()), + Value::Bool(false) => Some(String::new()), + _ => None, + }; + Some(match kind { + "string" => Typed::String(text().filter(|text| text.chars().count() <= MAX_STRING_LENGTH)?), + "keyword" => Typed::Keyword(text().filter(|text| text.chars().count() <= MAX_KEYWORD_LENGTH)?), + "keyword_list" => Typed::KeywordList( + value + .as_array()? + .iter() + .map(|item| { + item.as_str() + .filter(|text| text.chars().count() <= MAX_KEYWORD_LENGTH) + .map(str::to_owned) + }) + .collect::>()?, + ), + "int" => Typed::Int(value.as_i64()?), + "float" => Typed::Float(value.as_f64().filter(|_| value.is_number())?), + "bool" => Typed::Bool(value.as_bool()?), + "datetime" => Typed::Datetime(value.as_str().and_then(parse_datetime)?), + _ => return None, + }) +} + +/// A validated, normalized upsert ready to store. +pub(super) struct PreparedUpsert { + /// Normalized values; `null` deletes the key. + pub(super) attributes: BTreeMap, + /// `canonicalTypes`: the stored type of every typed key. + pub(super) attribute_types: BTreeMap, + /// Typed rows to write, by key. + pub(super) rows: BTreeMap, +} + +/// The bridge's `normalizeUpsertSearchAttributesCommand` and +/// `canonicalTypes` after the Server resolved `attribute_types`. +pub(super) fn prepare( + attributes: &Map, + attribute_types: &BTreeMap, + index: usize, +) -> Result { + let invalid = || refuse(StatusCode::UNPROCESSABLE_ENTITY, "invalid_commands"); + let attributes = normalized_call(attributes).map_err(|message| { + failure_messages(vec![(format!("commands.{index}.attributes"), vec![message])]).expect("message") + })?; + let mut rows = BTreeMap::new(); + for (key, kind) in attribute_types { + let value = attributes.get(key).ok_or_else(invalid)?; + if !value.is_null() { + rows.insert(key.clone(), typed(value, kind).ok_or_else(invalid)?); + } + } + let mut types = attribute_types.clone(); + for (key, value) in &attributes { + if !value.is_null() && !types.contains_key(key) { + let kind = infer_type(value); + rows.insert(key.clone(), typed(value, kind).ok_or_else(invalid)?); + types.insert(key.clone(), kind); + } + } + Ok(PreparedUpsert { + attributes, + attribute_types: types, + rows, + }) +} + +/// `validateAttributeSet` over the merged rows. PHP raises an uncaught +/// `InvalidArgumentException` here; native refuses the completion instead. +pub(super) fn check_attribute_set<'a>(rows: impl Iterator) -> Result<()> { + let (count, bytes) = rows.fold((0usize, 0usize), |(count, bytes), row| (count + 1, bytes + row.size())); + let message = if count > MAX_ATTRIBUTES_PER_RUN { + format!("Search attributes count exceeds maximum ({count} > {MAX_ATTRIBUTES_PER_RUN})") + } else if bytes > MAX_TOTAL_SIZE_BYTES { + format!("Total search attributes size exceeds maximum ({bytes} > {MAX_TOTAL_SIZE_BYTES} bytes)") + } else { + return Ok(()); + }; + Err(super::RuntimeError::Protocol { + status: StatusCode::UNPROCESSABLE_ENTITY, + response: json!({"reason":"search_attribute_limit_exceeded","message":message,"recorded":false}), + }) +} + +/// The upsert's `attributes`, as PHP decodes them: a JSON list becomes an +/// array with integer keys, which the validator then refuses. +fn attribute_map(command: &Value) -> Option> { + match &command["attributes"] { + Value::Object(map) => Some(map.clone()), + Value::Array(items) => Some(items.iter().enumerate().map(|(index, item)| (index.to_string(), item.clone())).collect()), + _ => None, + } +} + +/// Laravel's `commands.*.attribute_types` rules: `nullable|array`, each a string. +pub(super) fn validate_shape(command: &Value, index: usize) -> Result<()> { + // Laravel shows a wildcard rule's expanded attribute as it is. + let field = format!("commands.{index}.attribute_types"); + let messages = match &command["attribute_types"] { + Value::Null => Vec::new(), + Value::Object(types) => types + .iter() + .filter(|(_, kind)| !kind.is_string()) + .map(|(key, _)| (format!("commands.{index}.attribute_types.{key}"), vec![format!("The {field}.{key} field must be a string.")])) + .collect(), + Value::Array(types) => types + .iter() + .enumerate() + .filter(|(_, kind)| !kind.is_string()) + .map(|(key, _)| (format!("commands.{index}.attribute_types.{key}"), vec![format!("The {field}.{key} field must be a string.")])) + .collect(), + _ => vec![(format!("commands.{index}.attribute_types"), vec![format!("The {field} field must be an array.")])], + }; + if let Some(error) = failure_messages(messages) { + return Err(error); + } + if !matches!(command["attributes"], Value::Null | Value::Object(_) | Value::Array(_)) { + return Err(failure_messages(vec![( + format!("commands.{index}.attributes"), + vec![format!("The commands.{index}.attributes field must be an array.")], + )]) + .expect("one message")); + } + Ok(()) +} + +impl Store +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> Option<&'q str>: Encode<'q, DB>, + for<'q> Option: Encode<'q, DB>, + for<'q> String: Encode<'q, DB> + Type, + for<'q> i64: Encode<'q, DB> + Type, + for<'r> i64: Decode<'r, DB>, + usize: ColumnIndex, +{ + /// The run's typed rows by key, as `WorkflowRun::typedSearchAttributes` + /// reads them. Rows of an unknown type are left out. + pub(super) async fn typed_rows( + connection: &mut DB::Connection, + run_id: &str, + ) -> Result> { + let rows = Self::query(DB::SEARCH_ATTRIBUTE_ROWS) + .bind(run_id) + .fetch_all(&mut *connection) + .await?; + let mut typed = BTreeMap::new(); + for row in rows { + let value = match DB::string(&row, "type")?.as_str() { + "string" => DB::optional_string(&row, "value_string")?.map(Typed::String), + "keyword" => DB::optional_string(&row, "value_keyword")?.map(Typed::Keyword), + "keyword_list" => Some(Typed::KeywordList( + DB::optional_document_row(&row, "value_keyword_list")? + .and_then(|list| serde_json::from_value::>(list).ok()) + .unwrap_or_default(), + )), + "int" => DB::optional_number(&row, "value_int")?.map(Typed::Int), + "float" => DB::optional_float(&row, "value_float")?.map(Typed::Float), + "bool" => DB::optional_number(&row, "value_bool")?.map(|flag| Typed::Bool(flag != 0)), + "datetime" => DB::optional_instant(&row, "value_datetime")?.map(Typed::Datetime), + _ => None, + }; + if let Some(value) = value { + typed.insert(DB::string(&row, "attribute_key")?, value); + } + } + Ok(typed) + } + + /// The bridge's `applyUpsertSearchAttributes` after the Server's + /// `validateWorkflowTaskSearchAttributeCommands`. + pub(super) async fn upsert_search_attributes( + &self, + tx: &mut Transaction<'_, DB>, + run: &DB::Row, + task: &DB::Row, + command: &Value, + index: usize, + sequence: i64, + ) -> Result<()> { + let run_id = DB::string(run, "id")?; + let task_id = DB::string(task, "id")?; + if command.get("attribute_types").is_some() { + let worker_id = DB::optional_string(task, "lease_owner")? + .ok_or_else(|| refuse(StatusCode::CONFLICT, "task_not_leased"))?; + let worker = Self::query("SELECT capabilities FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2") + .bind(self.namespace.as_ref()).bind(&worker_id).fetch_optional(&mut **tx).await?; + let capable = match &worker { + Some(worker) => DB::optional_document_row(worker, "capabilities")? + .is_some_and(|flags| flags.as_array().is_some_and(|flags| flags.iter().any(|flag| flag == CAPABILITY))), + None => false, + }; + if !capable { + return Err(super::RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"task_id":task_id,"workflow_task_attempt":DB::number(task, "attempt_count")?, + "outcome":"rejected","recorded":false,"reason":"workflow_metadata_capability_not_advertised", + "capability":CAPABILITY,"command_type":"upsert_search_attributes", + "minimum_protocol_version":MINIMUM_PROTOCOL_VERSION, + "remediation":format!("Re-register lease owner {worker_id} with the {CAPABILITY} capability before emitting upsert_search_attributes.")}), + }); + } + } + let attributes = attribute_map(command).unwrap_or_default(); + let declared = command["attribute_types"].as_object().cloned().unwrap_or_default(); + let definitions = Self::query("SELECT name,type FROM search_attribute_definitions WHERE namespace=$1") + .bind(self.namespace.as_ref()) + .fetch_all(&mut **tx) + .await? + .iter() + .map(|row| Ok((DB::string(row, "name")?, DB::string(row, "type")?))) + .collect::>>()?; + let types = validate(&definitions, &attributes, &declared, &Limits::from_env()?).map_err(|messages| { + failure_messages(vec![(format!("commands.{index}.attributes"), messages)]).expect("messages") + })?; + let prepared = prepare(&attributes, &types, index)?; + let existing = Self::typed_rows(&mut **tx, &run_id).await?; + let mut stored = existing.clone(); + let mut merged: BTreeMap = + existing.iter().map(|(key, value)| (key.clone(), value.json())).collect(); + for (key, value) in &prepared.attributes { + if value.is_null() { + stored.remove(key); + merged.remove(key); + } else { + stored.insert(key.clone(), prepared.rows[key].clone()); + merged.insert(key.clone(), value.clone()); + } + } + check_attribute_set(stored.values())?; + let workflow_id = DB::string(run, "workflow_instance_id")?; + let written = now(); + for (key, value) in &prepared.attributes { + if value.is_null() { + Self::query(DB::SEARCH_ATTRIBUTE_DELETE) + .bind(&run_id) + .bind(key.as_str()) + .execute(&mut **tx) + .await?; + continue; + } + let row = &prepared.rows[key]; + let text = |kind| match row { + Typed::String(text) if kind == "string" => Some(text.as_str()), + Typed::Keyword(text) if kind == "keyword" => Some(text.as_str()), + _ => None, + }; + Self::query(DB::SEARCH_ATTRIBUTE_UPSERT) + .bind(&run_id) + .bind(workflow_id.as_str()) + .bind(key.as_str()) + .bind(row.kind()) + .bind(text("string")) + .bind(text("keyword")) + .bind(match row { + Typed::KeywordList(items) => Some(DB::document(&json!(items))), + _ => None, + }) + .bind(DB::optional_integer(match row { + Typed::Int(number) => Some(*number), + _ => None, + })) + .bind(DB::optional_float_value(match row { + Typed::Float(number) => Some(*number), + _ => None, + })) + .bind(DB::optional_flag(match row { + Typed::Bool(flag) => Some(*flag), + _ => None, + })) + .bind(DB::optional_time(match row { + Typed::Datetime(time) => Some(*time), + _ => None, + })) + .bind(sequence) + .bind(DB::bind_time(written)) + .bind(DB::bind_time(written)) + .execute(&mut **tx) + .await?; + } + // PHP serializes an empty map as a JSON list. + let object = |map: Map| if map.is_empty() { json!([]) } else { Value::Object(map) }; + Self::append( + tx, + &run_id, + "SearchAttributesUpserted", + json!({"sequence":sequence, + "attributes":object(prepared.attributes.into_iter().collect()), + "attribute_types":object(prepared.attribute_types.into_iter().map(|(key, kind)| (key, json!(kind))).collect()), + "merged":object(merged.into_iter().collect()), + "task":Self::task_snapshot(task)?}), + Some(&task_id), + None, + ) + .await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn limits() -> Limits { + Limits { key_length: 128, value_bytes: 2048 } + } + + fn map(value: Value) -> Map { + value.as_object().cloned().unwrap() + } + + #[test] + fn registered_declared_and_inferred_types_resolve_like_php() { + let definitions = BTreeMap::from([ + ("Ratio".to_owned(), "double".to_owned()), + ("Note".to_owned(), "text".to_owned()), + ]); + let types = validate( + &definitions, + &map(json!({"Ratio": 2, "Note": "a b", "Tags": ["x"], "WorkflowType": "manual", "Gone": null})), + &map(json!({"Tags": "keyword_list", "Gone": "keyword"})), + &limits(), + ) + .unwrap(); + assert_eq!( + types, + BTreeMap::from([ + ("Gone".to_owned(), "keyword"), + ("Note".to_owned(), "string"), + ("Ratio".to_owned(), "float"), + ("Tags".to_owned(), "keyword_list"), + ("WorkflowType".to_owned(), "keyword"), + ]) + ); + // Without custom definitions, values infer their type. + let inferred = validate( + &BTreeMap::new(), + &map(json!({"Id": "a1", "Text": "two words", "Count": 3, "Score": 1.5, "Flag": true, "Gone": null})), + &Map::new(), + &limits(), + ) + .unwrap(); + assert_eq!( + inferred, + BTreeMap::from([ + ("Count".to_owned(), "int"), + ("Flag".to_owned(), "bool"), + ("Id".to_owned(), "keyword"), + ("Score".to_owned(), "float"), + ("Text".to_owned(), "string"), + ]) + ); + } + + #[test] + fn validation_messages_match_php() { + let definitions = BTreeMap::from([("Count".to_owned(), "int".to_owned())]); + let messages = validate( + &definitions, + &map(json!({"Count": "3", "1bad": 1, "Other": 1, "7": 1})), + &map(json!({"Count": "keyword", "Missing": "int"})), + &limits(), + ) + .unwrap_err(); + assert_eq!( + messages, + [ + "Declared search attribute types must name keys present in the attribute update.", + "Search attribute keys must be non-empty strings.", + "Search attribute key [1bad] must start with a letter and contain only letters, numbers, and underscores.", + "Search attribute [Count] declares type [keyword] but is registered as [int].", + "Search attribute [Other] is not registered for this namespace.", + ] + ); + let declared = validate( + &BTreeMap::new(), + &map(json!({"A": 1.5, "B": "x", "C": 2, "D": "not a time", "E": ["ok", 3], "F": "界".repeat(86), "G": {"k": 1}})), + &map(json!({"A": "int", "B": "bool", "C": "text", "D": "datetime", "E": "keyword_list", "F": "keyword"})), + &limits(), + ) + .unwrap_err(); + assert_eq!( + declared, + [ + "Search attribute [C] declares unsupported type [text]; allowed types are string, keyword, keyword_list, int, float, bool, datetime.", + "Search attribute [A] is registered as int and must be an integer.", + "Search attribute [B] is registered as bool and must be a boolean.", + "Search attribute [C] is registered as string and must be a string.", + "Search attribute [D] is registered as datetime and must be an RFC3339 datetime string.", + "Search attribute [E] is registered as keyword_list and must contain only strings.", + "Search attribute [F] value exceeds the maximum of 255 bytes.", + "Search attribute [G] must be a scalar value or null unless it is registered as keyword_list.", + ] + ); + } + + #[test] + fn prepared_values_are_trimmed_typed_and_sized_like_php() { + let attributes = map(json!({"Tags": [" urgent ", "urgent"], "Name": " ", "Count": 9_007_199_254_740_993_i64, + "Ratio": 2, "At": "2026-10-08T12:34:56.123456Z", "Gone": null})); + let types = BTreeMap::from([ + ("At".to_owned(), "datetime"), + ("Count".to_owned(), "int"), + ("Name".to_owned(), "keyword"), + ("Ratio".to_owned(), "float"), + ("Tags".to_owned(), "keyword_list"), + ]); + let prepared = prepare(&attributes, &types, 0).unwrap(); + assert_eq!(prepared.attributes["Tags"], json!(["urgent", "urgent"])); + // A blank string deletes its key but keeps its declared type. + assert_eq!(prepared.attributes["Name"], Value::Null); + assert_eq!(prepared.attribute_types["Name"], "keyword"); + assert!(!prepared.rows.contains_key("Name") && !prepared.rows.contains_key("Gone")); + assert_eq!(prepared.rows["Ratio"], Typed::Float(2.0)); + assert_eq!(prepared.rows["Count"], Typed::Int(9_007_199_254_740_993)); + assert_eq!(prepared.rows["At"].json(), json!("2026-10-08T12:34:56.123456Z")); + let long_key = "k".repeat(65); + assert!(prepare(&map(json!({long_key: 1})), &BTreeMap::new(), 0).is_err()); + let many: Vec = (0..101).map(Typed::Int).collect(); + assert!(check_attribute_set(many.iter()).is_err()); + assert!(check_attribute_set(many[..100].iter()).is_ok()); + } +} diff --git a/rust/src/runtime/search_attributes.rs b/rust/src/runtime/search_attributes.rs index 4713cbac3..cbeb1e6a9 100644 --- a/rust/src/runtime/search_attributes.rs +++ b/rust/src/runtime/search_attributes.rs @@ -14,7 +14,7 @@ use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; use std::env; /// PHP's `SearchAttributeDefinition::SYSTEM_ATTRIBUTES`. -const SYSTEM_ATTRIBUTES: [(&str, &str); 11] = [ +pub(super) const SYSTEM_ATTRIBUTES: [(&str, &str); 11] = [ ("WorkflowType", "keyword"), ("WorkflowId", "keyword"), ("RunId", "keyword"), diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index e86052ed9..b1ac6da21 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -1215,6 +1215,25 @@ where continue; } } + // `WorkflowMetadataCapabilityPolicy::canReplayRun`: typed upsert + // history needs a worker that advertises typed search attributes. + if kind == "workflow" + && !DB::optional_document_row(&worker, "capabilities")?.is_some_and(|flags| { + flags.as_array().is_some_and(|flags| { + flags.iter().any(|flag| flag == super::search_attribute_upserts::CAPABILITY) + }) + }) + { + let typed_history = Self::query("SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SearchAttributesUpserted' ORDER BY sequence LIMIT 1001") + .bind(&run_id).fetch_all(&mut *tx).await?; + let mut typed = false; + for event in &typed_history { + typed |= DB::document_row(event, "payload")?.get("attribute_types").is_some(); + } + if typed { + continue; + } + } let run = Self::query("SELECT * FROM workflow_runs WHERE id=$1") .bind(&run_id) .fetch_one(&mut *tx) @@ -1720,7 +1739,7 @@ where if index + 1 == commands.len() && commands[..index] .iter() - .all(|prior| prior["type"] == "upsert_memo") => + .all(|prior| matches!(prior["type"].as_str(), Some("upsert_memo" | "upsert_search_attributes"))) => { super::continuations::validate(command)?; } @@ -1755,6 +1774,10 @@ where "record_version_marker" => { version_marker_payload(command)?; } + "upsert_search_attributes" => { + reject_fields(command, &["type", "attributes", "attribute_types"])?; + super::search_attribute_upserts::validate_shape(command, index)?; + } "upsert_memo" => { reject_fields(command, &["type", "entries"])?; envelope(command, "entries")?; @@ -1939,6 +1962,10 @@ where sequence += 1; self.upsert_memo(&mut tx, &run, &task, command, sequence) .await?; + } else if command["type"] == "upsert_search_attributes" { + sequence += 1; + self.upsert_search_attributes(&mut tx, &run, &task, command, index, sequence) + .await?; } else if command["type"] == "schedule_activity" { sequence += 1; let metadata = super::selections::metadata(command, "activity")?; @@ -3224,6 +3251,16 @@ pub(crate) fn capabilities() -> Value { "workflow_memo_updates": false, "cooperative_cancellation": true, "prepared_local_activities": false, "worker_sessions": false, "sticky_execution": false, "local_activities": false, "message_streams": false, "workflow_updates": true, "query_tasks": true, "query_task_authority":true, "query_task_poll_request_idempotency": true}); + value["supported_workflow_task_commands"] + .as_array_mut() + .expect("commands") + .push(json!("upsert_search_attributes")); + value["typed_search_attributes"] = json!({"supported":true, + "minimum_worker_protocol_version":super::search_attribute_upserts::MINIMUM_PROTOCOL_VERSION, + "worker_capability":super::search_attribute_upserts::CAPABILITY, + "canonical_types":["string","keyword","keyword_list","int","float","bool","datetime"], + "command_field":"attribute_types","history_event":"SearchAttributesUpserted", + "history_field":"attribute_types","legacy_history_rule":"absent_metadata_is_unknown_type_identity"}); if super::memos::UPDATES_SUPPORTED { value["supported_workflow_task_commands"] .as_array_mut() diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 89fa3a9cd..8eeb8192f 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -43,6 +43,8 @@ mod namespaces; mod late_workflow_claims; #[path = "execution/search_attributes.rs"] mod search_attributes; +#[path = "execution/search_attribute_upserts.rs"] +mod search_attribute_upserts; #[path = "execution/storage_test.rs"] mod storage_test; diff --git a/rust/tests/execution/search_attribute_upserts.rs b/rust/tests/execution/search_attribute_upserts.rs new file mode 100644 index 000000000..ee894e05f --- /dev/null +++ b/rust/tests/execution/search_attribute_upserts.rs @@ -0,0 +1,158 @@ +//! `upsert_search_attributes` as published PHP applies it: values checked +//! against the namespace's registered types, typed rows, a +//! `SearchAttributesUpserted` event and the run's typed view. +use super::*; + +async fn typed_worker(app: &Router, worker: &str) { + let (status, body) = request(app, "POST", "/api/worker/register", json!({"worker_id":worker, + "task_queue":"test","runtime":"rust","supported_workflow_types":["echo"],"supported_activity_types":[], + "capabilities":["typed_search_attributes"]})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); +} + +async fn define(app: &Router, name: &str, kind: &str) { + let (status, body) = + request(app, "POST", "/api/search-attributes", json!({"name":name,"type":kind})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); +} + +fn upsert(attributes: Value, types: Value) -> Value { + json!({"type":"upsert_search_attributes","attributes":attributes,"attribute_types":types}) +} + +fn events<'a>(history: &'a Value, kind: &str) -> Vec<&'a Value> { + history.as_array().unwrap().iter().filter(|event| event["event_type"] == kind).collect() +} + +async fn described(app: &Router, workflow: &str) -> Value { + let (status, body) = request(app, "GET", &format!("/api/workflows/{workflow}"), Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + body["search_attributes"].clone() +} + +#[tokio::test] +async fn typed_upserts_store_rows_history_and_the_described_view() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + for (name, kind) in [("SearchKeyword", "keyword"), ("SearchTags", "keyword_list"), ("SearchCount", "int"), + ("SearchRatio", "double"), ("SearchFlag", "bool"), ("SearchTime", "datetime"), ("SearchText", "string"), + ("SearchRemoved", "keyword")] + { + define(&app, name, kind).await; + } + typed_worker(&app, "typed-worker").await; + let started = start(&app, "typed-upserts").await; + let task = poll(&app, "typed-worker", "workflow").await; + let initial = json!({"SearchText":"two words","SearchKeyword":"界".repeat(85), + "SearchTags":[" urgent ","urgent"],"SearchCount":9_007_199_254_740_993_i64,"SearchRatio":1.25, + "SearchFlag":true,"SearchTime":"2026-10-08T12:34:56.123456Z","SearchRemoved":"erase"}); + let types = json!({"SearchText":"string","SearchKeyword":"keyword","SearchTags":"keyword_list", + "SearchCount":"int","SearchRatio":"float","SearchFlag":"bool","SearchTime":"datetime","SearchRemoved":"keyword"}); + let first = finish_task(&app, &task, json!([upsert(initial.clone(), types.clone()), + {"type":"open_signal_wait","signal_name":"finish"}])).await; + assert_eq!(first.0, StatusCode::OK, "{}", first.1); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let upserted = events(&history, "SearchAttributesUpserted"); + assert_eq!(upserted.len(), 1, "{history}"); + let payload = &upserted[0]["payload"]; + let mut expected = initial.clone(); + expected["SearchTags"] = json!(["urgent", "urgent"]); + assert_eq!(payload["attributes"], expected); + assert_eq!(payload["attribute_types"], types); + assert_eq!(payload["merged"], expected); + assert_eq!(payload["sequence"], 1); + assert_eq!(described(&app, "typed-upserts").await, expected); + + // Change, replace and delete values in a later task. + assert_eq!(request(&app, "POST", "/api/workflows/typed-upserts/signal/finish", + json!({"input":envelope(Payload::Array(vec![]))})).await.0, StatusCode::ACCEPTED); + let resumed = poll(&app, "typed-worker", "workflow").await; + let mutation = json!({"SearchText":"finished café 界","SearchKeyword":"finished","SearchTags":["done","café"], + "SearchFlag":false,"SearchRemoved":null}); + let second = finish_task(&app, &resumed, json!([upsert(mutation.clone(), json!({"SearchText":"string", + "SearchKeyword":"keyword","SearchTags":"keyword_list","SearchFlag":"bool"})), + {"type":"complete_workflow","result":envelope(Payload::Null)}])).await; + assert_eq!(second.0, StatusCode::OK, "{}", second.1); + let mut finished = expected.clone(); + for (key, value) in mutation.as_object().unwrap() { + finished[key] = value.clone(); + } + finished.as_object_mut().unwrap().remove("SearchRemoved"); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let upserted = events(&history, "SearchAttributesUpserted"); + assert_eq!(upserted[1]["payload"]["attributes"], mutation); + assert_eq!(upserted[1]["payload"]["merged"], finished); + assert_eq!(described(&app, "typed-upserts").await, finished); + runtime.close().await; + database.remove().await; +} + +#[tokio::test] +async fn invalid_upserts_are_refused_without_history() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + define(&app, "SearchCount", "int").await; + typed_worker(&app, "typed-worker").await; + register(&app, "plain-worker", json!(["echo"]), json!([])).await; + let started = start(&app, "refused-upserts").await; + let task = poll(&app, "typed-worker", "workflow").await; + let before = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + // The registered schema refuses an unregistered key and a wrong type. + let refused = finish_task(&app, &task, json!([ + {"type":"upsert_search_attributes","attributes":{"SearchCount":"3","Other":1}}])).await; + assert_eq!(refused.0, StatusCode::UNPROCESSABLE_ENTITY, "{}", refused.1); + assert_eq!(refused.1["errors"]["commands.0.attributes"], json!([ + "Search attribute [Other] is not registered for this namespace.", + "Search attribute [SearchCount] is registered as int and must be an integer.", + ])); + // Declared types need the worker protocol and capability that carry them. + let path = format!("/api/worker/workflow-tasks/{}/complete", task["task_id"].as_str().unwrap()); + let typed = completion(&task, json!([upsert(json!({"SearchCount":3}), json!({"SearchCount":"int"}))])); + let old_protocol = request_protocol(&app, "POST", &path, typed.clone(), "1.15").await; + assert_eq!(old_protocol.0, StatusCode::CONFLICT, "{}", old_protocol.1); + assert_eq!(old_protocol.1["reason"], "typed_search_attributes_unavailable"); + assert_eq!(old_protocol.1["minimum_protocol_version"], "1.16"); + assert_eq!(run_history(&app, &started["workflow_id"], &started["run_id"]).await, before); + let mut typed = typed; + typed["commands"].as_array_mut().unwrap().push(json!({"type":"open_signal_wait","signal_name":"finish"})); + let accepted = request(&app, "POST", &path, typed).await; + assert_eq!(accepted.0, StatusCode::OK, "{}", accepted.1); + // Typed history needs a worker that advertises the capability to replay it. + assert_eq!(request(&app, "POST", "/api/workflows/refused-upserts/signal/finish", + json!({"input":envelope(Payload::Array(vec![]))})).await.0, StatusCode::ACCEPTED); + assert!(poll(&app, "plain-worker", "workflow").await.is_null()); + assert!(poll(&app, "typed-worker", "workflow").await.is_object()); + // A worker registering the capability needs protocol 1.16. + let registered = request_protocol(&app, "POST", "/api/worker/register", json!({"worker_id":"old-worker", + "task_queue":"test","runtime":"rust","supported_workflow_types":["echo"],"supported_activity_types":[], + "capabilities":["typed_search_attributes"]}), "1.15").await; + assert_eq!(registered.0, StatusCode::CONFLICT, "{}", registered.1); + assert_eq!(registered.1["reason"], "workflow_metadata_capability_protocol_mismatch"); + runtime.close().await; + database.remove().await; +} + +#[tokio::test] +async fn continued_runs_inherit_typed_attributes() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + typed_worker(&app, "typed-worker").await; + let started = start(&app, "inherited-upserts").await; + let task = poll(&app, "typed-worker", "workflow").await; + // Without registered definitions, untyped values infer their type. + let continued = finish_task(&app, &task, json!([ + {"type":"upsert_search_attributes","attributes":{"Region":"eu-west","Attempt":2}}, + {"type":"continue_as_new"}])).await; + assert_eq!(continued.0, StatusCode::OK, "{}", continued.1); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + assert_eq!(events(&history, "SearchAttributesUpserted")[0]["payload"]["attribute_types"], + json!({"Attempt":"int","Region":"keyword"})); + let current = request(&app, "GET", "/api/workflows/inherited-upserts", Value::Null).await.1; + assert_ne!(current["run_id"], started["run_id"], "{current}"); + assert_eq!(current["search_attributes"], json!({"Attempt":2,"Region":"eu-west"})); + runtime.close().await; + database.remove().await; +} From 1d356d8a0bc86d51e05573b6da9047fb24ed19b8 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 23:03:01 +0000 Subject: [PATCH 22/30] Give signal task claims the signal's arguments, like PHP Published PHP's workflow task claim for a signal task carries signal_arguments, workflow_signal_id and workflow_command_id. The Rust SDK resumes a signal wait with signal_arguments. Native sent only the signal name, so a Rust workflow resumed after a worker restart received no arguments; the Sample App's search attribute scenario returned signal [] instead of ["released"]. Co-Authored-By: Claude Opus 5.5 --- .../2026-10-21-signal-claim-arguments.md | 18 +++++++++++++ rust/src/runtime/store.rs | 27 +++++++++++++++++++ rust/tests/execution.rs | 2 ++ rust/tests/execution/signal_claims.rs | 25 +++++++++++++++++ 4 files changed, 72 insertions(+) create mode 100644 docs/rust-server-port-checkpoints/2026-10-21-signal-claim-arguments.md create mode 100644 rust/tests/execution/signal_claims.rs diff --git a/docs/rust-server-port-checkpoints/2026-10-21-signal-claim-arguments.md b/docs/rust-server-port-checkpoints/2026-10-21-signal-claim-arguments.md new file mode 100644 index 000000000..39cc20a0e --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-21-signal-claim-arguments.md @@ -0,0 +1,18 @@ +# October 21, signal arguments in workflow task claims + +- **Found by.** The Sample App's search attribute scenario against native. + The Rust workflow was parked at a signal wait, its worker was killed, and + the signal `["released"]` arrived while no worker was running. The + replacement worker returned `signal: []`. +- **Cause.** Published PHP's `WorkflowTaskPoller::workflowTaskResumeContext` + gives a signal task's claim `signal_arguments`, the signal record's + argument envelope, along with `workflow_signal_id` and + `workflow_command_id`. The Rust SDK resumes the wait with + `signal_arguments`. Native sent `signal_name` without the arguments, so + the SDK resumed the wait with none. +- **Native now.** The claim of a signal task carries `signal_arguments`, + `workflow_signal_id` and `workflow_command_id`. The command ID comes from + the signal record when an older native task payload lacks it. +- **Verification.** A new test in `tests/execution/signal_claims.rs` claims a + signal task and checks the three fields against the signal request. The + signal and claim tests pass on SQLite. diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index e86052ed9..04610edeb 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -1392,6 +1392,33 @@ where claim[field] = value.clone(); } } + // `WorkflowTaskPoller::workflowTaskResumeContext`: a signal task + // names its signal and command and carries the signal's + // arguments, which the Rust SDK resumes the wait with. + for field in ["workflow_signal_id", "workflow_command_id"] { + if let Some(value) = task_payload[field].as_str().map(str::trim).filter(|value| !value.is_empty()) { + claim[field] = json!(value); + } + } + if let Some(signal_id) = task_payload["workflow_signal_id"].as_str() { + let signal = Self::query("SELECT arguments,workflow_command_id FROM workflow_signal_records WHERE id=$1 AND workflow_run_id=$2") + .bind(signal_id) + .bind(&run_id) + .fetch_optional(&mut *tx) + .await?; + if let Some(signal) = signal { + if let Some(arguments) = + DB::optional_string(&signal, "arguments")?.filter(|arguments| !arguments.is_empty()) + { + claim["signal_arguments"] = wire(&arguments); + } + // PHP's signal task payload names its command; older + // native tasks take it from the signal record. + if claim.get("workflow_command_id").is_none() { + claim["workflow_command_id"] = json!(DB::string(&signal, "workflow_command_id")?); + } + } + } let page_size = body["history_page_size"].as_i64().unwrap_or(100); let (events, next) = Self::history_page_connection(&mut tx, &run_id, 0, page_size).await?; diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 89fa3a9cd..e4915e333 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -53,6 +53,8 @@ mod named_timers; #[path = "execution/named_signal_timeouts.rs"] mod named_signal_timeouts; +#[path = "execution/signal_claims.rs"] +mod signal_claims; #[path = "execution/selected_run_signals.rs"] mod selected_run_signals; diff --git a/rust/tests/execution/signal_claims.rs b/rust/tests/execution/signal_claims.rs new file mode 100644 index 000000000..6f9bef06d --- /dev/null +++ b/rust/tests/execution/signal_claims.rs @@ -0,0 +1,25 @@ +//! A signal task's claim carries the signal's identity and arguments, as +//! published PHP's `workflowTaskResumeContext` delivers them. The Rust SDK +//! resumes a signal wait with these arguments. +use super::*; + +#[tokio::test] +async fn a_signal_claim_carries_the_signal_arguments() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + update_worker(&app, "signal-worker").await; + start(&app, "claimed-signal").await; + open_update_wait(&app, "signal-worker").await; + let input = envelope(Payload::Array(vec![Payload::Long(9007199254740993)])); + let signal = request(&app, "POST", "/api/workflows/claimed-signal/signal/finish", json!({"input":input})).await; + assert_eq!(signal.0, StatusCode::ACCEPTED, "{}", signal.1); + let task = poll(&app, "signal-worker", "workflow").await; + assert_eq!(task["resume_source_kind"], "workflow_signal", "{task}"); + assert_eq!(task["signal_name"], "finish"); + assert_eq!(task["signal_arguments"], input); + assert_eq!(task["workflow_command_id"], signal.1["command_id"]); + assert!(task["workflow_signal_id"].as_str().is_some_and(|id| !id.is_empty()), "{task}"); + runtime.close().await; + database.remove().await; +} From c0da48c13316c0ab7bde19a3d96d359081e53da6 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 23:05:19 +0000 Subject: [PATCH 23/30] Filter workflow lists with PHP's typed visibility queries The Sample App's search attribute scenario lists workflows with typed predicates and checks each run's typed attributes. Native refused any query with a comparison and listed no attributes. Native now parses the list query like published PHP's WorkflowVisibilityQuery: OR and AND groups, NOT, comparisons, IN lists and bare boolean fields over Status, the run's system columns and registered search attributes, with PHP's validation messages. A plain term keeps the free-text search. Every listed run carries its typed search_attributes. Native evaluates the query in memory over candidate runs, 500 at a time, so its static SQL stays unchanged. Co-Authored-By: Claude Opus 5.5 --- .../2026-10-21-search-attribute-upserts.md | 8 +- .../2026-10-21-visibility-queries.md | 39 + docs/rust-server-port.md | 15 +- rust/src/runtime/backend.rs | 2 + rust/src/runtime/mod.rs | 1 + rust/src/runtime/search_attribute_upserts.rs | 36 +- rust/src/runtime/visibility.rs | 138 ++- rust/src/runtime/visibility_query.rs | 795 ++++++++++++++++++ .../execution/search_attribute_upserts.rs | 53 ++ 9 files changed, 1046 insertions(+), 41 deletions(-) create mode 100644 docs/rust-server-port-checkpoints/2026-10-21-visibility-queries.md create mode 100644 rust/src/runtime/visibility_query.rs diff --git a/docs/rust-server-port-checkpoints/2026-10-21-search-attribute-upserts.md b/docs/rust-server-port-checkpoints/2026-10-21-search-attribute-upserts.md index 4fb1a3a68..e6a65cd61 100644 --- a/docs/rust-server-port-checkpoints/2026-10-21-search-attribute-upserts.md +++ b/docs/rust-server-port-checkpoints/2026-10-21-search-attribute-upserts.md @@ -36,10 +36,10 @@ `continue_as_new` in the same task. - **Describe.** Reads keyword lists through each backend's JSON decoding. PostgreSQL stores them in a `json` column. -- **Remaining.** Workflow lists still show no attributes and refuse typed - query predicates, so the Sample App scenario is not complete yet. Other - remaining differences are recorded as known gaps: start-time search - attributes, child-run inheritance, datetime forms and the limit refusal. +- **Remaining.** Lists and typed list queries follow in the same branch + (`2026-10-21-visibility-queries.md`). Other remaining differences are + recorded as known gaps: start-time search attributes, child-run + inheritance, datetime forms and the limit refusal. - **Differential probe.** Published PHP 2.5.14 and native received the same requests: twelve upsert completions covering registered, declared and inferred types, bad keys, wrong types, malformed `attribute_types`, empty, diff --git a/docs/rust-server-port-checkpoints/2026-10-21-visibility-queries.md b/docs/rust-server-port-checkpoints/2026-10-21-visibility-queries.md new file mode 100644 index 000000000..276da5f62 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-21-visibility-queries.md @@ -0,0 +1,39 @@ +# October 21, structured workflow list queries + +- **Found by.** The Sample App's search attribute scenario lists workflows + with typed predicates such as + `SearchCount > 9007199254740992 AND SearchCount <= 9007199254740993`. It + checks that each listed run carries its typed attributes. Native refused + any `query` containing `=`, `<`, `>` or a quote with + `422 unsupported_visibility_query`, and listed `search_attributes: []`. +- **Grammar.** Native parses the query like published PHP's + `WorkflowVisibilityQuery`: + - groups joined by `OR`, each joined by `AND`, outside quotes and + parentheses; + - `NOT`, the comparison operators, `IN (...)` lists and bare boolean + fields; + - quoted strings with PHP's `stripcslashes` escapes, booleans, integers, + floats and bare identifiers. + + A plain term that does not look like a query keeps the free-text workflow + ID and business key search. +- **Fields.** `Status` and `ExecutionStatus` compare the status or its + bucket. `WorkflowType`, `WorkflowId`, `RunId`, `TaskQueue`, `BuildId`, + `StartTime`, `ExecutionTime` and `CloseTime` compare run columns. Any + other field must be a registered search attribute and compares its typed + row: a run without the row never matches. Keyword lists match by + membership. +- **Refusals.** Parse and type errors answer `422 validation_failed` with + PHP's message on `query`. Examples: an undefined attribute, an ordering + operator on a keyword, a literal of the wrong type, or `NOT IN` on a + keyword list. +- **Lists.** Every listed run carries its typed `search_attributes`, or `[]` + when it has none, as PHP's `getTypedSearchAttributes` returns them. +- **Remaining.** The differences in evaluation cost, text collation and + error order are recorded as known gaps. +- **Verification.** + - Unit tests in `visibility_query.rs` cover the Sample App's queries, plain + terms, PHP's refusals, a missing attribute and the `NOT ... IN` form. + - A new test lists two runs through keyword, integer-range, + keyword-list, boolean, `OR`, `NOT`, `IN`, status and free-text queries. + It checks the listed attributes and an undefined-attribute refusal. diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 9f96a5969..f6c35598c 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -198,11 +198,16 @@ contract fixture before its correction. a signal therefore waits on PHP and is refused on native. The port takes the snapshot at claim time and must keep query lease replay stable, so it needs its own fixture. -- **Search attributes in workflow lists.** Native lists show - `search_attributes: []`, and the list `query` refuses any comparison with - `422 unsupported_visibility_query`. Published PHP returns each run's typed - attributes and filters with `WorkflowVisibilityQuery` over typed - predicates. The Sample App's search attribute scenario needs both. +- **Visibility query evaluation.** Published PHP filters a structured list + `query` in SQL. Native reads the queried attribute keys once, then + evaluates candidate runs in list order, 500 at a time, until it has the + requested page. A query that matches few runs in a large namespace reads + every run of that namespace. +- **Visibility query text comparisons.** PHP compares keyword and text + values with the database collation, which is case-insensitive in the + published MySQL schema. Native compares them exactly on every backend. +- **Visibility query key order.** When one command or query has several + errors, native reports them in sorted key order rather than request order. - **Inherited memos and search attributes on child runs.** Published PHP copies the parent's memos and search attributes to a child run and to a retried child (`inheritTypedVisibilityMetadata`). Native copies both only diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index a7e757907..93f27de88 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -61,6 +61,7 @@ pub(super) trait Backend: Database { const SEARCH_ATTRIBUTE_ROWS: &'static str = "SELECT \"key\" AS attribute_key,type,value_string,value_keyword,value_int,value_float,CASE WHEN value_bool IS NULL THEN NULL WHEN value_bool THEN 1 ELSE 0 END AS value_bool,value_datetime,value_keyword_list FROM workflow_search_attributes WHERE workflow_run_id=$1 ORDER BY \"key\" LIMIT 1001"; const SEARCH_ATTRIBUTE_UPSERT: &'static str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,\"key\",type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,false,$13,$14) ON CONFLICT(workflow_run_id,\"key\") DO UPDATE SET workflow_instance_id=excluded.workflow_instance_id,type=excluded.type,value_string=excluded.value_string,value_keyword=excluded.value_keyword,value_keyword_list=excluded.value_keyword_list,value_int=excluded.value_int,value_float=excluded.value_float,value_bool=excluded.value_bool,value_datetime=excluded.value_datetime,upserted_at_sequence=excluded.upserted_at_sequence,inherited_from_parent=false,updated_at=excluded.updated_at"; const SEARCH_ATTRIBUTE_DELETE: &'static str = "DELETE FROM workflow_search_attributes WHERE workflow_run_id=$1 AND \"key\"=$2"; + const SEARCH_ATTRIBUTE_KEY_ROWS: &'static str = "SELECT a.workflow_run_id,a.\"key\" AS attribute_key,a.type,a.value_string,a.value_keyword,a.value_int,a.value_float,CASE WHEN a.value_bool IS NULL THEN NULL WHEN a.value_bool THEN 1 ELSE 0 END AS value_bool,a.value_datetime,a.value_keyword_list FROM workflow_search_attributes a JOIN workflow_runs r ON r.id=a.workflow_run_id WHERE r.namespace=$1 AND a.\"key\"=$2"; const SEARCH_ATTRIBUTE_INHERIT: &'static str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,\"key\",type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) SELECT $1,$2,\"key\",type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,1,true,$3,$4 FROM workflow_search_attributes WHERE workflow_run_id=$5"; fn statement(sql: &'static str) -> Cow<'static, str> { Cow::Borrowed(sql) @@ -409,6 +410,7 @@ impl Backend for MySql { const SEARCH_ATTRIBUTE_ROWS: &'static str = "SELECT `key` AS attribute_key,type,value_string,value_keyword,value_int,value_float,CASE WHEN value_bool IS NULL THEN NULL WHEN value_bool THEN 1 ELSE 0 END AS value_bool,value_datetime,value_keyword_list FROM workflow_search_attributes WHERE workflow_run_id=$1 ORDER BY `key` LIMIT 1001"; const SEARCH_ATTRIBUTE_UPSERT: &'static str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,`key`,type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,false,$13,$14) ON DUPLICATE KEY UPDATE workflow_instance_id=VALUES(workflow_instance_id),type=VALUES(type),value_string=VALUES(value_string),value_keyword=VALUES(value_keyword),value_keyword_list=VALUES(value_keyword_list),value_int=VALUES(value_int),value_float=VALUES(value_float),value_bool=VALUES(value_bool),value_datetime=VALUES(value_datetime),upserted_at_sequence=VALUES(upserted_at_sequence),inherited_from_parent=false,updated_at=VALUES(updated_at)"; const SEARCH_ATTRIBUTE_DELETE: &'static str = "DELETE FROM workflow_search_attributes WHERE workflow_run_id=$1 AND `key`=$2"; + const SEARCH_ATTRIBUTE_KEY_ROWS: &'static str = "SELECT a.workflow_run_id,a.`key` AS attribute_key,a.type,a.value_string,a.value_keyword,a.value_int,a.value_float,CASE WHEN a.value_bool IS NULL THEN NULL WHEN a.value_bool THEN 1 ELSE 0 END AS value_bool,a.value_datetime,a.value_keyword_list FROM workflow_search_attributes a JOIN workflow_runs r ON r.id=a.workflow_run_id WHERE r.namespace=$1 AND a.`key`=$2"; const SEARCH_ATTRIBUTE_INHERIT: &'static str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,`key`,type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) SELECT $1,$2,`key`,type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,1,true,$3,$4 FROM workflow_search_attributes WHERE workflow_run_id=$5"; const READ_SNAPSHOT_BEGIN: &'static str = "START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY"; const ACTIVITY_METADATA_HISTORY: &'static str = "SELECT h.sequence,h.event_type,CASE WHEN page.projection_bytes<=8388608 THEN h.payload ELSE NULL END AS payload,h.workflow_task_id,h.recorded_at,page.projection_count FROM (SELECT id,sequence,COUNT(*) OVER () AS projection_count,SUM(payload_bytes) OVER (ORDER BY sequence) AS projection_bytes FROM (SELECT id,sequence,octet_length(CAST(payload AS CHAR CHARACTER SET utf8mb4)) AS payload_bytes FROM workflow_history_events FORCE INDEX (workflow_history_events_workflow_run_id_sequence_unique) WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled')) AS sizes ORDER BY sequence LIMIT 10001) AS page JOIN workflow_history_events h ON h.id=page.id ORDER BY page.sequence"; diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index c766d187e..acb01dc95 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -85,6 +85,7 @@ mod update_summary; mod updates; pub mod upgrade; mod visibility; +mod visibility_query; mod wait_projection; mod wait_timeouts; mod worker_contracts; diff --git a/rust/src/runtime/search_attribute_upserts.rs b/rust/src/runtime/search_attribute_upserts.rs index f1d7b891b..ff717f81a 100644 --- a/rust/src/runtime/search_attribute_upserts.rs +++ b/rust/src/runtime/search_attribute_upserts.rs @@ -124,7 +124,7 @@ fn infer_type(value: &Value) -> &'static str { /// `Carbon::parse` for the forms workers send: RFC 3339, or a date and time /// without an offset, read as UTC like the server's timezone. -fn parse_datetime(text: &str) -> Option> { +pub(super) fn parse_datetime(text: &str) -> Option> { if let Ok(time) = DateTime::parse_from_rfc3339(text) { return Some(time.with_timezone(&Utc)); } @@ -569,27 +569,31 @@ where .await?; let mut typed = BTreeMap::new(); for row in rows { - let value = match DB::string(&row, "type")?.as_str() { - "string" => DB::optional_string(&row, "value_string")?.map(Typed::String), - "keyword" => DB::optional_string(&row, "value_keyword")?.map(Typed::Keyword), - "keyword_list" => Some(Typed::KeywordList( - DB::optional_document_row(&row, "value_keyword_list")? - .and_then(|list| serde_json::from_value::>(list).ok()) - .unwrap_or_default(), - )), - "int" => DB::optional_number(&row, "value_int")?.map(Typed::Int), - "float" => DB::optional_float(&row, "value_float")?.map(Typed::Float), - "bool" => DB::optional_number(&row, "value_bool")?.map(|flag| Typed::Bool(flag != 0)), - "datetime" => DB::optional_instant(&row, "value_datetime")?.map(Typed::Datetime), - _ => None, - }; - if let Some(value) = value { + if let Some(value) = Self::typed_value(&row)? { typed.insert(DB::string(&row, "attribute_key")?, value); } } Ok(typed) } + /// One typed row's value; a row of an unknown type has none. + pub(super) fn typed_value(row: &DB::Row) -> Result> { + Ok(match DB::string(row, "type")?.as_str() { + "string" => DB::optional_string(row, "value_string")?.map(Typed::String), + "keyword" => DB::optional_string(row, "value_keyword")?.map(Typed::Keyword), + "keyword_list" => Some(Typed::KeywordList( + DB::optional_document_row(row, "value_keyword_list")? + .and_then(|list| serde_json::from_value::>(list).ok()) + .unwrap_or_default(), + )), + "int" => DB::optional_number(row, "value_int")?.map(Typed::Int), + "float" => DB::optional_float(row, "value_float")?.map(Typed::Float), + "bool" => DB::optional_number(row, "value_bool")?.map(|flag| Typed::Bool(flag != 0)), + "datetime" => DB::optional_instant(row, "value_datetime")?.map(Typed::Datetime), + _ => None, + }) + } + /// The bridge's `applyUpsertSearchAttributes` after the Server's /// `validateWorkflowTaskSearchAttributeCommands`. pub(super) async fn upsert_search_attributes( diff --git a/rust/src/runtime/visibility.rs b/rust/src/runtime/visibility.rs index bd8f0ca77..f54a0e710 100644 --- a/rust/src/runtime/visibility.rs +++ b/rust/src/runtime/visibility.rs @@ -6,7 +6,9 @@ use serde::Deserialize; use serde_json::{Value, json}; use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; -use super::{Result, RuntimeError, backend::Backend, refuse, store::Store}; +use std::collections::BTreeMap; + +use super::{Result, RuntimeError, backend::Backend, refuse, search_attribute_upserts::Typed, store::Store}; #[derive(Default, Deserialize)] pub(super) struct VisibilityQuery { @@ -21,10 +23,15 @@ pub(super) struct VisibilityFilter { workflow_type: Option, status: Option, pattern: Option, + /// A structured query, which replaces the free-text `pattern`. + visibility: Option, limit: i64, offset: i64, } +/// Candidate runs read per page while a structured query filters them. +const QUERY_SCAN_PAGE: i64 = 500; + impl VisibilityQuery { pub(super) fn validate(self) -> Result { if self @@ -49,13 +56,14 @@ impl VisibilityQuery { })?, None => 50, }; - let term = self.query.as_deref().unwrap_or("").trim(); - if term.len() > 4096 || term.contains(['=', '<', '>', '"', '\'']) { - return Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "unsupported_visibility_query", - )); - } + // `WorkflowVisibilityQuery::apply`; a plain term keeps the free-text + // workflow ID and business key search. + let term = self.query.as_deref().unwrap_or("").trim_matches([' ', '\t', '\n', '\r', '\0', '\x0b']); + let visibility = if term.is_empty() { + None + } else { + super::visibility_query::parse(term).map_err(query_failure)? + }; let offset = self .next_page_token .as_deref() @@ -66,7 +74,8 @@ impl VisibilityQuery { Ok(VisibilityFilter { workflow_type: self.workflow_type, status: self.status, - pattern: (!term.is_empty()).then(|| format!("%{term}%")), + pattern: (!term.is_empty() && visibility.is_none()).then(|| format!("%{term}%")), + visibility, limit, offset, }) @@ -93,6 +102,14 @@ fn response(body: Value) -> Value { super::control_plane::Operation::List.response(body) } +fn query_failure(message: String) -> RuntimeError { + RuntimeError::Protocol { + status: StatusCode::UNPROCESSABLE_ENTITY, + response: response(json!({"reason":"validation_failed", "message":message, + "errors":{"query":[message]}, "validation_errors":{"query":[message]}})), + } +} + fn validation(field: &'static str, message: &'static str) -> RuntimeError { RuntimeError::Protocol { status: StatusCode::UNPROCESSABLE_ENTITY, @@ -130,10 +147,10 @@ where for<'r> i64: Decode<'r, DB>, usize: ColumnIndex, { - pub(super) async fn list_workflows(&self, filter: &VisibilityFilter) -> Result { + async fn visibility_page(&self, filter: &VisibilityFilter, limit: i64, offset: i64) -> Result> { // PHP's index filters cancelled and terminated runs by their exact // status and the other values by status bucket. - let rows = Self::query(DB::VISIBILITY_PAGE) + Ok(Self::query(DB::VISIBILITY_PAGE) .bind(self.namespace.as_ref()) .bind(self.namespace.as_ref()) .bind(filter.workflow_type.as_deref()) @@ -144,24 +161,113 @@ where .bind(filter.pattern.as_deref()) .bind(filter.pattern.as_deref()) .bind(filter.pattern.as_deref()) - .bind(filter.limit + 1) - .bind(filter.offset) + .bind(limit) + .bind(offset) + .fetch_all(&self.pool) + .await?) + } + + /// The runs a structured query selects, in list order, until one past + /// the requested page. + async fn query_matches( + &self, + filter: &VisibilityFilter, + parsed: &super::visibility_query::Parsed, + ) -> Result> { + let definitions = Self::query("SELECT name,type FROM search_attribute_definitions WHERE namespace=$1") + .bind(self.namespace.as_ref()) .fetch_all(&self.pool) - .await?; + .await? + .iter() + .map(|row| Ok((DB::string(row, "name")?, DB::string(row, "type")?))) + .collect::>>()?; + let compiled = super::visibility_query::compile(parsed, &definitions, &self.namespace) + .map_err(query_failure)?; + let mut attributes: BTreeMap> = BTreeMap::new(); + for key in compiled.keys() { + let rows = Self::query(DB::SEARCH_ATTRIBUTE_KEY_ROWS) + .bind(self.namespace.as_ref()) + .bind(key.as_str()) + .fetch_all(&self.pool) + .await?; + for row in rows { + if let Some(value) = Self::typed_value(&row)? { + attributes + .entry(DB::string(&row, "workflow_run_id")?) + .or_default() + .insert(DB::string(&row, "attribute_key")?, value); + } + } + } + let wanted = (filter.offset.saturating_add(filter.limit).saturating_add(1)) as usize; + let none = BTreeMap::new(); + let mut matches = Vec::new(); + let mut scanned = 0; + loop { + let rows = self.visibility_page(filter, QUERY_SCAN_PAGE, scanned).await?; + let exhausted = (rows.len() as i64) < QUERY_SCAN_PAGE; + scanned += rows.len() as i64; + for row in rows { + let status = DB::string(&row, "status")?; + let run_id = DB::string(&row, "id")?; + let fields = super::visibility_query::RunFields { + workflow_type: DB::string(&row, "workflow_type")?, + workflow_id: DB::string(&row, "workflow_instance_id")?, + run_id: run_id.clone(), + queue: DB::string(&row, "queue")?, + compatibility: DB::optional_string(&row, "compatibility")?, + started_at: DB::optional_instant(&row, "started_at")?, + closed_at: DB::optional_instant(&row, "closed_at")?, + status_bucket: status_bucket(&status)?.to_owned(), + status, + }; + if compiled.matches(&fields, attributes.get(&run_id).unwrap_or(&none)) { + matches.push(row); + if matches.len() >= wanted { + return Ok(matches); + } + } + } + if exhausted { + return Ok(matches); + } + } + } + + pub(super) async fn list_workflows(&self, filter: &VisibilityFilter) -> Result { + let rows = match &filter.visibility { + Some(parsed) => self + .query_matches(filter, parsed) + .await? + .into_iter() + .skip(filter.offset.max(0) as usize) + .collect(), + None => self.visibility_page(filter, filter.limit + 1, filter.offset).await?, + }; let more = rows.len() > filter.limit as usize; let mut workflows = Vec::new(); + let mut connection = self.pool.acquire().await?; for row in rows.iter().take(filter.limit as usize) { if DB::optional_string(row, "compatibility")?.is_some() { return Err(refuse(StatusCode::CONFLICT, "unqualified_build_visibility")); } let status = DB::string(row, "status")?; let bucket = status_bucket(&status)?; - workflows.push(json!({"workflow_id":DB::string(row,"workflow_instance_id")?, "run_id":DB::string(row,"id")?, + let run_id = DB::string(row, "id")?; + // `getTypedSearchAttributes`; PHP renders an empty set as `[]`. + let typed = Self::typed_rows(&mut connection, &run_id).await?; + let search_attributes = if typed.is_empty() { + json!([]) + } else { + Value::Object(typed.into_iter().map(|(key, value)| (key, value.json())).collect()) + }; + workflows.push(json!({"workflow_id":DB::string(row,"workflow_instance_id")?, "run_id":run_id, "workflow_type":DB::string(row,"workflow_type")?, "business_key":DB::optional_string(row,"business_key")?, "status":status, "status_bucket":bucket, "task_queue":DB::string(row,"queue")?, "is_terminal":bucket!="running", "compatibility":null, "compatibility_status":"compatible", "compatibility_supported_in_fleet":true, "compatibility_fleet_reason":null, - "started_at":DB::instant(row,"started_at")?, "closed_at":DB::optional_instant(row,"closed_at")?, "search_attributes":[]})); + "started_at":DB::instant(row,"started_at")?, "closed_at":DB::optional_instant(row,"closed_at")?, + "search_attributes":search_attributes})); } let next = more.then(|| STANDARD.encode((filter.offset.saturating_add(filter.limit)).to_string())); diff --git a/rust/src/runtime/visibility_query.rs b/rust/src/runtime/visibility_query.rs new file mode 100644 index 000000000..86593f29d --- /dev/null +++ b/rust/src/runtime/visibility_query.rs @@ -0,0 +1,795 @@ +//! Published PHP's `WorkflowVisibilityQuery`: a Temporal-style filter over a +//! run's system fields and its typed search attributes. A plain term is not +//! a query; the caller keeps its free-text search for it. +use std::collections::BTreeMap; + +use chrono::{DateTime, Utc}; + +use super::search_attribute_upserts::{Typed, parse_datetime}; + +/// A parsed literal, as PHP's `parseLiteral` returns it. +#[derive(Clone, Debug, PartialEq)] +pub(super) enum Literal { + Text(String), + Bool(bool), + Int(i64), + Float(f64), + List(Vec), +} + +#[derive(Clone, Debug, PartialEq)] +struct Comparison { + field: String, + operator: &'static str, + literal: Literal, + bare: bool, +} + +/// A query's comparisons: groups joined by `OR`, each joined by `AND`. +#[derive(Clone, Debug, PartialEq)] +pub(super) struct Parsed(Vec>); + +fn invalid(message: impl Into) -> String { + message.into() +} + +fn identifier(text: &str) -> bool { + let mut chars = text.chars(); + chars.next().is_some_and(|first| first.is_ascii_alphabetic()) + && chars.all(|character| character.is_ascii_alphanumeric() || character == '_') +} + +fn word(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || byte == b'_' +} + +/// PHP's `looksLikeVisibilityQuery`. +fn looks_like_query(query: &str) -> bool { + let bytes = query.as_bytes(); + let starts_identifier = |index: usize| { + bytes[index].is_ascii_alphabetic() && (index == 0 || !word(bytes[index - 1])) + }; + for start in 0..bytes.len() { + if !starts_identifier(start) { + continue; + } + let mut end = start; + while end < bytes.len() && word(bytes[end]) { + end += 1; + } + let rest = query[end..].trim_start(); + if rest.starts_with(['=', '<', '>']) || rest.starts_with("!=") { + return true; + } + if end < bytes.len() && bytes[end].is_ascii_whitespace() { + let after = rest; + if after.len() >= 2 + && after[..2].eq_ignore_ascii_case("IN") + && after[2..].trim_start().starts_with('(') + { + return true; + } + } + let name = &query[start..end]; + if name.eq_ignore_ascii_case("NOT") + && end < bytes.len() + && bytes[end].is_ascii_whitespace() + && rest.chars().next().is_some_and(|first| first.is_ascii_alphabetic()) + { + let before = query[..start].trim_end(); + if before.is_empty() + || (before.len() >= 3 + && before[before.len() - 3..].eq_ignore_ascii_case("AND") + && (before.len() == 3 || !word(before.as_bytes()[before.len() - 4])) + && start > before.len()) + { + return true; + } + } + } + false +} + +/// PHP's `splitConditionsByKeyword`: `None` for an empty part, an +/// unbalanced parenthesis or an unterminated quote. +fn split_by_keyword(query: &str, keyword: &str) -> Option> { + if query.is_empty() { + return None; + } + let bytes = query.as_bytes(); + let mut parts = Vec::new(); + let mut current = Vec::new(); + let mut quote = None; + let mut depth = 0i64; + let mut index = 0; + while index < bytes.len() { + let byte = bytes[index]; + if let Some(open) = quote { + current.push(byte); + if byte == b'\\' && index + 1 < bytes.len() { + index += 1; + current.push(bytes[index]); + } else if byte == open { + quote = None; + } + index += 1; + continue; + } + match byte { + b'(' => depth += 1, + b')' => { + depth -= 1; + if depth < 0 { + return None; + } + } + b'"' | b'\'' => quote = Some(byte), + _ => {} + } + let keyword_here = depth == 0 + && !matches!(byte, b'(' | b')' | b'"' | b'\'') + && bytes.len() >= index + keyword.len() + && query.is_char_boundary(index) + && query.is_char_boundary(index + keyword.len()) + && query[index..index + keyword.len()].eq_ignore_ascii_case(keyword) + && (index == 0 || !word(bytes[index - 1])) + && bytes.get(index + keyword.len()).is_none_or(|after| !word(*after)); + if keyword_here { + let part = String::from_utf8_lossy(¤t).trim().to_owned(); + if part.is_empty() { + return None; + } + parts.push(part); + current.clear(); + index += keyword.len(); + continue; + } + current.push(byte); + index += 1; + } + let part = String::from_utf8_lossy(¤t).trim().to_owned(); + if quote.is_some() || depth != 0 || part.is_empty() { + return None; + } + parts.push(part); + Some(parts) +} + +/// PHP's `stripcslashes` for the escapes a query literal can carry. +fn strip_c_slashes(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut chars = text.chars().peekable(); + while let Some(character) = chars.next() { + if character != '\\' { + out.push(character); + continue; + } + match chars.next() { + Some('n') => out.push('\n'), + Some('t') => out.push('\t'), + Some('r') => out.push('\r'), + Some('a') => out.push('\x07'), + Some('v') => out.push('\x0b'), + Some('b') => out.push('\x08'), + Some('f') => out.push('\x0c'), + Some('x') => { + let mut digits = String::new(); + while digits.len() < 2 && chars.peek().is_some_and(char::is_ascii_hexdigit) { + digits.push(chars.next().expect("peeked")); + } + match u8::from_str_radix(&digits, 16) { + Ok(value) if !digits.is_empty() => out.push(char::from(value)), + _ => out.push('x'), + } + } + Some(digit @ '0'..='7') => { + let mut digits = digit.to_string(); + while digits.len() < 3 && chars.peek().is_some_and(|next| ('0'..='7').contains(next)) { + digits.push(chars.next().expect("peeked")); + } + out.push(char::from(u8::from_str_radix(&digits, 8).unwrap_or(0))); + } + Some(other) => out.push(other), + None => {} + } + } + out +} + +/// PHP's `parseLiteral`; `None` is its invalid literal. +fn parse_literal(raw: &str) -> Option { + let raw = raw.trim(); + if raw.is_empty() { + return None; + } + let first = raw.as_bytes()[0]; + if (first == b'"' || first == b'\'') && raw.len() >= 2 && raw.as_bytes()[raw.len() - 1] == first { + return Some(Literal::Text(strip_c_slashes(&raw[1..raw.len() - 1]))); + } + if raw.eq_ignore_ascii_case("true") { + return Some(Literal::Bool(true)); + } + if raw.eq_ignore_ascii_case("false") { + return Some(Literal::Bool(false)); + } + let unsigned = raw.strip_prefix(['+', '-']).unwrap_or(raw); + if !unsigned.is_empty() && unsigned.bytes().all(|byte| byte.is_ascii_digit()) { + // PHP's (int) saturates an out-of-range decimal string. + return Some(Literal::Int(raw.parse::().unwrap_or(if raw.starts_with('-') { + i64::MIN + } else { + i64::MAX + }))); + } + if float_literal(unsigned) { + return raw.parse::().ok().map(Literal::Float); + } + if raw.bytes().all(|byte| byte.is_ascii_alphanumeric() || b"_.:-".contains(&byte)) { + return Some(Literal::Text(raw.to_owned())); + } + None +} + +/// `\d+\.\d*`, `\.\d+`, and either with an exponent, or `\d+[eE][+-]?\d+`. +fn float_literal(text: &str) -> bool { + let (mantissa, exponent) = match text.find(['e', 'E']) { + Some(at) => (&text[..at], Some(&text[at + 1..])), + None => (text, None), + }; + let digits = |part: &str| !part.is_empty() && part.bytes().all(|byte| byte.is_ascii_digit()); + let mantissa_ok = match mantissa.split_once('.') { + Some((whole, fraction)) => { + (digits(whole) && (fraction.is_empty() || digits(fraction))) || (whole.is_empty() && digits(fraction)) + } + None => exponent.is_some() && digits(mantissa), + }; + mantissa_ok && exponent.is_none_or(|exponent| digits(exponent.strip_prefix(['+', '-']).unwrap_or(exponent))) +} + +/// PHP's `splitCommaList`: `None` for an empty item or an unterminated quote. +fn split_commas(raw: &str) -> Option> { + let bytes = raw.as_bytes(); + let mut parts = Vec::new(); + let mut current = Vec::new(); + let mut quote = None; + let mut index = 0; + while index < bytes.len() { + let byte = bytes[index]; + if let Some(open) = quote { + current.push(byte); + if byte == b'\\' && index + 1 < bytes.len() { + index += 1; + current.push(bytes[index]); + } else if byte == open { + quote = None; + } + } else if byte == b'"' || byte == b'\'' { + quote = Some(byte); + current.push(byte); + } else if byte == b',' { + let part = String::from_utf8_lossy(¤t).trim().to_owned(); + if part.is_empty() { + return None; + } + parts.push(part); + current.clear(); + } else { + current.push(byte); + } + index += 1; + } + let part = String::from_utf8_lossy(¤t).trim().to_owned(); + if quote.is_some() || part.is_empty() { + return None; + } + parts.push(part); + Some(parts) +} + +const OPERATORS: [&str; 6] = ["!=", ">=", "<=", "=", ">", "<"]; + +fn parse_predicate(part: &str) -> Result, String> { + let part = part.trim(); + if part.len() > 4 && part[..3].eq_ignore_ascii_case("NOT") && part.as_bytes()[3].is_ascii_whitespace() { + return Ok(parse_predicate(part[3..].trim_start())?.map(invert)); + } + let name_end = part.find(|character: char| !(character.is_ascii_alphanumeric() || character == '_')).unwrap_or(part.len()); + let name = &part[..name_end]; + if !identifier(name) { + return Ok(None); + } + let rest = part[name_end..].trim_start(); + if let Some(operator) = OPERATORS.iter().find(|operator| rest.starts_with(**operator)) { + let literal_text = rest[operator.len()..].trim_start(); + if literal_text.is_empty() { + return Ok(None); + } + let literal = parse_literal(literal_text) + .ok_or_else(|| invalid(format!("Visibility query literal [{}] is not valid.", literal_text.trim())))?; + return Ok(Some(Comparison { field: name.to_owned(), operator, literal, bare: false })); + } + if name_end < part.len() + && part.as_bytes()[name_end].is_ascii_whitespace() + && rest.len() >= 2 + && rest[..2].eq_ignore_ascii_case("IN") + { + let list = rest[2..].trim_start(); + if let Some(inner) = list.strip_prefix('(').and_then(|list| list.strip_suffix(')')) { + let literals = split_commas(inner) + .and_then(|parts| parts.iter().map(|part| parse_literal(part)).collect::>>()) + .filter(|literals| !literals.is_empty()) + .ok_or_else(|| invalid(format!("Visibility query literal list [{}] is not valid.", inner.trim())))?; + return Ok(Some(Comparison { field: name.to_owned(), operator: "IN", literal: Literal::List(literals), bare: false })); + } + } + if rest.is_empty() { + return Ok(Some(Comparison { field: name.to_owned(), operator: "=", literal: Literal::Bool(true), bare: true })); + } + Ok(None) +} + +fn invert(mut comparison: Comparison) -> Comparison { + if comparison.bare && comparison.operator == "=" { + comparison.literal = Literal::Bool(false); + return comparison; + } + comparison.operator = match comparison.operator { + "=" => "!=", + "!=" => "=", + ">" => "<=", + ">=" => "<", + "<" => ">=", + "<=" => ">", + "IN" => "NOT IN", + "NOT IN" => "IN", + other => other, + }; + comparison +} + +/// PHP's `parse`: `Ok(None)` for a plain search term. +pub(super) fn parse(query: &str) -> Result, String> { + let query = query.trim(); + if !looks_like_query(query) { + return Ok(None); + } + let predicates = || invalid("Visibility query predicates must use: Field = literal."); + let mut groups = Vec::new(); + for group in split_by_keyword(query, "OR").ok_or_else(predicates)? { + let mut comparisons = Vec::new(); + for part in split_by_keyword(&group, "AND").ok_or_else(predicates)? { + comparisons.push(parse_predicate(&part)?.ok_or_else(predicates)?); + } + groups.push(comparisons); + } + Ok(Some(Parsed(groups))) +} + +/// A system column of a run, as `SYSTEM_COLUMNS` maps it. +#[derive(Clone, Copy, Debug, PartialEq)] +enum Column { + WorkflowType, + WorkflowId, + RunId, + TaskQueue, + Compatibility, + StartedAt, + ClosedAt, +} + +impl Column { + fn named(field: &str) -> Option<(Self, &'static str)> { + Some(match field { + "WorkflowType" => (Self::WorkflowType, "keyword"), + "WorkflowId" => (Self::WorkflowId, "keyword"), + "RunId" => (Self::RunId, "keyword"), + "TaskQueue" => (Self::TaskQueue, "keyword"), + "BuildId" | "BuildIds" => (Self::Compatibility, "keyword"), + "StartTime" | "ExecutionTime" => (Self::StartedAt, "datetime"), + "CloseTime" => (Self::ClosedAt, "datetime"), + _ => return None, + }) + } +} + +/// A coerced comparison value. +#[derive(Clone, Debug, PartialEq)] +enum Coerced { + Text(String), + Int(i64), + Float(f64), + Bool(bool), + Time(DateTime), +} + +#[derive(Clone, Debug, PartialEq)] +enum Check { + /// `status` or `status_bucket` compared for equality. + Status { bucket: bool, value: String, equal: bool }, + Column { column: Column, operator: &'static str, values: Vec }, + Attribute { key: String, kind: &'static str, operator: &'static str, values: Vec }, +} + +/// A query checked against the namespace's definitions. +#[derive(Clone, Debug, PartialEq)] +pub(super) struct Compiled(Vec>); + +fn value_error(field: &str, expected: &str) -> String { + format!("[{field}] must be compared with {expected}.") +} + +fn equality(operator: &str, field: &str) -> Result<(), String> { + if matches!(operator, "=" | "!=") { + Ok(()) + } else { + Err(format!("[{field}] only supports equality comparisons.")) + } +} + +/// PHP's `normalizeSearchAttributeType`. +fn query_type(kind: &str) -> Result<&'static str, String> { + Ok(match kind { + "keyword" => "keyword", + "keyword_list" => "keyword_list", + "string" | "text" => "text", + "int" => "int", + "bool" => "bool", + "datetime" => "datetime", + "double" | "float" => "double", + other => return Err(format!("Search attribute type [{other}] is not supported in visibility queries.")), + }) +} + +fn coerce(literal: &Literal, kind: &str, field: &str) -> Result { + Ok(match kind { + "keyword" | "keyword_list" | "text" => Coerced::Text(match literal { + Literal::Bool(flag) => if *flag { "true" } else { "false" }.to_owned(), + Literal::Text(text) => text.clone(), + Literal::Int(number) => number.to_string(), + Literal::Float(number) => php_float_string(*number), + Literal::List(_) => return Err(value_error(field, "a string literal")), + }), + "int" => match literal { + Literal::Int(number) => Coerced::Int(*number), + _ => return Err(value_error(field, "an integer literal")), + }, + "double" => match literal { + Literal::Int(number) => Coerced::Float(*number as f64), + Literal::Float(number) => Coerced::Float(*number), + _ => return Err(value_error(field, "a numeric literal")), + }, + "bool" => match literal { + Literal::Bool(flag) => Coerced::Bool(*flag), + _ => return Err(value_error(field, "a boolean literal")), + }, + "datetime" => match literal { + Literal::Text(text) => Coerced::Time(parse_datetime(text).ok_or_else(|| value_error(field, "a datetime literal"))?), + _ => return Err(value_error(field, "a datetime literal")), + }, + other => return Err(format!("Search attribute type [{other}] is not supported in visibility queries.")), + }) +} + +/// PHP's `(string)` of a float: up to 14 significant digits. +fn php_float_string(number: f64) -> String { + if number.fract() == 0.0 && number.abs() < 1e15 { + return format!("{}", number as i64); + } + let text = format!("{:.*e}", 13, number); + let parsed: f64 = text.parse().unwrap_or(number); + format!("{parsed}") +} + +fn coerce_list(literal: &Literal, kind: &str, field: &str) -> Result, String> { + match literal { + Literal::List(items) if !items.is_empty() => items.iter().map(|item| coerce(item, kind, field)).collect(), + _ => Err(value_error(field, "a non-empty literal list")), + } +} + +/// PHP's `applyComparison` checks, in query order. `definitions` holds the +/// namespace's custom registered types. +pub(super) fn compile(parsed: &Parsed, definitions: &BTreeMap, namespace: &str) -> Result { + let mut groups = Vec::new(); + for group in &parsed.0 { + let mut checks = Vec::new(); + for comparison in group { + let field = comparison.field.as_str(); + let operator = comparison.operator; + let list = matches!(operator, "IN" | "NOT IN"); + if field == "Status" || field == "ExecutionStatus" { + let value = match &comparison.literal { + Literal::List(_) => return Err(value_error("Status", "a string literal")), + literal => match coerce(literal, "keyword", "Status")? { + Coerced::Text(text) => text.to_lowercase(), + _ => unreachable!("keyword coercion is text"), + }, + }; + equality(operator, "Status")?; + let bucket = matches!(value.as_str(), "running" | "completed" | "failed"); + checks.push(Check::Status { bucket, value, equal: operator == "=" }); + continue; + } + if let Some((column, kind)) = Column::named(field) { + let values = if list { + coerce_list(&comparison.literal, kind, field)? + } else { + let value = coerce(&comparison.literal, kind, field)?; + if kind == "keyword" { + equality(operator, field)?; + } + vec![value] + }; + checks.push(Check::Column { column, operator, values }); + continue; + } + let Some(definition) = definitions.get(field) else { + return Err(format!("Search attribute [{field}] is not defined in namespace [{namespace}].")); + }; + let kind = query_type(definition)?; + if comparison.bare && kind != "bool" { + return Err(value_error(field, "an explicit comparison")); + } + let values = if list { + let values = coerce_list(&comparison.literal, kind, field)?; + if kind == "keyword_list" && operator == "NOT IN" { + return Err("keyword_list search attributes do not support NOT IN comparisons.".to_owned()); + } + values + } else { + let value = coerce(&comparison.literal, kind, field)?; + match kind { + "keyword" => equality(operator, "keyword search attribute")?, + "text" => equality(operator, "text search attribute")?, + "keyword_list" if operator != "=" => { + return Err("keyword_list search attributes only support membership equality comparisons.".to_owned()); + } + _ => {} + } + vec![value] + }; + checks.push(Check::Attribute { key: field.to_owned(), kind, operator, values }); + } + groups.push(checks); + } + Ok(Compiled(groups)) +} + +impl Compiled { + /// The search attribute keys the query reads. + pub(super) fn keys(&self) -> Vec { + let mut keys: Vec = self + .0 + .iter() + .flatten() + .filter_map(|check| match check { + Check::Attribute { key, .. } => Some(key.clone()), + _ => None, + }) + .collect(); + keys.sort(); + keys.dedup(); + keys + } + + pub(super) fn matches(&self, run: &RunFields, attributes: &BTreeMap) -> bool { + self.0.iter().any(|group| group.iter().all(|check| check.matches(run, attributes))) + } +} + +/// The run columns PHP's summary query compares. +pub(super) struct RunFields { + pub(super) workflow_type: String, + pub(super) workflow_id: String, + pub(super) run_id: String, + pub(super) queue: String, + pub(super) compatibility: Option, + pub(super) started_at: Option>, + pub(super) closed_at: Option>, + pub(super) status: String, + pub(super) status_bucket: String, +} + +fn ordered(left: &T, operator: &str, right: &T) -> bool { + match operator { + "=" => left == right, + "!=" => left != right, + ">" => left > right, + ">=" => left >= right, + "<" => left < right, + "<=" => left <= right, + _ => false, + } +} + +/// SQL `IN` or `NOT IN` with a non-null value. +fn membership(found: bool, operator: &str) -> bool { + if operator == "NOT IN" { !found } else { found } +} + +impl Check { + fn matches(&self, run: &RunFields, attributes: &BTreeMap) -> bool { + match self { + Self::Status { bucket, value, equal } => { + let stored = if *bucket { &run.status_bucket } else { &run.status }; + (stored == value) == *equal + } + Self::Column { column, operator, values } => { + let text = |text: &str| -> bool { + let candidate = Coerced::Text(text.to_owned()); + match *operator { + "IN" | "NOT IN" => membership(values.contains(&candidate), operator), + _ => values.first().is_some_and(|value| compare(&candidate, operator, value)), + } + }; + let time = |time: Option>| -> bool { + // SQL comparisons with a NULL column never match. + let Some(time) = time else { return false }; + let candidate = Coerced::Time(time); + match *operator { + "IN" | "NOT IN" => membership(values.contains(&candidate), operator), + _ => values.first().is_some_and(|value| compare(&candidate, operator, value)), + } + }; + match column { + Column::WorkflowType => text(&run.workflow_type), + Column::WorkflowId => text(&run.workflow_id), + Column::RunId => text(&run.run_id), + Column::TaskQueue => text(&run.queue), + Column::Compatibility => run.compatibility.as_deref().is_some_and(text), + Column::StartedAt => time(run.started_at), + Column::ClosedAt => time(run.closed_at), + } + } + Self::Attribute { key, kind, operator, values } => { + // `whereHas`: the run needs a row for the key whose value matches. + let Some(row) = attributes.get(key) else { return false }; + let candidates: Vec = match (*kind, row) { + ("keyword", Typed::Keyword(text)) => vec![Coerced::Text(text.clone())], + ("text", Typed::String(text) | Typed::Keyword(text)) => vec![Coerced::Text(text.clone())], + ("int", Typed::Int(number)) => vec![Coerced::Int(*number)], + ("double", Typed::Float(number)) => vec![Coerced::Float(*number)], + ("double", Typed::Int(number)) => vec![Coerced::Float(*number as f64)], + ("bool", Typed::Bool(flag)) => vec![Coerced::Bool(*flag)], + ("datetime", Typed::Datetime(time)) => vec![Coerced::Time(*time)], + ("keyword_list", Typed::KeywordList(items)) => { + let items: Vec = items.iter().cloned().map(Coerced::Text).collect(); + // `whereJsonContains`: the list holds the value. + return match *operator { + "IN" => values.iter().any(|value| items.contains(value)), + _ => values.first().is_some_and(|value| items.contains(value)), + }; + } + _ => return false, + }; + candidates.iter().any(|candidate| match *operator { + "IN" | "NOT IN" => membership(values.contains(candidate), operator), + _ => values.first().is_some_and(|value| compare(candidate, operator, value)), + }) + } + } + } +} + +fn compare(left: &Coerced, operator: &str, right: &Coerced) -> bool { + match (left, right) { + (Coerced::Text(left), Coerced::Text(right)) => ordered(left, operator, right), + (Coerced::Int(left), Coerced::Int(right)) => ordered(left, operator, right), + (Coerced::Float(left), Coerced::Float(right)) => ordered(left, operator, right), + (Coerced::Bool(left), Coerced::Bool(right)) => ordered(left, operator, right), + (Coerced::Time(left), Coerced::Time(right)) => ordered(left, operator, right), + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn definitions() -> BTreeMap { + [("SearchKeyword", "keyword"), ("SearchCount", "int"), ("SearchRatio", "double"), ("SearchFlag", "bool"), + ("SearchTags", "keyword_list"), ("SearchTime", "datetime"), ("SearchText", "string")] + .into_iter() + .map(|(name, kind)| (name.to_owned(), kind.to_owned())) + .collect() + } + + fn compiled(query: &str) -> Result { + compile(&parse(query)?.expect("a visibility query"), &definitions(), "default") + } + + fn run() -> RunFields { + RunFields { + workflow_type: "sample".into(), + workflow_id: "wf-1".into(), + run_id: "run-1".into(), + queue: "q".into(), + compatibility: None, + started_at: parse_datetime("2026-10-08T12:00:00Z"), + closed_at: None, + status: "waiting".into(), + status_bucket: "running".into(), + } + } + + fn attributes() -> BTreeMap { + BTreeMap::from([ + ("SearchKeyword".into(), Typed::Keyword("界界".into())), + ("SearchCount".into(), Typed::Int(9_007_199_254_740_993)), + ("SearchRatio".into(), Typed::Float(1.25)), + ("SearchFlag".into(), Typed::Bool(true)), + ("SearchTags".into(), Typed::KeywordList(vec!["urgent".into(), "x".into()])), + ("SearchTime".into(), Typed::Datetime(parse_datetime("2026-10-08T12:34:56.123456Z").unwrap())), + ("SearchText".into(), Typed::String("two words".into())), + ]) + } + + #[test] + fn plain_terms_are_not_queries() { + assert_eq!(parse("order-123").unwrap(), None); + assert_eq!(parse(" ").unwrap(), None); + assert!(parse("NOT SearchFlag").unwrap().is_some()); + assert_eq!(parse("SearchFlag").unwrap(), None); + assert!(parse("SearchCount IN (1, 2)").unwrap().is_some()); + } + + #[test] + fn the_sample_app_queries_match_like_php() { + for query in [ + "SearchKeyword = \"界界\"", + "SearchCount > 9007199254740992 AND SearchCount <= 9007199254740993", + "SearchRatio > 1.0 AND SearchRatio < 2.0", + "SearchFlag = true", + // A bare field is a boolean comparison only inside a query; on + // its own PHP treats it as a free-text term. + "SearchFlag AND SearchCount > 1", + "SearchTags = \"urgent\"", + "SearchTime >= \"2026-10-08T12:34:56Z\" AND SearchTime < \"2026-10-08T12:34:57Z\"", + "SearchText = 'two words' OR SearchKeyword = missing", + "Status = Running AND WorkflowType = sample", + "SearchCount IN (1, 9007199254740993)", + "StartTime < \"2026-10-09\"", + ] { + let compiled = compiled(query).unwrap_or_else(|error| panic!("{query}: {error}")); + assert!(compiled.matches(&run(), &attributes()), "{query}"); + } + for query in [ + "SearchKeyword = \"missing\"", + "NOT SearchFlag", + "SearchTags = \"done\"", + "CloseTime < \"2030-01-01\"", + "NOT SearchRatio IN (1.25)", + "Status = completed", + ] { + let compiled = compiled(query).unwrap_or_else(|error| panic!("{query}: {error}")); + assert!(!compiled.matches(&run(), &attributes()), "{query}"); + } + // A run without the attribute never matches, not even `!=`. + assert!(!compiled("SearchKeyword != \"x\"").unwrap().matches(&run(), &BTreeMap::new())); + } + + #[test] + fn refusals_use_php_messages() { + for (query, message) in [ + ("SearchCount = ", "Visibility query predicates must use: Field = literal."), + ("SearchCount = 1 AND", "Visibility query predicates must use: Field = literal."), + ("SearchCount = $5", "Visibility query literal [$5] is not valid."), + ("SearchCount IN ()", "Visibility query literal list [] is not valid."), + ("Unknown = 1", "Search attribute [Unknown] is not defined in namespace [default]."), + ("SearchKeyword > a", "[keyword search attribute] only supports equality comparisons."), + ("SearchTags != a", "keyword_list search attributes only support membership equality comparisons."), + // PHP has no `Field NOT IN (...)` form; `NOT Field IN (...)` negates. + ("SearchTags NOT IN (a)", "Visibility query predicates must use: Field = literal."), + ("NOT SearchTags IN (a)", "keyword_list search attributes do not support NOT IN comparisons."), + ("SearchCount = 1.5", "[SearchCount] must be compared with an integer literal."), + ("SearchRatio = x", "[SearchRatio] must be compared with a numeric literal."), + ("SearchFlag = 1", "[SearchFlag] must be compared with a boolean literal."), + ("SearchTime > soon", "[SearchTime] must be compared with a datetime literal."), + ("SearchCount AND SearchFlag = true", "[SearchCount] must be compared with an explicit comparison."), + ("Status > running", "[Status] only supports equality comparisons."), + ("WorkflowType > a", "[WorkflowType] only supports equality comparisons."), + ] { + assert_eq!(compiled(query).unwrap_err(), message, "{query}"); + } + } +} diff --git a/rust/tests/execution/search_attribute_upserts.rs b/rust/tests/execution/search_attribute_upserts.rs index ee894e05f..1c3fd5967 100644 --- a/rust/tests/execution/search_attribute_upserts.rs +++ b/rust/tests/execution/search_attribute_upserts.rs @@ -156,3 +156,56 @@ async fn continued_runs_inherit_typed_attributes() { runtime.close().await; database.remove().await; } + +async fn listed(app: &Router, query: &str) -> (StatusCode, Value) { + let encoded: String = form_urlencoded::byte_serialize(query.as_bytes()).collect(); + request(app, "GET", &format!("/api/workflows?workflow_type=echo&query={encoded}"), Value::Null).await +} + +#[tokio::test] +async fn workflow_lists_filter_and_show_typed_attributes_like_php() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + for (name, kind) in [("SearchKeyword", "keyword"), ("SearchCount", "int"), ("SearchTags", "keyword_list"), + ("SearchFlag", "bool")] + { + define(&app, name, kind).await; + } + typed_worker(&app, "typed-worker").await; + for (workflow, keyword, count, flag) in [("listed-a", "alpha", 1, true), ("listed-b", "beta", 9_007_199_254_740_993_i64, false)] { + start(&app, workflow).await; + let task = poll(&app, "typed-worker", "workflow").await; + let done = finish_task(&app, &task, json!([upsert( + json!({"SearchKeyword":keyword,"SearchCount":count,"SearchTags":[keyword,"shared"],"SearchFlag":flag}), + json!({"SearchKeyword":"keyword","SearchCount":"int","SearchTags":"keyword_list","SearchFlag":"bool"})), + {"type":"open_signal_wait","signal_name":"finish"}])).await; + assert_eq!(done.0, StatusCode::OK, "{}", done.1); + } + let ids = |body: &Value| -> Vec { + body["workflows"].as_array().unwrap().iter().map(|row| row["workflow_id"].as_str().unwrap().to_owned()).collect() + }; + for (query, expected) in [ + ("SearchKeyword = \"alpha\"", vec!["listed-a"]), + ("SearchCount > 9007199254740992 AND SearchCount <= 9007199254740993", vec!["listed-b"]), + ("SearchTags = \"shared\"", vec!["listed-b", "listed-a"]), + ("SearchFlag = false OR SearchKeyword = alpha", vec!["listed-b", "listed-a"]), + ("NOT SearchFlag AND Status = running", vec!["listed-b"]), + ("SearchKeyword IN (\"beta\", gamma)", vec!["listed-b"]), + ("SearchKeyword = \"missing\"", vec![]), + ] { + let (status, body) = listed(&app, query).await; + assert_eq!(status, StatusCode::OK, "{query}: {body}"); + assert_eq!(ids(&body), expected, "{query}"); + } + let (_, body) = listed(&app, "SearchKeyword = alpha").await; + assert_eq!(body["workflows"][0]["search_attributes"], + json!({"SearchCount":1,"SearchFlag":true,"SearchKeyword":"alpha","SearchTags":["alpha","shared"]})); + // A plain term keeps the free-text workflow ID search. + assert_eq!(ids(&listed(&app, "listed-a").await.1), ["listed-a"]); + let refused = listed(&app, "Unknown = 1").await; + assert_eq!(refused.0, StatusCode::UNPROCESSABLE_ENTITY, "{}", refused.1); + assert_eq!(refused.1["errors"]["query"], json!(["Search attribute [Unknown] is not defined in namespace [default]."])); + runtime.close().await; + database.remove().await; +} From 5b31949651bbbc05a16f4042987ba70986d7ad42 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 23:14:26 +0000 Subject: [PATCH 24/30] Deliver cancellation to a parallel timer group, like PHP The Sample App's timers scenario cancels a workflow waiting on three concurrent timers. Every SDK delivers that cancellation with call_kind parallel and sequence_span 3, which native refused with cancellation_delivery_boundary_not_available. Native now accepts a parallel boundary whose members are all timers. As in PHP's CooperativeCancellationDelivery, each member must belong to the group at its index, a group resolved entirely before the request is not eligible, pending member timers are cancelled, and the delivery records its sequence_span. Groups with other member kinds keep the refusal. Co-Authored-By: Claude Opus 5.5 --- .../2026-10-21-parallel-cancellation.md | 34 ++++++ docs/rust-server-port.md | 6 + rust/src/runtime/cooperative.rs | 110 ++++++++++++++++-- rust/tests/execution.rs | 2 + rust/tests/execution/parallel_cancellation.rs | 61 ++++++++++ 5 files changed, 205 insertions(+), 8 deletions(-) create mode 100644 docs/rust-server-port-checkpoints/2026-10-21-parallel-cancellation.md create mode 100644 rust/tests/execution/parallel_cancellation.rs diff --git a/docs/rust-server-port-checkpoints/2026-10-21-parallel-cancellation.md b/docs/rust-server-port-checkpoints/2026-10-21-parallel-cancellation.md new file mode 100644 index 000000000..acb3fbc88 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-21-parallel-cancellation.md @@ -0,0 +1,34 @@ +# October 21, cancellation delivered to a parallel timer group + +- **Found by.** The Sample App's timers scenario (`scripts/sdk-timers.sh`) + against native, in its `concurrent-cancellation` case. Each SDK's workflow + waits on three concurrent timers and then receives a cooperative + cancellation request. All three SDKs called `deliver-cancellation` with + `call_kind: parallel` and `sequence_span: 3`. Native refused with + `422 cancellation_delivery_boundary_not_available`, the Rust worker + exited, and the run never recorded its delivery. +- **PHP's behavior.** `CooperativeCancellationDelivery` accepts a + `parallel` boundary whose span is at most a command batch. At the next + sequence it records the delivery directly. Otherwise `parallelEligibility` + requires every member to belong to the group at its index. A group whose + members all resolved before the request is not eligible. + `deliverPortableCancellation` then cancels each member's open wait and + records `CooperativeCancellationDelivered` with `sequence_span`. +- **Native now.** Native applies the same checks to a group whose members + are all timers: + - It cancels each pending member timer (`TimerCancelled`). + - It records the delivery with `sequence_span` and returns it in the + receipt. + - A repeated delivery returns the same receipt. + - A wrong span answers `409 cancellation_delivery_shape_mismatch`. + + A group with an activity, child or wait member keeps its earlier refusal. + The remaining boundaries are recorded as a known gap. +- **Verification.** + - A new test in `tests/execution/parallel_cancellation.rs` schedules three + parallel timers, requests cancellation and refuses a wrong span. It then + delivers to the group, repeats the delivery, and checks the history: + three `TimerCancelled`, one delivery with `sequence_span: 3`, then + `WorkflowCancelled`. + - The full suite passes on SQLite. The cooperative and cancellation tests + (53) pass on PostgreSQL and MySQL. diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 0a7394960..796260a90 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -198,6 +198,12 @@ contract fixture before its correction. a signal therefore waits on PHP and is refused on native. The port takes the snapshot at claim time and must keep query lease replay stable, so it needs its own fixture. +- **Cooperative cancellation boundaries.** Published PHP delivers a run's + cancellation at an activity, local activity, timer, condition, signal, + child, parallel or selection-handle boundary. Native delivers it at a + timer, a child and a parallel group whose members are all timers, and + refuses the other boundaries, including parallel groups with other member + kinds, with `422 cancellation_delivery_boundary_not_available`. - **Terminating a child run.** Published Workflow resolves the parent with `ChildRunTerminated` and the `terminated` child call status. Native refuses this with `422 termination_propagation_not_available` until the parent-side diff --git a/rust/src/runtime/cooperative.rs b/rust/src/runtime/cooperative.rs index 52158880e..03b41229a 100644 --- a/rust/src/runtime/cooperative.rs +++ b/rust/src/runtime/cooperative.rs @@ -103,6 +103,9 @@ async fn encode_command(value: Value, semaphore: Arc) -> Result Store where for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, @@ -190,11 +193,17 @@ where ) })?; let call_kind = super::text(&body, "call_kind")?; - if !matches!(call_kind, "timer" | "child") - || body - .get("sequence_span") - .filter(|v| !v.is_null()) - .is_some_and(|v| v.as_i64() != Some(1)) + // `validateBoundarySyntax`: only a parallel boundary spans more than + // one sequence, at most a command batch of them. + let span = match body.get("sequence_span").filter(|v| !v.is_null()) { + None => 1, + Some(value) => value + .as_i64() + .filter(|span| (1..=PARALLEL_SPAN_LIMIT).contains(span) && sequence.checked_add(*span).is_some()) + .ok_or_else(|| refuse(StatusCode::UNPROCESSABLE_ENTITY, "invalid_cancellation_delivery"))?, + }; + if !matches!(call_kind, "timer" | "child" | "parallel") + || (call_kind != "parallel" && span != 1) || body.get("operation_sequence").is_some_and(|v| !v.is_null()) || body .get("operation_sequence_span") @@ -245,7 +254,7 @@ where )); } let response = json!({"delivered":true,"task_id":task_id,"workflow_run_id":run_id,"request_id":request_id, - "sequence":sequence,"call_kind":call_kind,"sequence_span":1,"operation_sequence":null, + "sequence":sequence,"call_kind":call_kind,"sequence_span":span,"operation_sequence":null, "operation_sequence_span":1,"reason":null}); let requested=Self::query("SELECT sequence,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND workflow_command_id=$2 AND event_type='CooperativeCancellationRequested' ORDER BY sequence LIMIT 1") .bind(&run_id).bind(request_id).fetch_optional(&mut *tx).await? @@ -264,6 +273,7 @@ where .bind(&run_id).fetch_optional(&mut *tx).await? { let payload=DB::document_row(&delivery,"payload")?; if payload["workflow_command_id"]!=request_id || payload["sequence"]!=sequence || payload["call_kind"]!=call_kind + || payload["sequence_span"].as_i64().unwrap_or(1)!=span || payload["cancellation"]!=requested["cancellation"] || DB::optional_number(&run,"cancellation_delivery_sequence")?!=Some(sequence) { return Err(refuse(StatusCode::CONFLICT,"cancellation_delivery_mismatch")); } @@ -289,7 +299,17 @@ where "cancellation_delivery_sequence_mismatch", )); } - if call_kind == "child" { + if call_kind == "parallel" { + // `parallelEligibility`; a boundary at the next sequence has no + // authored member yet. + if sequence < next { + for timer in Self::parallel_timer_members(&mut tx, &run_id, sequence, span, request_sequence).await? { + if DB::string(&timer, "status")? == "pending" { + Self::cancel_root_timer(&mut tx, &run, request_id, &timer, Some(task_id), now()).await?; + } + } + } + } else if call_kind == "child" { if sequence < next { Self::validate_child_cancellation_boundary( &mut tx, @@ -331,11 +351,15 @@ where .bind(task_id) .execute(&mut *tx) .await?; - let payload = json!({"workflow_command_id":request_id,"workflow_run_id":run_id,"sequence":sequence,"call_kind":call_kind, + let mut payload = json!({"workflow_command_id":request_id,"workflow_run_id":run_id,"sequence":sequence,"call_kind":call_kind, "cancellation":requested["cancellation"],"task":{"id":task_id,"type":"workflow","status":"leased", "available_at":DB::optional_instant(&task,"available_at")?,"leased_at":DB::optional_instant(&task,"leased_at")?, "lease_owner":DB::optional_string(&task,"lease_owner")?,"lease_expires_at":expires, "attempt_count":DB::number(&task,"attempt_count")?}}); + // PHP records `sequence_span` only beyond one sequence. + if span != 1 { + payload["sequence_span"] = json!(span); + } Self::query("UPDATE workflow_runs SET cancellation_delivery_sequence=$1,cancellation_delivered_at=$2,last_progress_at=$3 WHERE id=$4") .bind(sequence).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(&run_id).execute(&mut *tx).await?; Self::append( @@ -351,6 +375,76 @@ where Ok(response) } + /// The pending timers of the parallel group a delivery names. Like + /// PHP's `parallelEligibility`, every member must belong to the group + /// at its index, and a group whose members all resolved before the + /// request is not eligible. Native cancels timer members only; a group + /// with another kind of member keeps its earlier refusal. + async fn parallel_timer_members( + tx: &mut Transaction<'_, DB>, + run_id: &str, + sequence: i64, + span: i64, + request_sequence: i64, + ) -> Result> { + let mismatch = || refuse(StatusCode::CONFLICT, "cancellation_delivery_shape_mismatch"); + let unavailable = || refuse(StatusCode::UNPROCESSABLE_ENTITY, "cancellation_delivery_boundary_not_available"); + let events = Self::query("SELECT sequence,event_type,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type IN ('TimerScheduled','TimerFired','TimerCancelled','ActivityScheduled','ChildWorkflowScheduled','ConditionWaitOpened','SignalWaitOpened') ORDER BY sequence LIMIT 10001") + .bind(run_id) + .fetch_all(&mut **tx) + .await?; + if events.len() > 10000 { + return Err(refuse(StatusCode::SERVICE_UNAVAILABLE, "cancellation_inventory_exceeds_budget")); + } + let mut members = Vec::new(); + let mut group_id = None; + let mut all_resolved = true; + for offset in 0..span { + let member = sequence + offset; + let mut scheduled = None; + let mut resolved_before_request = false; + for event in &events { + let payload = DB::document_row(event, "payload")?; + if payload["sequence"].as_i64() != Some(member) { + continue; + } + match DB::string(event, "event_type")?.as_str() { + "TimerScheduled" => scheduled = Some(payload), + "TimerFired" | "TimerCancelled" => { + resolved_before_request |= DB::number(event, "sequence")? < request_sequence; + } + _ => return Err(unavailable()), + } + } + let scheduled = scheduled.ok_or_else(mismatch)?; + let path = scheduled["parallel_group_path"].as_array().ok_or_else(mismatch)?; + let group = path + .iter() + .find(|group| { + group["parallel_group_base_sequence"].as_i64() == Some(sequence) + && group["parallel_group_size"].as_i64() == Some(span) + }) + .ok_or_else(mismatch)?; + if group["parallel_group_index"].as_i64() != Some(offset) + || group_id.get_or_insert_with(|| group["parallel_group_id"].clone()) != &group["parallel_group_id"] + { + return Err(mismatch()); + } + all_resolved &= resolved_before_request; + let timer = Self::query("SELECT * FROM workflow_run_timers WHERE workflow_run_id=$1 AND sequence=$2") + .bind(run_id) + .bind(member) + .fetch_optional(&mut **tx) + .await? + .ok_or_else(mismatch)?; + members.push(timer); + } + if all_resolved { + return Err(refuse(StatusCode::CONFLICT, "cancellation_delivery_not_eligible")); + } + Ok(members) + } + async fn validate_child_cancellation_boundary( tx: &mut Transaction<'_, DB>, run_id: &str, diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 89fa3a9cd..2b28c0df6 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -53,6 +53,8 @@ mod named_timers; #[path = "execution/named_signal_timeouts.rs"] mod named_signal_timeouts; +#[path = "execution/parallel_cancellation.rs"] +mod parallel_cancellation; #[path = "execution/selected_run_signals.rs"] mod selected_run_signals; diff --git a/rust/tests/execution/parallel_cancellation.rs b/rust/tests/execution/parallel_cancellation.rs new file mode 100644 index 000000000..9a8fbee5e --- /dev/null +++ b/rust/tests/execution/parallel_cancellation.rs @@ -0,0 +1,61 @@ +//! Cooperative cancellation delivered to a parallel group of timers, as +//! published PHP's `CooperativeCancellationDelivery` accepts a `parallel` +//! boundary and cancels each member's open timer. +use super::*; + +fn parallel_timer(index: i64) -> Value { + let group = json!({"parallel_group_id":"parallel-timers:1:3","parallel_group_kind":"timer", + "parallel_group_base_sequence":1,"parallel_group_size":3,"parallel_group_index":index}); + let mut command = group.clone(); + command["type"] = json!("start_timer"); + command["delay_seconds"] = json!(60); + command["parallel_group_path"] = json!([group]); + command +} + +async fn deliver_parallel(app: &Router, task: &Value, request_id: &Value, span: i64) -> (StatusCode, Value) { + request_protocol(app, "POST", + &format!("/api/worker/workflow-tasks/{}/deliver-cancellation", task["task_id"].as_str().unwrap()), + json!({"lease_owner":task["lease_owner"],"workflow_task_attempt":task["workflow_task_attempt"], + "request_id":request_id,"sequence":1,"call_kind":"parallel","sequence_span":span}), "1.20").await +} + +#[tokio::test] +async fn a_parallel_timer_group_is_cancelled_by_one_delivery() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let registered = request_protocol(&app, "POST", "/api/worker/register", + json!({"worker_id":"cleanup-worker","task_queue":"test","runtime":"python", + "supported_workflow_types":["echo"],"supported_activity_types":[],"capabilities":["cooperative_cancellation"]}), "1.20").await; + assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); + let started = start(&app, "parallel-cancel").await; + let task = cooperative_poll(&app, "workflow").await; + let scheduled = finish_task(&app, &task, json!([parallel_timer(0), parallel_timer(1), parallel_timer(2)])).await; + assert_eq!(scheduled.0, StatusCode::OK, "{}", scheduled.1); + let requested = request(&app, "POST", "/api/workflows/parallel-cancel/request-cancellation", + json!({"reason":"stop the group","cleanup_timeout_seconds":30})).await; + assert_eq!(requested.0, StatusCode::ACCEPTED, "{}", requested.1); + let request_id = requested.1["cancellation_request"]["request_id"].clone(); + let resumed = cooperative_poll(&app, "workflow").await; + assert_eq!(resumed["cancellation_request"]["request_id"], request_id, "{resumed}"); + // The span must name the group's actual size. + let wrong = deliver_parallel(&app, &resumed, &request_id, 2).await; + assert_eq!((wrong.0, wrong.1["reason"].clone()), (StatusCode::CONFLICT, json!("cancellation_delivery_shape_mismatch"))); + let delivered = deliver_parallel(&app, &resumed, &request_id, 3).await; + assert_eq!(delivered.0, StatusCode::OK, "{}", delivered.1); + assert_eq!((delivered.1["delivered"].clone(), delivered.1["sequence_span"].clone()), (json!(true), json!(3))); + // A repeated delivery returns the same receipt. + assert_eq!(deliver_parallel(&app, &resumed, &request_id, 3).await.1, delivered.1); + let finished = finish_task(&app, &resumed, json!([{"type":"complete_workflow","result":envelope(Payload::Null)}])).await; + assert_eq!(finished.0, StatusCode::OK, "{}", finished.1); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let types: Vec<_> = history.as_array().unwrap().iter().map(|event| event["event_type"].as_str().unwrap()).collect(); + assert_eq!(types, ["StartAccepted", "WorkflowStarted", "TimerScheduled", "TimerScheduled", "TimerScheduled", + "CooperativeCancellationRequested", "TimerCancelled", "TimerCancelled", "TimerCancelled", + "CooperativeCancellationDelivered", "WorkflowCancelled"]); + let delivery = history.as_array().unwrap().iter().find(|event| event["event_type"] == "CooperativeCancellationDelivered").unwrap(); + assert_eq!((delivery["payload"]["call_kind"].clone(), delivery["payload"]["sequence_span"].clone()), (json!("parallel"), json!(3))); + runtime.close().await; + database.remove().await; +} From 0dc4c6be59eb3e68958c31073e03a800a17538ce Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 22 Oct 2026 00:18:27 +0000 Subject: [PATCH 25/30] Authenticate database runtime credentials and the none driver like PHP Native refused DW_RUNTIME_CREDENTIALS_ENABLED and DW_AUTH_DRIVER=none at startup, which the Sample App's children and namespaces scenarios use. - Serve PHP's admin-only /api/runtime-credentials list, show, upsert, rotate and revoke, with the controller's validation and conflict refusals. Only the token's SHA-256 and 12-byte prefix are stored. - With the setting on, authenticate an unknown bearer token by its hash as a principal with its subject, roles, tenant and claims, and confine a tenant principal to its namespace, as ConfiguredAuthProvider::tenantAllows does, including a namespace route's target from the path or the name query. - Admit every request as the anonymous administrator with the none driver. - Record the principal's identity in task repair contexts. Co-Authored-By: Claude Opus 5.5 --- .../2026-10-22-runtime-credentials.md | 76 +++ docs/rust-server-port.md | 7 +- rust/src/main.rs | 40 +- rust/src/runtime/execution.rs | 24 + rust/src/runtime/http.rs | 125 ++++- rust/src/runtime/laravel_rules.rs | 5 +- rust/src/runtime/mod.rs | 1 + rust/src/runtime/runtime_credentials.rs | 432 ++++++++++++++++++ rust/src/runtime/task_repair_http.rs | 22 +- rust/tests/execution.rs | 2 + rust/tests/execution/runtime_credentials.rs | 112 +++++ 11 files changed, 809 insertions(+), 37 deletions(-) create mode 100644 docs/rust-server-port-checkpoints/2026-10-22-runtime-credentials.md create mode 100644 rust/src/runtime/runtime_credentials.rs create mode 100644 rust/tests/execution/runtime_credentials.rs diff --git a/docs/rust-server-port-checkpoints/2026-10-22-runtime-credentials.md b/docs/rust-server-port-checkpoints/2026-10-22-runtime-credentials.md new file mode 100644 index 000000000..98a8985ac --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-22-runtime-credentials.md @@ -0,0 +1,76 @@ +# October 22, runtime credentials + +- **Found by.** The Sample App's children and namespaces scenarios on the + second disposable VM run (Server #325 comment 6328544797). Both set + `DW_RUNTIME_CREDENTIALS_ENABLED=true` and give each worker its own + namespace-scoped token. Native refused that setting at startup. The + namespaces scenario's anonymous phase also restarts the server with + `DW_AUTH_DRIVER=none`, which native refused too. +- **Source.** Native follows published PHP 2.5.14: + - **Management API.** `RuntimeCredentialController`, admin-only: + - `GET /api/runtime-credentials` lists every credential, oldest first; + - `GET /api/runtime-credentials/{id}` shows one; + - `PUT /api/runtime-credentials/{id}` creates one (201), confirms an + identical one (200, `created: false`), or refuses with + `409 runtime_credential_conflict` or, for a token another credential + uses, `409 runtime_credential_token_conflict`; + - `POST /api/runtime-credentials/{id}/rotate` replaces the token and + clears a revocation; + - `DELETE /api/runtime-credentials/{id}` revokes it. + - **Validation.** The controller's rules and Laravel's messages: the + credential id format, a token of 32 to 512 characters, roles from + `operator` and `worker`, the tenant format, claims as an object, and a + valid `expires_at`. Roles are deduplicated and the tenant is lowercased. + - **Storage.** The `runtime_credentials` table holds the SHA-256 of the + token and its 12-byte prefix, never the token. Responses carry the + prefix and PHP's six-digit `+00:00` timestamps. + - **Authentication.** `ConfiguredAuthProvider`: with the setting on, a + bearer token that matches no configured token is looked up by its hash. + An unexpired, unrevoked credential becomes a principal with its subject, + roles, tenant and claims. + - **Authorization.** `RouteAuthorizationResource` and `tenantAllows`. A + principal with a tenant reaches only the requested namespace equal to + its tenant (the `X-Namespace` header, else the `namespace` query, else + `default`). On a namespace route, the target is the path segment or else + the `name` query field, and it must match the tenant too. Then one of + the principal's roles must be allowed on the route. A refusal is PHP's + 403 with the principal's first role. + - **Settings.** `DW_RUNTIME_CREDENTIALS_ENABLED`, or its + `WORKFLOW_SERVER_` name, is read with PHP's `FILTER_VALIDATE_BOOLEAN`, so + an unrecognized value turns it off. `DW_AUTH_DRIVER=none` admits every + request as the anonymous administrator. + - **Task repair.** Repair records the principal's subject, roles, method, + tenant and claims instead of a fixed legacy token identity. +- **Differences from PHP.** + - Native still requires `DW_AUTH_TOKEN` at startup, with the `none` + driver or with database credentials. PHP needs no legacy token then. + - Principal tokens (`DW_PRINCIPAL_TOKENS`), custom auth providers and the + signature driver still refuse startup. Runtime credentials grant only + `operator` and `worker`, so no tenant-bound administrator exists natively. + For that reason native does not read a namespace target from a JSON body + `name`, which PHP does; only creating a namespace sends one, and that + needs an administrator. + - Native has no service endpoints, so it does not read `caller_namespace`. + - Commands and history do not yet record the authenticated principal. + That remains the existing command-context gap. +- **Verification.** + - Three integration tests pass on SQLite, PostgreSQL and MySQL: managing + credentials, authenticating within a tenant through rotation and + revocation, and the `none` driver. Two unit tests cover the tenant rule + and token hashing. The full SQLite suite passes (331 integration, 189 + library). The auth, role, token and namespace tests pass on PostgreSQL + and MySQL (60 each). + - In the Waterline harness, 14 management requests went to published PHP + 2.5.14 and to native: create, confirm, both conflicts, four validation + failures, show, missing, rotate refusals and revoke. Apart from + timestamps and JSON key order, the outputs were identical. + - With `DW_RUNTIME_CREDENTIALS_ENABLED=true` on both servers, 24 + authentication requests matched in status: worker, operator and + cross-tenant credentials on worker, operator, admin and namespace routes, + expired, unknown, malformed and revoked tokens. The 401 and 403 bodies + were identical. Nine more requests checked the namespace route target + from the path and from `name`, with case and whitespace; all matched. + - The remaining body differences were outside this change: native answers + an empty worker registration with `invalid_request` (the registration + manifest gap), advertises fewer worker capabilities, and PHP's list held + rows from its shared database. diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 0a7394960..050522f8a 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -77,7 +77,6 @@ CLI and Waterline use the first group. - Durable workflow stream appends and closes. - Service catalogs and endpoints. - Nexus operations. - - Runtime credentials. - Deployments. - Bridge adapters. ## Known gaps found in review @@ -203,6 +202,12 @@ contract fixture before its correction. this with `422 termination_propagation_not_available` until the parent-side wait, child summary and failure projections for `ChildRunTerminated` have a fixture. +- **Authentication beyond role tokens and runtime credentials.** Native + refuses principal tokens (`DW_PRINCIPAL_TOKENS`), custom providers and the + signature driver at startup. It still requires `DW_AUTH_TOKEN` with the + `none` driver or database runtime credentials, which published PHP does + not. Without a tenant-bound administrator, native does not read a namespace + route's target from a JSON body `name`. ## Contract and verification inventory diff --git a/rust/src/main.rs b/rust/src/main.rs index 22003f39f..e3f4bfe0b 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -81,12 +81,33 @@ fn auth_value(key: &str) -> Result, Box> { } } +/// `server.auth.driver`: DW-primary, WORKFLOW-legacy, token by default. +fn anonymous_driver() -> Result> { + let driver = match auth_value("DW_AUTH_DRIVER")? { + Some(value) => Some(value), + None => auth_value("WORKFLOW_SERVER_AUTH_DRIVER")?, + }; + Ok(driver.as_deref() == Some("none")) +} + +/// `server.auth.runtime_credentials.enabled`: PHP's `FILTER_VALIDATE_BOOLEAN` +/// with `FILTER_NULL_ON_FAILURE`, so an unrecognized value disables it. +fn runtime_credentials_enabled() -> Result> { + let value = match auth_value("DW_RUNTIME_CREDENTIALS_ENABLED")? { + Some(value) => Some(value), + None => auth_value("WORKFLOW_SERVER_RUNTIME_CREDENTIALS_ENABLED")?, + }; + Ok(value.is_some_and(|value| { + matches!(value.trim().to_ascii_lowercase().as_str(), "1" | "true" | "on" | "yes") + })) +} + fn validate_auth_configuration() -> Result<(), Box> { // Unqualified credentials must not silently leave a full-access legacy // guard in place. These profiles require their own contract qualification. for key in ["DW_AUTH_DRIVER", "WORKFLOW_SERVER_AUTH_DRIVER"] { if let Some(value) = auth_value(key)? { - if value != "token" { + if value != "token" && value != "none" { return Err("unsupported development authentication configuration".into()); } } @@ -104,19 +125,6 @@ fn validate_auth_configuration() -> Result<(), Box> { return Err("unsupported development authentication configuration".into()); } } - for key in [ - "DW_RUNTIME_CREDENTIALS_ENABLED", - "WORKFLOW_SERVER_RUNTIME_CREDENTIALS_ENABLED", - ] { - if auth_value(key)?.is_some_and(|value| { - !matches!( - value.to_ascii_lowercase().as_str(), - "" | "0" | "false" | "off" | "no" - ) - }) { - return Err("unsupported development authentication configuration".into()); - } - } for key in [ "DW_AUTH_BACKWARD_COMPATIBLE", "WORKFLOW_SERVER_AUTH_BACKWARD_COMPATIBLE", @@ -444,7 +452,9 @@ async fn main() -> Result<(), Box> { return Err("unsupported development database".into()); } }; - let runtime = runtime.with_role_tokens(worker_token, operator_token, admin_token); + let runtime = runtime + .with_role_tokens(worker_token, operator_token, admin_token) + .with_auth_options(anonymous_driver()?, runtime_credentials_enabled()?); let listener = tokio::net::TcpListener::bind(address).await?; eprintln!( "Rust development runtime listening on {}", diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index 021692ed0..0557e2959 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -34,6 +34,8 @@ pub struct Runtime { storage: Arc, pub(crate) token: Arc, pub(crate) role_tokens: [Option>; 3], + /// `DW_AUTH_DRIVER=none` and `DW_RUNTIME_CREDENTIALS_ENABLED`. + pub(crate) auth: AuthOptions, scheduler: Arc, signal_codec: Arc, // Codec work done while holding the transition lock takes this permit, @@ -110,6 +112,15 @@ impl Drop for TimerScheduler { } } +/// How requests authenticate beyond the configured tokens. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct AuthOptions { + /// `DW_AUTH_DRIVER=none`: every request is the anonymous administrator. + pub(crate) anonymous: bool, + /// Database runtime credentials authenticate bearer tokens. + pub(crate) runtime_credentials: bool, +} + macro_rules! delegate { ($name:ident($($arg:ident: $ty:ty),* $(,)?) -> $output:ty) => { pub(crate) async fn $name(&self, $($arg:$ty),*) -> $output { @@ -144,6 +155,11 @@ impl Runtime { self } + pub fn with_auth_options(mut self, anonymous: bool, runtime_credentials: bool) -> Self { + self.auth = AuthOptions { anonymous, runtime_credentials }; + self + } + fn validate_token(token: &str) -> Result<()> { if token.trim().is_empty() { return Err(refuse( @@ -209,6 +225,7 @@ impl Runtime { storage: Arc::new(storage), token: token.into(), role_tokens: [None, None, None], + auth: AuthOptions::default(), scheduler: Arc::new(TimerScheduler::new()), signal_codec: Arc::new(Semaphore::new(1)), transition_codec: Arc::new(Semaphore::new(1)), @@ -278,6 +295,7 @@ impl Runtime { storage: Arc::new(storage), token: self.token.clone(), role_tokens: self.role_tokens.clone(), + auth: self.auth, scheduler: self.scheduler.clone(), signal_codec: self.signal_codec.clone(), transition_codec: self.transition_codec.clone(), @@ -305,6 +323,12 @@ impl Runtime { delegate!(activity_timeout_pass(limit: i64, ids: Vec) -> Result<(Value, bool)>); delegate!(create_namespace(body: Value) -> Result); delegate!(list_search_attributes() -> Result); + delegate!(runtime_credential_principal(token: &str) -> Result>); + delegate!(list_runtime_credentials() -> Result); + delegate!(show_runtime_credential(id: &str) -> Result); + delegate!(upsert_runtime_credential(id: &str, body: Value) -> Result<(axum::http::StatusCode, Value)>); + delegate!(rotate_runtime_credential(id: &str, body: Value) -> Result); + delegate!(revoke_runtime_credential(id: &str) -> Result); delegate!( storage_test_round_trip( local: &crate::external_payload::LocalStorage, diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index f307045bb..41f8554d3 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -12,13 +12,14 @@ use serde::Deserialize; use serde_json::{Value, json}; use subtle::ConstantTimeEq; +use super::runtime_credentials::Principal; use super::{Result, Runtime, cancellation::TerminalCommand, refuse, store::capabilities}; #[derive(Clone)] pub(super) struct AuthenticatedPrincipal { pub(super) role: &'static str, - pub(super) legacy_credential: bool, pub(super) legacy_full_access: bool, + pub(super) principal: super::runtime_credentials::Principal, } pub fn router(runtime: Runtime) -> Router { @@ -51,6 +52,17 @@ pub fn router(runtime: Runtime) -> Router { "/api/search-attributes", get(list_search_attributes).post(create_search_attribute), ) + .route("/api/runtime-credentials", get(list_runtime_credentials)) + .route( + "/api/runtime-credentials/{credential_id}", + get(show_runtime_credential) + .put(upsert_runtime_credential) + .delete(revoke_runtime_credential), + ) + .route( + "/api/runtime-credentials/{credential_id}/rotate", + post(rotate_runtime_credential), + ) .route( "/api/search-attributes/{name}", delete(delete_search_attribute), @@ -305,7 +317,7 @@ async fn guard(State(runtime): State, mut request: Request, next: Next) .and_then(|v| v.to_str().ok()); let provided = authorization.and_then(|v| v.strip_prefix("Bearer ")); let roles = ["worker", "operator", "admin"]; - let authenticated = provided.and_then(|provided| { + let configured = provided.and_then(|provided| { // Match configured roles before the legacy credential, matching PHP's // collision precedence. Equal-length secret contents use constant-time // comparison; secrets never enter response or diagnostic values. @@ -314,18 +326,42 @@ async fn guard(State(runtime): State, mut request: Request, next: Next) .as_ref() .is_some_and(|secret| bool::from(provided.as_bytes().ct_eq(secret.as_bytes()))) { - return Some((role, false, false)); + return Some((Principal::role(role, "token", false), false)); } } bool::from(provided.as_bytes().ct_eq(runtime.token.as_bytes())).then(|| { - ( + let mut principal = Principal::role( "admin", + "token", runtime.role_tokens.iter().all(Option::is_none), - true, - ) + ); + principal.subject = "legacy-token".into(); + principal.claims = json!({"legacy_credential": true}); + (principal, true) }) }); - let Some((role, legacy_full_access, legacy_credential)) = authenticated else { + // `ConfiguredAuthProvider::authenticate`: the none driver admits every + // request as the anonymous administrator; otherwise database runtime + // credentials follow the configured tokens. + let authenticated = if runtime.auth.anonymous { + let mut principal = Principal::role("admin", "none", true); + principal.subject = "anonymous".into(); + Some((principal, false)) + } else if configured.is_some() { + configured + } else if let (true, Some(provided)) = (runtime.auth.runtime_credentials, provided) { + match runtime.runtime_credential_principal(provided).await { + Ok(principal) => principal.map(|principal| (principal, false)), + Err(_) => { + return admission_error(request, worker, StatusCode::INTERNAL_SERVER_ERROR, + json!({"reason":"server_error", + "message":"Database runtime credential authentication is enabled but unavailable."})).await; + } + } + } else { + None + }; + let Some((principal, _legacy_credential)) = authenticated else { if payload { return admission_error( request, @@ -346,6 +382,12 @@ async fn guard(State(runtime): State, mut request: Request, next: Next) ) .await; }; + let legacy_full_access = principal.legacy_full_access; + let role = match principal.primary_role() { + Some("worker") => "worker", + Some("operator") => "operator", + _ => "admin", + }; let allowed_roles: &[&str] = if path == "/api/external-payloads/v1" && request.method() == Method::POST && request @@ -364,15 +406,39 @@ async fn guard(State(runtime): State, mut request: Request, next: Next) || path.starts_with("/api/namespaces/") && request.method() == Method::PUT // PHP's system routes are admin-only. || path.starts_with("/api/system/") + || path == "/api/runtime-credentials" + || path.starts_with("/api/runtime-credentials/") { &["admin"] } else { &["operator", "admin"] }; - if !legacy_full_access && !allowed_roles.contains(&role) { + // `RouteAuthorizationResource`: the requested namespace, and the target + // of a namespace route. + let requested_namespace = match request.headers().get("x-namespace") { + Some(header) => header.to_str().unwrap_or("").to_lowercase(), + None => Query::>::try_from_uri(request.uri()) + .ok() + .and_then(|Query(fields)| fields.get("namespace").cloned()) + .unwrap_or_else(|| "default".into()) + .to_lowercase(), + }; + let namespace_family = path == "/api/namespaces" || path.starts_with("/api/namespaces/"); + // A namespace route names its target in the path, or else in `name`. + // PHP also reads `name` from a JSON body; only creating a namespace sends + // one, and that needs an admin, which native never binds to a tenant. + let target_namespace = match path.strip_prefix("/api/namespaces/").and_then(|rest| rest.split('/').next()) { + Some(segment) if !segment.is_empty() => Some(segment.trim().to_lowercase()), + _ if namespace_family => Query::>::try_from_uri(request.uri()) + .ok() + .and_then(|Query(fields)| fields.get("name").map(|name| name.trim().to_lowercase())) + .filter(|name| !name.is_empty()), + _ => None, + }; + if !principal.allows(allowed_roles, &requested_namespace, target_namespace.as_deref(), namespace_family) { return admission_error(request, worker, StatusCode::FORBIDDEN, json!({ "reason":"forbidden", "message":"Authenticated role is not allowed to access this endpoint.", - "role":role, "allowed_roles":allowed_roles, + "role":principal.primary_role(), "allowed_roles":allowed_roles, })).await; } let header = if worker { @@ -476,8 +542,8 @@ async fn guard(State(runtime): State, mut request: Request, next: Next) } request.extensions_mut().insert(AuthenticatedPrincipal { role, - legacy_credential, legacy_full_access, + principal, }); let mut response = next.run(request).await; response @@ -487,6 +553,10 @@ async fn guard(State(runtime): State, mut request: Request, next: Next) } fn named_namespace_operation(method: &str, path: &str) -> bool { + // Runtime credentials are server-wide; the namespace only scopes a tenant. + if path == "/api/runtime-credentials" || path.starts_with("/api/runtime-credentials/") { + return true; + } if path == "/api/external-payloads/v1" { return method == "POST"; } @@ -641,6 +711,41 @@ async fn storage_test( Ok(Json(super::storage_test::storage_test(&runtime, body).await?)) } +async fn list_runtime_credentials(Extension(runtime): Extension) -> Result> { + Ok(Json(runtime.list_runtime_credentials().await?)) +} + +async fn show_runtime_credential( + Extension(runtime): Extension, + Path(credential_id): Path, +) -> Result> { + Ok(Json(runtime.show_runtime_credential(&credential_id).await?)) +} + +async fn upsert_runtime_credential( + Extension(runtime): Extension, + Path(credential_id): Path, + Json(body): Json, +) -> Result<(StatusCode, Json)> { + let (status, body) = runtime.upsert_runtime_credential(&credential_id, body).await?; + Ok((status, Json(body))) +} + +async fn rotate_runtime_credential( + Extension(runtime): Extension, + Path(credential_id): Path, + Json(body): Json, +) -> Result> { + Ok(Json(runtime.rotate_runtime_credential(&credential_id, body).await?)) +} + +async fn revoke_runtime_credential( + Extension(runtime): Extension, + Path(credential_id): Path, +) -> Result> { + Ok(Json(runtime.revoke_runtime_credential(&credential_id).await?)) +} + async fn list_search_attributes(Extension(runtime): Extension) -> Result> { Ok(Json(runtime.list_search_attributes().await?)) } diff --git a/rust/src/runtime/laravel_rules.rs b/rust/src/runtime/laravel_rules.rs index cd068fef3..66cc2a192 100644 --- a/rust/src/runtime/laravel_rules.rs +++ b/rust/src/runtime/laravel_rules.rs @@ -32,6 +32,8 @@ pub(crate) enum Rule { In(&'static [&'static str]), /// `date`, for the ISO-8601 shapes `schedule_backfill::php_date` reads. Date, + /// `regex:...`, as a predicate on the string. + Pattern(fn(&str) -> bool), } pub(crate) type Rules = [(&'static str, &'static [Rule])]; @@ -188,7 +190,7 @@ fn message(rule: Rule, attribute: &str, rules: &[Rule], value: Option<&Value>) - "array" => format!("The {attribute} field must not have more than {limit} items."), _ => format!("The {attribute} field must not be greater than {limit} characters."), }, - Rule::NotBlank => format!("The {attribute} field format is invalid."), + Rule::NotBlank | Rule::Pattern(_) => format!("The {attribute} field format is invalid."), Rule::Date => format!("The {attribute} field must be a valid date."), Rule::Nullable | Rule::Sometimes => String::new(), } @@ -223,6 +225,7 @@ fn passes(rule: Rule, data: &Value, value: Option<&Value>, rules: &[Rule]) -> bo Rule::Nullable | Rule::Sometimes => true, Rule::NotBlank => value.and_then(Value::as_str).is_some_and(|text| text.chars().any(|character| !character.is_whitespace())), Rule::String => value.is_some_and(Value::is_string), + Rule::Pattern(matches) => value.and_then(Value::as_str).is_some_and(matches), Rule::Date => value .and_then(Value::as_str) .is_some_and(|text| super::schedule_backfill::php_date(text).is_some()), diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index d0a7938e8..0a31d6e82 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -46,6 +46,7 @@ mod previews; mod queries; mod query_authority; mod retention; +mod runtime_credentials; mod run_observation; mod run_observation_store; mod run_observation_waits; diff --git a/rust/src/runtime/runtime_credentials.rs b/rust/src/runtime/runtime_credentials.rs new file mode 100644 index 000000000..1257fed65 --- /dev/null +++ b/rust/src/runtime/runtime_credentials.rs @@ -0,0 +1,432 @@ +//! Database runtime credentials, as published PHP Server stores, manages +//! and authenticates them (`RuntimeCredential`, `RuntimeCredentialController` +//! and `ConfiguredAuthProvider`). A credential is a hashed bearer token for a +//! principal with worker or operator roles, scoped to one namespace (its +//! tenant). +use axum::http::StatusCode; +use chrono::{DateTime, Utc}; +use serde_json::{Map, Value, json}; +use sha2::{Digest, Sha256}; +use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; + +use super::{ + Result, RuntimeError, + backend::Backend, + laravel_rules::{Rule, Rules, failure, normalize, validate}, + store::{Store, now}, +}; + +/// An authenticated caller: PHP's `App\Auth\Principal`. +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct Principal { + pub(crate) subject: String, + pub(crate) roles: Vec, + pub(crate) method: &'static str, + pub(crate) tenant: Option, + pub(crate) claims: Value, + pub(crate) legacy_full_access: bool, +} + +impl Principal { + /// `Principal::role`. + pub(crate) fn role(role: &str, method: &'static str, legacy_full_access: bool) -> Self { + Self { + subject: format!("role:{role}"), + roles: vec![role.to_owned()], + method, + tenant: None, + claims: json!({}), + legacy_full_access, + } + } + + pub(crate) fn primary_role(&self) -> Option<&str> { + self.roles.first().map(String::as_str) + } + + /// `ConfiguredAuthProvider::authorize`: a role the route allows, within + /// the principal's tenant. + pub(crate) fn allows(&self, allowed: &[&str], requested_namespace: &str, target_namespace: Option<&str>, namespace_family: bool) -> bool { + self.legacy_full_access + || (self.tenant_allows(requested_namespace, target_namespace, namespace_family) + && self.roles.iter().any(|role| allowed.contains(&role.as_str()))) + } + + /// `ConfiguredAuthProvider::tenantAllows`. + fn tenant_allows(&self, requested: &str, target: Option<&str>, namespace_family: bool) -> bool { + let Some(tenant) = self.tenant.as_deref().map(str::trim).filter(|tenant| !tenant.is_empty()) else { + return true; + }; + let tenant = tenant.to_lowercase(); + let requested = requested.trim().to_lowercase(); + if requested.is_empty() || requested != tenant { + return false; + } + let target = target.map(|target| target.trim().to_lowercase()).filter(|target| !target.is_empty()); + if target.as_ref().is_some_and(|target| *target != tenant) { + return false; + } + !namespace_family || target.is_some() + } +} + +/// `RuntimeCredential::hashToken`. +fn hash_token(token: &str) -> String { + Sha256::digest(token.as_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +/// `RuntimeCredential::prefixFor`: the first 12 bytes. +fn prefix(token: &str) -> String { + let mut end = token.len().min(12); + while !token.is_char_boundary(end) { + end -= 1; + } + token[..end].to_owned() +} + +fn tenant_format(text: &str) -> bool { + !text.is_empty() && text.bytes().all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte)) +} + +const CREDENTIAL_RULES: &Rules = &[ + ("token", &[Rule::Required, Rule::String, Rule::Min(32), Rule::Max(512)]), + ("name", &[Rule::Nullable, Rule::String, Rule::Max(255)]), + ("subject", &[Rule::Required, Rule::String, Rule::Max(255)]), + ("roles", &[Rule::Required, Rule::Array, Rule::Min(1)]), + ("roles.*", &[Rule::Required, Rule::String, Rule::In(&["operator", "worker"])]), + ("tenant", &[Rule::Required, Rule::String, Rule::Max(128), Rule::Pattern(tenant_format)]), + ("claims", &[Rule::Sometimes, Rule::Array]), + ("expires_at", &[Rule::Sometimes, Rule::Nullable, Rule::Date]), +]; + +const ROTATE_RULES: &Rules = &[ + ("token", &[Rule::Required, Rule::String, Rule::Min(32), Rule::Max(512)]), + ("expires_at", &[Rule::Sometimes, Rule::Nullable, Rule::Date]), +]; + +fn invalid(field: &str, message: &str) -> RuntimeError { + failure(vec![(field.to_owned(), vec![message.to_owned()])]).expect("one message") +} + +/// `validateCredentialId`. +fn credential_id(id: &str) -> Result<()> { + let mut bytes = id.bytes(); + let valid = id.len() <= 64 + && bytes.next().is_some_and(|first| first.is_ascii_alphanumeric()) + && bytes.all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte)); + if valid { Ok(()) } else { Err(invalid("credential_id", "The credential id format is invalid.")) } +} + +/// `CarbonImmutable::parse(...)->utc()` for a validated date. +fn date(value: &Value) -> Option> { + value.as_str().and_then(super::schedule_backfill::php_date) +} + +/// Carbon's `toIso8601String` in UTC. +fn iso(time: Option>) -> Value { + time.map_or(Value::Null, |time| json!(time.format("%Y-%m-%dT%H:%M:%S+00:00").to_string())) +} + +fn not_found(id: &str) -> RuntimeError { + RuntimeError::Protocol { + status: StatusCode::NOT_FOUND, + response: json!({"message":"Runtime credential not found.","reason":"runtime_credential_not_found","credential_id":id}), + } +} + +fn token_conflict(error: sqlx::Error) -> RuntimeError { + match error { + sqlx::Error::Database(error) if error.is_unique_violation() => RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"message":"The supplied runtime credential token is already assigned.", + "reason":"runtime_credential_token_conflict"}), + }, + error => error.into(), + } +} + +/// The attributes an upsert stores, as PHP's `attributes` derives them. +struct Attributes { + name: Option, + subject: String, + roles: Vec, + tenant: String, + claims: Value, + token_prefix: String, + token_hash: String, + expires_at: Option>, +} + +fn attributes(body: &Value) -> Result { + let claims = match &body["claims"] { + Value::Array(items) if !items.is_empty() => { + return Err(invalid("claims", "The claims field must be a JSON object, not a list.")); + } + Value::Object(map) => Value::Object(map.clone()), + _ => json!({}), + }; + let mut roles: Vec = body["roles"] + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect(); + roles.sort(); + roles.dedup(); + let token = body["token"].as_str().unwrap_or_default(); + Ok(Attributes { + name: body["name"].as_str().map(str::to_owned), + subject: body["subject"].as_str().unwrap_or_default().trim().to_owned(), + roles, + tenant: body["tenant"].as_str().unwrap_or_default().to_lowercase(), + claims, + token_prefix: prefix(token), + token_hash: hash_token(token), + expires_at: date(&body["expires_at"]), + }) +} + +impl Store +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> Option<&'q str>: Encode<'q, DB>, + for<'q> Option: Encode<'q, DB>, + for<'q> String: Encode<'q, DB> + Type, + for<'q> i64: Encode<'q, DB> + Type, + for<'r> i64: Decode<'r, DB>, + usize: ColumnIndex, +{ + fn serialize_credential(row: &DB::Row) -> Result { + let claims = DB::optional_document_row(row, "claims")? + .filter(|claims| claims.as_object().is_some_and(|map| !map.is_empty())) + .unwrap_or_else(|| json!([])); + Ok(json!({"id":DB::string(row,"id")?,"name":DB::optional_string(row,"name")?,"subject":DB::string(row,"subject")?, + "roles":DB::document_row(row,"roles")?,"tenant":DB::string(row,"tenant")?,"claims":claims, + "token_prefix":DB::string(row,"token_prefix")?, + "expires_at":iso(DB::optional_instant(row,"expires_at")?),"revoked_at":iso(DB::optional_instant(row,"revoked_at")?), + "rotated_at":iso(DB::optional_instant(row,"rotated_at")?),"created_at":iso(DB::optional_instant(row,"created_at")?), + "updated_at":iso(DB::optional_instant(row,"updated_at")?)})) + } + + /// `RuntimeCredential::activeForToken` and its `principal`. + pub(crate) async fn runtime_credential_principal(&self, token: &str) -> Result> { + let Some(row) = Self::query("SELECT * FROM runtime_credentials WHERE token_hash=$1") + .bind(hash_token(token)) + .fetch_optional(&self.pool) + .await? + else { + return Ok(None); + }; + if DB::optional_instant(&row, "revoked_at")?.is_some() + || DB::optional_instant(&row, "expires_at")?.is_some_and(|expires| expires <= now()) + { + return Ok(None); + } + let roles = DB::document_row(&row, "roles")? + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(|role| role.trim().to_owned()) + .filter(|role| !role.is_empty()) + .fold(Vec::new(), |mut roles, role| { + if !roles.contains(&role) { + roles.push(role); + } + roles + }); + Ok(Some(Principal { + subject: DB::string(&row, "subject")?, + roles, + method: "runtime-token", + tenant: Some(DB::string(&row, "tenant")?), + claims: DB::optional_document_row(&row, "claims")?.unwrap_or_else(|| json!({})), + legacy_full_access: false, + })) + } + + pub(crate) async fn list_runtime_credentials(&self) -> Result { + let rows = Self::query("SELECT * FROM runtime_credentials ORDER BY created_at,id") + .fetch_all(&self.pool) + .await?; + let credentials = rows.iter().map(Self::serialize_credential).collect::>>()?; + Ok(json!({"credentials":credentials})) + } + + pub(crate) async fn show_runtime_credential(&self, id: &str) -> Result { + Self::query("SELECT * FROM runtime_credentials WHERE id=$1") + .bind(id) + .fetch_optional(&self.pool) + .await? + .map(|row| Self::serialize_credential(&row)) + .transpose()? + .ok_or_else(|| not_found(id)) + } + + /// `RuntimeCredentialController::upsert`: create, or confirm an + /// identical credential; different attributes conflict. + pub(crate) async fn upsert_runtime_credential(&self, id: &str, body: Value) -> Result<(StatusCode, Value)> { + credential_id(id)?; + let body = normalize(&body); + validate(&body, CREDENTIAL_RULES)?; + let attributes = attributes(&body)?; + let mut tx = self.begin().await?; + if let Some(row) = Self::query("SELECT * FROM runtime_credentials WHERE id=$1") + .bind(id) + .fetch_optional(&mut *tx) + .await? + { + let same_claims = DB::optional_document_row(&row, "claims")?.unwrap_or_else(|| json!({})); + let same_claims = if same_claims.as_object().is_some_and(Map::is_empty) || same_claims == json!([]) { + json!({}) + } else { + same_claims + }; + let matches = DB::optional_string(&row, "name")? == attributes.name + && DB::string(&row, "subject")? == attributes.subject + && DB::document_row(&row, "roles")? == json!(attributes.roles) + && DB::string(&row, "tenant")? == attributes.tenant + && same_claims == attributes.claims + && DB::string(&row, "token_hash")? == attributes.token_hash + && DB::optional_instant(&row, "expires_at")? == attributes.expires_at + && DB::optional_instant(&row, "revoked_at")?.is_none(); + tx.commit().await?; + if !matches { + return Err(RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"message":"Runtime credential already exists with different attributes.", + "reason":"runtime_credential_conflict","credential_id":id}), + }); + } + let mut credential = Self::serialize_credential(&row)?; + credential["created"] = json!(false); + return Ok((StatusCode::OK, credential)); + } + let at = now(); + Self::query("INSERT INTO runtime_credentials(id,name,subject,roles,tenant,claims,token_prefix,token_hash,expires_at,revoked_at,rotated_at,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,NULL,NULL,$10,$11)") + .bind(id) + .bind(attributes.name.as_deref()) + .bind(attributes.subject.as_str()) + .bind(Some(DB::document(&json!(attributes.roles)))) + .bind(attributes.tenant.as_str()) + .bind(Some(DB::document(&attributes.claims))) + .bind(attributes.token_prefix.as_str()) + .bind(attributes.token_hash.as_str()) + .bind(DB::optional_time(attributes.expires_at)) + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .execute(&mut *tx) + .await + .map_err(token_conflict)?; + let row = Self::query("SELECT * FROM runtime_credentials WHERE id=$1").bind(id).fetch_one(&mut *tx).await?; + tx.commit().await?; + let mut credential = Self::serialize_credential(&row)?; + credential["created"] = json!(true); + Ok((StatusCode::CREATED, credential)) + } + + /// `RuntimeCredentialController::rotate`: a new token, which also + /// restores a revoked credential. + pub(crate) async fn rotate_runtime_credential(&self, id: &str, body: Value) -> Result { + credential_id(id)?; + let body = normalize(&body); + validate(&body, ROTATE_RULES)?; + let token = body["token"].as_str().unwrap_or_default(); + let mut tx = self.begin().await?; + let row = Self::query("SELECT * FROM runtime_credentials WHERE id=$1") + .bind(id) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| not_found(id))?; + let hash = hash_token(token); + let expires_at = if body.get("expires_at").is_some() { + date(&body["expires_at"]) + } else { + DB::optional_instant(&row, "expires_at")? + }; + let changed = DB::string(&row, "token_hash")? != hash + || DB::optional_instant(&row, "revoked_at")?.is_some() + || DB::optional_instant(&row, "expires_at")? != expires_at; + if changed { + let at = now(); + Self::query("UPDATE runtime_credentials SET token_hash=$1,token_prefix=$2,expires_at=$3,revoked_at=NULL,rotated_at=$4,updated_at=$5 WHERE id=$6") + .bind(hash.as_str()) + .bind(prefix(token)) + .bind(DB::optional_time(expires_at)) + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .bind(id) + .execute(&mut *tx) + .await + .map_err(token_conflict)?; + } + let row = Self::query("SELECT * FROM runtime_credentials WHERE id=$1").bind(id).fetch_one(&mut *tx).await?; + tx.commit().await?; + Self::serialize_credential(&row) + } + + /// `RuntimeCredentialController::revoke`. + pub(crate) async fn revoke_runtime_credential(&self, id: &str) -> Result { + credential_id(id)?; + let mut tx = self.begin().await?; + let at = now(); + Self::query("UPDATE runtime_credentials SET revoked_at=$1,updated_at=$2 WHERE id=$3 AND revoked_at IS NULL") + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .bind(id) + .execute(&mut *tx) + .await?; + let row = Self::query("SELECT * FROM runtime_credentials WHERE id=$1") + .bind(id) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| not_found(id))?; + tx.commit().await?; + Self::serialize_credential(&row) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tenant_principal(tenant: &str, roles: &[&str]) -> Principal { + Principal { + subject: "s".into(), + roles: roles.iter().map(|role| (*role).to_owned()).collect(), + method: "runtime-token", + tenant: Some(tenant.into()), + claims: json!({}), + legacy_full_access: false, + } + } + + #[test] + fn tenants_reach_only_their_namespace() { + let worker = tenant_principal("Team-A", &["worker"]); + assert!(worker.allows(&["worker"], "team-a", None, false)); + assert!(!worker.allows(&["worker"], "default", None, false)); + assert!(!worker.allows(&["operator", "admin"], "team-a", None, false)); + // Namespace routes need a target, and it must be the tenant. + assert!(!worker.allows(&["worker"], "team-a", None, true)); + assert!(worker.allows(&["worker"], "team-a", Some("TEAM-A"), true)); + assert!(!worker.allows(&["worker"], "team-a", Some("team-b"), true)); + let legacy = Principal::role("admin", "token", true); + assert!(legacy.allows(&["worker"], "anything", None, true)); + } + + #[test] + fn tokens_hash_and_prefix_like_php() { + assert_eq!(hash_token("abc"), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + assert_eq!(prefix("dwr_fixture_worker_0123456789"), "dwr_fixture_"); + assert!(credential_id("cred-1.a_b").is_ok()); + assert!(credential_id("-bad").is_err()); + assert!(credential_id(&"a".repeat(65)).is_err()); + } +} diff --git a/rust/src/runtime/task_repair_http.rs b/rust/src/runtime/task_repair_http.rs index ba5af0c37..53b5b98fd 100644 --- a/rust/src/runtime/task_repair_http.rs +++ b/rust/src/runtime/task_repair_http.rs @@ -94,13 +94,10 @@ pub(super) async fn request(request: Request, workflow_id: &str) -> Result<(Valu if !headers.is_array() { request["headers"] = headers; } - let subject = if principal.legacy_credential { - "legacy-token".into() - } else { - format!("role:{}", principal.role) - }; - let claims = if principal.legacy_credential { - json!({"legacy_credential":true}) + // The authenticated principal, as `Principal::toAuditContext` records it. + let subject = principal.principal.subject.clone(); + let claims = if principal.principal.claims.as_object().is_some_and(|claims| !claims.is_empty()) { + principal.principal.claims.clone() } else { json!([]) }; @@ -109,14 +106,19 @@ pub(super) async fn request(request: Request, workflow_id: &str) -> Result<(Valu "operator" => "Operator", _ => "Worker", }; + let mut audit = json!({"subject":subject,"roles":principal.principal.roles,"claims":claims, + "legacy_full_access":principal.legacy_full_access}); + if let Some(tenant) = &principal.principal.tenant { + audit["tenant"] = json!(tenant); + } let metadata = payload["request_id"] .as_str() .map_or(json!([]), |value| json!({"request_id":value})); let context = json!({ "caller":{"type":"server","label":"Standalone Server"}, - "auth":{"status":"authorized","method":"token","role":principal.role,"roles":[principal.role], - "principal":{"subject":subject,"roles":[principal.role],"claims":claims,"legacy_full_access":principal.legacy_full_access}}, - "principal":{"type":"auth:token","id":subject,"label":label}, + "auth":{"status":"authorized","method":principal.principal.method,"role":principal.role,"roles":principal.principal.roles, + "principal":audit}, + "principal":{"type":format!("auth:{}",principal.principal.method),"id":subject,"label":label}, "request":request,"server":{"operation_id":id(),"namespace":"default","workflow_id":workflow_id, "command":"repair","metadata":metadata}}); Ok((payload, context)) diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 89fa3a9cd..927c13209 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -53,6 +53,8 @@ mod named_timers; #[path = "execution/named_signal_timeouts.rs"] mod named_signal_timeouts; +#[path = "execution/runtime_credentials.rs"] +mod runtime_credentials; #[path = "execution/selected_run_signals.rs"] mod selected_run_signals; diff --git a/rust/tests/execution/runtime_credentials.rs b/rust/tests/execution/runtime_credentials.rs new file mode 100644 index 000000000..88b5b627e --- /dev/null +++ b/rust/tests/execution/runtime_credentials.rs @@ -0,0 +1,112 @@ +//! Database runtime credentials, as published PHP Server manages them +//! (`RuntimeCredentialController`) and authenticates their tokens for one +//! tenant namespace (`ConfiguredAuthProvider`). +use super::*; +use admission::admission_http; + +const WORKER_TOKEN: &str = "dwr_fixture_worker_default_0123456789"; +const OPERATOR_TOKEN: &str = "dwr_fixture_operator_team_a_0123456789"; + +async fn admin(app: &Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) { + let (status, _, body) = admission_http(app, method, path, false, Some("test-token"), Some("2"), None, body).await; + (status, body) +} + +async fn as_token(app: &Router, token: &str, method: &str, path: &str, worker: bool, namespace: Option<&str>) -> StatusCode { + let version = if worker { "1.20" } else { "2" }; + admission_http(app, method, path, worker, Some(token), Some(version), namespace, json!({})).await.0 +} + +fn credential(token: &str, roles: Value, tenant: &str) -> Value { + json!({"token":token,"subject":"svc:sample","roles":roles,"tenant":tenant,"claims":{"team":"core"}}) +} + +#[tokio::test] +async fn runtime_credentials_are_managed_like_php() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone().with_auth_options(false, true)); + let (status, created) = admin(&app, "PUT", "/api/runtime-credentials/worker-a", + credential(WORKER_TOKEN, json!(["worker", "worker"]), "Default")).await; + assert_eq!(status, StatusCode::CREATED, "{created}"); + assert_eq!((created["created"].clone(), created["roles"].clone(), created["tenant"].clone(), created["token_prefix"].clone()), + (json!(true), json!(["worker"]), json!("default"), json!("dwr_fixture_"))); + assert!(created.get("token_hash").is_none() && created.get("token").is_none(), "{created}"); + // The same attributes confirm the credential; different ones conflict. + let (status, again) = admin(&app, "PUT", "/api/runtime-credentials/worker-a", + credential(WORKER_TOKEN, json!(["worker"]), "default")).await; + assert_eq!((status, again["created"].clone()), (StatusCode::OK, json!(false)), "{again}"); + let (status, conflict) = admin(&app, "PUT", "/api/runtime-credentials/worker-a", + credential(WORKER_TOKEN, json!(["operator"]), "default")).await; + assert_eq!((status, conflict["reason"].clone()), (StatusCode::CONFLICT, json!("runtime_credential_conflict"))); + let (status, reused) = admin(&app, "PUT", "/api/runtime-credentials/worker-b", + credential(WORKER_TOKEN, json!(["worker"]), "default")).await; + assert_eq!((status, reused["reason"].clone()), (StatusCode::CONFLICT, json!("runtime_credential_token_conflict"))); + // Validation follows the controller's rules and messages. + let (status, invalid) = admin(&app, "PUT", "/api/runtime-credentials/-bad", credential(WORKER_TOKEN, json!(["worker"]), "default")).await; + assert_eq!((status, invalid["errors"]["credential_id"].clone()), (StatusCode::UNPROCESSABLE_ENTITY, json!(["The credential id format is invalid."]))); + let (status, invalid) = admin(&app, "PUT", "/api/runtime-credentials/short", + json!({"token":"short","subject":"s","roles":["admin"],"tenant":"bad tenant","claims":["x"]})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{invalid}"); + assert_eq!(invalid["errors"]["token"], json!(["The token field must be at least 32 characters."])); + assert_eq!(invalid["errors"]["tenant"], json!(["The tenant field format is invalid."])); + assert_eq!(invalid["errors"]["roles.0"], json!(["The selected roles.0 is invalid."])); + let (status, listed) = admin(&app, "GET", "/api/runtime-credentials", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{listed}"); + assert_eq!(listed["credentials"].as_array().unwrap().len(), 1); + assert_eq!(listed["credentials"][0]["claims"], json!({"team":"core"})); + let (status, missing) = admin(&app, "GET", "/api/runtime-credentials/nobody", Value::Null).await; + assert_eq!((status, missing["reason"].clone()), (StatusCode::NOT_FOUND, json!("runtime_credential_not_found"))); + // Only an administrator manages credentials. + assert_eq!(as_token(&app, WORKER_TOKEN, "GET", "/api/runtime-credentials", false, None).await, StatusCode::FORBIDDEN); + runtime.close().await; + database.remove().await; +} + +#[tokio::test] +async fn runtime_credentials_authenticate_within_their_tenant() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone().with_auth_options(false, true)); + assert_eq!(admin(&app, "PUT", "/api/runtime-credentials/worker-a", credential(WORKER_TOKEN, json!(["worker"]), "default")).await.0, StatusCode::CREATED); + assert_eq!(admin(&app, "PUT", "/api/runtime-credentials/operator-a", credential(OPERATOR_TOKEN, json!(["operator"]), "team-a")).await.0, StatusCode::CREATED); + // A worker credential reaches worker routes of its own namespace only. + assert_eq!(as_token(&app, WORKER_TOKEN, "GET", "/api/cluster/info", false, None).await, StatusCode::OK); + assert_eq!(as_token(&app, WORKER_TOKEN, "GET", "/api/workflows", false, None).await, StatusCode::FORBIDDEN); + // An operator credential for another tenant cannot read the default namespace. + assert_eq!(as_token(&app, OPERATOR_TOKEN, "GET", "/api/workflows", false, None).await, StatusCode::FORBIDDEN); + assert_eq!(as_token(&app, OPERATOR_TOKEN, "GET", "/api/namespaces", false, Some("team-a")).await, StatusCode::FORBIDDEN); + // A namespace route without a path target takes it from `name`. + assert_eq!(as_token(&app, OPERATOR_TOKEN, "GET", "/api/namespaces?name=Team-A", false, Some("team-a")).await, StatusCode::OK); + assert_eq!(as_token(&app, OPERATOR_TOKEN, "GET", "/api/namespaces?name=default", false, Some("team-a")).await, StatusCode::FORBIDDEN); + assert_eq!(as_token(&app, "dwr_unknown_token_0123456789_0123456789", "GET", "/api/cluster/info", false, None).await, StatusCode::UNAUTHORIZED); + // Rotation replaces the token; revocation ends it; rotation restores it. + let rotated = "dwr_fixture_worker_default_0123456789_rotated"; + let (status, body) = admin(&app, "POST", "/api/runtime-credentials/worker-a/rotate", json!({"token":rotated})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert!(body["rotated_at"].is_string(), "{body}"); + assert_eq!(as_token(&app, WORKER_TOKEN, "GET", "/api/cluster/info", false, None).await, StatusCode::UNAUTHORIZED); + assert_eq!(as_token(&app, rotated, "GET", "/api/cluster/info", false, None).await, StatusCode::OK); + let (status, body) = admin(&app, "DELETE", "/api/runtime-credentials/worker-a", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert!(body["revoked_at"].is_string(), "{body}"); + assert_eq!(as_token(&app, rotated, "GET", "/api/cluster/info", false, None).await, StatusCode::UNAUTHORIZED); + assert_eq!(admin(&app, "POST", "/api/runtime-credentials/worker-a/rotate", json!({"token":rotated})).await.0, StatusCode::OK); + assert_eq!(as_token(&app, rotated, "GET", "/api/cluster/info", false, None).await, StatusCode::OK); + // With database credentials disabled, the token is unknown. + let disabled = router(runtime.clone()); + assert_eq!(as_token(&disabled, rotated, "GET", "/api/cluster/info", false, None).await, StatusCode::UNAUTHORIZED); + runtime.close().await; + database.remove().await; +} + +#[tokio::test] +async fn the_none_driver_admits_every_request() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone().with_auth_options(true, false)); + let (status, _, body) = admission_http(&app, "GET", "/api/workflows", false, None, Some("2"), None, json!({})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + runtime.close().await; + database.remove().await; +} From a3faf631147de2ecbfa5d6f1931ac30819283ec4 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 22 Oct 2026 00:51:44 +0000 Subject: [PATCH 26/30] Compare stored credential claims and expiry through typed locals SonarCloud's Rust analyzer read the inline comparisons as unit comparisons (S7423) and failed the reliability gate. Clippy reports no unit comparison; reading the stored values into typed locals first is clearer either way. --- rust/src/runtime/runtime_credentials.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/rust/src/runtime/runtime_credentials.rs b/rust/src/runtime/runtime_credentials.rs index 1257fed65..8034ee416 100644 --- a/rust/src/runtime/runtime_credentials.rs +++ b/rust/src/runtime/runtime_credentials.rs @@ -5,7 +5,7 @@ //! tenant). use axum::http::StatusCode; use chrono::{DateTime, Utc}; -use serde_json::{Map, Value, json}; +use serde_json::{Value, json}; use sha2::{Digest, Sha256}; use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; @@ -282,19 +282,20 @@ where .fetch_optional(&mut *tx) .await? { - let same_claims = DB::optional_document_row(&row, "claims")?.unwrap_or_else(|| json!({})); - let same_claims = if same_claims.as_object().is_some_and(Map::is_empty) || same_claims == json!([]) { - json!({}) - } else { - same_claims + // PHP stores empty claims as an empty array. + let stored_claims: Value = match DB::optional_document_row(&row, "claims")? { + Some(Value::Array(items)) if items.is_empty() => json!({}), + Some(claims) => claims, + None => json!({}), }; + let stored_expiry: Option> = DB::optional_instant(&row, "expires_at")?; let matches = DB::optional_string(&row, "name")? == attributes.name && DB::string(&row, "subject")? == attributes.subject && DB::document_row(&row, "roles")? == json!(attributes.roles) && DB::string(&row, "tenant")? == attributes.tenant - && same_claims == attributes.claims + && stored_claims == attributes.claims && DB::string(&row, "token_hash")? == attributes.token_hash - && DB::optional_instant(&row, "expires_at")? == attributes.expires_at + && stored_expiry == attributes.expires_at && DB::optional_instant(&row, "revoked_at")?.is_none(); tx.commit().await?; if !matches { From a1be32b8af6a9a85b90e42d6bde8c62fda505626 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 22 Oct 2026 00:57:04 +0000 Subject: [PATCH 27/30] Integrate the October 21-22 port train: restore the startup test module and drop gaps this train closes The union merge of the database wait and legacy setting branches nested the setting tests inside the startup tests and re-added three known-gap bullets that their own branches removed or that later branches close: malformed input envelopes (#606), queries while a workflow task is open (#619) and search attribute upserts (closed by #622). --- docs/rust-server-port.md | 25 ------------------------- rust/src/main.rs | 3 +++ 2 files changed, 3 insertions(+), 25 deletions(-) diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 2bfd70cd2..f2a61b692 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -92,7 +92,6 @@ contract fixture before its correction. PHP fills them from `workflow_run_waits` and `workflow_run_timeline_entries`. No SDK replayer or bundle verifier reads either section. - - **Search attributes on start.** Published PHP validates `search_attributes` against the namespace registry and stores them as typed rows. Native answers a start that names any with @@ -119,17 +118,9 @@ contract fixture before its correction. signal and update counts with their payload size at intake, and the payload size of an activity's result. Native enforces only the memo size. #612 tracks the rest. -- **Malformed input envelopes.** Published PHP refuses a malformed `input` - envelope on start, signal, update and query with `422 validation_failed`, - using Laravel messages on `input.codec` or `input.blob`. The cases are a - codec that is not `avro`, a missing codec, a blob that is empty or not a - string, and a blob Avro cannot decode. Native answers with its own reasons - instead: `unsupported_payload_codec`, `invalid_payload_envelope`, or an - `invalid_*_arguments` reason. - **Key order inside plain JSON input.** For input that is not an envelope, PHP encodes nested objects as Avro maps in request order; native encodes them with sorted keys. The decoded values are equal, but the blobs differ. - - **Child completion while the parent has an open workflow task.** Published Workflow records the child resolution and reuses the parent's existing ready or leased workflow task (`WorkflowExecutor::dispatchParentResumeTasks`). @@ -161,7 +152,6 @@ contract fixture before its correction. and native then correctly fences the attempt. A heartbeat timeout comfortably above the request timeout, still inside the 9-second load wave, needs its own reviewed profile change. - - **A valid workflow task claim held by a stale worker.** Published PHP refuses a request whose lease is otherwise valid when its owner's registration is no longer fresh (`409 stale_worker_registration`). Native's @@ -204,27 +194,12 @@ contract fixture before its correction. `upgrade_source_catalog_unqualified` until its recipe lands. The recipe must also refuse rows that carry trace context until native implements the contract. The boundary is agreed on #325. -- **Queries while a workflow task is open.** Native answers a query from a - history snapshot taken at admission, so it refuses with 409 - `query_snapshot_busy` while the run has a ready or leased workflow task. - Published PHP admits the query. A workflow task leased at admission fixes - the query's history cutoff at that point. The query's claim then waits - while a workflow task is leased, or while an earlier resume task (signal, - update, activity, child or timer) is ready and not yet in history. A - run's first workflow task does not hold the claim. A query sent right after - a signal therefore waits on PHP and is refused on native. The port takes the - snapshot at claim time and must keep query lease replay stable, so it needs - its own fixture. - **Stop signal in the published image.** The published Server image declares `STOPSIGNAL SIGWINCH`, Apache's graceful stop. Native stops gracefully on `SIGTERM` or `SIGINT` and ignores `SIGWINCH`, so `docker stop` on the published image with the native binary as its entrypoint waits out its timeout and kills the process. A native image must declare `STOPSIGNAL SIGTERM`. -- **Search attribute upserts.** Native has no `upsert_search_attributes` - workflow command and refuses it with `unsupported_workflow_command`. The - Sample App's search attribute scenario stops there, on its Rust worker's - first upsert. - **Visibility query evaluation.** Published PHP filters a structured list `query` in SQL. Native reads the queried attribute keys once, then evaluates candidate runs in list order, 500 at a time, until it has the diff --git a/rust/src/main.rs b/rust/src/main.rs index acbe494e6..47f0911fd 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -623,7 +623,10 @@ mod startup_tests { )))); assert!(!unreachable(&RuntimeError::Database(sqlx::Error::RowNotFound))); assert!(!unreachable(&RuntimeError::Io(refused()))); + } } + +#[cfg(test)] mod auth_env_tests { use super::*; From 09c0a128009a3b14e9b4540a51ba651f54b1d6e1 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 22 Oct 2026 01:02:26 +0000 Subject: [PATCH 28/30] Expect native to accept the none driver and runtime credentials in the role profile The role profile asserted that native refuses DW_AUTH_DRIVER=none and DW_RUNTIME_CREDENTIALS_ENABLED=true at startup. Both are now qualified, so the refusal cases use the signature driver instead and drop runtime credentials. --- scripts/conformance/server-parity/role-profile.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/conformance/server-parity/role-profile.sh b/scripts/conformance/server-parity/role-profile.sh index 44c2e993e..43119caaf 100644 --- a/scripts/conformance/server-parity/role-profile.sh +++ b/scripts/conformance/server-parity/role-profile.sh @@ -43,7 +43,8 @@ database() { if test "$backend" = sqlite; then printf '/state/%s.sqlite' "$1"; else printf '%s' "$1"; fi } configuration_case=0 -for setting in DW_AUTH_DRIVER=none DW_AUTH_DRIVER= DW_PRINCIPAL_TOKENS=[] DW_AUTH_BACKWARD_COMPATIBLE=false DW_AUTH_BACKWARD_COMPATIBLE= DW_RUNTIME_CREDENTIALS_ENABLED=true; do +# The none driver and runtime credentials are qualified; these settings are not. +for setting in DW_AUTH_DRIVER=signature DW_AUTH_DRIVER= DW_PRINCIPAL_TOKENS=[] DW_AUTH_BACKWARD_COMPATIBLE=false DW_AUTH_BACKWARD_COMPATIBLE=; do configuration_case=$((configuration_case + 1)) key="${setting%%=*}" if docker run --rm --user=1000:1000 --network=none --entrypoint /target/debug/durable-workflow-server \ From 6c4ef3912822609c87c333ee98efd17a3bf4d853 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 22 Oct 2026 01:38:01 +0000 Subject: [PATCH 29/30] Dispatch a replay-blocked run's repaired task, like PHP Published Workflow's attemptRepair sends both an overdue run and a replay-blocked run through TaskRepair::repairRun and then dispatches the task. Native marked only the overdue path's task dispatched. A replay-blocked repair left the task's last dispatch at its creation, so once three seconds had passed the run read as repair_needed right after its repair; the slow MySQL suite on the combined train showed that. --- rust/src/runtime/task_repair.rs | 8 +++++++- rust/tests/execution/task_retry.rs | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/rust/src/runtime/task_repair.rs b/rust/src/runtime/task_repair.rs index 5e09532cb..c39be68cf 100644 --- a/rust/src/runtime/task_repair.rs +++ b/rust/src/runtime/task_repair.rs @@ -259,7 +259,13 @@ where .fetch_one(&mut **tx) .await?; let recorded = Repair { liveness: Some("workflow_replay_blocked"), wait_kind, payload: &payload, task: Some(&repaired) }; - Self::record_repair(tx, workflow_id, run, context, recorded).await + let response = Self::record_repair(tx, workflow_id, run, context, recorded).await?; + // `TaskDispatcher::markDispatched`: PHP dispatches a replay-blocked + // run's repaired task after the commit, as it does an overdue one. + let at = now(); + Self::query("UPDATE workflow_tasks SET last_dispatch_attempt_at=$1,last_dispatched_at=$2,last_dispatch_error=NULL,repair_available_at=NULL WHERE id=$3 AND status='ready'") + .bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(&task_id).execute(&mut **tx).await?; + Ok(response) } /// The accepted repair command, its `RepairRequested` event and PHP's diff --git a/rust/tests/execution/task_retry.rs b/rust/tests/execution/task_retry.rs index ee50b7866..9766fd1d6 100644 --- a/rust/tests/execution/task_retry.rs +++ b/rust/tests/execution/task_retry.rs @@ -516,6 +516,8 @@ async fn worker_reported_range_failure_requires_explicit_original_task_repair_ac assert_eq!(repaired["requested_run_id"], Value::Null); assert_eq!(repaired["resolved_run_id"], started["run_id"]); assert_eq!(rows(&database, "SELECT COUNT(*) FROM workflow_tasks WHERE status='ready' AND attempt_count=1 AND repair_count=1 AND lease_owner IS NULL AND lease_expires_at IS NULL AND last_error IS NULL").await, 1); + // PHP dispatches the repaired task, so it is not overdue right after. + assert_eq!(rows(&database, "SELECT COUNT(*) FROM workflow_tasks WHERE status='ready' AND repair_count=1 AND last_dispatched_at>=updated_at AND last_dispatch_attempt_at=last_dispatched_at").await, 1); assert_eq!(rows(&database, "SELECT COUNT(*) FROM workflow_commands WHERE command_type='repair' AND target_scope='instance' AND requested_workflow_run_id IS NULL").await, 1); let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; assert_eq!( From faac0d4533bf2df37a8da2a932efd2f5e94f9de3 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 22 Oct 2026 01:45:46 +0000 Subject: [PATCH 30/30] Prove a failed credential lookup refuses without falling back Security's review of #627 asked for a fault-injected test of the database lookup. With the credentials table gone, an unknown bearer token gets the generic 500 server_error without table or token text, configured tokens still authenticate, and a request without a token is still refused. --- rust/tests/execution/runtime_credentials.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/rust/tests/execution/runtime_credentials.rs b/rust/tests/execution/runtime_credentials.rs index 88b5b627e..54e41f7b2 100644 --- a/rust/tests/execution/runtime_credentials.rs +++ b/rust/tests/execution/runtime_credentials.rs @@ -110,3 +110,23 @@ async fn the_none_driver_admits_every_request() { runtime.close().await; database.remove().await; } + +#[tokio::test] +async fn a_failed_credential_lookup_refuses_without_falling_back() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone().with_auth_options(false, true)); + assert_eq!(admin(&app, "PUT", "/api/runtime-credentials/worker-a", credential(WORKER_TOKEN, json!(["worker"]), "default")).await.0, StatusCode::CREATED); + // The lookup's table is gone, so the database refuses every lookup. + database.execute("ALTER TABLE runtime_credentials RENAME TO runtime_credentials_moved").await; + let (status, _, body) = admission_http(&app, "GET", "/api/cluster/info", false, Some(WORKER_TOKEN), Some("2"), None, json!({})).await; + assert_eq!((status, body["reason"].clone()), (StatusCode::INTERNAL_SERVER_ERROR, json!("server_error")), "{body}"); + assert!(!body.to_string().contains("runtime_credentials") && !body.to_string().contains(WORKER_TOKEN), "{body}"); + // Configured tokens are matched first and still authenticate; a request + // without a token is still refused. + assert_eq!(admin(&app, "GET", "/api/cluster/info", Value::Null).await.0, StatusCode::OK); + let (status, _, _) = admission_http(&app, "GET", "/api/cluster/info", false, None, Some("2"), None, json!({})).await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + runtime.close().await; + database.remove().await; +}