diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index fc1b843..05354ba 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -8,11 +8,25 @@ {"id":"runledger-nxe","title":"Unify validated duplicate runtime abstractions","description":"Implement the three actionable dup-unifier findings sequentially: reuse runledger-test-support ScopedEnv in runtime tests, centralize runtime panic payload normalization, and remove the redundant private RuntimeTaskExit enum. Preserve public behavior and verify affected crates.","status":"closed","priority":2,"issue_type":"task","created_at":"2026-09-04T09:48:46.266695Z","created_by":"aa","updated_at":"2026-09-04T09:53:21.335789Z","closed_at":"2026-09-04T09:53:21.335646Z","close_reason":"Completed all three validated unifications; focused and full runtime tests, all-target checks/clippy, and post-change duplicate scan passed.","source_repo":"runledger","source_repo_path":"/Users/aa/Documents/runledger","compaction_level":0,"original_size":0} {"id":"runledger-qq4","title":"Scope lifecycle timeouts to bounded queue work","description":"Move lock-wait policy to initial lifecycle row acquisition and keep unbounded workflow terminal propagation outside library-imposed transaction deadlines; preserve caller timeout policy and add PostgreSQL 18 regression coverage.","status":"closed","priority":1,"issue_type":"bug","created_at":"2026-08-26T12:18:25.068498Z","created_by":"aa","updated_at":"2026-08-26T12:25:28.196269Z","closed_at":"2026-08-26T12:25:28.196159Z","close_reason":"Scoped lifecycle lock caps to initial completion row acquisition, kept workflow propagation outside library transaction deadlines, preserved caller settings across success/failure/continuation, and passed PostgreSQL 18.4 focused tests plus clippy.","source_repo":"runledger","source_repo_path":"/Users/aa/Documents/runledger","compaction_level":0,"original_size":0} {"id":"runledger-ri9","title":"Address progress-heartbeat remediation review findings","description":"Research heartbeat lock-timeout semantics, extend defensive timeout coverage to completion transactions, remove hot-path timeout setup round trips, isolate heartbeat lock regression observations, align lease TTL coverage, and add timeout/reaper tests.","status":"closed","priority":1,"issue_type":"bug","created_at":"2026-08-26T11:50:50.839070Z","created_by":"aa","updated_at":"2026-08-26T12:00:33.417588Z","closed_at":"2026-08-26T12:00:33.417421Z","close_reason":"Resolved heartbeat timeout semantics; combined timeout setup into one round trip; capped success/failure/continuation transactions; isolated lock observations; aligned lease TTL coverage; added completion and heartbeat timeout/reaper PostgreSQL 18.4 regressions; passed full postgres/runtime tests and clippy.","source_repo":"runledger","source_repo_path":"/Users/aa/Documents/runledger","compaction_level":0,"original_size":0} +{"id":"runledger-runledger-simplification-audit-0ju","title":"Measure run-ID payload lookup indexing for large scopes","description":"Follow-up to docs/review-hot-path-fixes-2026-09-05.md. On PostgreSQL 18.6, an exploratory fixture with 20,000 interleaved tenant rows and 10,000 old global rows caused a custom-plan run-ID JSON lookup to scan the global creation-order index and reject 19,980 unrelated rows (~39,995 buffer hits). Custom planning already simplified the scope predicate, so per-scope SQL alone does not address this separate distribution. Measure actual tenant/global payload/run-ID distributions and compare expression or partial indexes against write amplification and index-build rollout cost before selecting a schema change. Preserve exact scope and newest-created/id ordering. No universal latency guarantee is established by the small-scope regression fixture.","status":"open","priority":2,"issue_type":"task","created_at":"2026-09-05T18:02:55.338138072Z","created_by":"aa","updated_at":"2026-09-05T18:02:55.338138072Z","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0} +{"id":"runledger-runledger-simplification-audit-34a","title":"Fix scoped list plans and durable progress validation after branch review","description":"Research scope/prelude questions; preserve indexable tenant/global listing queries; centralize progress validation and retain typed errors across persistence/runtime; add PostgreSQL 18 regression, typed worker, legacy metrics/payload and packaged API coverage.","status":"closed","priority":2,"issue_type":"bug","created_at":"2026-09-05T13:22:01.563016734Z","created_by":"aa","updated_at":"2026-09-05T13:50:18.650614278Z","closed_at":"2026-09-05T13:50:18.650249968Z","close_reason":"Fixed scoped list access with SQLx-checked per-scope statements, centralized durable progress validation and typed runtime errors, restored prelude export, and added regression/compatibility tests. 879 workspace tests, targeted PostgreSQL 18.6 diagnostics, lint, metadata/package checks, cargo deny, and packaged smoke passed. Separate API coverage and standalone container-startup failures are tracked in 60g and z0s.","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0} +{"id":"runledger-runledger-simplification-audit-3ir","title":"Resolve review hot-path regressions and deadline coverage","description":"Research handler deadline precedence; reproduce payload lookup prepared plans on PostgreSQL 18; simplify progress persistence without weakening atomic validation or lease fencing; cover query plans, progress costs, deadline boundaries and migration compatibility; update upgrade notes.","status":"closed","priority":2,"issue_type":"task","created_at":"2026-09-05T17:44:28.417779785Z","created_by":"aa","updated_at":"2026-09-05T18:08:10.439398485Z","closed_at":"2026-09-05T18:08:10.439139640Z","close_reason":"Researched deadline and migration contracts; fixed explicit-scope query construction and redundant progress lock policy; added regression and cost coverage. PostgreSQL 18.6 workspace tests 915 passed, lint and packaged smoke passed, synchronized 159 SQLx records. Large-scope JSON indexing tracked separately in 0ju.","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0} +{"id":"runledger-runledger-simplification-audit-4a2","title":"Expose handler execution budgets and fenced progress services","description":"Source audit: docs/api-audit-2026-09-05.md, AP-RUST-001.\n\nAdd an opt-in execution service alongside legacy JobHandler with authoritative deadline, remaining budget, checkpoint access, and awaited lease-fenced progress. Keep SQLx outside serializable core context. Prove removal of OneSales CRM queue reads and identity plumbing; validate timeout, lease loss, and durable checkpoint behavior on PostgreSQL 18.","status":"closed","priority":1,"issue_type":"feature","created_at":"2026-09-05T09:20:36.022804734Z","created_by":"aa","updated_at":"2026-09-05T10:07:52.143976859Z","closed_at":"2026-09-05T10:07:52.143687842Z","close_reason":"Implemented opt-in JobExecutionHandler and borrowed execution services with authoritative deadline/budget, typed checkpoint access, awaited lease-fenced progress, and legacy adaptation. PostgreSQL 18.6: workspace tests 864 passed, 3 existing ignored; lint and core without default features passed. Validated downstream patches: 79 OneSales CRM tests and 8 IdentityPro tests, both all-target checks passed.","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0,"labels":["api-audit"]} +{"id":"runledger-runledger-simplification-audit-60g","title":"Extend exact scope coverage to metrics and payload lookups","description":"Review research: API-004 deliberately covered job/event/log/intent inspection. Legacy job, continuation, and intent metrics retain documented None=all-scopes semantics. Add exact-global metric APIs without changing wrappers. Payload lookup keys are unique only inside a tenant/global scope, so use an exact scope contract (no Admin wildcard for a single-row idempotency lookup); preserve tenant-only legacy signatures. Add global/tenant/admin metric tests and cross-scope duplicate-key lookup tests. See docs/review-fixes-2026-09-05.md.","design":"Reuse JobReadScope for metrics. Prefer the existing JobScope::{Global, Organization} for exact payload lookups; broaden its mutation-specific documentation if reused for reads. Do not introduce a competing scope enum without a concrete need. Preserve the existing JobEnqueueIntentMetricsFilter contract while adding an explicit-scope entry point/filter.\n\nImplementation locations: runledger-postgres/src/jobs/admin/metrics.rs; jobs/queue/intents.rs; jobs/admin/read.rs; jobs/types/admin.rs; jobs/types/enqueue.rs; jobs.rs; lib.rs. The two queue idempotency indexes enforce uniqueness per job type and exact tenant/global scope, not across all scopes. The latest-payload run_id is a JSON field and does not identify a globally unique queue row.\n\nExtend tests/legacy_read_contracts.rs, tests/job_continuation.rs, and tests/job_enqueue_intents.rs. Existing tests establish legacy behavior but do not prove the proposed exact-global APIs. See docs/review-fixes-2026-09-05.md and API-004 in docs/api-audit-2026-09-05.md.","acceptance_criteria":"1. Add explicit-scope APIs for job metrics, continuation metrics, and enqueue-intent metrics covering Global, Organization, and Admin. Preserve legacy signatures and None=all-scopes behavior; legacy Some(id) remains tenant-only.\n2. Preserve registered job types with zero counts in job and continuation metrics when the selected scope has no matching rows. Keep scope predicates in the LEFT JOIN (or an equivalent query preserving unmatched definitions), not a WHERE predicate that drops those rows. Preserve existing aggregation and duration semantics.\n3. Apply intent scope filtering to all three lifecycle populations: PENDING, CONFLICTED, and PROMOTED. Preserve pending-only backlog/retry/oldest-pending semantics, 24-hour terminal windows, omission of groups represented only by old terminal history, exact job-type filtering, stable ordering, and pagination validation.\n4. Add exact-global and exact-tenant variants of both get_job_payload_by_idempotency_key and get_latest_job_payload_for_run without an Admin wildcard. Preserve tenant-only legacy signatures and Option<(Uuid, Value)> results, including None for an absent match.\n5. Test metrics with global, tenant A, tenant B, Admin, and an unrelated tenant using distinguishable fixture values. Cover every intent lifecycle population and zero-count job/continuation rows. Assert legacy None matches Admin and legacy tenant calls match the corresponding explicit scope.\n6. Test identical idempotency keys and identical JSON run_id values across global and two tenant scopes; verify each exact-scope payload lookup returns the correct row and payload. For latest-payload reads, include multiple rows within a scope and verify created_at DESC, id DESC ordering, including timestamp ties and a newer out-of-scope row. Preserve missing-scope and nil-UUID behavior.\n7. Export the new APIs and any new filter through jobs and prelude; cover downstream imports. Document explicit scopes, legacy compatibility, and application responsibility for authorizing selected read scopes.\n8. Run relevant database-backed regression tests on PostgreSQL 18 and record the exact server version. Refresh SQLx metadata against PostgreSQL 18 with current migrations and keep .sqlx/, runledger-postgres/.sqlx/, and runledger-runtime/.sqlx/ synchronized.","status":"closed","priority":2,"issue_type":"feature","created_at":"2026-09-05T13:31:38.137006500Z","created_by":"aa","updated_at":"2026-09-05T14:35:27.062994084Z","closed_at":"2026-09-05T14:35:27.062722458Z","close_reason":"Implemented explicit metric scopes and exact payload scopes with legacy compatibility; 46 PostgreSQL 18.6 regressions and two packaged-consumer tests pass, lint passes, and all three SQLx caches are synchronized.","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0} +{"id":"runledger-runledger-simplification-audit-7cy","title":"Publish migration pipeline identity for downstream composition","description":"Source audit: docs/api-audit-2026-09-05.md, API-008.\n\nExpose library/migration-pipeline identity and bundle metadata usable by downstream schema/test-template fingerprints. Document composition with host SQLx history without taking ownership of app migration ordering. Validate against IdentityPro manual pipeline fingerprints and HOCR vendored-bundle use; retain current migration/compatibility helpers.","acceptance_criteria":"Publish crate-owned library identity and a documented deterministic bundle manifest and pipeline fingerprint. Distinguish bundle content identity from helper/release identity, including the policy for same-version patched builds. Preserve MIGRATOR and all existing startup/compatibility helpers. Compile downstream composition that retains host pipeline ordering/domain and other SQLx histories. Validate IdentityPro-style template inputs and HOCR-style pinned historical SQL/checksum vendoring, including changed/missing entry failures, through packaged external-consumer tests. Metadata must work without database access or workspace files. See docs/migration-pipeline-identity-audit-2026-09-05.md for source evidence and acceptance checks.","notes":"Implemented RUNLEDGER_POSTGRES_VERSION and migration_bundle()/MigrationBundle with separate bundle and release-pipeline SHA-256 fingerprints, documented v1 encoding and same-version helper policy. Preserved existing migration/compatibility helpers. Added composition example, IdentityPro adapter patch, and independent HOCR historical fixtures. Validation: 4 focused unit tests, 10 doctests, example execution, 2 local identity consumer tests, all 4 packaged consumer tests on PostgreSQL 18.6 (180006), repository lint and targeted final Clippy. Details: docs/migration-identity/README.md. Full downstream applications were not migrated.","status":"closed","priority":2,"issue_type":"feature","created_at":"2026-09-05T09:20:44.207359063Z","created_by":"aa","updated_at":"2026-09-05T15:08:07.170697956Z","closed_at":"2026-09-05T15:08:07.170387566Z","close_reason":"Implemented and validated migration identity, manifest, and downstream composition; all acceptance criteria covered.","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0,"labels":["api-audit"]} {"id":"runledger-runledger-simplification-audit-8ee","title":"Enforce Clippy function-length limit","description":"Enable clippy::too_many_lines as deny in both Rust workspaces with an explicit 150-line threshold. Refactor every violation, including integration and external-consumer tests; do not add lint exclusions.","status":"closed","priority":3,"issue_type":"task","created_at":"2026-09-02T08:04:03.454316549Z","created_by":"aa","updated_at":"2026-09-02T09:05:03.381465470Z","closed_at":"2026-09-02T09:05:03.381141390Z","close_reason":"Completed: enforced the 150-line limit with zero too_many_lines exemptions; strict lint, full PostgreSQL 18 workspace tests, and external consumer smoke test pass.","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0} {"id":"runledger-runledger-simplification-audit-9tf","title":"Make dependency and key-dispatch invariants explicit","description":"Address comprehensive-review follow-ups: model direct-dependent updates as aligned values and document/test TUI key-dispatch precedence across guarded bindings.","status":"closed","priority":3,"issue_type":"task","created_at":"2026-09-02T07:46:23.897866455Z","created_by":"aa","updated_at":"2026-09-02T07:52:32.372943132Z","closed_at":"2026-09-02T07:52:32.372704682Z","close_reason":"Completed: direct-dependent updates now preserve ID/unsatisfied alignment structurally; TUI dispatcher precedence is documented and covered by active/inactive context matrices. PostgreSQL 18, full TUI tests, checks, and Clippy pass.","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0} +{"id":"runledger-runledger-simplification-audit-bo0","title":"Document missing public API additions in Unreleased changelog","description":"Address the branch review finding by documenting scoped reads, shared specifications and typed handlers, execution services, enqueue outcomes, compact summary reads, migration identity, and fluent DAG helpers.","status":"closed","priority":3,"issue_type":"docs","created_at":"2026-09-05T18:24:42.407111680Z","created_by":"aa","updated_at":"2026-09-05T18:24:58.958176486Z","closed_at":"2026-09-05T18:24:58.957892637Z","close_reason":"Documented all missing public API groups and verified names and behavior against source; git diff --check passed.","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0} +{"id":"runledger-runledger-simplification-audit-dxt","title":"Evaluate durable lifecycle subscriptions with a CreditKit pilot","description":"Source audit: docs/api-audit-2026-09-05.md, API-007.\n\nCurrent lifecycle observers and dead-letter hooks are deliberately best effort. Evaluate a separate durable lifecycle inbox/subscription or terminal follow-up job captured atomically with lifecycle changes. Use creditkit-platform document reconciliation to identify generic delivery work that can be removed while retaining application artifact checks and business recovery policy. Specify stable delivery identity, acknowledgements, retries, retention, and at-least-once semantics. Enqueue intents are not lifecycle subscriptions. This is discovery, not a commitment to replace existing observers.","acceptance_criteria":"Deliver a source-backed design and consumer prototype demonstrating which reconciliation responsibilities move into Runledger. Cover crash-after-commit, duplicate delivery, failed acknowledgement, poison events, and retention. Preserve application authorization and side-effect idempotency; any database experiments use PostgreSQL 18 and record its exact version.","status":"open","priority":2,"issue_type":"task","created_at":"2026-09-05T09:24:30.169901378Z","created_by":"aa","updated_at":"2026-09-05T09:24:30.169901378Z","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0,"labels":["api-audit","discovery"]} {"id":"runledger-runledger-simplification-audit-fdc","title":"Document repair of legacy JobCompletion progress payloads","description":"Investigate the deserialization compatibility finding and document a staged repair path for application-persisted JobCompletion values with partial or invalid progress, preserving strict validated completion invariants.","status":"closed","priority":3,"issue_type":"docs","created_at":"2026-09-05T09:05:43.282910112Z","created_by":"aa","updated_at":"2026-09-05T09:08:13.147686992Z","closed_at":"2026-09-05T09:08:13.147440510Z","close_reason":"Confirmed legacy partial progress compatibility gap. Added staged repair guidance and a runnable JSON migration example while retaining strict validation. Validated with 110 core tests, 22 doctests, formatting, and rustdoc with warnings denied.","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0} +{"id":"runledger-runledger-simplification-audit-fmh","title":"Extend fluent workflow builders to cover advanced step policies","description":"Source audit: docs/api-audit-2026-09-05.md, API-005; verified against current builders and OneSales buyer enrichment. Extend WorkflowDagBuilder and CatalogWorkflowDagBuilder with configurable step composition, external steps, and active-key set/clear support. Reuse WorkflowStepEnqueueBuilder and JobCatalog::workflow_step for per-step organizations, queue settings, continuation and execution-resource policies. Preserve borrowed requests, low-level builders, existing validation, and persistence semantics. Owned requests remain optional: defer unless a concrete consumer requires ownership beyond fluent policy parity.","acceptance_criteria":"Both fluent builders compose configured job and external steps with success/terminal edges and active keys. Reproduce the OneSales buyer-enrichment graph in repository tests, asserting per-step tenants, continuation, shared resource, terminal chain, run scope, idempotency, active key and payloads against the existing low-level representation. Independently test mixed external/job graphs, forbidden external settings, invalid active/resource keys, queue limits, duplicate/missing/self/cyclic edges and result selection. Configured steps inserted into a catalog DAG must be checked against that receiving catalog, including steps created by a different catalog; reject unknown/disabled jobs and accept external steps without job registration. Preserve existing simple job behavior and validation timing, document new APIs with compiling examples, and run relevant core/runtime tests and lint checks.","notes":"Implemented additive step(WorkflowStepEnqueue), external, active_key and clear_active_key methods on both fluent DAG builders. Configurable policies reuse existing validated step builders; catalog insertion checks the receiving catalog. Preserved appended edge order, omitted release modes, low-level APIs and step-error precedence. Buyer-enrichment acceptance fixture compares complete borrowed requests from both fluent builders with the low-level graph for global, organization and account scopes. Owned requests deferred because this consumer retains its inputs for enqueue and policy parity requires no ownership API. Validation passed: core 116 unit + 3 integration + 23 doctests; runtime 5 fluent integration + 6 existing workflow integration + 28 catalog unit + 5 doctests; full scripts/lint.sh and final affected-crate all-target/all-feature Clippy. No database/schema changes or database tests.","status":"closed","priority":2,"issue_type":"feature","created_at":"2026-09-05T09:20:36.454876658Z","created_by":"aa","updated_at":"2026-09-05T15:24:41.332947308Z","closed_at":"2026-09-05T15:24:41.332711652Z","close_reason":"Implemented fluent advanced-policy parity with shared validation, catalog enforcement, documented APIs and passing acceptance tests.","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0,"labels":["api-audit"]} +{"id":"runledger-runledger-simplification-audit-kqm","title":"Measure and improve operational reads and bulk enqueue costs","description":"Source audit: docs/api-audit-2026-09-05.md, API-006.\n\nMeasure compact versus full job reads, cursor versus offset pages, and per-step/per-edge workflow inserts on PostgreSQL 18, recording exact server version. Design scoped compact summaries and bounded batch writes with explicit atomicity/outcomes and unchanged fencing/audit/idempotency. Shared result listeners are secondary: no sampled production wait consumer was found. Report measurements before claiming speedups.","status":"closed","priority":2,"issue_type":"task","created_at":"2026-09-05T09:20:42.280923180Z","created_by":"aa","updated_at":"2026-09-05T17:28:13.734082532Z","closed_at":"2026-09-05T17:28:13.733715428Z","close_reason":"Measured PostgreSQL 18.6 reads and graph/direct enqueue costs; shipped scoped compact cursor pages, batch statuses, and 256-row graph inserts preserving audit/idempotency/rollback. Published raw measurements and bounded direct-batch design in docs/operational-costs-2026-09-05.md. Workspace, focused regressions, migration tests, lint, and synchronized SQLx refresh passed.","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0,"labels":["api-audit"]} +{"id":"runledger-runledger-simplification-audit-m9g","title":"Add exact-scope direct-job operational reads","description":"Source audit: docs/api-audit-2026-09-05.md, API-004.\n\nAdd explicit Global, Organization, and Admin read scopes for job/event/log/intent reads while preserving legacy None-as-wildcard wrappers. Include global-only filtering and tenant isolation coverage on PostgreSQL 18. Scope selection remains application-authorized.","status":"closed","priority":1,"issue_type":"feature","created_at":"2026-09-05T09:20:36.216013290Z","created_by":"aa","updated_at":"2026-09-05T09:34:36.083449868Z","closed_at":"2026-09-05T09:34:36.083187252Z","close_reason":"Implemented JobReadScope and six scoped job/event/log/intent read APIs with legacy wildcard wrappers, explicit-scope examples, and synchronized SQLx caches. Verified PostgreSQL 18.6 (180006): 317 tests passed, 2 existing diagnostics ignored; scripts/lint.sh passed.","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0,"labels":["api-audit"]} {"id":"runledger-runledger-simplification-audit-o6q","title":"Couple workflow dependent locking and counter updates","description":"Address review finding in workflow terminal propagation: ensure direct-dependent counter updates cannot be called without first acquiring the stable ordered row locks.","status":"closed","priority":3,"issue_type":"task","created_at":"2026-09-02T07:28:03.028552837Z","created_by":"aa","updated_at":"2026-09-02T07:31:32.175578167Z","closed_at":"2026-09-02T07:31:32.175362347Z","close_reason":"Completed: counter updates now acquire and validate ordered dependent locks within the same helper; focused PostgreSQL 18 tests and Clippy pass.","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0} {"id":"runledger-runledger-simplification-audit-riw","title":"Contain confirmed Rust abstraction leaks","description":"Address AP-RUST-001 through AP-RUST-004 from the repository abstraction audit: ScopedEnv safety, checked JobCompletion progress, documented continuation representability, and CatalogError SQLx containment.","status":"closed","priority":1,"issue_type":"task","created_at":"2026-09-04T17:11:32.477818843Z","created_by":"aa","updated_at":"2026-09-04T17:52:26.336464010Z","closed_at":"2026-09-04T17:52:26.336084544Z","close_reason":"Implemented and verified all four abstraction-boundary remediations","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0} +{"id":"runledger-runledger-simplification-audit-s9o","title":"Assess Perdify upgrade and enqueue coalescing compatibility","description":"Source audit: docs/api-audit-2026-09-05.md, adoption opportunities.\n\nPerdify and perdify-checkout-1 lock Runledger 0.1.1 and share perdify-runledger-support/src/lib.rs. Inventory legacy definition setup, duplicate lookup, savepoints, and enqueue-or-update policies against current strict idempotent enqueue/outcomes. Identify removable wrappers and migration prerequisites. Payload refresh, especially PendingOrLeased behavior, has different semantics from identical-request deduplication. Treat duplicate checkouts as one demand signal and retain this as a Runledger adoption investigation.","acceptance_criteria":"Produce a compatibility map and staged upgrade recommendations with exact call sites. Separate strict enqueue replacements from pending/leased payload mutation. Document generation fencing, audit history, immutable request snapshots, and evidence required before proposing a generic coalescing API. Do not mechanically replace payload-update behavior.","status":"open","priority":2,"issue_type":"task","created_at":"2026-09-05T09:24:30.381244865Z","created_by":"aa","updated_at":"2026-09-05T09:24:30.381244865Z","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0,"labels":["api-audit","discovery"]} +{"id":"runledger-runledger-simplification-audit-sfc","title":"Audit public API against local downstream integrations","description":"Source-based audit of Runledger 0.12 public contracts, runtime, PostgreSQL APIs, and local downstream projects under ~/Documents. Deliver a source-linked report distinguishing current gaps, adoption debt, and unbenchmarked performance opportunities; no runtime implementation changes.","status":"closed","priority":2,"issue_type":"task","created_at":"2026-09-05T09:14:38.117260138Z","created_by":"aa","updated_at":"2026-09-05T09:21:31.388225058Z","closed_at":"2026-09-05T09:21:31.387900489Z","close_reason":"Completed source-linked audit in docs/api-audit-2026-09-05.md; recorded seven prioritized API follow-ups. No runtime implementation changes or database experiments.","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0} +{"id":"runledger-runledger-simplification-audit-tor","title":"Share producer and worker job specs with typed adapters","description":"Source audit: docs/api-audit-2026-09-05.md, AP-RUST-002 and API-003.\n\nIntroduce an additive storage-agnostic job specification shared by producer submission and worker binding, plus opt-in typed payload adapters. Preserve existing JSON handlers, durable payload compatibility, idempotency snapshots, and operator disables. Validate API-only construction without provider clients and migrate representative IdentityPro/CreditKit adapters.","notes":"API-003 also includes a direct-job builder and ordinary pool-owning enqueue-with-outcome convenience. Preserve strict identical-request semantics; expose Inserted/Existing so consumers such as IdentityPro billing can count new work without legacy duplicate-error handling. Centralize safe malformed-payload defaults while allowing application codes and diagnostics.","status":"closed","priority":1,"issue_type":"feature","created_at":"2026-09-05T09:20:35.762987791Z","created_by":"aa","updated_at":"2026-09-05T13:02:46.448861816Z","closed_at":"2026-09-05T13:02:46.448601575Z","close_reason":"Implemented storage-independent shared specs, typed submission/handlers, direct-job builder, and pool enqueue outcomes. PostgreSQL 18.6 compatibility and binding tests passed; IdentityPro (84 tests) and CreditKit (47 renewal + 59 job tests) migration pilots are saved as applicable patches. Runledger lint, Clippy, rustdoc, catalog and idempotency checks passed.","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0,"labels":["api-audit"]} {"id":"runledger-runledger-simplification-audit-w2f","title":"Runledger whole-codebase simplification audit","description":"# Canonical Runledger simplification audit\n\nAudit date: 2026-08-22. This epic is the durable Beads record for the completed read-only whole-repository audit.\n\n## Outcome\n\n- 783 tracked files reviewed: 209 Rust, 78 migration SQL, 456 SQLx cache JSON, and 40 remaining manifests/scripts/docs/policies/lockfiles.\n- 44 exact, non-overlapping subsystem rows.\n- 32 subsystem rows contain recommendations; 12 are explicit skips.\n- 49 recommendations survived source verification, schema review, materiality review, overlap/deduplication, and dependency ranking.\n- 12 candidates were rejected, demoted, or superseded.\n- No tests, builds, repository scripts, database commands, source edits, commits, pushes, publishing, or remote operations were performed during the audit.\n- PostgreSQL 18 is authoritative for every future database implementation/validation. Record `SHOW server_version` or `SHOW server_version_num`, apply current migrations, and keep all three SQLx caches synchronized.\n\n## Coverage contract\n\n- **CORE-01 — Core API and handler contracts** — `skip`; boundary: `runledger-core/src/{lib.rs,jobs.rs,jobs/handler.rs}`\n- **CORE-02 — Identifier types and macro-generated APIs** — `skip`; boundary: `runledger-core/src/jobs/{identifier_macros.rs,identifiers.rs}`\n- **CORE-03 — Runtime DTOs and persisted status contracts** — `recommend`; boundary: `runledger-core/src/jobs/{runtime_types.rs,status.rs}`\n- **CORE-04 — Workflow enqueue model and builders** — `recommend`; boundary: `runledger-core/src/jobs/workflow_enqueue{.rs,/types.rs,/errors.rs,/run_builder.rs,/step_builder.rs,/dag_builder.rs,/tests.rs,/tests/builders.rs,/tests/errors.rs}`\n- **CORE-05 — Workflow enqueue validation and DAG rules** — `skip`; boundary: `runledger-core/src/jobs/workflow_enqueue/{build_validation.rs,step_validation.rs,dag_validation.rs,tests/dag_validation.rs}`\n- **PG-01 — PostgreSQL API, errors, and transaction plumbing** — `recommend`; boundary: `runledger-postgres/src/{lib.rs,jobs.rs,error.rs,error/classify.rs,error/classify/jobs.rs,error/classify/workflows.rs,jobs/errors.rs,jobs/transaction_isolation.rs,jobs/transaction_settings.rs}; runledger-postgres/tests/owned_transaction_wrappers.rs`\n- **PG-02 — Persistence rows and public record types** — `recommend`; boundary: `runledger-postgres/src/jobs/{row_decode.rs,rows.rs,types.rs,types/*.rs,workflow_types.rs}`\n- **PG-03 — Migration engine and canonical PostgreSQL schema** — `recommend`; boundary: `runledger-postgres/src/migrations.rs; migrations/**; runledger-postgres/migrations/**; runledger-postgres/tests/migrations.rs`\n- **PG-04 — Job-definition catalog persistence** — `recommend`; boundary: `runledger-postgres/src/jobs/{schedule_definition_guard.rs,queue/definitions/**}`\n- **PG-05 — Direct enqueue and durable enqueue intents** — `recommend`; boundary: `runledger-postgres/src/jobs/queue/{enqueue.rs,intents.rs}; runledger-postgres/tests/{enqueue_outcome.rs,job_enqueue_intents.rs}; runledger-runtime/tests/idempotency.rs`\n- **PG-06 — Queue claiming, leases, resources, advance, and release** — `recommend`; boundary: `runledger-postgres/src/jobs/queue/{claim.rs,advance.rs,attempts.rs,release.rs}; runledger-postgres/tests/{queue_lease_validation.rs,job_execution_resources.rs}`\n- **PG-07 — Queue heartbeat, progress, success, and continuation** — `recommend`; boundary: `runledger-postgres/src/jobs/queue/{lifecycle.rs,lifecycle/common.rs,lifecycle/heartbeat.rs,lifecycle/progress.rs,lifecycle/success.rs,lifecycle/continuation.rs}; runledger-postgres/tests/{queue_completion_progress_validation.rs,job_continuation.rs}`\n- **PG-08 — Queue failure, retry, dead-letter, and lease reaping** — `recommend`; boundary: `runledger-postgres/src/jobs/queue/{failure_transition.rs,lifecycle/failure.rs,reaper.rs}; runledger-postgres/tests/{queue_retry_delay_validation.rs,queue_reaper_errors.rs}`\n- **PG-09 — Job events and logs** — `skip`; boundary: `runledger-postgres/src/jobs/{logs.rs,queue/events.rs}`\n- **PG-10 — Administrative reads, metrics, and payload mutation** — `skip`; boundary: `runledger-postgres/src/jobs/{admin.rs,admin/read.rs,admin/metrics.rs,admin/payload.rs}; runledger-postgres/tests/{admin_payload_mutation.rs,pagination_validation.rs}`\n- **PG-11 — Direct-job recovery and replay** — `recommend`; boundary: `runledger-postgres/src/jobs/{admin/recovery.rs,replay.rs}; runledger-postgres/tests/{compare_and_requeue.rs,succeeded_job_replay.rs}`\n- **PG-12 — Schedule definitions, claims, locking, and validation** — `recommend`; boundary: `runledger-postgres/src/jobs/{schedules.rs,schedules/**}; runledger-postgres/tests/schedules.rs`\n- **PG-13 — Persisted runtime configuration** — `skip`; boundary: `runledger-postgres/src/jobs/runtime_configs.rs`\n- **PG-14 — Workflow enqueue, snapshots, errors, and validation** — `recommend`; boundary: `runledger-postgres/src/jobs/{workflows.rs,workflows/enqueue.rs,workflows/enqueue_persistence.rs,workflows/snapshot.rs,workflows/errors.rs,workflows/validation.rs}`\n- **PG-15 — Workflow read models and handles** — `recommend`; boundary: `runledger-postgres/src/jobs/workflows/{read.rs,handles.rs}; runledger-postgres/tests/{workflow_ordering.rs,workflow_results.rs}`\n- **PG-16 — Workflow append, cancel, and idempotent mutation** — `recommend`; boundary: `runledger-postgres/src/jobs/workflows/{mutate.rs,mutate/append.rs,mutate/cancel.rs,mutate/idempotency.rs}`\n- **PG-17 — Workflow runtime transitions, claims, locking, hooks, and release** — `recommend`; boundary: `runledger-postgres/src/jobs/workflows/{runtime.rs,runtime/*.rs,active_claims.rs,locking.rs,release.rs,hooks.rs}; runledger-postgres/tests/{workflow_active_claims.rs}; runledger-runtime/tests/workflow_cancel_lock_order.rs`\n- **PG-18 — Workflow recovery** — `recommend`; boundary: `runledger-postgres/src/jobs/workflows/recovery.rs; runledger-postgres/tests/workflow_recovery.rs`\n- **PG-19 — Queue module facade and export ownership** — `skip`; boundary: `runledger-postgres/src/jobs/queue.rs`\n- **PG-20 — Shared workflow-step persistence and definition-default resolution** — `recommend`; boundary: `runledger-postgres/src/jobs/workflows/steps.rs`\n- **RT-01 — Runtime catalog model, registration, and synchronization** — `recommend`; boundary: `runledger-runtime/src/{catalog.rs,catalog/**}; runledger-runtime/tests/catalog.rs`\n- **RT-02 — Runtime configuration** — `skip`; boundary: `runledger-runtime/src/config.rs; runledger-runtime/tests/config_validation.rs`\n- **RT-03 — Worker claim and handler execution** — `recommend`; boundary: `runledger-runtime/src/{worker.rs,worker/execution.rs,worker/tests/mod.rs,worker/tests/support.rs,worker/tests/lease_fencing.rs,worker/tests/prestart_recovery.rs}; runledger-runtime/tests/worker_loop.rs`\n- **RT-04 — Worker completion, retry, continuation, and dead letters** — `skip`; boundary: `runledger-runtime/src/worker/{completion.rs,dead_letter.rs,tests/completion_and_retry.rs,tests/terminal_hooks.rs}`\n- **RT-05 — Observer event model and dispatch** — `recommend`; boundary: `runledger-runtime/src/{observer.rs,worker/observers.rs,reaper/observers.rs,worker/tests/observer_tasks.rs}`\n- **RT-06 — Lease reaper loop and terminal hooks** — `recommend`; boundary: `runledger-runtime/src/{reaper.rs,reaper/terminal_hooks.rs}; runledger-runtime/tests/reaper_loop.rs`\n- **RT-07 — Scheduler loop** — `recommend`; boundary: `runledger-runtime/src/{scheduler.rs,scheduler/tests.rs}`\n- **RT-08 — Supervisor, task lifecycle, shutdown, and runtime shell** — `recommend`; boundary: `runledger-runtime/src/{lib.rs,error.rs,shutdown.rs,supervisor.rs,task_group.rs}; runledger-runtime/tests/{supervisor_loop.rs,prelude_smoke.rs}`\n- **RT-09 — Handler registry and retry overrides** — `skip`; boundary: `runledger-runtime/src/registry.rs`\n- **RT-10 — Durable enqueue-intent promoter loop** — `skip`; boundary: `runledger-runtime/src/intent_promoter.rs`\n- **TUI-01 — TUI application state, commands, input, selection, and search** — `recommend`; boundary: `runledger-tui/src/{app.rs,app/input.rs,app/input/**,app/search.rs}`\n- **TUI-02 — TUI data access, models, and scope** — `recommend`; boundary: `runledger-tui/src/{scope.rs,app/fetch.rs,data.rs,data/**}`\n- **TUI-03 — TUI startup, configuration, formatting, terminal bridge, and UI rendering** — `recommend`; boundary: `runledger-tui/src/{main.rs,config.rs,format.rs,terminal.rs,ui.rs,ui/**}`\n- **TEST-01 — PostgreSQL container and shared test harness** — `recommend`; boundary: `runledger-test-support/{src/**,tests/**}; runledger-postgres/tests/support.rs; runledger-runtime/tests/{support.rs,support/**}`\n- **INF-01 — Generated SQLx metadata and schema packaging/synchronization** — `recommend`; boundary: `{.sqlx/**,runledger-postgres/.sqlx/**,runledger-runtime/.sqlx/**,runledger-postgres/build.rs,runledger-test-support/build.rs,runledger-test-support/migrations/**,scripts/refresh-sqlx-cache.sh}`\n- **INF-02 — Workspace manifests, CI, lint, licensing, and repository policy** — `recommend`; boundary: `{Cargo.toml,Cargo.lock,runledger-*/Cargo.toml,rust-toolchain.toml,rustfmt.toml,.cargo/**,.github/**,deny.toml,scripts/check-package-licenses.sh,scripts/lint.sh,.gitignore,AGENTS.md,runledger-*/AGENTS.md,LICENSE,runledger-*/LICENSE}`\n- **INF-03 — Public documentation, examples, and agent plans** — `skip`; boundary: `{README.md,CHANGELOG.md,llms.txt,docs/**,.agent/**,runledger-postgres/examples/**,runledger-runtime/examples/**}`\n- **INF-04 — Release automation** — `recommend`; boundary: `{cliff.toml,scripts/prepare-release.sh,scripts/publish-release.sh,scripts/verify-release-ci.sh}`\n- **INF-05 — Packaged downstream-consumer compatibility** — `recommend`; boundary: `{smoke/external-consumer/**,scripts/run-external-consumer-smoke.sh}`\n\n## Explicit subsystem skips\n\n- **CORE-01** — Curated prelude is a deliberate semver boundary; glob exports would weaken ownership.\n- **CORE-02** — Only candidate was allocation/type-path polish that added a second construction path.\n- **CORE-05** — Validation DTO is a useful adapter shared by two representations.\n- **PG-09** — Typed event payloads already own shape; persistence is straight-line.\n- **PG-10** — SQLx projections, one-use classifier, and metric API split are intentional.\n- **PG-13** — Straightforward CRUD with genuinely distinct conflict policies.\n- **PG-19** — Small declarative facade is the correct visibility owner.\n- **RT-02** — Configuration validation is cohesive; no material finding survived registry extraction.\n- **RT-04** — Completion identity already uses one exact lease tuple; proposed restyling was non-material.\n- **RT-09** — Static lifetime API cleanup was public type polish below materiality.\n- **RT-10** — Promoter wait/drain and cancellation semantics are already cohesive.\n- **INF-03** — Passive docs/examples are cohesive; packaged compatibility is separately owned by INF-05.\n\n## Accepted recommendation index\n\nEach accepted recommendation is a child task of this epic with full evidence, scope, risk, validation, and confidence.\n\n- **CORE-03-R1** — Centralize JobFailureKind retry eligibility (`accepted`, confidence high)\n- **CORE-04-R2** — Centralize effective dependency release mode (`accepted`, confidence high)\n- **PG-01-R1** — Represent typed query-error classification as one discriminated value (`accepted`, confidence high) — Limit the enum to fixed runtime-policy classifications and derive category/code/client message from it; keep generic classified database errors generic.\n- **PG-01-R2** — Derive PostgreSQL timeout text and milliseconds from one duration (`accepted`, confidence high)\n- **PG-02-R1** — Decode enqueue-intent lifecycle into a discriminated state (`accepted`, confidence high) — Representation must cover initial pending, retry-pending with attempt/error metadata, promoted, and conflicted (or a nested pending state), preserving every database-approved state and separately considering JobEnqueueIntentOutcome.\n- **PG-02-R2** — Represent external workflow terminal outcome as an enum (`accepted`, confidence high) — Use common status-independent completion metadata plus Succeeded { output } | Failed | Canceled; preserve succeeded-with-metadata, canceled-with-error-metadata, and exact idempotent comparison.\n- **PG-03-R1** — Make workflow_steps.job_id the sole stored workflow-step/job relationship (`accepted-long-horizon`, confidence medium-high) — Narrowed to retaining workflow_steps.job_id UNIQUE and deriving reverse lookup; staged mixed-version rollout required before dropping job_queue.workflow_step_id, reciprocal FK/unique constraint, deferred symmetry triggers, dual writes, and mismatch checks.\n- **PG-03-R2** — Factor only continuation-event validity into a NOT MATERIALIZED view CTE (`accepted`, confidence medium) — Keep branch-specific run correlation and 24-hour conditions local; PostgreSQL 18 plan/equivalence validation is mandatory, and no reusable SQL function/public view is warranted.\n- **PG-04-R1** — Represent exact definition sync by its computed absent-definition plan (`accepted`, confidence high) — Use a concrete absent-definition set/list, not a planning framework.\n- **PG-06-R1** — Give filtered and unfiltered claiming one authoritative SQL algorithm (`accepted-conditional`, confidence medium) — Verified ~100-line branch duplication differs only by two type predicates and bind numbering. Accept only if PostgreSQL 18 NULL/non-NULL custom+generic plan parity is demonstrated; otherwise retain duplication.\n- **PG-06-R2** — Use JobLeaseIdentity for strict unstarted-claim release (`accepted`, confidence high) — Require JobLeaseIdentity only for strict live-worker release; nullable reaper recovery stays separate.\n- **PG-05-R1** — Represent prepared enqueue idempotency as keyed or unkeyed (`accepted`, confidence high) — Keep Keyed/Unkeyed as a private prepared-enqueue representation, not a public hierarchy.\n- **PG-05-R2** — Classify intent promotion failure with one exhaustive action (`accepted`, confidence high) — Verified terminal errors also satisfy deferred predicate; current branch order is the only exclusivity guarantee. One total action enum preserves the exact decision table.\n- **PG-07-R1** — Type the running transition separately while preserving atomic checkpoint/progress (`accepted`, confidence medium-high) — Dedicated running update must still atomically persist Running, checkpoint, progress, execution start, and event in one transaction; zero-argument mark_running followed by progress would change crash semantics.\n- **PG-08-R1** — Make typed failure details authoritative for lease-expiry metadata (`accepted`, confidence high) — Verified raw failure-kind string and six repeated lease-expiry triplets across queue/attempt/event writes. Preserve the distinct prestart-release reason and exact SQL/event payloads.\n- **PG-08-R2** — Encode resolved retry selection as a discriminated union (`accepted`, confidence high) — Union must retain policy-only, policy-won-over-handler-hint, and handler-hint-won states (or an optional superseded hint on policy), because policy can legitimately win while the handler request remains persisted/audited.\n- **PG-11-R1** — Replace nullable cancellation scope with an explicit exact/admin scope (`accepted`, confidence high) — Current None is privileged all-tenant wildcard, while JobScope::Global means exact NULL. Use an explicit Admin wildcard variant or separate admin entrypoint plus exact global/organization variants; do not silently remove wildcard semantics.\n- **PG-11-R2** — Remove deprecated in-place requeue_job at the next breaking release (`accepted-breaking-release`, confidence high) — Remove only after exact-scope APIs exist, in a deliberate breaking release.\n- **PG-12-R1** — Centralize schedule upsert persistence behind an explicit active-state policy (`accepted`, confidence high) — Share only the SQL persistence primitive with explicit preserve/apply-active policy; keep guard and transaction orchestration separate.\n- **PG-14-R1** — Use one typed canonical append request for comparison and persistence (`accepted`, confidence high) — Use one internal canonical append value; retain the tolerant legacy decoder.\n- **PG-14-R2** — Derive strict recovery-field validation from the Serde schema (`accepted`, confidence medium-high) — Strict unknown-field observation applies only to structural recovery structs; payload/metadata remain opaque and tolerant append compatibility remains.\n- **PG-16-R1** — Load and represent only immediately resolvable appended steps (`accepted`, confidence high) — Verified the query decodes every appended payload/candidate then immediately skips pending rows. Filter dependency_count_pending=0, preserve append-input order and born-unsatisfied cancellation; intentionally drop or separately retain impossible missing-row diagnostic.\n- **PG-15-R1** — Replace ambiguous workflow Option scopes with explicit visibility scopes (`accepted`, confidence high) — Distinguish exact Organization, exact Global/NULL, and Admin; never replace one ambiguous Option with another.\n- **PG-20-R1** — Make step insertion own definition-default resolution (`accepted-low-priority`, confidence high) — Fresh review and coordinator source check confirmed callers reclassify Job/External solely to pass a permissive Option that insertion reclassifies again. Preserve the complete locked prevalidation scan and public override Options.\n- **PG-18-R1** — Reconcile recovery data into paired step blueprints during load (`accepted`, confidence high) — Verified RecoveryBlueprint stores mismatched collections and a partially consumed snapshot, then re-proves pairing during build. Destructure run settings and pair canonical/source state once in deterministic key order.\n- **PG-17-R1** — Batch dependency-counter transitions per terminal prerequisite (`accepted`, confidence high) — Batch only direct dependents for each popped prerequisite; preserve BFS order, deterministic locks, and per-candidate effects. Depends on PG-17-R2.\n- **PG-17-R2** — Make terminal propagation and run-status recomputation explicitly single-run (`accepted`, confidence high) — Expose the already schema-enforced single-run invariant; do not introduce a general propagation model.\n- **RT-01-R1** — Make the handler the sole catalog registration identity (`accepted`, confidence high) — Derive identity from the handler with a compatibility/deprecation path for current public registration methods.\n- **RT-03-R1** — Use JoinSet as the single in-flight capacity owner (`accepted`, confidence high) — Preserve join-error logging and shutdown draining when JoinSet becomes the sole capacity owner.\n- **RT-03-R2** — Validate claimed lease ownership instead of inventing unknown-worker (`accepted`, confidence medium) — Reject missing owner at claim/execution boundary and leave the lease for recovery; add no public type.\n- **RT-05-R1** — Make terminal event ObservedJob the sole worker-observer identity (`accepted`, confidence high) — Make TerminalJobObserverEvent authoritative and remove only the parallel &JobQueueRecord argument; depends on RT-04's committed event outcome semantics, not rejected identity restyling.\n- **RT-05-R2** — Couple reaper observer metadata to its JoinSet task outcome (`accepted`, confidence medium) — Narrowed by overlap audit to reaper/observers.rs only: each observer task returns/carries its metadata and catches panic, eliminating the side map and divergence repair without sharing a generic manager.\n- **RT-06-R1** — Centralize dead-letter hook execution policy (`accepted`, confidence high) — Extract only timeout, unwind-catching, and normalized hook outcome; keep fan-out and shutdown orchestration separate.\n- **RT-07-R1** — Bound stale-jitter search by the best candidate instead of a global occurrence count (`accepted`, confidence high) — Use streaming best-candidate search with a sound early bound; retain the occurrence cap defensively.\n- **RT-08-R1** — Use the watch channel as the sole shutdown state (`accepted`, confidence high) — Use send_replace/borrow as authority while preserving receiver-closed-means-shutdown.\n- **RT-08-R2** — Encode supervisor registry selection as one enum state (`accepted`, confidence high) — Verified three private fields admit contradictory states and implement sticky history manually. One Direct/Catalog/Mixed selection preserves same-source last-write-wins, mixed stickiness, config-validation precedence, and public APIs.\n- **TUI-01-R1** — Centralize query-filter transitions and dependent cache invalidation (`accepted`, confidence high) — Use small local transition methods; do not add an application-wide reducer.\n- **TUI-01-R2** — Represent the current screen and view state as one navigation frame (`accepted`, confidence medium-high) — Make the existing ScreenFrame authoritative; do not add routing infrastructure.\n- **TUI-02-R1** — Use one character-indexed truncation algorithm (`accepted`, confidence high) — Verified current code compares char count then uses max_chars as a byte offset; multibyte prefixes truncate too early. Keep Unicode scalar semantics explicit, not grapheme/display-width semantics.\n- **TUI-03-R1** — Make the terminal event loop the sole owner of fetch lifecycle state (`accepted`, confidence high) — Event loop owns lifecycle; workers return generation/outcome/duration values.\n- **TUI-03-R2** — Make job-detail rendering read-only and return only viewport state (`accepted`, confidence high) — Separate immutable detail data from the small viewport; do not use Option::take transient state.\n- **TEST-01-R1** — Make EphemeralDatabase own its permit and immutable identity (`accepted`, confidence high) — Store optional permit and immutable private identity in EphemeralDatabase, with explicit async teardown plus best-effort Drop fallback.\n- **TEST-01-R2** — Use Tokio OnceCell as the single async PostgreSQL initializer (`accepted-low-priority`, confidence high) — Verified OnceLock plus async mutex manually implements double-checked initialization and repeats set/get branches. One async initializer preserves external/owned modes; characterize cancellation/panic and container cleanup.\n- **INF-01-R1** — Synchronize vendored migrations before SQLx preparation (`accepted`, confidence high) — Verified prescribed refresh first compiles the workspace, whose build scripts reject stale vendored migrations before the later repair step. Reorder only; retain fail-fast drift enforcement and visible partial state if prepare fails.\n- **INF-01-R2** — Enforce and record PostgreSQL 18 in the SQLx refresh command (`accepted`, confidence medium) — Enforce PostgreSQL 18 and current migrations before prepare; avoid a provenance framework unless one synchronized marker proves necessary.\n- **INF-02-R1** — Make the lockstep release version one workspace package value (`accepted-low-priority`, confidence high) — Verified one release version is repeated in five package manifests plus four registry pins and bumped through separate loops. Use workspace.package inheritance for member versions; retain explicit published path-dependency pins and equality checks.\n- **INF-05-R1** — Make runledger-test-support the sole packaged-smoke PostgreSQL harness owner (`accepted`, confidence high) — Fresh bounded review and coordinator verification confirmed exact duplication and existing packaged test-support dependency. Use setup_unmigrated_ephemeral_pool so packaged runledger-postgres migration invocation remains under test; remove only the local harness and direct testcontainers edge.\n- **PG-20-R2** — Give workflow-step persistence sole ownership of dependency writes (`accepted`, confidence high) — Fresh review and coordinator source check confirmed enqueue and append duplicate dependency traversal/key lookup/release-mode persistence. Generalize the private helper to &[WorkflowStepEnqueue] while preserving diagnostic context and ordering.\n- **INF-04-R2** — Make the remote tag authoritative for final publication state (`accepted`, confidence high) — Fresh review and coordinator source check confirmed final push failure leaves a local-only tag that blocks the script's otherwise resumable publish flow. Push HEAD directly to branch and remote tag atomically; create a local tag only after success if desired.\n\n## Rejected, demoted, and superseded candidates\n\n- **CORE-02-R1** — Preserve owned String buffers in identifier conversions (`rejected-materiality`) — Pure allocation/type-path optimization; preserving the buffer creates a second validation construction path without material organizing benefit.\n- **CORE-03-R2** — Make dead-letter max_attempts non-optional (`demoted-breaking-cleanup`) — Schema audit confirmed the invariant but found only two Some wrappers and no meaningful branching. Keep as opportunistic next-breaking-release cleanup, not a material audit recommendation.\n- **CORE-01-R1** — Glob-reexport all jobs from core prelude (`rejected`)\n- **CORE-04-R1** — Remove WorkflowDagBuilder StepSlot's duplicate key/temporary Option (`rejected-materiality`) — StepSlot's Option is a bounded take/restore ownership mechanism; removal needs placeholders or more complex mutable ownership, while the duplicate key supports stable reconstruction.\n- **CORE-05-R1** — Validate built workflow steps directly without permissive DTO projection (`rejected-materiality`) — The projection DTO is a shared adapter for two input representations; direct validation would duplicate the algorithm or replace it with generic complexity.\n- **PG-04-R2** — Make the complete definition-disable guard one owned operation (`rejected-materiality`) — The lock protocol is already centralized; CRUD and catalog sync intentionally differ in ordering/error mapping, and a transaction-owning guard cannot prevent arbitrary SQL.\n- **RT-09-R1** — Return the registry's actual static identifier lifetime (`rejected-materiality`) — Although type-correct and locally clean, this removes only a five-line lifetime wrapper at two call sites; it is public type polish below this audit's materiality threshold.\n- **RT-01-R2** — Stop mirroring the PostgreSQL definition-sync error hierarchy (`rejected-materiality`) — The runtime error hierarchy is an intentional semantic boundary; transparent persistence errors would leak storage semantics or relocate exhaustive mapping to callers.\n- **RT-04-R1** — Build continuation observer identity from the committed outcome (`rejected-materiality`) — Claim identity already flows through the same exact lease tuple into the committed outcome; reconstructing ObservedJob from that outcome is stylistic consistency, not structural simplification.\n- **TUI-02-R2** — Compose dashboard queue and continuation metrics once per job type (`rejected-materiality`) — The present composition is two stores plus one lookup; a composed metrics model relocates a tiny join and risks losing continuation-only job types.\n- **INF-04-R1** — Give shared release topology and pure policy one side-effect-free module (`rejected-materiality`) — Fresh exact-boundary review found two short arrays in agreement; sourcing a shell policy file would add implicit-global/path coupling without removing Cargo-manifest ownership.\n- **RT-06-R2** — Couple reaper terminal-hook metadata to its JoinSet task outcome (`rejected-semantics`) — Superseded after cancellation tracing: terminal-hook side metadata intentionally retains per-job identity when JoinSet tasks are aborted/cancelled. Task-returned metadata would be unavailable, losing diagnostics or recreating a side channel.\n\n## Cross-cutting patterns\n\n1. Correlated fields represented independently: intent lifecycle, terminal outcome, retry selection, scopes, supervisor registry source, and TUI navigation/fetch state.\n2. Policy with multiple owners: retry eligibility, failure metadata, schedule persistence, hook invocation, timeout representation, and dependency defaults.\n3. One relationship/state stored twice: workflow-step/job linkage, semaphore plus JoinSet, task metadata mirrors, and shutdown mirrors.\n4. Persistence orchestration leaking into callers: workflow defaults, dependency writes, absent-definition computation, and canonical append conversion.\n5. Repeated or unbounded work: duplicate claim SQL, per-edge transitions, unnecessary step decoding, and cron jitter scans.\n6. Infrastructure provenance drift: SQLx refresh order, PostgreSQL baseline, release tag state, package versions, and duplicate smoke harnesses.\n\nThe final overlap pass found no duplicate accepted IDs. Similar scope types remain nominally separate because read visibility and destructive capability have different semantics. Observer and terminal-hook task managers also remain separate because abort diagnostics differ.\n\n## Priority tiers\n\n- **Tier A — first slices:** `INF-01-R1`, `INF-01-R2`, `CORE-03-R1`, `RT-03-R2`, `TUI-02-R1`, `RT-07-R1`, `INF-04-R2`, `PG-05-R2`, `RT-08-R1`\n- **Tier B — internal consolidation:** `PG-01-R2`, `PG-07-R1`, `PG-12-R1`, `PG-08-R2`, `RT-06-R1`, `TUI-03-R1`, `TUI-03-R2`, `PG-16-R1`, `PG-17-R2`, `RT-03-R1`, `RT-05-R1`, `RT-05-R2`, `PG-20-R2`, `PG-20-R1`, `PG-14-R2`, `PG-06-R2`, `PG-05-R1`, `PG-08-R1`, `TUI-01-R1`, `TEST-01-R1`, `INF-05-R1`\n- **Tier C — staged/evidence-gated:** `PG-11-R1`, `PG-15-R1`, `PG-17-R1`, `PG-18-R1`, `PG-01-R1`, `PG-02-R1`, `PG-02-R2`, `RT-01-R1`, `PG-03-R2`, `PG-03-R1`, `PG-06-R1`, `PG-11-R2`\n- **Tier D — opportunistic fold-ins:** `CORE-04-R2`, `PG-04-R1`, `PG-14-R1`, `RT-08-R2`, `TUI-01-R2`, `TEST-01-R2`, `INF-02-R1`\n\n## Dependency and evidence gates\n\n- INF-01-R1 -> INF-01-R2\n- PG-11-R1 -> PG-11-R2\n- PG-17-R2 -> PG-17-R1\n- CORE-04-R2 -> PG-20-R1 + PG-20-R2 (preferred fold-in)\n- TUI-03-R1 -> TUI-01-R1 and TUI-03-R2\n- PG-14-R2 -> PG-18-R1 (preferred)\n- CORE-03-R1 -> PG-08-R2 (preferred)\n- RT-03-R2 -> PG-06-R2 (preferred)\n- TEST-01-R1 -> INF-05-R1 (preferred)\n- PG-06-R1: PostgreSQL 18 custom/generic plan and throughput parity\n- PG-03-R2: PostgreSQL 18 plan/equivalence\n- PG-17-R1: PostgreSQL 18 plan, lock-order, rollback, and concurrency\n- PG-03-R1: staged mixed-version schema rollout\n- PG-11-R2 and legacy cutovers: breaking release\n\n## Audit log\n\n1. 2026-08-22: confirmed clean initial git status\n2. read all repository AGENTS guides and Rust audit skill references\n3. inventoried 783 tracked files, 209 Rust files, 78 SQL migration files, 456 SQLx metadata files\n4. ran read-only Fowler heuristic scanner over all 209 Rust files (1000-candidate summary)\n5. ran read-only duplicate-abstraction scanner over all 209 Rust files including tests, then production-only summary\n6. froze 38 non-overlapping subsystem rows; CORE-01..03 opened in first review batch\n7. Harvested CORE-01..03; coordinator rejected CORE-01 prelude glob, accepted CORE-02 owned-string conversion and CORE-03 retry policy; dead-letter optionality awaits materiality pass.\n8. Harvested CORE-04/05 and PG-01; source-verified five candidates and opened PG-02..04.\n9. Coordinator path-to-row check found unassigned runledger-postgres/src/jobs/queue.rs; added explicit PG-19 rather than broadening an existing boundary. All 209 Rust files now assigned.\n10. PG-04 independently verified: accepted R1 (computed absent-definition plan) and R2 (owned definition-disable guard protocol); no commands executed beyond source inspection.\n11. PG-06 independently verified: accepted R2; accepted R1 conditionally on PostgreSQL 18 plan parity because nullable-filter consolidation can change generic/custom plans.\n12. PG-05 independently verified: accepted private keyed/unkeyed enqueue preparation and exhaustive promotion failure action; public request/schema shapes remain unchanged.\n13. PG-07 independently verified: accepted explicit mark-running versus stage-free progress commands; assigned to PG-07 behavioral ownership, with PG-02 public DTO touched as a dependency.\n14. PG-09 independently verified skip: typed event payload enums already own variant shape; nullable audit columns are independently meaningful; log persistence is straight-line and cross-query abstraction would add complexity.\n15. PG-08 independently verified: accepted typed failure metadata authority and discriminated retry selection; CORE-03 retryability predicate recorded as duplicate/cross-owner, not repeated.\n16. PG-10 independently verified skip: repeated job projections preserve SQLx checking, payload candidate is a one-use ordered classifier, metric API split is intentional compatibility, and timeout pairs are owned by PG-01.\n17. PG-11 independently verified: accepted exact cancellation scope and removal of legacy requeue_job at a breaking release; recovery/replay lock flows remain separate.\n18. PG-12 independently verified: accepted shared schedule-upsert persistence with explicit PreserveStored/ApplyRequested policy; guard flows remain distinct.\n19. PG-13 independently verified skip: straightforward CRUD; conflict policies are genuinely distinct; helper/general-executor or raw-row abstraction would add structure without reducing material complexity.\n20. PG-14 independently verified: accepted typed canonical append source and schema-derived strict recovery decoding; append idempotency remains tolerant of additive legacy fields.\n21. PG-16 independently verified: accepted ready-only appended-step load/model; PG-14 canonical round-trip remains authoritative despite PG-16 noting its current incidental compatibility check, because tests can own that invariant.\n22. PG-15 independently verified: accepted explicit workflow scope representation and authoritative defaults map at insertion; scope API requires compatibility staging and tenant-leakage matrix.\n23. PG-19 independently verified skip: 45-line declarative visibility/export facade; globs or exposing submodules would weaken public API ownership without behavioral simplification.\n24. PG-18 independently verified: accepted paired recovery step blueprints; preserve length-before-key error precedence and deterministic BTreeMap order.\n25. PG-17 independently verified: accepted singular run-scoped propagation first, then set-based dependency counter transition; concurrency/order validation is a PostgreSQL 18 prerequisite.\n26. RT-02 independently verified: accepted actual 'static registered-type lifetime and removal of duplicate private accessor; low blast, ownership-focused.\n27. RT-01 independently verified: accepted handler-derived catalog identity and source-preserving definition-sync error ownership; both need staged public migration, error wrapping lower confidence due match ergonomics.\n28. RT-04 independently verified: accepted committed continuation outcome as observer identity source; local, no public/schema change.\n29. RT-03 independently verified: accepted JoinSet-only capacity ownership and validated claimed-owner phase; no public API changes.\n30. RT-05 independently verified: accepted event-owned terminal identity and outcome-coupled reaper observer metadata; RT-04 precedes R1.\n31. RT-06 independently verified: accepted shared dead-letter hook execution policy; reaper fanout/task ownership remains RT-05, so duplicate task-lifecycle suggestions were not repeated.\n32. RT-07 independently verified: accepted candidate-bounded stale-jitter search; rejected generic scheduler/promoter loop and savepoint abstractions as intentional semantic differences.\n33. RT-08 independently verified: accepted watch-owned shutdown state and one registry-selection state; left TaskGroup's deliberately distinct concurrency lifecycle unchanged.\n34. TUI-01 independently verified: accepted targeted filter/cache transitions and unified current navigation frame; no generic key-command or filtered-index abstraction accepted.\n35. TUI-02 independently verified: accepted character-indexed truncation and one composed dashboard metric model; Scope and FetchRequest remain intentionally simple.\n36. TUI-03 independently verified: accepted event-loop-owned revisioned fetch lifecycle and clone-free job-detail rendering; bounded table/render scans remain local.\n37. TEST-01 independently verified: accepted EphemeralDatabase-owned lifecycle permit/identity and Tokio OnceCell initialization; all future database validation remains PostgreSQL 18 with server version recorded.\n38. INF-01 independently verified: accepted migration-before-prepare ordering and command-owned PostgreSQL 18 provenance check; generated JSON and independent package/build guards remain untouched.\n39. INF-02 independently verified: accepted workspace-owned lockstep version and side-effect-free shared release policy; CI test isolation, per-package licenses, and remote attestation remain intentionally separate.\n40. INF-03 independently verified: accepted shared test-support ownership of smoke database provisioning while preserving packaged PostgreSQL migration ownership and external-consumer behavior.\n41. Independent coverage audit found five hidden ownership boundaries. Coverage contract revised from 39 to 44 rows: added PG-20 shared workflow-step persistence, RT-09 handler registry, RT-10 intent promoter, INF-04 release automation, and INF-05 packaged downstream-consumer compatibility.\n42. Coverage ownership corrections: workflow_results -> PG-15; runtime idempotency test -> PG-05; workflow cancel lock-order test -> PG-17; worker terminal-hook tests -> RT-04; prelude smoke -> RT-08; TUI data/presentation files separated. Candidates PG-15-R2, RT-02-R1, INF-02-R2, and INF-03-R1 reassigned to new authoritative rows pending fresh bounded review.\n43. Independent schema-completeness pass accepted narrowed PG-03-R1 (workflow_steps.job_id authoritative) and PG-03-R2 (local NOT MATERIALIZED validity CTE), demoted CORE-03-R2 to opportunistic breaking-release cleanup, and repaired state-shape requirements for PG-02-R1/R2, PG-07-R1, PG-08-R2, and PG-11-R1.\n44. Schema pass conditions: PG-06-R1 remains conditional on PostgreSQL 18 custom/generic-plan evidence; PG-17-R1 depends on PG-17-R2; RT-05-R1 depends on RT-04-R1; query-changing validation must name PostgreSQL 18 and synchronized SQLx caches.\n45. RT-09 fresh bounded review completed: accepted only the actual-static-lifetime registry API; rejected nested-map/entry rewrites as minor or behavior-complicating.\n46. PG-20 fresh bounded review and coordinator verification completed: accepted insertion-owned default resolution and a shared private dependency-row persistence helper; both remain internal and preserve prevalidation/transaction ordering.\n47. RT-10 fresh bounded review and coordinator source verification completed: explicit skip. The boolean selects exactly wait versus immediate drain, precise config is already authoritative, and cancellation/backlog semantics are directly covered.\n48. INF-04 fresh bounded review rejected shared shell topology as below materiality and accepted remote-tag-authoritative publication state; coordinator verified the preflight already uses the recoverable direct refspec while final execution creates an unrecoverable local-only state first.\n49. Independent materiality/over-abstraction pass rejected CORE-02-R1, CORE-04-R1, CORE-05-R1, PG-04-R2, RT-01-R2, RT-04-R1, RT-09-R1, and TUI-02-R2; affirmed CORE-01-R1 rejection. CORE-02, CORE-05, RT-04, and RT-09 therefore close as explicit skips.\n50. Materiality pass retained PG-03-R1 as long-horizon, PG-20-R1/TEST-01-R2/INF-02-R1 as low-priority, and CORE-03-R2 as non-material breaking cleanup. All surviving abstractions were narrowed to their smallest state/ownership boundary.\n51. INF-05 fresh bounded review and coordinator verification completed: accepted packaged test-support as sole PostgreSQL infrastructure owner while deliberately retaining the packaged runledger-postgres migration call and isolated archive workflow.\n52. PG-03 closed as recommend after schema/materiality validation; RT-02 closes as explicit config skip after registry extraction left no surviving config finding.\n53. Overlap audit found one ownership defect: split former cross-boundary RT-05-R2. RT-05-R2 now owns observer-task metadata only; new RT-06-R2 owns terminal-hook task metadata. No generic task manager is proposed because admission/shutdown policies differ; RT-06-R1 precedes RT-06-R2.\n54. Overlap audit superseded the provisional RT-06-R2 split after tracing cancellation: terminal-hook metadata map is intentional because aborted tasks cannot return identity. RT-05-R2 remains narrowly valid for observers, whose shutdown only aggregates cancellation counts. Survivor count returns to 49.\n55. Fresh duplication/ownership pass closed: 49 surviving IDs are deduplicated after narrowing RT-05-R2 to observer tasks; terminal-hook side metadata is an explicit semantic skip. Related clusters were recorded as dependency edges rather than forced shared abstractions.\n56. Fresh dependency-priority pass ranked all 49 survivors: 9 Tier A first slices, 21 Tier B internal consolidations, 12 Tier C staged/evidence-backed changes, and 7 Tier D fold-ins. PostgreSQL 18 gates and breaking/staged rollout constraints are explicit.\n57. Final repository integrity check: git status --short, unstaged/staged diff stats, and unstaged/staged diff --exit-code were all empty/successful. No tests, builds, scripts, database commands, edits, commits, pushes, or remote operations were run.\n\n## Integrity\n\nFinal staged and unstaged status/diff checks were empty. The only changes made after the audit are the Beads records requested by the user.","acceptance_criteria":"This epic remains open until its accepted child tasks are implemented or explicitly closed. The audit-storage operation itself is complete when:\n- all 49 accepted recommendations exist as child issues with stable audit IDs;\n- each child records exact evidence, current complexity, proposed representation, smallest scope, risks/migration, existing validation, additional validation, and confidence;\n- hard blocking dependencies are represented without cycles;\n- rejected findings and subsystem skips remain preserved in this epic;\n- `br sync --flush-only` succeeds;\n- repository changes are limited to `.beads/`.","notes":"# Coverage interface, call-site, and test index\n\n- **CORE-01 (skip)** — JobHandler, JobContext, completion/failure contracts and prelude; consumed by runtime registry/catalog/worker and examples; inline contract/compile coverage.\n- **CORE-02 (skip)** — JobType, WorkflowType, StepKey and owned/borrowed identifiers; used by every crate; macro parse/serde/equality tests.\n- **CORE-03 (recommend)** — Status/failure enums and runtime/dead-letter DTOs; consumed by PostgreSQL row conversion and runtime worker/observer paths; core, retry and terminal-hook tests.\n- **CORE-04 (recommend)** — WorkflowRunEnqueue, step/dependency specs and builders; called by PostgreSQL enqueue/append and external consumers; builder/error tests.\n- **CORE-05 (skip)** — Shared DAG/step validation used by builders and persistence entry points; dedicated DAG regressions.\n- **PG-01 (recommend)** — DbPool, DbTx, Error, QueryError and transaction wrappers/settings; used throughout PostgreSQL and runtime; owned transaction tests.\n- **PG-02 (recommend)** — Public job/workflow records, mutation inputs, scopes and row decoding; consumed by runtime, TUI, admin and downstream callers.\n- **PG-03 (recommend)** — Embedded migrators and canonical schema; called by test support, examples and packaged smoke; migration regressions.\n- **PG-04 (recommend)** — Definition CRUD/catalog synchronization and disable guard; called by runtime catalog synchronization; catalog/schedule tests.\n- **PG-05 (recommend)** — Enqueue/idempotency APIs and intent promotion; called by applications, promoter, workflow paths and smoke consumer; enqueue/intent/idempotency tests.\n- **PG-06 (recommend)** — Claim/release and resource-fencing APIs; called by worker and reaper; lease/resource tests.\n- **PG-07 (recommend)** — Live-lease heartbeat/progress/success/continuation updates; called by worker execution/completion; completion/progress/continuation tests.\n- **PG-08 (recommend)** — Failure/retry/reaper transitions; called by worker and runtime reaper; retry/reaper tests.\n- **PG-09 (skip)** — Typed audit events and job logs; consumed by runtime/TUI and lifecycle operations; lifecycle integration coverage.\n- **PG-10 (skip)** — Administrative listing, metrics and payload CAS; consumed by TUI and applications; payload/pagination tests.\n- **PG-11 (recommend)** — Cancel, compare-and-requeue, deprecated requeue and replay APIs; used by admin callers and smoke; recovery/replay tests.\n- **PG-12 (recommend)** — Schedule CRUD/catalog sync/claim APIs; called by scheduler and catalog; schedules tests.\n- **PG-13 (skip)** — Persisted runtime configuration CRUD consumed by runtime applications.\n- **PG-14 (recommend)** — Workflow enqueue, canonical snapshot and compatibility decoding; used by applications, recovery and smoke; idempotency/recovery coverage.\n- **PG-15 (recommend)** — Workflow read/list/wait/result handles; used by TUI and downstream applications; ordering/results tests.\n- **PG-16 (recommend)** — Append/cancel mutation APIs; used by downstream applications and recovery tests.\n- **PG-17 (recommend)** — Internal workflow execution state machine; called by queue lifecycle, worker and reaper; active-claim/results/lock-order tests.\n- **PG-18 (recommend)** — Workflow reconstruction/recovery API and blueprint model; workflow_recovery tests.\n- **PG-19 (skip)** — Declarative public visibility/re-export boundary for queue APIs; downstream compile/prelude consumers.\n- **PG-20 (recommend)** — Private step/dependency persistence called by PG-14 enqueue and PG-16 append; workflow persistence/recovery tests.\n- **RT-01 (recommend)** — JobCatalog registration and persistence synchronization; called by supervisor/examples; catalog tests.\n- **RT-02 (skip)** — Worker/reaper/scheduler/promoter configuration and validation; config_validation tests.\n- **RT-03 (recommend)** — Worker loop, claim admission and handler execution; started by supervisor; worker-loop/lease/prestart tests.\n- **RT-04 (skip)** — Completion orchestration and worker-side dead-letter hooks; called by execution; completion/terminal-hook tests.\n- **RT-05 (recommend)** — Observer traits/events, worker dispatch and reaper observer tasks; observer task/reaper tests.\n- **RT-06 (recommend)** — Reaper lifecycle and reaper-side terminal hooks; started by supervisor; reaper-loop tests.\n- **RT-07 (recommend)** — Schedule claiming, cron evaluation and enqueue loop; started by supervisor; scheduler tests.\n- **RT-08 (recommend)** — Supervisor, runtime loop exits, task groups and shutdown API; public embedding boundary; supervisor/prelude tests.\n- **RT-09 (skip)** — JobRegistry handler lookup and retry overrides; called by catalog, worker, reaper, promoter and supervisor; registry tests.\n- **RT-10 (skip)** — Public promoter loop and precise/broad config adapters; called by supervisor/custom orchestration; worker/config tests.\n- **TUI-01 (recommend)** — App navigation/filter/search state and input commands; used by terminal/UI; input/search/selection coverage.\n- **TUI-02 (recommend)** — Fetch requests, database models and tenant scope; used by event loop/rendering; data/fetch coverage.\n- **TUI-03 (recommend)** — Binary startup, terminal event loop, render tree and formatting utilities; format/render behavior.\n- **TEST-01 (recommend)** — Public ephemeral database/container APIs used by every database-backed suite and packaged consumers; lifecycle tests.\n- **INF-01 (recommend)** — Three SQLx caches, vendored migration contract, build guards and refresh command.\n- **INF-02 (recommend)** — Workspace/package metadata, CI, lint, licenses and repository policy.\n- **INF-03 (skip)** — Passive documentation, executable examples and public consumer guidance.\n- **INF-04 (recommend)** — Release preparation, CI verification, irreversible publication and tag workflow.\n- **INF-05 (recommend)** — Isolated workspace consuming extracted .crate archives; invoked by CI/release preparation.\n\n# Dependency-aware evaluation matrix\n\n- **INF-01-R1** — impact / confidence / effort / blast: H / H / S / L; none\n- **INF-01-R2** — impact / confidence / effort / blast: H / M / S / L; blocked by INF-01-R1\n- **CORE-03-R1** — impact / confidence / effort / blast: H / H / S-M / M; none\n- **RT-03-R2** — impact / confidence / effort / blast: H / M / S / L-M; none\n- **TUI-02-R1** — impact / confidence / effort / blast: M / H / S / L; none\n- **RT-07-R1** — impact / confidence / effort / blast: H worst-case / H / M / L; none\n- **INF-04-R2** — impact / confidence / effort / blast: H / H / S / M; none\n- **PG-05-R2** — impact / confidence / effort / blast: M-H / H / S-M / M; related to PG-01-R1\n- **RT-08-R1** — impact / confidence / effort / blast: M / H / S / L; none\n- **PG-01-R2** — impact / confidence / effort / blast: M-H / H / M / M; none\n- **PG-07-R1** — impact / confidence / effort / blast: H / H / M / M; compatibility staging\n- **PG-12-R1** — impact / confidence / effort / blast: M-H / H / M / M; PostgreSQL 18/SQLx prerequisite\n- **PG-08-R2** — impact / confidence / effort / blast: M-H / H / M / M; prefer CORE-03-R1\n- **RT-06-R1** — impact / confidence / effort / blast: M-H / H / M / M; coordinate RT-04 call site\n- **TUI-03-R1** — impact / confidence / effort / blast: M-H / H / M / M; none\n- **TUI-03-R2** — impact / confidence / effort / blast: M / H / M / L-M; blocked by TUI-03-R1\n- **PG-16-R1** — impact / confidence / effort / blast: M / H / S-M / M; PostgreSQL 18/SQLx prerequisite\n- **PG-17-R2** — impact / confidence / effort / blast: M-H / H / M / M; before PG-17-R1\n- **RT-03-R1** — impact / confidence / effort / blast: M-H / H / M / M; none\n- **RT-05-R1** — impact / confidence / effort / blast: M / H / M / M-H; preserve commit-before-observe\n- **RT-05-R2** — impact / confidence / effort / blast: M / M / M / M; observer-only narrowing\n- **PG-20-R2** — impact / confidence / effort / blast: M / H / M / M; related to CORE-04-R2\n- **PG-20-R1** — impact / confidence / effort / blast: L-M / H / S / L; blocked by/co-land with PG-20-R2\n- **PG-14-R2** — impact / confidence / effort / blast: H / M-H / M / M; preferably before PG-18-R1\n- **PG-06-R2** — impact / confidence / effort / blast: M / H / M / M; related to RT-03-R2\n- **PG-05-R1** — impact / confidence / effort / blast: M / H / M / M; none\n- **PG-08-R1** — impact / confidence / effort / blast: M / H / M / M; none\n- **TUI-01-R1** — impact / confidence / effort / blast: M / H / M / L; blocked by TUI-03-R1\n- **TEST-01-R1** — impact / confidence / effort / blast: M / H / M / L; preferably before INF-05-R1\n- **INF-05-R1** — impact / confidence / effort / blast: M / H / S-M / L; related to TEST-01-R1\n- **PG-11-R1** — impact / confidence / effort / blast: H / H / M / H; additive API before cutover\n- **PG-15-R1** — impact / confidence / effort / blast: H / H / M / H; additive API before cutover\n- **PG-17-R1** — impact / confidence / effort / blast: H / M / L / H; blocked by PG-17-R2 and PostgreSQL 18/SQLx work\n- **PG-18-R1** — impact / confidence / effort / blast: H / M-H / L / H; related to PG-14-R2\n- **PG-01-R1** — impact / confidence / effort / blast: M-H / H / M / H; preserve generic/public constructor compatibility\n- **PG-02-R1** — impact / confidence / effort / blast: H / H / M / H; public compatibility phase\n- **PG-02-R2** — impact / confidence / effort / blast: M-H / H / M / H; public compatibility phase\n- **RT-01-R1** — impact / confidence / effort / blast: M / H / M / H; deprecation/compatibility path\n- **PG-03-R2** — impact / confidence / effort / blast: M / M / M / M; PostgreSQL 18 plan equivalence and SQLx prerequisite\n- **PG-03-R1** — impact / confidence / effort / blast: H / M-H / XL / XL; staged schema rollout and SQLx prerequisite\n- **PG-06-R1** — impact / confidence / effort / blast: M / M / M / H; conditional PostgreSQL 18 plan/throughput parity\n- **PG-11-R2** — impact / confidence / effort / blast: M / H / S / H; blocked by PG-11-R1 and breaking release\n- **CORE-04-R2** — impact / confidence / effort / blast: L-M / H / S / L-M; fold into PG-20 work\n- **PG-04-R1** — impact / confidence / effort / blast: L-M / H / S / L; next catalog-sync change\n- **PG-14-R1** — impact / confidence / effort / blast: L-M / H / S / L; PG-14 codec work\n- **RT-08-R2** — impact / confidence / effort / blast: L-M / H / S / L; next supervisor edit\n- **TUI-01-R2** — impact / confidence / effort / blast: L-M / M-H / M / L-M; TUI state work\n- **TEST-01-R2** — impact / confidence / effort / blast: L / H / S / L; test-harness work\n- **INF-02-R1** — impact / confidence / effort / blast: L / H / S-M / M; next version/release tooling change\n\n# Graph semantics\n\n- Blocking edges in Beads represent implementation order that should affect `br ready`.\n- Softer coordination edges use the `related` dependency type and do not imply a shared abstraction.\n- Conditional PostgreSQL findings remain open but must be rejected/closed without implementation if their evidence gate fails.\n- The parent-child links attach all 49 actionable tasks to the canonical audit epic.","status":"closed","priority":1,"issue_type":"epic","created_at":"2026-08-22T20:49:48.486233Z","created_by":"aa","updated_at":"2026-08-24T09:48:21.092117Z","closed_at":"2026-08-24T09:48:21.091991Z","close_reason":"Completed all 49 accepted simplification recommendations. The final INF-02-R1 change established one workspace-owned package version, preserved explicit published dependency pins, and passed release, packaging, lockfile, workspace, external-consumer, and PostgreSQL 18 validation.","external_ref":"audit:runledger:2026-08-22","source_repo":"runledger","source_repo_path":"/Users/aa/Documents/runledger","compaction_level":0,"original_size":0,"labels":["canonical-report","read-only-audit","simplification-audit"]} {"id":"runledger-runledger-simplification-audit-w2f.1","title":"CORE-03-R1: Centralize JobFailureKind retry eligibility","description":"Audit ID: CORE-03-R1\nVerdict: recommend\nPriority tier: A\nConfidence: high\n\n## Evidence\nrunledger-core/src/jobs/status.rs:148-169; runledger-postgres/src/jobs/queue/lifecycle/failure.rs:84-105; runledger-runtime/src/worker/completion.rs:654-656.\n\n## Current complexity or invalid states\nJobFailureKind owns the semantic vocabulary, but PostgreSQL and runtime independently match variants to decide retryability, so policy can drift.\n\n## Proposed representation\nAdd one exhaustive JobFailureKind retry-eligibility predicate and replace the duplicated matches. Do not introduce a policy object.\n\n## Smallest credible implementation scope\nrunledger-core/src/jobs/status.rs and the two PostgreSQL/runtime call sites, plus focused tests.\n\n## Regression risks and migration concerns\nPreserve current handling of Timeout, Panicked, LeaseExpired, Terminal, and Retryable; no wire-format change.\n\n## Existing validation\nQueue retry-delay and worker completion coverage.\n\n## Additional validation required\nAdd an exhaustive core test for every JobFailureKind and cross-layer regressions.\n\nParent audit: runledger-runledger-simplification-audit-w2f.","acceptance_criteria":"- Implement only the bounded representation described above.\n- Preserve the stated compatibility and semantic constraints.\n- Add the required validation; database-backed work must use PostgreSQL 18 and record server_version/server_version_num.\n- If SQL/schema metadata changes, keep all three SQLx caches synchronized.\n- Close with a concise implementation and validation summary.","status":"closed","priority":1,"issue_type":"task","assignee":"aa","created_at":"2026-08-22T20:50:47.599230Z","created_by":"aa","updated_at":"2026-08-22T22:30:38.896534Z","closed_at":"2026-08-22T22:30:38.896359Z","close_reason":"Added an exhaustive JobFailureKind::is_retryable policy, replaced PostgreSQL and runtime duplicate matches, and added all-variant core and persistence regressions. Validated core, PostgreSQL, and runtime suites plus strict Clippy; PostgreSQL-backed coverage ran on 18.4 (server_version_num 180004).","external_ref":"audit:CORE-03-R1","source_repo":"runledger","source_repo_path":"/Users/aa/Documents/runledger","compaction_level":0,"original_size":0,"labels":["audit-core-03-r1","core","simplification-audit","tier-a"],"dependencies":[{"issue_id":"runledger-runledger-simplification-audit-w2f.1","depends_on_id":"runledger-runledger-simplification-audit-w2f","type":"parent-child","created_at":"2026-08-22T20:50:47.599230Z","created_by":"aa","metadata":"{}","thread_id":""}]} {"id":"runledger-runledger-simplification-audit-w2f.10","title":"PG-05-R1: Represent prepared enqueue as keyed or unkeyed","description":"Audit ID: PG-05-R1\nVerdict: recommend\nPriority tier: B\nConfidence: high\n\n## Evidence\nrunledger-postgres/src/jobs/queue/enqueue.rs:239-317; runledger-postgres/src/jobs/queue/intents.rs:73-143,872-907.\n\n## Current complexity or invalid states\nCanonical request state is conditionally correlated with an optional idempotency key and later branches assume that relationship.\n\n## Proposed representation\nUse a private PreparedEnqueue::Keyed | Unkeyed representation. Do not expose a public hierarchy or change request/schema shapes.\n\n## Smallest credible implementation scope\nQueue enqueue preparation and intent call sites only.\n\n## Regression risks and migration concerns\nPreserve dynamic SQL bind order, conflict outcomes, and canonical bytes.\n\n## Existing validation\nenqueue_outcome, idempotency, and intent integration tests.\n\n## Additional validation required\nTest that keyed requests always carry canonical comparison state and unkeyed requests never do, on PostgreSQL 18.\n\nParent audit: runledger-runledger-simplification-audit-w2f.","acceptance_criteria":"- Implement only the bounded representation described above.\n- Preserve the stated compatibility and semantic constraints.\n- Add the required validation; database-backed work must use PostgreSQL 18 and record server_version/server_version_num.\n- If SQL/schema metadata changes, keep all three SQLx caches synchronized.\n- Close with a concise implementation and validation summary.","status":"closed","priority":2,"issue_type":"task","created_at":"2026-08-22T20:50:49.099151Z","created_by":"aa","updated_at":"2026-08-23T08:42:26.310766Z","closed_at":"2026-08-23T08:42:26.310620Z","close_reason":"Implemented private PreparedEnqueue::Keyed { idempotency_key, enqueue_request } | Unkeyed representation; routed dynamic SQL binds, conflict clauses, and existing-row resolution through it without changing request/schema shapes or bind order. Added unit and PostgreSQL 18.4 persistence coverage for keyed/unkeyed correlation; enqueue outcome, intent convergence/conflict, cargo check, and clippy validations passed.","external_ref":"audit:PG-05-R1","source_repo":"runledger","source_repo_path":"/Users/aa/Documents/runledger","compaction_level":0,"original_size":0,"labels":["audit-pg-05-r1","postgresql","simplification-audit","tier-b"],"dependencies":[{"issue_id":"runledger-runledger-simplification-audit-w2f.10","depends_on_id":"runledger-runledger-simplification-audit-w2f","type":"parent-child","created_at":"2026-08-22T20:50:49.099151Z","created_by":"aa","metadata":"{}","thread_id":""}]} @@ -63,4 +77,7 @@ {"id":"runledger-runledger-simplification-audit-w2f.7","title":"PG-03-R1: Make workflow_steps.job_id the sole stored workflow-step/job relationship","description":"Audit ID: PG-03-R1\nVerdict: recommend\nPriority tier: C\nConfidence: medium-high\n\n## Evidence\nrunledger-postgres/migrations/202603280001_runledger_baseline.up.sql:362-455,484-584; runledger-postgres/src/jobs/workflows/release.rs:259-309; runtime/terminal.rs:135-188.\n\n## Current complexity or invalid states\nThe relationship is stored in reciprocal nullable columns and defended by two unique/FK sets, deferred triggers, dual writes, and runtime disagreement checks.\n\n## Proposed representation\nRetain workflow_steps.job_id UNIQUE REFERENCES job_queue(id) ON DELETE SET NULL and derive reverse lookup. Eventually remove job_queue.workflow_step_id and symmetry machinery.\n\n## Smallest credible implementation scope\nNew staged migrations in all mirrors; workflow release/terminal/locking/mutation, queue lifecycle/admin/replay readers, classifiers, and all SQLx caches.\n\n## Regression risks and migration concerns\nXL schema and concurrency blast radius, reverse-lookup plans, lock order, external SQL users, and mixed-version binaries. Never rewrite the historical baseline.\n\n## Existing validation\nBroad workflow release, continuation, cancel, recovery, results, admin, replay, and lock-order coverage; no dedicated symmetry regression.\n\n## Additional validation required\nStage audit/backfill, dual-compatible readers/writers, anti-joins, one-to-one/delete tests, mixed-version rollout tests, and PostgreSQL 18 EXPLAIN comparisons; record version and sync all caches.\n\nParent audit: runledger-runledger-simplification-audit-w2f.","acceptance_criteria":"- Implement only the bounded representation described above.\n- Preserve the stated compatibility and semantic constraints.\n- Add the required validation; database-backed work must use PostgreSQL 18 and record server_version/server_version_num.\n- If SQL/schema metadata changes, keep all three SQLx caches synchronized.\n- Close with a concise implementation and validation summary.","status":"closed","priority":3,"issue_type":"task","created_at":"2026-08-22T20:50:48.595934Z","created_by":"aa","updated_at":"2026-08-24T08:25:51.402357Z","closed_at":"2026-08-24T08:25:51.402246Z","close_reason":"Implemented staged workflow-step/job relationship cutover: expand audit/backfill plus one-way mixed-version projection, current-binary expand-schema guard, fenced contract removal of job_queue.workflow_step_id/FK/unique/symmetry machinery, authoritative workflow_steps.job_id readers/writers across release, terminal, continuation, admin, recovery, and replay, rollout docs, classifiers, and PostgreSQL 18 regression/plan coverage. Validated on PostgreSQL 18.4 (server_version_num=180004): reverse and legacy point lookups both used indexed cost-8.29 plans; full runledger-postgres and runledger-runtime suites passed; all 21 migration tests passed; clippy passed; SQLx caches refreshed and synchronized across all three locations.","external_ref":"audit:PG-03-R1","source_repo":"runledger","source_repo_path":"/Users/aa/Documents/runledger","compaction_level":0,"original_size":0,"labels":["audit-pg-03-r1","long-horizon","postgresql","simplification-audit","staged-rollout","tier-c"],"dependencies":[{"issue_id":"runledger-runledger-simplification-audit-w2f.7","depends_on_id":"runledger-runledger-simplification-audit-w2f","type":"parent-child","created_at":"2026-08-22T20:50:48.595934Z","created_by":"aa","metadata":"{}","thread_id":""},{"issue_id":"runledger-runledger-simplification-audit-w2f.7","depends_on_id":"runledger-runledger-simplification-audit-w2f.46","type":"blocks","created_at":"2026-08-22T20:53:34.204531Z","created_by":"aa","metadata":"{\"reason\":\"Schema rollout requires the corrected PostgreSQL 18 SQLx refresh contract.\",\"audit_edge\":\"INF-01-R2 -> PG-03-R1\"}","thread_id":""}]} {"id":"runledger-runledger-simplification-audit-w2f.8","title":"PG-03-R2: Factor continuation-event validity into a local NOT MATERIALIZED CTE","description":"Audit ID: PG-03-R2\nVerdict: recommend\nPriority tier: C\nConfidence: medium\n\n## Evidence\nrunledger-postgres/migrations/202607250001_harden_continuation_metrics_payload_validation.up.sql:21-46,65-92; runledger-postgres/tests/migrations.rs:726-940.\n\n## Current complexity or invalid states\nA subtle JSON type/range/timestamp/legacy validity predicate is duplicated in two branches, while only one branch needs current-run correlation.\n\n## Proposed representation\nCreate a view-local valid_continuation_events AS NOT MATERIALIZED CTE for only the shared predicate; keep branch-specific run and time conditions local.\n\n## Smallest credible implementation scope\nOne new up/down migration in all migration trees and migration tests.\n\n## Regression risks and migration concerns\nPostgreSQL 18 CTE plan behavior and CREATE OR REPLACE VIEW column compatibility. Do not create a reusable function or public view.\n\n## Existing validation\nDedicated malformed-payload migration regression.\n\n## Additional validation required\nProve result equivalence and compare PostgreSQL 18 custom/generic EXPLAIN ANALYZE plans; record version and sync caches.\n\nParent audit: runledger-runledger-simplification-audit-w2f.","acceptance_criteria":"- Implement only the bounded representation described above.\n- Preserve the stated compatibility and semantic constraints.\n- Add the required validation; database-backed work must use PostgreSQL 18 and record server_version/server_version_num.\n- If SQL/schema metadata changes, keep all three SQLx caches synchronized.\n- Close with a concise implementation and validation summary.","status":"closed","priority":2,"issue_type":"task","created_at":"2026-08-22T20:50:48.764164Z","created_by":"aa","updated_at":"2026-08-23T08:49:40.025419Z","closed_at":"2026-08-23T08:49:40.025284Z","close_reason":"Added reversible 202608230001 migration in all three migration trees. Factored only shared continuation-event validity into local valid_continuation_events AS NOT MATERIALIZED; retained 24h and current-run predicates in their branches. PostgreSQL 18.4 (server_version_num 180004) result/down-up equivalence, malformed-payload regression, forced custom/generic EXPLAIN ANALYZE operator parity, and all 19 migration tests passed. Refreshed and verified all three SQLx caches.","external_ref":"audit:PG-03-R2","source_repo":"runledger","source_repo_path":"/Users/aa/Documents/runledger","compaction_level":0,"original_size":0,"labels":["audit-pg-03-r2","postgresql","postgresql18-evidence","simplification-audit","tier-c"],"dependencies":[{"issue_id":"runledger-runledger-simplification-audit-w2f.8","depends_on_id":"runledger-runledger-simplification-audit-w2f","type":"parent-child","created_at":"2026-08-22T20:50:48.764164Z","created_by":"aa","metadata":"{}","thread_id":""},{"issue_id":"runledger-runledger-simplification-audit-w2f.8","depends_on_id":"runledger-runledger-simplification-audit-w2f.46","type":"blocks","created_at":"2026-08-22T20:53:34.362011Z","created_by":"aa","metadata":"{\"reason\":\"View migration requires the corrected PostgreSQL 18 SQLx refresh contract.\",\"audit_edge\":\"INF-01-R2 -> PG-03-R2\"}","thread_id":""}]} {"id":"runledger-runledger-simplification-audit-w2f.9","title":"PG-04-R1: Represent exact definition sync by its absent-definition set","description":"Audit ID: PG-04-R1\nVerdict: recommend\nPriority tier: D\nConfidence: high\n\n## Evidence\nrunledger-postgres/src/jobs/queue/definitions/catalog_sync.rs:137-199,255-340,386-413.\n\n## Current complexity or invalid states\nExact sync stores scope/catalog lists plus a derived boolean and later re-derives which definitions are absent.\n\n## Proposed representation\nCompute one concrete absent-definition set/list during validation and consume it directly. Do not introduce a planning framework.\n\n## Smallest credible implementation scope\nDefinition catalog synchronization internals and focused report tests.\n\n## Regression risks and migration concerns\nPreserve deterministic error/report order, definition locks, and non-empty SQL guards.\n\n## Existing validation\nRuntime catalog and definition synchronization tests.\n\n## Additional validation required\nCover empty, partial, all-absent, additive, and exact cases on PostgreSQL 18.\n\nParent audit: runledger-runledger-simplification-audit-w2f.","acceptance_criteria":"- Implement only the bounded representation described above.\n- Preserve the stated compatibility and semantic constraints.\n- Add the required validation; database-backed work must use PostgreSQL 18 and record server_version/server_version_num.\n- If SQL/schema metadata changes, keep all three SQLx caches synchronized.\n- Close with a concise implementation and validation summary.","status":"closed","priority":3,"issue_type":"task","created_at":"2026-08-22T20:50:48.933544Z","created_by":"aa","updated_at":"2026-08-24T09:05:46.528353Z","closed_at":"2026-08-24T09:05:46.528197Z","close_reason":"Exact definition sync now computes one sorted, deduplicated absent-definition list during validation and consumes it directly for guards, schedule checks, and disabling. Added empty/partial/all-absent/additive/exact report coverage. Validated on PostgreSQL 18.4, full schedule and runtime catalog suites, strict clippy, and synchronized all SQLx caches.","external_ref":"audit:PG-04-R1","source_repo":"runledger","source_repo_path":"/Users/aa/Documents/runledger","compaction_level":0,"original_size":0,"labels":["audit-pg-04-r1","postgresql","simplification-audit","tier-d"],"dependencies":[{"issue_id":"runledger-runledger-simplification-audit-w2f.9","depends_on_id":"runledger-runledger-simplification-audit-w2f","type":"parent-child","created_at":"2026-08-22T20:50:48.933544Z","created_by":"aa","metadata":"{}","thread_id":""}]} +{"id":"runledger-runledger-simplification-audit-x9i","title":"Make installation and producer examples match the current release","description":"Source audit: docs/api-audit-2026-09-05.md, API-008.\n\nREADME installation recommends 0.9 while workspace is 0.12.0; producer quick start contains an incomplete JobEnqueue placeholder. Provide complete current-version submission and shared producer/worker examples; add appropriate release validation for version references. Do not require new proposed APIs before fixing existing examples.","status":"closed","priority":2,"issue_type":"docs","created_at":"2026-09-05T09:20:44.398715755Z","created_by":"aa","updated_at":"2026-09-05T14:15:28.253729639Z","closed_at":"2026-09-05T14:15:28.253446279Z","close_reason":"Updated installation dependencies to workspace 0.12.0; replaced the incomplete quick start with compiled producer/worker binaries sharing typed payloads and a complete request, transactional enqueue, completion progress, and graceful shutdown. Added README version/source validation to lint and both release scripts; prepare-release updates current references while preserving historical guidance. Verified standalone consumer compilation from README, full scripts/lint.sh, negative version/snippet checks, prerelease rewrite and idempotence, and transactional rollback/idempotent retry/worker completion on PostgreSQL 18.6 (Debian 18.6-1.pgdg13+2). Cargo default runtime testing includes the example via test=true. One verification run hit a database container startup timeout; a fresh example run passed.","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0,"labels":["api-audit"]} +{"id":"runledger-runledger-simplification-audit-xtp","title":"Assess Vatbot document jobs versus continuation pump adoption","description":"Source audit: docs/api-audit-2026-09-05.md, adoption opportunities.\n\nVatbot locks Runledger 0.4.0. extraction_worker/controller.rs schedules a one-second pump whose run_one implementation owns another claim/retry lifecycle. Evaluate an upgrade followed by document-level Runledger jobs or a checkpointed continuation pump. Establish which application lifecycle invariants are intentional before proposing consolidation.","acceptance_criteria":"Compare both integration options with source-backed migration steps and an explicit ownership map. Preserve paid-provider completion budgets, application claims, durable outcomes, retry semantics, and shutdown behavior. Record reusable Runledger API gaps separately from application policy. Do not infer measured enqueue rates from cron cadence.","status":"open","priority":2,"issue_type":"task","created_at":"2026-09-05T09:24:30.616360437Z","created_by":"aa","updated_at":"2026-09-05T09:24:30.616360437Z","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0,"labels":["api-audit","discovery"]} +{"id":"runledger-runledger-simplification-audit-z0s","title":"Investigate container lifecycle probes failing PostgreSQL bootstrap on Linux","description":"During branch-review fix validation on 2026-09-05, standalone runledger-test-support container_lifecycle tests for normal exit and forced termination fail before readiness: PostgreSQL connection attempts exhausted after 40 retries. Missing/stalled optional reaper tests pass. Reproduction: env -u RUNLEDGER_TEST_ADMIN_DATABASE_URL RUNLEDGER_TEST_PG_IMAGE=postgres:18 cargo test -p runledger-test-support --test container_lifecycle -- --nocapture --test-threads=1. runledger-test-support, Cargo.lock, and toolchain are unchanged. Application suites and packaged smoke pass using an explicitly provisioned PostgreSQL 18.6 container. Investigate startup/reaper interaction; no cause established yet.","notes":"Investigated on Linux with postgres:18 image sha256:a6638641707cdf047e5d5c2781f437e2e809323cab22c70b280be8389fbb7878; SHOW server_version returned 18.6 (Debian 18.6-1.pgdg13+2), server_version_num=180006. The original lifecycle command passed twice before changes, so the historical transient itself was not reproduced. A direct child probe observed 16 ConnectionReset errors before becoming ready. No premature reaper removal was observed.\n\nConfirmed a startup timing defect: 40 retries with 250 ms backoff exhaust in about 10 seconds on fast connection resets, while slow attempts can each consume the SQLx default 30-second timeout. A real PostgreSQL 18 container deliberately delayed by 12 seconds reproduced the exact 40-attempt failure before the fix (11.36 seconds total). The stalled optional reaper already delays connection probing by 15 seconds, which can mask slow startup.\n\nReplaced attempt counting with a 30-second connection deadline, capped individual acquisition attempts at two seconds and backoff at the remaining budget, and preserved the final SQLx error without printing the admin URL. Kept version and uuidv7 validation. Added the delayed PostgreSQL regression (skipped in external-admin mode) and a local stalled-handshake test for the deadline. Delayed startup now passes against PostgreSQL 18.6.\n\nValidation: env -u RUNLEDGER_TEST_ADMIN_DATABASE_URL RUNLEDGER_TEST_PG_IMAGE=postgres:18 cargo test -p runledger-test-support -- --nocapture --test-threads=1 passed (10 library tests, 4 lifecycle tests; helper entrypoint intentionally ignored). Normal exit, forced termination, missing CLI and stalled CLI cases all passed. cargo clippy -p runledger-test-support --all-targets --locked -- -D warnings, cargo fmt --all -- --check, and git diff --check passed. No labeled lifecycle containers remained after validation.","status":"closed","priority":2,"issue_type":"bug","created_at":"2026-09-05T13:49:46.195011590Z","created_by":"aa","updated_at":"2026-09-05T14:00:47.146467503Z","closed_at":"2026-09-05T14:00:47.146071484Z","close_reason":"Fixed reproduced PostgreSQL startup retry-window defect with bounded readiness and regression coverage on PostgreSQL 18.6","source_repo":"runledger","source_repo_path":"/home/aa/Documents/runledger","compaction_level":0,"original_size":0} {"id":"runledger-y1o","title":"Harden lifecycle timeout recovery after 0.11 review","description":"Resolve comprehensive-review findings structurally: centralize queue timeout policy, bound prestart release recovery, type PostgreSQL lock contention for runtime policy, retry transient heartbeat lock waits within one fixed lease-maintenance deadline, and close PostgreSQL 18 regression gaps. Commit each logical slice separately and finish with the full repository suite.","status":"closed","priority":1,"issue_type":"bug","created_at":"2026-08-26T13:03:13.908525Z","created_by":"aa","updated_at":"2026-08-26T13:22:54.908241Z","closed_at":"2026-08-26T13:22:54.907662Z","close_reason":"Implemented and verified structural lifecycle timeout recovery across PostgreSQL and runtime; full workspace tests and Clippy pass on PostgreSQL 18.4.","source_repo":"runledger","source_repo_path":"/Users/aa/Documents/runledger","compaction_level":0,"original_size":0} diff --git a/.sqlx/query-07c64366e2b4e12cd65cad0e26ff8e2442eafaa21bc515a90063c7750ed0a070.json b/.sqlx/query-07c64366e2b4e12cd65cad0e26ff8e2442eafaa21bc515a90063c7750ed0a070.json new file mode 100644 index 0000000..7da9f8b --- /dev/null +++ b/.sqlx/query-07c64366e2b4e12cd65cad0e26ff8e2442eafaa21bc515a90063c7750ed0a070.json @@ -0,0 +1,105 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, job_type, organization_id, status::text AS \"status!\",\n priority, run_number, attempt, max_attempts, next_run_at, stage,\n progress_done, progress_total, created_at, updated_at\n FROM job_queue WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type = $3) AND (created_at, id) < ($5, $6) ORDER BY created_at DESC, id DESC LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Timestamptz", + "Uuid" + ] + }, + "nullable": [ + false, + false, + true, + null, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + }, + "hash": "07c64366e2b4e12cd65cad0e26ff8e2442eafaa21bc515a90063c7750ed0a070" +} diff --git a/runledger-postgres/.sqlx/query-a0514692991ed79f5c48f8b8758234202b016da9c5c40cac67f72e000455b9b8.json b/.sqlx/query-10269f3f8cd79707daaba3c26771d5aa97e7c9c08801cbfd18f70bbd8670e17a.json similarity index 57% rename from runledger-postgres/.sqlx/query-a0514692991ed79f5c48f8b8758234202b016da9c5c40cac67f72e000455b9b8.json rename to .sqlx/query-10269f3f8cd79707daaba3c26771d5aa97e7c9c08801cbfd18f70bbd8670e17a.json index 9fe67e1..704da5f 100644 --- a/runledger-postgres/.sqlx/query-a0514692991ed79f5c48f8b8758234202b016da9c5c40cac67f72e000455b9b8.json +++ b/.sqlx/query-10269f3f8cd79707daaba3c26771d5aa97e7c9c08801cbfd18f70bbd8670e17a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, payload\n FROM job_queue\n WHERE organization_id = $1\n AND job_type = $2\n AND payload->>'run_id' = $3\n ORDER BY created_at DESC, id DESC\n LIMIT 1", + "query": "SELECT id, payload FROM job_queue WHERE organization_id = $1 AND job_type = $2\n AND payload->>'run_id' = $3\n ORDER BY created_at DESC, id DESC\n LIMIT 1", "describe": { "columns": [ { @@ -26,5 +26,5 @@ false ] }, - "hash": "a0514692991ed79f5c48f8b8758234202b016da9c5c40cac67f72e000455b9b8" + "hash": "10269f3f8cd79707daaba3c26771d5aa97e7c9c08801cbfd18f70bbd8670e17a" } diff --git a/runledger-runtime/.sqlx/query-dbcca04872cf174ca1a23862f2f632fcf5da4599e75c8f21b1c134b34b1a876a.json b/.sqlx/query-1059ac0fa26c3192fe4160b03fe6f8fadf3727d67e102953a0793af534de602a.json similarity index 83% rename from runledger-runtime/.sqlx/query-dbcca04872cf174ca1a23862f2f632fcf5da4599e75c8f21b1c134b34b1a876a.json rename to .sqlx/query-1059ac0fa26c3192fe4160b03fe6f8fadf3727d67e102953a0793af534de602a.json index b05c115..db05c86 100644 --- a/runledger-runtime/.sqlx/query-dbcca04872cf174ca1a23862f2f632fcf5da4599e75c8f21b1c134b34b1a876a.json +++ b/.sqlx/query-1059ac0fa26c3192fe4160b03fe6f8fadf3727d67e102953a0793af534de602a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n je.id,\n je.job_id,\n je.run_number,\n je.attempt,\n je.event_type::text AS \"event_type!\",\n je.stage,\n je.progress_done,\n je.progress_total,\n je.payload,\n je.occurred_at\n FROM job_events je\n JOIN job_queue jq ON jq.id = je.job_id\n WHERE je.job_id = $1\n AND ($2::uuid IS NULL OR jq.organization_id = $2)\n AND ($3::bigint IS NULL OR je.id > $3)\n ORDER BY je.id ASC\n LIMIT $4", + "query": "SELECT\n je.id,\n je.job_id,\n je.run_number,\n je.attempt,\n je.event_type::text AS \"event_type!\",\n je.stage,\n je.progress_done,\n je.progress_total,\n je.payload,\n je.occurred_at\n FROM job_events je\n JOIN job_queue jq ON jq.id = je.job_id\n WHERE je.job_id = $1\n AND ($5::bool OR jq.organization_id IS NOT DISTINCT FROM $2::uuid)\n AND ($3::bigint IS NULL OR je.id > $3)\n ORDER BY je.id ASC\n LIMIT $4", "describe": { "columns": [ { @@ -59,7 +59,8 @@ "Uuid", "Uuid", "Int8", - "Int8" + "Int8", + "Bool" ] }, "nullable": [ @@ -75,5 +76,5 @@ false ] }, - "hash": "dbcca04872cf174ca1a23862f2f632fcf5da4599e75c8f21b1c134b34b1a876a" + "hash": "1059ac0fa26c3192fe4160b03fe6f8fadf3727d67e102953a0793af534de602a" } diff --git a/runledger-runtime/.sqlx/query-a435ae7cc1a59f8a0c1fa1b5b54924319d9d321e60520c00480c5d220248e79c.json b/.sqlx/query-290943331ceeb17e0d9e726a3bb5a2afb203b501dccdf13baa3b2390800515f7.json similarity index 91% rename from runledger-runtime/.sqlx/query-a435ae7cc1a59f8a0c1fa1b5b54924319d9d321e60520c00480c5d220248e79c.json rename to .sqlx/query-290943331ceeb17e0d9e726a3bb5a2afb203b501dccdf13baa3b2390800515f7.json index dca5600..399cc9a 100644 --- a/runledger-runtime/.sqlx/query-a435ae7cc1a59f8a0c1fa1b5b54924319d9d321e60520c00480c5d220248e79c.json +++ b/.sqlx/query-290943331ceeb17e0d9e726a3bb5a2afb203b501dccdf13baa3b2390800515f7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n next_run_at,\n idempotency_key,\n stage,\n enqueue_request_version,\n execution_resource_key,\n promotion_attempts,\n next_promotion_at,\n last_attempted_at,\n status,\n promoted_job_id,\n promoted_at,\n conflicted_at,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_enqueue_intents\n WHERE ($1::uuid IS NULL OR organization_id = $1)\n AND ($2::text IS NULL OR status = $2)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n next_run_at,\n idempotency_key,\n stage,\n enqueue_request_version,\n execution_resource_key,\n promotion_attempts,\n next_promotion_at,\n last_attempted_at,\n status,\n promoted_job_id,\n promoted_at,\n conflicted_at,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_enqueue_intents\n WHERE $1::uuid IS NULL AND ($2::text IS NULL OR status = $2)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", "describe": { "columns": [ { @@ -154,5 +154,5 @@ false ] }, - "hash": "a435ae7cc1a59f8a0c1fa1b5b54924319d9d321e60520c00480c5d220248e79c" + "hash": "290943331ceeb17e0d9e726a3bb5a2afb203b501dccdf13baa3b2390800515f7" } diff --git a/runledger-runtime/.sqlx/query-c2350f3a1f05bb5bf576e20872873d7135e8dfff31412ed42bda795dbeee7586.json b/.sqlx/query-2b0c48cf0a53ca60dfb663b242b1b65b0833bf0b32fe4cacb637e18df8ae77bb.json similarity index 92% rename from runledger-runtime/.sqlx/query-c2350f3a1f05bb5bf576e20872873d7135e8dfff31412ed42bda795dbeee7586.json rename to .sqlx/query-2b0c48cf0a53ca60dfb663b242b1b65b0833bf0b32fe4cacb637e18df8ae77bb.json index 9150683..b9f7f28 100644 --- a/runledger-runtime/.sqlx/query-c2350f3a1f05bb5bf576e20872873d7135e8dfff31412ed42bda795dbeee7586.json +++ b/.sqlx/query-2b0c48cf0a53ca60dfb663b242b1b65b0833bf0b32fe4cacb637e18df8ae77bb.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n status::text AS \"status!\",\n priority,\n run_number,\n attempt,\n max_attempts,\n timeout_seconds,\n next_run_at,\n lease_expires_at,\n last_heartbeat_at,\n worker_id,\n started_at,\n finished_at,\n stage,\n progress_done,\n progress_total,\n progress_pct::float8 AS progress_pct,\n checkpoint,\n output,\n idempotency_key,\n status_reason,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_queue\n WHERE ($1::uuid IS NULL OR organization_id = $1)\n AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n status::text AS \"status!\",\n priority,\n run_number,\n attempt,\n max_attempts,\n timeout_seconds,\n next_run_at,\n lease_expires_at,\n last_heartbeat_at,\n worker_id,\n started_at,\n finished_at,\n stage,\n progress_done,\n progress_total,\n progress_pct::float8 AS progress_pct,\n checkpoint,\n output,\n idempotency_key,\n status_reason,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_queue\n WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", "describe": { "columns": [ { @@ -184,5 +184,5 @@ false ] }, - "hash": "c2350f3a1f05bb5bf576e20872873d7135e8dfff31412ed42bda795dbeee7586" + "hash": "2b0c48cf0a53ca60dfb663b242b1b65b0833bf0b32fe4cacb637e18df8ae77bb" } diff --git a/.sqlx/query-2ca138d0f77ca8b5d61381edcdb32a7f98c016c7214a13f69be251c78058b0e7.json b/.sqlx/query-2ca138d0f77ca8b5d61381edcdb32a7f98c016c7214a13f69be251c78058b0e7.json new file mode 100644 index 0000000..d18086b --- /dev/null +++ b/.sqlx/query-2ca138d0f77ca8b5d61381edcdb32a7f98c016c7214a13f69be251c78058b0e7.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, status::text AS \"status!\", run_number, attempt, updated_at\n FROM job_queue WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND id = ANY($2::uuid[]) ORDER BY id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "UuidArray" + ] + }, + "nullable": [ + false, + null, + false, + false, + false + ] + }, + "hash": "2ca138d0f77ca8b5d61381edcdb32a7f98c016c7214a13f69be251c78058b0e7" +} diff --git a/runledger-postgres/.sqlx/query-a435ae7cc1a59f8a0c1fa1b5b54924319d9d321e60520c00480c5d220248e79c.json b/.sqlx/query-2f5d35c92e35e69d3d0260eeb2bac01332025306d05cb43392686d9cc73d5eba.json similarity index 91% rename from runledger-postgres/.sqlx/query-a435ae7cc1a59f8a0c1fa1b5b54924319d9d321e60520c00480c5d220248e79c.json rename to .sqlx/query-2f5d35c92e35e69d3d0260eeb2bac01332025306d05cb43392686d9cc73d5eba.json index dca5600..6abf7e3 100644 --- a/runledger-postgres/.sqlx/query-a435ae7cc1a59f8a0c1fa1b5b54924319d9d321e60520c00480c5d220248e79c.json +++ b/.sqlx/query-2f5d35c92e35e69d3d0260eeb2bac01332025306d05cb43392686d9cc73d5eba.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n next_run_at,\n idempotency_key,\n stage,\n enqueue_request_version,\n execution_resource_key,\n promotion_attempts,\n next_promotion_at,\n last_attempted_at,\n status,\n promoted_job_id,\n promoted_at,\n conflicted_at,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_enqueue_intents\n WHERE ($1::uuid IS NULL OR organization_id = $1)\n AND ($2::text IS NULL OR status = $2)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n next_run_at,\n idempotency_key,\n stage,\n enqueue_request_version,\n execution_resource_key,\n promotion_attempts,\n next_promotion_at,\n last_attempted_at,\n status,\n promoted_job_id,\n promoted_at,\n conflicted_at,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_enqueue_intents\n WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND ($2::text IS NULL OR status = $2)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", "describe": { "columns": [ { @@ -154,5 +154,5 @@ false ] }, - "hash": "a435ae7cc1a59f8a0c1fa1b5b54924319d9d321e60520c00480c5d220248e79c" + "hash": "2f5d35c92e35e69d3d0260eeb2bac01332025306d05cb43392686d9cc73d5eba" } diff --git a/runledger-runtime/.sqlx/query-9f9c494b36d03026a351373e70483c382d2e53386fac1d737a5f2e20818007d8.json b/.sqlx/query-321da426e59fb877a1e48430f03013433f919b3f4edc06e322f195e682e7be0f.json similarity index 88% rename from runledger-runtime/.sqlx/query-9f9c494b36d03026a351373e70483c382d2e53386fac1d737a5f2e20818007d8.json rename to .sqlx/query-321da426e59fb877a1e48430f03013433f919b3f4edc06e322f195e682e7be0f.json index 9b99123..fb67a3d 100644 --- a/runledger-runtime/.sqlx/query-9f9c494b36d03026a351373e70483c382d2e53386fac1d737a5f2e20818007d8.json +++ b/.sqlx/query-321da426e59fb877a1e48430f03013433f919b3f4edc06e322f195e682e7be0f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n jd.job_type AS \"job_type!\",\n COALESCE(SUM(jmr.pending_count), 0)::bigint AS \"pending_count!\",\n COALESCE(SUM(jmr.leased_count), 0)::bigint AS \"leased_count!\",\n COALESCE(SUM(jmr.stale_leases), 0)::bigint AS \"stale_leases!\",\n COALESCE(SUM(jmr.succeeded_24h), 0)::bigint AS \"succeeded_24h!\",\n COALESCE(SUM(jmr.retryable_24h), 0)::bigint AS \"retryable_24h!\",\n COALESCE(SUM(jmr.terminal_24h), 0)::bigint AS \"terminal_24h!\",\n COALESCE(SUM(jmr.panicked_24h), 0)::bigint AS \"panicked_24h!\",\n COALESCE(SUM(jmr.timeout_24h), 0)::bigint AS \"timeout_24h!\",\n COALESCE(SUM(jmr.dead_lettered_24h), 0)::bigint AS \"dead_lettered_24h!\",\n AVG(jmr.p50_duration_ms_24h) AS p50_duration_ms_24h,\n AVG(jmr.p95_duration_ms_24h) AS p95_duration_ms_24h\n FROM job_definitions jd\n LEFT JOIN job_metrics_rollup jmr\n ON jmr.job_type = jd.job_type\n AND ($1::uuid IS NULL OR jmr.organization_id = $1)\n WHERE ($2::text IS NULL OR jd.job_type = $2)\n GROUP BY jd.job_type\n ORDER BY jd.job_type ASC", + "query": "SELECT\n jd.job_type AS \"job_type!\",\n COALESCE(SUM(jmr.pending_count), 0)::bigint AS \"pending_count!\",\n COALESCE(SUM(jmr.leased_count), 0)::bigint AS \"leased_count!\",\n COALESCE(SUM(jmr.stale_leases), 0)::bigint AS \"stale_leases!\",\n COALESCE(SUM(jmr.succeeded_24h), 0)::bigint AS \"succeeded_24h!\",\n COALESCE(SUM(jmr.retryable_24h), 0)::bigint AS \"retryable_24h!\",\n COALESCE(SUM(jmr.terminal_24h), 0)::bigint AS \"terminal_24h!\",\n COALESCE(SUM(jmr.panicked_24h), 0)::bigint AS \"panicked_24h!\",\n COALESCE(SUM(jmr.timeout_24h), 0)::bigint AS \"timeout_24h!\",\n COALESCE(SUM(jmr.dead_lettered_24h), 0)::bigint AS \"dead_lettered_24h!\",\n AVG(jmr.p50_duration_ms_24h) AS p50_duration_ms_24h,\n AVG(jmr.p95_duration_ms_24h) AS p95_duration_ms_24h\n FROM job_definitions jd\n LEFT JOIN job_metrics_rollup jmr\n ON jmr.job_type = jd.job_type\n AND ($3::boolean OR (jmr.organization_id = $1 OR ($1::uuid IS NULL AND jmr.organization_id IS NULL)))\n WHERE ($2::text IS NULL OR jd.job_type = $2)\n GROUP BY jd.job_type\n ORDER BY jd.job_type ASC", "describe": { "columns": [ { @@ -67,7 +67,8 @@ "parameters": { "Left": [ "Uuid", - "Text" + "Text", + "Bool" ] }, "nullable": [ @@ -85,5 +86,5 @@ null ] }, - "hash": "9f9c494b36d03026a351373e70483c382d2e53386fac1d737a5f2e20818007d8" + "hash": "321da426e59fb877a1e48430f03013433f919b3f4edc06e322f195e682e7be0f" } diff --git a/.sqlx/query-48789cd60eb5ace54e41f24bafc110420c7e67fe6175f5bcb786579a66f9af8e.json b/.sqlx/query-48789cd60eb5ace54e41f24bafc110420c7e67fe6175f5bcb786579a66f9af8e.json new file mode 100644 index 0000000..cc80fa9 --- /dev/null +++ b/.sqlx/query-48789cd60eb5ace54e41f24bafc110420c7e67fe6175f5bcb786579a66f9af8e.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, payload FROM job_queue WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND job_type = $2\n AND payload->>'run_id' = $3\n ORDER BY created_at DESC, id DESC\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "payload", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "48789cd60eb5ace54e41f24bafc110420c7e67fe6175f5bcb786579a66f9af8e" +} diff --git a/.sqlx/query-809f796d49a1830f8338380883e2ee58cce341f4e12b85bb982b01bf09d6cc50.json b/.sqlx/query-51b23cabe44bba220eb95d76cfcc135768be65ed533785bea2d9d668e9305ebd.json similarity index 82% rename from .sqlx/query-809f796d49a1830f8338380883e2ee58cce341f4e12b85bb982b01bf09d6cc50.json rename to .sqlx/query-51b23cabe44bba220eb95d76cfcc135768be65ed533785bea2d9d668e9305ebd.json index 2ccb7de..9ced603 100644 --- a/.sqlx/query-809f796d49a1830f8338380883e2ee58cce341f4e12b85bb982b01bf09d6cc50.json +++ b/.sqlx/query-51b23cabe44bba220eb95d76cfcc135768be65ed533785bea2d9d668e9305ebd.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n jl.id,\n jl.job_id,\n jl.run_number,\n jl.attempt,\n jl.level,\n jl.message,\n jl.payload,\n jl.occurred_at\n FROM job_logs jl\n JOIN job_queue jq ON jq.id = jl.job_id\n WHERE jl.job_id = $1\n AND ($2::uuid IS NULL OR jq.organization_id = $2)\n AND ($3::bigint IS NULL OR jl.id > $3)\n ORDER BY jl.id ASC\n LIMIT $4", + "query": "SELECT\n jl.id,\n jl.job_id,\n jl.run_number,\n jl.attempt,\n jl.level,\n jl.message,\n jl.payload,\n jl.occurred_at\n FROM job_logs jl\n JOIN job_queue jq ON jq.id = jl.job_id\n WHERE jl.job_id = $1\n AND ($5::bool OR jq.organization_id IS NOT DISTINCT FROM $2::uuid)\n AND ($3::bigint IS NULL OR jl.id > $3)\n ORDER BY jl.id ASC\n LIMIT $4", "describe": { "columns": [ { @@ -49,7 +49,8 @@ "Uuid", "Uuid", "Int8", - "Int8" + "Int8", + "Bool" ] }, "nullable": [ @@ -63,5 +64,5 @@ false ] }, - "hash": "809f796d49a1830f8338380883e2ee58cce341f4e12b85bb982b01bf09d6cc50" + "hash": "51b23cabe44bba220eb95d76cfcc135768be65ed533785bea2d9d668e9305ebd" } diff --git a/.sqlx/query-6ab23505891fce132239b6cd02ab480be547302313aeb4be81f2c14bb20c0569.json b/.sqlx/query-6ab23505891fce132239b6cd02ab480be547302313aeb4be81f2c14bb20c0569.json new file mode 100644 index 0000000..0f09710 --- /dev/null +++ b/.sqlx/query-6ab23505891fce132239b6cd02ab480be547302313aeb4be81f2c14bb20c0569.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE job_queue\n SET stage = COALESCE($5, stage),\n progress_done = COALESCE($6, progress_done),\n progress_total = COALESCE($7, progress_total),\n checkpoint = COALESCE($8::jsonb, checkpoint),\n updated_at = now()\n WHERE id = $1\n AND run_number = $2\n AND attempt = $3\n AND worker_id = $4\n AND status = 'LEASED'\n AND lease_expires_at > clock_timestamp()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int4", + "Int4", + "Text", + "Text", + "Int8", + "Int8", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "6ab23505891fce132239b6cd02ab480be547302313aeb4be81f2c14bb20c0569" +} diff --git a/.sqlx/query-6c5d8cb4f663423a38334a1418faf9050dee8d8fd943820afd0bcb6e0bc32565.json b/.sqlx/query-6c5d8cb4f663423a38334a1418faf9050dee8d8fd943820afd0bcb6e0bc32565.json deleted file mode 100644 index d56ae1f..0000000 --- a/.sqlx/query-6c5d8cb4f663423a38334a1418faf9050dee8d8fd943820afd0bcb6e0bc32565.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workflow_steps (\n workflow_run_id,\n step_key,\n execution_kind,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n stage,\n allow_handler_continuation,\n execution_resource_key,\n status,\n dependency_count_total,\n dependency_count_pending,\n dependency_count_unsatisfied\n )\n VALUES (\n $1,\n $2,\n $3::text::workflow_step_execution_kind,\n $4,\n $5,\n $6::jsonb,\n $7,\n $8,\n $9,\n $10,\n $11,\n $12,\n 'BLOCKED',\n $13,\n $14,\n $15\n )\n RETURNING id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Text", - "Text", - "Uuid", - "Jsonb", - "Int4", - "Int4", - "Int4", - "Text", - "Bool", - "Text", - "Int4", - "Int4", - "Int4" - ] - }, - "nullable": [ - false - ] - }, - "hash": "6c5d8cb4f663423a38334a1418faf9050dee8d8fd943820afd0bcb6e0bc32565" -} diff --git a/runledger-runtime/.sqlx/query-76c178ef26a5e63480d6f03da8016bc623846f7f78de717025ebbd40c1b848e0.json b/.sqlx/query-76e0bb60c9fe984d8b52e294bc657bc1aece09a76f064220c406b146277929cc.json similarity index 61% rename from runledger-runtime/.sqlx/query-76c178ef26a5e63480d6f03da8016bc623846f7f78de717025ebbd40c1b848e0.json rename to .sqlx/query-76e0bb60c9fe984d8b52e294bc657bc1aece09a76f064220c406b146277929cc.json index 1e847d2..49786b4 100644 --- a/runledger-runtime/.sqlx/query-76c178ef26a5e63480d6f03da8016bc623846f7f78de717025ebbd40c1b848e0.json +++ b/.sqlx/query-76e0bb60c9fe984d8b52e294bc657bc1aece09a76f064220c406b146277929cc.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, payload\n FROM job_queue\n WHERE organization_id = $1\n AND job_type = $2\n AND idempotency_key = $3\n LIMIT 1", + "query": "SELECT id, payload FROM job_queue WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND job_type = $2\n AND idempotency_key = $3\n LIMIT 1", "describe": { "columns": [ { @@ -26,5 +26,5 @@ false ] }, - "hash": "76c178ef26a5e63480d6f03da8016bc623846f7f78de717025ebbd40c1b848e0" + "hash": "76e0bb60c9fe984d8b52e294bc657bc1aece09a76f064220c406b146277929cc" } diff --git a/.sqlx/query-118f48ae11853ca23401d3569a8c10677a04ac4114f2fbc5d832d866a46e778c.json b/.sqlx/query-7e287b60fb932a4795c61e25e247e4313adf268188c94931c8f9ebb995c8812e.json similarity index 94% rename from .sqlx/query-118f48ae11853ca23401d3569a8c10677a04ac4114f2fbc5d832d866a46e778c.json rename to .sqlx/query-7e287b60fb932a4795c61e25e247e4313adf268188c94931c8f9ebb995c8812e.json index b0eef8b..39f343e 100644 --- a/.sqlx/query-118f48ae11853ca23401d3569a8c10677a04ac4114f2fbc5d832d866a46e778c.json +++ b/.sqlx/query-7e287b60fb932a4795c61e25e247e4313adf268188c94931c8f9ebb995c8812e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n next_run_at,\n idempotency_key,\n stage,\n enqueue_request_version,\n execution_resource_key,\n promotion_attempts,\n next_promotion_at,\n last_attempted_at,\n status,\n promoted_job_id,\n promoted_at,\n conflicted_at,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_enqueue_intents\n WHERE id = $1\n AND ($2::uuid IS NULL OR organization_id = $2)\n LIMIT 1", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n next_run_at,\n idempotency_key,\n stage,\n enqueue_request_version,\n execution_resource_key,\n promotion_attempts,\n next_promotion_at,\n last_attempted_at,\n status,\n promoted_job_id,\n promoted_at,\n conflicted_at,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_enqueue_intents\n WHERE id = $1\n AND ($3::bool OR organization_id IS NOT DISTINCT FROM $2::uuid)\n LIMIT 1", "describe": { "columns": [ { @@ -122,7 +122,8 @@ "parameters": { "Left": [ "Uuid", - "Uuid" + "Uuid", + "Bool" ] }, "nullable": [ @@ -151,5 +152,5 @@ false ] }, - "hash": "118f48ae11853ca23401d3569a8c10677a04ac4114f2fbc5d832d866a46e778c" + "hash": "7e287b60fb932a4795c61e25e247e4313adf268188c94931c8f9ebb995c8812e" } diff --git a/.sqlx/query-80abaede7d6da749be3c1a5a6f5ff2da49f9b39c07bb5b2a0bfb35d86d7b7c8e.json b/.sqlx/query-80abaede7d6da749be3c1a5a6f5ff2da49f9b39c07bb5b2a0bfb35d86d7b7c8e.json new file mode 100644 index 0000000..0fcf450 --- /dev/null +++ b/.sqlx/query-80abaede7d6da749be3c1a5a6f5ff2da49f9b39c07bb5b2a0bfb35d86d7b7c8e.json @@ -0,0 +1,188 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n status::text AS \"status!\",\n priority,\n run_number,\n attempt,\n max_attempts,\n timeout_seconds,\n next_run_at,\n lease_expires_at,\n last_heartbeat_at,\n worker_id,\n started_at,\n finished_at,\n stage,\n progress_done,\n progress_total,\n progress_pct::float8 AS progress_pct,\n checkpoint,\n output,\n idempotency_key,\n status_reason,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_queue\n WHERE organization_id = $1 AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "payload", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 9, + "name": "timeout_seconds", + "type_info": "Int4" + }, + { + "ordinal": 10, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 11, + "name": "lease_expires_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 12, + "name": "last_heartbeat_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "worker_id", + "type_info": "Text" + }, + { + "ordinal": 14, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 15, + "name": "finished_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 16, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 17, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 18, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 19, + "name": "progress_pct", + "type_info": "Float8" + }, + { + "ordinal": 20, + "name": "checkpoint", + "type_info": "Jsonb" + }, + { + "ordinal": 21, + "name": "output", + "type_info": "Jsonb" + }, + { + "ordinal": 22, + "name": "idempotency_key", + "type_info": "Text" + }, + { + "ordinal": 23, + "name": "status_reason", + "type_info": "Text" + }, + { + "ordinal": 24, + "name": "last_error_code", + "type_info": "Text" + }, + { + "ordinal": 25, + "name": "last_error_message", + "type_info": "Text" + }, + { + "ordinal": 26, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 27, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false, + null, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + true, + false, + true, + true, + null, + true, + true, + true, + true, + true, + true, + false, + false + ] + }, + "hash": "80abaede7d6da749be3c1a5a6f5ff2da49f9b39c07bb5b2a0bfb35d86d7b7c8e" +} diff --git a/.sqlx/query-822a3544ce03dde10c9f57947ccbb25546b94f29cfdcf2151d28361d3a04d461.json b/.sqlx/query-822a3544ce03dde10c9f57947ccbb25546b94f29cfdcf2151d28361d3a04d461.json new file mode 100644 index 0000000..464adbd --- /dev/null +++ b/.sqlx/query-822a3544ce03dde10c9f57947ccbb25546b94f29cfdcf2151d28361d3a04d461.json @@ -0,0 +1,105 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, job_type, organization_id, status::text AS \"status!\",\n priority, run_number, attempt, max_attempts, next_run_at, stage,\n progress_done, progress_total, created_at, updated_at\n FROM job_queue WHERE $1::uuid IS NULL AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type = $3) AND (created_at, id) < ($5, $6) ORDER BY created_at DESC, id DESC LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Timestamptz", + "Uuid" + ] + }, + "nullable": [ + false, + false, + true, + null, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + }, + "hash": "822a3544ce03dde10c9f57947ccbb25546b94f29cfdcf2151d28361d3a04d461" +} diff --git a/.sqlx/query-85eca198e5990ee9fa5ec1abef7c0abaf3750dfb0d4e56e6a242e2921a46d995.json b/.sqlx/query-85eca198e5990ee9fa5ec1abef7c0abaf3750dfb0d4e56e6a242e2921a46d995.json new file mode 100644 index 0000000..9b9f66e --- /dev/null +++ b/.sqlx/query-85eca198e5990ee9fa5ec1abef7c0abaf3750dfb0d4e56e6a242e2921a46d995.json @@ -0,0 +1,105 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, job_type, organization_id, status::text AS \"status!\",\n priority, run_number, attempt, max_attempts, next_run_at, stage,\n progress_done, progress_total, created_at, updated_at\n FROM job_queue WHERE organization_id = $1 AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type = $3) AND (created_at, id) < ($5, $6) ORDER BY created_at DESC, id DESC LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Timestamptz", + "Uuid" + ] + }, + "nullable": [ + false, + false, + true, + null, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + }, + "hash": "85eca198e5990ee9fa5ec1abef7c0abaf3750dfb0d4e56e6a242e2921a46d995" +} diff --git a/.sqlx/query-89faf4374de8090be4461c194856d94ddc63813894ceaf9fef0c0a00cb2598e7.json b/.sqlx/query-89faf4374de8090be4461c194856d94ddc63813894ceaf9fef0c0a00cb2598e7.json new file mode 100644 index 0000000..9ac167b --- /dev/null +++ b/.sqlx/query-89faf4374de8090be4461c194856d94ddc63813894ceaf9fef0c0a00cb2598e7.json @@ -0,0 +1,62 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH status_metrics AS (\n SELECT\n job_type,\n COUNT(*)::bigint AS pending_count,\n COUNT(*) FILTER (WHERE promotion_attempts > 0)::bigint AS retrying_count,\n MAX(promotion_attempts)::integer AS max_promotion_attempts,\n 0::bigint AS conflicted_24h,\n 0::bigint AS promoted_24h,\n MIN(created_at) AS oldest_pending_at\n FROM job_enqueue_intents\n WHERE status = 'PENDING'\n AND ($5::boolean OR (organization_id = $1 OR ($1::uuid IS NULL AND organization_id IS NULL)))\n AND ($2::text IS NULL OR job_type = $2)\n GROUP BY job_type\n\n UNION ALL\n\n SELECT\n job_type,\n 0::bigint AS pending_count,\n 0::bigint AS retrying_count,\n 0::integer AS max_promotion_attempts,\n COUNT(*)::bigint AS conflicted_24h,\n 0::bigint AS promoted_24h,\n NULL::timestamptz AS oldest_pending_at\n FROM job_enqueue_intents\n WHERE status = 'CONFLICTED'\n AND conflicted_at >= now() - interval '24 hours'\n AND ($5::boolean OR (organization_id = $1 OR ($1::uuid IS NULL AND organization_id IS NULL)))\n AND ($2::text IS NULL OR job_type = $2)\n GROUP BY job_type\n\n UNION ALL\n\n SELECT\n job_type,\n 0::bigint AS pending_count,\n 0::bigint AS retrying_count,\n 0::integer AS max_promotion_attempts,\n 0::bigint AS conflicted_24h,\n COUNT(*)::bigint AS promoted_24h,\n NULL::timestamptz AS oldest_pending_at\n FROM job_enqueue_intents\n WHERE status = 'PROMOTED'\n AND promoted_at >= now() - interval '24 hours'\n AND ($5::boolean OR (organization_id = $1 OR ($1::uuid IS NULL AND organization_id IS NULL)))\n AND ($2::text IS NULL OR job_type = $2)\n GROUP BY job_type\n )\n SELECT\n job_type AS \"job_type!\",\n MAX(pending_count)::bigint AS \"pending_count!\",\n MAX(retrying_count)::bigint AS \"retrying_count!\",\n MAX(max_promotion_attempts)::integer AS \"max_promotion_attempts!\",\n MAX(conflicted_24h)::bigint AS \"conflicted_24h!\",\n MAX(promoted_24h)::bigint AS \"promoted_24h!\",\n MIN(oldest_pending_at) AS oldest_pending_at\n FROM status_metrics\n GROUP BY job_type\n ORDER BY job_type\n LIMIT $3\n OFFSET $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_type!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "pending_count!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "retrying_count!", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "max_promotion_attempts!", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "conflicted_24h!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "promoted_24h!", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "oldest_pending_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Int8", + "Int8", + "Bool" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "89faf4374de8090be4461c194856d94ddc63813894ceaf9fef0c0a00cb2598e7" +} diff --git a/runledger-postgres/.sqlx/query-76c178ef26a5e63480d6f03da8016bc623846f7f78de717025ebbd40c1b848e0.json b/.sqlx/query-8a04ed3b1b3497fbb7ec16ef976e9be2b03e4d64d5448bd90ce15cf8a9e5acc0.json similarity index 61% rename from runledger-postgres/.sqlx/query-76c178ef26a5e63480d6f03da8016bc623846f7f78de717025ebbd40c1b848e0.json rename to .sqlx/query-8a04ed3b1b3497fbb7ec16ef976e9be2b03e4d64d5448bd90ce15cf8a9e5acc0.json index 1e847d2..a47e779 100644 --- a/runledger-postgres/.sqlx/query-76c178ef26a5e63480d6f03da8016bc623846f7f78de717025ebbd40c1b848e0.json +++ b/.sqlx/query-8a04ed3b1b3497fbb7ec16ef976e9be2b03e4d64d5448bd90ce15cf8a9e5acc0.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, payload\n FROM job_queue\n WHERE organization_id = $1\n AND job_type = $2\n AND idempotency_key = $3\n LIMIT 1", + "query": "SELECT id, payload FROM job_queue WHERE organization_id = $1 AND job_type = $2\n AND idempotency_key = $3\n LIMIT 1", "describe": { "columns": [ { @@ -26,5 +26,5 @@ false ] }, - "hash": "76c178ef26a5e63480d6f03da8016bc623846f7f78de717025ebbd40c1b848e0" + "hash": "8a04ed3b1b3497fbb7ec16ef976e9be2b03e4d64d5448bd90ce15cf8a9e5acc0" } diff --git a/.sqlx/query-8de6d5625581aff24f7bb689361c8dcd9319042f9883e4cc748bcd771b06c09e.json b/.sqlx/query-8de6d5625581aff24f7bb689361c8dcd9319042f9883e4cc748bcd771b06c09e.json new file mode 100644 index 0000000..c506de2 --- /dev/null +++ b/.sqlx/query-8de6d5625581aff24f7bb689361c8dcd9319042f9883e4cc748bcd771b06c09e.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, status::text AS \"status!\", run_number, attempt, updated_at\n FROM job_queue WHERE organization_id = $1 AND id = ANY($2::uuid[]) ORDER BY id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "UuidArray" + ] + }, + "nullable": [ + false, + null, + false, + false, + false + ] + }, + "hash": "8de6d5625581aff24f7bb689361c8dcd9319042f9883e4cc748bcd771b06c09e" +} diff --git a/.sqlx/query-90841ab00a885ed00fd2c3a6b49c90b549a262963d281b863f4283af861ea5e5.json b/.sqlx/query-90841ab00a885ed00fd2c3a6b49c90b549a262963d281b863f4283af861ea5e5.json deleted file mode 100644 index bf1cfb5..0000000 --- a/.sqlx/query-90841ab00a885ed00fd2c3a6b49c90b549a262963d281b863f4283af861ea5e5.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH status_metrics AS (\n SELECT\n job_type,\n COUNT(*)::bigint AS pending_count,\n COUNT(*) FILTER (WHERE promotion_attempts > 0)::bigint AS retrying_count,\n MAX(promotion_attempts)::integer AS max_promotion_attempts,\n 0::bigint AS conflicted_24h,\n 0::bigint AS promoted_24h,\n MIN(created_at) AS oldest_pending_at\n FROM job_enqueue_intents\n WHERE status = 'PENDING'\n AND ($1::uuid IS NULL OR organization_id = $1)\n AND ($2::text IS NULL OR job_type = $2)\n GROUP BY job_type\n\n UNION ALL\n\n SELECT\n job_type,\n 0::bigint AS pending_count,\n 0::bigint AS retrying_count,\n 0::integer AS max_promotion_attempts,\n COUNT(*)::bigint AS conflicted_24h,\n 0::bigint AS promoted_24h,\n NULL::timestamptz AS oldest_pending_at\n FROM job_enqueue_intents\n WHERE status = 'CONFLICTED'\n AND conflicted_at >= now() - interval '24 hours'\n AND ($1::uuid IS NULL OR organization_id = $1)\n AND ($2::text IS NULL OR job_type = $2)\n GROUP BY job_type\n\n UNION ALL\n\n SELECT\n job_type,\n 0::bigint AS pending_count,\n 0::bigint AS retrying_count,\n 0::integer AS max_promotion_attempts,\n 0::bigint AS conflicted_24h,\n COUNT(*)::bigint AS promoted_24h,\n NULL::timestamptz AS oldest_pending_at\n FROM job_enqueue_intents\n WHERE status = 'PROMOTED'\n AND promoted_at >= now() - interval '24 hours'\n AND ($1::uuid IS NULL OR organization_id = $1)\n AND ($2::text IS NULL OR job_type = $2)\n GROUP BY job_type\n )\n SELECT\n job_type AS \"job_type!\",\n MAX(pending_count)::bigint AS \"pending_count!\",\n MAX(retrying_count)::bigint AS \"retrying_count!\",\n MAX(max_promotion_attempts)::integer AS \"max_promotion_attempts!\",\n MAX(conflicted_24h)::bigint AS \"conflicted_24h!\",\n MAX(promoted_24h)::bigint AS \"promoted_24h!\",\n MIN(oldest_pending_at) AS oldest_pending_at\n FROM status_metrics\n GROUP BY job_type\n ORDER BY job_type\n LIMIT $3\n OFFSET $4", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "job_type!", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "pending_count!", - "type_info": "Int8" - }, - { - "ordinal": 2, - "name": "retrying_count!", - "type_info": "Int8" - }, - { - "ordinal": 3, - "name": "max_promotion_attempts!", - "type_info": "Int4" - }, - { - "ordinal": 4, - "name": "conflicted_24h!", - "type_info": "Int8" - }, - { - "ordinal": 5, - "name": "promoted_24h!", - "type_info": "Int8" - }, - { - "ordinal": 6, - "name": "oldest_pending_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Int8", - "Int8" - ] - }, - "nullable": [ - null, - null, - null, - null, - null, - null, - null - ] - }, - "hash": "90841ab00a885ed00fd2c3a6b49c90b549a262963d281b863f4283af861ea5e5" -} diff --git a/.sqlx/query-92a41242e40c65c1eaad5aec4b2bd0b868b3e1fe7a7ba1ab015bf4c93c6fd431.json b/.sqlx/query-92a41242e40c65c1eaad5aec4b2bd0b868b3e1fe7a7ba1ab015bf4c93c6fd431.json new file mode 100644 index 0000000..b7ae055 --- /dev/null +++ b/.sqlx/query-92a41242e40c65c1eaad5aec4b2bd0b868b3e1fe7a7ba1ab015bf4c93c6fd431.json @@ -0,0 +1,103 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, job_type, organization_id, status::text AS \"status!\",\n priority, run_number, attempt, max_attempts, next_run_at, stage,\n progress_done, progress_total, created_at, updated_at\n FROM job_queue WHERE organization_id = $1 AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type = $3) ORDER BY created_at DESC, id DESC LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + null, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + }, + "hash": "92a41242e40c65c1eaad5aec4b2bd0b868b3e1fe7a7ba1ab015bf4c93c6fd431" +} diff --git a/.sqlx/query-a0383524b739ee8a06fba3e337e07e608049f1a18f43a548859d9683d6bae1f6.json b/.sqlx/query-a0383524b739ee8a06fba3e337e07e608049f1a18f43a548859d9683d6bae1f6.json new file mode 100644 index 0000000..c515cda --- /dev/null +++ b/.sqlx/query-a0383524b739ee8a06fba3e337e07e608049f1a18f43a548859d9683d6bae1f6.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, status::text AS \"status!\", run_number, attempt, updated_at\n FROM job_queue WHERE $1::uuid IS NULL AND id = ANY($2::uuid[]) ORDER BY id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "UuidArray" + ] + }, + "nullable": [ + false, + null, + false, + false, + false + ] + }, + "hash": "a0383524b739ee8a06fba3e337e07e608049f1a18f43a548859d9683d6bae1f6" +} diff --git a/.sqlx/query-a0514692991ed79f5c48f8b8758234202b016da9c5c40cac67f72e000455b9b8.json b/.sqlx/query-a0514692991ed79f5c48f8b8758234202b016da9c5c40cac67f72e000455b9b8.json deleted file mode 100644 index 9fe67e1..0000000 --- a/.sqlx/query-a0514692991ed79f5c48f8b8758234202b016da9c5c40cac67f72e000455b9b8.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT id, payload\n FROM job_queue\n WHERE organization_id = $1\n AND job_type = $2\n AND payload->>'run_id' = $3\n ORDER BY created_at DESC, id DESC\n LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "payload", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Text" - ] - }, - "nullable": [ - false, - false - ] - }, - "hash": "a0514692991ed79f5c48f8b8758234202b016da9c5c40cac67f72e000455b9b8" -} diff --git a/.sqlx/query-a611f047bb82ca7e1794ff283b09aee3ef1d9656babed381e73eb12626f3648c.json b/.sqlx/query-a611f047bb82ca7e1794ff283b09aee3ef1d9656babed381e73eb12626f3648c.json new file mode 100644 index 0000000..6d0321f --- /dev/null +++ b/.sqlx/query-a611f047bb82ca7e1794ff283b09aee3ef1d9656babed381e73eb12626f3648c.json @@ -0,0 +1,103 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, job_type, organization_id, status::text AS \"status!\",\n priority, run_number, attempt, max_attempts, next_run_at, stage,\n progress_done, progress_total, created_at, updated_at\n FROM job_queue WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type = $3) ORDER BY created_at DESC, id DESC LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + null, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + }, + "hash": "a611f047bb82ca7e1794ff283b09aee3ef1d9656babed381e73eb12626f3648c" +} diff --git a/.sqlx/query-a435ae7cc1a59f8a0c1fa1b5b54924319d9d321e60520c00480c5d220248e79c.json b/.sqlx/query-a74a899a375ff31b61d663d73cc4a13e467c246b649d4658484f183501c5d0c8.json similarity index 91% rename from .sqlx/query-a435ae7cc1a59f8a0c1fa1b5b54924319d9d321e60520c00480c5d220248e79c.json rename to .sqlx/query-a74a899a375ff31b61d663d73cc4a13e467c246b649d4658484f183501c5d0c8.json index dca5600..c375c3f 100644 --- a/.sqlx/query-a435ae7cc1a59f8a0c1fa1b5b54924319d9d321e60520c00480c5d220248e79c.json +++ b/.sqlx/query-a74a899a375ff31b61d663d73cc4a13e467c246b649d4658484f183501c5d0c8.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n next_run_at,\n idempotency_key,\n stage,\n enqueue_request_version,\n execution_resource_key,\n promotion_attempts,\n next_promotion_at,\n last_attempted_at,\n status,\n promoted_job_id,\n promoted_at,\n conflicted_at,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_enqueue_intents\n WHERE ($1::uuid IS NULL OR organization_id = $1)\n AND ($2::text IS NULL OR status = $2)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n next_run_at,\n idempotency_key,\n stage,\n enqueue_request_version,\n execution_resource_key,\n promotion_attempts,\n next_promotion_at,\n last_attempted_at,\n status,\n promoted_job_id,\n promoted_at,\n conflicted_at,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_enqueue_intents\n WHERE organization_id = $1 AND ($2::text IS NULL OR status = $2)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", "describe": { "columns": [ { @@ -154,5 +154,5 @@ false ] }, - "hash": "a435ae7cc1a59f8a0c1fa1b5b54924319d9d321e60520c00480c5d220248e79c" + "hash": "a74a899a375ff31b61d663d73cc4a13e467c246b649d4658484f183501c5d0c8" } diff --git a/.sqlx/query-ba0f6f24f4ce9dcb1d40cb75eb26228c07a691e183d68a876e0e8f778120d137.json b/.sqlx/query-ba0f6f24f4ce9dcb1d40cb75eb26228c07a691e183d68a876e0e8f778120d137.json new file mode 100644 index 0000000..c8ad8b2 --- /dev/null +++ b/.sqlx/query-ba0f6f24f4ce9dcb1d40cb75eb26228c07a691e183d68a876e0e8f778120d137.json @@ -0,0 +1,188 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n status::text AS \"status!\",\n priority,\n run_number,\n attempt,\n max_attempts,\n timeout_seconds,\n next_run_at,\n lease_expires_at,\n last_heartbeat_at,\n worker_id,\n started_at,\n finished_at,\n stage,\n progress_done,\n progress_total,\n progress_pct::float8 AS progress_pct,\n checkpoint,\n output,\n idempotency_key,\n status_reason,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_queue\n WHERE $1::uuid IS NULL AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "payload", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 9, + "name": "timeout_seconds", + "type_info": "Int4" + }, + { + "ordinal": 10, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 11, + "name": "lease_expires_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 12, + "name": "last_heartbeat_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "worker_id", + "type_info": "Text" + }, + { + "ordinal": 14, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 15, + "name": "finished_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 16, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 17, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 18, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 19, + "name": "progress_pct", + "type_info": "Float8" + }, + { + "ordinal": 20, + "name": "checkpoint", + "type_info": "Jsonb" + }, + { + "ordinal": 21, + "name": "output", + "type_info": "Jsonb" + }, + { + "ordinal": 22, + "name": "idempotency_key", + "type_info": "Text" + }, + { + "ordinal": 23, + "name": "status_reason", + "type_info": "Text" + }, + { + "ordinal": 24, + "name": "last_error_code", + "type_info": "Text" + }, + { + "ordinal": 25, + "name": "last_error_message", + "type_info": "Text" + }, + { + "ordinal": 26, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 27, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false, + null, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + true, + false, + true, + true, + null, + true, + true, + true, + true, + true, + true, + false, + false + ] + }, + "hash": "ba0f6f24f4ce9dcb1d40cb75eb26228c07a691e183d68a876e0e8f778120d137" +} diff --git a/.sqlx/query-ba9acdf8f6c1427f2488aec9ab85ac3eddf200e7f69494de198b3560888d03f7.json b/.sqlx/query-ba9acdf8f6c1427f2488aec9ab85ac3eddf200e7f69494de198b3560888d03f7.json new file mode 100644 index 0000000..fa50003 --- /dev/null +++ b/.sqlx/query-ba9acdf8f6c1427f2488aec9ab85ac3eddf200e7f69494de198b3560888d03f7.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workflow_step_dependencies (\n workflow_run_id, prerequisite_step_id, dependent_step_id, release_mode\n ) SELECT $1, r.prerequisite_step_id, r.dependent_step_id,\n r.release_mode::workflow_dependency_release_mode\n FROM jsonb_to_recordset($2::jsonb) AS r(\n prerequisite_step_id uuid, dependent_step_id uuid, release_mode text)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "ba9acdf8f6c1427f2488aec9ab85ac3eddf200e7f69494de198b3560888d03f7" +} diff --git a/.sqlx/query-bd7c8cceca8d42541d5816afd514778bd665cb721f5a44bcc700a08a8057fd95.json b/.sqlx/query-bd7c8cceca8d42541d5816afd514778bd665cb721f5a44bcc700a08a8057fd95.json deleted file mode 100644 index f427afe..0000000 --- a/.sqlx/query-bd7c8cceca8d42541d5816afd514778bd665cb721f5a44bcc700a08a8057fd95.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workflow_step_dependencies (\n workflow_run_id,\n prerequisite_step_id,\n dependent_step_id,\n release_mode\n )\n VALUES ($1, $2, $3, $4::text::workflow_dependency_release_mode)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Uuid", - "Uuid", - "Text" - ] - }, - "nullable": [] - }, - "hash": "bd7c8cceca8d42541d5816afd514778bd665cb721f5a44bcc700a08a8057fd95" -} diff --git a/runledger-runtime/.sqlx/query-af553a066c6cd1b24770d60492216a8a192bebc3257165496ee1523797b2df5d.json b/.sqlx/query-d3fdc5ee034100f4b1d0feca12b9e0cca4d8a97b4d1700f4f1cd4b2380e6ba43.json similarity index 74% rename from runledger-runtime/.sqlx/query-af553a066c6cd1b24770d60492216a8a192bebc3257165496ee1523797b2df5d.json rename to .sqlx/query-d3fdc5ee034100f4b1d0feca12b9e0cca4d8a97b4d1700f4f1cd4b2380e6ba43.json index 871d4e7..232afe9 100644 --- a/runledger-runtime/.sqlx/query-af553a066c6cd1b24770d60492216a8a192bebc3257165496ee1523797b2df5d.json +++ b/.sqlx/query-d3fdc5ee034100f4b1d0feca12b9e0cca4d8a97b4d1700f4f1cd4b2380e6ba43.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n jd.job_type AS \"job_type!\",\n COALESCE(SUM(jcmr.continued_24h), 0)::bigint AS \"continued_24h!\",\n COALESCE(SUM(jcmr.active_continued_count), 0)::bigint AS \"active_continued_count!\",\n COALESCE(MAX(jcmr.max_active_run_number), 0)::int4 AS \"max_active_run_number!\"\n FROM job_definitions jd\n LEFT JOIN job_continuation_metrics_rollup jcmr\n ON jcmr.job_type = jd.job_type\n AND ($1::uuid IS NULL OR jcmr.organization_id = $1)\n WHERE ($2::text IS NULL OR jd.job_type = $2)\n GROUP BY jd.job_type\n ORDER BY jd.job_type ASC", + "query": "SELECT\n jd.job_type AS \"job_type!\",\n COALESCE(SUM(jcmr.continued_24h), 0)::bigint AS \"continued_24h!\",\n COALESCE(SUM(jcmr.active_continued_count), 0)::bigint AS \"active_continued_count!\",\n COALESCE(MAX(jcmr.max_active_run_number), 0)::int4 AS \"max_active_run_number!\"\n FROM job_definitions jd\n LEFT JOIN job_continuation_metrics_rollup jcmr\n ON jcmr.job_type = jd.job_type\n AND ($3::boolean OR (jcmr.organization_id = $1 OR ($1::uuid IS NULL AND jcmr.organization_id IS NULL)))\n WHERE ($2::text IS NULL OR jd.job_type = $2)\n GROUP BY jd.job_type\n ORDER BY jd.job_type ASC", "describe": { "columns": [ { @@ -27,7 +27,8 @@ "parameters": { "Left": [ "Uuid", - "Text" + "Text", + "Bool" ] }, "nullable": [ @@ -37,5 +38,5 @@ null ] }, - "hash": "af553a066c6cd1b24770d60492216a8a192bebc3257165496ee1523797b2df5d" + "hash": "d3fdc5ee034100f4b1d0feca12b9e0cca4d8a97b4d1700f4f1cd4b2380e6ba43" } diff --git a/.sqlx/query-d72de6e5b2099c445d943a72a081987f9b1cc6ca13c877861d41fcf86466d6e6.json b/.sqlx/query-d72de6e5b2099c445d943a72a081987f9b1cc6ca13c877861d41fcf86466d6e6.json new file mode 100644 index 0000000..4783ff9 --- /dev/null +++ b/.sqlx/query-d72de6e5b2099c445d943a72a081987f9b1cc6ca13c877861d41fcf86466d6e6.json @@ -0,0 +1,103 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, job_type, organization_id, status::text AS \"status!\",\n priority, run_number, attempt, max_attempts, next_run_at, stage,\n progress_done, progress_total, created_at, updated_at\n FROM job_queue WHERE $1::uuid IS NULL AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type = $3) ORDER BY created_at DESC, id DESC LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + null, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + }, + "hash": "d72de6e5b2099c445d943a72a081987f9b1cc6ca13c877861d41fcf86466d6e6" +} diff --git a/.sqlx/query-d74db24f58285ef5796dcfe06b49136fea475553bcf65423c59c3067a2ef49b1.json b/.sqlx/query-d74db24f58285ef5796dcfe06b49136fea475553bcf65423c59c3067a2ef49b1.json new file mode 100644 index 0000000..b67bac0 --- /dev/null +++ b/.sqlx/query-d74db24f58285ef5796dcfe06b49136fea475553bcf65423c59c3067a2ef49b1.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workflow_steps (\n workflow_run_id, step_key, execution_kind, job_type, organization_id, payload,\n priority, max_attempts, timeout_seconds, stage, allow_handler_continuation,\n execution_resource_key, status, dependency_count_total,\n dependency_count_pending, dependency_count_unsatisfied\n ) SELECT $1, r.step_key, r.execution_kind::workflow_step_execution_kind,\n r.job_type, r.organization_id, COALESCE(r.payload, 'null'::jsonb), r.priority, r.max_attempts,\n r.timeout_seconds, r.stage, r.allow_handler_continuation, r.execution_resource_key,\n 'BLOCKED', r.dependency_count_total, r.dependency_count_pending, r.dependency_count_unsatisfied\n FROM jsonb_to_recordset($2::jsonb) AS r(\n step_key text, execution_kind text, job_type text, organization_id uuid, payload jsonb,\n priority int, max_attempts int, timeout_seconds int, stage text,\n allow_handler_continuation bool, execution_resource_key text,\n dependency_count_total int, dependency_count_pending int, dependency_count_unsatisfied int)\n RETURNING id, step_key", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "step_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Jsonb" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "d74db24f58285ef5796dcfe06b49136fea475553bcf65423c59c3067a2ef49b1" +} diff --git a/runledger-postgres/.sqlx/query-26acb4ac4804ccb2f1231f1e2bfe017f40f2e13c61b74158d5e09a66aac6c06d.json b/.sqlx/query-ddd4e90290e03da64cf22463877941b79baa32e938e95d66339a91fc3756d599.json similarity index 95% rename from runledger-postgres/.sqlx/query-26acb4ac4804ccb2f1231f1e2bfe017f40f2e13c61b74158d5e09a66aac6c06d.json rename to .sqlx/query-ddd4e90290e03da64cf22463877941b79baa32e938e95d66339a91fc3756d599.json index e8d6762..28b299f 100644 --- a/runledger-postgres/.sqlx/query-26acb4ac4804ccb2f1231f1e2bfe017f40f2e13c61b74158d5e09a66aac6c06d.json +++ b/.sqlx/query-ddd4e90290e03da64cf22463877941b79baa32e938e95d66339a91fc3756d599.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n status::text AS \"status!\",\n priority,\n run_number,\n attempt,\n max_attempts,\n timeout_seconds,\n next_run_at,\n lease_expires_at,\n last_heartbeat_at,\n worker_id,\n started_at,\n finished_at,\n stage,\n progress_done,\n progress_total,\n progress_pct::float8 AS progress_pct,\n checkpoint,\n output,\n idempotency_key,\n status_reason,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_queue\n WHERE id = $1\n AND ($2::uuid IS NULL OR organization_id = $2)\n LIMIT 1", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n status::text AS \"status!\",\n priority,\n run_number,\n attempt,\n max_attempts,\n timeout_seconds,\n next_run_at,\n lease_expires_at,\n last_heartbeat_at,\n worker_id,\n started_at,\n finished_at,\n stage,\n progress_done,\n progress_total,\n progress_pct::float8 AS progress_pct,\n checkpoint,\n output,\n idempotency_key,\n status_reason,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_queue\n WHERE id = $1\n AND ($3::bool OR organization_id IS NOT DISTINCT FROM $2::uuid)\n LIMIT 1", "describe": { "columns": [ { @@ -147,7 +147,8 @@ "parameters": { "Left": [ "Uuid", - "Uuid" + "Uuid", + "Bool" ] }, "nullable": [ @@ -181,5 +182,5 @@ false ] }, - "hash": "26acb4ac4804ccb2f1231f1e2bfe017f40f2e13c61b74158d5e09a66aac6c06d" + "hash": "ddd4e90290e03da64cf22463877941b79baa32e938e95d66339a91fc3756d599" } diff --git a/.sqlx/query-f6b58ff379b8af9e2ab6346743ad1da85494d295268da6627d59c2bcc70b9fb0.json b/.sqlx/query-f6b58ff379b8af9e2ab6346743ad1da85494d295268da6627d59c2bcc70b9fb0.json deleted file mode 100644 index ab6c006..0000000 --- a/.sqlx/query-f6b58ff379b8af9e2ab6346743ad1da85494d295268da6627d59c2bcc70b9fb0.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH locked_job AS MATERIALIZED (\n SELECT id\n FROM job_queue\n WHERE id = $1\n AND run_number = $2\n AND attempt = $3\n AND worker_id = $4\n AND status = 'LEASED'\n AND lease_expires_at IS NOT NULL\n FOR UPDATE\n )\n UPDATE job_queue\n SET stage = COALESCE($5, stage),\n progress_done = COALESCE($6, progress_done),\n progress_total = COALESCE($7, progress_total),\n checkpoint = COALESCE($8::jsonb, checkpoint),\n updated_at = now()\n FROM locked_job\n WHERE job_queue.id = locked_job.id\n AND job_queue.lease_expires_at > clock_timestamp()", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Int4", - "Int4", - "Text", - "Text", - "Int8", - "Int8", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "f6b58ff379b8af9e2ab6346743ad1da85494d295268da6627d59c2bcc70b9fb0" -} diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e3bfce..5d1a4be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,50 @@ All notable changes to this workspace are documented here. ## [Unreleased] +### Added + +- Add explicit `JobReadScope` selection to job, event, log, and enqueue-intent + reads through `get_job_by_id_with_scope`, `list_jobs_with_scope`, + `list_job_events_with_scope`, `list_job_logs_with_scope`, + `get_job_enqueue_intent_by_id_with_scope`, and + `list_job_enqueue_intents_with_scope`, with `JobReadListFilter` and + `JobEnqueueIntentReadListFilter` for scoped lists. +- Add `get_job_metrics_with_scope`, `get_job_continuation_metrics_with_scope`, + and `get_job_enqueue_intent_metrics_with_scope` for exact global, exact tenant, + or administrative visibility through `JobReadScope`. Intent metrics use + `JobEnqueueIntentReadMetricsFilter`; job and continuation metrics preserve + zero-count registered definitions. +- Add `get_job_payload_by_idempotency_key_with_scope` and + `get_latest_job_payload_for_run_with_scope` using `JobScope::{Global, + Organization}`. Payload lookups have no Admin wildcard because keys and JSON + run IDs can repeat across scopes. These metric and payload APIs and the new + intent metrics filter are available through `jobs` and `prelude`. +- Add shared `JobSpec`, `JobSpecs`, and `JobDefinitionSettings` for producers + and workers, plus `JobCatalog::from_specs` and `try_handler_for_spec` to + validate handler bindings against those definitions. +- Add `JobContract` and owned `JobSubmission` requests for typed producers, + and opt-in `TypedJobHandler` adapters for decoding payloads at dispatch. + Malformed payloads produce a terminal failure by default. +- Add `JobExecution` and `JobExecutionServices` for handler deadline and budget + access, checkpoint reads, and lease-fenced progress and checkpoint writes. + Handlers can opt in through `execute_with_services` or `JobExecutionHandler`. +- Add `enqueue_job_with_outcome` to manage its own transaction and return + whether a job was inserted or an existing strict-idempotent request matched. +- Add `list_job_summaries` with `JobSummaryFilter`, `JobSummaryCursor`, and + `JobSummary` for keyset pagination without JSON bodies, plus + `get_job_statuses_with_scope` and `JobStatusRecord` for scoped bulk status reads. +- Add `migration_bundle`, `MigrationBundle`, and `RUNLEDGER_POSTGRES_VERSION` + to inspect embedded migrations and compose application migration fingerprints. + Live database compatibility still requires the existing startup checks. +- Add `WorkflowDagBuilder::step` and `external` for fluent step insertion, plus + `active_key` and `clear_active_key` for workflow active-key configuration. + ### Changed +- Keep exact-scope payload lookups indexable under generic prepared plans. +- Remove redundant timeout-policy statements and duplicate locking from progress + writes while preserving locked-state validation, lease fencing, and audit + atomicity. Document and test strict handler deadline precedence. - Breaking: `JobCompletion` now keeps progress and checkpoint state private. `.progress(done, total)` validates non-negative values and `done <= total`, returning `Result`; use the new accessors when inspecting a completion. @@ -19,6 +61,18 @@ All notable changes to this workspace are documented here. ### Upgrade notes +- Apply `202609050001_job_summary_pagination` before starting the new runtime. + Its two ordinary index builds block queue writes until the SQLx migration + commits; schedule an appropriate deployment window. The migration is recorded + in `_sqlx_migrations`, but is deliberately outside the custom + `runledger_migration_history` compatibility fence. The current startup guard + still requires it. See the README migration guide for rollout details. + +- Scope APIs are additive and require no new database migration. Legacy metric + calls still aggregate all scopes when the organization filter is absent; + legacy payload lookups remain tenant-only. Use the `_with_scope` variants for + exact-global reads and authorize the chosen scope in application code. + - Update completion builders to handle the `Result` returned by `.progress` and replace direct `progress_done`, `progress_total`, or `checkpoint` field access with `progress_done()`, `progress_total()`, and `checkpoint_value()`. diff --git a/Cargo.lock b/Cargo.lock index 3c55caf..9b4d0fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2463,6 +2463,7 @@ dependencies = [ "runledger-test-support", "serde", "serde_json", + "sha2", "sqlx", "tokio", "tracing", @@ -2479,6 +2480,7 @@ dependencies = [ "runledger-core", "runledger-postgres", "runledger-test-support", + "serde", "serde_json", "sqlx", "thiserror", diff --git a/README.md b/README.md index cd8d4db..a5ae696 100644 --- a/README.md +++ b/README.md @@ -94,12 +94,16 @@ Add the libraries to your service: ```toml [dependencies] -runledger-core = "0.9" -runledger-postgres = "0.9" -runledger-runtime = "0.9" +runledger-core = "0.12.0" +runledger-postgres = "0.12.0" +runledger-runtime = "0.12.0" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sqlx = { version = "0.8.6", features = ["runtime-tokio", "postgres"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal"] } [dev-dependencies] -runledger-test-support = "0.9" +runledger-test-support = "0.12.0" ``` The published crates require **Rust 1.88+** and **PostgreSQL 18+**. Older @@ -116,49 +120,91 @@ use runledger_runtime::prelude::*; ## Quick start -Downstream services typically run a web/API process that enqueues work and a -separate worker process that runs handlers against the same database. A minimal -worker: +Run a producer and a worker as separate processes against the same PostgreSQL 18 +database. This example prints a greeting, using one shared job identity and typed +payload. It needs only the dependencies above. For a new service, create the +following files under `src/bin/`, with the shared module at +`src/bin/shared/mod.rs` (so Cargo does not treat it as another binary). +Shared contract (`src/bin/shared/mod.rs`): + + +```rust +use runledger_core::jobs::JobType; +use runledger_postgres::jobs::JobEnqueue; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +pub const GREETING_JOB: JobType<'static> = JobType::new("jobs.greeting.print"); + +#[derive(Serialize, Deserialize)] +pub struct Greeting { + pub name: String, +} + +pub fn request<'a>(payload: &'a Value, key: &'a str) -> JobEnqueue<'a> { + JobEnqueue { + job_type: GREETING_JOB, + organization_id: None, + payload, + priority: None, + max_attempts: None, + timeout_seconds: None, + next_run_at: None, + idempotency_key: Some(key), + stage: None, + } +} +``` + +Worker (`src/bin/worker.rs`): + + ```rust +pub mod shared; + use std::time::Duration; use runledger_core::jobs::{JobCompletion, JobContext, JobFailure, JobType}; use runledger_core::prelude::async_trait; -use runledger_runtime::Supervisor; -use runledger_runtime::catalog::JobCatalog; -use runledger_runtime::config::JobsConfig; -use runledger_runtime::registry::JobHandler; +use runledger_runtime::{Supervisor, catalog::JobCatalog, registry::JobHandler}; use serde_json::Value; +use shared::{GREETING_JOB, Greeting}; use sqlx::postgres::PgPoolOptions; -struct SendEmail; +struct PrintGreeting; #[async_trait] -impl JobHandler for SendEmail { +impl JobHandler for PrintGreeting { fn job_type(&self) -> JobType<'static> { - JobType::new("jobs.email.send") + GREETING_JOB } - async fn execute(&self, _context: JobContext, _payload: Value) -> Result { - // do the work - Ok(JobCompletion::success()) + async fn execute( + &self, + _context: JobContext, + payload: Value, + ) -> Result { + let greeting: Greeting = serde_json::from_value(payload) + .map_err(|_| JobFailure::terminal("greeting.invalid_payload", "Expected a name."))?; + println!("Hello, {}!", greeting.name); + JobCompletion::success().progress(1, 1).map_err(|_| { + JobFailure::terminal("greeting.invalid_progress", "Invalid completion counts.") + }) } } -async fn run_worker() -> Result<(), Box> { +#[tokio::main] +async fn main() -> Result<(), Box> { let pool = PgPoolOptions::new() .connect(&std::env::var("DATABASE_URL")?) .await?; - - // Apply the bundled schema (or validate it; see "Database schema and migrations"). + // For a fresh database. Existing deployments must follow the migration runbook. runledger_postgres::migrate_after_idempotency_cutover(&pool).await?; - - // Register handlers and sync their job definitions. - let catalog = JobCatalog::new().handler(SendEmail); + let catalog = JobCatalog::new().handler(PrintGreeting); catalog.sync_definitions(&pool).await?; + println!("worker ready; producers can now enqueue greetings"); - // Run the supervisor until Ctrl-C, with a 30s shutdown drain deadline. let supervisor = Supervisor::builder_from_env(&pool)? .with_catalog(&catalog) .build()?; @@ -172,20 +218,74 @@ async fn run_worker() -> Result<(), Box> { Duration::from_secs(30), ) .await; - - // Keep pool cleanup independent from the shutdown result. pool.close().await; shutdown_result?; Ok(()) } ``` -From anywhere else (such as your API), enqueue a job against the same pool: +Producer (`src/bin/producer.rs`): + ```rust -let job = runledger_postgres::jobs::enqueue_job(&pool, /* JobEnqueue */).await?; +pub mod shared; + +use runledger_postgres::jobs::enqueue_job_tx; +use shared::{Greeting, request}; +use sqlx::postgres::PgPoolOptions; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let name = std::env::args() + .nth(1) + .ok_or("usage: producer ")?; + let key = std::env::args().nth(2).ok_or("missing request-key")?; + let pool = PgPoolOptions::new() + .connect(&std::env::var("DATABASE_URL")?) + .await?; + runledger_postgres::ensure_schema_compatible_after_idempotency_cutover(&pool).await?; + + let payload = serde_json::to_value(Greeting { name })?; + let mut tx = pool.begin().await?; + // Persist application changes with this same transaction when needed. + let job_id = enqueue_job_tx(&mut tx, &request(&payload, &key)).await?; + tx.commit().await?; + println!("enqueued {job_id}"); + pool.close().await; + Ok(()) +} ``` +Start the worker first; it applies the schema to a fresh database and syncs the +job definition. Existing deployments should follow the +[migration runbook](#database-schema-and-migrations) before starting this worker. +Wait for `worker ready`, then submit from a second terminal using the same +`DATABASE_URL`: + +```bash +# Terminal 1 +export DATABASE_URL=postgres://postgres:postgres@localhost:5432/runledger +cargo run --bin worker + +# Terminal 2 +export DATABASE_URL=postgres://postgres:postgres@localhost:5432/runledger +cargo run --bin producer -- Ada greeting:1 +``` + +Inside this repository, use `cargo run -p runledger-runtime --example worker` +and `cargo run -p runledger-runtime --example producer -- Ada greeting:1` instead. +The worker prints `Hello, Ada!` and persists completion progress of 1/1. Press +Ctrl-C to drain and stop it. Execution is at least once, so even this print can +repeat after an interrupted attempt; real external effects need their own +idempotency protection. + +The producer commits the enqueue before reporting success. Application writes +can share that transaction: rolling it back also removes the enqueue. Reuse the +same request key and payload to retry a submission; use a new key for new work. +A changed payload with the same key is an idempotency conflict. This direct +submission requires an enabled job definition. If the producer must commit +before worker registration, use the [durable transactional handoff](#durable-transactional-handoff). + Notes on the worker lifecycle: - `run_until_shutdown()` is the preferred facade for worker binaries: it observes @@ -240,7 +340,7 @@ feature, not something to recreate by polling jobs or chaining handlers by hand. | --- | --- | | One independent retried unit of work | `runledger_postgres::jobs::enqueue_job` | | Commit application state and a future job request before its definition exists | `JobEnqueueIntent` and `record_job_enqueue_intent_tx` | -| Multi-step work with dependencies | `WorkflowDagBuilder` (simple DAGs), or `WorkflowRunEnqueueBuilder` / `WorkflowStepEnqueueBuilder` (advanced), then `enqueue_workflow_run` | +| Multi-step work with dependencies | `WorkflowDagBuilder` (`.step(...)` for configured steps, `.external(...)` for external work), or the low-level run/step builders, then `enqueue_workflow_run` | | Multi-step work with a durable JSON result | Declare a result step, enqueue with `enqueue_workflow_run_handle`, then call `WorkflowRunHandle::get_result` | | Fan-out, fan-in, or ordered stages | `WorkflowDagBuilder::after_success` / `after_terminal` (or lower-level `depends_on_success` / `depends_on_terminal`) | | Human/API approval or another external gate | External workflow steps and `complete_external_workflow_step` | @@ -337,7 +437,9 @@ deferred, so an operator can restore consistency and retry without replacing the durable request. Runledger keeps conflicted intents as immutable evidence; if replacement work is safe, the application must submit it deliberately under a new idempotency key. -Query `get_job_enqueue_intent_metrics` to alert on oldest pending age, +Query `get_job_enqueue_intent_metrics_with_scope` with a +`JobEnqueueIntentReadMetricsFilter` selecting the authorized `JobReadScope` to +alert on oldest pending age, `retrying_count`, `max_promotion_attempts`, and increases in `conflicted_24h`. The retry count and maximum attempt count describe only intents that are still pending, so resolved promoted or conflicted history cannot inflate the active @@ -521,6 +623,50 @@ compatibility APIs while callers migrate. Completion disposition and final output are intentionally private; inspect them with `disposition()` / `output()` and use constructors rather than struct literals. +### Live handler execution services + +Implement `JobExecutionHandler` when a handler needs the runtime's deadline or +durable progress writes. Register `handler.into_job_handler()` with the usual +`JobRegistry` or `JobCatalog`. Existing `JobHandler` implementations continue +to work unchanged; the worker dispatches through the new default +`execute_with_services` method. See the compiling +[counter example](runledger-runtime/examples/checkpointed_counter.rs). + +`JobExecution::deadline()` is the same monotonic deadline the worker enforces, +starting after the running transition succeeds. `remaining_budget()` includes +time spent awaiting progress writes. `remaining_work_budget(reserve)` subtracts +an application-selected reserve for its final checkpoint or cleanup and +saturates at zero. Runtime completion persistence happens after the handler +returns; this reserve does not impose a deadline on that persistence. +The worker must observe the handler result strictly before the deadline. At or +after it, timeout takes precedence over success or continuation, including when +the handler and timer become ready together or a long poll crosses the deadline. +Already committed checkpoints and external effects are not undone. + +`checkpoint::()` decodes the claimed resume snapshot; applications still +validate their checkpoint versions and domain invariants. +`persist_progress(JobExecutionUpdate { .. })` atomically commits ordinary +progress and a checkpoint using the exact live lease, without a queue reread +or caller-supplied worker/run/attempt arguments. `save_checkpoint(&value)` +serializes and commits only a checkpoint. Both operations must be awaited. +Omitted fields retain their durable values; successful writes do not mutate +the invocation's resume snapshot. + +The handle borrows runtime services, so it cannot escape into a detached task. +`JobExecutionError` distinguishes lease loss, deadline expiry, persistence +failure, and invalid input, and converts to `JobFailure` for `?` propagation. +The worker stops polling a handler when its progress write discovers lease loss, +even if the handler ignores that error. Successful writes acknowledge commit; +an error or cancellation can leave an indeterminate commit outcome. +External effects still require application idempotency. + +Custom runtimes must supply `JobExecutionServices` and invoke +`JobHandler::execute_with_services`. Calling legacy `execute` directly on +an adapted execution-services handler returns `job.execution_services_required`. +SQLx pools and persistence errors remain outside the serializable `JobContext`. +Validated [OneSales and IdentityPro migration patches](docs/execution-services-migrations/README.md) +show how to replace existing execution-state reconstruction. + ### Handler-selected retry timing When a provider supplies a dynamic reset time, a handler can attach either a @@ -646,7 +792,8 @@ condition; use a nonzero delay for polling-style work and do not return `continue_now()` forever. Production handlers should version their checkpoint shape, make every slice idempotent, enforce a logical deadline or run limit, canary activation by job type or tenant, and alert on continuation rate and run -depth. `get_job_continuation_metrics` returns a +depth. `get_job_continuation_metrics_with_scope` takes an explicit +`JobReadScope` and returns a `JobContinuationMetricsRecord` per job type with `continued_24h`, `active_continued_count`, and `max_active_run_number` for canary and runaway-loop alerts. Active counts include only jobs whose current run was created by a @@ -920,6 +1067,13 @@ not provide an implicit union helper because that can hide ownership mistakes. ### Job definition catalog +Producer processes can share provider-free `JobSpec` / `JobContract` definitions +with workers. Use `JobCatalog::from_specs` to validate complete worker bindings, +`TypedJobHandler` for opt-in payload decoding, and `JobContract::submit` for typed +direct requests. The pool-owning `enqueue_job_with_outcome` distinguishes new +work from identical retries. See the [shared contracts guide](docs/shared-job-specs.md) +for synchronization, wire compatibility, and migration examples. + `sync_definitions` is **additive**: it owns the definition fields it writes (`version`, retry limits, timeout, priority), restoring them to effective catalog values on each startup. It preserves an existing *disabled* row, so operator @@ -975,6 +1129,7 @@ These examples and integration references are compile-checked: - [External workflow gate](runledger-postgres/examples/external_gate.rs) - [Append workflow steps](runledger-postgres/examples/append_workflow_steps.rs) - [Scheduled job entrypoint](runledger-postgres/examples/schedule_job.rs) +- [Shared producer/worker quick start](runledger-runtime/examples/producer_worker/) - [Worker binary skeleton](runledger-runtime/examples/worker_binary.rs) - [Packaged continuation, retry timing, direct recovery, replay, and metrics smoke test](smoke/external-consumer/tests/smoke.rs) - [Active-workflow key integration reference](runledger-postgres/tests/workflow_active_claims.rs) @@ -994,14 +1149,65 @@ trusted all-tenant surface. The legacy nullable read helpers remain available: their `None` scope retains the historical admin wildcard. These helpers use the same workflow-type substring filtering as the TUI. -Use `get_job_continuation_metrics` for continuation canaries and runaway-loop -alerts. Each `JobContinuationMetricsRecord` reports the prior 24 hours' successful +Use `get_job_metrics_with_scope` and `get_job_continuation_metrics_with_scope` +with `JobReadScope::{Global, Organization, Admin}` for queue counters and +continuation canaries. Registered job types remain visible with zero counts +when the selected scope has no matching rows. Job duration metrics retain the +average of per-scope percentiles when aggregating scopes. Each +`JobContinuationMetricsRecord` reports the prior 24 hours' successful continuations, the number of pending/leased jobs whose current run was created by continuation, and the highest current run number among those active jobs. -Passing no organization filter aggregates all scopes; it does not mean exact -global scope. -Durable event consumers should call `list_job_events` and prefer +Use `get_job_enqueue_intent_metrics_with_scope` with +`JobEnqueueIntentReadMetricsFilter::new(scope, limit, offset)` for intent metrics. +Its optional `with_job_type` filter is exact. Backlog, retries, and oldest age +include pending intents only; promoted and conflicted counts cover the last +24 hours. Types with only older terminal history are omitted. Results are +ordered by job type, with limits of 1–1000 and nonnegative offsets. +All three legacy metric APIs retain `None` (or no organization filter) as +all scopes and `Some(id)` as exactly that tenant. + +For payload reads, use `get_job_payload_by_idempotency_key_with_scope` or +`get_latest_job_payload_for_run_with_scope` with `JobScope::Global` or +`JobScope::Organization(id)`. Keys and JSON `run_id` values may repeat across +scopes, so these single-result lookups have no admin wildcard. The latest +lookup orders by `created_at DESC, id DESC`. Both return `None` for an absent +match; nil UUIDs are ordinary UUID values, not global/admin sentinels. Their +legacy counterparts still require a tenant UUID. These APIs and the new intent +filter are exported through both `jobs` and `prelude`. Applications must +authorize every selected read scope, including exact payload scopes. + +Use `JobReadScope::Global` for jobs and intents with no organization, +`JobReadScope::Organization(id)` for one tenant, and `JobReadScope::Admin` +for visibility across all tenants and global rows. Applications must authorize +the selected scope; the enum only controls row filtering. + +Use `get_job_by_id_with_scope`, `list_jobs_with_scope` (with +`JobReadListFilter`), `list_job_events_with_scope`, and +`list_job_logs_with_scope` for job inspection. Intent inspection uses +`get_job_enqueue_intent_by_id_with_scope` and +`list_job_enqueue_intents_with_scope` (with +`JobEnqueueIntentReadListFilter::new(scope, limit, offset)`). + +For dashboards and recovery scans, `list_job_summaries` accepts a +`JobSummaryFilter` with an explicit `JobReadScope`, optional status and exact +`JobType`, a limit of 1–1,000, and an optional `after` cursor. It omits payload, +checkpoint, output, and free-form errors. Continue with the last row's +`cursor()`; preserve timestamp microseconds and keep the scope and filters +unchanged. Pages use descending `(created_at, id)` order and are observations, +not a snapshot across concurrent status changes. `get_job_statuses_with_scope` +reads up to 1,000 input IDs in one statement, returning unique visible rows in +ID order; absent and unauthorized-scope IDs are both omitted. + +Initial and appended workflow graphs insert steps and dependencies in chunks +of at most 256 rows within the existing transaction. Root releases still use +the ordinary enqueue/audit path. See the +[PostgreSQL 18 measurements and batch design](docs/operational-costs-2026-09-05.md) +for results, atomicity, and direct-job batch considerations. +The legacy APIs retain their existing behavior: an absent organization filter +means unrestricted visibility, including organization-owned rows. + +Durable event consumers should call `list_job_events_with_scope` and prefer `JobEventRecord::decoded_payload()` for Runledger-authored continuation, administrative requeue, and successful-replay payloads. The decoded enums are non-exhaustive: keep wildcard arms and retain `JobEventRecord::payload` as the @@ -1191,6 +1397,10 @@ forward migrations: anti-joins, removes the compatibility triggers, reciprocal FK/unique constraint, and `job_queue.workflow_step_id`, then advances the custom compatibility fence so pre-contract binaries refuse the destructive schema. +- `202609050001_job_summary_pagination` — adds scope/creation/ID and creation/ID + indexes for compact cursor pages. This additive migration is outside the + custom compatibility fence. Its ordinary index builds block queue writes + until the migration commits; schedule it for an appropriate deployment window. Every forward migration from `202607190001_job_replays_and_continuation_metrics` through @@ -1222,6 +1432,41 @@ in-place upgrade from the older multi-file standalone history; apply later forward migrations normally. The workspace-root `migrations/` directory is the canonical source for development and review. +### Migration identity and bundle manifest + +`runledger_postgres::migration_bundle()` inspects the compiled crate without +opening a database or reading workspace files: + +```rust +let bundle = runledger_postgres::migration_bundle(); +let library_version = bundle.library_version(); // also RUNLEDGER_POSTGRES_VERSION +let content: [u8; 32] = bundle.bundle_fingerprint(); +let pipeline: [u8; 32] = bundle.pipeline_fingerprint(); +for migration in bundle.migrations() { + // version, description, migration_type, checksum, no_tx, and exact SQL + println!("{} {}", migration.version, migration.description); +} +``` + +The manifest includes up and down entries, sorted by version then direction +(Simple, ReversibleUp, ReversibleDown). The content fingerprint hashes their +metadata and raw SQLx checksums. The pipeline fingerprint additionally includes +the compiled `runledger-postgres` version, so a new library release invalidates +cached templates even when its SQL is unchanged. Rustdoc specifies the versioned, +length-framed SHA-256 encoding; fingerprints are 32 raw bytes, not hex strings. + +Use the pipeline fingerprint as one input to your application's template/schema +fingerprint. Retain the host pipeline revision, other libraries' inputs, and host +migration ordering. Helper-only changes in same-version path/patched builds +require an additional host-owned source revision. Neither fingerprint proves a +live database is compatible or includes your application's SQLx history. + +The [composition example](runledger-postgres/examples/migration_identity.rs) +shows how to combine these inputs. The [consumer guide](docs/migration-identity/README.md) +includes an IdentityPro adapter patch and validation against HOCR's historical +vendored SQL. Keep application migration ordering and cutover decisions in the +application, and use the startup helpers below to apply or validate live state. + ### Applying or validating the schema Two supported startup modes: @@ -1547,7 +1792,7 @@ The preparation script starts from a clean working tree or resumes an existing generated release diff whose manifests are already at the requested version. It rejects changes outside the files it generates. The script bumps publishable crates through their shared workspace package version, updates the explicit -published workspace dependency pins, refreshes the root and standalone smoke +published workspace dependency pins and README installation/release versions, refreshes the root and standalone smoke lockfiles plus SQLx offline metadata, runs workspace tests and the locked packaged smoke test, dry-runs `runledger-core`, packages the library crates, and build-verifies the packaged `runledger-tui` binary. It also verifies that @@ -1555,6 +1800,12 @@ every crate archive contains the repository license. If publishing manually, run `./scripts/refresh-sqlx-cache.sh` before publishing `runledger-postgres` or `runledger-runtime` and commit any resulting `.sqlx/` changes. +`python3 scripts/check-readme.py` (Python 3.11+) checks current installation and release command +versions against `Cargo.toml` and checks the quick-start snippets against the +compiled example sources. CI and both release scripts run this check; historical +upgrade notes retain their original versions. The PostgreSQL example test runs +with `cargo test -p runledger-runtime --example worker`. + After reviewing and committing the prepared diff: ```bash diff --git a/docs/api-audit-2026-09-05.md b/docs/api-audit-2026-09-05.md new file mode 100644 index 0000000..861a5d9 --- /dev/null +++ b/docs/api-audit-2026-09-05.md @@ -0,0 +1,174 @@ +# Runledger API and downstream integration audit — 2026-09-05 + +Runledger's biggest opportunity is to reduce the infrastructure each application must implement around its durable engine. The engine already supplies workflows, continuations, enqueue intents, execution resources, and recovery. The convenient application interfaces do not yet cover those capabilities consistently. + +This is a source-based audit of Runledger 0.12.0 at `3ed14048c5dcbabcba95938549cd8649b92c949f` and the local downstream working trees. Recommendations below distinguish confirmed boundary problems, ergonomic additions, performance investigations, and adoption work. No production incident, throughput result, or database reproduction is claimed. + +**Downstream inventory** + +Versions below come from each checkout's `Cargo.lock`, cross-checked against its workspace manifest. Alternate checkouts are not counted as independent product demand. + +| Application / checkout | Locked Runledger version | Integration evidence and implication | +| --- | --- | --- | +| `onesales` | 0.12.0 | Broad workflow and CRM use; shared handler-support crate; manually composed definitions and handler registration; progress/checkpoint adapters. Strongest evidence for execution services and composable contracts. | +| `onesales-checkout-1` | 0.1.2 | Older checkout inventoried for version/adoption context; not treated as a second current consumer. | +| `creditkit-platform` | 0.12.0 | Catalog/Supervisor, document workflows, reconciliation, tenant checks, and field-by-field payload parsing. Useful second current integration for typed handlers and application projections. | +| `creditkit` | 0.6.0 | Older integration of the same application family. Its workarounds must be assessed against newer features before adding APIs. | +| `identitypro` | 0.12.0 | Separate producer/worker features, local job contracts, bounded continuations, recovery scans, and composed migrations. Strongest evidence for producer contracts, explicit scopes, and execution budgets. | +| `perdify`, `perdify-checkout-1` | 0.1.1 | Shared integration crate implements definition setup, duplicate lookup, savepoints, and payload refresh. Evidence for upgrade assistance and a possible coalescing API; not evidence that current enqueue lacks idempotency. | +| `vatbot` | 0.4.0 | Runledger schedules an extraction pump over an application-owned claim/retry engine. Runledger currently owns the pump's lifecycle, not each document's lifecycle. | +| `hocr-next` | postgres/core 0.5.0; no runtime in lockfile | Migration/bootstrap integration. Its migration test verifies vendored SQL/checksums against the upstream bundle. No production Runledger handler integration was found in the searched Rust source. | + +The inventory searched manifests across `~/Documents`, then inspected representative production call sites and relevant tests in the application families above. It did not read every matching source file or deeply audit the alternate historical checkouts. + +**AP-RUST-001 — Handlers reconstruct execution state that the runtime already owns** + +Severity: **medium**. Confidence: **high**. Boundary: handler execution through `JobHandler` and `JobContext`. Leak kind: execution lifecycle and behavioral coupling. Priority: **P1**. + +`JobContext` carries identity, tenant, worker, and checkpoint, but no effective execution deadline, remaining budget, or operation for persisting progress. The runtime owns the timeout, while consumers read the queue and reconstruct parts of that state themselves. + +Evidence: + +- [JobContext](/home/aa/Documents/runledger/runledger-core/src/jobs/runtime_types.rs:758) contains the execution snapshot; [worker deadline calculation](/home/aa/Documents/runledger/runledger-runtime/src/worker/execution.rs:292) remains private. +- [OneSales CRM execution state](/home/aa/Documents/onesales/crates/onesales-jobs/src/crm_sync_common/checkpoint.rs:94) fetches `JobQueueRecord`, checks the lease, reads timeout/checkpoint, derives a local deadline, and reconstructs progress-write arguments. +- [Shared lease/progress helpers](/home/aa/Documents/onesales/crates/onesales-job-support/src/lib.rs:362) compare status/run/attempt/worker and translate the stable lease-mismatch code. Similar consumers exist in stakeholder, signal-classification, and champion-tracking progress modules. +- [IdentityPro execution budgets](/home/aa/Documents/identitypro/crates/identitypro-jobs/src/definitions.rs:59) maintain work timeouts and completion reserves outside the runtime. + +Consequence: extra database reads and repeated lifecycle plumbing; locally calculated budgets can diverge from the actual handler deadline. The OneSales helper is an observation, not authoritative proof of a still-live lease: it does not test lease expiry, and any observation can become stale. Runledger's actual progress mutation remains database-fenced. + +Change test: extending the runtime's execution context or adjusting timeout accounting should not require every handler family to update queue-row reads and deadline reconstruction. + +Intent/counterevidence: storage independence in `runledger-core` is deliberate. The PostgreSQL progress API and `JobLeaseIdentity` already exist, and strings such as `job.lease_owner_mismatch` are documented stable contracts. The gap is the lack of an application-facing execution service, not the existence of low-level APIs or string codes. + +Smallest repair: add an opt-in execution interface alongside the current handler contract. Supply a runtime-owned execution handle with the authoritative deadline, remaining work budget, typed checkpoint access, and lease-fenced progress/checkpoint methods. Keep database machinery outside the serializable core context. Adapt legacy handlers; do not add required fields or arguments to their existing public contract in an additive release. Progress writes must remain awaited and explicit about durability. A cancellation signal can aid cooperative cleanup, but cannot promise cancellation of already-issued external effects. + +Validation: migrate one CRM handler and one bounded IdentityPro handler; remove their timeout/lease reconstruction; exercise timeout, lease loss, and checkpoint persistence with PostgreSQL 18. + +**AP-RUST-002 — The single-source catalog requires instantiated workers** + +Severity: **medium**. Confidence: **high**. Boundary: `JobCatalog`. Leak kind: dependency/ownership coupling. Priority: **P1**. + +The catalog couples reusable job metadata and enqueue validation to concrete handler objects. A producer process cannot use its complete job-contract surface without building those handlers or maintaining a separate representation. + +Evidence: + +- [CatalogJob](/home/aa/Documents/runledger/runledger-runtime/src/catalog/types.rs:191) contains an `Arc`; registration methods require handlers. [Definition materialization](/home/aa/Documents/runledger/runledger-runtime/src/catalog/sync.rs:122) is private. +- [IdentityPro JobDefinitionSpec](/home/aa/Documents/identitypro/crates/identitypro-jobs/src/definitions.rs:336) and [JobContract](/home/aa/Documents/identitypro/crates/identitypro-jobs/src/definitions.rs:380) provide an application-owned, feature-safe representation for storage clients and workers. +- [OneSales definitions and registration](/home/aa/Documents/onesales/crates/onesales-jobs/src/lib.rs:85) remain separate collections and wiring paths. Its crate guide explicitly requires keeping them aligned. + +Consequence: consumers build another catalog layer, duplicate conversions, and test that definitions and handlers agree. Provider credentials and service construction belong to worker startup, not producer-side validation. + +Change test: adding a provider dependency to a handler should leave the producer's job specification and enqueue interface untouched. An independently usable specification makes that guarantee explicit. + +Intent/counterevidence: a worker registry necessarily owns handlers, and the raw PostgreSQL definition API already supports producers. The missing value is reusable metadata and validation between those two interfaces; the worker registry itself does not need to disappear. + +Smallest repair: introduce a storage-agnostic `JobSpec`/contract collection, bind handlers to those specs at worker construction, and let PostgreSQL setup and producer enqueue helpers consume the same specs. Preserve operator-disable semantics and make exact versus additive synchronization explicit. Separate the durable payload schema version from operational definition settings; do not silently make every setting change alter existing idempotency requests. + +Validation: an API-only fixture must build and enqueue from the shared specs without constructing provider clients; a worker fixture must reject missing or duplicate bindings. Migrate IdentityPro's local conversion layer first. + +**API-003 — Add typed job submission and decoding on top of the JSON contract** + +Priority: **P1**, after or with shared specs. Confidence: **high**. This is an ergonomic extension, not a claim that JSON storage is an abstraction defect. + +[JobHandler::execute](/home/aa/Documents/runledger/runledger-core/src/jobs/handler.rs:16) receives `Value`. [JobEnqueue](/home/aa/Documents/runledger/runledger-postgres/src/jobs/types/enqueue.rs:10) accepts an independently selected job type and borrowed JSON. No type relationship proves that the submitted payload belongs to that handler. + +Consumers compensate through [OneSales payload policies](/home/aa/Documents/onesales/crates/onesales-job-support/src/lib.rs:204), [IdentityPro decoding and safe diagnostics](/home/aa/Documents/identitypro/crates/identitypro-jobs/src/handler_support.rs:92), and [CreditKit field parsers](/home/aa/Documents/creditkit-platform/crates/creditkit/src/jobs/payload.rs:6). IdentityPro's [billing submission](/home/aa/Documents/identitypro/crates/identitypro-jobs/src/billing_renewal.rs:70) separately serializes, selects the string identity, and fills default options. + +Add an optional typed handler/submit adapter linking a spec to `Payload`, with typed output/checkpoint support where useful. Keep the existing JSON handler as the compatibility and dynamic-dispatch boundary. Centralize default malformed-payload classification with static durable messages; allow applications to choose their own codes and safe diagnostics. Runtime decoding remains necessary for old durable rows, and payload upgrade policy remains explicit. + +Also provide a direct-job builder and a pool-owning enqueue-with-outcome convenience. `CatalogJobEnqueueInput` currently repeats the full `JobEnqueue` shape, and `enqueue_job_with_outcome_tx` is available only through a caller transaction. IdentityPro increments its `enqueued` counter for every UUID-returning success and still catches `job.already_enqueued`; current identical retries succeed, so an `Inserted`/`Existing` result would let that caller distinguish new work correctly. + +Success criterion: implementing a simple handler requires business logic and its payload type, not another parser, type-string pairing, serialization block, and list of `None` fields. Add this through adapters rather than a mandatory derive macro or wire-format migration. + +**API-004 — Extend exact scope semantics to direct-job reads** + +Implementation status: scoped job/event/log/intent inspection is available. +The completed `runledger-runledger-simplification-audit-60g` follow-up also adds +scoped metrics and exact global/tenant payload lookups while preserving legacy +signatures and visibility. See the [current read API guide](../README.md#admin-reads) +and [follow-up evidence](review-fixes-2026-09-05.md#exact-scope-metrics-and-payload-follow-up-60g). +The audit rationale below records the original findings. + +Priority: **P1**. Severity of API inconsistency: **medium**. Confidence: **high**. No downstream authorization breach was established. + +[get_job_by_id/list_jobs](/home/aa/Documents/runledger/runledger-postgres/src/jobs/admin/read.rs:67) interpret `organization_id: None` as unrestricted. Enqueue interprets `None` as a global job. [WorkflowRunReadScope](/home/aa/Documents/runledger/runledger-postgres/src/jobs/workflow_types.rs:222) and job mutation scope types already distinguish exact global, exact organization, and administrative visibility where applicable. + +A caller translating an optional organization from an enqueue or domain record into a read must know that its meaning changes. Global-only job listing cannot be expressed through the existing filter. [IdentityPro's recovery scan](/home/aa/Documents/identitypro/crates/identitypro-db/src/dead_letter_recovery.rs:103) explicitly uses `IS NOT DISTINCT FROM` to retain exact nullable scope. + +Add `JobReadScope::{Global, Organization, Admin}` and scoped job, event, log, and intent reads. Keep legacy wrappers with their existing wildcard semantics and migrate examples to explicit scopes. A scoped client/handle can bind the choice once for repeated operations. It is not authentication: applications must still authorize the chosen scope. + +Do not mechanically merge every scope enum: read, cancellation, and recovery permissions differ. The goal is consistent semantics, not identical types everywhere. The wildcard is intentional legacy behavior, so this is an API coverage gap rather than a hidden SQL bug. + +**API-005 — Keep advanced workflow features in the convenient builder** + +Priority: **P2**. Confidence: **high**. + +[WorkflowDagBuilder](/home/aa/Documents/runledger/runledger-core/src/jobs/workflow_enqueue/dag_builder.rs:15) explicitly sends consumers to lower-level builders for external steps and per-step settings. It also lacks the active-key, execution-resource, and continuation methods provided by the run/step builders. [CatalogWorkflowDagBuilder](/home/aa/Documents/runledger/runledger-runtime/src/catalog/workflow.rs:29) inherits that limitation. + +[OneSales buyer enrichment](/home/aa/Documents/onesales/crates/onesales-jobs/src/buyer_enrichment/launcher.rs:484) builds steps manually to combine per-step tenants, continuations, a shared execution resource, terminal dependencies, and an active run key. These are already-supported engine features that outgrow the convenient API. + +Add a configurable `step` entry point, external-step entry point, and active-key support to both fluent builders. An additive owned workflow request would also let dynamic callers return graphs without separately retaining payload/key storage; retain borrowed builders for callers that benefit from them. Reuse one validation implementation. Do not introduce a second workflow DSL. + +Use the buyer-enrichment graph as the acceptance fixture: express all of its existing policies through the convenient API without dropping validation or changing persisted semantics. + +**API-006 — Provide efficient operational reads and batch writes** + +Priority: **P2**. Confidence: **high** about source behavior; actual performance impact is **unmeasured**. + +Three distinct cost centers are visible: + +1. [Job listing](/home/aa/Documents/runledger/runledger-postgres/src/jobs/admin/read.rs:11) always selects payload, checkpoint, and output, uses offset pagination, and offers substring job-type matching. A status dashboard or recovery scan often needs compact fields, exact types, and a stable cursor. Add scoped `JobSummary` pages, exact job-type filters, batch status lookup, and keyset pagination. Retain detail reads for payload inspection. IdentityPro's custom scan demonstrates the cursor/exact-filter need, but its application ownership joins must remain application-owned. +2. [Workflow step insertion](/home/aa/Documents/runledger/runledger-postgres/src/jobs/workflows/steps.rs:173) executes one insert per step, then [one per dependency](/home/aa/Documents/runledger/runledger-postgres/src/jobs/workflows/steps.rs:237). For V steps and E edges, that is V+E awaited insert executions before counting run creation and root release. [OneSales batch submission](/home/aa/Documents/onesales/crates/onesales-account-research/src/batch_pipeline.rs:68) and IdentityPro billing also enqueue in loops. Investigate set-based graph inserts and a bounded `enqueue_many_tx` with per-item outcomes, deterministic locking, and explicit atomicity. A loop hidden behind a new API would improve convenience but would not establish a performance improvement. +3. [Each workflow-result waiter](/home/aa/Documents/runledger/runledger-postgres/src/jobs/workflows/handles.rs:224) attempts a `PgListener` connection and keeps it when the pool appears to have query capacity. It already closes the read/LISTEN race and falls back to polling. A shared process-level listener could reduce per-waiter connection demand for high-fanout HTTP consumers. No production caller of this waiting API was found in the sampled current integrations, so this ranks below the observed read/write needs. + +Benchmark database round trips, latency, allocations/JSON bytes, and connection use on PostgreSQL 18 with its exact server version recorded. Preserve snapshot idempotency, graph invariants, events, and lock ordering; avoid speculative concurrency changes inside a shared transaction. + +**API-007 — Offer durable lifecycle delivery as a separate capability** + +Priority: **P2 discovery**, after application-facing basics. Confidence: **medium** that the shared feature would remove enough code to justify its cost. + +The [runtime contract](/home/aa/Documents/runledger/runledger-runtime/src/lib.rs:13) promises best-effort post-commit observers. [Dead-letter hooks](/home/aa/Documents/runledger/runledger-runtime/src/dead_letter_hook.rs:12) have a ten-second production timeout and no durable acknowledgement protocol. These are appropriate telemetry hooks; they do not guarantee that application projections or business notifications complete after a process crash. + +[CreditKit reconciliation](/home/aa/Documents/creditkit-platform/crates/creditkit/src/documents/extraction_workflow/reconciliation.rs:22) scans application records and workflow state to repair convergence. This includes product-specific artifact checks and retry policy, so a lifecycle subscription cannot replace the entire reconciler. + +Explore a durable lifecycle inbox/subscription or declarative terminal follow-up job, recorded in the lifecycle transaction and processed with acknowledgement, retry, stable delivery identity, and retention. State explicitly that delivery is at least once. Generic capture/delivery could be shared; business policy and side-effect idempotency stay in the application. + +This is new functionality, not a bug in explicitly best-effort observers. Existing enqueue intents solve producer handoff before definition availability; they do not by themselves create subscriptions to lifecycle transitions. + +**API-008 — Treat upgrades, migrations, and examples as part of the product** + +Priority: **P2**, with the installation fix an immediate small task. Confidence: **high**. + +The [installation block](/home/aa/Documents/runledger/README.md:97) still recommends `0.9` for all crates while this workspace is `0.12.0`. The [producer quick-start](/home/aa/Documents/runledger/README.md:186) uses an incomplete `/* JobEnqueue */` placeholder. A new adopter can therefore begin on an older minor version and immediately leave the main tutorial to learn submission. + +The migration burden is concrete: [IdentityPro](/home/aa/Documents/identitypro/crates/identitypro-db/src/migrations/bundle.rs:248) manually includes the Runledger helper version in its template fingerprint, alongside raw migration checksums. [HOCR's migration test](/home/aa/Documents/hocr-next/apps/hocr-migrate/src/main.rs:678) verifies its vendored Runledger migration history. Runledger already exports migration and compatibility helpers; replacing all host migration orchestration would overreach. + +Add an exported library version/migration-pipeline fingerprint, a documented migration-bundle manifest, and examples for composing with an existing SQLx history. Extend compatibility diagnostics only where a concrete consumer needs more than the existing checker. Keep application migration ordering and historical cutover decisions with the application. + +Provide tested producer + worker examples sharing one job contract, with transactional handoff, typed payloads, progress, and ordinary shutdown. Check README version references as part of release verification. A large glob prelude is convenient but exposes application and runtime-internal operations together; task-oriented documentation and optional client handles would improve discovery without removing advanced exports. + +**Adoption opportunities that do not require inventing another engine** + +- Perdify: upgrade analysis should replace legacy duplicate-detection/savepoint code with current strict idempotent enqueue/outcomes where semantics match. Its [enqueue-or-update policy](/home/aa/Documents/perdify/crates/perdify-runledger-support/src/lib.rs:203) deliberately refreshes runnable payloads, including leased jobs under one policy. That is coalescing/mutation, not identical-request deduplication. Do not mechanically substitute strict enqueue. If repeated demand warrants a coalescing API, define pending-versus-leased behavior, generation fencing, audit history, and request-snapshot semantics first. +- Vatbot: [the one-second pump](/home/aa/Documents/vatbot/crates/vatbot/src/extraction_worker/controller.rs:21) calls `run_one`, which performs [its own claim/retry processing](/home/aa/Documents/vatbot/crates/vatbot/src/extraction_worker/mod.rs:70). It is a candidate for document-level Runledger jobs or a checkpointed continuation pump after an upgrade. Preserve its paid-provider completion budgets and domain-specific claims; the current layering may be intentional. The schedule is one-second eligible cadence, not evidence of a measured one-job-per-second creation rate. +- OneSales: prefer the already-published lease-identity and continuation APIs where current adapters use older forms; then evaluate what code remains before adding convenience APIs. +- IdentityPro: its successful continuations and explicit ownership-aware recovery are positive examples. Keep its business recovery gates and durable attention outbox policies in the application. + +**Recommended delivery order** + +1. Shared producer/worker specs plus typed submit/handler adapters. Prove the contract with IdentityPro and CreditKit; measure deleted integration code and the number of places edited to add a job. +2. Execution services and explicit direct-job read scopes. Prove them with OneSales CRM and IdentityPro recovery; measure removed queue-row reads and lifecycle helpers. +3. Fluent workflow parity and compact/batched persistence. Use buyer enrichment and batch enrollment as representative workloads; publish PostgreSQL 18 measurements before claiming speedups. +4. Migration metadata and complete current-version examples can ship independently. Investigate durable lifecycle delivery after a consumer prototype shows which reconciliation responsibilities it can actually remove. + +All public API additions should coexist with current entry points first. Adding fields to externally constructible structs or required trait methods can break consumers; use new types/adapters and an explicit later breaking release where needed. Durable payloads and database contracts need compatibility across queued work and rolling deployments, even when application source can be updated together. + +**Boundaries and rejected candidates** + +Reviewed: core handler/context and workflow contracts; effective crate exports; catalog and supervisor APIs; worker timeout/progress behavior; PostgreSQL enqueue, reads, scopes, workflow persistence/results, and migration entry points; representative downstream setup, handlers, submission, recovery, and migration consumers. Test-support exports and the external-consumer/example layout were inspected as validation infrastructure, not comprehensively audited. + +Confirmed abstraction findings: **0 high, 2 medium, 0 low**. Other numbered items are explicitly API coverage, ergonomics, performance, or product opportunities, not additional confirmed leaks or runtime bugs. + +Rejected: SQLx pool/transaction types in `runledger-postgres` are appropriate for a PostgreSQL integration library and enable atomic business writes plus enqueue. Tokio ownership in the runtime is deliberate. Stable error-code matching is documented. JSON persistence is deliberate. Application SQL joining queue state to domain ownership is not automatically missing functionality. Best-effort observers are not defective because they lack durable delivery. Private validated workflow fields and distinct recovery outcome types are useful boundaries to preserve. + +Limits: source inspection and offline Cargo metadata only; no compilation, tests, database execution, load test, security penetration test, or verification of deployed versions. No PostgreSQL-major-dependent behavior was experimentally asserted. Proposed APIs and migration sequences have not been compiled. TUI internals, every feature combination, every migration/locking path, and all downstream business rules were outside the detailed review. Local source links refer to the inspected filesystem and may move after this dated audit. diff --git a/docs/downstream-agent-guide.md b/docs/downstream-agent-guide.md index bfaa630..9349448 100644 --- a/docs/downstream-agent-guide.md +++ b/docs/downstream-agent-guide.md @@ -124,7 +124,11 @@ retain coverage for every job type that can receive intents. Database failures retry indefinitely so a prolonged outage cannot silently discard work; alert on oldest pending age, pending-only `retrying_count`, and pending-only `max_promotion_attempts` from -`get_job_enqueue_intent_metrics`, then repair the database policy. Conflicted +`get_job_enqueue_intent_metrics_with_scope` with +`JobEnqueueIntentReadMetricsFilter::new(scope, limit, offset)`, then repair the +database policy. Choose an authorized `JobReadScope::Global`, +`Organization(id)`, or `Admin`; the legacy `JobEnqueueIntentMetricsFilter` +without an organization still aggregates all scopes. Conflicted intents remain immutable evidence; safe replacement work requires a deliberately new application idempotency key. Metrics aggregate pending, conflicted, and recently promoted populations through separate selective predicates; @@ -145,7 +149,7 @@ Deploy this capability in order: 3. Switch application writers to `record_job_enqueue_intent_tx` only after the retention prerequisite is complete. 4. Alert on oldest pending age and `conflicted_24h` from - `get_job_enqueue_intent_metrics`. + `get_job_enqueue_intent_metrics_with_scope` for the authorized read scope. Queue retention must remove promoted-intent links first. In the same transaction that deletes selected queue rows, call @@ -242,10 +246,17 @@ in 0.11.0 and later. If the task requires step dependencies, build a workflow DAG: 1. Prefer `WorkflowDagBuilder` with `.job(...)`, `.after_success(...)`, and `.build()`. -2. Use `WorkflowStepEnqueueBuilder` and `WorkflowRunEnqueueBuilder` for advanced per-step - settings, external steps, hand-authored dependency specs, or call sites that - pass explicit `StepKey` and `JobType` values. -3. Persist the run with `runledger_postgres::jobs::enqueue_workflow_run`. +2. For per-step tenants, queue settings, continuations, execution resources, or + hand-authored dependencies, configure `WorkflowStepEnqueueBuilder`, call + `.try_build()`, and pass the result to `.step(...)`. Use `.external(...)` for + external work. `JobCatalog::workflow_dag` supports the same composition and + checks configured job steps against its own enabled catalog entries; + `JobCatalog::workflow_step` supplies a configurable step builder. + `WorkflowRunEnqueueBuilder` remains available for direct step collections. +3. Persist the run with `runledger_postgres::jobs::enqueue_workflow_run`. For + reusable active-cycle coordination, set `.active_key(...)` and use + `enqueue_or_get_active_workflow`. The active key is separate from request + idempotency and can be cleared with `.clear_active_key()`. If callers need a durable workflow result, declare one DAG step as the result step with `WorkflowDagBuilder::result_step(...)` or @@ -412,9 +423,40 @@ The packaged external-consumer compile-checked reference for returning a checkpointed continuation, reading it on the next run, and completing typed recovery. +### Exact payload reads + +For status dashboards or recovery scans, use `list_job_summaries` with +`JobSummaryFilter { scope, status, job_type, limit, after }`. `job_type` is an +optional exact `JobType`, not an ILIKE expression. `after: None` starts the scan; +use the last returned summary's `cursor()` for the next page. Preserve timestamp +microseconds and keep filters/scope fixed. Limits are 1–1,000. Empty pages end +the scan; concurrent status changes can move rows into or out of the filter. +`JobSummary` never reads payload, checkpoint, output, or free-form error text. + +Use `get_job_statuses_with_scope(pool, scope, ids)` for at most 1,000 input IDs. +Empty input performs no query, duplicates collapse, and missing/out-of-scope +IDs are omitted without distinguishing them. Results are sorted by ID. Neither +API authorizes the caller or provides a mutation fence: continue to use the +exact-scope recovery and lease APIs for writes. Retain application ownership +joins and authorization checks. Both APIs and their input/record types are +exported from `runledger_postgres::jobs` and `prelude`. + +Use `get_job_payload_by_idempotency_key_with_scope` or +`get_latest_job_payload_for_run_with_scope` with an authorized `JobScope::Global` +or `JobScope::Organization(id)`. These lookups select one exact scope and job +type; they have no Admin wildcard. The latest-run helper matches the JSON +`run_id` and orders by `created_at DESC, id DESC`. Both return +`Option<(Uuid, Value)>`, including `None` for an absent match. Nil UUIDs are +ordinary tenant/run values, not sentinels. Legacy helpers keep their tenant UUID +signatures. The scoped metrics and payload APIs and the intent filter are +exported through both `runledger_postgres::jobs` and `prelude`. + ### Continuation operational queries -Use `get_job_continuation_metrics` for service dashboards and alerts. It returns +Use `get_job_continuation_metrics_with_scope` for service dashboards and alerts. +Select `JobReadScope::Global` for global jobs, `Organization(id)` for one tenant, +or `Admin` for aggregation across all scopes. Authorize that selection in the +application. The function returns one `JobContinuationMetricsRecord` per job type: - `continued_24h` is the number of committed handler continuations in the last @@ -425,8 +467,10 @@ one `JobContinuationMetricsRecord` per job type: continuation-created runs. A later admin recovery is deliberately excluded, so this is a focused runaway-depth signal rather than lifetime ancestry. -The optional organization argument follows the admin-metrics convention: -`None` aggregates every scope rather than selecting only global jobs. Choose +The legacy `get_job_continuation_metrics` optional organization argument retains +its meaning: `None` aggregates every scope, and `Some(id)` selects one tenant. +`get_job_metrics_with_scope` uses the same explicit scopes for queue metrics; +both APIs retain registered definitions with zero counts in empty scopes. Choose alert thresholds from the expected slice size and schedule; a fixed global threshold is usually misleading. @@ -516,10 +560,16 @@ fallback for malformed, historical, custom, and future payloads: ```rust use runledger_postgres::prelude::{ DecodedJobEventPayload, DecodedRequeuedEventPayload, - SuccessfulReplayEnqueuedEventPayload, list_job_events, + SuccessfulReplayEnqueuedEventPayload, JobReadScope, list_job_events_with_scope, }; -let events = list_job_events(&pool, organization_id, job_id, 200, None).await?; +// Authorize access to this exact job scope in the application first. +// Here organization_id comes from the application's job ownership record. +let scope = match organization_id { + Some(id) => JobReadScope::Organization(id), + None => JobReadScope::Global, +}; +let events = list_job_events_with_scope(&pool, scope, job_id, 200, None).await?; for event in events { match event.decoded_payload() { diff --git a/docs/execution-services-migrations/README.md b/docs/execution-services-migrations/README.md new file mode 100644 index 0000000..e3b47fe --- /dev/null +++ b/docs/execution-services-migrations/README.md @@ -0,0 +1,113 @@ +# Execution-services migration pilots + +These patches migrate real downstream handlers for Runledger issue +`runledger-runledger-simplification-audit-4a2` (audit AP-RUST-001). They were +applied and tested in isolated worktrees using this Runledger workspace through +Cargo path overrides. They are integration pilots, not application releases: +neither application was deployed or had its main checkout's code changed. +Downstream maintainers can apply them once they select a Runledger release +containing these APIs, then regenerate the application lockfile. + +## OneSales CRM account sync + +[onesales.patch](onesales.patch) applies to OneSales +`c5beb8f1bcd6345df181cbd79f07622bf346522e`. It migrates +`CrmSyncAccountsHandler`, including Salesforce and HubSpot, to +`JobExecutionHandler` and registers its `into_job_handler()` adapter. + +The account checkpoint loader now parses the supplied resume snapshot, derives +its page quota from the remaining runtime budget, and uses the supplied absolute +deadline. Its progress writes call `JobExecution::persist_progress`. The +production account path no longer calls `get_job_by_id`, compares lease fields, +copies a worker/run/attempt tuple, or reconstructs a deadline from handler start +time. Checkpoint versions, source-job lineage, provider validation, and +continuation policies remain application-owned. + +The shared Salesforce page-runner state still supports legacy contact sync. +Account states select runtime-managed progress; accidentally routing one through +the positional legacy writer fails explicitly. Contact sync is not migrated by +this pilot. + +Existing direct-invocation tests now supply a test execution-service driver. +Provider-only tests without a queue record retain their explicit no-persistence +fixture; queued checkpoint tests use PostgreSQL-fenced writes. Two resume +fixtures now seed checkpoints before claiming, so their invocation snapshot +contains the intended state. The driver can read a queue row to simulate the +runtime's timeout input; the migrated production handler cannot. + +Verification: all 79 CRM account tests passed, including bounded continuation +and checkpoint resume. The resume test recorded PostgreSQL +`18.6 (Debian 18.6-1.pgdg13+2)`. HTTP CRM providers are mocked by the existing +test suite; this is not evidence of live Salesforce or HubSpot execution. +`cargo check -p onesales-jobs --all-targets` also passed. + +## IdentityPro bounded protection-enrollment recovery + +[identitypro.patch](identitypro.patch) applies to IdentityPro +`ee655fd0a40c6fd047bafa19ea3a1a3ade1e3e38`. It migrates +`ProtectionEnrollmentRecoveryHandler` to the execution-services interface and +registers its adapter in the catalog. + +The recovery service receives the runtime's absolute deadline minus the +existing 15-second application reserve. It no longer starts a new 130-second +budget when execution reaches the service call. Definition policy, ownership +identity passed to the domain service, continuation versus failure decisions, +and durable payloads remain unchanged. + +All eight protection-enrollment tests passed. The new test supplies a +25-second invocation deadline and asserts that the recovery service receives +that exact deadline minus 15 seconds, so reconstructing the old fixed timeout +fails the assertion. These are domain-service unit tests; runtime timeout and +database lease behavior are exercised separately in Runledger's PostgreSQL tests. +`cargo check -p identitypro-jobs --features worker,test-support --all-targets` +also passed. The public test-support helper retains its legacy direct invocation +contract through an explicit test execution driver. + +## Reproduction + +Start from clean worktrees at the revisions above. From each application root, +run `git apply --check /path/to/patch`, followed by `git apply /path/to/patch`. +Both patches passed that applicability check against the original checkouts. + +Create a temporary Cargo configuration file, replacing the paths below with +the location of this Runledger checkout: + + [patch.crates-io] + runledger-core = { path = "/path/to/runledger/runledger-core" } + runledger-postgres = { path = "/path/to/runledger/runledger-postgres" } + runledger-runtime = { path = "/path/to/runledger/runledger-runtime" } + +From OneSales: + + SQLX_OFFLINE=true cargo check -p onesales-jobs --all-targets --config /path/to/overrides.toml + SQLX_OFFLINE=true cargo test -p onesales-jobs --lib crm_accounts_sync::tests --config /path/to/overrides.toml + SQLX_OFFLINE=true cargo test -p onesales-jobs --lib crm_accounts_sync::tests::checkpoint_resume_tests::sync_resumes_from_checkpoint_after_page_cap_continuation --config /path/to/overrides.toml -- --nocapture + +The database test harness uses Docker and PostgreSQL 18. The last command prints +the exact server version. + +From IdentityPro: + + SQLX_OFFLINE=true cargo check -p identitypro-jobs --features worker,test-support --all-targets --config /path/to/overrides.toml + SQLX_OFFLINE=true cargo test -p identitypro-jobs --features worker,test-support protection_enrollment --config /path/to/overrides.toml + +The patches also replace older test accesses to private `JobCompletion` fields +with the existing public accessors required by this Runledger checkout. +Assertions retain their original values. Local Cargo lockfile changes caused +by path overrides are intentionally not included. + +## Runledger verification + +`cargo test --workspace --all-features` passed: 864 tests, zero failures, and +three existing ignored test entries (two manual diagnostics and a child-process +entrypoint exercised by its parent tests). Database tests used PostgreSQL 18.6. +The five new runtime tests prove commit acknowledgement before handler return, +checkpoint resume across continuation, timeout with durable checkpoint delivery +to the dead-letter hook, rejection of expired/replaced run/attempt/worker leases, +typed persistence failure, and cancellation of a progress write blocked on a row +lock. Lease loss stops handler polling even when the handler swallows the error. + +`scripts/lint.sh` and `cargo check -p runledger-core --no-default-features` +passed. No SQL queries or migrations changed; the existing three SQLx cache +directories remain identical. This was a solo implementation and verification; +no independent reviewer or production rollout is claimed. diff --git a/docs/execution-services-migrations/identitypro.patch b/docs/execution-services-migrations/identitypro.patch new file mode 100644 index 0000000..99f49a4 --- /dev/null +++ b/docs/execution-services-migrations/identitypro.patch @@ -0,0 +1,260 @@ +diff --git a/crates/identitypro-jobs/src/catalog.rs b/crates/identitypro-jobs/src/catalog.rs +index b3bf6b1..5cfbc71 100644 +--- a/crates/identitypro-jobs/src/catalog.rs ++++ b/crates/identitypro-jobs/src/catalog.rs +@@ -3,7 +3,7 @@ use std::{sync::Arc, time::Duration}; + use ::identitypro_core::auth::{EmailVerificationTokenCodec, PasswordResetTokenCodec}; + use ::identitypro_db::Db; + use identitypro_billing::BillingService; +-use runledger_core::jobs::{JobHandler, JobType}; ++use runledger_core::jobs::{JobExecutionHandler, JobHandler, JobType}; + use runledger_postgres::jobs::{ + deactivate_schedules_absent_from_names_tx, prepare_schedule_exact_sync_critical_section_tx, + sync_catalog_job_schedules_tx, +@@ -282,7 +282,7 @@ fn build_catalog( + let catalog = match protection_enrollment { + Some(recovery) => register_protection_enrollment_recovery( + catalog, +- ProtectionEnrollmentRecoveryHandler::new(recovery), ++ ProtectionEnrollmentRecoveryHandler::new(recovery).into_job_handler(), + billing_enabled, + ), + None => catalog, +diff --git a/crates/identitypro-jobs/src/definitions.rs b/crates/identitypro-jobs/src/definitions.rs +index d98de37..9e2d7af 100644 +--- a/crates/identitypro-jobs/src/definitions.rs ++++ b/crates/identitypro-jobs/src/definitions.rs +@@ -64,7 +64,7 @@ pub const PROTECTION_ENROLLMENT_RECOVERY_WORK_TIMEOUT: std::time::Duration = + pub const PROTECTION_ENROLLMENT_RECOVERY_CONTINUATION_SECONDS: u16 = 60; + /// Runledger's timeout includes both service work and the time needed to + /// durably record the resulting completion. +-const PROTECTION_ENROLLMENT_DURABLE_COMPLETION_MARGIN_SECONDS: u64 = 15; ++pub(crate) const PROTECTION_ENROLLMENT_DURABLE_COMPLETION_MARGIN_SECONDS: u64 = 15; + pub const PROTECTION_ENROLLMENT_RECOVERY_TIMEOUT_SECONDS: i32 = + (PROTECTION_ENROLLMENT_RECOVERY_WORK_TIMEOUT.as_secs() + + PROTECTION_ENROLLMENT_DURABLE_COMPLETION_MARGIN_SECONDS) as i32; +diff --git a/crates/identitypro-jobs/src/identity_protect_alert.rs b/crates/identitypro-jobs/src/identity_protect_alert.rs +index 97ac44b..0617f99 100644 +--- a/crates/identitypro-jobs/src/identity_protect_alert.rs ++++ b/crates/identitypro-jobs/src/identity_protect_alert.rs +@@ -444,7 +444,7 @@ mod tests { + JobCompletionDisposition::ContinueAfter(std::time::Duration::from_secs(60)) + ); + assert!( +- continuation.checkpoint.is_none(), ++ continuation.checkpoint_value().cloned().is_none(), + "capacity waiting must not enter the exhaustion phase" + ); + assert_eq!( +@@ -582,7 +582,7 @@ mod tests { + JobCompletionDisposition::ContinueAfter(Duration::ZERO) + ); + assert_eq!( +- hydration_phase(continuation.checkpoint.as_ref()).unwrap(), ++ hydration_phase(continuation.checkpoint_value()).unwrap(), + Some(IdentityProtectAlertHydrationCheckpoint::PersistExhaustion { + attempt_id: expected_attempt_id, + }) +@@ -612,7 +612,7 @@ mod tests { + JobCompletionDisposition::ContinueAfter(HYDRATION_CONTENTION_DELAY) + ); + assert!( +- continuation.checkpoint.is_none(), ++ continuation.checkpoint_value().cloned().is_none(), + "claim contention must not enter the exhaustion phase" + ); + } +@@ -642,7 +642,7 @@ mod tests { + continuation.disposition(), + JobCompletionDisposition::Succeed + ); +- assert!(continuation.checkpoint.is_none()); ++ assert!(continuation.checkpoint_value().cloned().is_none()); + assert_eq!(hydration.exhaustion_calls.load(Ordering::SeqCst), 1); + } + +@@ -717,7 +717,8 @@ mod tests { + JobCompletionDisposition::ContinueAfter(std::time::Duration::ZERO) + ); + let checkpoint = continuation +- .checkpoint ++ .checkpoint_value() ++ .cloned() + .expect("the continuation persists its exhaustion phase"); + assert_eq!( + hydration_phase(Some(&checkpoint)).unwrap(), +diff --git a/crates/identitypro-jobs/src/protection_enrollment.rs b/crates/identitypro-jobs/src/protection_enrollment.rs +index 66d9d33..565e17c 100644 +--- a/crates/identitypro-jobs/src/protection_enrollment.rs ++++ b/crates/identitypro-jobs/src/protection_enrollment.rs +@@ -5,7 +5,9 @@ use std::time::Duration; + use async_trait::async_trait; + use runledger_core::jobs::JobType; + #[cfg(feature = "worker")] +-use runledger_core::jobs::{JobCompletion, JobContext, JobFailure, JobHandler}; ++use runledger_core::jobs::{ ++ JobCompletion, JobContext, JobExecution, JobExecutionHandler, JobFailure, ++}; + use runledger_postgres::jobs::{JobEnqueue, enqueue_job_tx}; + use serde::{Deserialize, Serialize}; + #[cfg(feature = "worker")] +@@ -16,7 +18,7 @@ use uuid::Uuid; + #[cfg(feature = "worker")] + use crate::{ + PROTECTION_ENROLLMENT_RECOVERY_CONTINUATION_SECONDS, +- PROTECTION_ENROLLMENT_RECOVERY_WORK_TIMEOUT, ++ definitions::PROTECTION_ENROLLMENT_DURABLE_COMPLETION_MARGIN_SECONDS, + }; + use crate::{ + PROTECTION_ENROLLMENT_RECOVERY_JOB_TYPE, definitions::PROTECTION_ENROLLMENT_RECOVERY_DEFINITION, +@@ -210,22 +212,23 @@ impl ProtectionEnrollmentRecoveryHandler { + #[cfg(all(feature = "worker", any(test, feature = "test-support")))] + pub fn protection_enrollment_recovery_handler_for_tests( + recovery: Arc, +-) -> impl JobHandler { +- ProtectionEnrollmentRecoveryHandler::new(recovery) ++) -> impl runledger_core::jobs::JobHandler { ++ TestProtectionEnrollmentHandler(ProtectionEnrollmentRecoveryHandler::new(recovery)) + } + + #[cfg(feature = "worker")] + #[async_trait] +-impl JobHandler for ProtectionEnrollmentRecoveryHandler { ++impl JobExecutionHandler for ProtectionEnrollmentRecoveryHandler { + fn job_type(&self) -> JobType<'static> { + JobType::new(PROTECTION_ENROLLMENT_RECOVERY_JOB_TYPE) + } + + async fn execute( + &self, +- context: JobContext, ++ execution: JobExecution<'_>, + payload: Value, + ) -> Result { ++ let context = execution.context(); + let payload: ProtectionEnrollmentRecoveryJobPayload = serde_json::from_value(payload) + .map_err(|error| { + tracing::warn!( +@@ -241,10 +244,11 @@ impl JobHandler for ProtectionEnrollmentRecoveryHandler { + .recovery + .recover_protection_enrollment( + ProtectionEnrollmentRecoveryRequest { +- job_id: context.job_id, ++ job_id: execution.context().job_id, + user_id: payload.user_id, + }, +- tokio::time::Instant::now() + PROTECTION_ENROLLMENT_RECOVERY_WORK_TIMEOUT, ++ tokio::time::Instant::from_std(execution.deadline()) ++ - Duration::from_secs(PROTECTION_ENROLLMENT_DURABLE_COMPLETION_MARGIN_SECONDS), + ) + .await + .map_err(|error| { +@@ -347,6 +351,43 @@ mod tests { + } + } + ++ struct DeadlineRecovery(std::sync::Mutex>); ++ ++ #[async_trait] ++ impl ProtectionEnrollmentRecovery for DeadlineRecovery { ++ async fn recover_protection_enrollment( ++ &self, ++ _request: ProtectionEnrollmentRecoveryRequest, ++ deadline: tokio::time::Instant, ++ ) -> anyhow::Result { ++ *self.0.lock().unwrap() = Some(deadline); ++ Ok(ProtectionEnrollmentRecoveryOutcome::Complete) ++ } ++ } ++ ++ #[tokio::test] ++ async fn uses_supplied_runtime_deadline_and_preserves_completion_reserve() { ++ let recovery = Arc::new(DeadlineRecovery(std::sync::Mutex::new(None))); ++ let handler = ProtectionEnrollmentRecoveryHandler::new(recovery.clone()); ++ // Different from the configured 145-second timeout: the handler must ++ // obey this invocation's effective budget rather than reconstruct it. ++ let deadline = std::time::Instant::now() + Duration::from_secs(25); ++ let services = TestExecutionBudget(deadline); ++ let context = context(1); ++ let completion = JobExecutionHandler::execute( ++ &handler, ++ JobExecution::new(&context, &services), ++ payload(), ++ ) ++ .await ++ .unwrap(); ++ assert_eq!(completion.disposition(), JobCompletionDisposition::Succeed); ++ assert_eq!( ++ *recovery.0.lock().unwrap(), ++ Some(tokio::time::Instant::from_std(deadline) - Duration::from_secs(15)) ++ ); ++ } ++ + fn context(run_number: i32) -> JobContext { + JobContext { + job_id: Uuid::now_v7(), +@@ -503,3 +544,64 @@ mod tests { + ); + } + } ++ ++#[cfg(all(any(test, feature = "test-support"), feature = "worker"))] ++struct TestExecutionBudget(std::time::Instant); ++ ++#[cfg(all(any(test, feature = "test-support"), feature = "worker"))] ++#[async_trait] ++impl runledger_core::jobs::JobExecutionServices for TestExecutionBudget { ++ fn deadline(&self) -> std::time::Instant { ++ self.0 ++ } ++ fn remaining_budget(&self) -> Duration { ++ self.0.saturating_duration_since(std::time::Instant::now()) ++ } ++ async fn persist_progress( ++ &self, ++ _: runledger_core::jobs::JobExecutionUpdate<'_>, ++ ) -> Result<(), runledger_core::jobs::JobExecutionError> { ++ panic!("protection-enrollment test handler does not persist progress"); ++ } ++} ++ ++#[cfg(all(any(test, feature = "test-support"), feature = "worker"))] ++impl ProtectionEnrollmentRecoveryHandler { ++ async fn execute( ++ &self, ++ context: JobContext, ++ payload: Value, ++ ) -> Result { ++ let services = TestExecutionBudget( ++ std::time::Instant::now() ++ + Duration::from_secs(crate::PROTECTION_ENROLLMENT_RECOVERY_TIMEOUT_SECONDS as u64), ++ ); ++ JobExecutionHandler::execute(self, JobExecution::new(&context, &services), payload).await ++ } ++} ++ ++#[cfg(all(any(test, feature = "test-support"), feature = "worker"))] ++struct TestProtectionEnrollmentHandler(ProtectionEnrollmentRecoveryHandler); ++ ++#[cfg(all(any(test, feature = "test-support"), feature = "worker"))] ++#[async_trait] ++impl runledger_core::jobs::JobHandler for TestProtectionEnrollmentHandler { ++ fn job_type(&self) -> JobType<'static> { ++ JobExecutionHandler::job_type(&self.0) ++ } ++ async fn execute( ++ &self, ++ context: JobContext, ++ payload: Value, ++ ) -> Result { ++ self.0.execute(context, payload).await ++ } ++ async fn on_dead_letter( ++ &self, ++ context: JobContext, ++ payload: Value, ++ info: runledger_core::jobs::JobDeadLetterInfo, ++ ) { ++ JobExecutionHandler::on_dead_letter(&self.0, context, payload, info).await; ++ } ++} diff --git a/docs/execution-services-migrations/onesales.patch b/docs/execution-services-migrations/onesales.patch new file mode 100644 index 0000000..4696edf --- /dev/null +++ b/docs/execution-services-migrations/onesales.patch @@ -0,0 +1,966 @@ +diff --git a/crates/onesales-jobs/src/buyer_enrichment/account_handler_tests.rs b/crates/onesales-jobs/src/buyer_enrichment/account_handler_tests.rs +index 25e1b5ac0..4f0c391e9 100644 +--- a/crates/onesales-jobs/src/buyer_enrichment/account_handler_tests.rs ++++ b/crates/onesales-jobs/src/buyer_enrichment/account_handler_tests.rs +@@ -78,7 +78,10 @@ async fn account_handler_continues_across_slices_until_every_contact_is_complete + first.disposition(), + JobCompletionDisposition::ContinueAfter(Duration::ZERO) + ); +- let checkpoint = first.checkpoint.expect("first slice checkpoint"); ++ let checkpoint = first ++ .checkpoint_value() ++ .cloned() ++ .expect("first slice checkpoint"); + assert_eq!(checkpoint["contactsProcessed"], 25); + assert_eq!(checkpoint["slicesCompleted"], 1); + +@@ -216,7 +219,10 @@ async fn account_handler_preserves_email_completion_when_profile_lookup_is_defer + first.disposition(), + JobCompletionDisposition::ContinueAfter(delay) if delay > Duration::ZERO + )); +- let checkpoint = first.checkpoint.expect("deferred run checkpoint"); ++ let checkpoint = first ++ .checkpoint_value() ++ .cloned() ++ .expect("deferred run checkpoint"); + assert!(checkpoint["afterContactId"].is_null()); + assert_eq!(checkpoint["stats"]["email_found"], 1); + assert_eq!(checkpoint["stats"]["provider_attempted_candidates"], 1); +diff --git a/crates/onesales-jobs/src/buyer_enrichment_tests.rs b/crates/onesales-jobs/src/buyer_enrichment_tests.rs +index e54ef963f..91ee1e014 100644 +--- a/crates/onesales-jobs/src/buyer_enrichment_tests.rs ++++ b/crates/onesales-jobs/src/buyer_enrichment_tests.rs +@@ -94,7 +94,10 @@ fn continuation_persists_cursor_and_more_than_the_old_500_contact_cap() { + completion.disposition(), + JobCompletionDisposition::ContinueAfter(std::time::Duration::ZERO) + ); +- let persisted = completion.checkpoint.expect("continuation checkpoint"); ++ let persisted = completion ++ .checkpoint_value() ++ .cloned() ++ .expect("continuation checkpoint"); + assert_eq!(persisted["contactsProcessed"], 625); + assert_eq!(persisted["slicesCompleted"], 1); + assert_eq!(persisted["afterContactId"], after_contact_id.to_string()); +@@ -118,7 +121,7 @@ fn delayed_continuation_keeps_the_last_completed_cursor() { + JobCompletionDisposition::ContinueAfter(delay) + ); + assert_eq!( +- completion.checkpoint.expect("checkpoint")["afterContactId"], ++ completion.checkpoint_value().cloned().expect("checkpoint")["afterContactId"], + after_contact_id.to_string() + ); + } +diff --git a/crates/onesales-jobs/src/crm_accounts_sync.rs b/crates/onesales-jobs/src/crm_accounts_sync.rs +index 42a3e21e7..c1e68834a 100644 +--- a/crates/onesales-jobs/src/crm_accounts_sync.rs ++++ b/crates/onesales-jobs/src/crm_accounts_sync.rs +@@ -1,5 +1,3 @@ +-use std::time::Instant; +- + use chrono::{DateTime, Duration, Utc}; + use onesales_core::crm_sync_contracts::{CrmSyncAccountsPayload, CrmSyncContactsPayload}; + use onesales_db::crm::{ +@@ -11,7 +9,9 @@ use onesales_db::crm::{ + }; + use onesales_hubspot as hubspot_api; + use onesales_salesforce as salesforce_api; +-use runledger_core::jobs::{JobContext, JobFailure, JobFailureKind, JobHandler}; ++use runledger_core::jobs::{ ++ JobContext, JobExecution, JobExecutionHandler, JobFailure, JobFailureKind, ++}; + use runledger_postgres::jobs::{JobEnqueue, JobEnqueueDisposition}; + use runledger_runtime::registry::JobRegistry; + use serde_json::Value; +@@ -141,11 +141,9 @@ impl CrmSyncAccountsHandler { + context: &JobContext, + organization_id: Uuid, + connection: &mut SalesforceSyncConnectionRecord, +- handler_started_at: Instant, ++ execution: JobExecution<'_>, + ) -> Result { +- let mut execution_state = self +- .load_sync_execution_state(context, organization_id, handler_started_at) +- .await?; ++ let mut execution_state = self.load_sync_execution_state(execution).await?; + execution_state.checkpoint.ensure_source_context(context); + + info!( +@@ -183,6 +181,7 @@ impl CrmSyncAccountsHandler { + let mut page_adapter = AccountPageSyncAdapter::new( + self, + context, ++ execution, + organization_id, + connection, + account_query_capabilities, +@@ -788,7 +787,7 @@ pub fn register_crm_sync_accounts_handler( + pool: onesales_db::DbPool, + http_client: reqwest::Client, + ) { +- registry.register(CrmSyncAccountsHandler::new(pool, http_client)); ++ registry.register(CrmSyncAccountsHandler::new(pool, http_client).into_job_handler()); + } + + fn account_payload_invalid_message(error: &serde_json::Error) -> String { +diff --git a/crates/onesales-jobs/src/crm_accounts_sync/checkpoint.rs b/crates/onesales-jobs/src/crm_accounts_sync/checkpoint.rs +index e526947f0..5378ce348 100644 +--- a/crates/onesales-jobs/src/crm_accounts_sync/checkpoint.rs ++++ b/crates/onesales-jobs/src/crm_accounts_sync/checkpoint.rs +@@ -1,15 +1,13 @@ ++use runledger_core::jobs::JobExecution; + use runledger_core::jobs::{JobContext, JobFailure}; + use serde::{Deserialize, Serialize}; + use serde_json::Value; +-use std::time::Instant; + use uuid::Uuid; + + use super::CrmSyncAccountsHandler; + use super::constants::{HUBSPOT_CHECKPOINT_VERSION, SALESFORCE_CHECKPOINT_VERSION}; + use crate::crm_sync_common::{self, SyncExecutionState as SharedSyncExecutionState}; + +-const ENTITY_LABEL: &str = "CRM account sync"; +- + #[derive(Clone, Debug, Deserialize, Serialize)] + pub(super) struct SalesforceSyncCheckpoint { + pub(super) version: u8, +@@ -149,69 +147,79 @@ pub(super) type HubSpotSyncExecutionState = SharedSyncExecutionState, + ) -> Result { +- crm_sync_common::load_sync_execution_state( +- &self.pool, +- context, +- organization_id, +- ENTITY_LABEL, +- handler_started_at, +- parse_salesforce_sync_checkpoint, +- super::derive_max_pages_per_attempt, +- ) +- .await ++ Ok(SharedSyncExecutionState { ++ checkpoint: parse_salesforce_sync_checkpoint(execution.checkpoint_value())?, ++ progress_mode: crm_sync_common::SyncProgressMode::RuntimeServices, ++ max_pages_per_attempt: Some(super::derive_max_pages_per_attempt( ++ remaining_timeout_seconds(execution), ++ )), ++ attempt_deadline: Some(execution.deadline()), ++ }) + } + + pub(super) async fn persist_sync_progress( + &self, ++ execution: JobExecution<'_>, + execution_state: &SyncExecutionState, + ) -> Result<(), JobFailure> { +- crm_sync_common::persist_sync_progress( +- &self.pool, +- execution_state, ++ persist_checkpoint( ++ execution, ++ &execution_state.checkpoint, + crm_sync_common::SALESFORCE_CHECKPOINT_INVALID_FAILURE_CODE, + "CRM account sync produced an invalid checkpoint payload.", +- ENTITY_LABEL, + ) + .await + } + + pub(super) async fn load_hubspot_sync_execution_state( + &self, +- context: &JobContext, +- organization_id: Uuid, +- handler_started_at: Instant, ++ execution: JobExecution<'_>, + ) -> Result { +- crm_sync_common::load_sync_execution_state( +- &self.pool, +- context, +- organization_id, +- ENTITY_LABEL, +- handler_started_at, +- parse_hubspot_sync_checkpoint, +- super::derive_hubspot_max_pages_per_attempt, +- ) +- .await ++ Ok(SharedSyncExecutionState { ++ checkpoint: parse_hubspot_sync_checkpoint(execution.checkpoint_value())?, ++ progress_mode: crm_sync_common::SyncProgressMode::RuntimeServices, ++ max_pages_per_attempt: Some(super::derive_hubspot_max_pages_per_attempt( ++ remaining_timeout_seconds(execution), ++ )), ++ attempt_deadline: Some(execution.deadline()), ++ }) + } + + pub(super) async fn persist_hubspot_sync_progress( + &self, ++ execution: JobExecution<'_>, + execution_state: &HubSpotSyncExecutionState, + ) -> Result<(), JobFailure> { +- crm_sync_common::persist_sync_progress( +- &self.pool, +- execution_state, ++ persist_checkpoint( ++ execution, ++ &execution_state.checkpoint, + crm_sync_common::SYNC_CHECKPOINT_INVALID_FAILURE_CODE, + "CRM account sync produced an invalid HubSpot checkpoint payload.", +- ENTITY_LABEL, + ) + .await + } + } + ++async fn persist_checkpoint( ++ execution: JobExecution<'_>, ++ checkpoint: &(impl Serialize + Sync + crm_sync_common::SyncCheckpoint), ++ invalid_code: &'static str, ++ invalid_message: &'static str, ++) -> Result<(), JobFailure> { ++ let value = serde_json::to_value(checkpoint) ++ .map_err(|_| JobFailure::terminal(invalid_code, invalid_message))?; ++ execution ++ .persist_progress(runledger_core::jobs::JobExecutionUpdate { ++ progress_done: Some(i64::try_from(checkpoint.records_processed()).unwrap_or(i64::MAX)), ++ progress_total: None, ++ checkpoint: Some(&value), ++ }) ++ .await ++ .map_err(Into::into) ++} ++ + pub(super) fn parse_salesforce_sync_checkpoint( + raw_checkpoint: Option<&Value>, + ) -> Result { +@@ -252,3 +260,11 @@ pub(super) fn parse_hubspot_sync_checkpoint( + )?; + Ok(checkpoint) + } ++ ++fn remaining_timeout_seconds(execution: JobExecution<'_>) -> i32 { ++ let remaining = execution.remaining_budget(); ++ // Preserve page-quota rounding at startup; exact deadline guards still ++ // bound every provider page and all progress writes. ++ let seconds = remaining.as_secs() + u64::from(remaining.subsec_nanos() != 0); ++ i32::try_from(seconds).unwrap_or(i32::MAX) ++} +diff --git a/crates/onesales-jobs/src/crm_accounts_sync/hubspot/sync.rs b/crates/onesales-jobs/src/crm_accounts_sync/hubspot/sync.rs +index f211aec4c..78f92b966 100644 +--- a/crates/onesales-jobs/src/crm_accounts_sync/hubspot/sync.rs ++++ b/crates/onesales-jobs/src/crm_accounts_sync/hubspot/sync.rs +@@ -1,7 +1,6 @@ + use onesales_db::crm::HubSpotSyncConnectionRecord; + use onesales_hubspot as hubspot; +-use runledger_core::jobs::{JobContext, JobFailure}; +-use std::time::Instant; ++use runledger_core::jobs::{JobContext, JobExecution, JobFailure}; + use tracing::{info, warn}; + use uuid::Uuid; + +@@ -19,7 +18,7 @@ impl CrmSyncAccountsHandler { + context: &JobContext, + organization_id: Uuid, + connection: &HubSpotSyncConnectionRecord, +- handler_started_at: Instant, ++ execution: JobExecution<'_>, + token_fingerprint: &str, + ) -> Result { + ensure_hubspot_sync_scopes(connection.scope.as_deref())?; +@@ -34,9 +33,7 @@ impl CrmSyncAccountsHandler { + + self.ensure_hubspot_account_mapper_version().await?; + +- let mut execution_state = self +- .load_hubspot_sync_execution_state(context, organization_id, handler_started_at) +- .await?; ++ let mut execution_state = self.load_hubspot_sync_execution_state(execution).await?; + execution_state.checkpoint.ensure_source_context(context); + let max_pages_per_attempt = execution_state.max_pages_per_attempt.unwrap_or(u32::MAX); + let mut pages_processed_this_attempt = 0_u32; +@@ -55,7 +52,8 @@ impl CrmSyncAccountsHandler { + return Ok(SyncJobOutcome::Completed); + } + if pages_processed_this_attempt >= max_pages_per_attempt { +- self.persist_hubspot_sync_progress(&execution_state).await?; ++ self.persist_hubspot_sync_progress(execution, &execution_state) ++ .await?; + return SyncJobOutcome::continue_with_checkpoint( + &execution_state.checkpoint, + crate::crm_sync_common::SYNC_CHECKPOINT_INVALID_FAILURE_CODE, +@@ -68,7 +66,8 @@ impl CrmSyncAccountsHandler { + constants::HUBSPOT_PAGE_CAP_PROCESSING_GUARD_SECONDS, + constants::HUBSPOT_PAGE_CAP_FINALIZATION_GUARD_SECONDS, + ) { +- self.persist_hubspot_sync_progress(&execution_state).await?; ++ self.persist_hubspot_sync_progress(execution, &execution_state) ++ .await?; + return SyncJobOutcome::continue_with_checkpoint( + &execution_state.checkpoint, + crate::crm_sync_common::SYNC_CHECKPOINT_INVALID_FAILURE_CODE, +@@ -160,18 +159,21 @@ impl CrmSyncAccountsHandler { + execution_state.checkpoint.after = next_after; + + if execution_state.checkpoint.after.is_some() { +- self.persist_hubspot_sync_progress(&execution_state).await?; ++ self.persist_hubspot_sync_progress(execution, &execution_state) ++ .await?; + continue; + } + + if !execution_state.checkpoint.archived { + execution_state.checkpoint.archived = true; +- self.persist_hubspot_sync_progress(&execution_state).await?; ++ self.persist_hubspot_sync_progress(execution, &execution_state) ++ .await?; + continue; + } + + execution_state.checkpoint.phase = HubSpotSyncPhase::ContactEnqueuePending; +- self.persist_hubspot_sync_progress(&execution_state).await?; ++ self.persist_hubspot_sync_progress(execution, &execution_state) ++ .await?; + break; + } + } +@@ -209,7 +211,8 @@ impl CrmSyncAccountsHandler { + ); + })?; + execution_state.checkpoint.phase = HubSpotSyncPhase::ContactEnqueueCompleted; +- self.persist_hubspot_sync_progress(&execution_state).await?; ++ self.persist_hubspot_sync_progress(execution, &execution_state) ++ .await?; + } + + info!( +diff --git a/crates/onesales-jobs/src/crm_accounts_sync/job_handler.rs b/crates/onesales-jobs/src/crm_accounts_sync/job_handler.rs +index efe9f9dde..894a8cc05 100644 +--- a/crates/onesales-jobs/src/crm_accounts_sync/job_handler.rs ++++ b/crates/onesales-jobs/src/crm_accounts_sync/job_handler.rs +@@ -2,17 +2,17 @@ use super::*; + use runledger_core::jobs::JobCompletion; + + #[async_trait::async_trait] +-impl JobHandler for CrmSyncAccountsHandler { ++impl JobExecutionHandler for CrmSyncAccountsHandler { + fn job_type(&self) -> runledger_core::jobs::JobType<'static> { + CRM_SYNC_ACCOUNTS_JOB_TYPE + } + + async fn execute( + &self, +- context: JobContext, ++ execution: JobExecution<'_>, + payload: Value, + ) -> Result { +- let handler_started_at = Instant::now(); ++ let context = execution.context().clone(); + let organization_id = context.organization_id.ok_or_else(|| { + terminal_failure( + "job.organization_required", +@@ -75,7 +75,7 @@ impl JobHandler for CrmSyncAccountsHandler { + &context, + organization_id, + &mut connection, +- handler_started_at, ++ execution, + ) + .await; + +@@ -148,7 +148,7 @@ impl JobHandler for CrmSyncAccountsHandler { + &context, + organization_id, + &connection, +- handler_started_at, ++ execution, + ¤t_token_fingerprint, + ) + .await; +diff --git a/crates/onesales-jobs/src/crm_accounts_sync/salesforce/page_adapter.rs b/crates/onesales-jobs/src/crm_accounts_sync/salesforce/page_adapter.rs +index 8dca127fa..52c279b96 100644 +--- a/crates/onesales-jobs/src/crm_accounts_sync/salesforce/page_adapter.rs ++++ b/crates/onesales-jobs/src/crm_accounts_sync/salesforce/page_adapter.rs +@@ -1,7 +1,7 @@ + use async_trait::async_trait; + use onesales_db::crm::SalesforceSyncConnectionRecord; + use onesales_salesforce as salesforce; +-use runledger_core::jobs::{JobContext, JobFailure}; ++use runledger_core::jobs::{JobContext, JobExecution, JobFailure}; + use serde_json::Value; + use tracing::{error, info, warn}; + use url::Url; +@@ -23,6 +23,7 @@ use super::{ + pub(in crate::crm_accounts_sync) struct AccountPageSyncAdapter<'a> { + handler: &'a CrmSyncAccountsHandler, + context: &'a JobContext, ++ execution: JobExecution<'a>, + organization_id: Uuid, + connection: &'a mut SalesforceSyncConnectionRecord, + account_query_capabilities: AccountQueryCapabilities, +@@ -35,6 +36,7 @@ impl<'a> AccountPageSyncAdapter<'a> { + pub(in crate::crm_accounts_sync) fn new( + handler: &'a CrmSyncAccountsHandler, + context: &'a JobContext, ++ execution: JobExecution<'a>, + organization_id: Uuid, + connection: &'a mut SalesforceSyncConnectionRecord, + account_query_capabilities: AccountQueryCapabilities, +@@ -43,6 +45,7 @@ impl<'a> AccountPageSyncAdapter<'a> { + Self { + handler, + context, ++ execution, + organization_id, + connection, + account_query_capabilities, +@@ -310,7 +313,9 @@ impl SalesforcePageSyncAdapter for AccountPageSyncAdapter<'_> { + &mut self, + execution_state: &checkpoint::SyncExecutionState, + ) -> Result<(), JobFailure> { +- self.handler.persist_sync_progress(execution_state).await ++ self.handler ++ .persist_sync_progress(self.execution, execution_state) ++ .await + } + + fn prepare_checkpoint_snapshot( +diff --git a/crates/onesales-jobs/src/crm_accounts_sync/tests.rs b/crates/onesales-jobs/src/crm_accounts_sync/tests.rs +index f54e2e12f..d1b61ab4c 100644 +--- a/crates/onesales-jobs/src/crm_accounts_sync/tests.rs ++++ b/crates/onesales-jobs/src/crm_accounts_sync/tests.rs +@@ -16,3 +16,89 @@ mod sync_flow_tests; + mod token_refresh_tests; + #[path = "tests/unit_tests.rs"] + mod unit_tests; ++ ++#[async_trait::async_trait] ++impl runledger_core::jobs::JobExecutionServices for TestExecutionServices<'_> { ++ fn deadline(&self) -> std::time::Instant { ++ self.deadline ++ } ++ fn remaining_budget(&self) -> std::time::Duration { ++ self.deadline ++ .saturating_duration_since(std::time::Instant::now()) ++ } ++ async fn persist_progress( ++ &self, ++ update: runledger_core::jobs::JobExecutionUpdate<'_>, ++ ) -> Result<(), runledger_core::jobs::JobExecutionError> { ++ use runledger_core::jobs::JobExecutionError; ++ // Older provider-only tests deliberately have no queue row. ++ if !self.queued { ++ return Ok(()); ++ } ++ runledger_postgres::jobs::update_job_ordinary_progress_for_lease( ++ self.pool, ++ runledger_postgres::jobs::JobLeaseIdentity::new( ++ self.context.job_id, ++ self.context.run_number, ++ self.context.attempt, ++ &self.context.worker_id, ++ ), ++ &runledger_postgres::jobs::JobOrdinaryProgressUpdate { ++ progress_done: update.progress_done, ++ progress_total: update.progress_total, ++ checkpoint: update.checkpoint, ++ }, ++ ) ++ .await ++ .map_err(|error| match error { ++ runledger_postgres::Error::QueryError(ref error) ++ if error.code() == "job.lease_owner_mismatch" => ++ { ++ JobExecutionError::LeaseLost ++ } ++ _ => JobExecutionError::PersistenceFailed, ++ }) ++ } ++} ++ ++struct TestExecutionServices<'a> { ++ pool: &'a onesales_db::DbPool, ++ context: &'a runledger_core::jobs::JobContext, ++ deadline: std::time::Instant, ++ queued: bool, ++} ++ ++impl super::CrmSyncAccountsHandler { ++ async fn execute( ++ &self, ++ context: runledger_core::jobs::JobContext, ++ payload: serde_json::Value, ++ ) -> Result { ++ use runledger_core::jobs::{JobExecution, JobExecutionHandler}; ++ // Invalid-input unit tests must not need a database. ++ let needs_database = context.organization_id.is_some() ++ && serde_json::from_value::( ++ payload.clone(), ++ ) ++ .is_ok(); ++ let job = if needs_database { ++ runledger_postgres::jobs::get_job_by_id( ++ &self.pool, ++ context.organization_id, ++ context.job_id, ++ ) ++ .await ++ .expect("load test execution fixture") ++ } else { ++ None ++ }; ++ let timeout = job.as_ref().map_or(3600, |job| job.timeout_seconds.max(1)) as u64; ++ let services = TestExecutionServices { ++ pool: &self.pool, ++ context: &context, ++ queued: job.is_some(), ++ deadline: std::time::Instant::now() + std::time::Duration::from_secs(timeout), ++ }; ++ JobExecutionHandler::execute(self, JobExecution::new(&context, &services), payload).await ++ } ++} +diff --git a/crates/onesales-jobs/src/crm_accounts_sync/tests/checkpoint_resume_tests.rs b/crates/onesales-jobs/src/crm_accounts_sync/tests/checkpoint_resume_tests.rs +index 14de1ea78..23bd38c5c 100644 +--- a/crates/onesales-jobs/src/crm_accounts_sync/tests/checkpoint_resume_tests.rs ++++ b/crates/onesales-jobs/src/crm_accounts_sync/tests/checkpoint_resume_tests.rs +@@ -6,8 +6,6 @@ use runledger_postgres::jobs::{ + }; + use serde_json::json; + +-use crate::registry::JobHandler; +- + use super::super::checkpoint::parse_salesforce_sync_checkpoint; + use super::super::*; + use super::db_helpers::{ +@@ -22,6 +20,12 @@ use super::mock_salesforce_server::{ + #[tokio::test] + async fn sync_resumes_from_checkpoint_after_page_cap_continuation() { + let harness = db_harness("crm-sync-resume-after-page-cap").await; ++ let version: String = sqlx::query_scalar("SHOW server_version") ++ .fetch_one(&harness.pool) ++ .await ++ .expect("read PostgreSQL version"); ++ eprintln!("CRM execution-services migration: PostgreSQL {version}"); ++ assert!(version.starts_with("18.")); + let timeout_seconds = PAGE_CAP_GUARD_SECONDS + (PAGE_CAP_SECONDS_PER_PAGE * 2); + register_crm_sync_job_definition(&harness.pool, timeout_seconds).await; + let (instance_url, query_all_hits, page_two_hits, page_three_hits, server) = +@@ -88,7 +92,7 @@ async fn sync_resumes_from_checkpoint_after_page_cap_continuation() { + assert_eq!(checkpoint_after_first.source_job_id, Some(job_id)); + assert_eq!(checkpoint_after_first.source_run_number, Some(1)); + assert_eq!( +- first_completion.checkpoint.as_ref(), ++ first_completion.checkpoint_value(), + after_first_attempt.checkpoint.as_ref() + ); + +@@ -100,9 +104,9 @@ async fn sync_resumes_from_checkpoint_after_page_cap_continuation() { + "resume-worker", + &JobContinuationUpdate { + delay: std::time::Duration::ZERO, +- progress_done: first_completion.progress_done, +- progress_total: first_completion.progress_total, +- checkpoint: first_completion.checkpoint.as_ref(), ++ progress_done: first_completion.progress_done(), ++ progress_total: first_completion.progress_total(), ++ checkpoint: first_completion.checkpoint_value(), + }, + ) + .await +@@ -134,9 +138,9 @@ async fn sync_resumes_from_checkpoint_after_page_cap_continuation() { + claimed_second_job.attempt, + "resume-worker", + Some(&JobCompletionUpdate { +- progress_done: final_completion.progress_done, +- progress_total: final_completion.progress_total, +- checkpoint: final_completion.checkpoint.as_ref(), ++ progress_done: final_completion.progress_done(), ++ progress_total: final_completion.progress_total(), ++ checkpoint: final_completion.checkpoint_value(), + output: final_completion.output(), + }), + ) +@@ -217,12 +221,6 @@ async fn sync_returns_terminal_failure_for_invalid_checkpoint_shape() { + .await + .expect("enqueue crm.sync.accounts"); + +- let claimed = claim_jobs(&harness.pool, "checkpoint-worker", 60, 1) +- .await +- .expect("claim job"); +- assert_eq!(claimed.len(), 1); +- let claimed_job = &claimed[0]; +- + sqlx::query( + "UPDATE job_queue + SET checkpoint = $2::jsonb +@@ -239,6 +237,13 @@ async fn sync_returns_terminal_failure_for_invalid_checkpoint_shape() { + .await + .expect("set invalid checkpoint payload"); + ++ // Resume state must be durable before the runtime captures the claimed snapshot. ++ let claimed = claim_jobs(&harness.pool, "checkpoint-worker", 60, 1) ++ .await ++ .expect("claim job"); ++ assert_eq!(claimed.len(), 1); ++ let claimed_job = &claimed[0]; ++ + let err = handler + .execute(context_from_claimed_job(claimed_job), payload) + .await +@@ -292,12 +297,6 @@ async fn sync_retry_without_next_records_url_uses_base_query_when_owner_profile_ + .await + .expect("enqueue crm.sync.accounts"); + +- let claimed = claim_jobs(&harness.pool, "resume-worker", 60, 1) +- .await +- .expect("claim job"); +- assert_eq!(claimed.len(), 1); +- let claimed_job = &claimed[0]; +- + sqlx::query( + "UPDATE job_queue + SET checkpoint = $2::jsonb +@@ -315,6 +314,13 @@ async fn sync_retry_without_next_records_url_uses_base_query_when_owner_profile_ + .await + .expect("set retry checkpoint payload"); + ++ // Resume state must be durable before the runtime captures the claimed snapshot. ++ let claimed = claim_jobs(&harness.pool, "resume-worker", 60, 1) ++ .await ++ .expect("claim job"); ++ assert_eq!(claimed.len(), 1); ++ let claimed_job = &claimed[0]; ++ + handler + .execute(context_from_claimed_job(claimed_job), payload) + .await +diff --git a/crates/onesales-jobs/src/crm_accounts_sync/tests/contact_fanout_tests.rs b/crates/onesales-jobs/src/crm_accounts_sync/tests/contact_fanout_tests.rs +index 4a6198bc9..c42e7c926 100644 +--- a/crates/onesales-jobs/src/crm_accounts_sync/tests/contact_fanout_tests.rs ++++ b/crates/onesales-jobs/src/crm_accounts_sync/tests/contact_fanout_tests.rs +@@ -3,8 +3,6 @@ use runledger_core::jobs::JobContext; + use serde_json::json; + use uuid::Uuid; + +-use crate::registry::JobHandler; +- + use super::db_helpers::{ + insert_connected_salesforce_connection, seed_additional_mapping_for_existing_account, + seed_deleted_mapping, seed_mapping_with_external_id, +diff --git a/crates/onesales-jobs/src/crm_accounts_sync/tests/sync_flow_tests.rs b/crates/onesales-jobs/src/crm_accounts_sync/tests/sync_flow_tests.rs +index 5e61ca5f1..f3f2a1f78 100644 +--- a/crates/onesales-jobs/src/crm_accounts_sync/tests/sync_flow_tests.rs ++++ b/crates/onesales-jobs/src/crm_accounts_sync/tests/sync_flow_tests.rs +@@ -15,8 +15,6 @@ use serde_json::json; + use tokio::net::TcpListener; + use uuid::Uuid; + +-use crate::registry::JobHandler; +- + use super::db_helpers::context_from_claimed_job; + use super::db_helpers::{ + count_connection_user_mappings, insert_connected_hubspot_connection, +diff --git a/crates/onesales-jobs/src/crm_accounts_sync/tests/sync_flow_tests/hubspot_enqueue.rs b/crates/onesales-jobs/src/crm_accounts_sync/tests/sync_flow_tests/hubspot_enqueue.rs +index b4f1d68f2..1171bda8a 100644 +--- a/crates/onesales-jobs/src/crm_accounts_sync/tests/sync_flow_tests/hubspot_enqueue.rs ++++ b/crates/onesales-jobs/src/crm_accounts_sync/tests/sync_flow_tests/hubspot_enqueue.rs +@@ -220,8 +220,7 @@ async fn hubspot_company_sync_page_cap_continues_same_job_with_checkpoint() { + assert_eq!(page_hits.load(Ordering::SeqCst), 1); + + let checkpoint = completion +- .checkpoint +- .as_ref() ++ .checkpoint_value() + .expect("continuation should carry checkpoint"); + assert_eq!(checkpoint["after"], "page-2"); + assert_eq!(checkpoint["pages_completed"], 1); +@@ -238,9 +237,9 @@ async fn hubspot_company_sync_page_cap_continues_same_job_with_checkpoint() { + "hubspot-account-continuation-worker", + &JobContinuationUpdate { + delay: std::time::Duration::ZERO, +- progress_done: completion.progress_done, +- progress_total: completion.progress_total, +- checkpoint: completion.checkpoint.as_ref(), ++ progress_done: completion.progress_done(), ++ progress_total: completion.progress_total(), ++ checkpoint: completion.checkpoint_value(), + }, + ) + .await +diff --git a/crates/onesales-jobs/src/crm_accounts_sync/tests/token_refresh_tests.rs b/crates/onesales-jobs/src/crm_accounts_sync/tests/token_refresh_tests.rs +index 05220736a..adf55b0fb 100644 +--- a/crates/onesales-jobs/src/crm_accounts_sync/tests/token_refresh_tests.rs ++++ b/crates/onesales-jobs/src/crm_accounts_sync/tests/token_refresh_tests.rs +@@ -6,8 +6,6 @@ use runledger_core::jobs::{JobContext, JobFailureKind}; + use serde_json::json; + use uuid::Uuid; + +-use crate::registry::JobHandler; +- + use super::db_helpers::{ + insert_connected_salesforce_connection, + insert_connected_salesforce_connection_with_oauth_client, load_connection_status_and_error, +diff --git a/crates/onesales-jobs/src/crm_accounts_sync/tests/unit_tests.rs b/crates/onesales-jobs/src/crm_accounts_sync/tests/unit_tests.rs +index e00dc608f..2cf91a7ad 100644 +--- a/crates/onesales-jobs/src/crm_accounts_sync/tests/unit_tests.rs ++++ b/crates/onesales-jobs/src/crm_accounts_sync/tests/unit_tests.rs +@@ -5,8 +5,6 @@ use runledger_postgres::QueryErrorCategory; + use serde_json::json; + use uuid::Uuid; + +-use crate::registry::JobHandler; +- + use super::super::checkpoint::{parse_hubspot_sync_checkpoint, parse_salesforce_sync_checkpoint}; + use super::super::*; + use super::db_helpers::test_handler; +diff --git a/crates/onesales-jobs/src/crm_contacts_sync/tests/checkpoint_resume_tests.rs b/crates/onesales-jobs/src/crm_contacts_sync/tests/checkpoint_resume_tests.rs +index 92e626a81..b3d96f95b 100644 +--- a/crates/onesales-jobs/src/crm_contacts_sync/tests/checkpoint_resume_tests.rs ++++ b/crates/onesales-jobs/src/crm_contacts_sync/tests/checkpoint_resume_tests.rs +@@ -88,7 +88,7 @@ async fn sync_resumes_from_checkpoint_after_page_cap_continuation_and_preserves_ + ); + assert_eq!(after_first_attempt.progress_done, Some(2)); + assert_eq!( +- first_completion.checkpoint.as_ref(), ++ first_completion.checkpoint_value(), + after_first_attempt.checkpoint.as_ref() + ); + +@@ -100,9 +100,9 @@ async fn sync_resumes_from_checkpoint_after_page_cap_continuation_and_preserves_ + "resume-worker", + &JobContinuationUpdate { + delay: std::time::Duration::ZERO, +- progress_done: first_completion.progress_done, +- progress_total: first_completion.progress_total, +- checkpoint: first_completion.checkpoint.as_ref(), ++ progress_done: first_completion.progress_done(), ++ progress_total: first_completion.progress_total(), ++ checkpoint: first_completion.checkpoint_value(), + }, + ) + .await +@@ -134,9 +134,9 @@ async fn sync_resumes_from_checkpoint_after_page_cap_continuation_and_preserves_ + claimed_second_job.attempt, + "resume-worker", + Some(&JobCompletionUpdate { +- progress_done: final_completion.progress_done, +- progress_total: final_completion.progress_total, +- checkpoint: final_completion.checkpoint.as_ref(), ++ progress_done: final_completion.progress_done(), ++ progress_total: final_completion.progress_total(), ++ checkpoint: final_completion.checkpoint_value(), + output: final_completion.output(), + }), + ) +diff --git a/crates/onesales-jobs/src/crm_contacts_sync/tests/sync_flow_tests/hubspot_reconciliation.rs b/crates/onesales-jobs/src/crm_contacts_sync/tests/sync_flow_tests/hubspot_reconciliation.rs +index a492b54dd..efc8e0fdc 100644 +--- a/crates/onesales-jobs/src/crm_contacts_sync/tests/sync_flow_tests/hubspot_reconciliation.rs ++++ b/crates/onesales-jobs/src/crm_contacts_sync/tests/sync_flow_tests/hubspot_reconciliation.rs +@@ -117,8 +117,7 @@ async fn hubspot_contact_sync_reconciliation_resumes_from_checkpoint_without_ref + assert_eq!(archived_hits.load(Ordering::SeqCst), 1); + + let checkpoint = first_completion +- .checkpoint +- .as_ref() ++ .checkpoint_value() + .expect("reconciliation continuation should carry checkpoint"); + assert_eq!( + checkpoint["phase"], +@@ -135,9 +134,9 @@ async fn hubspot_contact_sync_reconciliation_resumes_from_checkpoint_without_ref + "hubspot-contact-reconcile-source-worker", + &JobContinuationUpdate { + delay: std::time::Duration::ZERO, +- progress_done: first_completion.progress_done, +- progress_total: first_completion.progress_total, +- checkpoint: first_completion.checkpoint.as_ref(), ++ progress_done: first_completion.progress_done(), ++ progress_total: first_completion.progress_total(), ++ checkpoint: first_completion.checkpoint_value(), + }, + ) + .await +diff --git a/crates/onesales-jobs/src/crm_contacts_sync/tests/sync_flow_tests/hubspot_token_and_pagination.rs b/crates/onesales-jobs/src/crm_contacts_sync/tests/sync_flow_tests/hubspot_token_and_pagination.rs +index c1bc42839..9c4781ef1 100644 +--- a/crates/onesales-jobs/src/crm_contacts_sync/tests/sync_flow_tests/hubspot_token_and_pagination.rs ++++ b/crates/onesales-jobs/src/crm_contacts_sync/tests/sync_flow_tests/hubspot_token_and_pagination.rs +@@ -376,8 +376,7 @@ async fn hubspot_contact_sync_page_cap_continues_same_job_with_checkpoint() { + assert_eq!(page_hits.load(Ordering::SeqCst), 1); + + let checkpoint = completion +- .checkpoint +- .as_ref() ++ .checkpoint_value() + .expect("continuation should carry checkpoint"); + assert_eq!(checkpoint["after"], "page-2"); + assert_eq!(checkpoint["pages_completed"], 1); +@@ -394,9 +393,9 @@ async fn hubspot_contact_sync_page_cap_continues_same_job_with_checkpoint() { + "hubspot-contact-continuation-worker", + &JobContinuationUpdate { + delay: std::time::Duration::ZERO, +- progress_done: completion.progress_done, +- progress_total: completion.progress_total, +- checkpoint: completion.checkpoint.as_ref(), ++ progress_done: completion.progress_done(), ++ progress_total: completion.progress_total(), ++ checkpoint: completion.checkpoint_value(), + }, + ) + .await +@@ -509,8 +508,7 @@ async fn hubspot_contact_sync_continuation_preserves_source_identity_across_runs + JobCompletionDisposition::ContinueAfter(std::time::Duration::ZERO) + ); + let first_checkpoint = first_completion +- .checkpoint +- .as_ref() ++ .checkpoint_value() + .expect("first continuation should carry checkpoint"); + assert_eq!(first_checkpoint["source_job_id"], source_job_id.to_string()); + assert_eq!(first_checkpoint["source_run_number"], claimed[0].run_number); +@@ -523,9 +521,9 @@ async fn hubspot_contact_sync_continuation_preserves_source_identity_across_runs + "hubspot-contact-source-retry-worker", + &JobContinuationUpdate { + delay: std::time::Duration::ZERO, +- progress_done: first_completion.progress_done, +- progress_total: first_completion.progress_total, +- checkpoint: first_completion.checkpoint.as_ref(), ++ progress_done: first_completion.progress_done(), ++ progress_total: first_completion.progress_total(), ++ checkpoint: first_completion.checkpoint_value(), + }, + ) + .await +@@ -553,8 +551,7 @@ async fn hubspot_contact_sync_continuation_preserves_source_identity_across_runs + JobCompletionDisposition::ContinueAfter(std::time::Duration::ZERO) + ); + let second_checkpoint = second_completion +- .checkpoint +- .as_ref() ++ .checkpoint_value() + .expect("second continuation should carry checkpoint"); + assert_eq!( + second_checkpoint["source_job_id"], +diff --git a/crates/onesales-jobs/src/crm_sync_common.rs b/crates/onesales-jobs/src/crm_sync_common.rs +index 51538b8cf..e4b660628 100644 +--- a/crates/onesales-jobs/src/crm_sync_common.rs ++++ b/crates/onesales-jobs/src/crm_sync_common.rs +@@ -1,8 +1,8 @@ + pub(crate) use auth::refresh_salesforce_access_token; + pub(crate) use checkpoint::{ + SyncCheckpoint, SyncCheckpointFailurePolicy, SyncCheckpointParseConfig, SyncExecutionState, +- derive_max_pages_per_attempt, load_sync_execution_state, parse_sync_checkpoint, +- persist_sync_progress, ++ SyncProgressMode, derive_max_pages_per_attempt, load_sync_execution_state, ++ parse_sync_checkpoint, persist_sync_progress, + }; + pub(crate) use failures::{ + SALESFORCE_CHECKPOINT_INVALID_FAILURE_CODE, SYNC_CHECKPOINT_INVALID_FAILURE_CODE, +diff --git a/crates/onesales-jobs/src/crm_sync_common/checkpoint.rs b/crates/onesales-jobs/src/crm_sync_common/checkpoint.rs +index 0b95f253c..cc1375632 100644 +--- a/crates/onesales-jobs/src/crm_sync_common/checkpoint.rs ++++ b/crates/onesales-jobs/src/crm_sync_common/checkpoint.rs +@@ -19,6 +19,7 @@ use super::{ + #[derive(Clone, Debug)] + pub(crate) enum SyncProgressMode { + DisabledCompatibility, ++ RuntimeServices, + Enabled { + job_id: Uuid, + run_number: i32, +@@ -151,6 +152,15 @@ pub(crate) async fn persist_sync_progress( + where + TCheckpoint: Serialize + SyncCheckpoint, + { ++ if matches!( ++ execution_state.progress_mode, ++ SyncProgressMode::RuntimeServices ++ ) { ++ return Err(JobFailure::terminal( ++ "job.execution_services_required", ++ "Runtime-managed progress requires execution services.", ++ )); ++ } + let SyncProgressMode::Enabled { + job_id, + run_number, +diff --git a/crates/onesales-jobs/src/crm_sync_common/outcome.rs b/crates/onesales-jobs/src/crm_sync_common/outcome.rs +index f34ed289c..0f4c8ad4e 100644 +--- a/crates/onesales-jobs/src/crm_sync_common/outcome.rs ++++ b/crates/onesales-jobs/src/crm_sync_common/outcome.rs +@@ -57,6 +57,6 @@ mod tests { + completion.disposition(), + JobCompletionDisposition::ContinueAfter(std::time::Duration::ZERO) + ); +- assert_eq!(completion.checkpoint.as_ref(), Some(&checkpoint)); ++ assert_eq!(completion.checkpoint_value(), Some(&checkpoint)); + } + } +diff --git a/crates/onesales-jobs/src/outbound_drafts_cleanup.rs b/crates/onesales-jobs/src/outbound_drafts_cleanup.rs +index f88530d12..dae070298 100644 +--- a/crates/onesales-jobs/src/outbound_drafts_cleanup.rs ++++ b/crates/onesales-jobs/src/outbound_drafts_cleanup.rs +@@ -211,7 +211,10 @@ mod tests { + completion.disposition(), + JobCompletionDisposition::ContinueAfter(std::time::Duration::ZERO) + ); +- let checkpoint = completion.checkpoint.expect("continuation checkpoint"); ++ let checkpoint = completion ++ .checkpoint_value() ++ .cloned() ++ .expect("continuation checkpoint"); + assert_eq!(checkpoint["version"], EPHEMERAL_CLEANUP_CHECKPOINT_VERSION); + assert_eq!(checkpoint["runs_completed"], 1); + assert_eq!( +@@ -230,7 +233,7 @@ mod tests { + .expect("short batch should finish"); + + assert_eq!(completion.disposition(), JobCompletionDisposition::Succeed); +- assert_eq!(completion.checkpoint, None); ++ assert_eq!(completion.checkpoint_value().cloned(), None); + } + + #[test] +diff --git a/crates/onesales-jobs/tests/account_research_handlers/champion_tracking_monthly_scan_tests/helpers_jobs_and_pipeline.rs b/crates/onesales-jobs/tests/account_research_handlers/champion_tracking_monthly_scan_tests/helpers_jobs_and_pipeline.rs +index 46fbe49b1..ba681d505 100644 +--- a/crates/onesales-jobs/tests/account_research_handlers/champion_tracking_monthly_scan_tests/helpers_jobs_and_pipeline.rs ++++ b/crates/onesales-jobs/tests/account_research_handlers/champion_tracking_monthly_scan_tests/helpers_jobs_and_pipeline.rs +@@ -110,9 +110,9 @@ async fn persist_monthly_scan_wait_continuation( + worker_id, + &JobContinuationUpdate { + delay: std::time::Duration::from_secs(30 * 60), +- progress_done: completion.progress_done, +- progress_total: completion.progress_total, +- checkpoint: completion.checkpoint.as_ref(), ++ progress_done: completion.progress_done(), ++ progress_total: completion.progress_total(), ++ checkpoint: completion.checkpoint_value(), + }, + ) + .await +diff --git a/crates/onesales-jobs/tests/account_research_handlers/common/pipeline.rs b/crates/onesales-jobs/tests/account_research_handlers/common/pipeline.rs +index 150eefb61..0608b10f9 100644 +--- a/crates/onesales-jobs/tests/account_research_handlers/common/pipeline.rs ++++ b/crates/onesales-jobs/tests/account_research_handlers/common/pipeline.rs +@@ -37,8 +37,7 @@ pub(crate) fn assert_robot_poll_continuation( + JobCompletionDisposition::ContinueAfter(std::time::Duration::from_secs(delay_seconds)) + ); + let checkpoint = completion +- .checkpoint +- .as_ref() ++ .checkpoint_value() + .expect("robot poll continuation checkpoint"); + assert_eq!( + checkpoint +@@ -64,7 +63,8 @@ pub(crate) fn assert_direct_continuation( + JobCompletionDisposition::ContinueAfter(std::time::Duration::from_secs(delay_seconds)) + ); + completion +- .checkpoint ++ .checkpoint_value() ++ .cloned() + .clone() + .expect("direct continuation checkpoint") + } diff --git a/docs/measurements/operational-costs-2026-09-05/after.txt b/docs/measurements/operational-costs-2026-09-05/after.txt new file mode 100644 index 0000000..410630f --- /dev/null +++ b/docs/measurements/operational-costs-2026-09-05/after.txt @@ -0,0 +1,32 @@ + Compiling runledger-postgres v0.12.0 (/home/aa/Documents/runledger/runledger-postgres) + Finished `test` profile [unoptimized + debuginfo] target(s) in 6.13s + Running tests/operational_costs.rs (target/debug/deps/operational_costs-0c0064ed170bb3b9) + +running 1 test +PostgreSQL 18.6 (Debian 18.6-1.pgdg13+2); server_version_num=180006; pool max=1 +full offset=0 json_bytes=1232100: median_ms=14.819 p95_ms=18.150 +full offset=9000 json_bytes=1232100: median_ms=18.316 p95_ms=21.799 +compact after=false json_bytes=0: median_ms=2.250 p95_ms=2.809 +compact after=true json_bytes=0: median_ms=1.912 p95_ms=2.758 +compact raw cursor at depth 9000: median_ms=1.481 p95_ms=1.796 +compact raw offset=9000: median_ms=4.679 p95_ms=5.539 +workflow V=10 E=9: median_ms=13.083 p95_ms=57.564 +12 enqueues: [("edge inserts", 12), ("other statements", 168), ("step inserts", 12)] +workflow V=10 E=30: median_ms=13.475 p95_ms=33.997 +12 enqueues: [("edge inserts", 12), ("other statements", 168), ("step inserts", 12)] +workflow V=100 E=99: median_ms=23.513 p95_ms=51.518 +12 enqueues: [("edge inserts", 12), ("other statements", 168), ("step inserts", 12)] +workflow V=100 E=390: median_ms=42.835 p95_ms=275.887 +12 enqueues: [("edge inserts", 24), ("other statements", 168), ("step inserts", 12)] +workflow V=600 E=599: median_ms=147.887 p95_ms=203.130 +12 enqueues: [("edge inserts", 36), ("other statements", 168), ("step inserts", 36)] +workflow V=600 E=2390: median_ms=335.263 p95_ms=453.667 +12 enqueues: [("edge inserts", 120), ("other statements", 168), ("step inserts", 36)] +100 direct jobs own_transaction=true: median_ms=895.000 p95_ms=1124.848 +12 direct groups: total statements=4800 +100 direct jobs own_transaction=false: median_ms=77.239 p95_ms=94.182 +12 direct groups: total statements=2424 +connections=1, idle=0 +test measure_operational_costs ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 30.70s diff --git a/docs/measurements/operational-costs-2026-09-05/baseline.txt b/docs/measurements/operational-costs-2026-09-05/baseline.txt new file mode 100644 index 0000000..ce7dfed --- /dev/null +++ b/docs/measurements/operational-costs-2026-09-05/baseline.txt @@ -0,0 +1,24 @@ + Compiling runledger-postgres v0.12.0 (/home/aa/Documents/runledger/runledger-postgres) + Finished `test` profile [unoptimized + debuginfo] target(s) in 10.64s + Running tests/operational_costs.rs (target/debug/deps/operational_costs-0c0064ed170bb3b9) + +running 1 test +PostgreSQL 18.6 (Debian 18.6-1.pgdg13+2); server_version_num=180006; pool max=1 +full offset=0 json_bytes=1232100: median_ms=20.422 p95_ms=23.199 +full offset=9000 json_bytes=1232100: median_ms=20.490 p95_ms=22.162 +workflow V=10 E=9: median_ms=13.449 p95_ms=16.268 +12 enqueues: [("edge inserts", 108), ("other statements", 168), ("step inserts", 120)] +workflow V=10 E=30: median_ms=17.232 p95_ms=20.165 +12 enqueues: [("edge inserts", 360), ("other statements", 168), ("step inserts", 120)] +workflow V=100 E=99: median_ms=70.956 p95_ms=80.737 +12 enqueues: [("edge inserts", 1188), ("other statements", 168), ("step inserts", 1200)] +workflow V=100 E=390: median_ms=142.869 p95_ms=153.638 +12 enqueues: [("edge inserts", 4680), ("other statements", 168), ("step inserts", 1200)] +workflow V=600 E=599: median_ms=378.741 p95_ms=409.424 +12 enqueues: [("edge inserts", 7188), ("other statements", 168), ("step inserts", 7200)] +workflow V=600 E=2390: median_ms=816.635 p95_ms=859.004 +12 enqueues: [("edge inserts", 28680), ("other statements", 168), ("step inserts", 7200)] +connections=1, idle=0 +test measure_operational_costs ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 26.84s diff --git a/docs/measurements/operational-costs-2026-09-05/first-after.txt b/docs/measurements/operational-costs-2026-09-05/first-after.txt new file mode 100644 index 0000000..de57d92 --- /dev/null +++ b/docs/measurements/operational-costs-2026-09-05/first-after.txt @@ -0,0 +1,28 @@ + Compiling runledger-postgres v0.12.0 (/home/aa/Documents/runledger/runledger-postgres) + Finished `test` profile [unoptimized + debuginfo] target(s) in 6.92s + Running tests/operational_costs.rs (target/debug/deps/operational_costs-0c0064ed170bb3b9) + +running 1 test +PostgreSQL 18.6 (Debian 18.6-1.pgdg13+2); server_version_num=180006; pool max=1 +full offset=0 json_bytes=1232100: median_ms=13.655 p95_ms=14.652 +full offset=9000 json_bytes=1232100: median_ms=17.592 p95_ms=19.048 +compact after=false json_bytes=0: median_ms=1.912 p95_ms=2.588 +compact after=true json_bytes=0: median_ms=2.227 p95_ms=2.722 +compact raw cursor at depth 9000: median_ms=1.499 p95_ms=1.822 +compact raw offset=9000: median_ms=4.967 p95_ms=5.767 +workflow V=10 E=9: median_ms=13.859 p95_ms=737.573 +12 enqueues: [("edge inserts", 12), ("other statements", 168), ("step inserts", 12)] +workflow V=10 E=30: median_ms=12.125 p95_ms=33.737 +12 enqueues: [("edge inserts", 12), ("other statements", 168), ("step inserts", 12)] +workflow V=100 E=99: median_ms=24.653 p95_ms=27.136 +12 enqueues: [("edge inserts", 12), ("other statements", 168), ("step inserts", 12)] +workflow V=100 E=390: median_ms=39.526 p95_ms=47.272 +12 enqueues: [("edge inserts", 24), ("other statements", 168), ("step inserts", 12)] +workflow V=600 E=599: median_ms=137.274 p95_ms=142.243 +12 enqueues: [("edge inserts", 36), ("other statements", 168), ("step inserts", 36)] +workflow V=600 E=2390: median_ms=354.841 p95_ms=369.156 +12 enqueues: [("edge inserts", 120), ("other statements", 168), ("step inserts", 36)] +connections=1, idle=0 +test measure_operational_costs ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 24.83s diff --git a/docs/measurements/operational-costs-2026-09-05/plans.txt b/docs/measurements/operational-costs-2026-09-05/plans.txt new file mode 100644 index 0000000..961d274 --- /dev/null +++ b/docs/measurements/operational-costs-2026-09-05/plans.txt @@ -0,0 +1,15 @@ + Compiling runledger-postgres v0.12.0 (/home/aa/Documents/runledger/runledger-postgres) + Finished `test` profile [unoptimized + debuginfo] target(s) in 5.44s + Running tests/job_summary_plans.rs (target/debug/deps/job_summary_plans-7a506079c94e6d52) + +running 1 test +summary plans PostgreSQL 18.6 (Debian 18.6-1.pgdg13+2) +force_custom_plan Global: execution=0.064 ms, shared buffers=4 +force_custom_plan Organization(c4ca4238-a0b9-2382-0dcc-509a6f75849b): execution=0.031 ms, shared buffers=5 +force_custom_plan Admin: execution=0.058 ms, shared buffers=3 +force_generic_plan Global: execution=0.065 ms, shared buffers=4 +force_generic_plan Organization(c4ca4238-a0b9-2382-0dcc-509a6f75849b): execution=0.055 ms, shared buffers=4 +force_generic_plan Admin: execution=0.044 ms, shared buffers=3 +test actual_summary_cursor_constrains_index_with_custom_and_generic_plans ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 1.56s diff --git a/docs/measurements/operational-costs-2026-09-05/second-after.txt b/docs/measurements/operational-costs-2026-09-05/second-after.txt new file mode 100644 index 0000000..2c22f67 --- /dev/null +++ b/docs/measurements/operational-costs-2026-09-05/second-after.txt @@ -0,0 +1,34 @@ + Compiling runledger-core v0.12.0 (/home/aa/Documents/runledger/runledger-core) + Compiling runledger-test-support v0.12.0 (/home/aa/Documents/runledger/runledger-test-support) + Compiling runledger-postgres v0.12.0 (/home/aa/Documents/runledger/runledger-postgres) + Finished `test` profile [unoptimized + debuginfo] target(s) in 8.99s + Running tests/operational_costs.rs (target/debug/deps/operational_costs-0c0064ed170bb3b9) + +running 1 test +PostgreSQL 18.6 (Debian 18.6-1.pgdg13+2); server_version_num=180006; pool max=1 +full offset=0 json_bytes=1232100: median_ms=13.680 p95_ms=15.665 +full offset=9000 json_bytes=1232100: median_ms=18.089 p95_ms=20.593 +compact after=false json_bytes=0: median_ms=2.239 p95_ms=2.675 +compact after=true json_bytes=0: median_ms=2.192 p95_ms=3.041 +compact raw cursor at depth 9000: median_ms=1.936 p95_ms=3.116 +compact raw offset=9000: median_ms=6.860 p95_ms=8.327 +workflow V=10 E=9: median_ms=14.975 p95_ms=21.292 +12 enqueues: [("edge inserts", 12), ("other statements", 168), ("step inserts", 12)] +workflow V=10 E=30: median_ms=12.503 p95_ms=14.025 +12 enqueues: [("edge inserts", 12), ("other statements", 168), ("step inserts", 12)] +workflow V=100 E=99: median_ms=29.641 p95_ms=36.293 +12 enqueues: [("edge inserts", 12), ("other statements", 168), ("step inserts", 12)] +workflow V=100 E=390: median_ms=49.734 p95_ms=51.762 +12 enqueues: [("edge inserts", 24), ("other statements", 168), ("step inserts", 12)] +workflow V=600 E=599: median_ms=139.296 p95_ms=163.562 +12 enqueues: [("edge inserts", 36), ("other statements", 168), ("step inserts", 36)] +workflow V=600 E=2390: median_ms=342.758 p95_ms=353.946 +12 enqueues: [("edge inserts", 120), ("other statements", 168), ("step inserts", 36)] +100 direct jobs own_transaction=true: median_ms=874.670 p95_ms=894.247 +12 direct groups: total statements=4800 +100 direct jobs own_transaction=false: median_ms=80.602 p95_ms=91.991 +12 direct groups: total statements=2424 +connections=1, idle=0 +test measure_operational_costs ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 29.16s diff --git a/docs/measurements/operational-costs-2026-09-05/verification.txt b/docs/measurements/operational-costs-2026-09-05/verification.txt new file mode 100644 index 0000000..64f7b33 --- /dev/null +++ b/docs/measurements/operational-costs-2026-09-05/verification.txt @@ -0,0 +1,27 @@ +PostgreSQL: 18.6 (Debian 18.6-1.pgdg13+2), server_version_num=180006 + +cargo test --workspace --all-features + Exit 0; 909 passed, 4 ignored across 60 test/doc-test groups. + The final JSON-null SQL change was then covered by the focused rerun below. + +cargo test -p runledger-postgres --test workflow_batch_inserts --test workflow_step_defaults --test workflow_dependency_persistence + Exit 0; 12 passed (4 batch, 3 defaults, 5 dependency/concurrency). + +cargo test -p runledger-postgres --test migrations + Exit 0; 21 passed after moving additive-index setup into the rollout fixture. + +bash scripts/lint.sh + Exit 0; README checks, workspace/external-consumer formatting and Clippy, + SQLx migration-info parser checks, and workspace rustdoc with warnings denied. + +bash scripts/refresh-sqlx-cache.sh + Exit 0 against PostgreSQL 18.6 with current migrations; workspace build and + packaged SQLx-cache presence checks passed. Repeated after JSON-null SQL fix. + +Directory comparisons + Root .sqlx equals runledger-postgres/.sqlx and runledger-runtime/.sqlx. + Root migrations equal runledger-postgres/migrations and runledger-test-support/migrations. + +Manual timing diagnostic + Final ignored operational_costs test explicitly run and passed; see after.txt. + timings are not assertions in ordinary CI. diff --git a/docs/migration-identity/README.md b/docs/migration-identity/README.md new file mode 100644 index 0000000..3a2c764 --- /dev/null +++ b/docs/migration-identity/README.md @@ -0,0 +1,130 @@ +# Composing migration identity + +Runledger issue `runledger-runledger-simplification-audit-7cy` adds +`RUNLEDGER_POSTGRES_VERSION`, `migration_bundle()`, and the `MigrationBundle` +manifest. These APIs are available without feature flags, runtime setup, or a +database. The manifest borrows the same SQLx entries as `MIGRATOR`; all existing +migration and compatibility helpers remain exported. + +| Input | Meaning | Use | +| --- | --- | --- | +| `library_version()` | Compiled `runledger-postgres` package version | Diagnostics; conservative release identity | +| `migrations()` | All embedded up/down entries in documented order, including exact SQL and SQLx checksums | Vendoring and individual-entry inspection | +| `bundle_fingerprint()` | SHA-256 of ordered versions, descriptions, directions, checksums, and transaction modes | Exact bundle metadata identity | +| `pipeline_fingerprint()` | SHA-256 of library version and bundle fingerprint | One component of a host template/schema fingerprint | + +The v1 encoding is specified in the API rustdoc. Fingerprints are 32 raw bytes; +frame or label them when adding them to a host hash. The content identity omits +the library version; the pipeline identity conservatively changes on every +release, including releases without schema changes. It identifies released +helper behavior through the release version, not by hashing Rust source or the +dependency graph. Same-version helper patches, host configuration, dependency +overrides, and changes to application ordering need a host-owned revision input. + +## IdentityPro template adapter + +[identitypro.patch](identitypro.patch) is a narrow adaptation of local IdentityPro +commit `5a74dfa3bdbb90d04cc17f029282e71f0cd90788`, in +`crates/identitypro-db/src/migrations/bundle.rs`. Apply it only after selecting a +Runledger release or local dependency that includes this API; the original +published 0.12.0 dependency does not gain APIs when this repository changes. + +The patch removes the duplicated Runledger version literal and its raw migrator +loop, replacing both with the exported pipeline fingerprint. It preserves +IdentityPro's `identitypro-db-migration-pipeline-v16` domain, Runlimit helper +version and metadata, application migration metadata, and the existing +`FingerprintBuilder` adapter. Existing template identities change once because +the Runledger input changes representation. The host remains responsible for +template retention and cleanup. + +The host coordinator in `crates/identitypro-db/src/migrations.rs` still owns +interrupted-index recovery and the Runledger → Runlimit → IdentityPro execution +sequence. Revise the host pipeline domain when that behavior changes. An identity +API does not authorize reordering it or replacing its history validation. + +The packaged external-consumer test uses IdentityPro's actual +`postgres-test-harness` 0.2.0 `FingerprintBuilder`, with small host and Runlimit +migration fixtures. It checks repeatability and invalidation for changes to host +domain, host SQL, Runlimit SQL/version, owner ordering, and omission of Runledger. +This validates the composition interface, not IdentityPro's full migration +coordinator or live database behavior. + +## HOCR historical vendoring + +HOCR commit `4fb17323497dcbb98e09822cdab7b3b2e926f5b8` pins Runledger 0.5.0. +Its `apps/hocr-migrate/src/main.rs` test +`launch_migrations_vendor_the_exact_runledger_0_5_0_history` independently expects +five versions and compares vendored SQL and checksums. Keep that test while the +application is pinned to 0.5.0. + +The external-consumer fixture under +`smoke/external-consumer/tests/fixtures/hocr-runledger-0.5.0/` copies those five +exact SQL files from HOCR. Their bytes match the corresponding current Runledger +up migrations. The test uses the new manifest to verify this historical prefix, +with the original five-version expectation, and explicitly checks that the +current manifest has newer entries. It rejects changed SQL, SQL changed without +updating its checksum, checksum-only corruption, a missing vendored entry, and a +missing upstream entry. + +After an explicit dependency upgrade, a host may inspect `bundle.migrations()` +and select its historical versions to verify that prefix. It must separately +plan and validate every later migration and cutover. The current full bundle's +fingerprint is not the fingerprint of the old 0.5.0 bundle. The fixture checks +historical content preservation; it does not claim to upgrade HOCR or apply the +new migrations to its launch history. + +## Shared SQLx history and startup + +The [standalone example](../../runledger-postgres/examples/migration_identity.rs) +composes Runledger identity with a host pipeline domain and host SQLx migration +metadata, using length framing. Replace its empty demonstration migrator with +the application's existing migrator. Include other libraries at the positions +chosen by the host. Add non-SQL initialization inputs as needed. + +Host history rows and bundled metadata are different things: the manifest is +available content, while `_sqlx_migrations` records application of that content. +Keep using `migrate_after_idempotency_cutover` when Runledger should apply all +pending migrations, or `ensure_schema_compatible_after_idempotency_cutover` when +the host manages staged DDL. Both retain their existing shared-history behavior. +The host must allow other owners' versions in its own history validation while +checking its own checksums and append-only version set. Do not replace the host +coordinator with raw `MIGRATOR.run()` on a shared application pool. + +## Verification commands + +```sh +cargo test -p runledger-postgres --lib migration_identity --locked +cargo test -p runledger-postgres --doc --locked +cargo run -p runledger-postgres --example migration_identity --locked +cargo test --manifest-path smoke/external-consumer/Cargo.toml --test smoke migration_identity --locked +scripts/run-external-consumer-smoke.sh +scripts/lint.sh +``` + +The first four commands need no database. The packaged smoke script also runs +the existing database-backed consumer tests and requires PostgreSQL 18. Unit +tests pin independently computed SHA-256 vectors and an explicit complete +up/down version set, test individual manifest-field mutations, and establish +release invalidation with unchanged SQL. Database verification results belong +to the implementation completion record, not these command instructions. + +## Implementation verification, 2026-09-05 + +- Four focused unit tests passed, including the independent full-bundle content + snapshot `77005335e2e12fbcc96bd95c50d8a9c75b56293a0b91afee05b3a433ec96271c`. +- All ten `runledger-postgres` doctests passed, and the standalone composition + example ran without a database. +- Both local consumer identity tests passed. The packaged external-consumer + harness then passed all four tests against extracted `.crate` archives, + including both identity tests and the existing database embedding tests. + The server reported `18.6 (Debian 18.6-1.pgdg13+2)`, version number `180006`. +- `scripts/lint.sh` passed: README checks, formatting, migration-info parser + checks, workspace and external-consumer Clippy, and warning-free workspace + rustdoc. Focused Clippy was repeated after adding the full-bundle snapshot. +- The IdentityPro patch passed `git apply --check` against the source revision + cited above. The full downstream application was not built or migrated. + +No migration SQL, query SQL, SQLx cache, startup-helper implementation, or +compatibility-helper implementation changed. The runtime SHA-256 dependency +uses the 0.10.9 version already present through SQLx; the real IdentityPro +fingerprint builder is a dev dependency of the separate consumer test workspace. diff --git a/docs/migration-identity/identitypro.patch b/docs/migration-identity/identitypro.patch new file mode 100644 index 0000000..99029fb --- /dev/null +++ b/docs/migration-identity/identitypro.patch @@ -0,0 +1,40 @@ +--- a/crates/identitypro-db/src/migrations/bundle.rs ++++ b/crates/identitypro-db/src/migrations/bundle.rs +@@ -246,23 +246,20 @@ + "identitypro-db-migration-pipeline-v16"; + + #[cfg(feature = "test-support")] +-pub(crate) const RUNLEDGER_POSTGRES_MIGRATION_PIPELINE_VERSION: &str = "0.12.0"; +- +-#[cfg(feature = "test-support")] + pub(crate) const RUNLIMIT_POSTGRES_MIGRATION_PIPELINE_VERSION: &str = "0.3.0"; + + /// Returns the raw content identity inputs for the complete migration pipeline + /// used by shared test-database templates. + #[cfg(feature = "test-support")] + pub fn test_template_migration_fingerprint_inputs() -> Vec<(String, Vec)> { +- // These exact-pinned versions are load-bearing: they capture behavior +- // changes in the Runledger and Runlimit migration helpers beyond their raw +- // MIGRATOR data without making this package depend on workspace files. ++ // Library pipeline identities capture helper behavior beyond raw SQL. ++ // Runledger includes its own release and bundle identity; retain the host ++ // domain and Runlimit inputs for the rest of the migration pipeline. + let mut inputs = vec![ + ( +- "runledger-postgres:pipeline-version".to_owned(), +- RUNLEDGER_POSTGRES_MIGRATION_PIPELINE_VERSION +- .as_bytes() ++ "runledger-postgres:pipeline-fingerprint".to_owned(), ++ runledger_postgres::migration_bundle() ++ .pipeline_fingerprint() + .to_vec(), + ), + ( +@@ -272,7 +269,6 @@ + .to_vec(), + ), + ]; +- add_migrator_fingerprint_inputs(&mut inputs, "runledger", &runledger_postgres::MIGRATOR); + add_migrator_fingerprint_inputs(&mut inputs, "runlimit", &runlimit_postgres::MIGRATOR); + add_migrator_fingerprint_inputs(&mut inputs, "identitypro", &IDENTITYPRO_MIGRATOR); + inputs diff --git a/docs/migration-pipeline-identity-audit-2026-09-05.md b/docs/migration-pipeline-identity-audit-2026-09-05.md new file mode 100644 index 0000000..44a9dc0 --- /dev/null +++ b/docs/migration-pipeline-identity-audit-2026-09-05.md @@ -0,0 +1,120 @@ +# Migration pipeline identity audit + +Implementation follow-up: [migration identity and consumer composition](migration-identity/README.md). +The findings below record the source state at the audited commit. + +Audited `runledger-runledger-simplification-audit-7cy` on 2026-09-05 against +Runledger commit `d10037a647afc7f7ea73747c93be48ea7c4a5369` and the current local +consumer sources. The issue remains justified at P2 and is not implemented. +This report audits the requested feature; it does not change migration behavior. + +## Findings + +1. **P2: downstream callers still maintain migration-helper identity manually.** + The crate exports `MIGRATOR` and startup/compatibility functions, but no public + library version or pipeline identity + ([exports](../runledger-postgres/src/lib.rs#L334)). IdentityPro explicitly + duplicates `"0.12.0"` in + [bundle.rs](/home/aa/Documents/identitypro/crates/identitypro-db/src/migrations/bundle.rs:249) + and includes it in the test-template fingerprint. This is currently consistent + with its exact dependency pin, not evidence of an existing stale template. + The maintenance hazard is real: SQL checksums alone cannot identify changes + to Rust history filtering, legacy-row rejection, or constraint validation in + [the helper](../runledger-postgres/src/migrations.rs#L285). + Publish crate-owned identity and document when it changes. A library version + is sufficient for conservative invalidation across immutable releases; + same-version path dependencies or patched builds require an explicit policy + if they are also expected to invalidate templates. + +2. **P2: bundle identity and helper identity need distinct contracts.** + Raw metadata is already available through the documented, public + [`MIGRATOR`](../runledger-postgres/src/migrations.rs#L8). Do not describe this + as missing access to migration versions or checksums. What remains absent is + the promised documented manifest/identity contract and composition example. + IdentityPro consumes version, description, migration type, checksum, and + `no_tx`, in iteration order + ([adapter](/home/aa/Documents/identitypro/crates/identitypro-db/src/migrations/bundle.rs:282)). + HOCR's + [vendoring test](/home/aa/Documents/hocr-next/apps/hocr-migrate/src/main.rs:678) + requires the exact five up-migration versions in Runledger 0.5.0 and compares + both SQL and checksums. Its manifest must describe that pinned historical + bundle, not the latest Runledger schema. A combined opaque pipeline hash + alone cannot replace that verification. Retain access to individual entries + and SQL through `MIGRATOR`; specify ordering, direction, checksum encoding, + transaction metadata, and identity-format version for any new interface. + +3. **P2: consumer composition remains unproven.** + The [existing runbook](../README.md#applying-or-validating-the-schema) explains + supported startup helpers and migration inspection, but provides no example + composing the proposed identity with host migration inputs. IdentityPro + combines Runledger, Runlimit, and application migrators under its own + `identitypro-db-migration-pipeline-v16` domain, then uses those inputs to + [select a template](/home/aa/Documents/identitypro/crates/identitypro-test-support/src/db.rs:226). + Its [coordinator](/home/aa/Documents/identitypro/crates/identitypro-db/src/migrations.rs:100) + also owns recovery and library/application execution ordering. An exported + Runledger identity must replace only the duplicated Runledger input, while + retaining the host domain and other owners' inputs. Identity equality is a + cache identity, not proof of a live database's compatibility. Keep the + existing compatibility checker and host ordering decisions. + +## Requirement assessment + +| Issue requirement | Current evidence | Assessment | +| --- | --- | --- | +| Export library/migration-pipeline identity | No such item in crate exports or migration implementation; IdentityPro duplicates a version string | Missing | +| Bundle metadata usable in downstream fingerprints | `MIGRATOR` already supplies the fields used by IdentityPro and HOCR | Existing foundation; documented identity/manifest contract still needed | +| Document composition with host SQLx history | Startup/compatibility runbook exists; no identity composition example | Incomplete | +| Validate IdentityPro integration | Manual version and metadata adapter inspected; no replacement API or consumer fixture | Demand confirmed; implementation validation missing | +| Validate HOCR vendored-bundle use | Exact 0.5.0 version, SQL, and checksum assertions inspected | Demand confirmed; implementation validation missing | +| Retain migration and compatibility helpers | Explicit cutover helpers, `MIGRATOR`, and deprecated aliases remain exported | Satisfied by current source; preserve during implementation | +| Keep application ordering and cutovers application-owned | Existing helpers document staged external DDL; consumers own orchestration | Preserve this boundary | + +## Acceptance checks for implementation + +These are proposed checks to make the issue executable, not claims of completed +validation or mandatory API names. + +1. Provide a public identity available to ordinary downstream crate consumers, + without runtime setup, database access, or a source-checkout dependency. + Define library release identity, helper behavior identity, and bundle content + identity explicitly. If release version represents helper identity, state its + conservative invalidation and same-version development limitations. +2. Document the manifest's complete inputs and deterministic representation. + Cover up/down entries and transaction mode. If exposing a digest, specify a + versioned, unambiguous encoding with stable field boundaries. Verify changes + to relevant inputs change identity, while identical input reproduces it. + Test the helper identity independently of SQL changes. Do not treat a digest + derived from the same entries on both sides as proof of bundle completeness. +3. Compile an IdentityPro-shaped external consumer that replaces the local + Runledger version literal, preserves host/Runlimit inputs and the host + pipeline domain, and feeds the existing fingerprint builder. Demonstrate + that host ordering/domain changes still invalidate the composed identity. +4. Exercise an HOCR-shaped vendored-bundle fixture with an independently pinned + expected version set and SQL/checksum comparisons. Include a changed SQL + entry and a missing expected entry as failures. Preserve the historical + 0.5.0 check; adopting a new metadata API requires an explicit dependency + upgrade or a separately described historical adapter, not silently expecting + the current bundle to equal 0.5.0. +5. Check the packaged crate through the external-consumer harness so metadata + does not accidentally depend on workspace files. Keep existing helpers and + aliases usable. Do not add application orchestration to the metadata API. +6. If migration execution or compatibility behavior changes, run the relevant + existing migration tests on PostgreSQL 18 and record the exact server + version. Metadata-only behavior can be tested without a database. Do not + refresh SQLx caches merely for adding identity metadata. + +## Evidence and limits + +Inspected local IdentityPro commit +`5a74dfa3bdbb90d04cc17f029282e71f0cd90788` (dependency `=0.12.0`) and HOCR commit +`4fb17323497dcbb98e09822cdab7b3b2e926f5b8` (dependency `=0.5.0`). These observations +describe local source, not deployed or registry state. + +Existing Runledger tests cover fresh migration application, unrelated shared +SQLx history, conflicting versions, compatibility fences, and equality of +vendored migration copies +([migration tests](../runledger-postgres/tests/migrations.rs#L1811)). They do not +establish the absent identity contract. Test bodies and assertions were +inspected; no Cargo tests or database experiments were run for this source +audit. No PostgreSQL-version-dependent behavior was experimentally asserted. +The proposed API and consumer adaptations remain future implementation work. diff --git a/docs/operational-costs-2026-09-05.md b/docs/operational-costs-2026-09-05.md new file mode 100644 index 0000000..153451c --- /dev/null +++ b/docs/operational-costs-2026-09-05.md @@ -0,0 +1,278 @@ +# Operational reads and workflow enqueue costs + +This records the API-006 measurement/design work tracked by +`runledger-runledger-simplification-audit-kqm`. The implementation adds scoped +compact pages, exact job-type filtering, batch status lookup, and set-based +initial/append graph inserts. Direct-job `enqueue_many_tx` remains a design +below, as requested by the audit's investigation scope. No new wrapper merely +hides the existing per-job loop, and no shared result listener is introduced. + +## Measurements + +Server: **PostgreSQL 18.6 (Debian 18.6-1.pgdg13+2)**, +`server_version_num=180006`, official `postgres:18` container. All current +migrations were applied. Client: Rust debug test profile, SQLx 0.8.6, local TCP, +one pool connection, sequential requests, default PostgreSQL durability settings, +`pg_stat_statements` enabled. This shared development machine was not isolated +from other workloads. These are synthetic latency measurements, not production +throughput or capacity estimates. + +Baseline persistence source was commit `4844fe3`, before the new indexes and +batching. The diagnostic added only fixture/measurement code. Each read has +one warmup and 31 measured samples; each write case has one warmup and 11 +measured samples, including transaction commit. Fixture construction, workflow +builders, and cursor discovery are outside the measured regions. Workflow +validation, snapshot creation, graph writes, and root release are inside them. +The reported p95 is the sorted sample at `floor(n * .95)`; for 11 samples this +is the maximum, not a statistically robust tail estimate. + +Raw evidence: [baseline](measurements/operational-costs-2026-09-05/baseline.txt), +[first after run](measurements/operational-costs-2026-09-05/first-after.txt), +[second after run](measurements/operational-costs-2026-09-05/second-after.txt), +[final after run](measurements/operational-costs-2026-09-05/after.txt), and +[actual prepared query plans](measurements/operational-costs-2026-09-05/plans.txt). +The first after run includes a 737.573 ms small-graph outlier; it is retained, +not discarded. The final run includes the direct-job experiment, JSON-null preservation in the +shipped INSERT, and a prepared-statement selector that cannot accidentally select +its own inspection query. Its 100/390 graph sample also has a 275.887 ms tail outlier. + +### Reads + +The fixture has 10,000 global jobs, each with a 4,096-character deterministic +varied string in each of payload, checkpoint, and output. Pages have 100 jobs. +No status/type filter is applied for the timing comparison. + +| API/path | Baseline median / p95 ms | Final median / p95 ms | +| --- | ---: | ---: | +| Full page, offset 0 | 20.422 / 23.199 | 14.819 / 18.150 | +| Full page, offset 9,000 | 20.490 / 22.162 | 18.316 / 21.799 | +| Compact first page | — | 2.250 / 2.809 | +| Compact cursor at depth 9,000 | — | 1.912 / 2.758 | +| Compact raw cursor at depth 9,000 | — | 1.481 / 1.796 | +| Compact raw offset 9,000 | — | 4.679 / 5.539 | + +The raw comparison derives its SQL from the actual public prepared cursor +statement, replaces only pagination, fetches both into the same SQLx row type, +and asserts identical IDs. This separates cursor access from projection/typed +decode differences. The final compact first page is about 6.6 times faster +than the final full first page on this fixture; the raw deep cursor is about +3.2 times faster than its offset counterpart. Wider/narrower real payloads, +selective filters, cache state, and network distance change these ratios. + +Full pages materialize **1,232,100 JSON bytes** per page, calculated by compact +serialization of the three returned JSON fields outside the timed region. +Compact pages read **zero JSON fields/bytes**. This is not a PostgreSQL protocol +byte count or an allocator profile: both paths still allocate row vectors and +identifier strings. No claim about total heap allocation counts is made. +Each public page and nonempty batch status lookup executes one SELECT and uses +one query connection; an empty status lookup executes none. The diagnostic pool +remained at one connection. It does not measure concurrent connection pressure. + +The plan regression uses 30,000 rows spread across global and two tenant scopes. +It EXPLAINs the public prepared statement under both `force_custom_plan` and +`force_generic_plan`. All three scopes constrain an index with the cursor tuple; +the recorded plans use 3–5 shared buffer hits for a 20-row page near the end. +Optional exact type and status predicates may remain residual filters, especially +with generic plans. No claim is made that every filter distribution has constant +scan cost; specialized covering/filter indexes need workload evidence. + +### Graph writes + +Graphs have one root job. Each later step depends on the previous one or up to +four preceding steps, exercising chains and denser DAGs. Every request is new +and unkeyed; each root retains its ordinary queue insertion and ENQUEUED event. + +| Steps / edges | Baseline median / p95 ms | Final median / p95 ms | Graph INSERT executions before → after | +| --- | ---: | ---: | ---: | +| 10 / 9 | 13.449 / 16.268 | 13.083 / 57.564 | 19 → 2 | +| 10 / 30 | 17.232 / 20.165 | 13.475 / 33.997 | 40 → 2 | +| 100 / 99 | 70.956 / 80.737 | 23.513 / 51.518 | 199 → 2 | +| 100 / 390 | 142.869 / 153.638 | 42.835 / 275.887 | 490 → 3 | +| 600 / 599 | 378.741 / 409.424 | 147.887 / 203.130 | 1,199 → 6 | +| 600 / 2,390 | 816.635 / 859.004 | 335.263 / 453.667 | 2,990 → 13 | + +Statement counts are measured with `pg_stat_statements` over all 12 executions, +then divided by 12. They count awaited SQL executions, not TCP packets or +statements executed inside triggers. Each case retains 14 other top-level +statements per enqueue, including transaction control. The graph portion changes +from `V + E` to `ceil(V / 256) + ceil(E / 256)`. Empty edge sets execute no +dependency INSERT. Small graph improvements are inconsistent across runs: the second after run +had a 14.975 ms median for 10/9, slower than the 13.449 ms baseline. The largest graph improved about 2.4 times despite far fewer +statements, because graph validation, JSON serialization, constraints, indexes, +triggers, snapshot work, and root release remain. + +The same batched writers serve append, whose existing dependency-counter and +mutation-outcome logic is preserved. Append correctness is tested across chunk +boundaries; the timing table measures initial enqueue only. Fanout graphs with +many ready job roots still pay per-root queue/audit costs. The new queue indexes +also add storage and maintenance work to queue writes; this experiment does +not isolate that overhead or measure index-build duration on a production table. + +### Independent direct jobs + +The final diagnostic also enqueues 100 new, unkeyed jobs using existing APIs: + +| Transaction ownership | Median / p95 ms | SQL executions per 100 jobs | +| --- | ---: | ---: | +| Each `enqueue_job_with_outcome` owns its transaction | 895.000 / 1124.848 | 400 | +| 100 `enqueue_job_with_outcome_tx` calls, one caller transaction | 77.239 / 94.182 | 202 | + +This isolates transaction amortization on this machine. It changes the failure +boundary: individually committed jobs can survive a later failure, whereas the +caller must roll back the whole group on error for atomic submission. Both +still execute one queue INSERT and one event INSERT per new job. It is evidence +for deliberate transaction composition today, not a measured set-based direct +enqueue implementation. Keyed contention, disabled definitions, duplicates, +payload sizes, and application-side row locking were not timed in this case. + +## Shipped contracts and compatibility + +`list_job_summaries(pool, &JobSummaryFilter)` requires an application-authorized +`JobReadScope`, optional status, optional exact case-sensitive `JobType`, limit +1–1,000, and optional exclusive `(created_at, id)` cursor. Wildcard characters +in a job type are literal. The compact record includes identity, scope, status, +priority, run/attempt counters, retry time, stage/progress, and timestamps, without +payload/checkpoint/output or free-form errors. Callers retain the detail API. +Cursor timestamps must retain PostgreSQL microsecond precision. Cursors need no +live anchor row; deleting the previous page's last row does not invalidate them. +Keep scope/filters fixed, and treat pages as changing observations: a status +transition can enter/leave a filter, and new rows ahead of a cursor are excluded. +Application ownership joins and authorization remain application-owned. + +`get_job_statuses_with_scope(pool, scope, ids)` accepts at most 1,000 IDs, including +duplicates in that bound. Empty input performs no query. Each visible ID appears +once, in ascending ID order; absent and out-of-scope IDs are indistinguishably +omitted. Status/run/attempt observations do not grant lease or recovery authority. +Legacy offset/substring/detail APIs keep their contracts. + +Graph writes serialize at most 256 borrowed step records or dependency records +per statement into `jsonb_to_recordset`, with SQLx-checked columns and bound JSON. +The row bound limits scratch records and statement size growth with graph count; +it is **not a hard byte limit on caller payloads**. Existing callers can still +submit large individual payloads. IDs remain database-generated and are mapped +by step key; no dependence on RETURNING row order is introduced. Append results +remain in input order. The entire graph, root jobs, snapshots, mutation record, +and events share the existing transaction. Owned APIs commit all or roll back +on error; `_tx` callers retain transaction ownership and must roll back on error. +There are no per-chunk commits, partial success outcomes, or concurrent tasks +sharing a transaction. Initial keyed retries still reuse the original run only +after canonical snapshot equality, and append still reports `Appended` or +`AlreadyApplied`. Active-key outcomes remain unchanged. + +Definition validation/locks, run coordination locks, append step locks, +dependency orientation/release modes, per-step tenant/default/override policies, +continuation/resource fields, immutable snapshots, ordinary audit writes, lease +fences, and terminal propagation use the existing paths. Only graph INSERT +execution is grouped. No claim/recovery/fencing SQL is changed. + +Migration `202609050001_job_summary_pagination` adds +`(organization_id, created_at DESC, id DESC)` and `(created_at DESC, id DESC)` +indexes. It is additive and omitted from `runledger_migration_history` so older +filtered startup helpers can coexist; SQLx history still tracks/checksums it. +Apply it before deploying this build: the current startup guard requires it, +including during an expand-only workflow/job-link rollout. These are ordinary +transactional CREATE INDEX statements and block queue writes until commit; +schedule an appropriate deployment window. The down migration drops only these +indexes. The independently calculated manifest fingerprint is updated, and root, +packaged migrations, and all three SQLx caches are synchronized on PostgreSQL 18. + +## Bounded direct-job batch design + +The audit asked to investigate `enqueue_many_tx`, not commit to a new concurrency +protocol. The graph improvement above is shipped. The following is the concrete +contract/design for a subsequent direct-job API; it is not exported in this change. + +1. Accept at most 256 indexed entries and 1 MiB of encoded canonical requests + per call, validating the full input before any writes. Include optional + execution-resource keys in the canonical request using the existing snapshot + format. Empty input succeeds without SQL. Bound violations are validation + errors; do not silently split one atomic request into separately committed work. +2. Require READ COMMITTED. The owned wrapper commits all entries together. + The `_tx` wrapper creates a savepoint, releases it on complete success, and + rolls back to it on any per-item validation/conflict/SQL failure. It returns + the first failing input index with a classified error, not a partially + successful vector. Transaction/rollback failure makes the outer transaction + unusable and must be reported. This is an explicit stronger atomic-call + guarantee than merely looping the current `_tx` helper. +3. Return `Vec` in input order only on whole-batch success. + Repeated identical keyed requests share one row: first occurrence is Inserted + if new and later occurrences Existing. Different snapshots for the same + `(scope, job_type, key)` reject the whole call before writes. Every unkeyed + entry creates its own job. Existing means the observed status/run number, + not a payload refresh or requeue. +4. Deduplicate keys before insertion. Acquire definition SHARE locks in job-type + order, insert in a consistent `(global/tenant, organization UUID, job type, + key, input ordinal)` order, and acquire existing job mutation-ready locks in + a consistent order. Use the existing global/tenant partial unique indexes + as the authority shared with single-job writers. Do not invent an advisory + key scheme respected only by batch callers. This reduces batch-to-batch + inversions; it cannot eliminate deadlocks from locks already held by application + code or differently ordered single-item loops. Retry the whole transaction + on a classified deadlock, never an unknown partial subset. +5. Use bounded set-based queue inserts and event inserts from returned new IDs. + After `ON CONFLICT DO NOTHING`, resolve conflicts in a **second statement** + so READ COMMITTED can see a row committed while the unique insert waited. + A same-statement INSERT/SELECT CTE fallback can miss that row. Compare immutable + `enqueue_request`, never mutable payload/checkpoint/live options. Preserve the + current ability to return an identical existing keyed job even if its definition + was subsequently disabled; missing/disabled definitions only reject entries + that cannot resolve an existing request. Missing legacy snapshots still fail. +6. Emit exactly one ordinary ENQUEUED event for each newly inserted job and none + for retries. Preserve execution-resource fields and normal downstream claiming; + outcomes must never bypass lease identity or typed compare-and-requeue fences. + +Before implementing/exporting this design, benchmark it against the measured +202-statement caller-transaction loop and test mixed new/existing/duplicate +entries, opposite-order concurrent batches, single/batch races, a conflict in +the final item, commit failure, application locks, disabled definitions, and +rollback retaining unrelated caller writes. The source behavior requiring these +rules is in `queue/enqueue.rs` (`enqueue_job_with_existing_lock_tx_inner`, +`resolve_existing_idempotent_job_tx`, and `load_existing_idempotent_job_tx`). +OneSales/IdentityPro loop sites in the [source audit](api-audit-2026-09-05.md) +are candidate consumers; no measured production enqueue rate is inferred from them. + +Shared workflow-result listeners remain secondary. The source audit found no +sampled production wait consumer; this work neither changes LISTEN/poll fallback +behavior nor claims reduced waiter connection use. + +## Reproduction and validation + +Start an owned diagnostic server and pass its URL to the ignored diagnostic. +The test helper creates, migrates, and drops an isolated database; the extension +and statistics reset require the diagnostic server's administrator role. + +```sh +docker run -d --name runledger-costs-pg18 \ + -e POSTGRES_USER=runledger -e POSTGRES_PASSWORD=runledger \ + -e POSTGRES_DB=postgres -p 127.0.0.1::5432 \ + postgres:18 -c shared_preload_libraries=pg_stat_statements +docker port runledger-costs-pg18 5432 +# Substitute the reported local port below. +RUNLEDGER_TEST_ADMIN_DATABASE_URL=postgres://runledger:runledger@127.0.0.1:PORT/postgres \ + cargo test -p runledger-postgres --test operational_costs -- --ignored --nocapture +docker rm -f runledger-costs-pg18 +``` + +The baseline can be reproduced in a detached checkout of `4844fe3`: copy the +current `operational_costs.rs` test into that checkout and remove the +`compact_reads` and `direct_jobs` function definitions and their calls. The +remaining read/graph fixture, warmups, sample counts, and timers are the baseline +harness. Use the same server/profile and avoid concurrent test workloads during +timing. SQLx preparation uses the ordinary refresh script against a separate +PostgreSQL 18 database with the current root migrations applied. + +Behavior tests cover exact scopes/types/statuses, literal wildcards, tied +timestamps, deleted anchors, concurrent newer insertion, input bounds, duplicate +and missing IDs, and custom/generic cursor plans. Graph tests cross 256-row +step/edge boundaries, preserve JSON null payloads through recordset conversion, +assert edge orientation/fields/audit/snapshot reuse and +append outcomes/order, and inject later-chunk failures to prove owned rollback. +Existing default/nullability, dependency propagation, concurrent completion, +active-key, recovery, resource, and idempotency tests remain part of the workspace +regression suite. Timing is a manual diagnostic, not a flaky CI speed assertion. + +Validation completed: the workspace suite passed 909 tests (four intentionally +ignored). The final JSON-null SQL change passed 12 focused workflow tests; the +final upgrade fixture passed all 21 migration tests. `scripts/lint.sh` and the +PostgreSQL 18 SQLx refresh/package-cache checks passed. See the +[verification record](measurements/operational-costs-2026-09-05/verification.txt). diff --git a/docs/review-fixes-2026-09-05.md b/docs/review-fixes-2026-09-05.md new file mode 100644 index 0000000..5f0c8e4 --- /dev/null +++ b/docs/review-fixes-2026-09-05.md @@ -0,0 +1,207 @@ +# Branch review fixes and scope decisions + +This follow-up addresses the review of `3ed1404..b724223`. It covers scoped +list reads, handler progress persistence, typed worker integration, and the +producer prelude. Rust 1.88/edition 2024 and PostgreSQL 18 remain the baseline. +Public signatures, persisted formats, lease fencing, and legacy scope meanings +remain compatible. Generated SQLx metadata is refreshed from current migrations. + +## Open-question research + +The findings below describe the original review scope. The metrics and payload +coverage gaps were subsequently resolved by the +[60g follow-up](#exact-scope-metrics-and-payload-follow-up-60g). + +- API-004 in `api-audit-2026-09-05.md` explicitly proposed scoped job, event, + log, and intent inspection. The implemented surface matches that list. + `get_job_metrics` and `get_job_continuation_metrics` apply optional organization + filtering to rollup views; `get_job_enqueue_intent_metrics` filters three + lifecycle populations. For all three, `None` intentionally aggregates global + and tenant rows. The TUI dashboard uses that legacy aggregation contract. + Exact-global metrics were an additive coverage gap; the follow-up preserves + existing callers' results. +- Both legacy payload helpers require a tenant UUID. The queue has separate tenant + and global idempotency indexes; the same key can legally exist in multiple + scopes. An admin wildcard on a single-result key lookup would be ambiguous. + The new scoped lookups use an exact scope contract without an admin variant. + The latest-payload helper's `run_id` is a JSON field, not a globally unique + queue identity. Legacy signatures and behavior stay intact. Completed follow-up: + `runledger-runledger-simplification-audit-60g`. +- The producer outcome helper was exported through `jobs` but omitted from + `prelude` when introduced in `b724223`. There is no documented exclusion, + and the transaction counterpart is already in the prelude. Exporting it is + additive; both the producer integration test and packaged consumer now use it. + +## Design diagnosis and changes + +### Scope predicates: a translation mistake exposed by weak performance coverage + +The `JobReadScope` enum is an appropriate closed set. Translating it into a +boolean plus nullable UUID hid the distinct SQL access paths in +`jobs/admin/read.rs` and `jobs/queue/intents.rs`. `IS NOT DISTINCT FROM` gives +correct visibility but did not constrain the organization index in the +PostgreSQL 18 diagnostic. Merely replacing it with equality inside a wildcard +OR would still depend on custom-plan simplification. + +The fix selects one statement per enum variant: equality for a tenant, +`IS NULL` for global scope, and unrestricted admin filtering. A small private +`scoped_list` macro owns that choice while preserving SQLx's literal-SQL checks +and keeping projections/filters at their existing call sites. It is restricted +to these list reads; point lookups already narrow by unique identity. +The first bind position stays reserved for the organization in all variants. + +Fowler move: Remove Flag Argument and Extract Function, expressed as a private +macro because SQLx needs literal SQL. Impact medium; confidence high; +scope internal/cross-module; risk medium (plan and visibility behavior). +The DTOs, public wrappers, and handler traits are not refactoring targets. + +PostgreSQL documents B-tree support for equality and `IS NULL`: +. + +### Progress: invariant ownership and discarded domain errors + +The core execution validator checked only values present in a request. +The database correctly merged omissions with durable values, but its CHECK +failure was reduced to a generic retryable persistence error. A validated +request alone cannot solve this: another progress writer can change the row. + +The shared `validate_job_progress` function now owns the numerical rule. +Completion construction, execution prevalidation, completion persistence, and +ordinary progress use it. The existing live-lease lock helper was renamed for +reuse, then ordinary progress gained locked-state validation before mutation. +The SQL write still rechecks lease expiry. Original partial values remain in +audit events, and checkpoint writes remain atomic with progress. + +The private query-error classification retains `JobProgressValidationError`; +an additive accessor lets the runtime return terminal `InvalidProgress` without +parsing strings. Existing public exhaustive error enums were not extended. +Actual connection/commit failures remain retryable. Rejected validation awaits +rollback so the lease row is released before returning. Existing completion +error codes and diagnostic wording are retained. + +Fowler moves: Extract Function, Move Function, and preserve a typed recoverable +error. Impact medium; confidence high; scope cross-crate; risk medium +(transaction ordering and error policy). This is a bug behavior change after +the behavior-preserving validator extraction and lock-helper rename. + +## Verification sequence + +1. Baseline formatting, core tests/doctests, and existing scope tests passed. + The heuristic scan covered 35 changed Rust files; its 143 candidates were + treated as hints, not additional findings or cleanup work. +2. Added a worker regression before changing behavior. It failed with `Pending` + instead of `DeadLettered`; typed continuation and malformed-payload tests + already passed. Validator extraction passed core tests; lock-helper rename + passed the PostgreSQL compile check. +3. Added an actual-prepared-query plan test against production indexes and + 20,000 interleaved rows. Before the fix, the job query rejected 19,980 rows + in a sequential scan (1.833 ms, 385 shared buffers). The test covers tenant + and global scopes under custom and generic plans, including legitimate + global-only partial indexes. Timing is diagnostic, not a flaky threshold. +4. Added competing partial-update validation, rejected-write atomicity/audit + assertions, terminal worker classification, typed continuation resume, raw + malformed-payload cleanup, legacy metrics aggregation, tenant-local payload + lookup, and packaged prelude usage. +5. All four downstream patches passed `git apply --cached --check` against their + documented base commits using temporary indexes. This checks applicability; + downstream application suites are not rerun by this check. + +Database diagnostics use PostgreSQL `18.6 (Debian 18.6-1.pgdg13+2)` +(`server_version_num=180006`). + +Final checks: + +- `cargo test --workspace`: 879 passed, three explicitly ignored entries, + using `RUNLEDGER_TEST_ADMIN_DATABASE_URL` for an isolated PostgreSQL 18.6 + container. Container-lifecycle parent tests return early under that setting. + The ignored entries are a manual claim throughput benchmark, a slow promoter + transaction-timeout test, and the lifecycle child-process entrypoint. +- `scripts/lint.sh`: passed formatting, workspace/all-target/all-feature Clippy, + packaged-consumer Clippy, migration-info checks, and warning-free rustdoc. +- `scripts/refresh-sqlx-cache.sh`: passed on PostgreSQL 18.6 with all migrations; + all three cache directories contain the same 148 query records. Offline + workspace compilation and packaged metadata checks passed. +- `scripts/check-package-licenses.sh` and `cargo deny check`: passed. +- `scripts/run-external-consumer-smoke.sh`: passed from packaged crates against + PostgreSQL 18.6, including the prelude outcome helper. +- Standalone container-lifecycle tests, rerun without the external database + override: two passed and two failed before PostgreSQL readiness (40 exhausted + connection attempts). The failed cases cover normal exit and forced + termination; missing/stalled reaper CLI cases pass. The test-support source, + migrations, lockfile, and toolchain are unchanged by this patch. This remains + an infrastructure-test limitation; no cause or fix is claimed here. +- The four downstream patch applicability checks passed. Downstream application + suites and the CI semver action were not rerun (the local semver CLI is absent). + +The tenant job-list plan in the fixed 20,000-row fixture used 22 shared buffers +and 0.073 ms versus the baseline's 385 buffers and 1.833 ms. Both list APIs +passed tenant/global custom/generic plan checks. These are fixture diagnostics, +not production latency guarantees. + +## Remaining risks and deliberately separate work + +Progress now performs an additional locked read in its existing transaction. +It reuses existing timeout bounds and holds the lock through validation/write, +which prevents concurrent partial updates from validating against stale state. +Ordinary-progress throughput was not benchmarked; no throughput claim is made. +There are no migration, serialization, authorization, unsafe, dependency, or +runtime-dispatch changes. Exact-global metrics and payload APIs were implemented +in the follow-up below while preserving legacy optional arguments. + +## Exact-scope metrics and payload follow-up (`60g`) + +`runledger-runledger-simplification-audit-60g` adds +`get_job_metrics_with_scope`, `get_job_continuation_metrics_with_scope`, and +`get_job_enqueue_intent_metrics_with_scope`. Metrics use `JobReadScope`; intent +metrics take `JobEnqueueIntentReadMetricsFilter`. Legacy calls still interpret +`None` or an omitted organization filter as Admin, and a tenant UUID as that +exact tenant. Scope predicates remain in the job/continuation LEFT JOINs so +registered definitions survive with zero counts. Existing sums, maxima, and +averages of per-scope duration percentiles are unchanged. + +`get_job_payload_by_idempotency_key_with_scope` and +`get_latest_job_payload_for_run_with_scope` use the existing exact `JobScope`. +Neither accepts an Admin wildcard. Legacy payload helpers remain tenant-only, +and absent matches still return `None`. Latest payload ordering remains +`created_at DESC, id DESC`. All new APIs and the filter are exported through +`jobs` and `prelude`; README and API docs describe application authorization +responsibility and compatibility. + +Acceptance evidence: + +- `tests/legacy_read_contracts.rs`: global, two tenants, Admin, and an unrelated + tenant; all nine job counters and both duration metrics; zero-count + definitions; exact type filtering and ordering; legacy equivalence. Payload + fixtures repeat keys and JSON run IDs across global and two tenants, with + multiple rows per scope, timestamp ties, a larger but older UUID, a newer + unrelated row, wrong-type rows, missing matches, and absent/present nil UUIDs. +- `tests/job_continuation.rs`: distinct continuation populations in global and + two tenants, Admin sums/maxima, an unrelated tenant, empty definitions, and + equivalence with legacy aggregation and tenant calls. The existing plan + diagnostic now uses the new scope predicate. +- `tests/job_enqueue_intents.rs`: all three lifecycle populations in each + scope; pending-only age/retry/max-attempt values despite older creation times + and larger attempt counts on terminal rows; recent and expired terminal + windows; omission of old-only history; terminal-only groups; exact type + filters, stable pagination, invalid pagination, and legacy equivalence. +- `smoke/external-consumer/tests/smoke.rs`: imports and calls all five APIs and + the new filter from the prelude in a consumer built from packaged crates. + +Final validation used an isolated `postgres:18` container reporting +`18.6 (Debian 18.6-1.pgdg13+2)` (`server_version_num=180006`) with all 16 current +migrations applied. `RUNLEDGER_TEST_ADMIN_DATABASE_URL` selected this server for +regressions and the packaged consumer. + +- `cargo test -p runledger-postgres --test legacy_read_contracts --test + job_continuation --test job_enqueue_intents`: 46 passed, one existing slow + promotion transaction-timeout test explicitly ignored. +- `scripts/lint.sh`: passed workspace/all-target/all-feature Clippy, consumer + Clippy, formatting, README and migration-info checks, and warning-free rustdoc. +- `scripts/refresh-sqlx-cache.sh`: passed against the PostgreSQL 18.6 server with + current migrations, including offline workspace compilation and package + metadata checks. The three SQLx directories contain identical sets of 148 + query records; five query records were replaced in each directory. +- `scripts/run-external-consumer-smoke.sh`: both packaged-consumer tests passed. + +This follow-up adds read APIs only; it does not change migrations, persisted +formats, mutation permissions, or the legacy callers' scope meanings. diff --git a/docs/review-hot-path-fixes-2026-09-05.md b/docs/review-hot-path-fixes-2026-09-05.md new file mode 100644 index 0000000..3544d8c --- /dev/null +++ b/docs/review-hot-path-fixes-2026-09-05.md @@ -0,0 +1,177 @@ +# Review follow-up: scoped reads, progress, and deadline policy + +This follow-up addresses the comprehensive review of `3ed1404..c30fffe`. +Database diagnostics use PostgreSQL **18.6 (Debian 18.6-1.pgdg13+2)**, +`server_version_num=180006`, with the current migrations. Public signatures, +persisted formats, error categories, and timeout precedence are preserved. + +## Research before implementation + +The handler cutoff is intentional. `JobExecution::deadline` exposes the +worker's authoritative monotonic deadline, and the worker already explicitly +checks it after polling the handler. Tokio 1.53.1, the locked dependency, +[randomizes `select!` branch polling](https://docs.rs/tokio/1.53.1/tokio/macro.select.html#fairness). +Removing the result-side check would let polling order decide whether an +overdue result wins. Moreover, Tokio +[`timeout_at`](https://docs.rs/tokio/1.53.1/tokio/time/fn.timeout_at.html) +allows an immediately ready future to succeed irrespective of the deadline; +it is not by itself a strict result-acceptance policy. + +Keep the existing rule: the worker must observe a result strictly before the +deadline. An equal or later result is a timeout, including success and +continuation. Database completion persistence occurs afterward. Committed +checkpoints and external effects cannot be undone by discarding a late result. +The public rustdoc and README now state this rule. + +The migration question also has a concrete answer: +`ensure_schema_compatible_after_idempotency_cutover` checks every required +bundled up migration against `_sqlx_migrations`. The custom +`runledger_migration_history` fence is a separate compatibility mechanism. +The pagination indexes are required by current startup even though they do not +advance that fence. A regression now proves rejection before application and +acceptance afterward, without a custom fence entry. + +## Diagnosis and design + +### Preserve scope variants when constructing SQL + +The scope enum is sound. The problem was collapsing it to a nullable UUID and +an `OR` expression before PostgreSQL planned the query. A +[generic prepared plan](https://www.postgresql.org/docs/18/sql-prepare.html) +cannot simplify it using one tenant's bound value, and +[partial-index applicability](https://www.postgresql.org/docs/18/indexes-partial.html) +depends on what the planner can prove about the predicate. + +The new exact-scope lookup macro selects equality or `IS NULL` statements. +It lives beside the existing list-query mechanism in `jobs/scoped_read.rs`, +retains SQLx literal-SQL checking, and accepts only `JobScope`. An admin +wildcard cannot enter a payload lookup whose key is only scope-local. + +Fowler move: Remove Flag Argument / preserve the closed enum at the SQL +boundary. Impact medium; confidence high; scope internal; risk medium because +visibility and plans must both remain correct. No new indexes are needed for +the reproduced generic-plan regression. + +### Separate lock acquisition from completion timeout policy + +The shared live-lease helper bundled two responsibilities: acquiring the row +and temporarily applying a completion transaction's lock-timeout policy. +Progress had already installed its whole-transaction caps, so reusing that +wrapper added redundant cap/restore statements. Its following update also +contained another locking CTE. + +Extracting the row acquisition lets completion retain its existing wrapper +while progress uses the caps it already owns. Progress locks once, validates +the current row through the shared Rust validator, then updates and audits in +the same transaction. The update still rechecks wall-clock lease expiry. + +The locked read remains deliberate. Replacing it with optimistic validation +would reintroduce a race between partial updates. Duplicating the numerical +validation in SQL or parsing PostgreSQL CHECK-error text would add another +source of truth or a fragile error protocol. The existing typed validation +errors and awaited rejection rollback remain intact. + +Fowler move: Extract Function / separate acquisition from policy. Impact +medium; confidence high; scope internal; risk medium because cancellation, +lease fencing, and transaction ordering matter. The initial extraction passed +the existing progress race and rollback tests before progress adopted it. + +### Make the rollout requirement visible + +The ordinary index builds are an existing, documented deployment choice. +Changing an already checked-in migration or introducing a separate concurrent +DDL protocol would create additional compatibility work. The concrete omission +was in the Unreleased upgrade notes: they now identify the migration, its +write-lock window, and the difference between SQLx history and the custom fence. + +## Measurements and regression coverage + +The payload fixture uses 20,000 rows with repeated keys/run IDs across scopes, +twenty busy tenants, and old global rows. It explains the actual SQLx-prepared +public queries under forced custom and generic plans on one backend. + +| Measurement | Before | After | +| --- | ---: | ---: | +| Generic global run lookup: unrelated rows rejected | 19,980 | 0 | +| Same lookup: shared buffers | 868 | 4 | +| Same lookup: diagnostic execution time | 3.437 ms | 0.043 ms | +| Ordinary progress: statements per write, including BEGIN/COMMIT | 8 | 6 | +| Progress: local median over 64 writes | 7.286 ms | 6.701 ms | +| Progress: local p95 | 10.515 ms | 8.230 ms | + +The review's “4× statement count” description was incorrect for the complete +progress operation. The measured reduction is 25%. Timing is diagnostic; +the manual progress test asserts the six-statement ceiling and verifies durable +progress, checkpoint, and audit events. It does not assert a latency threshold. +The idempotency-key lookup remained efficient in the fixture; the severe +reproduced regression was the run lookup. A final repeat measured 6.868 ms +median and 10.594 ms p95 for progress while retaining exactly six statements +per write, illustrating why the timing samples are not a throughput guarantee. + +The scope change does not establish a universal performance bound for JSON +searches within a large scope. A separate exploratory fixture with 10,000 old +global rows also induced a broad ordering scan under a custom plan, where the +scope predicate was already simplified. Choosing payload-expression indexes +requires workload evidence; this follow-up does not claim to resolve that +different access-pattern limitation. +It is tracked as `runledger-runledger-simplification-audit-0ju`. + +Additional tests cover: + +- Exact deadline equality and one nanosecond after it with a paused clock; + success and continuation before the cutoff remain accepted. +- A real worker handler that returns success or continuation after a + non-yielding poll crosses the cutoff, proving timer-branch selection alone + cannot enforce the rule. +- Cancellation of blocked progress with one-, two-, and four-connection worker + pools; cancelled writes cannot commit after the holder releases the row. +- A live lease expiring while progress waits for its row lock, with no progress + or checkpoint/audit write surviving. +- Sparse status and job-type filters in summary pagination under custom and + generic plans, with independent fixture-derived expected rows. Selective + custom plans may use the existing type/status/time index and apply the UUID + tie-break as a residual filter; requiring one particular index would reject + valid plans. +- Current-startup enforcement of the additive index migration independently of + the custom compatibility fence. + +## Verification + +Focused query-plan, progress, deadline, small-pool cancellation, and migration +tests passed. SQLx metadata was refreshed against PostgreSQL 18.6 with current +migrations and synchronized across all three cache directories. + +The first full workspace build exhausted local disk space. Cargo's supported +profile cleanup removed regenerable development artifacts before retrying with +incremental compilation disabled. + +- `cargo test --workspace`: 915 passed, zero failed, five explicitly ignored. + The ignored entries are two manual cost diagnostics, a claim benchmark, a + slow promoter transaction-timeout test, and a lifecycle child entrypoint. + External PostgreSQL mode does not independently exercise owned-container + lifecycle teardown; its parent tests return early in that mode. +- `scripts/lint.sh`: passed workspace/all-target/all-feature Clippy, standalone + consumer Clippy, formatting, README checks, migration-info checks, and + warning-free rustdoc. +- `scripts/run-external-consumer-smoke.sh`: four tests passed from packaged + crates on PostgreSQL 18.6. Downstream application suites were not rerun. +- `scripts/refresh-sqlx-cache.sh`: passed PostgreSQL 18/current-migration checks, + offline compilation, and packaged-cache checks; all three directories contain + the same 159 query records. +- The ignored progress diagnostic was also run explicitly: its six-statement + ceiling and durable-state/audit assertions passed. The strengthened deadline + result-preservation test and final runtime Clippy check passed separately. + +The focused diagnostics can be rerun with an isolated PostgreSQL 18 server +configured with `shared_preload_libraries=pg_stat_statements`, exporting its +administrative URL as `RUNLEDGER_TEST_ADMIN_DATABASE_URL`: + +```bash +cargo test -p runledger-postgres --test job_read_plans -- --nocapture +cargo test -p runledger-postgres --test job_summary_plans -- --nocapture +cargo test -p runledger-postgres --test operational_costs measure_progress_costs -- --ignored --nocapture +cargo test -p runledger-postgres --test progress_validation -- --nocapture +cargo test -p runledger-runtime --lib execution_services -- --nocapture +cargo test -p runledger-runtime --lib success_and_continuation_must_be_observed_strictly_before_the_deadline +cargo test -p runledger-postgres --test migrations summary_indexes -- --nocapture +``` diff --git a/docs/shared-job-specs-migrations/README.md b/docs/shared-job-specs-migrations/README.md new file mode 100644 index 0000000..8ecbdce --- /dev/null +++ b/docs/shared-job-specs-migrations/README.md @@ -0,0 +1,112 @@ +# Shared job-spec migration pilots + +These patches migrate real IdentityPro and CreditKit adapters for +`runledger-runledger-simplification-audit-tor` (AP-RUST-002 and API-003). +They are integration pilots applied in isolated worktrees against this Runledger +workspace. Application releases and dependency updates remain separate; neither +application's main checkout was modified or deployed. + +## IdentityPro + +`identitypro.patch` applies to `ee655fd0a40c6fd047bafa19ea3a1a3ade1e3e38`. +Producer definition upserts and worker registration now consume the same +`JobSpec` conversion. The local compile-time timeout constants, schedule +activation, and capability policies remain application-owned. The duplicate +runtime definition conversion and manual handler identity assertion are removed; +Runledger validates the shared specification binding. + +Billing renewal uses `JobContract::submit` and `TypedJobHandler`. Its durable +payload, unknown-field rejection, tenant check, static malformed-payload code, +and explicitly snapshotted attempts/timeout are preserved. The dispatcher counts +only inserted jobs. Shape diagnostics no longer print deserializer input values. + +## CreditKit + +`creditkit.patch` applies to `808bf397f8729f862f11646ad1dc6f5bff488727`. +The billing crate owns the shared renewal contract, payload, and operational +settings. Worker registration consumes the spec and its typed adapter; business +logic retains tenant validation and provider-error classification. The old UUID +and timestamp parsing block is removed from the renewal handler. Unknown fields +remain accepted. A contract serializer preserves the legacy `+00:00` timestamp +spelling instead of changing existing request snapshots to `Z`. + +The producer preserves its explicit one-attempt/120-second request overrides and +counts only `Inserted`. The existing database duplicate-scheduler test now +requires counts of one followed by zero while still asserting one durable row +and the exact payload fields. + +## Compatibility with this Runledger revision + +Both patches include mechanical replacements of older test accesses to private +`JobCompletion` fields with public accessors, preserving expected values. +CreditKit also acknowledges the fallible progress builder for its fixed five-stage +mapping. These adjustments are needed by APIs already present in this checkout. +They do not relax progress or checkpoint assertions. + +## Reproduce + +From clean worktrees at the revisions above, apply the appropriate patch with +`git apply --check` followed by `git apply`. Create a temporary Cargo config: + +```toml +[patch.crates-io] +runledger-core = { path = "/path/to/runledger/runledger-core" } +runledger-postgres = { path = "/path/to/runledger/runledger-postgres" } +runledger-runtime = { path = "/path/to/runledger/runledger-runtime" } +``` + +Pass `--config /path/to/overrides.toml` to the commands below and set +`SQLX_OFFLINE=true`. Cargo lockfile changes from local overrides are excluded +from the patches. For an application release, select a published Runledger +version or pinned Git revision containing these APIs and regenerate its lockfile. + +IdentityPro: + +```sh +cargo check -p identitypro-jobs --no-default-features --features storage +cargo check -p identitypro-jobs --features worker,test-support --all-targets +cargo test -p identitypro-jobs --features worker,test-support --lib +``` + +CreditKit: + +```sh +cargo check -p creditkit -p creditkit-billing --features test-support --all-targets +cargo test -p creditkit-billing --features test-support --lib renewal -- --nocapture +cargo test -p creditkit --features test-support --lib jobs::tests +``` + +Database tests use the application's PostgreSQL 18 harness. Providers are mocked +by the existing suites; these pilots do not establish live payment-provider +behavior or replace downstream release gates. + +## Verification results + +IdentityPro: storage-only construction compiled without worker/provider clients; +worker/test-support all-target checking and Clippy passed. All 84 job unit tests +passed, including typed billing serialization, malformed input, unknown-field +policy, tenant mismatch, and shared definition/catalog validation. + +CreditKit: all-target checking for `creditkit` and `creditkit-billing` passed. +All 47 renewal-related billing tests and all 59 application job tests passed. +The duplicate-scheduler test recorded PostgreSQL +`18.6 (Debian 18.6-1.pgdg13+2)`, preserved the old payload spelling, and verified +that an identical retry returns zero new work. Application job tests include +real PostgreSQL enqueue-to-handler execution, stale renewal replay, tenant +checks, malformed timestamps, and catalog synchronization. + +Both patches passed `git apply --check` against their source checkouts. This was +solo implementation and verification; no independent reviewer or production +rollout is claimed. + +Runledger verification passed: the core suite and doctests; catalog unit tests; +63 catalog integration tests; 32 idempotency tests; three enqueue-outcome tests; +and the new core, API-only PostgreSQL, and runtime shared-spec tests. The latter +cover missing/duplicate/unknown bindings, disabled specs, legacy JSON decoding, +custom safe failures, execution-service and terminal-hook forwarding, metadata +parity, strict snapshots across definition changes, and operator disables. +The PostgreSQL acceptance test recorded the same 18.6 server version. +`scripts/lint.sh`, final workspace all-target/all-feature Clippy, warning-free +rustdoc, and a storage-free `runledger-core --no-default-features` check passed. +No SQL or migration changes were needed; all three SQLx cache directories remain +identical. diff --git a/docs/shared-job-specs-migrations/creditkit.patch b/docs/shared-job-specs-migrations/creditkit.patch new file mode 100644 index 0000000..dd12ea9 --- /dev/null +++ b/docs/shared-job-specs-migrations/creditkit.patch @@ -0,0 +1,552 @@ +diff --git a/crates/creditkit-billing/src/lib.rs b/crates/creditkit-billing/src/lib.rs +index 0b585311..b23623f8 100644 +--- a/crates/creditkit-billing/src/lib.rs ++++ b/crates/creditkit-billing/src/lib.rs +@@ -10,7 +10,9 @@ mod nmi_account; + mod product_analytics; + mod provider_evidence; + mod reconciliation; ++mod renewal_contract; + mod renewals; ++pub use renewal_contract::{RENEWAL_CHARGE_SETTINGS, RenewalCharge, RenewalChargePayload}; + mod repository; + mod responses; + mod service; +diff --git a/crates/creditkit-billing/src/renewal_contract.rs b/crates/creditkit-billing/src/renewal_contract.rs +new file mode 100644 +index 00000000..850730b1 +--- /dev/null ++++ b/crates/creditkit-billing/src/renewal_contract.rs +@@ -0,0 +1,68 @@ ++//! Durable renewal contract shared by the producer and worker. ++use chrono::{DateTime, Utc}; ++use runledger_core::jobs::{JobContract, JobDefinitionSettings, JobSpec, JobType}; ++use serde::{Deserialize, Serialize, Serializer}; ++use uuid::Uuid; ++ ++use crate::{RENEWAL_CHARGE_JOB_TYPE, RENEWAL_JOB_TIMEOUT_SECONDS}; ++ ++pub const RENEWAL_CHARGE_SETTINGS: JobDefinitionSettings = JobDefinitionSettings::new() ++ .max_attempts(1) ++ .timeout_seconds(RENEWAL_JOB_TIMEOUT_SECONDS) ++ .priority(100); ++ ++#[derive(Debug, Serialize, Deserialize)] ++pub struct RenewalChargePayload { ++ pub tenant_id: Uuid, ++ pub subscription_id: Uuid, ++ // Keep the old producer's +00:00 spelling in idempotency request snapshots. ++ #[serde(serialize_with = "serialize_period")] ++ pub period_start_at: DateTime, ++} ++ ++fn serialize_period( ++ value: &DateTime, ++ serializer: S, ++) -> Result { ++ serializer.serialize_str(&value.to_rfc3339()) ++} ++ ++pub struct RenewalCharge; ++impl JobContract for RenewalCharge { ++ type Payload = RenewalChargePayload; ++ fn spec() -> JobSpec { ++ JobSpec::new(JobType::new(RENEWAL_CHARGE_JOB_TYPE)) ++ .expect("static renewal identity") ++ .with_settings(RENEWAL_CHARGE_SETTINGS) ++ .expect("static renewal settings") ++ } ++} ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ use serde_json::json; ++ ++ #[test] ++ fn typed_submission_matches_legacy_json_and_preserves_old_rows() { ++ let payload = RenewalChargePayload { ++ tenant_id: Uuid::nil(), ++ subscription_id: Uuid::nil(), ++ period_start_at: "2026-09-05T10:00:00.123456+00:00" ++ .parse() ++ .expect("timestamp"), ++ }; ++ let request = RenewalCharge::submit(&payload).expect("submit"); ++ assert_eq!( ++ request.payload, ++ json!({ ++ "tenant_id":payload.tenant_id,"subscription_id":payload.subscription_id, ++ "period_start_at":payload.period_start_at.to_rfc3339(), ++ }) ++ ); ++ let mut old = request.payload; ++ old["extra_legacy_field"] = json!(true); ++ let decoded: RenewalChargePayload = serde_json::from_value(old).expect("old row"); ++ assert_eq!(decoded.period_start_at, payload.period_start_at); ++ } ++} +diff --git a/crates/creditkit-billing/src/renewals.rs b/crates/creditkit-billing/src/renewals.rs +index 42ba0640..924f8d60 100644 +--- a/crates/creditkit-billing/src/renewals.rs ++++ b/crates/creditkit-billing/src/renewals.rs +@@ -1,4 +1,6 @@ + use crate::*; ++use runledger_core::jobs::JobContract; ++use runledger_postgres::jobs::{JobEnqueueDisposition, enqueue_job_with_outcome}; + use syrup_rail::{ChargeRenewal, PaymentResolutionCode, RenewalDispatch}; + + impl BillingService { +@@ -8,29 +10,30 @@ impl BillingService { + let due = state.repository.due_subscriptions().await?; + let mut enqueued = 0; + for subscription in due { +- let payload = json!({ +- "tenant_id": subscription.billing_scope_id().into_uuid(), +- "subscription_id": subscription.subscription_id().into_uuid(), +- "period_start_at": subscription.period_start_at().to_rfc3339(), +- }); +- let idempotency_key = renewal_enqueue_idempotency_key(&subscription, enqueue_window); +- let job = JobEnqueue { +- job_type: JobType::new(RENEWAL_CHARGE_JOB_TYPE), +- organization_id: Some(subscription.billing_scope_id().into_uuid()), +- payload: &payload, +- priority: None, +- max_attempts: Some(1), +- timeout_seconds: Some(RENEWAL_JOB_TIMEOUT_SECONDS), +- next_run_at: None, +- // The enqueue window keeps a lagging worker from accumulating +- // duplicate renewal jobs while still letting future scheduler +- // ticks recover from stale failed queue rows. +- idempotency_key: Some(&idempotency_key), +- stage: None, ++ let payload = RenewalChargePayload { ++ tenant_id: subscription.billing_scope_id().into_uuid(), ++ subscription_id: subscription.subscription_id().into_uuid(), ++ period_start_at: *subscription.period_start_at(), + }; +- match enqueue_job(state.repository.database().pool(), &job).await { +- Ok(_) => enqueued += 1, +- Err(error) if is_job_already_enqueued(&error) => {} ++ let idempotency_key = renewal_enqueue_idempotency_key(&subscription, enqueue_window); ++ // Retain overrides and time spelling from the original durable request. ++ let request = RenewalCharge::submit(&payload) ++ .expect("static renewal contract and serializable payload") ++ .organization_id(payload.tenant_id) ++ .max_attempts(RENEWAL_CHARGE_SETTINGS.max_attempts) ++ .timeout_seconds(RENEWAL_CHARGE_SETTINGS.default_timeout_seconds) ++ .idempotency_key(idempotency_key); ++ match enqueue_job_with_outcome( ++ state.repository.database().pool(), ++ &JobEnqueue::from(&request), ++ ) ++ .await ++ { ++ Ok(outcome) => { ++ if outcome.disposition == JobEnqueueDisposition::Inserted { ++ enqueued += 1; ++ } ++ } + Err(error) => { + drop(error); + return Err(BillingError::job_enqueue( +@@ -191,11 +194,3 @@ pub(crate) fn renewal_enqueue_idempotency_key( + enqueue_window + ) + } +- +-fn is_job_already_enqueued(error: &runledger_postgres::Error) -> bool { +- matches!( +- error, +- runledger_postgres::Error::QueryError(query_error) +- if query_error.code() == "job.already_enqueued" +- ) +-} +diff --git a/crates/creditkit-billing/src/tests/renewal_attempt_limits.rs b/crates/creditkit-billing/src/tests/renewal_attempt_limits.rs +index eb600b64..bea0c1aa 100644 +--- a/crates/creditkit-billing/src/tests/renewal_attempt_limits.rs ++++ b/crates/creditkit-billing/src/tests/renewal_attempt_limits.rs +@@ -248,6 +248,15 @@ async fn renewal_enqueue_deduplicates_same_period_scheduler_ticks() { + Some("vault_enqueue"), + ))); + let (service, database, _postgres, user_id) = test_service(gateway).await; ++ let version: String = sqlx::query_scalar("SHOW server_version") ++ .fetch_one(database.pool()) ++ .await ++ .expect("server version"); ++ eprintln!("renewal migration server_version={version}"); ++ assert!( ++ version.starts_with("18."), ++ "PostgreSQL 18 baseline required: {version}" ++ ); + insert_renewal_charge_job_definition(&database).await; + + let enrollment = service +@@ -283,7 +292,7 @@ async fn renewal_enqueue_deduplicates_same_period_scheduler_ticks() { + .expect("same-window duplicate should resolve to the existing job row"); + + assert_eq!(first, 1); +- assert_eq!(second, 1); ++ assert_eq!(second, 0, "identical retries must not count as new work"); + let jobs: Vec<(Option, serde_json::Value, String)> = sqlx::query_as( + r#" + SELECT organization_id, payload, idempotency_key +diff --git a/crates/creditkit/src/jobs/billing.rs b/crates/creditkit/src/jobs/billing.rs +index 09694bee..a7c143b6 100644 +--- a/crates/creditkit/src/jobs/billing.rs ++++ b/crates/creditkit/src/jobs/billing.rs +@@ -1,16 +1,15 @@ + use async_trait::async_trait; + use creditkit_billing::{ + BillingError, BillingService, RECONCILE_NMI_ACCOUNT_JOB_TYPE, RECONCILE_NMI_ATTEMPTS_JOB_TYPE, +- RENEWAL_CHARGE_JOB_TYPE, RENEWAL_ENQUEUE_JOB_TYPE, ++ RENEWAL_ENQUEUE_JOB_TYPE, RenewalCharge, RenewalChargePayload, ++}; ++use runledger_core::jobs::{ ++ JobCompletion, JobContext, JobFailure, JobHandler, JobType, TypedJobHandler, + }; +-use runledger_core::jobs::{JobCompletion, JobContext, JobFailure, JobHandler, JobType}; + use serde_json::Value; + use syrup_rail::GatewayError; + +-use super::{ +- failure::REDACTED_TELEMETRY_ID, +- payload::{datetime_payload_field, uuid_payload_field}, +-}; ++use super::{failure::REDACTED_TELEMETRY_ID, payload::uuid_payload_field}; + + pub(super) struct BillingRenewalEnqueueHandler { + pub(super) billing: BillingService, +@@ -56,47 +55,26 @@ impl JobHandler for BillingRenewalEnqueueHandler { + } + + #[async_trait] +-impl JobHandler for BillingRenewalChargeHandler { +- fn job_type(&self) -> JobType<'static> { +- JobType::new(RENEWAL_CHARGE_JOB_TYPE) ++impl TypedJobHandler for BillingRenewalChargeHandler { ++ type Contract = RenewalCharge; ++ ++ fn malformed_payload(&self, _source: &serde_json::Error) -> JobFailure { ++ JobFailure::terminal( ++ "creditkit.billing.invalid_renewal_payload", ++ "Invalid billing renewal payload.", ++ ) + } + + async fn execute( + &self, + context: JobContext, +- payload: Value, ++ payload: RenewalChargePayload, + ) -> Result { +- let tenant_id = uuid_payload_field( +- &payload, +- "tenant_id", +- "creditkit.billing.invalid_renewal_payload", +- "billing renewal", +- )?; +- let subscription_id = uuid_payload_field( +- &payload, +- "subscription_id", +- "creditkit.billing.invalid_renewal_payload", +- "billing renewal", +- )?; +- let period_start_at = datetime_payload_field(&payload, "period_start_at") +- .map_err(|source| { +- tracing::warn!( +- payload_name = "billing renewal", +- field = "period_start_at", +- error = ?source, +- "failed to parse datetime field from job payload" +- ); +- JobFailure::terminal( +- "creditkit.billing.invalid_renewal_payload", +- "billing renewal payload must include period_start_at", +- ) +- })? +- .ok_or_else(|| { +- JobFailure::terminal( +- "creditkit.billing.invalid_renewal_payload", +- "billing renewal payload must include period_start_at", +- ) +- })?; ++ let RenewalChargePayload { ++ tenant_id, ++ subscription_id, ++ period_start_at, ++ } = payload; + if context.organization_id != Some(tenant_id) { + tracing::error!( + job_id = REDACTED_TELEMETRY_ID, +diff --git a/crates/creditkit/src/jobs/catalog.rs b/crates/creditkit/src/jobs/catalog.rs +index c299115d..bf4bce0f 100644 +--- a/crates/creditkit/src/jobs/catalog.rs ++++ b/crates/creditkit/src/jobs/catalog.rs +@@ -2,8 +2,9 @@ use chrono::Utc; + use creditkit_billing::{ + BILLING_PROVIDER_RATE_LIMIT_RETRY_AFTER, PRODUCT_ANALYTICS_DELIVERY_JOB_TYPE, + RECONCILE_JOB_TIMEOUT_SECONDS, RECONCILE_NMI_ACCOUNT_JOB_TYPE, RECONCILE_NMI_ATTEMPTS_JOB_TYPE, +- RENEWAL_CHARGE_JOB_TYPE, RENEWAL_ENQUEUE_JOB_TYPE, RENEWAL_JOB_TIMEOUT_SECONDS, ++ RENEWAL_CHARGE_JOB_TYPE, RENEWAL_CHARGE_SETTINGS, RENEWAL_ENQUEUE_JOB_TYPE, RenewalCharge, + }; ++use runledger_core::jobs::{JobContract, TypedJobHandler}; + use runledger_postgres::jobs::{ + JobScheduleUpsert, set_job_schedule_active_tx, upsert_job_schedule_tx, + }; +@@ -551,16 +552,22 @@ pub(super) const CREDITKIT_JOB_SPECS: &[CreditKitJobSpec] = &[ + }, + CreditKitJobSpec { + job_type: RENEWAL_CHARGE_JOB_TYPE, +- definition: CreditKitJobDefinition::standard(RENEWAL_JOB_TIMEOUT_SECONDS) +- .with_max_attempts(1), ++ definition: CreditKitJobDefinition { ++ version: RENEWAL_CHARGE_SETTINGS.version, ++ max_attempts: RENEWAL_CHARGE_SETTINGS.max_attempts, ++ timeout_seconds: RENEWAL_CHARGE_SETTINGS.default_timeout_seconds, ++ priority: RENEWAL_CHARGE_SETTINGS.default_priority, ++ enabled: RENEWAL_CHARGE_SETTINGS.is_enabled, ++ }, + schedule: None, + retry_delay: None, +- register: |catalog, dependencies, definition| { +- catalog.try_handler_with_definition_overrides( ++ register: |catalog, dependencies, _definition| { ++ catalog.try_handler_for_spec( ++ &RenewalCharge::spec(), + BillingRenewalChargeHandler { + billing: dependencies.billing.clone(), +- }, +- definition, ++ } ++ .into_job_handler(), + ) + }, + }, +diff --git a/crates/creditkit/src/jobs/documents.rs b/crates/creditkit/src/jobs/documents.rs +index 4efbd1a2..a73c5aaf 100644 +--- a/crates/creditkit/src/jobs/documents.rs ++++ b/crates/creditkit/src/jobs/documents.rs +@@ -354,6 +354,7 @@ fn native_text_credit_report_progress( + }; + completion + .progress(progress_done, 5) ++ .expect("document checkpoint progress is bounded by its five stages") + .checkpoint(serde_json::json!({ + "schema_version": 1, + "pipeline_version": pipeline_version, +diff --git a/crates/creditkit/src/jobs/payload.rs b/crates/creditkit/src/jobs/payload.rs +index e14ae022..d1f6ae9b 100644 +--- a/crates/creditkit/src/jobs/payload.rs ++++ b/crates/creditkit/src/jobs/payload.rs +@@ -1,4 +1,3 @@ +-use chrono::{DateTime, Utc}; + use runledger_core::jobs::JobFailure; + use serde_json::Value; + use uuid::Uuid; +@@ -34,18 +33,3 @@ pub(super) fn uuid_payload_field( + } + } + } +- +-pub(super) fn datetime_payload_field( +- payload: &Value, +- field: &'static str, +-) -> Result>, chrono::ParseError> { +- // Missing and non-string fields stay optional so callers can keep their +- // existing required-field failures; malformed strings preserve the parse source. +- let Some(value) = payload.get(field) else { +- return Ok(None); +- }; +- let Some(value) = value.as_str() else { +- return Ok(None); +- }; +- DateTime::parse_from_rfc3339(value).map(|value| Some(value.with_timezone(&Utc))) +-} +diff --git a/crates/creditkit/src/jobs/tests.rs b/crates/creditkit/src/jobs/tests.rs +index 1244a77c..b8dbfcbd 100644 +--- a/crates/creditkit/src/jobs/tests.rs ++++ b/crates/creditkit/src/jobs/tests.rs +@@ -76,7 +76,6 @@ use super::observability::{ + safe_job_failure_summary, + }; + use super::outbound_email::OUTBOUND_EMAIL_DISPATCH_JOB_TYPE; +-use super::payload::datetime_payload_field; + use crate::connections::{ConnectionStorageError, MonitoringImportError}; + use crate::documents::{ + DOCUMENT_ADVANCE_NATIVE_TEXT_CREDIT_REPORT_JOB_TYPE, DOCUMENT_EXTRACT_TEXT_JOB_TYPE, +diff --git a/crates/creditkit/src/jobs/tests/basic.rs b/crates/creditkit/src/jobs/tests/basic.rs +index ddabec49..0606cf0b 100644 +--- a/crates/creditkit/src/jobs/tests/basic.rs ++++ b/crates/creditkit/src/jobs/tests/basic.rs +@@ -14,10 +14,10 @@ fn native_text_completion_projects_privacy_safe_continuation_state() { + completion.disposition(), + JobCompletionDisposition::ContinueAfter(Duration::from_secs(7)) + ); +- assert_eq!(completion.progress_done, Some(1)); +- assert_eq!(completion.progress_total, Some(5)); ++ assert_eq!(completion.progress_done(), Some(1)); ++ assert_eq!(completion.progress_total(), Some(5)); + assert_eq!( +- completion.checkpoint, ++ completion.checkpoint_value().cloned(), + Some(json!({ + "schema_version": 1, + "pipeline_version": "native-text-credit-report-v1", +@@ -33,9 +33,9 @@ fn missing_native_root_completes_without_durable_worker_state() { + ); + + assert_eq!(completion.disposition(), JobCompletionDisposition::Succeed); +- assert_eq!(completion.progress_done, None); +- assert_eq!(completion.progress_total, None); +- assert_eq!(completion.checkpoint, None); ++ assert_eq!(completion.progress_done(), None); ++ assert_eq!(completion.progress_total(), None); ++ assert_eq!(completion.checkpoint_value().cloned(), None); + } + + #[test] +@@ -48,10 +48,10 @@ fn live_native_retry_exhaustion_returns_runledger_success() { + ); + + assert_eq!(completion.disposition(), JobCompletionDisposition::Succeed); +- assert_eq!(completion.progress_done, Some(5)); +- assert_eq!(completion.progress_total, Some(5)); ++ assert_eq!(completion.progress_done(), Some(5)); ++ assert_eq!(completion.progress_total(), Some(5)); + assert_eq!( +- completion.checkpoint, ++ completion.checkpoint_value().cloned(), + Some(json!({ + "schema_version": 1, + "pipeline_version": "native-text-credit-report-v1", +diff --git a/crates/creditkit/src/jobs/tests/billing_cases.rs b/crates/creditkit/src/jobs/tests/billing_cases.rs +index 58bc2f08..ac970815 100644 +--- a/crates/creditkit/src/jobs/tests/billing_cases.rs ++++ b/crates/creditkit/src/jobs/tests/billing_cases.rs +@@ -1,4 +1,5 @@ + use super::*; ++use runledger_core::jobs::TypedJobHandler; + + #[tokio::test] + async fn billing_renewal_charge_rejects_mismatched_tenant_context() { +@@ -7,7 +8,8 @@ async fn billing_renewal_charge_rejects_mismatched_tenant_context() { + let subscription_id = Uuid::now_v7(); + let handler = BillingRenewalChargeHandler { + billing: creditkit_billing::BillingService::disabled(), +- }; ++ } ++ .into_job_handler(); + + let error = handler + .execute( +@@ -317,7 +319,7 @@ async fn billing_renewal_charge_accepts_enqueued_payload_and_context() { + ); + assert!(payload["period_start_at"].as_str().is_some()); + +- let handler = BillingRenewalChargeHandler { billing }; ++ let handler = BillingRenewalChargeHandler { billing }.into_job_handler(); + handler + .execute( + JobContext { +@@ -425,7 +427,7 @@ async fn stale_renewal_job_after_recovery_is_a_noop() { + let resolution_count_before = gateway_resolutions.resolution_count(); + let account_mode_count_before = gateway.account_mode_query_count(); + let sale_count_before = gateway.sale_count(); +- let handler = BillingRenewalChargeHandler { billing }; ++ let handler = BillingRenewalChargeHandler { billing }.into_job_handler(); + handler + .execute( + JobContext { +@@ -578,7 +580,8 @@ async fn renewal_identity_layers_remain_distinct() { + let tenant_id = organization_id.expect("renewal job should carry tenant context"); + let handler = BillingRenewalChargeHandler { + billing: billing.clone(), +- }; ++ } ++ .into_job_handler(); + handler + .execute( + JobContext { +diff --git a/crates/creditkit/src/jobs/tests/payloads.rs b/crates/creditkit/src/jobs/tests/payloads.rs +index ef7b79bc..8466cc85 100644 +--- a/crates/creditkit/src/jobs/tests/payloads.rs ++++ b/crates/creditkit/src/jobs/tests/payloads.rs +@@ -295,25 +295,41 @@ fn native_text_credit_report_payload_rejects_identity_bearing_fields() { + .expect_err("identity-bearing native payload fields should be rejected"); + } + +-#[test] +-fn datetime_payload_field_rejects_malformed_timestamp_without_conflating_absence() { +- assert!( +- datetime_payload_field(&json!({}), "period_start_at") +- .expect("missing field should stay optional") +- .is_none() +- ); +- assert!( +- datetime_payload_field(&json!({ "period_start_at": 1 }), "period_start_at") +- .expect("non-string field should stay optional") +- .is_none() +- ); +- +- let error = datetime_payload_field( +- &json!({ "period_start_at": "not-a-timestamp" }), +- "period_start_at", +- ) +- .expect_err("malformed timestamp should preserve the parse source"); +- assert!(!error.to_string().is_empty()); ++#[tokio::test] ++async fn typed_renewal_rejects_missing_non_string_and_malformed_timestamps_safely() { ++ use runledger_core::jobs::TypedJobHandler; ++ let handler = BillingRenewalChargeHandler { ++ billing: creditkit_billing::BillingService::disabled(), ++ } ++ .into_job_handler(); ++ for period in [ ++ None, ++ Some(json!(1)), ++ Some(json!("private-invalid-timestamp")), ++ ] { ++ let tenant_id = Uuid::now_v7(); ++ let mut payload = json!({"tenant_id":tenant_id,"subscription_id":Uuid::now_v7()}); ++ if let Some(period) = period { ++ payload["period_start_at"] = period; ++ } ++ let error = handler ++ .execute( ++ JobContext { ++ job_id: Uuid::now_v7(), ++ run_number: 1, ++ attempt: 1, ++ organization_id: Some(tenant_id), ++ worker_id: "test".into(), ++ checkpoint: None, ++ }, ++ payload, ++ ) ++ .await ++ .expect_err("invalid timestamp"); ++ assert_eq!(error.kind, JobFailureKind::Terminal); ++ assert_eq!(error.code, "creditkit.billing.invalid_renewal_payload"); ++ assert_eq!(error.message, "Invalid billing renewal payload."); ++ } + } + + #[test] +diff --git a/crates/creditkit/src/jobs/tests/workflows.rs b/crates/creditkit/src/jobs/tests/workflows.rs +index d1a1c74c..3480d5d1 100644 +--- a/crates/creditkit/src/jobs/tests/workflows.rs ++++ b/crates/creditkit/src/jobs/tests/workflows.rs +@@ -674,9 +674,9 @@ async fn native_text_handler_completes_without_side_effects_after_its_root_is_de + .expect("missing root provider should drain before assertions"); + let (completion, counts) = scenario.expect("missing root scenario should complete"); + assert_eq!(completion.disposition(), JobCompletionDisposition::Succeed); +- assert_eq!(completion.progress_done, None); +- assert_eq!(completion.progress_total, None); +- assert_eq!(completion.checkpoint, None); ++ assert_eq!(completion.progress_done(), None); ++ assert_eq!(completion.progress_total(), None); ++ assert_eq!(completion.checkpoint_value().cloned(), None); + assert_eq!(counts, (0, 0, 0, 1)); + assert!( + provider_requests.is_empty(), diff --git a/docs/shared-job-specs-migrations/identitypro.patch b/docs/shared-job-specs-migrations/identitypro.patch new file mode 100644 index 0000000..b10193a --- /dev/null +++ b/docs/shared-job-specs-migrations/identitypro.patch @@ -0,0 +1,384 @@ +diff --git a/crates/identitypro-jobs/src/billing_renewal.rs b/crates/identitypro-jobs/src/billing_renewal.rs +index 2f2d128..8972411 100644 +--- a/crates/identitypro-jobs/src/billing_renewal.rs ++++ b/crates/identitypro-jobs/src/billing_renewal.rs +@@ -1,14 +1,17 @@ + use chrono::{DateTime, Utc}; + use identitypro_billing::{BillingError, BillingService, SubscriptionRenewalOutcome}; + use identitypro_db::Db; +-use runledger_core::jobs::{JobCompletion, JobContext, JobFailure, JobHandler, JobType}; +-use runledger_postgres::jobs::{JobEnqueue, enqueue_job}; ++use runledger_core::jobs::{ ++ JobCompletion, JobContext, JobContract, JobFailure, JobHandler, JobSpec, JobType, ++ TypedJobHandler, ++}; ++use runledger_postgres::jobs::{JobEnqueue, JobEnqueueDisposition, enqueue_job_with_outcome}; + use serde::{Deserialize, Serialize}; + use serde_json::Value; + use uuid::Uuid; + + use crate::{ +- BILLING_RENEWAL_CHARGE_JOB_TYPE, BILLING_RENEWAL_DISPATCH_JOB_TYPE, ++ BILLING_RENEWAL_DISPATCH_JOB_TYPE, + billing_failure::{BillingFailureDisposition, classify_billing_failure}, + definitions::BILLING_RENEWAL_CHARGE_DEFINITION, + }; +@@ -26,12 +29,22 @@ pub(crate) struct BillingRenewalChargeHandler { + + #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] + #[serde(deny_unknown_fields)] +-struct RenewalPayload { ++pub(crate) struct RenewalPayload { + billing_scope_id: Uuid, + subscription_id: Uuid, + period_start_at: DateTime, + } + ++pub(crate) struct BillingRenewalCharge; ++ ++impl JobContract for BillingRenewalCharge { ++ type Payload = RenewalPayload; ++ ++ fn spec() -> JobSpec { ++ BILLING_RENEWAL_CHARGE_DEFINITION.shared_spec() ++ } ++} ++ + impl BillingRenewalDispatchHandler { + pub(crate) fn new(db: Db, billing: Option) -> Self { + Self { db, billing } +@@ -67,7 +80,6 @@ impl JobHandler for BillingRenewalDispatchHandler { + subscription_id: dispatch.subscription_id().into_uuid(), + period_start_at: *dispatch.period_start_at(), + }; +- let value = serde_json::to_value(&payload).expect("renewal payload is serializable"); + let idempotency_key = format!( + "billing-renewal:{}:{}:{}:{}", + payload.subscription_id, +@@ -75,22 +87,19 @@ impl JobHandler for BillingRenewalDispatchHandler { + dispatch.attempt_sequence_count(), + enqueue_window, + ); +- let enqueue_overrides = BILLING_RENEWAL_CHARGE_DEFINITION.enqueue_overrides(); +- let job = JobEnqueue { +- job_type: JobType::new(BILLING_RENEWAL_CHARGE_JOB_TYPE), +- organization_id: Some(payload.billing_scope_id), +- payload: &value, +- priority: None, +- max_attempts: enqueue_overrides.max_attempts, +- timeout_seconds: enqueue_overrides.timeout_seconds, +- next_run_at: None, +- idempotency_key: Some(&idempotency_key), +- stage: None, +- }; +- match enqueue_job(self.db.pool(), &job).await { +- Ok(_) => enqueued += 1, +- Err(runledger_postgres::Error::QueryError(error)) +- if error.code() == "job.already_enqueued" => {} ++ // Retain the explicit overrides used in already-durable request snapshots. ++ let job = BillingRenewalCharge::submit(&payload) ++ .expect("static renewal contract and serializable payload") ++ .organization_id(payload.billing_scope_id) ++ .max_attempts(BILLING_RENEWAL_CHARGE_DEFINITION.max_attempts) ++ .timeout_seconds(BILLING_RENEWAL_CHARGE_DEFINITION.timeout_seconds) ++ .idempotency_key(idempotency_key); ++ match enqueue_job_with_outcome(self.db.pool(), &JobEnqueue::from(&job)).await { ++ Ok(outcome) => { ++ if outcome.disposition == JobEnqueueDisposition::Inserted { ++ enqueued += 1; ++ } ++ } + Err(error) => { + tracing::error!(error = ?error, "could not enqueue billing renewal"); + return Err(JobFailure::retryable( +@@ -108,23 +117,22 @@ impl JobHandler for BillingRenewalDispatchHandler { + } + + #[async_trait::async_trait] +-impl JobHandler for BillingRenewalChargeHandler { +- fn job_type(&self) -> JobType<'static> { +- JobType::new(BILLING_RENEWAL_CHARGE_JOB_TYPE) ++impl TypedJobHandler for BillingRenewalChargeHandler { ++ type Contract = BillingRenewalCharge; ++ ++ fn malformed_payload(&self, _source: &serde_json::Error) -> JobFailure { ++ tracing::warn!("billing renewal payload had an invalid shape"); ++ JobFailure::terminal( ++ "identitypro.billing.invalid_renewal_payload", ++ "invalid billing renewal payload", ++ ) + } + + async fn execute( + &self, + context: JobContext, +- payload: Value, ++ payload: RenewalPayload, + ) -> Result { +- let payload: RenewalPayload = serde_json::from_value(payload).map_err(|error| { +- tracing::warn!(error = ?error, "billing renewal payload had an invalid shape"); +- JobFailure::terminal( +- "identitypro.billing.invalid_renewal_payload", +- "invalid billing renewal payload", +- ) +- })?; + if context.organization_id != Some(payload.billing_scope_id) { + return Err(JobFailure::terminal( + "identitypro.billing.invalid_renewal_context", +@@ -169,3 +177,74 @@ fn billing_failure(error: BillingError) -> JobFailure { + ), + } + } ++ ++#[cfg(test)] ++mod typed_contract_tests { ++ use super::*; ++ use serde_json::json; ++ ++ fn context(scope: Uuid) -> JobContext { ++ JobContext { ++ job_id: Uuid::now_v7(), ++ run_number: 1, ++ attempt: 1, ++ organization_id: Some(scope), ++ worker_id: "test".into(), ++ checkpoint: None, ++ } ++ } ++ ++ #[test] ++ fn typed_submission_preserves_legacy_payload_and_explicit_overrides() { ++ let payload = RenewalPayload { ++ billing_scope_id: Uuid::nil(), ++ subscription_id: Uuid::nil(), ++ period_start_at: "2026-09-05T12:00:00Z".parse().expect("timestamp"), ++ }; ++ let request = BillingRenewalCharge::submit(&payload) ++ .expect("typed submit") ++ .max_attempts(BILLING_RENEWAL_CHARGE_DEFINITION.max_attempts) ++ .timeout_seconds(BILLING_RENEWAL_CHARGE_DEFINITION.timeout_seconds); ++ assert_eq!( ++ request.payload, ++ json!({"billing_scope_id":Uuid::nil(), "subscription_id":Uuid::nil(), "period_start_at":"2026-09-05T12:00:00Z"}) ++ ); ++ let old = BILLING_RENEWAL_CHARGE_DEFINITION.enqueue_overrides(); ++ assert_eq!(request.max_attempts, old.max_attempts); ++ assert_eq!(request.timeout_seconds, old.timeout_seconds); ++ } ++ ++ #[tokio::test] ++ async fn typed_handler_preserves_shape_failures_and_scope_validation() { ++ let handler = BillingRenewalChargeHandler::new(None).into_job_handler(); ++ let scope = Uuid::now_v7(); ++ let error = handler ++ .execute( ++ context(scope), ++ json!({"billing_scope_id":"private-invalid-value"}), ++ ) ++ .await ++ .expect_err("malformed"); ++ assert_eq!(error.kind, runledger_core::jobs::JobFailureKind::Terminal); ++ assert_eq!(error.code, "identitypro.billing.invalid_renewal_payload"); ++ assert_eq!(error.message, "invalid billing renewal payload"); ++ let payload = json!({"billing_scope_id":Uuid::nil(),"subscription_id":Uuid::nil(),"period_start_at":"2026-09-05T12:00:00Z"}); ++ let error = handler ++ .execute(context(scope), payload.clone()) ++ .await ++ .expect_err("scope mismatch"); ++ assert_eq!(error.code, "identitypro.billing.invalid_renewal_context"); ++ let error = handler ++ .execute(context(Uuid::nil()), payload.clone()) ++ .await ++ .expect_err("provider absent"); ++ assert_eq!(error.code, "identitypro.billing.not_configured"); ++ let mut unknown = payload; ++ unknown["unrecognized"] = json!(true); ++ let error = handler ++ .execute(context(Uuid::nil()), unknown) ++ .await ++ .expect_err("retain deny_unknown_fields"); ++ assert_eq!(error.code, "identitypro.billing.invalid_renewal_payload"); ++ } ++} +diff --git a/crates/identitypro-jobs/src/catalog.rs b/crates/identitypro-jobs/src/catalog.rs +index b3bf6b1..603d0df 100644 +--- a/crates/identitypro-jobs/src/catalog.rs ++++ b/crates/identitypro-jobs/src/catalog.rs +@@ -221,7 +221,9 @@ fn build_catalog( + ); + let cancellation_attention_dispatch = + SubscriptionCancellationAttentionDispatchHandler::new(db.clone()); +- let billing_charge = BillingRenewalChargeHandler::new(billing); ++ let billing_charge = runledger_core::jobs::TypedJobHandler::into_job_handler( ++ BillingRenewalChargeHandler::new(billing), ++ ); + let cleanup_handler = AuthCleanupHandler::new(db.clone()); + let admin_billing_view_audit_cleanup_handler = + AdminBillingViewAuditCleanupHandler::new(db.clone()); +@@ -368,12 +370,9 @@ where + H: JobHandler + 'static, + { + let payload = serde_json::json!({}); +- assert_eq!( +- handler.job_type().as_str(), +- contract.job_type(), +- "job handler type must match its IdentityPro job contract" +- ); +- let catalog = catalog.handler_with_definition_overrides(handler, contract.runtime_overrides()); ++ let catalog = catalog ++ .try_handler_for_spec(&contract.definition.shared_spec(), handler) ++ .expect("job handler type must match its IdentityPro job contract"); + + match contract.runtime_schedule(&payload, billing_enabled) { + Some(schedule) => catalog.schedule(schedule), +diff --git a/crates/identitypro-jobs/src/definitions.rs b/crates/identitypro-jobs/src/definitions.rs +index d98de37..9741a41 100644 +--- a/crates/identitypro-jobs/src/definitions.rs ++++ b/crates/identitypro-jobs/src/definitions.rs +@@ -1,11 +1,11 @@ + #[cfg(feature = "storage")] +-use runledger_core::jobs::JobType as ContractJobType; ++use runledger_core::jobs::{JobDefinitionSettings, JobSpec, JobType as ContractJobType}; + #[cfg(feature = "worker")] + use { + chrono::{DateTime, Utc}, + runledger_core::jobs::JobType, + runledger_postgres::jobs::{JobScheduleCatalogSyncEntry, JobScheduleUpsert}, +- runledger_runtime::catalog::{CatalogJobScheduleSpec, JobCatalogDefinitionOverrides}, ++ runledger_runtime::catalog::CatalogJobScheduleSpec, + serde_json::Value, + }; + +@@ -760,14 +760,18 @@ const _: () = assert!(LONGEST_JOB_TIMEOUT_SECONDS > 0); + + #[cfg(feature = "storage")] + impl JobDefinitionSpec { +- #[cfg(feature = "worker")] +- pub(crate) fn runtime_overrides(self) -> JobCatalogDefinitionOverrides { +- JobCatalogDefinitionOverrides::new() +- .version(self.version) +- .max_attempts(self.max_attempts) +- .timeout_seconds(self.timeout_seconds) +- .priority(self.priority) +- .enabled(self.enabled) ++ pub(crate) fn shared_spec(self) -> JobSpec { ++ JobSpec::new(ContractJobType::new(self.job_type)) ++ .expect("static IdentityPro job identity") ++ .with_settings( ++ JobDefinitionSettings::new() ++ .version(self.version) ++ .max_attempts(self.max_attempts) ++ .timeout_seconds(self.timeout_seconds) ++ .priority(self.priority) ++ .enabled(self.enabled), ++ ) ++ .expect("static IdentityPro job settings") + } + + pub(crate) const fn enqueue_overrides(self) -> JobEnqueueOverrides { +@@ -780,19 +784,13 @@ impl JobDefinitionSpec { + + #[cfg(feature = "storage")] + impl JobContract { ++ #[cfg(feature = "worker")] + pub(crate) const fn job_type(self) -> &'static str { + self.definition.job_type + } + + fn definition_upsert(self) -> runledger_postgres::jobs::JobDefinitionUpsert<'static> { +- runledger_postgres::jobs::JobDefinitionUpsert { +- job_type: ContractJobType::new(self.job_type()), +- version: self.definition.version, +- max_attempts: self.definition.max_attempts, +- default_timeout_seconds: self.definition.timeout_seconds, +- default_priority: self.definition.priority, +- is_enabled: self.definition.enabled, +- } ++ runledger_postgres::jobs::JobDefinitionUpsert::from(&self.definition.shared_spec()) + } + + #[cfg(feature = "worker")] +@@ -805,11 +803,6 @@ impl JobContract { + self.schedule + } + +- #[cfg(feature = "worker")] +- pub(crate) fn runtime_overrides(self) -> JobCatalogDefinitionOverrides { +- self.definition.runtime_overrides() +- } +- + #[cfg(feature = "worker")] + pub(crate) fn schedule_sync_entry<'a>( + self, +@@ -1003,15 +996,12 @@ mod tests { + assert_eq!(upsert.default_priority, definition.priority); + assert_eq!(upsert.is_enabled, definition.enabled); + +- let debug = format!("{:?}", contract.runtime_overrides()); +- assert!(debug.contains(&format!("version: Some({})", definition.version))); +- assert!(debug.contains(&format!("max_attempts: Some({})", definition.max_attempts))); +- assert!(debug.contains(&format!( +- "default_timeout_seconds: Some({})", +- definition.timeout_seconds +- ))); +- assert!(debug.contains(&format!("default_priority: Some({})", definition.priority))); +- assert!(debug.contains(&format!("is_enabled: Some({})", definition.enabled))); ++ let settings = contract.definition.shared_spec().settings(); ++ assert_eq!(settings.version, definition.version); ++ assert_eq!(settings.max_attempts, definition.max_attempts); ++ assert_eq!(settings.default_timeout_seconds, definition.timeout_seconds); ++ assert_eq!(settings.default_priority, definition.priority); ++ assert_eq!(settings.is_enabled, definition.enabled); + } + } + +diff --git a/crates/identitypro-jobs/src/identity_protect_alert.rs b/crates/identitypro-jobs/src/identity_protect_alert.rs +index 97ac44b..59d067f 100644 +--- a/crates/identitypro-jobs/src/identity_protect_alert.rs ++++ b/crates/identitypro-jobs/src/identity_protect_alert.rs +@@ -444,7 +444,7 @@ mod tests { + JobCompletionDisposition::ContinueAfter(std::time::Duration::from_secs(60)) + ); + assert!( +- continuation.checkpoint.is_none(), ++ continuation.checkpoint_value().is_none(), + "capacity waiting must not enter the exhaustion phase" + ); + assert_eq!( +@@ -582,7 +582,7 @@ mod tests { + JobCompletionDisposition::ContinueAfter(Duration::ZERO) + ); + assert_eq!( +- hydration_phase(continuation.checkpoint.as_ref()).unwrap(), ++ hydration_phase(continuation.checkpoint_value()).unwrap(), + Some(IdentityProtectAlertHydrationCheckpoint::PersistExhaustion { + attempt_id: expected_attempt_id, + }) +@@ -612,7 +612,7 @@ mod tests { + JobCompletionDisposition::ContinueAfter(HYDRATION_CONTENTION_DELAY) + ); + assert!( +- continuation.checkpoint.is_none(), ++ continuation.checkpoint_value().is_none(), + "claim contention must not enter the exhaustion phase" + ); + } +@@ -642,7 +642,7 @@ mod tests { + continuation.disposition(), + JobCompletionDisposition::Succeed + ); +- assert!(continuation.checkpoint.is_none()); ++ assert!(continuation.checkpoint_value().is_none()); + assert_eq!(hydration.exhaustion_calls.load(Ordering::SeqCst), 1); + } + +@@ -717,7 +717,8 @@ mod tests { + JobCompletionDisposition::ContinueAfter(std::time::Duration::ZERO) + ); + let checkpoint = continuation +- .checkpoint ++ .checkpoint_value() ++ .cloned() + .expect("the continuation persists its exhaustion phase"); + assert_eq!( + hydration_phase(Some(&checkpoint)).unwrap(), diff --git a/docs/shared-job-specs.md b/docs/shared-job-specs.md new file mode 100644 index 0000000..b23e183 --- /dev/null +++ b/docs/shared-job-specs.md @@ -0,0 +1,80 @@ +# Shared producer and worker contracts + +Keep job identity, payload, and operational settings in a provider-free module +that both API and worker builds can import. `runledger-core` owns these contracts; +only the worker constructs provider clients. + +```rust +use runledger_core::jobs::{JobContract, JobDefinitionSettings, JobSpec, JobType}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +pub struct DeliveryPayload { + pub request_id: uuid::Uuid, +} + +pub struct Delivery; +impl JobContract for Delivery { + type Payload = DeliveryPayload; + + fn spec() -> JobSpec { + JobSpec::new(JobType::new("app.delivery")) + .expect("static job identity") + .with_settings(JobDefinitionSettings::new().max_attempts(5)) + .expect("static job settings") + } +} +``` + +A producer builds `Delivery::submit(&payload)?.idempotency_key(key)` and borrows +it with `JobEnqueue::from(&submission)` for any existing PostgreSQL enqueue API. +`enqueue_job_with_outcome(&pool, &request)` commits its own transaction and returns +`JobEnqueueDisposition::Inserted` or `Existing`. Count only `Inserted` when +reporting new work. Use `enqueue_job_with_outcome_tx` inside a caller transaction; +the pool convenience's returned status is an observation, not a retained lock. +Dynamic JSON callers can use `JobSpec::submit(value)` or the direct owned builder +`JobSubmission::new(job_type, value)`. Organization, priority, attempts, timeout, +schedule, idempotency key, and stage are fluent, explicit options. + +Build `JobSpecs::new([Delivery::spec(), ...])` once to reject duplicate identities. +For producer definition setup, materialize +`specs.iter().map(JobDefinitionUpsert::from).collect::>()` and use the +existing PostgreSQL synchronization operations: + +- `sync_catalog_job_definitions_tx` with + `PreserveExistingEnabledForEnabledDefinitions` preserves operator disables. +- `sync_catalog_job_definitions_exact_tx` takes an explicit owned job-type scope, + disables absent scoped definitions, and restores the supplied enabled states. + +These modes retain the existing schedule checks and transaction guarantees. +Specification-enabled checks do not bypass the database's operator-disable check. + +A worker implements `TypedJobHandler`, selects `type Contract = Delivery`, and +receives `DeliveryPayload` in `execute`. Register `handler.into_job_handler()`. +`JobCatalog::from_specs(&specs, handlers)` requires exactly one handler per spec, +including disabled specs that may still have queued work. It rejects missing, +unknown, and duplicate bindings before returning a catalog. The same catalog +schedule, retry-override, supervisor, and sync APIs remain available. For staged +adoption in an existing catalog, `try_handler_for_spec(&spec, handler)` validates +the pairing and applies the shared metadata. Existing JSON handlers also bind. + +The typed adapter decodes existing JSON at execution time. It does not wrap the +payload, add a version field, deny unknown fields, or upgrade rows. Choose Serde +attributes deliberately and retain decoding for every durable payload version +that can remain queued. The default malformed-payload result is terminal +`job.invalid_payload` with the static message `Job payload has an invalid shape.` +Override `malformed_payload` for application codes or sanitized diagnostics; +Serde errors can contain attacker-controlled values. Raw JSON still reaches +`on_dead_letter`, including malformed rows. `execute_with_services` can be +overridden to use the existing typed checkpoint and lease-fenced progress APIs. + +Definition versions describe operational metadata, not payload schema versions. +Defaults do **not** become implicit request overrides: changing a timeout or +attempt policy must not change an identical request's stored snapshot. If an +existing producer explicitly snapshots attempts or timeout, retain those explicit +builder options during migration. Also retain original scheduled timestamps and +payload serialization; changed keyed requests still fail with +`job.idempotency_conflict`. + +See the [IdentityPro and CreditKit migration pilots](shared-job-specs-migrations/README.md) +for concrete downstream conversions and reproduction commands. diff --git a/migrations/202609050001_job_summary_pagination.down.sql b/migrations/202609050001_job_summary_pagination.down.sql new file mode 100644 index 0000000..113842c --- /dev/null +++ b/migrations/202609050001_job_summary_pagination.down.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS idx_job_queue_created_id; +DROP INDEX IF EXISTS idx_job_queue_scope_created_id; diff --git a/migrations/202609050001_job_summary_pagination.up.sql b/migrations/202609050001_job_summary_pagination.up.sql new file mode 100644 index 0000000..597676f --- /dev/null +++ b/migrations/202609050001_job_summary_pagination.up.sql @@ -0,0 +1,9 @@ +-- Support compact pages with and without an exact organization scope. +-- Include the UUID tie-breaker so deep cursors constrain the full ordering key. +CREATE INDEX idx_job_queue_scope_created_id + ON job_queue (organization_id, created_at DESC, id DESC); +CREATE INDEX idx_job_queue_created_id + ON job_queue (created_at DESC, id DESC); + +-- Additive indexes do not change the persisted contract. Omit this migration +-- from runledger_migration_history so older filtered startup helpers coexist. diff --git a/runledger-core/src/jobs.rs b/runledger-core/src/jobs.rs index 3743229..67c8158 100644 --- a/runledger-core/src/jobs.rs +++ b/runledger-core/src/jobs.rs @@ -6,21 +6,26 @@ //! reusable active keys, execution-resource keys, and the explicit //! handler-continuation opt-in consumed by `runledger-postgres`. +mod execution; mod handler; mod identifier_macros; mod identifiers; mod runtime_types; +mod spec; mod status; +mod submission; +mod typed_handler; mod workflow_enqueue; -pub use handler::{JobHandler, JobHandlerRegistry}; +pub use execution::{JobExecution, JobExecutionError, JobExecutionServices, JobExecutionUpdate}; +pub use handler::{ExecutionHandlerAdapter, JobExecutionHandler, JobHandler, JobHandlerRegistry}; pub use identifiers::{ IdentifierValidationError, JobType, JobTypeName, StepKey, StepKeyName, WorkflowType, WorkflowTypeName, }; pub use runtime_types::{ JobCompletion, JobCompletionDisposition, JobContext, JobDeadLetterInfo, JobDeadLetterReason, - JobFailure, JobProgressValidationError, JobRetryTiming, + JobFailure, JobProgressValidationError, JobRetryTiming, validate_job_progress, }; pub use status::{ JobEventType, JobFailureKind, JobStage, JobStatus, WorkflowRunStatus, WorkflowStepStatus, @@ -33,3 +38,7 @@ pub use workflow_enqueue::{ WorkflowStepExecution, WorkflowStepExecutionKind, validate_workflow_dag, validate_workflow_run_enqueue, validate_workflow_step_append, }; + +pub use spec::{JobDefinitionSettings, JobSpec, JobSpecError, JobSpecs}; +pub use submission::{JobContract, JobSubmission, JobSubmissionError}; +pub use typed_handler::{TypedHandlerAdapter, TypedJobHandler, malformed_job_payload}; diff --git a/runledger-core/src/jobs/execution.rs b/runledger-core/src/jobs/execution.rs new file mode 100644 index 0000000..539361f --- /dev/null +++ b/runledger-core/src/jobs/execution.rs @@ -0,0 +1,185 @@ +//! Optional live execution services, separate from the serializable job context. + +use std::time::{Duration, Instant}; + +use async_trait::async_trait; +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::Value; + +use super::{JobContext, JobFailure, JobProgressValidationError, validate_job_progress}; + +/// An atomic ordinary-progress/checkpoint update. Omitted fields retain their +/// durable values. This cannot change job stage, lease identity, or final output. +#[derive(Debug, Default, Clone, Copy)] +pub struct JobExecutionUpdate<'a> { + pub progress_done: Option, + pub progress_total: Option, + pub checkpoint: Option<&'a Value>, +} + +/// A live execution operation failed. +#[derive(Debug)] +#[non_exhaustive] +pub enum JobExecutionError { + /// The exact run/attempt/worker no longer holds a live lease. + LeaseLost, + /// The runtime's handler deadline has elapsed. + DeadlineElapsed, + /// Persistence did not acknowledge a successful commit. + PersistenceFailed, + InvalidProgress(JobProgressValidationError), + InvalidCheckpoint(serde_json::Error), +} + +impl std::fmt::Display for JobExecutionError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::LeaseLost => f.write_str("Job lease ownership was lost."), + Self::DeadlineElapsed => f.write_str("Job execution deadline elapsed."), + Self::PersistenceFailed => f.write_str("Job progress commit was not acknowledged."), + Self::InvalidProgress(error) => error.fmt(f), + Self::InvalidCheckpoint(error) => error.fmt(f), + } + } +} + +impl std::error::Error for JobExecutionError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::InvalidProgress(error) => Some(error), + Self::InvalidCheckpoint(error) => Some(error), + _ => None, + } + } +} + +impl From for JobFailure { + fn from(error: JobExecutionError) -> Self { + match error { + JobExecutionError::LeaseLost => { + Self::lease_expired("job.lease_owner_mismatch", error.to_string()) + } + JobExecutionError::DeadlineElapsed => { + Self::timeout("job.timeout_exceeded", error.to_string()) + } + JobExecutionError::PersistenceFailed => { + Self::retryable("job.progress_persist_failed", error.to_string()) + } + JobExecutionError::InvalidProgress(_) => { + Self::terminal("job.invalid_progress", error.to_string()) + } + JobExecutionError::InvalidCheckpoint(_) => { + Self::terminal("job.invalid_checkpoint", error.to_string()) + } + } + } +} + +/// Runtime implementation of services for one exact execution. +/// +/// Custom runtimes must bind writes to the context's live lease, return success +/// only after commit, and use the same deadline and clock as handler timeout +/// enforcement. Implementations must not extend that deadline on progress writes. +#[async_trait] +pub trait JobExecutionServices: Send + Sync { + fn deadline(&self) -> Instant; + fn remaining_budget(&self) -> Duration; + async fn persist_progress( + &self, + update: JobExecutionUpdate<'_>, + ) -> Result<(), JobExecutionError>; +} + +/// Borrowed services for the current handler invocation. +/// +/// The checkpoint is the resume snapshot captured before handler execution. +/// Successful writes become visible to subsequent runs and dead-letter hooks; +/// they do not mutate this snapshot. The borrow prevents carrying this handle +/// into a detached task beyond the runtime-owned execution. +/// +/// The runtime still enforces timeout and lease loss. These services cannot +/// cancel external effects already issued by a handler. +#[derive(Clone, Copy)] +pub struct JobExecution<'a> { + context: &'a JobContext, + services: &'a dyn JobExecutionServices, +} + +impl<'a> JobExecution<'a> { + /// Binds an execution snapshot to services supplied by a custom runtime. + #[must_use] + pub fn new(context: &'a JobContext, services: &'a dyn JobExecutionServices) -> Self { + Self { context, services } + } + + #[must_use] + pub fn context(&self) -> &JobContext { + self.context + } + + /// The authoritative monotonic handler deadline, including time spent + /// awaiting progress writes. Completion persistence occurs after execution. + /// The Runledger worker accepts a handler result only when observed strictly + /// before this instant. At or after it, timeout takes precedence over success + /// and continuation, even when the handler and timer become ready together. + /// Committed checkpoints and external effects are not undone by timeout. + #[must_use] + pub fn deadline(&self) -> Instant { + self.services.deadline() + } + + #[must_use] + pub fn remaining_budget(&self) -> Duration { + self.services.remaining_budget() + } + + /// Time available for application work after reserving time inside the + /// handler for its final checkpoint or cleanup. Saturates at zero. + #[must_use] + pub fn remaining_work_budget(&self, reserve: Duration) -> Duration { + self.remaining_budget().saturating_sub(reserve) + } + + #[must_use] + pub fn checkpoint_value(&self) -> Option<&Value> { + self.context.checkpoint.as_ref() + } + + /// Decodes the resume snapshot. Applications still own checkpoint versions + /// and domain validation; an absent checkpoint is distinct from malformed JSON. + pub fn checkpoint(&self) -> Result, serde_json::Error> { + self.checkpoint_value() + .cloned() + .map(serde_json::from_value) + .transpose() + } + + /// Awaits an atomic, lease-fenced commit of ordinary progress and checkpoint. + /// + /// Ok means the transaction committed. An error or cancellation does not + /// prove a write was absent: the connection can fail after the server commits. + /// Omitted fields retain existing values; a JSON null checkpoint is a value. + /// The application must make replay of external effects safe independently. + pub async fn persist_progress( + &self, + update: JobExecutionUpdate<'_>, + ) -> Result<(), JobExecutionError> { + validate_job_progress(update.progress_done, update.progress_total) + .map_err(JobExecutionError::InvalidProgress)?; + self.services.persist_progress(update).await + } + + /// Serializes and durably commits a checkpoint without changing progress. + pub async fn save_checkpoint( + &self, + checkpoint: &T, + ) -> Result<(), JobExecutionError> { + let checkpoint = + serde_json::to_value(checkpoint).map_err(JobExecutionError::InvalidCheckpoint)?; + self.persist_progress(JobExecutionUpdate { + checkpoint: Some(&checkpoint), + ..Default::default() + }) + .await + } +} diff --git a/runledger-core/src/jobs/handler.rs b/runledger-core/src/jobs/handler.rs index 9f1d841..54abb92 100644 --- a/runledger-core/src/jobs/handler.rs +++ b/runledger-core/src/jobs/handler.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use serde_json::Value; use super::identifiers::JobType; -use super::{JobCompletion, JobContext, JobDeadLetterInfo, JobFailure}; +use super::{JobCompletion, JobContext, JobDeadLetterInfo, JobExecution, JobFailure}; #[async_trait] pub trait JobHandler: Send + Sync { @@ -18,6 +18,16 @@ pub trait JobHandler: Send + Sync { payload: Value, ) -> Result; + /// Runtime dispatch with live execution services. Legacy handlers retain + /// their existing execute contract through this default implementation. + async fn execute_with_services( + &self, + execution: JobExecution<'_>, + payload: Value, + ) -> Result { + self.execute(execution.context().clone(), payload).await + } + async fn on_dead_letter( &self, _context: JobContext, @@ -38,3 +48,75 @@ pub trait JobHandlerRegistry { self.register_boxed(Arc::new(handler)); } } + +/// Opt-in handler contract with runtime-owned deadline and persistence services. +/// +/// Register `handler.into_job_handler()` with an existing registry or catalog. +/// Legacy `JobHandler` implementations do not need to change. +#[async_trait] +pub trait JobExecutionHandler: Send + Sync { + fn job_type(&self) -> JobType<'static>; + + async fn execute( + &self, + execution: JobExecution<'_>, + payload: Value, + ) -> Result; + + async fn on_dead_letter( + &self, + _context: JobContext, + _payload: Value, + _dead_letter: JobDeadLetterInfo, + ) { + } + + fn into_job_handler(self) -> ExecutionHandlerAdapter + where + Self: Sized, + { + ExecutionHandlerAdapter(self) + } +} + +/// Binds an execution-services handler to existing registries and catalogs. +/// +/// Custom runtimes must call `JobHandler::execute_with_services`. Calling the +/// legacy `execute` directly returns `job.execution_services_required`; it +/// cannot fabricate a deadline or a live lease for the adapted handler. +pub struct ExecutionHandlerAdapter(H); + +#[async_trait] +impl JobHandler for ExecutionHandlerAdapter { + fn job_type(&self) -> JobType<'static> { + self.0.job_type() + } + + async fn execute( + &self, + _context: JobContext, + _payload: Value, + ) -> Result { + Err(JobFailure::terminal( + "job.execution_services_required", + "This handler requires runtime-owned execution services.", + )) + } + + async fn execute_with_services( + &self, + execution: JobExecution<'_>, + payload: Value, + ) -> Result { + self.0.execute(execution, payload).await + } + + async fn on_dead_letter( + &self, + context: JobContext, + payload: Value, + dead_letter: JobDeadLetterInfo, + ) { + self.0.on_dead_letter(context, payload, dead_letter).await; + } +} diff --git a/runledger-core/src/jobs/runtime_types.rs b/runledger-core/src/jobs/runtime_types.rs index 62c0235..8b592e0 100644 --- a/runledger-core/src/jobs/runtime_types.rs +++ b/runledger-core/src/jobs/runtime_types.rs @@ -56,6 +56,32 @@ impl std::fmt::Display for JobProgressValidationError { impl std::error::Error for JobProgressValidationError {} +/// Validates the supplied progress values against the durable nonnegative and +/// done-at-most-total invariant. Missing values impose no constraint here. +/// Persistence implementations must first merge partial updates with the locked +/// durable row, then validate the effective values with this same function. +pub const fn validate_job_progress( + progress_done: Option, + progress_total: Option, +) -> Result<(), JobProgressValidationError> { + if let Some(actual) = progress_done + && actual < 0 + { + return Err(JobProgressValidationError::NegativeDone { actual }); + } + if let Some(actual) = progress_total + && actual < 0 + { + return Err(JobProgressValidationError::NegativeTotal { actual }); + } + if let (Some(done), Some(total)) = (progress_done, progress_total) + && done > total + { + return Err(JobProgressValidationError::DoneExceedsTotal { done, total }); + } + Ok(()) +} + /// Validated progress reported by a job handler. #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct JobProgress { @@ -66,14 +92,9 @@ struct JobProgress { impl JobProgress { /// Creates progress after validating its durable invariants. const fn new(done: i64, total: i64) -> Result { - if done < 0 { - return Err(JobProgressValidationError::NegativeDone { actual: done }); - } - if total < 0 { - return Err(JobProgressValidationError::NegativeTotal { actual: total }); - } - if done > total { - return Err(JobProgressValidationError::DoneExceedsTotal { done, total }); + match validate_job_progress(Some(done), Some(total)) { + Ok(()) => {} + Err(error) => return Err(error), } Ok(Self { done, total }) } diff --git a/runledger-core/src/jobs/spec.rs b/runledger-core/src/jobs/spec.rs new file mode 100644 index 0000000..46fa8d4 --- /dev/null +++ b/runledger-core/src/jobs/spec.rs @@ -0,0 +1,191 @@ +use std::collections::BTreeMap; + +use super::{JobType, JobTypeName}; + +/// Default values applied when syncing catalog jobs to `job_definitions`. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct JobDefinitionSettings { + /// Definition version written for catalog jobs. + pub version: i32, + /// Default maximum attempts written for catalog jobs. + pub max_attempts: i32, + /// Default execution timeout, in seconds, written for catalog jobs. + pub default_timeout_seconds: i32, + /// Default queue priority written for catalog jobs. + pub default_priority: i32, + /// Whether catalog jobs should be synced as enabled. + pub is_enabled: bool, +} + +impl Default for JobDefinitionSettings { + fn default() -> Self { + Self::new() + } +} + +impl JobDefinitionSettings { + /// Creates the default catalog definition values. + #[must_use] + pub const fn new() -> Self { + Self { + version: 1, + max_attempts: 3, + default_timeout_seconds: 300, + default_priority: 0, + is_enabled: true, + } + } + + /// Sets the definition version written for catalog jobs. + #[must_use] + pub const fn version(mut self, version: i32) -> Self { + self.version = version; + self + } + + /// Sets the default maximum attempts written for catalog jobs. + #[must_use] + pub const fn max_attempts(mut self, max_attempts: i32) -> Self { + self.max_attempts = max_attempts; + self + } + + /// Sets the default execution timeout, in seconds, written for catalog jobs. + #[must_use] + pub const fn timeout_seconds(mut self, default_timeout_seconds: i32) -> Self { + self.default_timeout_seconds = default_timeout_seconds; + self + } + + /// Sets the default queue priority written for catalog jobs. + #[must_use] + pub const fn priority(mut self, default_priority: i32) -> Self { + self.default_priority = default_priority; + self + } + + /// Sets whether catalog jobs should be synced as enabled. + #[must_use] + pub const fn enabled(mut self, is_enabled: bool) -> Self { + self.is_enabled = is_enabled; + self + } + + pub fn validate(self) -> Result<(), &'static str> { + if self.version <= 0 { + return Err("version"); + } + if self.max_attempts <= 0 { + return Err("max_attempts"); + } + if self.default_timeout_seconds <= 0 { + return Err("default_timeout_seconds"); + } + // default_priority intentionally accepts zero and negative values. + Ok(()) + } +} + +/// Storage-independent definition shared by producers and worker bindings. +/// +/// Definition version and settings are operational metadata. They are never +/// injected into payloads or enqueue request snapshots. Applications own any +/// durable payload versioning and must retain decoding for queued older rows. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct JobSpec { + job_type: JobType<'static>, + settings: JobDefinitionSettings, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum JobSpecError { + InvalidJobType, + InvalidSetting(&'static str), + DuplicateJobType(String), + DisabledJobType(String), +} + +impl std::fmt::Display for JobSpecError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::InvalidJobType => f.write_str("job type must be non-empty"), + Self::InvalidSetting(field) => write!(f, "invalid job definition setting: {field}"), + Self::DuplicateJobType(name) => write!(f, "duplicate job specification: {name}"), + Self::DisabledJobType(name) => write!(f, "job specification is disabled: {name}"), + } + } +} + +impl std::error::Error for JobSpecError {} + +impl JobSpec { + pub fn new(job_type: JobType<'static>) -> Result { + JobTypeName::new(job_type.as_str()).map_err(|_| JobSpecError::InvalidJobType)?; + Ok(Self { + job_type, + settings: JobDefinitionSettings::default(), + }) + } + + pub fn with_settings(mut self, settings: JobDefinitionSettings) -> Result { + settings.validate().map_err(JobSpecError::InvalidSetting)?; + self.settings = settings; + Ok(self) + } + + #[must_use] + pub fn job_type(&self) -> JobType<'static> { + self.job_type + } + + #[must_use] + pub fn settings(&self) -> JobDefinitionSettings { + self.settings + } + + /// Builds a JSON request without implicitly snapshotting definition settings. + pub fn submit(&self, payload: serde_json::Value) -> Result { + self.require_enabled()?; + Ok(super::JobSubmission::new(self.job_type, payload)) + } + + /// Checks code configuration; persistence still enforces operator disables. + pub fn require_enabled(&self) -> Result<(), JobSpecError> { + if self.settings.is_enabled { + Ok(()) + } else { + Err(JobSpecError::DisabledJobType( + self.job_type.as_str().to_owned(), + )) + } + } +} + +/// A validated producer-side collection requiring no handler or provider clients. +#[derive(Debug, Clone, Default)] +pub struct JobSpecs(BTreeMap); + +impl JobSpecs { + pub fn new(specs: impl IntoIterator) -> Result { + let mut entries = BTreeMap::new(); + for spec in specs { + let key = JobTypeName::new(spec.job_type.as_str()) + .map_err(|_| JobSpecError::InvalidJobType)?; + if entries.insert(key, spec).is_some() { + return Err(JobSpecError::DuplicateJobType( + spec.job_type.as_str().to_owned(), + )); + } + } + Ok(Self(entries)) + } + + pub fn iter(&self) -> impl ExactSizeIterator { + self.0.values() + } + + pub fn get(&self, job_type: JobType<'_>) -> Option<&JobSpec> { + self.0.get(job_type.as_str()) + } +} diff --git a/runledger-core/src/jobs/submission.rs b/runledger-core/src/jobs/submission.rs new file mode 100644 index 0000000..b7f2cb3 --- /dev/null +++ b/runledger-core/src/jobs/submission.rs @@ -0,0 +1,116 @@ +use chrono::{DateTime, Utc}; +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::Value; +use uuid::Uuid; + +use super::{JobSpec, JobSpecError, JobStage, JobType}; + +/// Associates a durable identity with its payload; no runtime clients are needed. +/// Serde attributes and application decoding policy define the wire format. +pub trait JobContract { + type Payload: Serialize + DeserializeOwned + Send; + + fn spec() -> JobSpec; + + fn submit(payload: &Self::Payload) -> Result { + let spec = Self::spec(); + spec.require_enabled().map_err(JobSubmissionError::Spec)?; + Ok(JobSubmission::new( + spec.job_type(), + serde_json::to_value(payload).map_err(JobSubmissionError::Serialize)?, + )) + } +} + +#[derive(Debug)] +pub enum JobSubmissionError { + Spec(JobSpecError), + Serialize(serde_json::Error), +} + +impl std::fmt::Display for JobSubmissionError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Spec(error) => error.fmt(f), + Self::Serialize(_) => f.write_str("could not serialize job payload"), + } + } +} + +impl std::error::Error for JobSubmissionError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(match self { + Self::Spec(error) => error, + Self::Serialize(error) => error, + }) + } +} + +/// Owned direct-job request. Only explicitly selected overrides enter the +/// idempotency snapshot; definition defaults are resolved by storage on insert. +#[derive(Debug, Clone)] +pub struct JobSubmission { + pub job_type: JobType<'static>, + pub payload: Value, + pub organization_id: Option, + pub priority: Option, + pub max_attempts: Option, + pub timeout_seconds: Option, + pub next_run_at: Option>, + pub idempotency_key: Option, + pub stage: Option, +} + +impl JobSubmission { + #[must_use] + pub fn new(job_type: JobType<'static>, payload: Value) -> Self { + Self { + job_type, + payload, + organization_id: None, + priority: None, + max_attempts: None, + timeout_seconds: None, + next_run_at: None, + idempotency_key: None, + stage: None, + } + } + + #[must_use] + pub fn organization_id(mut self, id: Uuid) -> Self { + self.organization_id = Some(id); + self + } + #[must_use] + pub fn priority(mut self, value: i32) -> Self { + self.priority = Some(value); + self + } + #[must_use] + pub fn max_attempts(mut self, value: i32) -> Self { + self.max_attempts = Some(value); + self + } + #[must_use] + pub fn timeout_seconds(mut self, value: i32) -> Self { + self.timeout_seconds = Some(value); + self + } + /// Keyed retries must reuse the original timestamp. + #[must_use] + pub fn next_run_at(mut self, value: DateTime) -> Self { + self.next_run_at = Some(value); + self + } + #[must_use] + pub fn idempotency_key(mut self, value: impl Into) -> Self { + self.idempotency_key = Some(value.into()); + self + } + #[must_use] + pub fn stage(mut self, value: JobStage) -> Self { + self.stage = Some(value); + self + } +} diff --git a/runledger-core/src/jobs/typed_handler.rs b/runledger-core/src/jobs/typed_handler.rs new file mode 100644 index 0000000..45a4b8c --- /dev/null +++ b/runledger-core/src/jobs/typed_handler.rs @@ -0,0 +1,89 @@ +use async_trait::async_trait; +use serde_json::Value; + +use super::{ + JobCompletion, JobContext, JobContract, JobDeadLetterInfo, JobExecution, JobFailure, + JobHandler, JobType, +}; + +/// Safe durable default: decoding details may contain input data and are never +/// included in the failure. Applications may use their own static code/message. +#[must_use] +pub fn malformed_job_payload() -> JobFailure { + JobFailure::terminal("job.invalid_payload", "Job payload has an invalid shape.") +} + +/// Opt-in typed execution over the existing JSON dispatch boundary. +#[async_trait] +pub trait TypedJobHandler: Send + Sync { + type Contract: JobContract; + + async fn execute( + &self, + context: JobContext, + payload: ::Payload, + ) -> Result; + + async fn execute_with_services( + &self, + execution: JobExecution<'_>, + payload: ::Payload, + ) -> Result { + self.execute(execution.context().clone(), payload).await + } + + /// Override for application classification or safe diagnostics. The source + /// can contain payload data; do not persist or log it without sanitizing. + fn malformed_payload(&self, _source: &serde_json::Error) -> JobFailure { + malformed_job_payload() + } + + /// Retains raw JSON so cleanup also runs for undecodable durable rows. + async fn on_dead_letter( + &self, + _context: JobContext, + _payload: Value, + _info: JobDeadLetterInfo, + ) { + } + + fn into_job_handler(self) -> TypedHandlerAdapter + where + Self: Sized, + { + TypedHandlerAdapter(self) + } +} + +pub struct TypedHandlerAdapter(H); + +#[async_trait] +impl JobHandler for TypedHandlerAdapter { + fn job_type(&self) -> JobType<'static> { + H::Contract::spec().job_type() + } + + async fn execute( + &self, + context: JobContext, + payload: Value, + ) -> Result { + let payload = + serde_json::from_value(payload).map_err(|error| self.0.malformed_payload(&error))?; + self.0.execute(context, payload).await + } + + async fn execute_with_services( + &self, + execution: JobExecution<'_>, + payload: Value, + ) -> Result { + let payload = + serde_json::from_value(payload).map_err(|error| self.0.malformed_payload(&error))?; + self.0.execute_with_services(execution, payload).await + } + + async fn on_dead_letter(&self, context: JobContext, payload: Value, info: JobDeadLetterInfo) { + self.0.on_dead_letter(context, payload, info).await; + } +} diff --git a/runledger-core/src/jobs/workflow_enqueue.rs b/runledger-core/src/jobs/workflow_enqueue.rs index 7ede4a0..2770211 100644 --- a/runledger-core/src/jobs/workflow_enqueue.rs +++ b/runledger-core/src/jobs/workflow_enqueue.rs @@ -5,9 +5,10 @@ //! up front, and lets the PostgreSQL workflow runtime release downstream steps //! as prerequisites finish. //! -//! For ordinary dependent work, prefer [`WorkflowDagBuilder`] for simple DAGs, or -//! [`WorkflowRunEnqueueBuilder`] and [`WorkflowStepEnqueueBuilder`] for advanced -//! per-step settings. Direct jobs are best for one independent unit of work. +//! For dependent work, prefer [`WorkflowDagBuilder`]. Compose configured +//! [`WorkflowStepEnqueueBuilder`] results with [`WorkflowDagBuilder::step`] for +//! advanced per-step policies. [`WorkflowRunEnqueueBuilder`] remains available +//! for assembling step collections directly. Direct jobs suit independent work. mod build_validation; mod dag_builder; diff --git a/runledger-core/src/jobs/workflow_enqueue/dag_builder.rs b/runledger-core/src/jobs/workflow_enqueue/dag_builder.rs index 776cac7..f3f5306 100644 --- a/runledger-core/src/jobs/workflow_enqueue/dag_builder.rs +++ b/runledger-core/src/jobs/workflow_enqueue/dag_builder.rs @@ -1,28 +1,26 @@ use uuid::Uuid; use super::super::identifiers::{StepKey, WorkflowType}; +use super::build_validation::validate_step_enqueue; use super::errors::WorkflowBuildError; use super::run_builder::WorkflowRunEnqueueBuilder; use super::step_builder::WorkflowStepEnqueueBuilder; -use super::types::WorkflowRunEnqueue; - -#[derive(Debug, Clone)] -struct StepSlot<'a> { - step_key: StepKey<'a>, - builder: Option>, -} +use super::types::{ + WorkflowDependencyReleaseMode, WorkflowRunEnqueue, WorkflowStepDependencySpec, + WorkflowStepEnqueue, +}; /// High-level builder for workflow DAG enqueue payloads. /// -/// Use this builder for common workflows expressed as a fluent chain of jobs and -/// dependency edges. For per-step priority, attempts, timeout, stage, external -/// steps, or hand-authored dependency specs, use -/// [`WorkflowStepEnqueueBuilder`] and [`WorkflowRunEnqueueBuilder`] instead. +/// Compose jobs, external steps, and dependency edges in a fluent chain. Use +/// [`Self::step`] with a validated [`WorkflowStepEnqueueBuilder`] result for +/// per-step organizations, queue settings, continuations, execution resources, +/// or hand-authored dependencies. The lower-level builders remain available. /// /// This helper accepts raw string identifiers for ergonomics. It validates the /// workflow shape before enqueueing, but it does not prove at compile time that /// a job type is registered with storage or with a runtime handler. Use the -/// lower-level builders when you want call sites to pass explicit [`StepKey`] +/// [`Self::step`] with the low-level step builder for explicit [`StepKey`] /// and [`JobType`](crate::jobs::JobType) values. /// /// # Validation Timing @@ -33,7 +31,9 @@ struct StepSlot<'a> { /// | [`Self::try_new`] | blank workflow type | empty step list and dependency graph errors | /// | [`Self::job`] | blank step key, blank job type, duplicate step key | job type registration is not checked by this builder | /// | [`Self::after_success`] / [`Self::after_terminal`] | blank target step key, blank prerequisite step key, unknown target step | missing prerequisite step, self-dependency, duplicate dependency, cycle | +/// | [`Self::step`] / [`Self::external`] | invalid or duplicate step key (configured steps are already shape-validated) | dependency graph errors | /// | [`Self::idempotency_key`] | never | blank idempotency key | +/// | [`Self::active_key`] | never | blank key or key longer than 512 bytes | /// /// # Examples /// ```rust @@ -63,8 +63,9 @@ pub struct WorkflowDagBuilder<'a> { organization_id: Option, metadata: &'a serde_json::Value, idempotency_key: Option<&'a str>, + active_key: Option<&'a str>, result_step_key: Option>, - steps: Vec>, + steps: Vec>, } impl<'a> WorkflowDagBuilder<'a> { @@ -79,6 +80,7 @@ impl<'a> WorkflowDagBuilder<'a> { organization_id: None, metadata, idempotency_key: None, + active_key: None, result_step_key: None, steps: Vec::new(), } @@ -99,6 +101,7 @@ impl<'a> WorkflowDagBuilder<'a> { organization_id: None, metadata, idempotency_key: None, + active_key: None, result_step_key: None, steps: Vec::new(), }) @@ -132,6 +135,25 @@ impl<'a> WorkflowDagBuilder<'a> { self } + /// Sets a reusable coordination key for one active workflow cycle. + /// + /// The key is shared across workflow types in the same organization/global + /// scope. It must be non-blank and at most 512 bytes, checked at build time. + /// Use the request with `enqueue_or_get_active_workflow`; this key is + /// independent of permanent request idempotency. + #[must_use] + pub fn active_key(mut self, active_key: &'a str) -> Self { + self.active_key = Some(active_key); + self + } + + /// Clears the reusable active workflow key. + #[must_use] + pub fn clear_active_key(mut self) -> Self { + self.active_key = None; + self + } + /// Declares the step whose successful output becomes the workflow result. /// /// # Errors @@ -164,36 +186,78 @@ impl<'a> WorkflowDagBuilder<'a> { job_type: &'a str, payload: &'a serde_json::Value, ) -> Result { - let validated_step_key = StepKey::try_new(step_key) + self.check_new_step_key(step_key)?; + let step = WorkflowStepEnqueueBuilder::try_new(step_key, job_type, payload)?.try_build()?; + self.steps.push(step); + Ok(self) + } + + /// Adds a configured job or external step built with [`WorkflowStepEnqueueBuilder`]. + /// + /// Preserves all step settings and dependencies. More edges can be appended + /// with [`Self::after_success`] or [`Self::after_terminal`]. Prerequisites + /// may be added later; the complete graph is validated at build time. + /// + /// # Errors + /// Returns [`WorkflowBuildError::DuplicateStepKey`] if the step already exists. + /// + /// # Examples + /// ```rust + /// use runledger_core::jobs::{WorkflowDagBuilder, WorkflowStepEnqueueBuilder}; + /// let payload = serde_json::json!({"account": "a"}); + /// let run = WorkflowDagBuilder::new("enrichment", &payload) + /// .active_key("enrichment:active") + /// .step(WorkflowStepEnqueueBuilder::try_new("account", "enrich", &payload)? + /// .allow_handler_continuation() + /// .execution_resource("provider") + /// .try_build()?)? + /// .external("approval", &payload)? + /// .after_success("approval", ["account"])? + /// .build()?; + /// assert!(run.steps()[0].allows_handler_continuation()); + /// # Ok::<_, runledger_core::jobs::WorkflowBuildError>(()) + /// ``` + pub fn step(mut self, step: WorkflowStepEnqueue<'a>) -> Result { + self.check_new_step_key(step.step_key().as_str())?; + self.steps.push(step); + Ok(self) + } + + /// Adds a step completed by an external actor, without queued-job settings. + /// + /// Use [`Self::step`] to supply a configured external step. + /// + /// # Errors + /// Returns [`WorkflowBuildError`] for a blank or duplicate step key. + pub fn external( + self, + step_key: &'a str, + payload: &'a serde_json::Value, + ) -> Result { + self.step(WorkflowStepEnqueueBuilder::try_new_external(step_key, payload)?.try_build()?) + } + + fn check_new_step_key(&self, step_key: &str) -> Result<(), WorkflowBuildError> { + let step_key = StepKey::try_new(step_key) .map_err(|_| WorkflowBuildError::BlankStepKey { step_index: None })?; - if self - .steps - .iter() - .any(|slot| slot.step_key == validated_step_key) - { + if self.steps.iter().any(|step| step.step_key() == step_key) { return Err(WorkflowBuildError::DuplicateStepKey { - step_key: validated_step_key.as_str().to_owned(), + step_key: step_key.as_str().to_owned(), }); } - - let builder = WorkflowStepEnqueueBuilder::try_new(step_key, job_type, payload)?; - self.steps.push(StepSlot { - step_key: validated_step_key, - builder: Some(builder), - }); - Ok(self) + Ok(()) } /// Adds success-only dependencies to an existing step. /// - /// The target `step_key` must already have been added with [`Self::job`]. + /// The target `step_key` must already have been added. /// Prerequisite step keys may be added later in the chain, but every /// prerequisite must exist before [`Self::build`] or [`Self::try_build`] /// succeeds. /// /// # Errors /// Returns [`WorkflowBuildError`] when the target or any prerequisite step key - /// is blank, when the target step was not added with [`Self::job`], or when + /// is blank, when the target step has not been added, or when /// dependency validation fails at build time. pub fn after_success( self, @@ -206,20 +270,20 @@ impl<'a> WorkflowDagBuilder<'a> { self.after( step_key, prerequisites, - WorkflowStepEnqueueBuilder::depends_on_success, + WorkflowDependencyReleaseMode::OnSuccess, ) } /// Adds terminal-state dependencies to an existing step. /// - /// The target `step_key` must already have been added with [`Self::job`]. + /// The target `step_key` must already have been added. /// Prerequisite step keys may be added later in the chain, but every /// prerequisite must exist before [`Self::build`] or [`Self::try_build`] /// succeeds. /// /// # Errors /// Returns [`WorkflowBuildError`] when the target or any prerequisite step key - /// is blank, when the target step was not added with [`Self::job`], or when + /// is blank, when the target step has not been added, or when /// dependency validation fails at build time. pub fn after_terminal( self, @@ -232,19 +296,18 @@ impl<'a> WorkflowDagBuilder<'a> { self.after( step_key, prerequisites, - WorkflowStepEnqueueBuilder::depends_on_terminal, + WorkflowDependencyReleaseMode::OnTerminal, ) } - fn after( + fn after( mut self, step_key: &'a str, prerequisites: I, - attach: F, + release_mode: WorkflowDependencyReleaseMode, ) -> Result where I: IntoIterator, - F: FnOnce(WorkflowStepEnqueueBuilder<'a>, &[StepKey<'a>]) -> WorkflowStepEnqueueBuilder<'a>, { let target_step_key = StepKey::try_new(step_key) .map_err(|_| WorkflowBuildError::BlankStepKey { step_index: None })?; @@ -261,25 +324,28 @@ impl<'a> WorkflowDagBuilder<'a> { }) .collect::, _>>()?; - let slot = self + let step = self .steps .iter_mut() - .find(|slot| slot.step_key == target_step_key) + .find(|step| step.step_key() == target_step_key) .ok_or(WorkflowBuildError::UnknownStepKey { step_key: target_step_key_string, })?; - let builder = slot - .builder - .take() - .expect("step builder is present until try_build consumes it"); - slot.builder = Some(attach(builder, &prerequisite_step_keys)); + step.dependencies.extend( + prerequisite_step_keys + .into_iter() + .map(|prerequisite_step_key| WorkflowStepDependencySpec { + prerequisite_step_key, + release_mode: Some(release_mode), + }), + ); Ok(self) } /// Finalizes the builder and returns a validated [`WorkflowRunEnqueue`]. /// - /// This validates the workflow type, idempotency key, non-empty step list, + /// This validates the workflow type, idempotency and active keys, non-empty step list, /// per-step enqueue fields, missing prerequisite steps, duplicate /// dependencies, self-dependencies, and cycles. It does not check whether job /// types have registered storage definitions or runtime handlers. @@ -294,7 +360,7 @@ impl<'a> WorkflowDagBuilder<'a> { /// Finalizes the builder and returns a validated [`WorkflowRunEnqueue`]. /// - /// This validates the workflow type, idempotency key, non-empty step list, + /// This validates the workflow type, idempotency and active keys, non-empty step list, /// per-step enqueue fields, missing prerequisite steps, duplicate /// dependencies, self-dependencies, and cycles. It does not check whether job /// types have registered storage definitions or runtime handlers. @@ -304,13 +370,9 @@ impl<'a> WorkflowDagBuilder<'a> { /// keys are invalid, dependencies reference missing steps, or the dependency /// graph contains a cycle. pub fn try_build(self) -> Result, WorkflowBuildError> { - let mut built_steps = Vec::with_capacity(self.steps.len()); - for mut slot in self.steps { - let builder = slot - .builder - .take() - .expect("step builder is present until try_build consumes it"); - built_steps.push(builder.try_build()?); + // Preserve per-step error precedence before validating run fields. + for step in &self.steps { + validate_step_enqueue(step)?; } let mut run_builder = WorkflowRunEnqueueBuilder::new(self.workflow_type, self.metadata); @@ -320,10 +382,13 @@ impl<'a> WorkflowDagBuilder<'a> { if let Some(idempotency_key) = self.idempotency_key { run_builder = run_builder.idempotency_key(idempotency_key); } + if let Some(active_key) = self.active_key { + run_builder = run_builder.active_key(active_key); + } if let Some(result_step_key) = self.result_step_key { run_builder = run_builder.result_step_key(result_step_key); } - run_builder.extend_steps(built_steps).try_build() + run_builder.extend_steps(self.steps).try_build() } } diff --git a/runledger-core/src/jobs/workflow_enqueue/tests.rs b/runledger-core/src/jobs/workflow_enqueue/tests.rs index 3fa05fa..225e19b 100644 --- a/runledger-core/src/jobs/workflow_enqueue/tests.rs +++ b/runledger-core/src/jobs/workflow_enqueue/tests.rs @@ -1,3 +1,4 @@ mod builders; mod dag_validation; mod errors; +mod fluent_policies; diff --git a/runledger-core/src/jobs/workflow_enqueue/tests/fluent_policies.rs b/runledger-core/src/jobs/workflow_enqueue/tests/fluent_policies.rs new file mode 100644 index 0000000..609f70d --- /dev/null +++ b/runledger-core/src/jobs/workflow_enqueue/tests/fluent_policies.rs @@ -0,0 +1,255 @@ +use serde_json::json; +use uuid::Uuid; + +use crate::jobs::{ + JobStage, StepKey, WorkflowBuildError, WorkflowDagBuilder, WorkflowDependencyReleaseMode, + WorkflowStepDependencySpec, WorkflowStepEnqueueBuilder, WorkflowStepExecution, +}; + +#[test] +fn configured_steps_preserve_all_fields_and_append_edges() { + let payload = json!({"account": "a"}); + let organization = Uuid::from_u128(42); + let step = WorkflowStepEnqueueBuilder::try_new("work", "enrich", &payload) + .expect("valid workflow input") + .organization_id(organization) + .priority(8) + .max_attempts(3) + .timeout_seconds(90) + .stage(JobStage::Scheduled) + .allow_handler_continuation() + .execution_resource("provider") + .dependency(WorkflowStepDependencySpec { + prerequisite_step_key: StepKey::new("approval"), + release_mode: None, + }) + .try_build() + .expect("valid workflow input"); + let run = WorkflowDagBuilder::new("workflow", &payload) + .step(step) + .expect("valid workflow input") + .external("approval", &payload) + .expect("valid workflow input") + .job("fetch", "fetch", &payload) + .expect("valid workflow input") + .after_success("work", ["fetch"]) + .expect("valid workflow input") + .result_step("work") + .expect("valid workflow input") + .build() + .expect("valid workflow input"); + let step = &run.steps()[0]; + assert_eq!(step.organization_id(), Some(organization)); + assert_eq!(step.payload(), &payload); + assert_eq!( + step.job_type().expect("valid workflow input").as_str(), + "enrich" + ); + assert_eq!(step.priority(), Some(8)); + assert_eq!(step.max_attempts(), Some(3)); + assert_eq!(step.timeout_seconds(), Some(90)); + assert_eq!(step.stage(), Some(JobStage::Scheduled)); + assert!(step.allows_handler_continuation()); + assert_eq!(step.execution_resource_key(), Some("provider")); + assert_eq!(step.dependencies().len(), 2); + assert_eq!(step.dependencies()[0].release_mode, None); + assert_eq!( + step.dependencies()[0].effective_release_mode(), + WorkflowDependencyReleaseMode::OnTerminal + ); + assert_eq!( + step.dependencies()[1].release_mode, + Some(WorkflowDependencyReleaseMode::OnSuccess) + ); + assert_eq!(run.result_step_key(), Some(StepKey::new("work"))); + assert_eq!(run.steps()[1].execution(), WorkflowStepExecution::External); +} + +#[test] +fn active_keys_validate_and_clear_independently_of_idempotency() { + let payload = json!({}); + let base = WorkflowDagBuilder::try_new("workflow", &payload) + .expect("valid workflow input") + .idempotency_key("request") + .external("approval", &payload) + .expect("valid workflow input"); + for blank in ["", " "] { + assert_eq!( + base.clone() + .active_key(blank) + .build() + .expect_err("invalid workflow input"), + WorkflowBuildError::BlankActiveKey + ); + } + let oversized = "é".repeat(257); + assert_eq!( + base.clone() + .active_key(&oversized) + .build() + .expect_err("invalid workflow input"), + WorkflowBuildError::ActiveKeyTooLong + ); + let boundary = "é".repeat(256); + let run = base + .clone() + .active_key(&boundary) + .build() + .expect("valid workflow input"); + assert_eq!(run.active_key(), Some(boundary.as_str())); + assert_eq!(run.idempotency_key(), Some("request")); + let run = base + .active_key(" ") + .clear_active_key() + .build() + .expect("valid workflow input"); + assert_eq!(run.active_key(), None); + assert_eq!(run.idempotency_key(), Some("request")); +} + +#[test] +fn external_steps_support_scope_dependencies_and_result_selection() { + let payload = json!({"ticket": 7}); + let organization = Uuid::from_u128(7); + let external = WorkflowStepEnqueueBuilder::try_new_external("approval", &payload) + .expect("valid workflow input") + .organization_id(organization) + .try_build() + .expect("valid workflow input"); + let run = WorkflowDagBuilder::new("workflow", &payload) + .step(external) + .expect("valid workflow input") + .job("work", "work", &payload) + .expect("valid workflow input") + .external("done", &payload) + .expect("valid workflow input") + .after_terminal("approval", ["work"]) + .expect("valid workflow input") + .after_success("done", ["approval"]) + .expect("valid workflow input") + .result_step("done") + .expect("valid workflow input") + .build() + .expect("valid workflow input"); + let step = &run.steps()[0]; + assert_eq!(step.organization_id(), Some(organization)); + assert_eq!(step.payload(), &payload); + assert_eq!(step.execution(), WorkflowStepExecution::External); + assert_eq!( + step.dependencies()[0].release_mode, + Some(WorkflowDependencyReleaseMode::OnTerminal) + ); + assert_eq!(run.result_step_key(), Some(StepKey::new("done"))); +} + +#[test] +fn all_entry_points_share_duplicate_key_detection() { + let payload = json!({}); + let configured = WorkflowStepEnqueueBuilder::try_new("step", "work", &payload) + .expect("valid workflow input") + .try_build() + .expect("valid workflow input"); + let bases = [ + WorkflowDagBuilder::new("workflow", &payload) + .job("step", "work", &payload) + .expect("valid workflow input"), + WorkflowDagBuilder::new("workflow", &payload) + .step(configured.clone()) + .expect("valid workflow input"), + WorkflowDagBuilder::new("workflow", &payload) + .external("step", &payload) + .expect("valid workflow input"), + ]; + for base in bases { + for result in [ + base.clone().job("step", "work", &payload), + base.clone().step(configured.clone()), + base.external("step", &payload), + ] { + assert_eq!( + result.expect_err("invalid workflow input"), + WorkflowBuildError::DuplicateStepKey { + step_key: "step".into() + } + ); + } + } + // Preserve the existing simple-job error precedence. + let error = WorkflowDagBuilder::new("workflow", &payload) + .external("step", &payload) + .expect("valid workflow input") + .job("step", " ", &payload) + .expect_err("invalid workflow input"); + assert!(matches!(error, WorkflowBuildError::DuplicateStepKey { .. })); + assert!(matches!( + WorkflowDagBuilder::new("workflow", &payload).external(" ", &payload), + Err(WorkflowBuildError::BlankStepKey { .. }) + )); +} + +#[test] +fn configured_dependencies_still_receive_complete_graph_validation() { + let payload = json!({}); + let configured = WorkflowStepEnqueueBuilder::try_new("work", "work", &payload) + .expect("valid workflow input") + .depends_on_terminal(&[StepKey::new("approval")]) + .try_build() + .expect("valid workflow input"); + let base = WorkflowDagBuilder::new("workflow", &payload) + .step(configured) + .expect("valid workflow input"); + assert!(matches!( + base.clone().build(), + Err(WorkflowBuildError::MissingDependency { .. }) + )); + let base = base + .external("approval", &payload) + .expect("valid workflow input"); + assert!(base.clone().build().is_ok()); + assert!(matches!( + base.clone() + .after_success("work", ["approval"]) + .expect("valid workflow input") + .build(), + Err(WorkflowBuildError::DuplicateDependency { .. }) + )); + assert!(matches!( + base.clone() + .after_terminal("approval", ["approval"]) + .expect("valid workflow input") + .build(), + Err(WorkflowBuildError::SelfDependency { .. }) + )); + assert!(matches!( + base.clone() + .after_success("approval", ["work"]) + .expect("valid workflow input") + .build(), + Err(WorkflowBuildError::CycleDetected) + )); + assert!(matches!( + base.result_step("missing") + .expect("valid workflow input") + .build(), + Err(WorkflowBuildError::UnknownResultStepKey { .. }) + )); +} + +#[test] +fn fluent_build_preserves_step_error_precedence_over_run_fields() { + let payload = json!({}); + let error = WorkflowDagBuilder::new(" ", &payload) + .idempotency_key(" ") + .job("work", "work", &payload) + .expect("valid job") + .after_success("work", ["work"]) + .expect("deferred self dependency") + .build() + .expect_err("self dependency precedes invalid run fields"); + assert_eq!( + error, + WorkflowBuildError::SelfDependency { + step_key: "work".into() + } + ); +} diff --git a/runledger-core/src/lib.rs b/runledger-core/src/lib.rs index a809580..cbaef9f 100644 --- a/runledger-core/src/lib.rs +++ b/runledger-core/src/lib.rs @@ -92,11 +92,13 @@ pub mod prelude { pub use async_trait::async_trait; pub use crate::jobs::{ - IdentifierValidationError, JobCompletion, JobCompletionDisposition, JobContext, - JobDeadLetterInfo, JobDeadLetterReason, JobEventType, JobFailure, JobFailureKind, - JobHandler, JobHandlerRegistry, JobProgressValidationError, JobRetryTiming, JobStage, - JobStatus, JobType, JobTypeName, StepKey, StepKeyName, WorkflowBuildError, - WorkflowDagBuilder, WorkflowDagDependencyValidationInput, WorkflowDagStepValidationInput, + ExecutionHandlerAdapter, IdentifierValidationError, JobCompletion, + JobCompletionDisposition, JobContext, JobDeadLetterInfo, JobDeadLetterReason, JobEventType, + JobExecution, JobExecutionError, JobExecutionHandler, JobExecutionServices, + JobExecutionUpdate, JobFailure, JobFailureKind, JobHandler, JobHandlerRegistry, + JobProgressValidationError, JobRetryTiming, JobStage, JobStatus, JobType, JobTypeName, + StepKey, StepKeyName, WorkflowBuildError, WorkflowDagBuilder, + WorkflowDagDependencyValidationInput, WorkflowDagStepValidationInput, WorkflowDagValidationError, WorkflowDependencyReleaseMode, WorkflowJobStepExecution, WorkflowRunEnqueue, WorkflowRunEnqueueBuilder, WorkflowRunStatus, WorkflowStepDependencySpec, WorkflowStepEnqueue, WorkflowStepEnqueueBuilder, diff --git a/runledger-core/tests/job_specs.rs b/runledger-core/tests/job_specs.rs new file mode 100644 index 0000000..53ca527 --- /dev/null +++ b/runledger-core/tests/job_specs.rs @@ -0,0 +1,88 @@ +use runledger_core::jobs::{ + JobContract, JobDefinitionSettings, JobSpec, JobSpecError, JobSpecs, JobSubmissionError, + JobType, +}; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +#[derive(Serialize, Deserialize)] +struct Payload { + request_id: String, +} +struct Email; +impl JobContract for Email { + type Payload = Payload; + fn spec() -> JobSpec { + JobSpec::new(JobType::new("email.send")) + .expect("static identity") + .with_settings(JobDefinitionSettings::new().version(4).max_attempts(7)) + .expect("static settings") + } +} + +#[test] +fn producer_contract_preserves_wire_payload_and_leaves_policy_out_of_request() { + let request = Email::submit(&Payload { + request_id: "old-row".into(), + }) + .expect("serialize"); + assert_eq!(request.job_type.as_str(), "email.send"); + assert_eq!(request.payload, json!({"request_id":"old-row"})); + assert_eq!(request.max_attempts, None); + assert_eq!(request.timeout_seconds, None); + assert_eq!(request.priority, None); + let spec = Email::spec() + .with_settings(JobDefinitionSettings::new().version(8).max_attempts(9)) + .expect("updated settings"); + let updated = spec.submit(request.payload.clone()).expect("submit"); + assert_eq!(updated.payload, request.payload); + assert_eq!(updated.max_attempts, request.max_attempts); +} + +#[test] +fn rejects_invalid_specs_duplicate_identities_and_disabled_submissions() { + assert_eq!( + JobSpec::new(JobType::new(" ")).expect_err("blank"), + JobSpecError::InvalidJobType + ); + for settings in [ + JobDefinitionSettings::new().version(0), + JobDefinitionSettings::new().max_attempts(0), + JobDefinitionSettings::new().timeout_seconds(-1), + ] { + assert!(Email::spec().with_settings(settings).is_err()); + } + assert!(matches!( + JobSpecs::new([Email::spec(), Email::spec()]), + Err(JobSpecError::DuplicateJobType(_)) + )); + let spec = Email::spec() + .with_settings(JobDefinitionSettings::new().enabled(false)) + .expect("disabled"); + assert!(matches!( + spec.submit(json!({})), + Err(JobSpecError::DisabledJobType(_)) + )); +} + +#[test] +fn serialization_failure_is_recoverable() { + #[derive(Deserialize)] + struct Unserializable; + impl Serialize for Unserializable { + fn serialize(&self, _: S) -> Result { + Err(serde::ser::Error::custom("cannot encode")) + } + } + struct Broken; + impl JobContract for Broken { + type Payload = Unserializable; + fn spec() -> JobSpec { + Email::spec() + } + } + assert!(matches!( + Broken::submit(&Unserializable), + Err(JobSubmissionError::Serialize(_)) + )); +} diff --git a/runledger-postgres/.sqlx/query-07c64366e2b4e12cd65cad0e26ff8e2442eafaa21bc515a90063c7750ed0a070.json b/runledger-postgres/.sqlx/query-07c64366e2b4e12cd65cad0e26ff8e2442eafaa21bc515a90063c7750ed0a070.json new file mode 100644 index 0000000..7da9f8b --- /dev/null +++ b/runledger-postgres/.sqlx/query-07c64366e2b4e12cd65cad0e26ff8e2442eafaa21bc515a90063c7750ed0a070.json @@ -0,0 +1,105 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, job_type, organization_id, status::text AS \"status!\",\n priority, run_number, attempt, max_attempts, next_run_at, stage,\n progress_done, progress_total, created_at, updated_at\n FROM job_queue WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type = $3) AND (created_at, id) < ($5, $6) ORDER BY created_at DESC, id DESC LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Timestamptz", + "Uuid" + ] + }, + "nullable": [ + false, + false, + true, + null, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + }, + "hash": "07c64366e2b4e12cd65cad0e26ff8e2442eafaa21bc515a90063c7750ed0a070" +} diff --git a/runledger-postgres/.sqlx/query-10269f3f8cd79707daaba3c26771d5aa97e7c9c08801cbfd18f70bbd8670e17a.json b/runledger-postgres/.sqlx/query-10269f3f8cd79707daaba3c26771d5aa97e7c9c08801cbfd18f70bbd8670e17a.json new file mode 100644 index 0000000..704da5f --- /dev/null +++ b/runledger-postgres/.sqlx/query-10269f3f8cd79707daaba3c26771d5aa97e7c9c08801cbfd18f70bbd8670e17a.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, payload FROM job_queue WHERE organization_id = $1 AND job_type = $2\n AND payload->>'run_id' = $3\n ORDER BY created_at DESC, id DESC\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "payload", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "10269f3f8cd79707daaba3c26771d5aa97e7c9c08801cbfd18f70bbd8670e17a" +} diff --git a/.sqlx/query-dbcca04872cf174ca1a23862f2f632fcf5da4599e75c8f21b1c134b34b1a876a.json b/runledger-postgres/.sqlx/query-1059ac0fa26c3192fe4160b03fe6f8fadf3727d67e102953a0793af534de602a.json similarity index 83% rename from .sqlx/query-dbcca04872cf174ca1a23862f2f632fcf5da4599e75c8f21b1c134b34b1a876a.json rename to runledger-postgres/.sqlx/query-1059ac0fa26c3192fe4160b03fe6f8fadf3727d67e102953a0793af534de602a.json index b05c115..db05c86 100644 --- a/.sqlx/query-dbcca04872cf174ca1a23862f2f632fcf5da4599e75c8f21b1c134b34b1a876a.json +++ b/runledger-postgres/.sqlx/query-1059ac0fa26c3192fe4160b03fe6f8fadf3727d67e102953a0793af534de602a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n je.id,\n je.job_id,\n je.run_number,\n je.attempt,\n je.event_type::text AS \"event_type!\",\n je.stage,\n je.progress_done,\n je.progress_total,\n je.payload,\n je.occurred_at\n FROM job_events je\n JOIN job_queue jq ON jq.id = je.job_id\n WHERE je.job_id = $1\n AND ($2::uuid IS NULL OR jq.organization_id = $2)\n AND ($3::bigint IS NULL OR je.id > $3)\n ORDER BY je.id ASC\n LIMIT $4", + "query": "SELECT\n je.id,\n je.job_id,\n je.run_number,\n je.attempt,\n je.event_type::text AS \"event_type!\",\n je.stage,\n je.progress_done,\n je.progress_total,\n je.payload,\n je.occurred_at\n FROM job_events je\n JOIN job_queue jq ON jq.id = je.job_id\n WHERE je.job_id = $1\n AND ($5::bool OR jq.organization_id IS NOT DISTINCT FROM $2::uuid)\n AND ($3::bigint IS NULL OR je.id > $3)\n ORDER BY je.id ASC\n LIMIT $4", "describe": { "columns": [ { @@ -59,7 +59,8 @@ "Uuid", "Uuid", "Int8", - "Int8" + "Int8", + "Bool" ] }, "nullable": [ @@ -75,5 +76,5 @@ false ] }, - "hash": "dbcca04872cf174ca1a23862f2f632fcf5da4599e75c8f21b1c134b34b1a876a" + "hash": "1059ac0fa26c3192fe4160b03fe6f8fadf3727d67e102953a0793af534de602a" } diff --git a/runledger-postgres/.sqlx/query-290943331ceeb17e0d9e726a3bb5a2afb203b501dccdf13baa3b2390800515f7.json b/runledger-postgres/.sqlx/query-290943331ceeb17e0d9e726a3bb5a2afb203b501dccdf13baa3b2390800515f7.json new file mode 100644 index 0000000..399cc9a --- /dev/null +++ b/runledger-postgres/.sqlx/query-290943331ceeb17e0d9e726a3bb5a2afb203b501dccdf13baa3b2390800515f7.json @@ -0,0 +1,158 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n next_run_at,\n idempotency_key,\n stage,\n enqueue_request_version,\n execution_resource_key,\n promotion_attempts,\n next_promotion_at,\n last_attempted_at,\n status,\n promoted_job_id,\n promoted_at,\n conflicted_at,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_enqueue_intents\n WHERE $1::uuid IS NULL AND ($2::text IS NULL OR status = $2)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "payload", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "timeout_seconds", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 8, + "name": "idempotency_key", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "enqueue_request_version", + "type_info": "Int2" + }, + { + "ordinal": 11, + "name": "execution_resource_key", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "promotion_attempts", + "type_info": "Int4" + }, + { + "ordinal": 13, + "name": "next_promotion_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 14, + "name": "last_attempted_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 15, + "name": "status", + "type_info": "Text" + }, + { + "ordinal": 16, + "name": "promoted_job_id", + "type_info": "Uuid" + }, + { + "ordinal": 17, + "name": "promoted_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 18, + "name": "conflicted_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 19, + "name": "last_error_code", + "type_info": "Text" + }, + { + "ordinal": 20, + "name": "last_error_message", + "type_info": "Text" + }, + { + "ordinal": 21, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 22, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false, + true, + true, + true, + true, + false, + false, + false, + true, + false, + false, + true, + false, + true, + true, + true, + true, + true, + false, + false + ] + }, + "hash": "290943331ceeb17e0d9e726a3bb5a2afb203b501dccdf13baa3b2390800515f7" +} diff --git a/runledger-postgres/.sqlx/query-c2350f3a1f05bb5bf576e20872873d7135e8dfff31412ed42bda795dbeee7586.json b/runledger-postgres/.sqlx/query-2b0c48cf0a53ca60dfb663b242b1b65b0833bf0b32fe4cacb637e18df8ae77bb.json similarity index 92% rename from runledger-postgres/.sqlx/query-c2350f3a1f05bb5bf576e20872873d7135e8dfff31412ed42bda795dbeee7586.json rename to runledger-postgres/.sqlx/query-2b0c48cf0a53ca60dfb663b242b1b65b0833bf0b32fe4cacb637e18df8ae77bb.json index 9150683..b9f7f28 100644 --- a/runledger-postgres/.sqlx/query-c2350f3a1f05bb5bf576e20872873d7135e8dfff31412ed42bda795dbeee7586.json +++ b/runledger-postgres/.sqlx/query-2b0c48cf0a53ca60dfb663b242b1b65b0833bf0b32fe4cacb637e18df8ae77bb.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n status::text AS \"status!\",\n priority,\n run_number,\n attempt,\n max_attempts,\n timeout_seconds,\n next_run_at,\n lease_expires_at,\n last_heartbeat_at,\n worker_id,\n started_at,\n finished_at,\n stage,\n progress_done,\n progress_total,\n progress_pct::float8 AS progress_pct,\n checkpoint,\n output,\n idempotency_key,\n status_reason,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_queue\n WHERE ($1::uuid IS NULL OR organization_id = $1)\n AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n status::text AS \"status!\",\n priority,\n run_number,\n attempt,\n max_attempts,\n timeout_seconds,\n next_run_at,\n lease_expires_at,\n last_heartbeat_at,\n worker_id,\n started_at,\n finished_at,\n stage,\n progress_done,\n progress_total,\n progress_pct::float8 AS progress_pct,\n checkpoint,\n output,\n idempotency_key,\n status_reason,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_queue\n WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", "describe": { "columns": [ { @@ -184,5 +184,5 @@ false ] }, - "hash": "c2350f3a1f05bb5bf576e20872873d7135e8dfff31412ed42bda795dbeee7586" + "hash": "2b0c48cf0a53ca60dfb663b242b1b65b0833bf0b32fe4cacb637e18df8ae77bb" } diff --git a/runledger-postgres/.sqlx/query-2ca138d0f77ca8b5d61381edcdb32a7f98c016c7214a13f69be251c78058b0e7.json b/runledger-postgres/.sqlx/query-2ca138d0f77ca8b5d61381edcdb32a7f98c016c7214a13f69be251c78058b0e7.json new file mode 100644 index 0000000..d18086b --- /dev/null +++ b/runledger-postgres/.sqlx/query-2ca138d0f77ca8b5d61381edcdb32a7f98c016c7214a13f69be251c78058b0e7.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, status::text AS \"status!\", run_number, attempt, updated_at\n FROM job_queue WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND id = ANY($2::uuid[]) ORDER BY id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "UuidArray" + ] + }, + "nullable": [ + false, + null, + false, + false, + false + ] + }, + "hash": "2ca138d0f77ca8b5d61381edcdb32a7f98c016c7214a13f69be251c78058b0e7" +} diff --git a/runledger-postgres/.sqlx/query-2f5d35c92e35e69d3d0260eeb2bac01332025306d05cb43392686d9cc73d5eba.json b/runledger-postgres/.sqlx/query-2f5d35c92e35e69d3d0260eeb2bac01332025306d05cb43392686d9cc73d5eba.json new file mode 100644 index 0000000..6abf7e3 --- /dev/null +++ b/runledger-postgres/.sqlx/query-2f5d35c92e35e69d3d0260eeb2bac01332025306d05cb43392686d9cc73d5eba.json @@ -0,0 +1,158 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n next_run_at,\n idempotency_key,\n stage,\n enqueue_request_version,\n execution_resource_key,\n promotion_attempts,\n next_promotion_at,\n last_attempted_at,\n status,\n promoted_job_id,\n promoted_at,\n conflicted_at,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_enqueue_intents\n WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND ($2::text IS NULL OR status = $2)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "payload", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "timeout_seconds", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 8, + "name": "idempotency_key", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "enqueue_request_version", + "type_info": "Int2" + }, + { + "ordinal": 11, + "name": "execution_resource_key", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "promotion_attempts", + "type_info": "Int4" + }, + { + "ordinal": 13, + "name": "next_promotion_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 14, + "name": "last_attempted_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 15, + "name": "status", + "type_info": "Text" + }, + { + "ordinal": 16, + "name": "promoted_job_id", + "type_info": "Uuid" + }, + { + "ordinal": 17, + "name": "promoted_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 18, + "name": "conflicted_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 19, + "name": "last_error_code", + "type_info": "Text" + }, + { + "ordinal": 20, + "name": "last_error_message", + "type_info": "Text" + }, + { + "ordinal": 21, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 22, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false, + true, + true, + true, + true, + false, + false, + false, + true, + false, + false, + true, + false, + true, + true, + true, + true, + true, + false, + false + ] + }, + "hash": "2f5d35c92e35e69d3d0260eeb2bac01332025306d05cb43392686d9cc73d5eba" +} diff --git a/.sqlx/query-9f9c494b36d03026a351373e70483c382d2e53386fac1d737a5f2e20818007d8.json b/runledger-postgres/.sqlx/query-321da426e59fb877a1e48430f03013433f919b3f4edc06e322f195e682e7be0f.json similarity index 88% rename from .sqlx/query-9f9c494b36d03026a351373e70483c382d2e53386fac1d737a5f2e20818007d8.json rename to runledger-postgres/.sqlx/query-321da426e59fb877a1e48430f03013433f919b3f4edc06e322f195e682e7be0f.json index 9b99123..fb67a3d 100644 --- a/.sqlx/query-9f9c494b36d03026a351373e70483c382d2e53386fac1d737a5f2e20818007d8.json +++ b/runledger-postgres/.sqlx/query-321da426e59fb877a1e48430f03013433f919b3f4edc06e322f195e682e7be0f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n jd.job_type AS \"job_type!\",\n COALESCE(SUM(jmr.pending_count), 0)::bigint AS \"pending_count!\",\n COALESCE(SUM(jmr.leased_count), 0)::bigint AS \"leased_count!\",\n COALESCE(SUM(jmr.stale_leases), 0)::bigint AS \"stale_leases!\",\n COALESCE(SUM(jmr.succeeded_24h), 0)::bigint AS \"succeeded_24h!\",\n COALESCE(SUM(jmr.retryable_24h), 0)::bigint AS \"retryable_24h!\",\n COALESCE(SUM(jmr.terminal_24h), 0)::bigint AS \"terminal_24h!\",\n COALESCE(SUM(jmr.panicked_24h), 0)::bigint AS \"panicked_24h!\",\n COALESCE(SUM(jmr.timeout_24h), 0)::bigint AS \"timeout_24h!\",\n COALESCE(SUM(jmr.dead_lettered_24h), 0)::bigint AS \"dead_lettered_24h!\",\n AVG(jmr.p50_duration_ms_24h) AS p50_duration_ms_24h,\n AVG(jmr.p95_duration_ms_24h) AS p95_duration_ms_24h\n FROM job_definitions jd\n LEFT JOIN job_metrics_rollup jmr\n ON jmr.job_type = jd.job_type\n AND ($1::uuid IS NULL OR jmr.organization_id = $1)\n WHERE ($2::text IS NULL OR jd.job_type = $2)\n GROUP BY jd.job_type\n ORDER BY jd.job_type ASC", + "query": "SELECT\n jd.job_type AS \"job_type!\",\n COALESCE(SUM(jmr.pending_count), 0)::bigint AS \"pending_count!\",\n COALESCE(SUM(jmr.leased_count), 0)::bigint AS \"leased_count!\",\n COALESCE(SUM(jmr.stale_leases), 0)::bigint AS \"stale_leases!\",\n COALESCE(SUM(jmr.succeeded_24h), 0)::bigint AS \"succeeded_24h!\",\n COALESCE(SUM(jmr.retryable_24h), 0)::bigint AS \"retryable_24h!\",\n COALESCE(SUM(jmr.terminal_24h), 0)::bigint AS \"terminal_24h!\",\n COALESCE(SUM(jmr.panicked_24h), 0)::bigint AS \"panicked_24h!\",\n COALESCE(SUM(jmr.timeout_24h), 0)::bigint AS \"timeout_24h!\",\n COALESCE(SUM(jmr.dead_lettered_24h), 0)::bigint AS \"dead_lettered_24h!\",\n AVG(jmr.p50_duration_ms_24h) AS p50_duration_ms_24h,\n AVG(jmr.p95_duration_ms_24h) AS p95_duration_ms_24h\n FROM job_definitions jd\n LEFT JOIN job_metrics_rollup jmr\n ON jmr.job_type = jd.job_type\n AND ($3::boolean OR (jmr.organization_id = $1 OR ($1::uuid IS NULL AND jmr.organization_id IS NULL)))\n WHERE ($2::text IS NULL OR jd.job_type = $2)\n GROUP BY jd.job_type\n ORDER BY jd.job_type ASC", "describe": { "columns": [ { @@ -67,7 +67,8 @@ "parameters": { "Left": [ "Uuid", - "Text" + "Text", + "Bool" ] }, "nullable": [ @@ -85,5 +86,5 @@ null ] }, - "hash": "9f9c494b36d03026a351373e70483c382d2e53386fac1d737a5f2e20818007d8" + "hash": "321da426e59fb877a1e48430f03013433f919b3f4edc06e322f195e682e7be0f" } diff --git a/runledger-postgres/.sqlx/query-48789cd60eb5ace54e41f24bafc110420c7e67fe6175f5bcb786579a66f9af8e.json b/runledger-postgres/.sqlx/query-48789cd60eb5ace54e41f24bafc110420c7e67fe6175f5bcb786579a66f9af8e.json new file mode 100644 index 0000000..cc80fa9 --- /dev/null +++ b/runledger-postgres/.sqlx/query-48789cd60eb5ace54e41f24bafc110420c7e67fe6175f5bcb786579a66f9af8e.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, payload FROM job_queue WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND job_type = $2\n AND payload->>'run_id' = $3\n ORDER BY created_at DESC, id DESC\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "payload", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "48789cd60eb5ace54e41f24bafc110420c7e67fe6175f5bcb786579a66f9af8e" +} diff --git a/runledger-postgres/.sqlx/query-809f796d49a1830f8338380883e2ee58cce341f4e12b85bb982b01bf09d6cc50.json b/runledger-postgres/.sqlx/query-51b23cabe44bba220eb95d76cfcc135768be65ed533785bea2d9d668e9305ebd.json similarity index 82% rename from runledger-postgres/.sqlx/query-809f796d49a1830f8338380883e2ee58cce341f4e12b85bb982b01bf09d6cc50.json rename to runledger-postgres/.sqlx/query-51b23cabe44bba220eb95d76cfcc135768be65ed533785bea2d9d668e9305ebd.json index 2ccb7de..9ced603 100644 --- a/runledger-postgres/.sqlx/query-809f796d49a1830f8338380883e2ee58cce341f4e12b85bb982b01bf09d6cc50.json +++ b/runledger-postgres/.sqlx/query-51b23cabe44bba220eb95d76cfcc135768be65ed533785bea2d9d668e9305ebd.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n jl.id,\n jl.job_id,\n jl.run_number,\n jl.attempt,\n jl.level,\n jl.message,\n jl.payload,\n jl.occurred_at\n FROM job_logs jl\n JOIN job_queue jq ON jq.id = jl.job_id\n WHERE jl.job_id = $1\n AND ($2::uuid IS NULL OR jq.organization_id = $2)\n AND ($3::bigint IS NULL OR jl.id > $3)\n ORDER BY jl.id ASC\n LIMIT $4", + "query": "SELECT\n jl.id,\n jl.job_id,\n jl.run_number,\n jl.attempt,\n jl.level,\n jl.message,\n jl.payload,\n jl.occurred_at\n FROM job_logs jl\n JOIN job_queue jq ON jq.id = jl.job_id\n WHERE jl.job_id = $1\n AND ($5::bool OR jq.organization_id IS NOT DISTINCT FROM $2::uuid)\n AND ($3::bigint IS NULL OR jl.id > $3)\n ORDER BY jl.id ASC\n LIMIT $4", "describe": { "columns": [ { @@ -49,7 +49,8 @@ "Uuid", "Uuid", "Int8", - "Int8" + "Int8", + "Bool" ] }, "nullable": [ @@ -63,5 +64,5 @@ false ] }, - "hash": "809f796d49a1830f8338380883e2ee58cce341f4e12b85bb982b01bf09d6cc50" + "hash": "51b23cabe44bba220eb95d76cfcc135768be65ed533785bea2d9d668e9305ebd" } diff --git a/runledger-postgres/.sqlx/query-6ab23505891fce132239b6cd02ab480be547302313aeb4be81f2c14bb20c0569.json b/runledger-postgres/.sqlx/query-6ab23505891fce132239b6cd02ab480be547302313aeb4be81f2c14bb20c0569.json new file mode 100644 index 0000000..0f09710 --- /dev/null +++ b/runledger-postgres/.sqlx/query-6ab23505891fce132239b6cd02ab480be547302313aeb4be81f2c14bb20c0569.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE job_queue\n SET stage = COALESCE($5, stage),\n progress_done = COALESCE($6, progress_done),\n progress_total = COALESCE($7, progress_total),\n checkpoint = COALESCE($8::jsonb, checkpoint),\n updated_at = now()\n WHERE id = $1\n AND run_number = $2\n AND attempt = $3\n AND worker_id = $4\n AND status = 'LEASED'\n AND lease_expires_at > clock_timestamp()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int4", + "Int4", + "Text", + "Text", + "Int8", + "Int8", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "6ab23505891fce132239b6cd02ab480be547302313aeb4be81f2c14bb20c0569" +} diff --git a/runledger-postgres/.sqlx/query-6c5d8cb4f663423a38334a1418faf9050dee8d8fd943820afd0bcb6e0bc32565.json b/runledger-postgres/.sqlx/query-6c5d8cb4f663423a38334a1418faf9050dee8d8fd943820afd0bcb6e0bc32565.json deleted file mode 100644 index d56ae1f..0000000 --- a/runledger-postgres/.sqlx/query-6c5d8cb4f663423a38334a1418faf9050dee8d8fd943820afd0bcb6e0bc32565.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workflow_steps (\n workflow_run_id,\n step_key,\n execution_kind,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n stage,\n allow_handler_continuation,\n execution_resource_key,\n status,\n dependency_count_total,\n dependency_count_pending,\n dependency_count_unsatisfied\n )\n VALUES (\n $1,\n $2,\n $3::text::workflow_step_execution_kind,\n $4,\n $5,\n $6::jsonb,\n $7,\n $8,\n $9,\n $10,\n $11,\n $12,\n 'BLOCKED',\n $13,\n $14,\n $15\n )\n RETURNING id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Text", - "Text", - "Uuid", - "Jsonb", - "Int4", - "Int4", - "Int4", - "Text", - "Bool", - "Text", - "Int4", - "Int4", - "Int4" - ] - }, - "nullable": [ - false - ] - }, - "hash": "6c5d8cb4f663423a38334a1418faf9050dee8d8fd943820afd0bcb6e0bc32565" -} diff --git a/.sqlx/query-76c178ef26a5e63480d6f03da8016bc623846f7f78de717025ebbd40c1b848e0.json b/runledger-postgres/.sqlx/query-76e0bb60c9fe984d8b52e294bc657bc1aece09a76f064220c406b146277929cc.json similarity index 61% rename from .sqlx/query-76c178ef26a5e63480d6f03da8016bc623846f7f78de717025ebbd40c1b848e0.json rename to runledger-postgres/.sqlx/query-76e0bb60c9fe984d8b52e294bc657bc1aece09a76f064220c406b146277929cc.json index 1e847d2..49786b4 100644 --- a/.sqlx/query-76c178ef26a5e63480d6f03da8016bc623846f7f78de717025ebbd40c1b848e0.json +++ b/runledger-postgres/.sqlx/query-76e0bb60c9fe984d8b52e294bc657bc1aece09a76f064220c406b146277929cc.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, payload\n FROM job_queue\n WHERE organization_id = $1\n AND job_type = $2\n AND idempotency_key = $3\n LIMIT 1", + "query": "SELECT id, payload FROM job_queue WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND job_type = $2\n AND idempotency_key = $3\n LIMIT 1", "describe": { "columns": [ { @@ -26,5 +26,5 @@ false ] }, - "hash": "76c178ef26a5e63480d6f03da8016bc623846f7f78de717025ebbd40c1b848e0" + "hash": "76e0bb60c9fe984d8b52e294bc657bc1aece09a76f064220c406b146277929cc" } diff --git a/runledger-runtime/.sqlx/query-118f48ae11853ca23401d3569a8c10677a04ac4114f2fbc5d832d866a46e778c.json b/runledger-postgres/.sqlx/query-7e287b60fb932a4795c61e25e247e4313adf268188c94931c8f9ebb995c8812e.json similarity index 94% rename from runledger-runtime/.sqlx/query-118f48ae11853ca23401d3569a8c10677a04ac4114f2fbc5d832d866a46e778c.json rename to runledger-postgres/.sqlx/query-7e287b60fb932a4795c61e25e247e4313adf268188c94931c8f9ebb995c8812e.json index b0eef8b..39f343e 100644 --- a/runledger-runtime/.sqlx/query-118f48ae11853ca23401d3569a8c10677a04ac4114f2fbc5d832d866a46e778c.json +++ b/runledger-postgres/.sqlx/query-7e287b60fb932a4795c61e25e247e4313adf268188c94931c8f9ebb995c8812e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n next_run_at,\n idempotency_key,\n stage,\n enqueue_request_version,\n execution_resource_key,\n promotion_attempts,\n next_promotion_at,\n last_attempted_at,\n status,\n promoted_job_id,\n promoted_at,\n conflicted_at,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_enqueue_intents\n WHERE id = $1\n AND ($2::uuid IS NULL OR organization_id = $2)\n LIMIT 1", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n next_run_at,\n idempotency_key,\n stage,\n enqueue_request_version,\n execution_resource_key,\n promotion_attempts,\n next_promotion_at,\n last_attempted_at,\n status,\n promoted_job_id,\n promoted_at,\n conflicted_at,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_enqueue_intents\n WHERE id = $1\n AND ($3::bool OR organization_id IS NOT DISTINCT FROM $2::uuid)\n LIMIT 1", "describe": { "columns": [ { @@ -122,7 +122,8 @@ "parameters": { "Left": [ "Uuid", - "Uuid" + "Uuid", + "Bool" ] }, "nullable": [ @@ -151,5 +152,5 @@ false ] }, - "hash": "118f48ae11853ca23401d3569a8c10677a04ac4114f2fbc5d832d866a46e778c" + "hash": "7e287b60fb932a4795c61e25e247e4313adf268188c94931c8f9ebb995c8812e" } diff --git a/runledger-postgres/.sqlx/query-80abaede7d6da749be3c1a5a6f5ff2da49f9b39c07bb5b2a0bfb35d86d7b7c8e.json b/runledger-postgres/.sqlx/query-80abaede7d6da749be3c1a5a6f5ff2da49f9b39c07bb5b2a0bfb35d86d7b7c8e.json new file mode 100644 index 0000000..0fcf450 --- /dev/null +++ b/runledger-postgres/.sqlx/query-80abaede7d6da749be3c1a5a6f5ff2da49f9b39c07bb5b2a0bfb35d86d7b7c8e.json @@ -0,0 +1,188 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n status::text AS \"status!\",\n priority,\n run_number,\n attempt,\n max_attempts,\n timeout_seconds,\n next_run_at,\n lease_expires_at,\n last_heartbeat_at,\n worker_id,\n started_at,\n finished_at,\n stage,\n progress_done,\n progress_total,\n progress_pct::float8 AS progress_pct,\n checkpoint,\n output,\n idempotency_key,\n status_reason,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_queue\n WHERE organization_id = $1 AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "payload", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 9, + "name": "timeout_seconds", + "type_info": "Int4" + }, + { + "ordinal": 10, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 11, + "name": "lease_expires_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 12, + "name": "last_heartbeat_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "worker_id", + "type_info": "Text" + }, + { + "ordinal": 14, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 15, + "name": "finished_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 16, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 17, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 18, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 19, + "name": "progress_pct", + "type_info": "Float8" + }, + { + "ordinal": 20, + "name": "checkpoint", + "type_info": "Jsonb" + }, + { + "ordinal": 21, + "name": "output", + "type_info": "Jsonb" + }, + { + "ordinal": 22, + "name": "idempotency_key", + "type_info": "Text" + }, + { + "ordinal": 23, + "name": "status_reason", + "type_info": "Text" + }, + { + "ordinal": 24, + "name": "last_error_code", + "type_info": "Text" + }, + { + "ordinal": 25, + "name": "last_error_message", + "type_info": "Text" + }, + { + "ordinal": 26, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 27, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false, + null, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + true, + false, + true, + true, + null, + true, + true, + true, + true, + true, + true, + false, + false + ] + }, + "hash": "80abaede7d6da749be3c1a5a6f5ff2da49f9b39c07bb5b2a0bfb35d86d7b7c8e" +} diff --git a/runledger-postgres/.sqlx/query-822a3544ce03dde10c9f57947ccbb25546b94f29cfdcf2151d28361d3a04d461.json b/runledger-postgres/.sqlx/query-822a3544ce03dde10c9f57947ccbb25546b94f29cfdcf2151d28361d3a04d461.json new file mode 100644 index 0000000..464adbd --- /dev/null +++ b/runledger-postgres/.sqlx/query-822a3544ce03dde10c9f57947ccbb25546b94f29cfdcf2151d28361d3a04d461.json @@ -0,0 +1,105 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, job_type, organization_id, status::text AS \"status!\",\n priority, run_number, attempt, max_attempts, next_run_at, stage,\n progress_done, progress_total, created_at, updated_at\n FROM job_queue WHERE $1::uuid IS NULL AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type = $3) AND (created_at, id) < ($5, $6) ORDER BY created_at DESC, id DESC LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Timestamptz", + "Uuid" + ] + }, + "nullable": [ + false, + false, + true, + null, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + }, + "hash": "822a3544ce03dde10c9f57947ccbb25546b94f29cfdcf2151d28361d3a04d461" +} diff --git a/runledger-postgres/.sqlx/query-85eca198e5990ee9fa5ec1abef7c0abaf3750dfb0d4e56e6a242e2921a46d995.json b/runledger-postgres/.sqlx/query-85eca198e5990ee9fa5ec1abef7c0abaf3750dfb0d4e56e6a242e2921a46d995.json new file mode 100644 index 0000000..9b9f66e --- /dev/null +++ b/runledger-postgres/.sqlx/query-85eca198e5990ee9fa5ec1abef7c0abaf3750dfb0d4e56e6a242e2921a46d995.json @@ -0,0 +1,105 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, job_type, organization_id, status::text AS \"status!\",\n priority, run_number, attempt, max_attempts, next_run_at, stage,\n progress_done, progress_total, created_at, updated_at\n FROM job_queue WHERE organization_id = $1 AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type = $3) AND (created_at, id) < ($5, $6) ORDER BY created_at DESC, id DESC LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Timestamptz", + "Uuid" + ] + }, + "nullable": [ + false, + false, + true, + null, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + }, + "hash": "85eca198e5990ee9fa5ec1abef7c0abaf3750dfb0d4e56e6a242e2921a46d995" +} diff --git a/runledger-postgres/.sqlx/query-89faf4374de8090be4461c194856d94ddc63813894ceaf9fef0c0a00cb2598e7.json b/runledger-postgres/.sqlx/query-89faf4374de8090be4461c194856d94ddc63813894ceaf9fef0c0a00cb2598e7.json new file mode 100644 index 0000000..9ac167b --- /dev/null +++ b/runledger-postgres/.sqlx/query-89faf4374de8090be4461c194856d94ddc63813894ceaf9fef0c0a00cb2598e7.json @@ -0,0 +1,62 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH status_metrics AS (\n SELECT\n job_type,\n COUNT(*)::bigint AS pending_count,\n COUNT(*) FILTER (WHERE promotion_attempts > 0)::bigint AS retrying_count,\n MAX(promotion_attempts)::integer AS max_promotion_attempts,\n 0::bigint AS conflicted_24h,\n 0::bigint AS promoted_24h,\n MIN(created_at) AS oldest_pending_at\n FROM job_enqueue_intents\n WHERE status = 'PENDING'\n AND ($5::boolean OR (organization_id = $1 OR ($1::uuid IS NULL AND organization_id IS NULL)))\n AND ($2::text IS NULL OR job_type = $2)\n GROUP BY job_type\n\n UNION ALL\n\n SELECT\n job_type,\n 0::bigint AS pending_count,\n 0::bigint AS retrying_count,\n 0::integer AS max_promotion_attempts,\n COUNT(*)::bigint AS conflicted_24h,\n 0::bigint AS promoted_24h,\n NULL::timestamptz AS oldest_pending_at\n FROM job_enqueue_intents\n WHERE status = 'CONFLICTED'\n AND conflicted_at >= now() - interval '24 hours'\n AND ($5::boolean OR (organization_id = $1 OR ($1::uuid IS NULL AND organization_id IS NULL)))\n AND ($2::text IS NULL OR job_type = $2)\n GROUP BY job_type\n\n UNION ALL\n\n SELECT\n job_type,\n 0::bigint AS pending_count,\n 0::bigint AS retrying_count,\n 0::integer AS max_promotion_attempts,\n 0::bigint AS conflicted_24h,\n COUNT(*)::bigint AS promoted_24h,\n NULL::timestamptz AS oldest_pending_at\n FROM job_enqueue_intents\n WHERE status = 'PROMOTED'\n AND promoted_at >= now() - interval '24 hours'\n AND ($5::boolean OR (organization_id = $1 OR ($1::uuid IS NULL AND organization_id IS NULL)))\n AND ($2::text IS NULL OR job_type = $2)\n GROUP BY job_type\n )\n SELECT\n job_type AS \"job_type!\",\n MAX(pending_count)::bigint AS \"pending_count!\",\n MAX(retrying_count)::bigint AS \"retrying_count!\",\n MAX(max_promotion_attempts)::integer AS \"max_promotion_attempts!\",\n MAX(conflicted_24h)::bigint AS \"conflicted_24h!\",\n MAX(promoted_24h)::bigint AS \"promoted_24h!\",\n MIN(oldest_pending_at) AS oldest_pending_at\n FROM status_metrics\n GROUP BY job_type\n ORDER BY job_type\n LIMIT $3\n OFFSET $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_type!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "pending_count!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "retrying_count!", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "max_promotion_attempts!", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "conflicted_24h!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "promoted_24h!", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "oldest_pending_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Int8", + "Int8", + "Bool" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "89faf4374de8090be4461c194856d94ddc63813894ceaf9fef0c0a00cb2598e7" +} diff --git a/runledger-postgres/.sqlx/query-8a04ed3b1b3497fbb7ec16ef976e9be2b03e4d64d5448bd90ce15cf8a9e5acc0.json b/runledger-postgres/.sqlx/query-8a04ed3b1b3497fbb7ec16ef976e9be2b03e4d64d5448bd90ce15cf8a9e5acc0.json new file mode 100644 index 0000000..a47e779 --- /dev/null +++ b/runledger-postgres/.sqlx/query-8a04ed3b1b3497fbb7ec16ef976e9be2b03e4d64d5448bd90ce15cf8a9e5acc0.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, payload FROM job_queue WHERE organization_id = $1 AND job_type = $2\n AND idempotency_key = $3\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "payload", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "8a04ed3b1b3497fbb7ec16ef976e9be2b03e4d64d5448bd90ce15cf8a9e5acc0" +} diff --git a/runledger-postgres/.sqlx/query-8de6d5625581aff24f7bb689361c8dcd9319042f9883e4cc748bcd771b06c09e.json b/runledger-postgres/.sqlx/query-8de6d5625581aff24f7bb689361c8dcd9319042f9883e4cc748bcd771b06c09e.json new file mode 100644 index 0000000..c506de2 --- /dev/null +++ b/runledger-postgres/.sqlx/query-8de6d5625581aff24f7bb689361c8dcd9319042f9883e4cc748bcd771b06c09e.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, status::text AS \"status!\", run_number, attempt, updated_at\n FROM job_queue WHERE organization_id = $1 AND id = ANY($2::uuid[]) ORDER BY id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "UuidArray" + ] + }, + "nullable": [ + false, + null, + false, + false, + false + ] + }, + "hash": "8de6d5625581aff24f7bb689361c8dcd9319042f9883e4cc748bcd771b06c09e" +} diff --git a/runledger-postgres/.sqlx/query-90841ab00a885ed00fd2c3a6b49c90b549a262963d281b863f4283af861ea5e5.json b/runledger-postgres/.sqlx/query-90841ab00a885ed00fd2c3a6b49c90b549a262963d281b863f4283af861ea5e5.json deleted file mode 100644 index bf1cfb5..0000000 --- a/runledger-postgres/.sqlx/query-90841ab00a885ed00fd2c3a6b49c90b549a262963d281b863f4283af861ea5e5.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH status_metrics AS (\n SELECT\n job_type,\n COUNT(*)::bigint AS pending_count,\n COUNT(*) FILTER (WHERE promotion_attempts > 0)::bigint AS retrying_count,\n MAX(promotion_attempts)::integer AS max_promotion_attempts,\n 0::bigint AS conflicted_24h,\n 0::bigint AS promoted_24h,\n MIN(created_at) AS oldest_pending_at\n FROM job_enqueue_intents\n WHERE status = 'PENDING'\n AND ($1::uuid IS NULL OR organization_id = $1)\n AND ($2::text IS NULL OR job_type = $2)\n GROUP BY job_type\n\n UNION ALL\n\n SELECT\n job_type,\n 0::bigint AS pending_count,\n 0::bigint AS retrying_count,\n 0::integer AS max_promotion_attempts,\n COUNT(*)::bigint AS conflicted_24h,\n 0::bigint AS promoted_24h,\n NULL::timestamptz AS oldest_pending_at\n FROM job_enqueue_intents\n WHERE status = 'CONFLICTED'\n AND conflicted_at >= now() - interval '24 hours'\n AND ($1::uuid IS NULL OR organization_id = $1)\n AND ($2::text IS NULL OR job_type = $2)\n GROUP BY job_type\n\n UNION ALL\n\n SELECT\n job_type,\n 0::bigint AS pending_count,\n 0::bigint AS retrying_count,\n 0::integer AS max_promotion_attempts,\n 0::bigint AS conflicted_24h,\n COUNT(*)::bigint AS promoted_24h,\n NULL::timestamptz AS oldest_pending_at\n FROM job_enqueue_intents\n WHERE status = 'PROMOTED'\n AND promoted_at >= now() - interval '24 hours'\n AND ($1::uuid IS NULL OR organization_id = $1)\n AND ($2::text IS NULL OR job_type = $2)\n GROUP BY job_type\n )\n SELECT\n job_type AS \"job_type!\",\n MAX(pending_count)::bigint AS \"pending_count!\",\n MAX(retrying_count)::bigint AS \"retrying_count!\",\n MAX(max_promotion_attempts)::integer AS \"max_promotion_attempts!\",\n MAX(conflicted_24h)::bigint AS \"conflicted_24h!\",\n MAX(promoted_24h)::bigint AS \"promoted_24h!\",\n MIN(oldest_pending_at) AS oldest_pending_at\n FROM status_metrics\n GROUP BY job_type\n ORDER BY job_type\n LIMIT $3\n OFFSET $4", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "job_type!", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "pending_count!", - "type_info": "Int8" - }, - { - "ordinal": 2, - "name": "retrying_count!", - "type_info": "Int8" - }, - { - "ordinal": 3, - "name": "max_promotion_attempts!", - "type_info": "Int4" - }, - { - "ordinal": 4, - "name": "conflicted_24h!", - "type_info": "Int8" - }, - { - "ordinal": 5, - "name": "promoted_24h!", - "type_info": "Int8" - }, - { - "ordinal": 6, - "name": "oldest_pending_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Int8", - "Int8" - ] - }, - "nullable": [ - null, - null, - null, - null, - null, - null, - null - ] - }, - "hash": "90841ab00a885ed00fd2c3a6b49c90b549a262963d281b863f4283af861ea5e5" -} diff --git a/runledger-postgres/.sqlx/query-92a41242e40c65c1eaad5aec4b2bd0b868b3e1fe7a7ba1ab015bf4c93c6fd431.json b/runledger-postgres/.sqlx/query-92a41242e40c65c1eaad5aec4b2bd0b868b3e1fe7a7ba1ab015bf4c93c6fd431.json new file mode 100644 index 0000000..b7ae055 --- /dev/null +++ b/runledger-postgres/.sqlx/query-92a41242e40c65c1eaad5aec4b2bd0b868b3e1fe7a7ba1ab015bf4c93c6fd431.json @@ -0,0 +1,103 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, job_type, organization_id, status::text AS \"status!\",\n priority, run_number, attempt, max_attempts, next_run_at, stage,\n progress_done, progress_total, created_at, updated_at\n FROM job_queue WHERE organization_id = $1 AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type = $3) ORDER BY created_at DESC, id DESC LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + null, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + }, + "hash": "92a41242e40c65c1eaad5aec4b2bd0b868b3e1fe7a7ba1ab015bf4c93c6fd431" +} diff --git a/runledger-postgres/.sqlx/query-a0383524b739ee8a06fba3e337e07e608049f1a18f43a548859d9683d6bae1f6.json b/runledger-postgres/.sqlx/query-a0383524b739ee8a06fba3e337e07e608049f1a18f43a548859d9683d6bae1f6.json new file mode 100644 index 0000000..c515cda --- /dev/null +++ b/runledger-postgres/.sqlx/query-a0383524b739ee8a06fba3e337e07e608049f1a18f43a548859d9683d6bae1f6.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, status::text AS \"status!\", run_number, attempt, updated_at\n FROM job_queue WHERE $1::uuid IS NULL AND id = ANY($2::uuid[]) ORDER BY id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "UuidArray" + ] + }, + "nullable": [ + false, + null, + false, + false, + false + ] + }, + "hash": "a0383524b739ee8a06fba3e337e07e608049f1a18f43a548859d9683d6bae1f6" +} diff --git a/runledger-postgres/.sqlx/query-a611f047bb82ca7e1794ff283b09aee3ef1d9656babed381e73eb12626f3648c.json b/runledger-postgres/.sqlx/query-a611f047bb82ca7e1794ff283b09aee3ef1d9656babed381e73eb12626f3648c.json new file mode 100644 index 0000000..6d0321f --- /dev/null +++ b/runledger-postgres/.sqlx/query-a611f047bb82ca7e1794ff283b09aee3ef1d9656babed381e73eb12626f3648c.json @@ -0,0 +1,103 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, job_type, organization_id, status::text AS \"status!\",\n priority, run_number, attempt, max_attempts, next_run_at, stage,\n progress_done, progress_total, created_at, updated_at\n FROM job_queue WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type = $3) ORDER BY created_at DESC, id DESC LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + null, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + }, + "hash": "a611f047bb82ca7e1794ff283b09aee3ef1d9656babed381e73eb12626f3648c" +} diff --git a/runledger-postgres/.sqlx/query-a74a899a375ff31b61d663d73cc4a13e467c246b649d4658484f183501c5d0c8.json b/runledger-postgres/.sqlx/query-a74a899a375ff31b61d663d73cc4a13e467c246b649d4658484f183501c5d0c8.json new file mode 100644 index 0000000..c375c3f --- /dev/null +++ b/runledger-postgres/.sqlx/query-a74a899a375ff31b61d663d73cc4a13e467c246b649d4658484f183501c5d0c8.json @@ -0,0 +1,158 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n next_run_at,\n idempotency_key,\n stage,\n enqueue_request_version,\n execution_resource_key,\n promotion_attempts,\n next_promotion_at,\n last_attempted_at,\n status,\n promoted_job_id,\n promoted_at,\n conflicted_at,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_enqueue_intents\n WHERE organization_id = $1 AND ($2::text IS NULL OR status = $2)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "payload", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "timeout_seconds", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 8, + "name": "idempotency_key", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "enqueue_request_version", + "type_info": "Int2" + }, + { + "ordinal": 11, + "name": "execution_resource_key", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "promotion_attempts", + "type_info": "Int4" + }, + { + "ordinal": 13, + "name": "next_promotion_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 14, + "name": "last_attempted_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 15, + "name": "status", + "type_info": "Text" + }, + { + "ordinal": 16, + "name": "promoted_job_id", + "type_info": "Uuid" + }, + { + "ordinal": 17, + "name": "promoted_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 18, + "name": "conflicted_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 19, + "name": "last_error_code", + "type_info": "Text" + }, + { + "ordinal": 20, + "name": "last_error_message", + "type_info": "Text" + }, + { + "ordinal": 21, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 22, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false, + true, + true, + true, + true, + false, + false, + false, + true, + false, + false, + true, + false, + true, + true, + true, + true, + true, + false, + false + ] + }, + "hash": "a74a899a375ff31b61d663d73cc4a13e467c246b649d4658484f183501c5d0c8" +} diff --git a/runledger-postgres/.sqlx/query-ba0f6f24f4ce9dcb1d40cb75eb26228c07a691e183d68a876e0e8f778120d137.json b/runledger-postgres/.sqlx/query-ba0f6f24f4ce9dcb1d40cb75eb26228c07a691e183d68a876e0e8f778120d137.json new file mode 100644 index 0000000..c8ad8b2 --- /dev/null +++ b/runledger-postgres/.sqlx/query-ba0f6f24f4ce9dcb1d40cb75eb26228c07a691e183d68a876e0e8f778120d137.json @@ -0,0 +1,188 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n status::text AS \"status!\",\n priority,\n run_number,\n attempt,\n max_attempts,\n timeout_seconds,\n next_run_at,\n lease_expires_at,\n last_heartbeat_at,\n worker_id,\n started_at,\n finished_at,\n stage,\n progress_done,\n progress_total,\n progress_pct::float8 AS progress_pct,\n checkpoint,\n output,\n idempotency_key,\n status_reason,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_queue\n WHERE $1::uuid IS NULL AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "payload", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 9, + "name": "timeout_seconds", + "type_info": "Int4" + }, + { + "ordinal": 10, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 11, + "name": "lease_expires_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 12, + "name": "last_heartbeat_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "worker_id", + "type_info": "Text" + }, + { + "ordinal": 14, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 15, + "name": "finished_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 16, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 17, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 18, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 19, + "name": "progress_pct", + "type_info": "Float8" + }, + { + "ordinal": 20, + "name": "checkpoint", + "type_info": "Jsonb" + }, + { + "ordinal": 21, + "name": "output", + "type_info": "Jsonb" + }, + { + "ordinal": 22, + "name": "idempotency_key", + "type_info": "Text" + }, + { + "ordinal": 23, + "name": "status_reason", + "type_info": "Text" + }, + { + "ordinal": 24, + "name": "last_error_code", + "type_info": "Text" + }, + { + "ordinal": 25, + "name": "last_error_message", + "type_info": "Text" + }, + { + "ordinal": 26, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 27, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false, + null, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + true, + false, + true, + true, + null, + true, + true, + true, + true, + true, + true, + false, + false + ] + }, + "hash": "ba0f6f24f4ce9dcb1d40cb75eb26228c07a691e183d68a876e0e8f778120d137" +} diff --git a/runledger-postgres/.sqlx/query-ba9acdf8f6c1427f2488aec9ab85ac3eddf200e7f69494de198b3560888d03f7.json b/runledger-postgres/.sqlx/query-ba9acdf8f6c1427f2488aec9ab85ac3eddf200e7f69494de198b3560888d03f7.json new file mode 100644 index 0000000..fa50003 --- /dev/null +++ b/runledger-postgres/.sqlx/query-ba9acdf8f6c1427f2488aec9ab85ac3eddf200e7f69494de198b3560888d03f7.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workflow_step_dependencies (\n workflow_run_id, prerequisite_step_id, dependent_step_id, release_mode\n ) SELECT $1, r.prerequisite_step_id, r.dependent_step_id,\n r.release_mode::workflow_dependency_release_mode\n FROM jsonb_to_recordset($2::jsonb) AS r(\n prerequisite_step_id uuid, dependent_step_id uuid, release_mode text)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "ba9acdf8f6c1427f2488aec9ab85ac3eddf200e7f69494de198b3560888d03f7" +} diff --git a/runledger-postgres/.sqlx/query-bd7c8cceca8d42541d5816afd514778bd665cb721f5a44bcc700a08a8057fd95.json b/runledger-postgres/.sqlx/query-bd7c8cceca8d42541d5816afd514778bd665cb721f5a44bcc700a08a8057fd95.json deleted file mode 100644 index f427afe..0000000 --- a/runledger-postgres/.sqlx/query-bd7c8cceca8d42541d5816afd514778bd665cb721f5a44bcc700a08a8057fd95.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workflow_step_dependencies (\n workflow_run_id,\n prerequisite_step_id,\n dependent_step_id,\n release_mode\n )\n VALUES ($1, $2, $3, $4::text::workflow_dependency_release_mode)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Uuid", - "Uuid", - "Text" - ] - }, - "nullable": [] - }, - "hash": "bd7c8cceca8d42541d5816afd514778bd665cb721f5a44bcc700a08a8057fd95" -} diff --git a/runledger-postgres/.sqlx/query-af553a066c6cd1b24770d60492216a8a192bebc3257165496ee1523797b2df5d.json b/runledger-postgres/.sqlx/query-d3fdc5ee034100f4b1d0feca12b9e0cca4d8a97b4d1700f4f1cd4b2380e6ba43.json similarity index 74% rename from runledger-postgres/.sqlx/query-af553a066c6cd1b24770d60492216a8a192bebc3257165496ee1523797b2df5d.json rename to runledger-postgres/.sqlx/query-d3fdc5ee034100f4b1d0feca12b9e0cca4d8a97b4d1700f4f1cd4b2380e6ba43.json index 871d4e7..232afe9 100644 --- a/runledger-postgres/.sqlx/query-af553a066c6cd1b24770d60492216a8a192bebc3257165496ee1523797b2df5d.json +++ b/runledger-postgres/.sqlx/query-d3fdc5ee034100f4b1d0feca12b9e0cca4d8a97b4d1700f4f1cd4b2380e6ba43.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n jd.job_type AS \"job_type!\",\n COALESCE(SUM(jcmr.continued_24h), 0)::bigint AS \"continued_24h!\",\n COALESCE(SUM(jcmr.active_continued_count), 0)::bigint AS \"active_continued_count!\",\n COALESCE(MAX(jcmr.max_active_run_number), 0)::int4 AS \"max_active_run_number!\"\n FROM job_definitions jd\n LEFT JOIN job_continuation_metrics_rollup jcmr\n ON jcmr.job_type = jd.job_type\n AND ($1::uuid IS NULL OR jcmr.organization_id = $1)\n WHERE ($2::text IS NULL OR jd.job_type = $2)\n GROUP BY jd.job_type\n ORDER BY jd.job_type ASC", + "query": "SELECT\n jd.job_type AS \"job_type!\",\n COALESCE(SUM(jcmr.continued_24h), 0)::bigint AS \"continued_24h!\",\n COALESCE(SUM(jcmr.active_continued_count), 0)::bigint AS \"active_continued_count!\",\n COALESCE(MAX(jcmr.max_active_run_number), 0)::int4 AS \"max_active_run_number!\"\n FROM job_definitions jd\n LEFT JOIN job_continuation_metrics_rollup jcmr\n ON jcmr.job_type = jd.job_type\n AND ($3::boolean OR (jcmr.organization_id = $1 OR ($1::uuid IS NULL AND jcmr.organization_id IS NULL)))\n WHERE ($2::text IS NULL OR jd.job_type = $2)\n GROUP BY jd.job_type\n ORDER BY jd.job_type ASC", "describe": { "columns": [ { @@ -27,7 +27,8 @@ "parameters": { "Left": [ "Uuid", - "Text" + "Text", + "Bool" ] }, "nullable": [ @@ -37,5 +38,5 @@ null ] }, - "hash": "af553a066c6cd1b24770d60492216a8a192bebc3257165496ee1523797b2df5d" + "hash": "d3fdc5ee034100f4b1d0feca12b9e0cca4d8a97b4d1700f4f1cd4b2380e6ba43" } diff --git a/runledger-postgres/.sqlx/query-d72de6e5b2099c445d943a72a081987f9b1cc6ca13c877861d41fcf86466d6e6.json b/runledger-postgres/.sqlx/query-d72de6e5b2099c445d943a72a081987f9b1cc6ca13c877861d41fcf86466d6e6.json new file mode 100644 index 0000000..4783ff9 --- /dev/null +++ b/runledger-postgres/.sqlx/query-d72de6e5b2099c445d943a72a081987f9b1cc6ca13c877861d41fcf86466d6e6.json @@ -0,0 +1,103 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, job_type, organization_id, status::text AS \"status!\",\n priority, run_number, attempt, max_attempts, next_run_at, stage,\n progress_done, progress_total, created_at, updated_at\n FROM job_queue WHERE $1::uuid IS NULL AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type = $3) ORDER BY created_at DESC, id DESC LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + null, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + }, + "hash": "d72de6e5b2099c445d943a72a081987f9b1cc6ca13c877861d41fcf86466d6e6" +} diff --git a/runledger-postgres/.sqlx/query-d74db24f58285ef5796dcfe06b49136fea475553bcf65423c59c3067a2ef49b1.json b/runledger-postgres/.sqlx/query-d74db24f58285ef5796dcfe06b49136fea475553bcf65423c59c3067a2ef49b1.json new file mode 100644 index 0000000..b67bac0 --- /dev/null +++ b/runledger-postgres/.sqlx/query-d74db24f58285ef5796dcfe06b49136fea475553bcf65423c59c3067a2ef49b1.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workflow_steps (\n workflow_run_id, step_key, execution_kind, job_type, organization_id, payload,\n priority, max_attempts, timeout_seconds, stage, allow_handler_continuation,\n execution_resource_key, status, dependency_count_total,\n dependency_count_pending, dependency_count_unsatisfied\n ) SELECT $1, r.step_key, r.execution_kind::workflow_step_execution_kind,\n r.job_type, r.organization_id, COALESCE(r.payload, 'null'::jsonb), r.priority, r.max_attempts,\n r.timeout_seconds, r.stage, r.allow_handler_continuation, r.execution_resource_key,\n 'BLOCKED', r.dependency_count_total, r.dependency_count_pending, r.dependency_count_unsatisfied\n FROM jsonb_to_recordset($2::jsonb) AS r(\n step_key text, execution_kind text, job_type text, organization_id uuid, payload jsonb,\n priority int, max_attempts int, timeout_seconds int, stage text,\n allow_handler_continuation bool, execution_resource_key text,\n dependency_count_total int, dependency_count_pending int, dependency_count_unsatisfied int)\n RETURNING id, step_key", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "step_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Jsonb" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "d74db24f58285ef5796dcfe06b49136fea475553bcf65423c59c3067a2ef49b1" +} diff --git a/.sqlx/query-26acb4ac4804ccb2f1231f1e2bfe017f40f2e13c61b74158d5e09a66aac6c06d.json b/runledger-postgres/.sqlx/query-ddd4e90290e03da64cf22463877941b79baa32e938e95d66339a91fc3756d599.json similarity index 95% rename from .sqlx/query-26acb4ac4804ccb2f1231f1e2bfe017f40f2e13c61b74158d5e09a66aac6c06d.json rename to runledger-postgres/.sqlx/query-ddd4e90290e03da64cf22463877941b79baa32e938e95d66339a91fc3756d599.json index e8d6762..28b299f 100644 --- a/.sqlx/query-26acb4ac4804ccb2f1231f1e2bfe017f40f2e13c61b74158d5e09a66aac6c06d.json +++ b/runledger-postgres/.sqlx/query-ddd4e90290e03da64cf22463877941b79baa32e938e95d66339a91fc3756d599.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n status::text AS \"status!\",\n priority,\n run_number,\n attempt,\n max_attempts,\n timeout_seconds,\n next_run_at,\n lease_expires_at,\n last_heartbeat_at,\n worker_id,\n started_at,\n finished_at,\n stage,\n progress_done,\n progress_total,\n progress_pct::float8 AS progress_pct,\n checkpoint,\n output,\n idempotency_key,\n status_reason,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_queue\n WHERE id = $1\n AND ($2::uuid IS NULL OR organization_id = $2)\n LIMIT 1", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n status::text AS \"status!\",\n priority,\n run_number,\n attempt,\n max_attempts,\n timeout_seconds,\n next_run_at,\n lease_expires_at,\n last_heartbeat_at,\n worker_id,\n started_at,\n finished_at,\n stage,\n progress_done,\n progress_total,\n progress_pct::float8 AS progress_pct,\n checkpoint,\n output,\n idempotency_key,\n status_reason,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_queue\n WHERE id = $1\n AND ($3::bool OR organization_id IS NOT DISTINCT FROM $2::uuid)\n LIMIT 1", "describe": { "columns": [ { @@ -147,7 +147,8 @@ "parameters": { "Left": [ "Uuid", - "Uuid" + "Uuid", + "Bool" ] }, "nullable": [ @@ -181,5 +182,5 @@ false ] }, - "hash": "26acb4ac4804ccb2f1231f1e2bfe017f40f2e13c61b74158d5e09a66aac6c06d" + "hash": "ddd4e90290e03da64cf22463877941b79baa32e938e95d66339a91fc3756d599" } diff --git a/runledger-postgres/.sqlx/query-f6b58ff379b8af9e2ab6346743ad1da85494d295268da6627d59c2bcc70b9fb0.json b/runledger-postgres/.sqlx/query-f6b58ff379b8af9e2ab6346743ad1da85494d295268da6627d59c2bcc70b9fb0.json deleted file mode 100644 index ab6c006..0000000 --- a/runledger-postgres/.sqlx/query-f6b58ff379b8af9e2ab6346743ad1da85494d295268da6627d59c2bcc70b9fb0.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH locked_job AS MATERIALIZED (\n SELECT id\n FROM job_queue\n WHERE id = $1\n AND run_number = $2\n AND attempt = $3\n AND worker_id = $4\n AND status = 'LEASED'\n AND lease_expires_at IS NOT NULL\n FOR UPDATE\n )\n UPDATE job_queue\n SET stage = COALESCE($5, stage),\n progress_done = COALESCE($6, progress_done),\n progress_total = COALESCE($7, progress_total),\n checkpoint = COALESCE($8::jsonb, checkpoint),\n updated_at = now()\n FROM locked_job\n WHERE job_queue.id = locked_job.id\n AND job_queue.lease_expires_at > clock_timestamp()", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Int4", - "Int4", - "Text", - "Text", - "Int8", - "Int8", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "f6b58ff379b8af9e2ab6346743ad1da85494d295268da6627d59c2bcc70b9fb0" -} diff --git a/runledger-postgres/Cargo.toml b/runledger-postgres/Cargo.toml index c1d85fc..29f15b2 100644 --- a/runledger-postgres/Cargo.toml +++ b/runledger-postgres/Cargo.toml @@ -25,6 +25,7 @@ cron = { workspace = true } runledger-core = { workspace = true, features = ["sqlx-postgres"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +sha2 = "0.10.9" sqlx = { version = "0.8.6", features = ["runtime-tokio", "postgres", "uuid", "chrono", "macros", "migrate"] } tokio = { workspace = true } tracing = { workspace = true } diff --git a/runledger-postgres/examples/enqueue_job.rs b/runledger-postgres/examples/enqueue_job.rs index 551dcdb..0ec2664 100644 --- a/runledger-postgres/examples/enqueue_job.rs +++ b/runledger-postgres/examples/enqueue_job.rs @@ -34,6 +34,10 @@ async fn main() -> Result<(), Box> { .await?; println!("enqueued job_id={job_id} job_type={SEND_EMAIL_JOB}"); + // This application enqueues global jobs, so it selects exact global visibility. + if let Some(job) = get_job_by_id_with_scope(&pool, JobReadScope::Global, job_id).await? { + println!("job status={:?}", job.status); + } Ok(()) } diff --git a/runledger-postgres/examples/migration_identity.rs b/runledger-postgres/examples/migration_identity.rs new file mode 100644 index 0000000..222ed18 --- /dev/null +++ b/runledger-postgres/examples/migration_identity.rs @@ -0,0 +1,41 @@ +//! Compose Runledger identity with an application-owned SQLx migration pipeline. +//! Run with `cargo run -p runledger-postgres --example migration_identity`. +//! No database is opened; replace the empty host migrator with your existing one. + +use sha2::{Digest, Sha256}; +use sqlx::migrate::Migrator; + +fn add_frame(digest: &mut Sha256, bytes: &[u8]) { + digest.update((bytes.len() as u64).to_be_bytes()); + digest.update(bytes); +} + +fn template_identity(host_pipeline_revision: &str, host: &Migrator) -> [u8; 32] { + let mut digest = Sha256::new(); + // The host owns this domain and revises it when ordering, recovery, + // configuration, or other non-SQL template initialization changes. + add_frame(&mut digest, host_pipeline_revision.as_bytes()); + add_frame(&mut digest, b"runledger-postgres:pipeline-fingerprint"); + add_frame( + &mut digest, + &runledger_postgres::migration_bundle().pipeline_fingerprint(), + ); + // Include each additional library's helper identity and migration inputs + // here, in the host's chosen order, before the host migration inputs. + for migration in host.iter() { + add_frame(&mut digest, b"application:migration"); + add_frame(&mut digest, &migration.version.to_be_bytes()); + add_frame(&mut digest, migration.description.as_bytes()); + add_frame(&mut digest, migration.migration_type.suffix().as_bytes()); + add_frame(&mut digest, &migration.checksum); + add_frame(&mut digest, &[u8::from(migration.no_tx)]); + } + digest.finalize().into() +} + +fn main() { + // In a host application: use its existing sqlx::migrate!("./migrations"). + let host = Migrator::DEFAULT; + let fingerprint = template_identity("my-application:migration-pipeline:v1", &host); + println!("composed template identity: {fingerprint:02x?}"); +} diff --git a/runledger-postgres/migrations/202609050001_job_summary_pagination.down.sql b/runledger-postgres/migrations/202609050001_job_summary_pagination.down.sql new file mode 100644 index 0000000..113842c --- /dev/null +++ b/runledger-postgres/migrations/202609050001_job_summary_pagination.down.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS idx_job_queue_created_id; +DROP INDEX IF EXISTS idx_job_queue_scope_created_id; diff --git a/runledger-postgres/migrations/202609050001_job_summary_pagination.up.sql b/runledger-postgres/migrations/202609050001_job_summary_pagination.up.sql new file mode 100644 index 0000000..597676f --- /dev/null +++ b/runledger-postgres/migrations/202609050001_job_summary_pagination.up.sql @@ -0,0 +1,9 @@ +-- Support compact pages with and without an exact organization scope. +-- Include the UUID tie-breaker so deep cursors constrain the full ordering key. +CREATE INDEX idx_job_queue_scope_created_id + ON job_queue (organization_id, created_at DESC, id DESC); +CREATE INDEX idx_job_queue_created_id + ON job_queue (created_at DESC, id DESC); + +-- Additive indexes do not change the persisted contract. Omit this migration +-- from runledger_migration_history so older filtered startup helpers coexist. diff --git a/runledger-postgres/src/error.rs b/runledger-postgres/src/error.rs index 0fac9cf..d7a5088 100644 --- a/runledger-postgres/src/error.rs +++ b/runledger-postgres/src/error.rs @@ -1,5 +1,6 @@ use std::{fmt, sync::Arc}; +use runledger_core::jobs::JobProgressValidationError; use sqlx::error::ErrorKind; mod classify; @@ -158,6 +159,7 @@ impl FrameworkConstraintSpec { enum QueryErrorClassification { Fixed(QueryErrorKind), Classified(QueryErrorSpec), + InvalidProgress(JobProgressValidationError), } impl QueryErrorClassification { @@ -165,13 +167,16 @@ impl QueryErrorClassification { match self { Self::Fixed(kind) => kind.spec(), Self::Classified(spec) => *spec, + Self::InvalidProgress(_) => { + QueryErrorSpec::validation("job.invalid_progress", "Job progress is invalid.") + } } } const fn kind(&self) -> Option { match self { Self::Fixed(kind) => Some(*kind), - Self::Classified(_) => None, + Self::Classified(_) | Self::InvalidProgress(_) => None, } } } @@ -186,6 +191,27 @@ pub struct QueryError { } impl QueryError { + pub(crate) fn from_invalid_progress(error: JobProgressValidationError) -> Self { + Self { + classification: QueryErrorClassification::InvalidProgress(error), + sqlstate: None, + constraint: None, + message: error.to_string(), + source: None, + } + } + + /// Returns the typed violation of the effective durable progress values. + /// Runtime adapters can preserve this validation failure without parsing + /// diagnostic strings or mistaking it for a transient database failure. + #[must_use] + pub const fn progress_validation_error(&self) -> Option { + match self.classification { + QueryErrorClassification::InvalidProgress(error) => Some(error), + _ => None, + } + } + #[must_use] pub fn from_classified( category: QueryErrorCategory, diff --git a/runledger-postgres/src/jobs.rs b/runledger-postgres/src/jobs.rs index fdb4ade..3810300 100644 --- a/runledger-postgres/src/jobs.rs +++ b/runledger-postgres/src/jobs.rs @@ -24,6 +24,7 @@ mod rows; mod runtime_configs; mod schedule_definition_guard; mod schedules; +mod scoped_read; mod transaction_isolation; mod transaction_settings; mod types; @@ -33,11 +34,14 @@ mod workflows; pub use admin::{ JobPayloadUuidArrayFieldUpdate, JobPayloadUuidArrayFieldUpdateRejection, cancel_job, cancel_job_with_scope, compare_and_requeue_job, compare_and_requeue_job_tx, get_job_by_id, - get_job_continuation_metrics, get_job_metrics, get_job_payload_by_idempotency_key, - get_latest_job_payload_for_run, list_job_events, list_jobs, - update_job_payload_uuid_array_field, + get_job_by_id_with_scope, get_job_continuation_metrics, + get_job_continuation_metrics_with_scope, get_job_metrics, get_job_metrics_with_scope, + get_job_payload_by_idempotency_key, get_job_payload_by_idempotency_key_with_scope, + get_job_statuses_with_scope, get_latest_job_payload_for_run, + get_latest_job_payload_for_run_with_scope, list_job_events, list_job_events_with_scope, + list_job_summaries, list_jobs, list_jobs_with_scope, update_job_payload_uuid_array_field, }; -pub use logs::{insert_job_log, list_job_logs}; +pub use logs::{insert_job_log, list_job_logs, list_job_logs_with_scope}; #[allow( deprecated, reason = "deprecated stage-bearing progress APIs remain re-exported for semver compatibility" @@ -53,13 +57,14 @@ pub use queue::{ complete_job_success_with_outcome_for_lease, delete_promoted_job_enqueue_intents_before, delete_promoted_job_enqueue_intents_for_jobs_tx, enqueue_job, enqueue_job_tx, enqueue_job_with_execution_resource, enqueue_job_with_execution_resource_tx, - enqueue_job_with_outcome_tx, get_job_definition_by_type, get_job_enqueue_intent_by_id, - get_job_enqueue_intent_metrics, heartbeat_job, heartbeat_job_for_lease, - insert_job_definition_if_missing_tx, list_job_definitions, list_job_enqueue_intents, - mark_job_running, mark_job_running_for_lease, promote_job_enqueue_intents_for_types, - reap_expired_leases, reap_expired_leases_with_diagnostics, - reap_expired_leases_with_terminal_records, record_job_enqueue_intent, - record_job_enqueue_intent_tx, release_unstarted_job_claim, + enqueue_job_with_outcome, enqueue_job_with_outcome_tx, get_job_definition_by_type, + get_job_enqueue_intent_by_id, get_job_enqueue_intent_by_id_with_scope, + get_job_enqueue_intent_metrics, get_job_enqueue_intent_metrics_with_scope, heartbeat_job, + heartbeat_job_for_lease, insert_job_definition_if_missing_tx, list_job_definitions, + list_job_enqueue_intents, list_job_enqueue_intents_with_scope, mark_job_running, + mark_job_running_for_lease, promote_job_enqueue_intents_for_types, reap_expired_leases, + reap_expired_leases_with_diagnostics, reap_expired_leases_with_terminal_records, + record_job_enqueue_intent, record_job_enqueue_intent_tx, release_unstarted_job_claim, sync_catalog_job_definitions_exact_tx, sync_catalog_job_definitions_tx, update_job_definition, update_job_ordinary_progress, update_job_ordinary_progress_for_lease, update_job_progress, update_job_progress_for_lease, upsert_job_definition_tx, @@ -92,14 +97,16 @@ pub use types::{ JobDefinitionUpdate, JobDefinitionUpsert, JobEnqueue, JobEnqueueDisposition, JobEnqueueIntent, JobEnqueueIntentDisposition, JobEnqueueIntentListFilter, JobEnqueueIntentMetricsFilter, JobEnqueueIntentMetricsRecord, JobEnqueueIntentOutcome, JobEnqueueIntentOutcomeState, - JobEnqueueIntentPromotionError, JobEnqueueIntentPromotionReport, JobEnqueueIntentRecord, + JobEnqueueIntentPromotionError, JobEnqueueIntentPromotionReport, + JobEnqueueIntentReadListFilter, JobEnqueueIntentReadMetricsFilter, JobEnqueueIntentRecord, JobEnqueueIntentState, JobEnqueueIntentStatus, JobEnqueueOutcome, JobEventRecord, JobFailureCompletionDisposition, JobFailureCompletionOutcome, JobFailureUpdate, JobLeaseIdentity, JobListFilter, JobLogRecord, JobLogRecordInput, JobMetricsRecord, - JobOrdinaryProgressUpdate, JobProgressUpdate, JobQueueRecord, JobRequeueStatePolicy, - JobRunningUpdate, JobRuntimeConfigListFilter, JobRuntimeConfigRecord, JobRuntimeConfigUpsert, - JobScheduleCatalogSyncEntry, JobScheduleCatalogSyncReport, JobScheduleJobTypeReference, - JobScheduleRecord, JobScheduleUpsert, JobScope, JobSuccessCompletionOutcome, + JobOrdinaryProgressUpdate, JobProgressUpdate, JobQueueRecord, JobReadListFilter, JobReadScope, + JobRequeueStatePolicy, JobRunningUpdate, JobRuntimeConfigListFilter, JobRuntimeConfigRecord, + JobRuntimeConfigUpsert, JobScheduleCatalogSyncEntry, JobScheduleCatalogSyncReport, + JobScheduleJobTypeReference, JobScheduleRecord, JobScheduleUpsert, JobScope, JobStatusRecord, + JobSuccessCompletionOutcome, JobSummary, JobSummaryCursor, JobSummaryFilter, NonRequeueableJobStatusError, ReapExpiredLeaseCleanupError, ReapExpiredLeaseCleanupOperation, ReapExpiredLeaseDeferredError, ReapExpiredLeasesDetailedResult, ReapExpiredLeasesResult, ReapedLeaseDisposition, ReapedLeaseRecord, ReapedTerminalLeaseRecord, RequeueableJobStatus, diff --git a/runledger-postgres/src/jobs/admin.rs b/runledger-postgres/src/jobs/admin.rs index ca71aa8..6fca61c 100644 --- a/runledger-postgres/src/jobs/admin.rs +++ b/runledger-postgres/src/jobs/admin.rs @@ -2,18 +2,26 @@ mod metrics; mod payload; mod read; mod recovery; +mod summary; -pub use metrics::{get_job_continuation_metrics, get_job_metrics}; +pub use metrics::{ + get_job_continuation_metrics, get_job_continuation_metrics_with_scope, get_job_metrics, + get_job_metrics_with_scope, +}; pub use payload::{ JobPayloadUuidArrayFieldUpdate, JobPayloadUuidArrayFieldUpdateRejection, update_job_payload_uuid_array_field, }; pub use read::{ - get_job_by_id, get_job_payload_by_idempotency_key, get_latest_job_payload_for_run, - list_job_events, list_jobs, + get_job_by_id, get_job_by_id_with_scope, get_job_payload_by_idempotency_key, + get_job_payload_by_idempotency_key_with_scope, get_latest_job_payload_for_run, + get_latest_job_payload_for_run_with_scope, list_job_events, list_job_events_with_scope, + list_jobs, list_jobs_with_scope, }; pub use recovery::{ cancel_job, cancel_job_with_scope, compare_and_requeue_job, compare_and_requeue_job_tx, }; +pub use summary::{get_job_statuses_with_scope, list_job_summaries}; + pub(crate) use recovery::cancel_job_with_scope_tx; diff --git a/runledger-postgres/src/jobs/admin/metrics.rs b/runledger-postgres/src/jobs/admin/metrics.rs index 5641020..fdb0320 100644 --- a/runledger-postgres/src/jobs/admin/metrics.rs +++ b/runledger-postgres/src/jobs/admin/metrics.rs @@ -3,13 +3,26 @@ use sqlx::types::Uuid; use crate::{DbPool, Error, Result}; use super::super::row_decode::parse_job_type_name; -use super::super::types::{JobContinuationMetricsRecord, JobMetricsRecord}; +use super::super::types::{JobContinuationMetricsRecord, JobMetricsRecord, JobReadScope}; +/// Legacy read: `None` aggregates all scopes; `Some(id)` selects one tenant. +/// Prefer [`get_job_metrics_with_scope`] for explicit visibility. pub async fn get_job_metrics( pool: &DbPool, organization_id: Option, job_type: Option<&str>, ) -> Result> { + get_job_metrics_with_scope(pool, JobReadScope::from_legacy(organization_id), job_type).await +} + +/// Returns metrics for exactly the selected visibility, including zero-count definitions. +/// Applications must authorize the selected [`JobReadScope`]. +pub async fn get_job_metrics_with_scope( + pool: &DbPool, + scope: JobReadScope, + job_type: Option<&str>, +) -> Result> { + let (is_admin, organization_id) = scope.visibility_predicate(); let rows = sqlx::query!( "SELECT jd.job_type AS \"job_type!\", @@ -27,12 +40,13 @@ pub async fn get_job_metrics( FROM job_definitions jd LEFT JOIN job_metrics_rollup jmr ON jmr.job_type = jd.job_type - AND ($1::uuid IS NULL OR jmr.organization_id = $1) + AND ($3::boolean OR (jmr.organization_id = $1 OR ($1::uuid IS NULL AND jmr.organization_id IS NULL))) WHERE ($2::text IS NULL OR jd.job_type = $2) GROUP BY jd.job_type ORDER BY jd.job_type ASC", organization_id, job_type, + is_admin, ) .fetch_all(pool) .await @@ -59,11 +73,29 @@ pub async fn get_job_metrics( } /// Returns continuation-specific canary and runaway-loop signals by job type. +/// Legacy read: `None` aggregates all scopes; `Some(id)` selects one tenant. +/// Prefer [`get_job_continuation_metrics_with_scope`] for explicit visibility. pub async fn get_job_continuation_metrics( pool: &DbPool, organization_id: Option, job_type: Option<&str>, ) -> Result> { + get_job_continuation_metrics_with_scope( + pool, + JobReadScope::from_legacy(organization_id), + job_type, + ) + .await +} + +/// Returns metrics for exactly the selected visibility, including zero-count definitions. +/// Applications must authorize the selected [`JobReadScope`]. +pub async fn get_job_continuation_metrics_with_scope( + pool: &DbPool, + scope: JobReadScope, + job_type: Option<&str>, +) -> Result> { + let (is_admin, organization_id) = scope.visibility_predicate(); let rows = sqlx::query!( "SELECT jd.job_type AS \"job_type!\", @@ -73,12 +105,13 @@ pub async fn get_job_continuation_metrics( FROM job_definitions jd LEFT JOIN job_continuation_metrics_rollup jcmr ON jcmr.job_type = jd.job_type - AND ($1::uuid IS NULL OR jcmr.organization_id = $1) + AND ($3::boolean OR (jcmr.organization_id = $1 OR ($1::uuid IS NULL AND jcmr.organization_id IS NULL))) WHERE ($2::text IS NULL OR jd.job_type = $2) GROUP BY jd.job_type ORDER BY jd.job_type ASC", organization_id, job_type, + is_admin, ) .fetch_all(pool) .await diff --git a/runledger-postgres/src/jobs/admin/read.rs b/runledger-postgres/src/jobs/admin/read.rs index 9579181..b494edf 100644 --- a/runledger-postgres/src/jobs/admin/read.rs +++ b/runledger-postgres/src/jobs/admin/read.rs @@ -6,15 +6,44 @@ use crate::{DbPool, Error, Result}; use super::super::errors::{validate_page_limit, validate_pagination}; use super::super::row_decode::{parse_job_event_type, parse_job_stage}; use super::super::rows::JobQueueRow; -use super::super::types::{JobEventRecord, JobListFilter, JobQueueRecord}; +use super::super::types::{ + JobEventRecord, JobListFilter, JobQueueRecord, JobReadListFilter, JobReadScope, JobScope, +}; +struct JobPayloadRow { + id: Uuid, + payload: serde_json::Value, +} + +/// Lists jobs with legacy visibility: a None organization matches every scope. +/// Prefer [`list_jobs_with_scope`] for new code. pub async fn list_jobs(pool: &DbPool, filter: &JobListFilter<'_>) -> Result> { + list_jobs_with_scope( + pool, + &JobReadListFilter { + scope: JobReadScope::from_legacy(filter.organization_id), + status: filter.status, + job_type: filter.job_type, + limit: filter.limit, + offset: filter.offset, + }, + ) + .await +} + +/// Lists jobs within an application-authorized, explicit visibility scope. +pub async fn list_jobs_with_scope( + pool: &DbPool, + filter: &JobReadListFilter<'_>, +) -> Result> { validate_pagination(filter.limit, filter.offset)?; let status_filter = filter.status.map(JobStatus::as_db_value); - let rows = sqlx::query_as!( + let rows = super::super::scoped_read::scoped_list!( JobQueueRow, + pool, + filter.scope, "SELECT id, job_type, @@ -45,30 +74,39 @@ pub async fn list_jobs(pool: &DbPool, filter: &JobListFilter<'_>) -> Result, job_id: Uuid, ) -> Result> { + get_job_by_id_with_scope(pool, JobReadScope::from_legacy(organization_id), job_id).await +} + +/// Reads within an application-authorized, explicit job visibility scope. +pub async fn get_job_by_id_with_scope( + pool: &DbPool, + scope: JobReadScope, + job_id: Uuid, +) -> Result> { + let (is_admin, organization_id) = scope.visibility_predicate(); let row = sqlx::query_as!( JobQueueRow, "SELECT @@ -102,10 +140,11 @@ pub async fn get_job_by_id( updated_at FROM job_queue WHERE id = $1 - AND ($2::uuid IS NULL OR organization_id = $2) + AND ($3::bool OR organization_id IS NOT DISTINCT FROM $2::uuid) LIMIT 1", job_id, organization_id, + is_admin, ) .fetch_optional(pool) .await @@ -114,25 +153,41 @@ pub async fn get_job_by_id( row.map(JobQueueRow::into_record).transpose() } +/// Tenant-only compatibility wrapper for [`get_job_payload_by_idempotency_key_with_scope`]. pub async fn get_job_payload_by_idempotency_key( pool: &DbPool, organization_id: Uuid, job_type: JobType<'_>, idempotency_key: &str, ) -> Result> { - let row = sqlx::query!( - "SELECT id, payload - FROM job_queue - WHERE organization_id = $1 - AND job_type = $2 + get_job_payload_by_idempotency_key_with_scope( + pool, + JobScope::Organization(organization_id), + job_type, + idempotency_key, + ) + .await +} + +/// Looks up a payload in one exact global or tenant scope, returning `None` if absent. +/// Applications must authorize the selected [`JobScope`]; keys are not unique across scopes. +pub async fn get_job_payload_by_idempotency_key_with_scope( + pool: &DbPool, + scope: JobScope, + job_type: JobType<'_>, + idempotency_key: &str, +) -> Result> { + let row = super::super::scoped_read::scoped_lookup!( + JobPayloadRow, + pool, + scope, + "SELECT id, payload FROM job_queue WHERE", + "AND job_type = $2 AND idempotency_key = $3 LIMIT 1", - organization_id, job_type as _, idempotency_key, ) - .fetch_optional(pool) - .await .map_err(|error| { Error::from_query_sqlx_with_context("get job payload by idempotency key", error) })?; @@ -140,27 +195,44 @@ pub async fn get_job_payload_by_idempotency_key( Ok(row.map(|row| (row.id, row.payload))) } +/// Tenant-only compatibility wrapper for [`get_latest_job_payload_for_run_with_scope`]. pub async fn get_latest_job_payload_for_run( pool: &DbPool, organization_id: Uuid, job_type: JobType<'_>, run_id: Uuid, +) -> Result> { + get_latest_job_payload_for_run_with_scope( + pool, + JobScope::Organization(organization_id), + job_type, + run_id, + ) + .await +} + +/// Looks up a payload in one exact global or tenant scope, returning `None` if absent. +/// Applications must authorize the selected [`JobScope`]; keys are not unique across scopes. +/// Selects the newest `created_at`, breaking timestamp ties by descending job ID. +pub async fn get_latest_job_payload_for_run_with_scope( + pool: &DbPool, + scope: JobScope, + job_type: JobType<'_>, + run_id: Uuid, ) -> Result> { let run_id_text = run_id.to_string(); - let row = sqlx::query!( - "SELECT id, payload - FROM job_queue - WHERE organization_id = $1 - AND job_type = $2 + let row = super::super::scoped_read::scoped_lookup!( + JobPayloadRow, + pool, + scope, + "SELECT id, payload FROM job_queue WHERE", + "AND job_type = $2 AND payload->>'run_id' = $3 ORDER BY created_at DESC, id DESC LIMIT 1", - organization_id, job_type as _, run_id_text, ) - .fetch_optional(pool) - .await .map_err(|error| { Error::from_query_sqlx_with_context("get latest job payload for run", error) })?; @@ -168,6 +240,8 @@ pub async fn get_latest_job_payload_for_run( Ok(row.map(|row| (row.id, row.payload))) } +/// Legacy read: None matches global and organization-owned jobs. +/// Prefer [`list_job_events_with_scope`] for new code. pub async fn list_job_events( pool: &DbPool, organization_id: Option, @@ -175,6 +249,25 @@ pub async fn list_job_events( limit: i64, after_id: Option, ) -> Result> { + list_job_events_with_scope( + pool, + JobReadScope::from_legacy(organization_id), + job_id, + limit, + after_id, + ) + .await +} + +/// Reads within an application-authorized, explicit job visibility scope. +pub async fn list_job_events_with_scope( + pool: &DbPool, + scope: JobReadScope, + job_id: Uuid, + limit: i64, + after_id: Option, +) -> Result> { + let (is_admin, organization_id) = scope.visibility_predicate(); validate_page_limit(limit)?; let rows = sqlx::query!( @@ -192,7 +285,7 @@ pub async fn list_job_events( FROM job_events je JOIN job_queue jq ON jq.id = je.job_id WHERE je.job_id = $1 - AND ($2::uuid IS NULL OR jq.organization_id = $2) + AND ($5::bool OR jq.organization_id IS NOT DISTINCT FROM $2::uuid) AND ($3::bigint IS NULL OR je.id > $3) ORDER BY je.id ASC LIMIT $4", @@ -200,6 +293,7 @@ pub async fn list_job_events( organization_id, after_id, limit, + is_admin, ) .fetch_all(pool) .await diff --git a/runledger-postgres/src/jobs/admin/summary.rs b/runledger-postgres/src/jobs/admin/summary.rs new file mode 100644 index 0000000..4ec247f --- /dev/null +++ b/runledger-postgres/src/jobs/admin/summary.rs @@ -0,0 +1,129 @@ +use chrono::{DateTime, Utc}; +use sqlx::types::Uuid; + +use crate::jobs::errors::validate_page_limit; +use crate::jobs::row_decode::{parse_job_stage, parse_job_status, parse_job_type_name}; +use crate::jobs::types::{JobReadScope, JobStatusRecord, JobSummary, JobSummaryFilter}; +use crate::{DbPool, Error, Result}; + +struct SummaryRow { + id: Uuid, + job_type: String, + organization_id: Option, + status: String, + priority: i32, + run_number: i32, + attempt: i32, + max_attempts: i32, + next_run_at: DateTime, + stage: Option, + progress_done: Option, + progress_total: Option, + created_at: DateTime, + updated_at: DateTime, +} + +/// Reads a compact page in descending `(created_at, id)` order within an +/// application-authorized scope. Empty pages end the scan; use the last row's +/// [`JobSummary::cursor`] to continue. No count query or JSON fields are read. +#[expect( + clippy::cognitive_complexity, + reason = "SQLx expands six statically checked scope/cursor query branches" +)] +pub async fn list_job_summaries( + pool: &DbPool, + filter: &JobSummaryFilter<'_>, +) -> Result> { + validate_page_limit(filter.limit)?; + // Separate first/subsequent-page SQL keeps the tuple comparison indexable + // even when PostgreSQL chooses a generic prepared plan. + macro_rules! page { + ($cursor_predicate:literal, $($cursor_arg:expr),* $(,)?) => { + crate::jobs::scoped_read::scoped_list!( + SummaryRow, pool, filter.scope, + "SELECT id, job_type, organization_id, status::text AS \"status!\", + priority, run_number, attempt, max_attempts, next_run_at, stage, + progress_done, progress_total, created_at, updated_at + FROM job_queue WHERE", + "AND ($2::text::job_status IS NULL OR status = $2::text::job_status) + AND ($3::text IS NULL OR job_type = $3) " + $cursor_predicate + + " ORDER BY created_at DESC, id DESC LIMIT $4", + filter.status.map(|s| s.as_db_value()), + filter.job_type.map(|t| t.as_str()), filter.limit, $($cursor_arg),* + ) + }; + } + let rows = match filter.after { + Some(after) => page!( + "AND (created_at, id) < ($5, $6)", + after.created_at, + after.id + ), + None => page!("",), + } + .map_err(|error| Error::from_query_sqlx_with_context("list job summaries", error))?; + rows.into_iter() + .map(|row| { + Ok(JobSummary { + id: row.id, + job_type: parse_job_type_name(row.job_type)?, + organization_id: row.organization_id, + status: parse_job_status(row.status)?, + priority: row.priority, + run_number: row.run_number, + attempt: row.attempt, + max_attempts: row.max_attempts, + next_run_at: row.next_run_at, + stage: row.stage.map(parse_job_stage).transpose()?, + progress_done: row.progress_done, + progress_total: row.progress_total, + created_at: row.created_at, + updated_at: row.updated_at, + }) + }) + .collect() +} + +struct StatusRow { + id: Uuid, + status: String, + run_number: i32, + attempt: i32, + updated_at: DateTime, +} + +/// Reads up to `JOB_LIST_PAGE_LIMIT_MAX` input IDs in one statement. Duplicate +/// IDs return one row; missing and out-of-scope IDs are both omitted. Results +/// are ordered by ID, not input order. Empty input returns without a query. +/// The application must authorize `scope`; observations confer no mutation rights. +pub async fn get_job_statuses_with_scope( + pool: &DbPool, + scope: JobReadScope, + job_ids: &[Uuid], +) -> Result> { + if job_ids.is_empty() { + return Ok(Vec::new()); + } + validate_page_limit(i64::try_from(job_ids.len()).unwrap_or(i64::MAX))?; + let rows = crate::jobs::scoped_read::scoped_list!( + StatusRow, + pool, + scope, + "SELECT id, status::text AS \"status!\", run_number, attempt, updated_at + FROM job_queue WHERE", + "AND id = ANY($2::uuid[]) ORDER BY id", + job_ids, + ) + .map_err(|error| Error::from_query_sqlx_with_context("get job statuses", error))?; + rows.into_iter() + .map(|row| { + Ok(JobStatusRecord { + id: row.id, + status: parse_job_status(row.status)?, + run_number: row.run_number, + attempt: row.attempt, + updated_at: row.updated_at, + }) + }) + .collect() +} diff --git a/runledger-postgres/src/jobs/errors.rs b/runledger-postgres/src/jobs/errors.rs index 766da9e..03f16f1 100644 --- a/runledger-postgres/src/jobs/errors.rs +++ b/runledger-postgres/src/jobs/errors.rs @@ -122,31 +122,19 @@ pub(super) fn validate_completion_progress( progress_done: Option, progress_total: Option, ) -> Result<()> { - if let Some(progress_done) = progress_done - && progress_done < 0 - { - return Err(invalid_completion_progress_error(format!( - "progress_done must be greater than or equal to zero, got {progress_done}" - ))); - } - - if let Some(progress_total) = progress_total - && progress_total < 0 - { - return Err(invalid_completion_progress_error(format!( - "progress_total must be greater than or equal to zero, got {progress_total}" - ))); - } - - if let (Some(progress_done), Some(progress_total)) = (progress_done, progress_total) - && progress_done > progress_total - { - return Err(invalid_completion_progress_error(format!( - "progress_done must not exceed progress_total, got progress_done={progress_done}, progress_total={progress_total}" - ))); - } - - Ok(()) + runledger_core::jobs::validate_job_progress(progress_done, progress_total).map_err(|error| { + use runledger_core::jobs::JobProgressValidationError; + let detail = match error { + JobProgressValidationError::NegativeDone { actual } => { + format!("progress_done must be greater than or equal to zero, got {actual}") + } + JobProgressValidationError::NegativeTotal { actual } => { + format!("progress_total must be greater than or equal to zero, got {actual}") + } + _ => error.to_string(), + }; + invalid_completion_progress_error(detail) + }) } pub(super) fn invalid_continuation_delay_error(detail: String) -> Error { diff --git a/runledger-postgres/src/jobs/logs.rs b/runledger-postgres/src/jobs/logs.rs index 6210d65..a18a96b 100644 --- a/runledger-postgres/src/jobs/logs.rs +++ b/runledger-postgres/src/jobs/logs.rs @@ -3,7 +3,7 @@ use sqlx::types::Uuid; use crate::{DbPool, Error, Result}; use super::errors::validate_page_limit; -use super::types::{JobLogRecord, JobLogRecordInput}; +use super::types::{JobLogRecord, JobLogRecordInput, JobReadScope}; pub async fn insert_job_log(pool: &DbPool, input: &JobLogRecordInput) -> Result<()> { sqlx::query!( @@ -30,6 +30,8 @@ pub async fn insert_job_log(pool: &DbPool, input: &JobLogRecordInput) -> Result< Ok(()) } +/// Legacy read: None matches logs from global and all organization-owned jobs. +/// Prefer [`list_job_logs_with_scope`] for new code. pub async fn list_job_logs( pool: &DbPool, organization_id: Option, @@ -37,6 +39,25 @@ pub async fn list_job_logs( limit: i64, after_id: Option, ) -> Result> { + list_job_logs_with_scope( + pool, + JobReadScope::from_legacy(organization_id), + job_id, + limit, + after_id, + ) + .await +} + +/// Lists logs within an application-authorized, explicit job visibility scope. +pub async fn list_job_logs_with_scope( + pool: &DbPool, + scope: JobReadScope, + job_id: Uuid, + limit: i64, + after_id: Option, +) -> Result> { + let (is_admin, organization_id) = scope.visibility_predicate(); validate_page_limit(limit)?; sqlx::query_as!( @@ -53,7 +74,7 @@ pub async fn list_job_logs( FROM job_logs jl JOIN job_queue jq ON jq.id = jl.job_id WHERE jl.job_id = $1 - AND ($2::uuid IS NULL OR jq.organization_id = $2) + AND ($5::bool OR jq.organization_id IS NOT DISTINCT FROM $2::uuid) AND ($3::bigint IS NULL OR jl.id > $3) ORDER BY jl.id ASC LIMIT $4", @@ -61,6 +82,7 @@ pub async fn list_job_logs( organization_id, after_id, limit, + is_admin, ) .fetch_all(pool) .await diff --git a/runledger-postgres/src/jobs/queue.rs b/runledger-postgres/src/jobs/queue.rs index b2ded20..6a210d2 100644 --- a/runledger-postgres/src/jobs/queue.rs +++ b/runledger-postgres/src/jobs/queue.rs @@ -23,11 +23,13 @@ pub use self::definitions::{ pub(in crate::jobs) use self::enqueue::enqueue_replayed_job_with_outcome_tx; pub use self::enqueue::{ enqueue_job, enqueue_job_tx, enqueue_job_with_execution_resource, - enqueue_job_with_execution_resource_tx, enqueue_job_with_outcome_tx, + enqueue_job_with_execution_resource_tx, enqueue_job_with_outcome, enqueue_job_with_outcome_tx, }; pub use self::intents::{ delete_promoted_job_enqueue_intents_before, delete_promoted_job_enqueue_intents_for_jobs_tx, - get_job_enqueue_intent_by_id, get_job_enqueue_intent_metrics, list_job_enqueue_intents, + get_job_enqueue_intent_by_id, get_job_enqueue_intent_by_id_with_scope, + get_job_enqueue_intent_metrics, get_job_enqueue_intent_metrics_with_scope, + list_job_enqueue_intents, list_job_enqueue_intents_with_scope, promote_job_enqueue_intents_for_types, record_job_enqueue_intent, record_job_enqueue_intent_tx, }; #[allow( diff --git a/runledger-postgres/src/jobs/queue/enqueue.rs b/runledger-postgres/src/jobs/queue/enqueue.rs index 899f2a7..bde8fe2 100644 --- a/runledger-postgres/src/jobs/queue/enqueue.rs +++ b/runledger-postgres/src/jobs/queue/enqueue.rs @@ -726,6 +726,24 @@ pub async fn enqueue_job(pool: &DbPool, payload: &JobEnqueue<'_>) -> Result, +) -> Result { + let mut tx = pool + .begin() + .await + .map_err(|error| Error::ConnectionError(error.to_string()))?; + let outcome = enqueue_job_with_outcome_tx(&mut tx, payload).await?; + tx.commit() + .await + .map_err(|error| Error::ConnectionError(error.to_string()))?; + Ok(outcome) +} + /// Enqueues a resource-constrained job in its own transaction. /// /// See [`enqueue_job_with_execution_resource_tx`] for key scope, validation, diff --git a/runledger-postgres/src/jobs/queue/intents.rs b/runledger-postgres/src/jobs/queue/intents.rs index 86fa531..15e521f 100644 --- a/runledger-postgres/src/jobs/queue/intents.rs +++ b/runledger-postgres/src/jobs/queue/intents.rs @@ -26,7 +26,8 @@ use super::super::types::{ JobEnqueue, JobEnqueueDisposition, JobEnqueueIntent, JobEnqueueIntentDisposition, JobEnqueueIntentListFilter, JobEnqueueIntentMetricsFilter, JobEnqueueIntentMetricsRecord, JobEnqueueIntentOutcome, JobEnqueueIntentOutcomeState, JobEnqueueIntentPromotionReport, - JobEnqueueIntentRecord, JobEnqueueIntentStatus, + JobEnqueueIntentReadListFilter, JobEnqueueIntentReadMetricsFilter, JobEnqueueIntentRecord, + JobEnqueueIntentStatus, JobReadScope, }; use super::enqueue::{ IntentEnqueueResolution, JOB_ENQUEUE_REQUEST_VERSION, canonical_job_enqueue_request_v1, @@ -493,11 +494,27 @@ async fn load_existing_intent_with_key_share( /// /// Passing an organization ID filters to that tenant. Passing `None` performs /// an administrator-wide lookup; authentication remains the caller's concern. +/// Prefer [`get_job_enqueue_intent_by_id_with_scope`] for new code. pub async fn get_job_enqueue_intent_by_id( pool: &DbPool, organization_id: Option, intent_id: Uuid, ) -> Result> { + get_job_enqueue_intent_by_id_with_scope( + pool, + JobReadScope::from_legacy(organization_id), + intent_id, + ) + .await +} + +/// Loads an intent within an application-authorized, explicit visibility scope. +pub async fn get_job_enqueue_intent_by_id_with_scope( + pool: &DbPool, + scope: JobReadScope, + intent_id: Uuid, +) -> Result> { + let (is_admin, organization_id) = scope.visibility_predicate(); let row = sqlx::query_as!( JobEnqueueIntentRecordRow, "SELECT @@ -526,10 +543,11 @@ pub async fn get_job_enqueue_intent_by_id( updated_at FROM job_enqueue_intents WHERE id = $1 - AND ($2::uuid IS NULL OR organization_id = $2) + AND ($3::bool OR organization_id IS NOT DISTINCT FROM $2::uuid) LIMIT 1", intent_id, organization_id, + is_admin, ) .fetch_optional(pool) .await @@ -538,16 +556,37 @@ pub async fn get_job_enqueue_intent_by_id( row.map(JobEnqueueIntentRecordRow::into_record).transpose() } -/// Lists durable enqueue intents with bounded pagination. +/// Lists intents with legacy visibility: no organization filter matches all scopes. +/// Prefer [`list_job_enqueue_intents_with_scope`] for new code. pub async fn list_job_enqueue_intents( pool: &DbPool, filter: &JobEnqueueIntentListFilter<'_>, +) -> Result> { + list_job_enqueue_intents_with_scope( + pool, + &JobEnqueueIntentReadListFilter { + scope: JobReadScope::from_legacy(filter.organization_id), + status: filter.status, + job_type_query: filter.job_type_query, + limit: filter.limit, + offset: filter.offset, + }, + ) + .await +} + +/// Lists intents within an application-authorized, explicit visibility scope. +pub async fn list_job_enqueue_intents_with_scope( + pool: &DbPool, + filter: &JobEnqueueIntentReadListFilter<'_>, ) -> Result> { validate_pagination(filter.limit, filter.offset)?; let status = filter.status.map(JobEnqueueIntentStatus::as_db_value); - let rows = sqlx::query_as!( + let rows = super::super::scoped_read::scoped_list!( JobEnqueueIntentRecordRow, + pool, + filter.scope, "SELECT id, job_type, @@ -573,20 +612,17 @@ pub async fn list_job_enqueue_intents( created_at, updated_at FROM job_enqueue_intents - WHERE ($1::uuid IS NULL OR organization_id = $1) - AND ($2::text IS NULL OR status = $2) + WHERE", + "AND ($2::text IS NULL OR status = $2) AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%') ORDER BY created_at DESC, id DESC LIMIT $4 OFFSET $5", - filter.organization_id, status, filter.job_type_query, filter.limit, filter.offset, ) - .fetch_all(pool) - .await .map_err(|error| Error::from_query_sqlx_with_context("list job enqueue intents", error))?; rows.into_iter() @@ -596,6 +632,9 @@ pub async fn list_job_enqueue_intents( /// Returns durable intent backlog and promotion signals grouped by job type. /// +/// Legacy scope: no organization filter aggregates all scopes; an organization +/// selects only that tenant. Prefer [`get_job_enqueue_intent_metrics_with_scope`]. +/// /// Each lifecycle population is aggregated behind its own selective predicate. /// `pending_count`, `retrying_count`, `max_promotion_attempts`, and /// `oldest_pending_at` describe only intents that are currently pending; @@ -610,6 +649,26 @@ pub async fn get_job_enqueue_intent_metrics( pool: &DbPool, filter: &JobEnqueueIntentMetricsFilter<'_>, ) -> Result> { + get_job_enqueue_intent_metrics_with_scope( + pool, + &JobEnqueueIntentReadMetricsFilter { + scope: JobReadScope::from_legacy(filter.organization_id), + job_type: filter.job_type, + limit: filter.limit, + offset: filter.offset, + }, + ) + .await +} + +/// Returns intent metrics for the selected visibility with the same lifecycle, +/// ordering, and pagination semantics as [`get_job_enqueue_intent_metrics`]. +/// Applications must authorize the selected [`JobReadScope`]. +pub async fn get_job_enqueue_intent_metrics_with_scope( + pool: &DbPool, + filter: &JobEnqueueIntentReadMetricsFilter<'_>, +) -> Result> { + let (is_admin, organization_id) = filter.scope.visibility_predicate(); validate_pagination(filter.limit, filter.offset)?; let job_type = filter.job_type.map(|job_type| job_type.as_str()); let rows = sqlx::query_as!( @@ -625,7 +684,7 @@ pub async fn get_job_enqueue_intent_metrics( MIN(created_at) AS oldest_pending_at FROM job_enqueue_intents WHERE status = 'PENDING' - AND ($1::uuid IS NULL OR organization_id = $1) + AND ($5::boolean OR (organization_id = $1 OR ($1::uuid IS NULL AND organization_id IS NULL))) AND ($2::text IS NULL OR job_type = $2) GROUP BY job_type @@ -642,7 +701,7 @@ pub async fn get_job_enqueue_intent_metrics( FROM job_enqueue_intents WHERE status = 'CONFLICTED' AND conflicted_at >= now() - interval '24 hours' - AND ($1::uuid IS NULL OR organization_id = $1) + AND ($5::boolean OR (organization_id = $1 OR ($1::uuid IS NULL AND organization_id IS NULL))) AND ($2::text IS NULL OR job_type = $2) GROUP BY job_type @@ -659,7 +718,7 @@ pub async fn get_job_enqueue_intent_metrics( FROM job_enqueue_intents WHERE status = 'PROMOTED' AND promoted_at >= now() - interval '24 hours' - AND ($1::uuid IS NULL OR organization_id = $1) + AND ($5::boolean OR (organization_id = $1 OR ($1::uuid IS NULL AND organization_id IS NULL))) AND ($2::text IS NULL OR job_type = $2) GROUP BY job_type ) @@ -676,10 +735,11 @@ pub async fn get_job_enqueue_intent_metrics( ORDER BY job_type LIMIT $3 OFFSET $4", - filter.organization_id, + organization_id, job_type, filter.limit, filter.offset, + is_admin, ) .fetch_all(pool) .await diff --git a/runledger-postgres/src/jobs/queue/lifecycle/common.rs b/runledger-postgres/src/jobs/queue/lifecycle/common.rs index 3104428..b86e790 100644 --- a/runledger-postgres/src/jobs/queue/lifecycle/common.rs +++ b/runledger-postgres/src/jobs/queue/lifecycle/common.rs @@ -22,7 +22,7 @@ pub(super) const COMPLETE_CONTINUATION_LEASE_MISMATCH_CONTEXT: &str = pub(super) const COMPLETE_FAILURE_LEASE_MISMATCH_CONTEXT: &str = "complete job failure transaction missing leased row"; -pub(super) struct CompletionLeaseRow { +pub(super) struct LiveJobLeaseRow { pub(super) job_type: String, pub(super) organization_id: Option, pub(super) max_attempts: i32, @@ -31,18 +31,36 @@ pub(super) struct CompletionLeaseRow { pub(super) completion_base_at: DateTime, } -pub(super) async fn lock_live_completion_lease_tx( +pub(super) async fn lock_live_job_lease_tx( tx: &mut DbTx<'_>, identity: JobLeaseIdentity<'_>, error_context: &'static str, -) -> Result> { +) -> Result> { let previous_lock_timeout = cap_completion_job_row_lock_timeout_tx( tx, "cap job completion row-lock acquisition timeout", ) .await?; + let row = lock_live_job_lease_with_current_timeouts_tx(tx, identity, error_context).await?; + restore_completion_job_row_lock_timeout_tx( + tx, + &previous_lock_timeout, + "restore job completion row-lock timeout", + ) + .await?; + Ok(row) +} + +/// Acquires the live row using the transaction's existing timeout policy. +/// Bounded progress transactions keep their caps until commit; completion +/// transactions use the wrapper above to cap only initial lock acquisition. +pub(super) async fn lock_live_job_lease_with_current_timeouts_tx( + tx: &mut DbTx<'_>, + identity: JobLeaseIdentity<'_>, + error_context: &'static str, +) -> Result> { let row = sqlx::query_as!( - CompletionLeaseRow, + LiveJobLeaseRow, r#"SELECT job_type AS "job_type!", organization_id, @@ -67,19 +85,13 @@ pub(super) async fn lock_live_completion_lease_tx( .fetch_optional(&mut **tx) .await .map_err(|error| Error::from_query_sqlx_with_context(error_context, error))?; - restore_completion_job_row_lock_timeout_tx( - tx, - &previous_lock_timeout, - "restore job completion row-lock timeout", - ) - .await?; Ok(row) } pub(super) fn coalesce_completion_progress( progress_done: &mut Option, progress_total: &mut Option, - existing: &CompletionLeaseRow, + existing: &LiveJobLeaseRow, ) -> Result<()> { *progress_done = progress_done.or(existing.progress_done); *progress_total = progress_total.or(existing.progress_total); diff --git a/runledger-postgres/src/jobs/queue/lifecycle/continuation.rs b/runledger-postgres/src/jobs/queue/lifecycle/continuation.rs index 497d1c2..013afe5 100644 --- a/runledger-postgres/src/jobs/queue/lifecycle/continuation.rs +++ b/runledger-postgres/src/jobs/queue/lifecycle/continuation.rs @@ -19,8 +19,7 @@ use super::super::events::{ }; use super::common::{ COMPLETE_CONTINUATION_LEASE_MISMATCH_CONTEXT, coalesce_completion_progress, - finish_successful_attempt_tx, lock_live_completion_lease_tx, - rollback_and_return_lease_mismatch, + finish_successful_attempt_tx, lock_live_job_lease_tx, rollback_and_return_lease_mismatch, }; struct ContinuationProgressUpdate<'a> { @@ -117,8 +116,7 @@ pub async fn complete_job_continuation_with_outcome_for_lease( .begin() .await .map_err(|error| Error::ConnectionError(error.to_string()))?; - let Some(lookup) = - lock_live_completion_lease_tx(&mut tx, identity, "lock job continuation").await? + let Some(lookup) = lock_live_job_lease_tx(&mut tx, identity, "lock job continuation").await? else { return rollback_and_return_lease_mismatch( tx, diff --git a/runledger-postgres/src/jobs/queue/lifecycle/progress.rs b/runledger-postgres/src/jobs/queue/lifecycle/progress.rs index 9b7157d..2455732 100644 --- a/runledger-postgres/src/jobs/queue/lifecycle/progress.rs +++ b/runledger-postgres/src/jobs/queue/lifecycle/progress.rs @@ -1,7 +1,7 @@ -use runledger_core::jobs::JobStage; +use runledger_core::jobs::{JobStage, validate_job_progress}; use sqlx::types::Uuid; -use crate::{DbPool, DbTx, Error, Result}; +use crate::{DbPool, DbTx, Error, QueryError, Result}; #[allow( deprecated, @@ -11,7 +11,7 @@ use super::super::super::types::JobProgressUpdate; use super::super::super::types::{JobLeaseIdentity, JobOrdinaryProgressUpdate, JobRunningUpdate}; use super::common::{ UPDATE_PROGRESS_LEASE_MISMATCH_CONTEXT, cap_bounded_job_lifecycle_timeouts_tx, - rollback_and_return_lease_mismatch, + lock_live_job_lease_with_current_timeouts_tx, rollback_and_return_lease_mismatch, }; #[derive(Clone, Copy)] @@ -60,27 +60,22 @@ async fn update_job_progress_row_tx( identity: JobLeaseIdentity<'_>, progress: ProgressMutation<'_>, ) -> Result { + // The caller holds the live lease row through validation and this write. + // Recheck wall-clock expiry immediately before mutation, without acquiring + // the same lock in a second CTE. let rows_affected = sqlx::query!( - "WITH locked_job AS MATERIALIZED ( - SELECT id - FROM job_queue - WHERE id = $1 - AND run_number = $2 - AND attempt = $3 - AND worker_id = $4 - AND status = 'LEASED' - AND lease_expires_at IS NOT NULL - FOR UPDATE - ) - UPDATE job_queue + "UPDATE job_queue SET stage = COALESCE($5, stage), progress_done = COALESCE($6, progress_done), progress_total = COALESCE($7, progress_total), checkpoint = COALESCE($8::jsonb, checkpoint), updated_at = now() - FROM locked_job - WHERE job_queue.id = locked_job.id - AND job_queue.lease_expires_at > clock_timestamp()", + WHERE id = $1 + AND run_number = $2 + AND attempt = $3 + AND worker_id = $4 + AND status = 'LEASED' + AND lease_expires_at > clock_timestamp()", identity.job_id, identity.run_number, identity.attempt, @@ -190,6 +185,25 @@ async fn persist_progress_mutation_for_lease( .map_err(|error| Error::ConnectionError(error.to_string()))?; cap_bounded_job_lifecycle_timeouts_tx(&mut tx, "cap progress lifecycle timeouts").await?; + // Partial updates can only be validated against the current locked values, + // never the handler's invocation snapshot. This owned transaction already + // has lifecycle timeout caps, which stay active through its audit writes. + // Keep the original partial values for persistence and audit events. + let Some(existing) = + lock_live_job_lease_with_current_timeouts_tx(&mut tx, identity, "lock job progress lease") + .await? + else { + return rollback_and_return_lease_mismatch(tx, UPDATE_PROGRESS_LEASE_MISMATCH_CONTEXT) + .await; + }; + if let Err(error) = validate_job_progress( + progress.progress_done.or(existing.progress_done), + progress.progress_total.or(existing.progress_total), + ) { + super::super::super::errors::ensure_rejection_rollback_succeeded(tx.rollback().await)?; + return Err(Error::QueryError(QueryError::from_invalid_progress(error))); + } + let updated = update_job_progress_row_tx(&mut tx, identity, progress).await?; if updated == 0 { diff --git a/runledger-postgres/src/jobs/queue/lifecycle/success.rs b/runledger-postgres/src/jobs/queue/lifecycle/success.rs index a0066bf..04766b0 100644 --- a/runledger-postgres/src/jobs/queue/lifecycle/success.rs +++ b/runledger-postgres/src/jobs/queue/lifecycle/success.rs @@ -14,8 +14,7 @@ use super::super::super::types::{ use super::super::super::workflows::on_terminal; use super::common::{ COMPLETE_SUCCESS_LEASE_MISMATCH_CONTEXT, coalesce_completion_progress, - finish_successful_attempt_tx, lock_live_completion_lease_tx, - rollback_and_return_lease_mismatch, + finish_successful_attempt_tx, lock_live_job_lease_tx, rollback_and_return_lease_mismatch, }; struct SuccessProgressUpdate<'a> { @@ -178,7 +177,7 @@ pub async fn complete_job_success_with_outcome_for_lease( .await .map_err(|error| Error::ConnectionError(error.to_string()))?; let Some(lookup) = - lock_live_completion_lease_tx(&mut tx, identity, "lock job success progress").await? + lock_live_job_lease_tx(&mut tx, identity, "lock job success progress").await? else { return rollback_and_return_lease_mismatch(tx, COMPLETE_SUCCESS_LEASE_MISMATCH_CONTEXT) .await; diff --git a/runledger-postgres/src/jobs/scoped_read.rs b/runledger-postgres/src/jobs/scoped_read.rs new file mode 100644 index 0000000..1635844 --- /dev/null +++ b/runledger-postgres/src/jobs/scoped_read.rs @@ -0,0 +1,56 @@ +// SQLx requires literal SQL for compile-time checking. Keep the projection and +// filters at each call site, but select the scope predicate here. Separate +// statements keep organization predicates indexable even with generic prepared +// plans; a nullable equality or an admin OR flag cannot guarantee that. +// +// $1 is reserved for organization identity in every branch. Global/Admin bind +// NULL and test that parameter so the remaining filter positions stay stable. +macro_rules! scoped_list { + ($row:path, $pool:expr, $scope:expr, $prefix:literal, $suffix:expr, $($arg:expr),+ $(,)?) => { + match $scope { + $crate::jobs::JobReadScope::Organization(id) => { + sqlx::query_as!( + $row, $prefix + " organization_id = $1 " + $suffix, + Some(id), $($arg),+ + ).fetch_all($pool).await + } + $crate::jobs::JobReadScope::Global => { + sqlx::query_as!( + $row, $prefix + " ($1::uuid IS NULL AND organization_id IS NULL) " + $suffix, + None::, $($arg),+ + ).fetch_all($pool).await + } + $crate::jobs::JobReadScope::Admin => { + sqlx::query_as!( + $row, $prefix + " $1::uuid IS NULL " + $suffix, + None::, $($arg),+ + ).fetch_all($pool).await + } + } + }; +} + +pub(super) use scoped_list; + +// Exact-scope lookups deliberately accept JobScope, so an administrative +// wildcard cannot accidentally turn tenant-local keys into global identities. +macro_rules! scoped_lookup { + ($row:path, $pool:expr, $scope:expr, $prefix:literal, $suffix:expr, $($arg:expr),+ $(,)?) => { + match $scope { + $crate::jobs::JobScope::Organization(id) => { + sqlx::query_as!( + $row, $prefix + " organization_id = $1 " + $suffix, + Some(id), $($arg),+ + ).fetch_optional($pool).await + } + $crate::jobs::JobScope::Global => { + sqlx::query_as!( + $row, $prefix + " ($1::uuid IS NULL AND organization_id IS NULL) " + $suffix, + None::, $($arg),+ + ).fetch_optional($pool).await + } + } + }; +} + +pub(super) use scoped_lookup; diff --git a/runledger-postgres/src/jobs/types.rs b/runledger-postgres/src/jobs/types.rs index c81c9d2..6654b8e 100644 --- a/runledger-postgres/src/jobs/types.rs +++ b/runledger-postgres/src/jobs/types.rs @@ -1,6 +1,6 @@ pub use self::admin::{ JOB_LIST_PAGE_LIMIT_MAX, JobCancellationScope, JobContinuationMetricsRecord, JobListFilter, - JobLogRecord, JobLogRecordInput, JobMetricsRecord, + JobLogRecord, JobLogRecordInput, JobMetricsRecord, JobReadListFilter, JobReadScope, }; pub use self::definitions::{ JobDefinitionListFilter, JobDefinitionRecord, JobDefinitionUpdate, JobDefinitionUpsert, @@ -12,9 +12,9 @@ pub use self::enqueue::{ JobEnqueueIntent, JobEnqueueIntentDisposition, JobEnqueueIntentListFilter, JobEnqueueIntentMetricsFilter, JobEnqueueIntentMetricsRecord, JobEnqueueIntentOutcome, JobEnqueueIntentOutcomeState, JobEnqueueIntentPromotionError, JobEnqueueIntentPromotionReport, - JobEnqueueIntentRecord, JobEnqueueIntentState, JobEnqueueIntentStatus, JobEnqueueOutcome, - JobQueueRecord, JobRequeueStatePolicy, JobScope, NonRequeueableJobStatusError, - RequeueableJobStatus, + JobEnqueueIntentReadListFilter, JobEnqueueIntentReadMetricsFilter, JobEnqueueIntentRecord, + JobEnqueueIntentState, JobEnqueueIntentStatus, JobEnqueueOutcome, JobQueueRecord, + JobRequeueStatePolicy, JobScope, NonRequeueableJobStatusError, RequeueableJobStatus, }; pub(crate) use self::events::{ BASIC_REQUEUE_KIND, COMPARE_AND_REQUEUE_KIND, HANDLER_CONTINUATION_KIND, @@ -44,6 +44,8 @@ pub use self::schedules::{ JobScheduleRecord, JobScheduleUpsert, }; +pub use self::summary::{JobStatusRecord, JobSummary, JobSummaryCursor, JobSummaryFilter}; + mod admin; mod definitions; mod enqueue; @@ -51,3 +53,4 @@ mod events; mod lifecycle; mod reaper; mod schedules; +mod summary; diff --git a/runledger-postgres/src/jobs/types/admin.rs b/runledger-postgres/src/jobs/types/admin.rs index 9b90153..9bd6aae 100644 --- a/runledger-postgres/src/jobs/types/admin.rs +++ b/runledger-postgres/src/jobs/types/admin.rs @@ -9,6 +9,49 @@ use sqlx::types::Uuid; /// allowing operators to inspect a large page when needed. pub const JOB_LIST_PAGE_LIMIT_MAX: i64 = 1_000; +/// Explicit visibility for job, event, log, enqueue-intent, and metrics reads. +/// +/// This selects rows, not authorization. Applications must authorize the chosen +/// scope, especially `Admin`, before calling a read API. It grants no mutation +/// or cancellation permission. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum JobReadScope { + /// Match only rows whose job or intent has a NULL organization. + Global, + /// Match only rows belonging to this exact organization. + Organization(Uuid), + /// Match both global and organization-owned rows. + Admin, +} + +impl JobReadScope { + pub(in crate::jobs) const fn from_legacy(organization_id: Option) -> Self { + match organization_id { + Some(organization_id) => Self::Organization(organization_id), + None => Self::Admin, + } + } + + pub(in crate::jobs) const fn visibility_predicate(self) -> (bool, Option) { + match self { + Self::Global => (false, None), + Self::Organization(organization_id) => (false, Some(organization_id)), + Self::Admin => (true, None), + } + } +} + +/// Explicit-scope input for listing jobs. +#[derive(Clone, Debug)] +pub struct JobReadListFilter<'a> { + pub scope: JobReadScope, + pub status: Option, + /// Case-insensitive job-type substring; SQL ILIKE wildcards retain their meaning. + pub job_type: Option<&'a str>, + pub limit: i64, + pub offset: i64, +} + /// Authorization scope for canceling a job. /// /// Unlike legacy cancellation APIs that use `None` as an admin wildcard, this @@ -79,6 +122,7 @@ pub struct JobLogRecordInput { #[derive(Clone, Debug)] pub struct JobListFilter<'a> { + /// Legacy visibility: None matches all organizations and global jobs. pub organization_id: Option, pub status: Option, /// Admin list query input used for `ILIKE` substring matching, not a canonical persisted diff --git a/runledger-postgres/src/jobs/types/definitions.rs b/runledger-postgres/src/jobs/types/definitions.rs index 9637de1..5dfde9b 100644 --- a/runledger-postgres/src/jobs/types/definitions.rs +++ b/runledger-postgres/src/jobs/types/definitions.rs @@ -13,6 +13,20 @@ pub struct JobDefinitionUpsert<'a> { pub is_enabled: bool, } +impl From<&runledger_core::jobs::JobSpec> for JobDefinitionUpsert<'static> { + fn from(spec: &runledger_core::jobs::JobSpec) -> Self { + let settings = spec.settings(); + Self { + job_type: spec.job_type(), + version: settings.version, + max_attempts: settings.max_attempts, + default_timeout_seconds: settings.default_timeout_seconds, + default_priority: settings.default_priority, + is_enabled: settings.is_enabled, + } + } +} + #[derive(Clone, Debug)] pub struct JobDefinitionRecord { pub job_type: JobTypeName, diff --git a/runledger-postgres/src/jobs/types/enqueue.rs b/runledger-postgres/src/jobs/types/enqueue.rs index 92adba9..4497a0c 100644 --- a/runledger-postgres/src/jobs/types/enqueue.rs +++ b/runledger-postgres/src/jobs/types/enqueue.rs @@ -6,6 +6,8 @@ use serde::Serialize; use serde_json::Value; use sqlx::types::Uuid; +use super::admin::JobReadScope; + #[derive(Clone, Debug)] pub struct JobEnqueue<'a> { pub job_type: JobType<'a>, @@ -22,6 +24,23 @@ pub struct JobEnqueue<'a> { pub stage: Option, } +/// Borrows an owned core submission without changing its JSON or request overrides. +impl<'a> From<&'a runledger_core::jobs::JobSubmission> for JobEnqueue<'a> { + fn from(request: &'a runledger_core::jobs::JobSubmission) -> Self { + Self { + job_type: request.job_type, + payload: &request.payload, + organization_id: request.organization_id, + priority: request.priority, + max_attempts: request.max_attempts, + timeout_seconds: request.timeout_seconds, + next_run_at: request.next_run_at, + idempotency_key: request.idempotency_key.as_deref(), + stage: request.stage, + } + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[non_exhaustive] pub enum JobEnqueueDisposition { @@ -420,7 +439,10 @@ impl JobEnqueueIntentRecord { } } -/// Bounded filters for listing durable enqueue intents. +/// Legacy filters for listing durable enqueue intents. +/// +/// Without an organization ID, reads include global and all organization-owned +/// intents. Prefer [`JobEnqueueIntentReadListFilter`] for explicit visibility. #[derive(Clone, Debug)] pub struct JobEnqueueIntentListFilter<'a> { pub(crate) organization_id: Option, @@ -465,7 +487,48 @@ impl<'a> JobEnqueueIntentListFilter<'a> { } } +/// Explicit-scope filters for listing durable enqueue intents. +#[derive(Clone, Debug)] +pub struct JobEnqueueIntentReadListFilter<'a> { + pub(crate) scope: JobReadScope, + pub(crate) status: Option, + pub(crate) job_type_query: Option<&'a str>, + pub(crate) limit: i64, + pub(crate) offset: i64, +} + +impl<'a> JobEnqueueIntentReadListFilter<'a> { + #[must_use] + pub const fn new(scope: JobReadScope, limit: i64, offset: i64) -> Self { + Self { + scope, + status: None, + job_type_query: None, + limit, + offset, + } + } + + #[must_use] + pub const fn with_status(mut self, status: JobEnqueueIntentStatus) -> Self { + self.status = Some(status); + self + } + + /// Filters by a case-insensitive job-type substring. + /// + /// PostgreSQL `ILIKE` metacharacters in `job_type_query` retain their + /// normal wildcard meaning, matching the crate's other admin filters. + #[must_use] + pub const fn with_job_type_query(mut self, job_type_query: &'a str) -> Self { + self.job_type_query = Some(job_type_query); + self + } +} + /// Bounded filters for durable enqueue-intent metrics grouped by job type. +/// Legacy metrics filter: no organization selects all scopes. +/// Use [`JobEnqueueIntentReadMetricsFilter`] for explicit visibility. #[derive(Clone, Debug)] pub struct JobEnqueueIntentMetricsFilter<'a> { pub(crate) organization_id: Option, @@ -498,6 +561,34 @@ impl<'a> JobEnqueueIntentMetricsFilter<'a> { } } +/// Explicit visibility and bounded pagination for enqueue-intent metrics. +/// Applications must authorize the selected [`JobReadScope`]. +#[derive(Clone, Debug)] +pub struct JobEnqueueIntentReadMetricsFilter<'a> { + pub(crate) scope: JobReadScope, + pub(crate) job_type: Option>, + pub(crate) limit: i64, + pub(crate) offset: i64, +} + +impl<'a> JobEnqueueIntentReadMetricsFilter<'a> { + #[must_use] + pub const fn new(scope: JobReadScope, limit: i64, offset: i64) -> Self { + Self { + scope, + job_type: None, + limit, + offset, + } + } + + #[must_use] + pub const fn with_job_type(mut self, job_type: JobType<'a>) -> Self { + self.job_type = Some(job_type); + self + } +} + /// Operational backlog signals for one intent job type. #[derive(Clone, Debug, Eq, PartialEq)] #[non_exhaustive] @@ -557,7 +648,10 @@ impl JobEnqueueIntentPromotionReport { } } -/// Exact tenant scope for a job mutation. +/// Exact global or tenant scope for job mutations and payload lookups. +/// +/// This selects rows, not authorization. Applications must authorize the chosen +/// scope. There is no admin wildcard: payload keys are unique only within a scope. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum JobScope { /// Match only a job whose `organization_id` is `NULL`. diff --git a/runledger-postgres/src/jobs/types/summary.rs b/runledger-postgres/src/jobs/types/summary.rs new file mode 100644 index 0000000..2450e94 --- /dev/null +++ b/runledger-postgres/src/jobs/types/summary.rs @@ -0,0 +1,70 @@ +use chrono::{DateTime, Utc}; +use runledger_core::jobs::{JobStage, JobStatus, JobType, JobTypeName}; +use sqlx::types::Uuid; + +use super::JobReadScope; + +/// Exclusive position in descending `(created_at, id)` order. +/// Preserve PostgreSQL microsecond timestamp precision when transporting this value. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct JobSummaryCursor { + pub created_at: DateTime, + pub id: Uuid, +} + +/// Compact operational page. Applications must authorize `scope` and keep the +/// scope/filters unchanged between pages. A cursor is a position, not a snapshot: +/// concurrent inserts ahead of it are excluded, and mutable status filters can +/// gain or lose rows while scanning. Use detail reads for payload inspection. +#[derive(Clone, Debug)] +pub struct JobSummaryFilter<'a> { + pub scope: JobReadScope, + pub status: Option, + /// Exact, case-sensitive identifier; no substring or wildcard matching. + pub job_type: Option>, + /// Between 1 and `JOB_LIST_PAGE_LIMIT_MAX`, inclusive. + pub limit: i64, + pub after: Option, +} + +/// Operational fields without payload, checkpoint, output, or free-form errors. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct JobSummary { + pub id: Uuid, + pub job_type: JobTypeName, + pub organization_id: Option, + pub status: JobStatus, + pub priority: i32, + pub run_number: i32, + pub attempt: i32, + pub max_attempts: i32, + pub next_run_at: DateTime, + pub stage: Option, + pub progress_done: Option, + pub progress_total: Option, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl JobSummary { + /// Continue a scan after this row, normally the last row in a full page. + #[must_use] + pub const fn cursor(&self) -> JobSummaryCursor { + JobSummaryCursor { + created_at: self.created_at, + id: self.id, + } + } +} + +/// A batch status observation, not authorization or a lease/mutation fence. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct JobStatusRecord { + pub id: Uuid, + pub status: JobStatus, + pub run_number: i32, + pub attempt: i32, + pub updated_at: DateTime, +} diff --git a/runledger-postgres/src/jobs/workflows/mutate/append.rs b/runledger-postgres/src/jobs/workflows/mutate/append.rs index 1771fa7..5ec36aa 100644 --- a/runledger-postgres/src/jobs/workflows/mutate/append.rs +++ b/runledger-postgres/src/jobs/workflows/mutate/append.rs @@ -35,9 +35,9 @@ use super::super::release::{ use super::super::runtime::{recompute_workflow_run_status_tx, resolve_terminal_step_queue_tx}; use super::super::snapshot::{CanonicalAppendRequest, canonical_append_request}; use super::super::steps::{ - WorkflowStepDependencyWriteContext, WorkflowStepIdsByKey, dependency_count_total, - fetch_job_definition_defaults_tx, insert_workflow_step_dependencies_tx, - insert_workflow_step_record_tx, workflow_step_effective_organization_id, + INSERT_CHUNK_ROWS, StepInsert, WorkflowStepDependencyWriteContext, WorkflowStepIdsByKey, + dependency_count_total, fetch_job_definition_defaults_tx, insert_step_chunk_tx, + insert_workflow_step_dependencies_tx, workflow_step_effective_organization_id, }; use super::super::validation::workflow_dag_validation_error; use super::idempotency::{ @@ -205,21 +205,31 @@ async fn insert_appended_step_records_tx( .collect::(); let mut appended_step_ids = Vec::with_capacity(steps.len()); - for step in steps { - let (dependency_count_pending, dependency_count_unsatisfied) = - initial_dependency_counters(&existing_statuses_by_key, &new_step_keys, step)?; - let step_id = insert_workflow_step_record_tx( - tx, - workflow_run_id, - workflow_step_effective_organization_id(workflow_organization_id, step), - step, - &defaults_by_job_type, - dependency_count_pending, - dependency_count_unsatisfied, - ) - .await?; - step_id_by_key.insert(step.step_key().as_str().to_owned(), step_id); - appended_step_ids.push(step_id); + for chunk in steps.chunks(INSERT_CHUNK_ROWS) { + let records = chunk + .iter() + .map(|step| { + let (pending, unsatisfied) = + initial_dependency_counters(&existing_statuses_by_key, &new_step_keys, step)?; + StepInsert::new( + step, + workflow_step_effective_organization_id(workflow_organization_id, step), + &defaults_by_job_type, + pending, + unsatisfied, + ) + }) + .collect::>>()?; + let ids = insert_step_chunk_tx(tx, workflow_run_id, &records).await?; + for step in chunk { + let id = super::super::steps::step_id_for_key( + &ids, + step.step_key().as_str(), + "missing inserted appended workflow step id", + )?; + appended_step_ids.push(id); + } + step_id_by_key.extend(ids); } Ok(InsertedAppend { diff --git a/runledger-postgres/src/jobs/workflows/steps.rs b/runledger-postgres/src/jobs/workflows/steps.rs index 6ef994a..8e95702 100644 --- a/runledger-postgres/src/jobs/workflows/steps.rs +++ b/runledger-postgres/src/jobs/workflows/steps.rs @@ -1,8 +1,13 @@ +mod batch; +pub(in crate::jobs::workflows) use batch::{ + INSERT_CHUNK_ROWS, StepInsert, insert_step_chunk_tx, insert_workflow_step_dependencies_tx, + insert_workflow_steps_tx, +}; + use std::collections::{BTreeMap, BTreeSet}; use runledger_core::jobs::{ - JobStage, WorkflowJobStepExecution, WorkflowRunEnqueue, WorkflowStepEnqueue, - WorkflowStepExecution, + JobStage, WorkflowJobStepExecution, WorkflowStepEnqueue, WorkflowStepExecution, }; use sqlx::types::Uuid; @@ -81,119 +86,6 @@ pub(in crate::jobs::workflows) fn workflow_step_defaults<'a>( .ok_or_else(|| workflow_definition_not_available_error(job_type.as_str())) } -pub(in crate::jobs::workflows) async fn insert_workflow_step_record_tx( - tx: &mut DbTx<'_>, - workflow_run_id: Uuid, - organization_id: Option, - step: &WorkflowStepEnqueue<'_>, - defaults_by_job_type: &DefaultsByJobType, - dependency_count_pending: i32, - dependency_count_unsatisfied: i32, -) -> Result { - let dependency_count_total = dependency_count_total(step)?; - let (job_type, priority, max_attempts, timeout_seconds, stage) = match step.execution() { - WorkflowStepExecution::Job(execution) => { - let defaults = workflow_step_defaults(defaults_by_job_type, execution)?; - - ( - Some(execution.job_type().as_str()), - Some(execution.priority().unwrap_or(defaults.default_priority)), - Some(execution.max_attempts().unwrap_or(defaults.max_attempts)), - Some( - execution - .timeout_seconds() - .unwrap_or(defaults.default_timeout_seconds), - ), - workflow_step_effective_stage(step), - ) - } - WorkflowStepExecution::External => (None, None, None, None, None), - }; - let step_id: Uuid = sqlx::query_scalar!( - "INSERT INTO workflow_steps ( - workflow_run_id, - step_key, - execution_kind, - job_type, - organization_id, - payload, - priority, - max_attempts, - timeout_seconds, - stage, - allow_handler_continuation, - execution_resource_key, - status, - dependency_count_total, - dependency_count_pending, - dependency_count_unsatisfied - ) - VALUES ( - $1, - $2, - $3::text::workflow_step_execution_kind, - $4, - $5, - $6::jsonb, - $7, - $8, - $9, - $10, - $11, - $12, - 'BLOCKED', - $13, - $14, - $15 - ) - RETURNING id", - workflow_run_id, - step.step_key() as _, - step.execution_kind().as_db_value(), - job_type, - organization_id, - step.payload(), - priority, - max_attempts, - timeout_seconds, - stage, - step.allows_handler_continuation(), - step.execution_resource_key(), - dependency_count_total, - dependency_count_pending, - dependency_count_unsatisfied, - ) - .fetch_one(&mut **tx) - .await - .map_err(|error| Error::from_query_sqlx_with_context("insert workflow step", error))?; - - Ok(step_id) -} - -pub(in crate::jobs::workflows) async fn insert_workflow_steps_tx( - tx: &mut DbTx<'_>, - payload: &WorkflowRunEnqueue<'_>, - workflow_run_id: Uuid, - defaults_by_job_type: &DefaultsByJobType, -) -> Result { - let mut step_id_by_key = WorkflowStepIdsByKey::new(); - for step in payload.steps() { - let step_id = insert_workflow_step_record_tx( - tx, - workflow_run_id, - workflow_step_effective_organization_id(payload.organization_id(), step), - step, - defaults_by_job_type, - dependency_count_total(step)?, - 0, - ) - .await?; - step_id_by_key.insert(step.step_key().as_str().to_owned(), step_id); - } - - Ok(step_id_by_key) -} - pub(in crate::jobs::workflows) fn step_id_for_key( step_id_by_key: &WorkflowStepIdsByKey, step_key: &str, @@ -205,69 +97,6 @@ pub(in crate::jobs::workflows) fn step_id_for_key( .ok_or_else(|| workflow_internal_state_error(missing_error)) } -pub(in crate::jobs::workflows) async fn insert_workflow_step_dependency_record_tx( - tx: &mut DbTx<'_>, - workflow_run_id: Uuid, - prerequisite_step_id: Uuid, - dependent_step_id: Uuid, - release_mode: &str, -) -> Result<()> { - sqlx::query!( - "INSERT INTO workflow_step_dependencies ( - workflow_run_id, - prerequisite_step_id, - dependent_step_id, - release_mode - ) - VALUES ($1, $2, $3, $4::text::workflow_dependency_release_mode)", - workflow_run_id, - prerequisite_step_id, - dependent_step_id, - release_mode, - ) - .execute(&mut **tx) - .await - .map_err(|error| { - Error::from_query_sqlx_with_context("insert workflow step dependency", error) - })?; - - Ok(()) -} - -pub(in crate::jobs::workflows) async fn insert_workflow_step_dependencies_tx( - tx: &mut DbTx<'_>, - steps: &[WorkflowStepEnqueue<'_>], - workflow_run_id: Uuid, - step_id_by_key: &WorkflowStepIdsByKey, - context: WorkflowStepDependencyWriteContext, -) -> Result<()> { - for step in steps { - let dependent_step_id = step_id_for_key( - step_id_by_key, - step.step_key().as_str(), - context.missing_dependent_step_id_error(), - )?; - for dependency in step.dependencies() { - let prerequisite_step_id = step_id_for_key( - step_id_by_key, - dependency.prerequisite_step_key.as_str(), - context.missing_prerequisite_step_id_error(), - )?; - let release_mode = dependency.effective_release_mode().as_db_value(); - insert_workflow_step_dependency_record_tx( - tx, - workflow_run_id, - prerequisite_step_id, - dependent_step_id, - release_mode, - ) - .await?; - } - } - - Ok(()) -} - pub(in crate::jobs::workflows) async fn fetch_job_definition_defaults_tx( tx: &mut DbTx<'_>, steps: &[WorkflowStepEnqueue<'_>], diff --git a/runledger-postgres/src/jobs/workflows/steps/batch.rs b/runledger-postgres/src/jobs/workflows/steps/batch.rs new file mode 100644 index 0000000..e0ef00d --- /dev/null +++ b/runledger-postgres/src/jobs/workflows/steps/batch.rs @@ -0,0 +1,196 @@ +use runledger_core::jobs::{WorkflowRunEnqueue, WorkflowStepEnqueue, WorkflowStepExecution}; +use serde::Serialize; +use sqlx::types::{Json, Uuid}; + +use crate::{DbTx, Error, Result}; + +use super::{ + DefaultsByJobType, WorkflowStepDependencyWriteContext, WorkflowStepIdsByKey, + dependency_count_total, step_id_for_key, workflow_step_defaults, + workflow_step_effective_organization_id, workflow_step_effective_stage, +}; + +// Bounds serialization scratch space and work per statement independently of +// graph size. Payload bytes remain caller-controlled, just as for single inserts. +pub(in crate::jobs::workflows) const INSERT_CHUNK_ROWS: usize = 256; + +#[derive(Serialize)] +pub(in crate::jobs::workflows) struct StepInsert<'a> { + step_key: &'a str, + execution_kind: &'static str, + job_type: Option<&'a str>, + organization_id: Option, + payload: &'a serde_json::Value, + priority: Option, + max_attempts: Option, + timeout_seconds: Option, + stage: Option<&'static str>, + allow_handler_continuation: bool, + execution_resource_key: Option<&'a str>, + dependency_count_total: i32, + dependency_count_pending: i32, + dependency_count_unsatisfied: i32, +} + +impl<'a> StepInsert<'a> { + pub(in crate::jobs::workflows) fn new( + step: &'a WorkflowStepEnqueue<'_>, + organization_id: Option, + defaults: &DefaultsByJobType, + pending: i32, + unsatisfied: i32, + ) -> Result { + let (job_type, priority, max_attempts, timeout_seconds) = match step.execution() { + WorkflowStepExecution::Job(execution) => { + let defaults = workflow_step_defaults(defaults, execution)?; + ( + Some(execution.job_type().as_str()), + Some(execution.priority().unwrap_or(defaults.default_priority)), + Some(execution.max_attempts().unwrap_or(defaults.max_attempts)), + Some( + execution + .timeout_seconds() + .unwrap_or(defaults.default_timeout_seconds), + ), + ) + } + WorkflowStepExecution::External => (None, None, None, None), + }; + Ok(Self { + step_key: step.step_key().as_str(), + execution_kind: step.execution_kind().as_db_value(), + job_type, + organization_id, + payload: step.payload(), + priority, + max_attempts, + timeout_seconds, + stage: workflow_step_effective_stage(step), + allow_handler_continuation: step.allows_handler_continuation(), + execution_resource_key: step.execution_resource_key(), + dependency_count_total: dependency_count_total(step)?, + dependency_count_pending: pending, + dependency_count_unsatisfied: unsatisfied, + }) + } +} + +pub(in crate::jobs::workflows) async fn insert_step_chunk_tx( + tx: &mut DbTx<'_>, + workflow_run_id: Uuid, + records: &[StepInsert<'_>], +) -> Result { + // recordset maps a JSON null field to SQL NULL. Payload is always present + // in StepInsert, so restore the JSON null that single-row binding stored. + let rows = sqlx::query!( + "INSERT INTO workflow_steps ( + workflow_run_id, step_key, execution_kind, job_type, organization_id, payload, + priority, max_attempts, timeout_seconds, stage, allow_handler_continuation, + execution_resource_key, status, dependency_count_total, + dependency_count_pending, dependency_count_unsatisfied + ) SELECT $1, r.step_key, r.execution_kind::workflow_step_execution_kind, + r.job_type, r.organization_id, COALESCE(r.payload, 'null'::jsonb), r.priority, r.max_attempts, + r.timeout_seconds, r.stage, r.allow_handler_continuation, r.execution_resource_key, + 'BLOCKED', r.dependency_count_total, r.dependency_count_pending, r.dependency_count_unsatisfied + FROM jsonb_to_recordset($2::jsonb) AS r( + step_key text, execution_kind text, job_type text, organization_id uuid, payload jsonb, + priority int, max_attempts int, timeout_seconds int, stage text, + allow_handler_continuation bool, execution_resource_key text, + dependency_count_total int, dependency_count_pending int, dependency_count_unsatisfied int) + RETURNING id, step_key", + workflow_run_id, Json(records) as _, + ).fetch_all(&mut **tx).await + .map_err(|error| Error::from_query_sqlx_with_context("insert workflow steps", error))?; + Ok(rows.into_iter().map(|row| (row.step_key, row.id)).collect()) +} + +pub(in crate::jobs::workflows) async fn insert_workflow_steps_tx( + tx: &mut DbTx<'_>, + payload: &WorkflowRunEnqueue<'_>, + workflow_run_id: Uuid, + defaults: &DefaultsByJobType, +) -> Result { + let mut ids = WorkflowStepIdsByKey::new(); + for chunk in payload.steps().chunks(INSERT_CHUNK_ROWS) { + let records = chunk + .iter() + .map(|step| { + StepInsert::new( + step, + workflow_step_effective_organization_id(payload.organization_id(), step), + defaults, + dependency_count_total(step)?, + 0, + ) + }) + .collect::>>()?; + ids.extend(insert_step_chunk_tx(tx, workflow_run_id, &records).await?); + } + Ok(ids) +} + +#[derive(Serialize)] +struct DependencyInsert { + prerequisite_step_id: Uuid, + dependent_step_id: Uuid, + release_mode: &'static str, +} + +async fn insert_dependency_chunk_tx( + tx: &mut DbTx<'_>, + workflow_run_id: Uuid, + records: &[DependencyInsert], +) -> Result<()> { + if records.is_empty() { + return Ok(()); + } + sqlx::query!( + "INSERT INTO workflow_step_dependencies ( + workflow_run_id, prerequisite_step_id, dependent_step_id, release_mode + ) SELECT $1, r.prerequisite_step_id, r.dependent_step_id, + r.release_mode::workflow_dependency_release_mode + FROM jsonb_to_recordset($2::jsonb) AS r( + prerequisite_step_id uuid, dependent_step_id uuid, release_mode text)", + workflow_run_id, + Json(records) as _, + ) + .execute(&mut **tx) + .await + .map_err(|error| { + Error::from_query_sqlx_with_context("insert workflow step dependencies", error) + })?; + Ok(()) +} + +pub(in crate::jobs::workflows) async fn insert_workflow_step_dependencies_tx( + tx: &mut DbTx<'_>, + steps: &[WorkflowStepEnqueue<'_>], + workflow_run_id: Uuid, + step_id_by_key: &WorkflowStepIdsByKey, + context: WorkflowStepDependencyWriteContext, +) -> Result<()> { + let mut records = Vec::with_capacity(INSERT_CHUNK_ROWS); + for step in steps { + let dependent_step_id = step_id_for_key( + step_id_by_key, + step.step_key().as_str(), + context.missing_dependent_step_id_error(), + )?; + for dependency in step.dependencies() { + records.push(DependencyInsert { + prerequisite_step_id: step_id_for_key( + step_id_by_key, + dependency.prerequisite_step_key.as_str(), + context.missing_prerequisite_step_id_error(), + )?, + dependent_step_id, + release_mode: dependency.effective_release_mode().as_db_value(), + }); + if records.len() == INSERT_CHUNK_ROWS { + insert_dependency_chunk_tx(tx, workflow_run_id, &records).await?; + records.clear(); + } + } + } + insert_dependency_chunk_tx(tx, workflow_run_id, &records).await +} diff --git a/runledger-postgres/src/lib.rs b/runledger-postgres/src/lib.rs index e4138a5..aa00545 100644 --- a/runledger-postgres/src/lib.rs +++ b/runledger-postgres/src/lib.rs @@ -324,6 +324,7 @@ use std::fmt; mod error; pub mod jobs; +mod migration_identity; mod migrations; pub use error::{ @@ -331,6 +332,7 @@ pub use error::{ classify_framework_constraint, classify_query_error, classify_query_error_with_constraint_classifier, has_framework_constraint_classifier, }; +pub use migration_identity::{MigrationBundle, RUNLEDGER_POSTGRES_VERSION, migration_bundle}; pub use migrations::{ MIGRATOR, SchemaCompatibilityError, WorkflowJobLinkTriggerDiagnostic, WorkflowJobLinkTriggerProblem, ensure_schema_compatible_after_idempotency_cutover, @@ -365,17 +367,20 @@ pub mod prelude { JobEnqueue, JobEnqueueDisposition, JobEnqueueIntent, JobEnqueueIntentDisposition, JobEnqueueIntentListFilter, JobEnqueueIntentMetricsFilter, JobEnqueueIntentMetricsRecord, JobEnqueueIntentOutcome, JobEnqueueIntentOutcomeState, JobEnqueueIntentPromotionError, - JobEnqueueIntentPromotionReport, JobEnqueueIntentRecord, JobEnqueueIntentState, + JobEnqueueIntentPromotionReport, JobEnqueueIntentReadListFilter, + JobEnqueueIntentReadMetricsFilter, JobEnqueueIntentRecord, JobEnqueueIntentState, JobEnqueueIntentStatus, JobEnqueueOutcome, JobEventRecord, JobFailureCompletionDisposition, JobFailureCompletionOutcome, JobFailureUpdate, JobLeaseIdentity, JobListFilter, JobLogRecord, JobLogRecordInput, JobMetricsRecord, JobOrdinaryProgressUpdate, JobPayloadUuidArrayFieldUpdate, JobPayloadUuidArrayFieldUpdateRejection, JobProgressUpdate, - JobQueueRecord, JobRequeueStatePolicy, JobRunningUpdate, JobRuntimeConfigListFilter, - JobRuntimeConfigRecord, JobRuntimeConfigUpsert, JobScheduleRecord, JobScheduleUpsert, - JobScope, JobSuccessCompletionOutcome, NonRequeueableJobStatusError, - ReapExpiredLeaseCleanupError, ReapExpiredLeaseCleanupOperation, - ReapExpiredLeaseDeferredError, ReapExpiredLeasesDetailedResult, ReapExpiredLeasesResult, - ReapedLeaseDisposition, ReapedLeaseRecord, ReapedTerminalLeaseRecord, RequeueableJobStatus, + JobQueueRecord, JobReadListFilter, JobReadScope, JobRequeueStatePolicy, JobRunningUpdate, + JobRuntimeConfigListFilter, JobRuntimeConfigRecord, JobRuntimeConfigUpsert, + JobScheduleRecord, JobScheduleUpsert, JobScope, JobStatusRecord, + JobSuccessCompletionOutcome, JobSummary, JobSummaryCursor, JobSummaryFilter, + NonRequeueableJobStatusError, ReapExpiredLeaseCleanupError, + ReapExpiredLeaseCleanupOperation, ReapExpiredLeaseDeferredError, + ReapExpiredLeasesDetailedResult, ReapExpiredLeasesResult, ReapedLeaseDisposition, + ReapedLeaseRecord, ReapedTerminalLeaseRecord, RequeueableJobStatus, SuccessfulReplayEnqueuedEventPayload, WorkflowRecoveryDisposition, WorkflowRecoveryMode, WorkflowRecoveryOutcome, WorkflowRecoveryRequest, WorkflowRunDbRecord, WorkflowRunHandle, WorkflowRunHandleError, WorkflowRunHandleScope, WorkflowRunListFilter, @@ -397,19 +402,25 @@ pub mod prelude { delete_promoted_job_enqueue_intents_before, delete_promoted_job_enqueue_intents_for_jobs_tx, enqueue_job, enqueue_job_tx, enqueue_job_with_execution_resource, enqueue_job_with_execution_resource_tx, - enqueue_job_with_outcome_tx, enqueue_or_get_active_workflow, + enqueue_job_with_outcome, enqueue_job_with_outcome_tx, enqueue_or_get_active_workflow, enqueue_or_get_active_workflow_tx, enqueue_workflow_run, enqueue_workflow_run_handle, - enqueue_workflow_run_tx, get_job_by_id, get_job_continuation_metrics, - get_job_definition_by_type, get_job_enqueue_intent_by_id, get_job_enqueue_intent_metrics, - get_job_metrics, get_job_payload_by_idempotency_key, get_job_runtime_config_by_type, - get_job_schedule_by_name, get_latest_job_payload_for_run, get_latest_workflow_run_by_type, - get_latest_workflow_run_by_type_with_scope, get_required_job_runtime_config_by_type, - get_workflow_run_by_id, get_workflow_run_by_id_with_scope, - get_workflow_run_by_type_and_idempotency_key, get_workflow_run_id_for_job, - heartbeat_job_for_lease, insert_job_definition_if_missing_tx, insert_job_log, - insert_job_runtime_config_if_missing, list_job_definitions, list_job_enqueue_intents, - list_job_events, list_job_logs, list_job_runtime_configs, list_jobs, list_workflow_runs, - list_workflow_runs_with_scope, list_workflow_step_dependencies, + enqueue_workflow_run_tx, get_job_by_id, get_job_by_id_with_scope, + get_job_continuation_metrics, get_job_continuation_metrics_with_scope, + get_job_definition_by_type, get_job_enqueue_intent_by_id, + get_job_enqueue_intent_by_id_with_scope, get_job_enqueue_intent_metrics, + get_job_enqueue_intent_metrics_with_scope, get_job_metrics, get_job_metrics_with_scope, + get_job_payload_by_idempotency_key, get_job_payload_by_idempotency_key_with_scope, + get_job_runtime_config_by_type, get_job_schedule_by_name, get_job_statuses_with_scope, + get_latest_job_payload_for_run, get_latest_job_payload_for_run_with_scope, + get_latest_workflow_run_by_type, get_latest_workflow_run_by_type_with_scope, + get_required_job_runtime_config_by_type, get_workflow_run_by_id, + get_workflow_run_by_id_with_scope, get_workflow_run_by_type_and_idempotency_key, + get_workflow_run_id_for_job, heartbeat_job_for_lease, insert_job_definition_if_missing_tx, + insert_job_log, insert_job_runtime_config_if_missing, list_job_definitions, + list_job_enqueue_intents, list_job_enqueue_intents_with_scope, list_job_events, + list_job_events_with_scope, list_job_logs, list_job_logs_with_scope, + list_job_runtime_configs, list_job_summaries, list_jobs, list_jobs_with_scope, + list_workflow_runs, list_workflow_runs_with_scope, list_workflow_step_dependencies, list_workflow_step_dependencies_page, list_workflow_step_dependencies_page_with_scope, list_workflow_step_dependencies_with_scope, list_workflow_steps, list_workflow_steps_page, list_workflow_steps_page_with_scope, list_workflow_steps_with_scope, diff --git a/runledger-postgres/src/migration_identity.rs b/runledger-postgres/src/migration_identity.rs new file mode 100644 index 0000000..f91c168 --- /dev/null +++ b/runledger-postgres/src/migration_identity.rs @@ -0,0 +1,150 @@ +use std::sync::LazyLock; + +use sha2::{Digest, Sha256}; +use sqlx::migrate::{Migration, MigrationType}; + +use crate::MIGRATOR; + +/// The version of the compiled `runledger-postgres` crate, not the host crate. +pub const RUNLEDGER_POSTGRES_VERSION: &str = env!("CARGO_PKG_VERSION"); + +static BUNDLE: LazyLock = LazyLock::new(|| { + let mut migrations: Vec<_> = MIGRATOR.iter().collect(); + migrations.sort_by_key(|migration| { + ( + migration.version, + migration_type_tag(migration.migration_type), + ) + }); + let bundle_fingerprint = fingerprint_bundle(&migrations); + MigrationBundle { + pipeline_fingerprint: fingerprint_pipeline(RUNLEDGER_POSTGRES_VERSION, &bundle_fingerprint), + bundle_fingerprint, + migrations, + } +}); + +/// Inspect the compiled migration bundle without a database or filesystem access. +/// +/// Use its pipeline fingerprint as one input to an application-owned schema or +/// test-template fingerprint. Keep the host's own pipeline revision, migration +/// inputs, and ordering policy in that composition. The fingerprint does not +/// validate a live database; retain [`crate::ensure_schema_compatible_after_idempotency_cutover`]. +/// +/// ``` +/// let bundle = runledger_postgres::migration_bundle(); +/// assert_eq!(bundle.library_version(), runledger_postgres::RUNLEDGER_POSTGRES_VERSION); +/// let pipeline_input: [u8; 32] = bundle.pipeline_fingerprint(); +/// for migration in bundle.migrations() { +/// // Compare version, migration_type, checksum, and sql with a vendored bundle. +/// assert!(!migration.checksum.is_empty()); +/// } +/// ``` +#[must_use] +pub fn migration_bundle() -> &'static MigrationBundle { + &BUNDLE +} + +/// A read-only manifest of the exact migrations embedded in this crate. +/// +/// Includes both up and down migrations. It describes available content, not +/// applied database history or a rollout plan. Existing [`MIGRATOR`] inspection +/// and startup helpers remain available. +#[derive(Debug)] +pub struct MigrationBundle { + migrations: Vec<&'static Migration>, + bundle_fingerprint: [u8; 32], + pipeline_fingerprint: [u8; 32], +} + +impl MigrationBundle { + /// Release identity, conservatively including changes unrelated to migrations. + #[must_use] + pub const fn library_version(&self) -> &'static str { + RUNLEDGER_POSTGRES_VERSION + } + + /// Entries sorted by version ascending, then Simple, ReversibleUp, ReversibleDown. + /// + /// Each SQLx entry exposes `version`, `description`, `migration_type`, raw + /// `checksum` bytes (SQLx SHA-384 of the SQL), `no_tx`, and exact `sql` text. + /// This ordering is for identity/inspection; it does not prescribe host DDL + /// ordering. Filter with `migration_type.is_up_migration()` for forward DDL. + pub fn migrations(&self) -> impl ExactSizeIterator + Clone + '_ { + self.migrations.iter().copied() + } + + /// SHA-256 identity of the complete bundle, independent of library version. + /// + /// Format v1 hashes, in order: + /// - literal bytes `runledger-postgres:migration-bundle:v1\0`; + /// - entry count as a big-endian u64; + /// - each sorted entry's version (big-endian i64), framed UTF-8 description, + /// type tag (one byte: Simple=0, ReversibleUp=1, ReversibleDown=2), framed + /// raw checksum, and `no_tx` (one byte: false=0, true=1). + /// + /// `\0` denotes one NUL byte. A frame is a big-endian u64 byte length followed + /// by those bytes. SQL is + /// represented by its SQLx checksum; timestamps, paths, and crate version + /// are excluded. Any future encoding change uses a new domain version. + #[must_use] + pub const fn bundle_fingerprint(&self) -> [u8; 32] { + self.bundle_fingerprint + } + + /// SHA-256 identity of the released migration pipeline, including its bundle. + /// + /// Format v1 hashes literal bytes `runledger-postgres:migration-pipeline:v1\0`, + /// the framed UTF-8 library version, then the 32 raw bundle fingerprint bytes. + /// Framing is the same as [`Self::bundle_fingerprint`]. + /// + /// The crate release version identifies Rust helper behavior beyond SQL. + /// Every new release invalidates this identity, even with unchanged SQL. + /// Helper-only edits in same-version path/patched builds are **not** detected: + /// add a host-owned source revision to the composed fingerprint for those + /// builds. Also include host migration ordering, configuration, and other + /// dependencies that affect template creation. This is not a build hash, + /// compatibility guarantee, or replacement for checking live schema state. + #[must_use] + pub const fn pipeline_fingerprint(&self) -> [u8; 32] { + self.pipeline_fingerprint + } +} + +fn fingerprint_bundle(migrations: &[&Migration]) -> [u8; 32] { + let mut digest = Sha256::new(); + digest.update(b"runledger-postgres:migration-bundle:v1\0"); + digest.update((migrations.len() as u64).to_be_bytes()); + for migration in migrations { + digest.update(migration.version.to_be_bytes()); + add_frame(&mut digest, migration.description.as_bytes()); + digest.update([migration_type_tag(migration.migration_type)]); + add_frame(&mut digest, &migration.checksum); + digest.update([u8::from(migration.no_tx)]); + } + digest.finalize().into() +} + +fn fingerprint_pipeline(version: &str, bundle_fingerprint: &[u8; 32]) -> [u8; 32] { + let mut digest = Sha256::new(); + digest.update(b"runledger-postgres:migration-pipeline:v1\0"); + add_frame(&mut digest, version.as_bytes()); + digest.update(bundle_fingerprint); + digest.finalize().into() +} + +fn migration_type_tag(migration_type: MigrationType) -> u8 { + match migration_type { + MigrationType::Simple => 0, + MigrationType::ReversibleUp => 1, + MigrationType::ReversibleDown => 2, + } +} + +fn add_frame(digest: &mut Sha256, bytes: &[u8]) { + digest.update((bytes.len() as u64).to_be_bytes()); + digest.update(bytes); +} + +#[cfg(test)] +mod tests; diff --git a/runledger-postgres/src/migration_identity/tests.rs b/runledger-postgres/src/migration_identity/tests.rs new file mode 100644 index 0000000..8b570cb --- /dev/null +++ b/runledger-postgres/src/migration_identity/tests.rs @@ -0,0 +1,138 @@ +use std::borrow::Cow; + +use super::*; + +fn vector_entries() -> [Migration; 2] { + [ + Migration { + version: -7, + description: Cow::Borrowed("alpha\0β"), + migration_type: MigrationType::Simple, + sql: Cow::Borrowed(""), + checksum: Cow::Borrowed(&[0, 255, 128]), + no_tx: false, + }, + Migration { + version: 42, + description: Cow::Borrowed("down"), + migration_type: MigrationType::ReversibleDown, + sql: Cow::Borrowed(""), + checksum: Cow::Borrowed(&[1, 2]), + no_tx: true, + }, + ] +} + +fn hex(bytes: [u8; 32]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +#[test] +fn encoding_matches_independently_computed_sha256_vectors() { + // Computed with Python hashlib/struct from the documented wire format, + // including a negative i64, UTF-8, NUL, binary checksum, and down migration. + let entries = vector_entries(); + let fingerprint = fingerprint_bundle(&entries.iter().collect::>()); + assert_eq!( + hex(fingerprint), + "e1c5a7c71d47180cf5ac552e5534ddccc95b37a6e324dd673e230d4f00518209" + ); + assert_eq!( + hex(fingerprint_pipeline("9.8.7-test", &fingerprint)), + "3fcc9cbe7a2c75e504fae7e5404b21d5f8c82bcf133bd2a27b2ec36cd9fd0f98" + ); +} + +#[test] +fn every_manifest_field_and_entry_affects_bundle_identity() { + let entries = vector_entries(); + let original = fingerprint_bundle(&[&entries[0], &entries[1]]); + let mutations: [fn(&mut Migration); 6] = [ + |entry| entry.version += 1, + |entry| entry.description = Cow::Borrowed("changed"), + |entry| entry.migration_type = MigrationType::ReversibleUp, + |entry| entry.migration_type = MigrationType::ReversibleDown, + |entry| entry.checksum = Cow::Borrowed(&[0, 255, 129]), + |entry| entry.no_tx = true, + ]; + for mutate in mutations { + let mut changed = entries[0].clone(); + mutate(&mut changed); + assert_ne!(original, fingerprint_bundle(&[&changed, &entries[1]])); + } + assert_ne!(original, fingerprint_bundle(&[&entries[0]])); + assert_ne!(original, fingerprint_bundle(&[])); + let mut changed_down = entries[1].clone(); + changed_down.checksum = Cow::Borrowed(&[3, 4]); + assert_ne!(original, fingerprint_bundle(&[&entries[0], &changed_down])); +} + +#[test] +fn release_identity_changes_without_sql_changes() { + let entries = vector_entries(); + let content = fingerprint_bundle(&[&entries[0], &entries[1]]); + assert_ne!( + fingerprint_pipeline("0.12.0", &content), + fingerprint_pipeline("0.12.1", &content) + ); + assert_eq!( + fingerprint_pipeline("0.12.0", &content), + fingerprint_pipeline("0.12.0", &content) + ); + let changed_content = fingerprint_bundle(&[&entries[0]]); + assert_ne!( + fingerprint_pipeline("0.12.0", &content), + fingerprint_pipeline("0.12.0", &changed_content) + ); +} + +#[test] +fn compiled_manifest_has_the_independently_pinned_up_and_down_history() { + let expected_versions = [ + 202603280001, + 202604100001, + 202605180001, + 202605220001, + 202606030001, + 202607190001, + 202607250001, + 202607280001, + 202607280002, + 202607280003, + 202607280004, + 202607280005, + 202608180001, + 202608230001, + 202608240001, + 202608240002, + 202609050001, + ]; + let actual: Vec<_> = migration_bundle() + .migrations() + .map(|entry| (entry.version, entry.migration_type)) + .collect(); + let expected: Vec<_> = expected_versions + .into_iter() + .flat_map(|version| { + [ + (version, MigrationType::ReversibleUp), + (version, MigrationType::ReversibleDown), + ] + }) + .collect(); + assert_eq!(actual, expected); + // Independently calculated from root migration files with Python SHA-384 + // checksums and the documented v1 SHA-256 encoding. Updating the migration + // history requires reviewing this content snapshot as well as the versions. + assert_eq!( + hex(migration_bundle().bundle_fingerprint()), + "a4fc356878542c81b29151d3636a7d267141a0b2573c0204257f0813733aac6f" + ); + assert_eq!( + migration_bundle().pipeline_fingerprint(), + fingerprint_pipeline( + RUNLEDGER_POSTGRES_VERSION, + &migration_bundle().bundle_fingerprint() + ) + ); +} diff --git a/runledger-postgres/tests/job_continuation.rs b/runledger-postgres/tests/job_continuation.rs index 09b765a..4972c30 100644 --- a/runledger-postgres/tests/job_continuation.rs +++ b/runledger-postgres/tests/job_continuation.rs @@ -6,10 +6,11 @@ use runledger_core::jobs::{ }; use runledger_postgres::jobs::{ CompareAndRequeueJob, CompareAndRequeueJobOutcome, DecodedJobEventPayload, - DecodedRequeuedEventPayload, JobContinuationUpdate, JobRequeueStatePolicy, cancel_job, - claim_jobs, compare_and_requeue_job, complete_job_continuation_with_outcome, + DecodedRequeuedEventPayload, JobContinuationUpdate, JobReadScope, JobRequeueStatePolicy, + cancel_job, claim_jobs, compare_and_requeue_job, complete_job_continuation_with_outcome, complete_job_success, enqueue_workflow_run, get_job_by_id, get_job_continuation_metrics, - get_workflow_run_by_id, list_job_events, list_workflow_steps, + get_job_continuation_metrics_with_scope, get_workflow_run_by_id, list_job_events, + list_workflow_steps, }; use runledger_postgres::{DbPool, Error, QueryErrorKind}; use runledger_test_support::{setup_ephemeral_pool, teardown_ephemeral_pool}; @@ -369,6 +370,12 @@ async fn continuation_metrics_filter_exact_tenants_and_aggregate_all_scopes() { let long_delay = Duration::from_secs(3_600); enqueue_test_job(&pool, JOB_TYPE, None, &json!({"scope": "global"})).await; + continue_next_due_job( + &pool, + "worker-continuation-metrics-global-first", + Duration::ZERO, + ) + .await; continue_next_due_job(&pool, "worker-continuation-metrics-global", long_delay).await; enqueue_test_job( @@ -395,44 +402,76 @@ async fn continuation_metrics_filter_exact_tenants_and_aggregate_all_scopes() { .await; continue_next_due_job(&pool, "worker-continuation-metrics-org-b-run-2", long_delay).await; - let organization_a_metrics = - get_job_continuation_metrics(&pool, Some(organization_a), Some(JOB_TYPE)) + // Make all three populations distinguishable, including active count and max run. + enqueue_test_job( + &pool, + JOB_TYPE, + Some(organization_b), + &json!({"extra": true}), + ) + .await; + continue_next_due_job(&pool, "worker-continuation-metrics-org-b-extra", long_delay).await; + for (scope, expected) in [ + (JobReadScope::Global, (2, 1, 3)), + (JobReadScope::Organization(organization_a), (1, 1, 2)), + (JobReadScope::Organization(organization_b), (3, 2, 3)), + (JobReadScope::Admin, (6, 4, 3)), + ( + JobReadScope::Organization(unrelated_organization), + (0, 0, 0), + ), + ] { + let rows = get_job_continuation_metrics_with_scope(&pool, scope, Some(JOB_TYPE)) .await - .expect("load organization A continuation metrics") - .pop() - .expect("registered job type has organization A metrics"); - assert_eq!(organization_a_metrics.continued_24h, 1); - assert_eq!(organization_a_metrics.active_continued_count, 1); - assert_eq!(organization_a_metrics.max_active_run_number, 2); - - let organization_b_metrics = - get_job_continuation_metrics(&pool, Some(organization_b), Some(JOB_TYPE)) + .expect("scoped continuation metrics"); + assert_eq!(rows.len(), 1, "preserve zero-count definitions"); + let row = &rows[0]; + assert_eq!( + ( + row.continued_24h, + row.active_continued_count, + row.max_active_run_number + ), + expected + ); + let legacy_scope = match scope { + JobReadScope::Global => continue, + JobReadScope::Organization(id) => Some(id), + JobReadScope::Admin => None, + }; + let legacy = get_job_continuation_metrics(&pool, legacy_scope, Some(JOB_TYPE)) .await - .expect("load organization B continuation metrics") - .pop() - .expect("registered job type has organization B metrics"); - assert_eq!(organization_b_metrics.continued_24h, 2); - assert_eq!(organization_b_metrics.active_continued_count, 1); - assert_eq!(organization_b_metrics.max_active_run_number, 3); - - let unrelated_metrics = - get_job_continuation_metrics(&pool, Some(unrelated_organization), Some(JOB_TYPE)) + .expect("legacy metrics"); + assert_eq!(legacy.len(), 1); + assert_eq!( + ( + legacy[0].continued_24h, + legacy[0].active_continued_count, + legacy[0].max_active_run_number + ), + expected + ); + } + let empty_type = "jobs.test.continuation.empty"; + register_test_job_definition(&pool, empty_type).await; + for scope in [ + JobReadScope::Global, + JobReadScope::Organization(organization_a), + JobReadScope::Admin, + ] { + let rows = get_job_continuation_metrics_with_scope(&pool, scope, Some(empty_type)) .await - .expect("load unrelated organization continuation metrics") - .pop() - .expect("registered job type has zero-valued unrelated metrics"); - assert_eq!(unrelated_metrics.continued_24h, 0); - assert_eq!(unrelated_metrics.active_continued_count, 0); - assert_eq!(unrelated_metrics.max_active_run_number, 0); - - let aggregate_metrics = get_job_continuation_metrics(&pool, None, Some(JOB_TYPE)) - .await - .expect("load aggregate continuation metrics") - .pop() - .expect("registered job type has aggregate metrics"); - assert_eq!(aggregate_metrics.continued_24h, 4); - assert_eq!(aggregate_metrics.active_continued_count, 3); - assert_eq!(aggregate_metrics.max_active_run_number, 3); + .expect("empty definition"); + assert_eq!(rows.len(), 1); + assert_eq!( + ( + rows[0].continued_24h, + rows[0].active_continued_count, + rows[0].max_active_run_number + ), + (0, 0, 0) + ); + } let scoped_plan = sqlx::query_scalar::<_, Value>( "EXPLAIN (FORMAT JSON) @@ -446,13 +485,14 @@ async fn continuation_metrics_filter_exact_tenants_and_aggregate_all_scopes() { FROM job_definitions jd LEFT JOIN job_continuation_metrics_rollup jcmr ON jcmr.job_type = jd.job_type - AND ($1::uuid IS NULL OR jcmr.organization_id = $1) + AND ($3::boolean OR (jcmr.organization_id = $1 OR ($1::uuid IS NULL AND jcmr.organization_id IS NULL))) WHERE ($2::text IS NULL OR jd.job_type = $2) GROUP BY jd.job_type ORDER BY jd.job_type ASC", ) .bind(organization_a) .bind(JOB_TYPE) + .bind(false) .fetch_one(&pool) .await .expect("explain scoped continuation metrics query"); diff --git a/runledger-postgres/tests/job_enqueue_intents.rs b/runledger-postgres/tests/job_enqueue_intents.rs index d8fd928..d3f722b 100644 --- a/runledger-postgres/tests/job_enqueue_intents.rs +++ b/runledger-postgres/tests/job_enqueue_intents.rs @@ -4,12 +4,13 @@ use chrono::{Duration as ChronoDuration, Utc}; use runledger_core::jobs::{JobStage, JobStatus, JobType}; use runledger_postgres::jobs::{ JobDefinitionUpdate, JobEnqueue, JobEnqueueIntent, JobEnqueueIntentDisposition, - JobEnqueueIntentListFilter, JobEnqueueIntentMetricsFilter, JobEnqueueIntentState, - JobEnqueueIntentStatus, delete_promoted_job_enqueue_intents_before, - delete_promoted_job_enqueue_intents_for_jobs_tx, enqueue_job, get_job_by_id, - get_job_enqueue_intent_by_id, get_job_enqueue_intent_metrics, list_job_enqueue_intents, - list_job_events, promote_job_enqueue_intents_for_types, record_job_enqueue_intent, - record_job_enqueue_intent_tx, update_job_definition, + JobEnqueueIntentListFilter, JobEnqueueIntentMetricsFilter, JobEnqueueIntentMetricsRecord, + JobEnqueueIntentReadMetricsFilter, JobEnqueueIntentState, JobEnqueueIntentStatus, JobReadScope, + delete_promoted_job_enqueue_intents_before, delete_promoted_job_enqueue_intents_for_jobs_tx, + enqueue_job, get_job_by_id, get_job_enqueue_intent_by_id, get_job_enqueue_intent_metrics, + get_job_enqueue_intent_metrics_with_scope, list_job_enqueue_intents, list_job_events, + promote_job_enqueue_intents_for_types, record_job_enqueue_intent, record_job_enqueue_intent_tx, + update_job_definition, }; use runledger_test_support::{setup_ephemeral_pool, teardown_ephemeral_pool}; use serde_json::{Value, json}; @@ -2780,3 +2781,312 @@ async fn enqueue_event_failure_defers_only_failed_intent_and_rolls_back_its_job( teardown_ephemeral_pool(pool, database).await; } + +async fn seed_metric_intent( + pool: &sqlx::PgPool, + organization_id: Option, + job_type: &str, + key: &str, + status: &str, + created_at: chrono::DateTime, + recent: bool, +) { + let payload = json!({"fixture": key}); + let request = JobEnqueueIntent::new(JobType::new(job_type), &payload, key); + let request = match organization_id { + Some(id) => request.with_organization_id(id), + None => request, + }; + let intent = record_job_enqueue_intent(pool, &request) + .await + .expect("intent metrics fixture or read succeeds"); + sqlx::query("UPDATE job_enqueue_intents SET created_at = $2 WHERE id = $1") + .bind(intent.intent_id) + .bind(created_at) + .execute(pool) + .await + .expect("intent metrics fixture or read succeeds"); + if status == "FRESH" { + return; + } + if status == "PROMOTED" { + let job = enqueue_job( + pool, + &JobEnqueue { + job_type: JobType::new(job_type), + organization_id, + payload: &payload, + idempotency_key: Some(key), + priority: None, + max_attempts: None, + timeout_seconds: None, + next_run_at: None, + stage: None, + }, + ) + .await + .expect("intent metrics fixture or read succeeds"); + sqlx::query("UPDATE job_enqueue_intents SET status = 'PROMOTED', promotion_attempts = 99, last_attempted_at = now(), promoted_at = now() - make_interval(hours => $3), promoted_job_id = $2 WHERE id = $1") + .bind(intent.intent_id).bind(job).bind(if recent { 1 } else { 25 }).execute(pool).await.expect("intent metrics fixture or read succeeds"); + } else { + sqlx::query("UPDATE job_enqueue_intents SET status = $2, promotion_attempts = $3, last_attempted_at = now(), last_error_code = 'fixture', last_error_message = 'fixture', conflicted_at = CASE WHEN $2 = 'CONFLICTED' THEN now() - make_interval(hours => $4) END WHERE id = $1") + .bind(intent.intent_id).bind(status).bind(if status == "PENDING" { 3 } else { 99 }).bind(if recent { 1 } else { 25 }).execute(pool).await.expect("intent metrics fixture or read succeeds"); + } +} + +#[tokio::test] +async fn exact_intent_metrics_scope_every_population_and_preserve_windows_and_pages() { + let (pool, database) = setup_ephemeral_pool("exact_intent_metrics", 4).await; + record_postgres_server_version(&pool, "exact intent metrics").await; + register_test_job_definition(&pool, JOB_TYPE).await; + let tenants = [Uuid::now_v7(), Uuid::now_v7()]; + let job_types = [ + "jobs.test.metrics.a", + "jobs.test.metrics.b", + "jobs.test.metrics.c", + ]; + let old_type = "jobs.test.metrics.old"; + let base = Utc::now() - ChronoDuration::days(3); + let base = chrono::DateTime::from_timestamp_micros(base.timestamp_micros()) + .expect("current timestamp fits microsecond precision"); + seed_metric_populations(&pool, tenants, job_types, old_type, base).await; + for (scope, pending, retries, conflicts, promoted, oldest) in [ + ( + JobReadScope::Global, + 2, + 1, + 2, + 3, + Some(base - ChronoDuration::hours(1)), + ), + ( + JobReadScope::Organization(tenants[0]), + 3, + 2, + 3, + 4, + Some(base - ChronoDuration::hours(2)), + ), + ( + JobReadScope::Organization(tenants[1]), + 5, + 4, + 5, + 6, + Some(base - ChronoDuration::hours(4)), + ), + ( + JobReadScope::Admin, + 10, + 7, + 10, + 13, + Some(base - ChronoDuration::hours(4)), + ), + (JobReadScope::Organization(Uuid::now_v7()), 0, 0, 0, 0, None), + ] { + let filter = JobEnqueueIntentReadMetricsFilter::new(scope, 10, 0) + .with_job_type(JobType::new(job_types[0])); + let rows = get_job_enqueue_intent_metrics_with_scope(&pool, &filter) + .await + .expect("intent metrics fixture or read succeeds"); + assert_eq!(rows.len(), usize::from(pending > 0)); + if let Some(row) = rows.first() { + assert_eq!( + ( + row.pending_count, + row.retrying_count, + row.max_promotion_attempts, + row.conflicted_24h, + row.promoted_24h, + row.oldest_pending_at + ), + (pending, retries, 3, conflicts, promoted, oldest) + ); + } + let all = get_job_enqueue_intent_metrics_with_scope( + &pool, + &JobEnqueueIntentReadMetricsFilter::new(scope, 10, 0), + ) + .await + .expect("intent metrics fixture or read succeeds"); + if pending > 0 { + assert_eq!( + all.iter() + .map(|row| row.job_type.as_str()) + .collect::>(), + job_types + ); + let terminal = &all[1]; + assert_eq!( + ( + terminal.pending_count, + terminal.retrying_count, + terminal.max_promotion_attempts, + terminal.oldest_pending_at + ), + (0, 0, 0, None) + ); + assert_eq!( + terminal.promoted_24h, + if scope == JobReadScope::Admin { 3 } else { 1 } + ); + assert_eq!(terminal.conflicted_24h, 0); + } else { + assert!(all.is_empty()); + } + assert_metric_page(&pool, scope, &all).await; + for nonmatch in [old_type, "jobs.test.metrics", "jobs.test.metrics.%"] { + assert!( + get_job_enqueue_intent_metrics_with_scope( + &pool, + &JobEnqueueIntentReadMetricsFilter::new(scope, 10, 0) + .with_job_type(JobType::new(nonmatch)) + ) + .await + .expect("intent metrics fixture or read succeeds") + .is_empty() + ); + } + for (limit, offset) in [(0, 0), (-1, 0), (1001, 0), (1, -1)] { + assert!( + get_job_enqueue_intent_metrics_with_scope( + &pool, + &JobEnqueueIntentReadMetricsFilter::new(scope, limit, offset) + ) + .await + .is_err() + ); + } + let legacy = match scope { + JobReadScope::Global => continue, + JobReadScope::Organization(id) => { + JobEnqueueIntentMetricsFilter::new(10, 0).with_organization_id(id) + } + JobReadScope::Admin => JobEnqueueIntentMetricsFilter::new(10, 0), + }; + assert_eq!( + get_job_enqueue_intent_metrics(&pool, &legacy) + .await + .expect("intent metrics fixture or read succeeds"), + all + ); + } + teardown_ephemeral_pool(pool, database).await; +} + +async fn seed_metric_populations( + pool: &sqlx::PgPool, + tenants: [Uuid; 2], + job_types: [&str; 3], + old_type: &str, + base: chrono::DateTime, +) { + for job_type in job_types.into_iter().chain([old_type]) { + register_test_job_definition(pool, job_type).await; + } + for (organization_id, count) in [(None, 1), (Some(tenants[0]), 2), (Some(tenants[1]), 4)] { + let oldest = base - ChronoDuration::hours(i64::from(count)); + for (status, total) in [ + ("PENDING", count), + ("CONFLICTED", count + 1), + ("PROMOTED", count + 2), + ] { + for index in 0..total { + seed_metric_intent( + pool, + organization_id, + job_types[0], + &format!("{status}-{index}"), + status, + if status == "PENDING" { + oldest + ChronoDuration::minutes(i64::from(index)) + } else { + base - ChronoDuration::days(30) + }, + true, + ) + .await; + } + } + seed_metric_intent( + pool, + organization_id, + job_types[0], + "fresh", + "FRESH", + base, + true, + ) + .await; + for status in ["CONFLICTED", "PROMOTED"] { + // Ancient terminal creation/attempt counts must not pollute pending age or retries. + seed_metric_intent( + pool, + organization_id, + job_types[0], + &format!("old-{status}"), + status, + base - ChronoDuration::days(30), + false, + ) + .await; + seed_metric_intent( + pool, + organization_id, + old_type, + &format!("only-old-{status}"), + status, + base, + false, + ) + .await; + } + seed_metric_intent( + pool, + organization_id, + job_types[1], + "terminal-only", + "PROMOTED", + base, + true, + ) + .await; + seed_metric_intent( + pool, + organization_id, + job_types[2], + "pending-only", + "FRESH", + base, + true, + ) + .await; + } +} + +async fn assert_metric_page( + pool: &sqlx::PgPool, + scope: JobReadScope, + all: &[JobEnqueueIntentMetricsRecord], +) { + let page = get_job_enqueue_intent_metrics_with_scope( + pool, + &JobEnqueueIntentReadMetricsFilter::new(scope, 1, 1), + ) + .await + .expect("intent metrics fixture or read succeeds"); + assert_eq!( + page, + all.iter().skip(1).take(1).cloned().collect::>() + ); + assert!( + get_job_enqueue_intent_metrics_with_scope( + pool, + &JobEnqueueIntentReadMetricsFilter::new(scope, 1, 3) + ) + .await + .expect("intent metrics fixture or read succeeds") + .is_empty() + ); +} diff --git a/runledger-postgres/tests/job_read_plans.rs b/runledger-postgres/tests/job_read_plans.rs new file mode 100644 index 0000000..efe9664 --- /dev/null +++ b/runledger-postgres/tests/job_read_plans.rs @@ -0,0 +1,308 @@ +//! Plans for the actual SQLx-prepared public list queries, using production indexes. +use runledger_core::jobs::JobType; +use runledger_postgres::prelude::*; +use runledger_test_support::{setup_ephemeral_pool, teardown_ephemeral_pool}; +use serde_json::Value; +use sqlx::Connection; +use sqlx::types::Uuid; + +mod support; + +const JOB_TYPE: &str = "jobs.test.read_plans"; + +async fn populate(pool: &DbPool) { + support::register_test_job_definition(pool, JOB_TYPE).await; + // Twenty rows per tenant, interleaved by creation time. Global rows are + // deliberately old so a global ordering scan cannot get lucky with LIMIT. + sqlx::raw_sql( + "INSERT INTO job_queue (job_type, organization_id, max_attempts, created_at) + SELECT 'jobs.test.read_plans', + CASE WHEN n <= 20 THEN NULL ELSE md5((n % 1000)::text)::uuid END, + 3, now() - (20001 - n) * interval '1 second' + FROM generate_series(1, 20000) n; + INSERT INTO job_enqueue_intents + (job_type, organization_id, payload, idempotency_key, enqueue_request, created_at) + SELECT job_type, organization_id, payload, id::text, '{}'::jsonb, created_at FROM job_queue; + ANALYZE job_queue; + ANALYZE job_enqueue_intents;", + ).execute(pool).await.expect("populate representative cross-tenant data"); +} + +fn has_scope_index_access(value: &Value, global_indexes: &[String]) -> bool { + match value { + Value::Object(object) => { + object + .get("Index Cond") + .and_then(Value::as_str) + .is_some_and(|condition| condition.contains("organization_id")) + || object + .get("Index Name") + .and_then(Value::as_str) + .is_some_and(|name| global_indexes.iter().any(|index| index == name)) + || object + .values() + .any(|value| has_scope_index_access(value, global_indexes)) + } + Value::Array(values) => values + .iter() + .any(|value| has_scope_index_access(value, global_indexes)), + _ => false, + } +} + +async fn assert_prepared_plan(pool: &DbPool, table: &str, organization: Option) { + let (name, parameter_count): (String, i32) = sqlx::query_as( + "SELECT name, cardinality(parameter_types) FROM pg_prepared_statements + WHERE statement LIKE '%' || $1 || '%' + AND statement LIKE '%ORDER BY created_at DESC, id DESC%' + AND statement LIKE '%OFFSET $5%'", + ) + .bind(format!("FROM {table}")) + .fetch_one(pool) + .await + .expect("public list prepared statement"); + // A global-only partial index also proves selective scope access, even + // when EXPLAIN has no Index Cond because the index predicate is sufficient. + let global_indexes: Vec = if organization.is_none() { + sqlx::query_scalar( + "SELECT indexrelid::regclass::text FROM pg_index + WHERE indrelid = $1::text::regclass + AND pg_get_expr(indpred, indrelid) LIKE '%organization_id IS NULL%'", + ) + .bind(table) + .fetch_all(pool) + .await + .expect("global partial indexes") + } else { + Vec::new() + }; + let organization = organization.map_or_else(|| "NULL".into(), |id| format!("'{id}'::uuid")); + // The baseline query has a sixth admin flag; retaining support here lets + // this regression test demonstrate failure before the fix as well. + let admin_argument = if parameter_count == 6 { ", false" } else { "" }; + let statement = format!( + "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON, TIMING OFF) EXECUTE \"{}\" ({organization}, NULL, NULL, 20, 0{admin_argument})", + name.replace('"', "\"\""), + ); + let plan: Value = sqlx::query_scalar(&statement) + .fetch_one(pool) + .await + .expect("explain actual query"); + eprintln!( + "{table}: execution={} ms, shared buffers={}", + plan[0]["Execution Time"], plan[0]["Plan"]["Shared Hit Blocks"] + ); + assert!( + has_scope_index_access(&plan, &global_indexes), + "scope must constrain an index scan: {plan}" + ); +} + +#[tokio::test] +async fn selective_scopes_use_indexes_with_custom_and_generic_prepared_plans() { + // A single connection keeps the public call and EXPLAIN on the same + // backend, so this checks the executed query rather than a copied predicate. + let (pool, database) = setup_ephemeral_pool("job_read_plans", 1).await; + let version: String = sqlx::query_scalar("SHOW server_version") + .fetch_one(&pool) + .await + .expect("version"); + eprintln!("job list plans PostgreSQL {version}"); + assert!(version.starts_with("18.")); + populate(&pool).await; + let tenant: Uuid = sqlx::query_scalar("SELECT md5('999')::uuid") + .fetch_one(&pool) + .await + .expect("tenant"); + for mode in ["force_custom_plan", "force_generic_plan"] { + sqlx::raw_sql(&format!("SET plan_cache_mode = {mode}")) + .execute(&pool) + .await + .expect("plan mode"); + for (scope, organization) in [ + (JobReadScope::Organization(tenant), Some(tenant)), + (JobReadScope::Global, None), + ] { + // SQLx retains statement IDs internally, so close the prepared + // cache through its API instead of issuing DEALLOCATE behind it. + pool.acquire() + .await + .expect("connection") + .clear_cached_statements() + .await + .expect("clear cache"); + let jobs = list_jobs_with_scope( + &pool, + &JobReadListFilter { + scope, + status: None, + job_type: None, + limit: 20, + offset: 0, + }, + ) + .await + .expect("list jobs"); + assert_eq!(jobs.len(), 20); + assert!(jobs.iter().all(|job| job.organization_id == organization)); + assert_prepared_plan(&pool, "job_queue", organization).await; + let intents = list_job_enqueue_intents_with_scope( + &pool, + &JobEnqueueIntentReadListFilter::new(scope, 20, 0), + ) + .await + .expect("list intents"); + assert_eq!(intents.len(), 20); + assert!( + intents + .iter() + .all(|intent| intent.organization_id == organization) + ); + assert_prepared_plan(&pool, "job_enqueue_intents", organization).await; + } + } + teardown_ephemeral_pool(pool, database).await; +} + +async fn assert_payload_plan(pool: &DbPool, predicate: &str, scope: JobScope, key: &str) { + let name: String = sqlx::query_scalar( + "SELECT name FROM pg_prepared_statements + WHERE statement LIKE 'SELECT id, payload%' + AND position($1 IN statement) > 0", + ) + .bind(predicate) + .fetch_one(pool) + .await + .expect("actual public payload statement"); + let organization = scope.organization_id(); + let global_indexes = if organization.is_none() { + sqlx::query_scalar::<_, String>( + "SELECT indexrelid::regclass::text FROM pg_index + WHERE indrelid = 'job_queue'::regclass + AND pg_get_expr(indpred, indrelid) LIKE '%organization_id IS NULL%'", + ) + .fetch_all(pool) + .await + .expect("global indexes") + } else { + Vec::new() + }; + let organization = organization.map_or_else(|| "NULL".into(), |id| format!("'{id}'::uuid")); + let statement = format!( + "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON, TIMING OFF) EXECUTE \"{}\" \ + ({organization}, '{JOB_TYPE}', '{}')", + name.replace('"', "\"\""), + key.replace('\'', "''"), + ); + let plan: Value = sqlx::query_scalar(&statement) + .fetch_one(pool) + .await + .expect("explain payload lookup"); + eprintln!( + "payload {predicate} {scope:?}: execution={} ms, shared buffers={}", + plan[0]["Execution Time"], plan[0]["Plan"]["Shared Hit Blocks"] + ); + assert!( + has_scope_index_access(&plan, &global_indexes), + "payload lookup must constrain scope through an index: {plan}" + ); + assert!( + plan[0]["Plan"]["Shared Hit Blocks"] + .as_u64() + .expect("buffer count") + < 100, + "one payload must not read hundreds of buffers in this indexed fixture: {plan}" + ); +} + +#[tokio::test] +async fn payload_scopes_use_indexes_with_custom_and_generic_prepared_plans() { + let (pool, database) = setup_ephemeral_pool("payload_read_plans", 1).await; + let version: String = sqlx::query_scalar("SHOW server_version") + .fetch_one(&pool) + .await + .expect("version"); + eprintln!("payload plans PostgreSQL {version}"); + assert!(version.starts_with("18.")); + support::register_test_job_definition(&pool, JOB_TYPE).await; + let run = Uuid::from_u128(42); + // Twenty busy tenants with interleaved creation order and repeated keys/run + // IDs. Global rows are oldest, preventing an unscoped ordering scan from + // getting lucky with LIMIT. Each tenant has roughly 1,000 matching rows. + sqlx::query( + "INSERT INTO job_queue (job_type, organization_id, max_attempts, created_at, + idempotency_key, enqueue_request, payload) + SELECT 'jobs.test.read_plans', organization, 3, + now() - (20001 - n) * interval '1 second', + CASE WHEN organization IS NULL THEN n::text ELSE (n / 20)::text END, + '{}'::jsonb, jsonb_build_object('run_id', $1::text, 'organization', organization) + FROM generate_series(1, 20000) n + CROSS JOIN LATERAL ( + SELECT CASE WHEN n <= 20 THEN NULL ELSE md5((n % 20)::text)::uuid END AS organization + ) scopes", + ) + .bind(run.to_string()) + .execute(&pool) + .await + .expect("payload fixture"); + sqlx::raw_sql("ANALYZE job_queue") + .execute(&pool) + .await + .expect("analyze"); + let tenant: Uuid = sqlx::query_scalar("SELECT md5('19')::uuid") + .fetch_one(&pool) + .await + .expect("tenant"); + for mode in ["force_custom_plan", "force_generic_plan"] { + sqlx::raw_sql(&format!("SET plan_cache_mode = {mode}")) + .execute(&pool) + .await + .expect("plan mode"); + for scope in [JobScope::Global, JobScope::Organization(tenant)] { + let key: String = sqlx::query_scalar( + "SELECT idempotency_key FROM job_queue + WHERE organization_id IS NOT DISTINCT FROM $1 + ORDER BY created_at DESC LIMIT 1", + ) + .bind(scope.organization_id()) + .fetch_one(&pool) + .await + .expect("key"); + pool.acquire() + .await + .expect("connection") + .clear_cached_statements() + .await + .expect("clear cache"); + let (_, payload) = get_job_payload_by_idempotency_key_with_scope( + &pool, + scope, + JobType::new(JOB_TYPE), + &key, + ) + .await + .expect("lookup key") + .expect("payload"); + assert_eq!( + payload["organization"], + serde_json::json!(scope.organization_id()) + ); + assert_payload_plan(&pool, "idempotency_key = $3", scope, &key).await; + let (_, payload) = get_latest_job_payload_for_run_with_scope( + &pool, + scope, + JobType::new(JOB_TYPE), + run, + ) + .await + .expect("lookup run") + .expect("payload"); + assert_eq!( + payload["organization"], + serde_json::json!(scope.organization_id()) + ); + assert_payload_plan(&pool, "payload->>'run_id' = $3", scope, &run.to_string()).await; + } + } + teardown_ephemeral_pool(pool, database).await; +} diff --git a/runledger-postgres/tests/job_read_scope.rs b/runledger-postgres/tests/job_read_scope.rs new file mode 100644 index 0000000..ba1b0bd --- /dev/null +++ b/runledger-postgres/tests/job_read_scope.rs @@ -0,0 +1,393 @@ +use std::collections::BTreeSet; + +use runledger_core::jobs::{JobStatus, JobType}; +use runledger_postgres::jobs::JOB_LIST_PAGE_LIMIT_MAX; +use runledger_postgres::prelude::*; +use runledger_test_support::{setup_ephemeral_pool, teardown_ephemeral_pool}; +use serde_json::json; +use sqlx::types::Uuid; + +mod support; + +const JOB_TYPE: &str = "jobs.test.read_scope"; + +struct Fixture { + organization_id: Option, + job_id: Uuid, + intent_id: Uuid, +} + +async fn fixtures(pool: &DbPool) -> Vec { + let version: String = sqlx::query_scalar("SHOW server_version") + .fetch_one(pool) + .await + .expect("server version"); + let version_num: i32 = sqlx::query_scalar("SELECT current_setting('server_version_num')::int") + .fetch_one(pool) + .await + .expect("server version number"); + eprintln!("job read scope: PostgreSQL {version} ({version_num})"); + assert_eq!(version_num / 10_000, 18); + support::register_test_job_definition(pool, JOB_TYPE).await; + let mut fixtures = Vec::new(); + for organization_id in [None, Some(Uuid::now_v7()), Some(Uuid::now_v7())] { + let payload = json!({"organization_id": organization_id}); + let job_id = support::enqueue_test_job(pool, JOB_TYPE, organization_id, &payload).await; + for message in ["first", "second"] { + insert_job_log( + pool, + &JobLogRecordInput { + job_id, + run_number: 1, + attempt: None, + level: "INFO".into(), + message: message.into(), + payload: payload.clone(), + }, + ) + .await + .expect("insert log"); + } + let intent = JobEnqueueIntent::new(JobType::new(JOB_TYPE), &payload, "scope-intent"); + let intent = match organization_id { + Some(id) => intent.with_organization_id(id), + None => intent, + }; + let intent_id = record_job_enqueue_intent(pool, &intent) + .await + .expect("record intent") + .intent_id; + fixtures.push(Fixture { + organization_id, + job_id, + intent_id, + }); + } + fixtures +} + +fn list_filter(scope: JobReadScope) -> JobReadListFilter<'static> { + JobReadListFilter { + scope, + status: None, + job_type: None, + limit: 100, + offset: 0, + } +} + +async fn assert_point_reads(pool: &DbPool, scope: JobReadScope, fixture: &Fixture, visible: bool) { + let job = get_job_by_id_with_scope(pool, scope, fixture.job_id) + .await + .expect("read job"); + assert_eq!( + job.as_ref().map(|row| row.id), + visible.then_some(fixture.job_id) + ); + let intent = get_job_enqueue_intent_by_id_with_scope(pool, scope, fixture.intent_id) + .await + .expect("read intent"); + assert_eq!( + intent.as_ref().map(|row| row.id), + visible.then_some(fixture.intent_id) + ); + let events = list_job_events_with_scope(pool, scope, fixture.job_id, 100, None) + .await + .expect("read events"); + assert_eq!(events.len(), usize::from(visible)); + assert!(events.iter().all(|row| row.job_id == fixture.job_id)); + let logs = list_job_logs_with_scope(pool, scope, fixture.job_id, 100, None) + .await + .expect("read logs"); + assert_eq!(logs.len(), if visible { 2 } else { 0 }); + assert!(logs.iter().all(|row| row.job_id == fixture.job_id)); +} + +#[tokio::test] +async fn exact_scopes_isolate_jobs_events_logs_and_intents() { + let (pool, database) = setup_ephemeral_pool("job_read_scope", 4).await; + let fixtures = fixtures(&pool).await; + // Expected indices are independent of the SQL predicate and scope conversion. + let cases = [ + (JobReadScope::Global, vec![0]), + ( + JobReadScope::Organization(fixtures[1].organization_id.expect("tenant A")), + vec![1], + ), + ( + JobReadScope::Organization(fixtures[2].organization_id.expect("tenant B")), + vec![2], + ), + (JobReadScope::Organization(Uuid::now_v7()), vec![]), + (JobReadScope::Admin, vec![0, 1, 2]), + ]; + for (scope, visible) in cases { + for (index, fixture) in fixtures.iter().enumerate() { + assert_point_reads(&pool, scope, fixture, visible.contains(&index)).await; + } + let jobs = list_jobs_with_scope(&pool, &list_filter(scope)) + .await + .expect("list jobs"); + assert_eq!( + jobs.iter().map(|row| row.id).collect::>(), + visible.iter().map(|&i| fixtures[i].job_id).collect() + ); + let intents = list_job_enqueue_intents_with_scope( + &pool, + &JobEnqueueIntentReadListFilter::new(scope, 100, 0), + ) + .await + .expect("list intents"); + assert_eq!( + intents.iter().map(|row| row.id).collect::>(), + visible.iter().map(|&i| fixtures[i].intent_id).collect() + ); + assert_point_reads( + &pool, + scope, + &Fixture { + organization_id: None, + job_id: Uuid::now_v7(), + intent_id: Uuid::now_v7(), + }, + false, + ) + .await; + } + teardown_ephemeral_pool(pool, database).await; +} + +async fn assert_legacy_point_reads( + pool: &DbPool, + organization_id: Option, + fixture: &Fixture, + visible: bool, +) { + assert_eq!( + get_job_by_id(pool, organization_id, fixture.job_id) + .await + .expect("legacy job") + .map(|row| row.id), + visible.then_some(fixture.job_id) + ); + assert_eq!( + get_job_enqueue_intent_by_id(pool, organization_id, fixture.intent_id) + .await + .expect("legacy intent") + .map(|row| row.id), + visible.then_some(fixture.intent_id) + ); + assert_eq!( + list_job_events(pool, organization_id, fixture.job_id, 100, None) + .await + .expect("legacy events") + .len(), + usize::from(visible) + ); + assert_eq!( + list_job_logs(pool, organization_id, fixture.job_id, 100, None) + .await + .expect("legacy logs") + .len(), + if visible { 2 } else { 0 } + ); +} + +#[tokio::test] +async fn legacy_none_remains_wildcard_and_some_remains_tenant_only() { + let (pool, database) = setup_ephemeral_pool("job_legacy_read_scope", 4).await; + let fixtures = fixtures(&pool).await; + for (organization_id, visible) in [ + (None, vec![0, 1, 2]), + (fixtures[1].organization_id, vec![1]), + (fixtures[2].organization_id, vec![2]), + (Some(Uuid::now_v7()), vec![]), + ] { + for (index, fixture) in fixtures.iter().enumerate() { + assert_legacy_point_reads(&pool, organization_id, fixture, visible.contains(&index)) + .await; + } + let jobs = list_jobs( + &pool, + &JobListFilter { + organization_id, + status: None, + job_type: None, + limit: 100, + offset: 0, + }, + ) + .await + .expect("legacy list"); + assert_eq!( + jobs.iter().map(|row| row.id).collect::>(), + visible.iter().map(|&i| fixtures[i].job_id).collect() + ); + let filter = JobEnqueueIntentListFilter::new(100, 0); + let filter = match organization_id { + Some(id) => filter.with_organization_id(id), + None => filter, + }; + let intents = list_job_enqueue_intents(&pool, &filter) + .await + .expect("legacy intent list"); + assert_eq!( + intents.iter().map(|row| row.id).collect::>(), + visible.iter().map(|&i| fixtures[i].intent_id).collect() + ); + } + teardown_ephemeral_pool(pool, database).await; +} + +#[tokio::test] +async fn scoped_lists_preserve_filters_order_and_pagination() { + let (pool, database) = setup_ephemeral_pool("job_scope_pagination", 4).await; + let fixtures = fixtures(&pool).await; + for scope in [JobReadScope::Global, JobReadScope::Admin] { + let expected: Vec<_> = if scope == JobReadScope::Global { + vec![0] + } else { + vec![2, 1, 0] + }; + for (offset, &index) in expected.iter().enumerate() { + let jobs = list_jobs_with_scope( + &pool, + &JobReadListFilter { + status: Some(JobStatus::Pending), + job_type: Some("TEST.READ_SCOPE"), + limit: 1, + offset: offset as i64, + ..list_filter(scope) + }, + ) + .await + .expect("filtered job page"); + assert_eq!( + jobs.iter().map(|job| job.id).collect::>(), + vec![fixtures[index].job_id] + ); + let intents = list_job_enqueue_intents_with_scope( + &pool, + &JobEnqueueIntentReadListFilter::new(scope, 1, offset as i64) + .with_status(JobEnqueueIntentStatus::Pending) + .with_job_type_query("TEST.READ_SCOPE"), + ) + .await + .expect("filtered intent page"); + assert_eq!( + intents.iter().map(|row| row.id).collect::>(), + vec![fixtures[index].intent_id] + ); + } + for (status, job_type, offset) in [ + (Some(JobStatus::Succeeded), None, 0), + (None, Some("missing-type"), 0), + (None, None, expected.len() as i64), + ] { + assert!( + list_jobs_with_scope( + &pool, + &JobReadListFilter { + status, + job_type, + offset, + ..list_filter(scope) + } + ) + .await + .expect("empty job filter") + .is_empty() + ); + } + for filter in [ + JobEnqueueIntentReadListFilter::new(scope, 100, 0) + .with_status(JobEnqueueIntentStatus::Promoted), + JobEnqueueIntentReadListFilter::new(scope, 100, 0).with_job_type_query("missing-type"), + JobEnqueueIntentReadListFilter::new(scope, 100, expected.len() as i64), + ] { + assert!( + list_job_enqueue_intents_with_scope(&pool, &filter) + .await + .expect("empty intent filter") + .is_empty() + ); + } + } + assert_stream_pages(&pool, &fixtures[0]).await; + assert_invalid_pages(&pool, fixtures[0].job_id).await; + teardown_ephemeral_pool(pool, database).await; +} + +async fn assert_stream_pages(pool: &DbPool, fixture: &Fixture) { + let scope = JobReadScope::Global; + let first = list_job_logs_with_scope(pool, scope, fixture.job_id, 1, None) + .await + .expect("first log"); + assert_eq!(first.len(), 1); + assert_eq!(first[0].message, "first"); + let second = list_job_logs_with_scope(pool, scope, fixture.job_id, 1, Some(first[0].id)) + .await + .expect("second log"); + assert_eq!(second.len(), 1); + assert_eq!(second[0].message, "second"); + assert!(second[0].id > first[0].id); + assert!( + list_job_logs_with_scope(pool, scope, fixture.job_id, 1, Some(second[0].id)) + .await + .expect("end of logs") + .is_empty() + ); + let events = list_job_events_with_scope(pool, scope, fixture.job_id, 1, None) + .await + .expect("first event"); + assert_eq!(events.len(), 1); + assert!( + list_job_events_with_scope(pool, scope, fixture.job_id, 1, Some(events[0].id)) + .await + .expect("end of events") + .is_empty() + ); +} + +async fn assert_invalid_pages(pool: &DbPool, job_id: Uuid) { + for scope in [ + JobReadScope::Global, + JobReadScope::Organization(Uuid::now_v7()), + JobReadScope::Admin, + ] { + for (limit, offset) in [(0, 0), (JOB_LIST_PAGE_LIMIT_MAX + 1, 0), (1, -1)] { + assert!( + list_jobs_with_scope( + pool, + &JobReadListFilter { + limit, + offset, + ..list_filter(scope) + } + ) + .await + .is_err() + ); + assert!( + list_job_enqueue_intents_with_scope( + pool, + &JobEnqueueIntentReadListFilter::new(scope, limit, offset), + ) + .await + .is_err() + ); + if offset == 0 { + assert!( + list_job_events_with_scope(pool, scope, job_id, limit, None) + .await + .is_err() + ); + assert!( + list_job_logs_with_scope(pool, scope, job_id, limit, None) + .await + .is_err() + ); + } + } + } +} diff --git a/runledger-postgres/tests/job_summaries.rs b/runledger-postgres/tests/job_summaries.rs new file mode 100644 index 0000000..a890763 --- /dev/null +++ b/runledger-postgres/tests/job_summaries.rs @@ -0,0 +1,173 @@ +use chrono::{TimeZone, Utc}; +use runledger_core::jobs::{JobStatus, JobType}; +use runledger_postgres::{DbPool, jobs::*}; +use runledger_test_support::{setup_ephemeral_pool, teardown_ephemeral_pool}; +use serde_json::json; +use sqlx::types::Uuid; + +mod support; + +fn filter(scope: JobReadScope) -> JobSummaryFilter<'static> { + JobSummaryFilter { + scope, + status: None, + job_type: None, + limit: 2, + after: None, + } +} + +#[tokio::test] +async fn summaries_and_statuses_preserve_scope_exact_filters_and_cursor_boundaries() { + let (pool, database) = setup_ephemeral_pool("job_summaries", 1).await; + for name in [ + "summary.job", + "summary.job.extra", + "SUMMARY.JOB", + "summary.%", + ] { + support::register_test_job_definition(&pool, name).await; + } + let tenant = Uuid::from_u128(1000); + let other_tenant = Uuid::from_u128(2000); + let timestamp = Utc + .with_ymd_and_hms(2026, 1, 1, 0, 0, 0) + .single() + .expect("timestamp"); + for (id, organization, job_type, status) in [ + (1, None, "summary.job", "PENDING"), + (2, None, "summary.job", "PENDING"), + (3, None, "summary.job", "CANCELED"), + (4, Some(tenant), "summary.job", "PENDING"), + (5, Some(other_tenant), "summary.job", "PENDING"), + (6, None, "summary.job.extra", "PENDING"), + (7, None, "SUMMARY.JOB", "PENDING"), + (8, None, "summary.%", "PENDING"), + ] { + sqlx::query("INSERT INTO job_queue (id, organization_id, job_type, status, max_attempts, + payload, checkpoint, output, created_at, run_number, attempt, progress_done, progress_total) + VALUES ($1, $2, $3, $4::text::job_status, 3, $5, $5, $5, $6, 2, 1, 4, 10)") + .bind(Uuid::from_u128(id)).bind(organization).bind(job_type).bind(status) + .bind(json!({"wide": "x".repeat(8192)})).bind(timestamp) + .execute(&pool).await.expect("fixture"); + } + for (scope, expected) in [ + (JobReadScope::Global, vec![8, 7, 6, 3, 2, 1]), + (JobReadScope::Organization(tenant), vec![4]), + (JobReadScope::Organization(other_tenant), vec![5]), + (JobReadScope::Organization(Uuid::nil()), vec![]), + (JobReadScope::Admin, vec![8, 7, 6, 5, 4, 3, 2, 1]), + ] { + assert_scan(&pool, scope, &expected).await; + let ids = [8, 7, 6, 5, 4, 3, 2, 1, 1, 999].map(Uuid::from_u128); + let statuses = get_job_statuses_with_scope(&pool, scope, &ids) + .await + .expect("statuses"); + let mut sorted = expected; + sorted.sort(); + assert_eq!( + statuses.iter().map(|s| s.id.as_u128()).collect::>(), + sorted + ); + for status in statuses { + assert_eq!( + status.status, + if status.id == Uuid::from_u128(3) { + JobStatus::Canceled + } else { + JobStatus::Pending + } + ); + assert_eq!((status.run_number, status.attempt), (2, 1)); + } + } + let mut exact = filter(JobReadScope::Global); + exact.job_type = Some(JobType::new("summary.job")); + exact.status = Some(JobStatus::Pending); + let page = list_job_summaries(&pool, &exact).await.expect("exact page"); + assert_eq!( + page.iter().map(|s| s.id.as_u128()).collect::>(), + vec![2, 1] + ); + assert_eq!( + (page[0].progress_done, page[0].progress_total), + (Some(4), Some(10)) + ); + exact.job_type = Some(JobType::new("summary.%")); + assert_eq!( + list_job_summaries(&pool, &exact) + .await + .expect("literal wildcard")[0] + .id, + Uuid::from_u128(8) + ); + // A cursor still works after its anchor is deleted; a newer row cannot shift the next page. + let mut scan = filter(JobReadScope::Global); + scan.after = Some(JobSummaryCursor { + created_at: timestamp, + id: Uuid::from_u128(3), + }); + sqlx::query("DELETE FROM job_queue WHERE id = $1") + .bind(Uuid::from_u128(3)) + .execute(&pool) + .await + .expect("delete anchor"); + support::enqueue_test_job(&pool, "summary.job", None, &json!({})).await; + assert_eq!( + list_job_summaries(&pool, &scan) + .await + .expect("continue") + .iter() + .map(|s| s.id.as_u128()) + .collect::>(), + vec![2, 1] + ); + teardown_ephemeral_pool(pool, database).await; +} + +async fn assert_scan(pool: &DbPool, scope: JobReadScope, expected: &[u128]) { + let mut request = filter(scope); + let mut seen = Vec::new(); + loop { + let page = list_job_summaries(pool, &request).await.expect("page"); + let Some(last) = page.last() else { + break; + }; + request.after = Some(last.cursor()); + seen.extend(page.iter().map(|s| s.id.as_u128())); + assert!( + seen.len() <= expected.len(), + "scan must terminate without duplicates" + ); + } + assert_eq!(seen, expected); +} + +#[tokio::test] +async fn compact_read_bounds_are_validated_before_database_access() { + let pool = sqlx::postgres::PgPoolOptions::new() + .connect_lazy("postgres://unused:unused@localhost/unused") + .expect("lazy pool"); + for limit in [0, -1, 1001, i64::MAX] { + let mut request = filter(JobReadScope::Admin); + request.limit = limit; + let error = list_job_summaries(&pool, &request) + .await + .expect_err("invalid limit"); + assert!( + matches!(error, runledger_postgres::Error::QueryError(e) if e.code() == "job.invalid_pagination") + ); + } + assert!( + get_job_statuses_with_scope(&pool, JobReadScope::Admin, &[]) + .await + .expect("empty input") + .is_empty() + ); + let error = get_job_statuses_with_scope(&pool, JobReadScope::Admin, &[Uuid::nil(); 1001]) + .await + .expect_err("too many IDs"); + assert!( + matches!(error, runledger_postgres::Error::QueryError(e) if e.code() == "job.invalid_pagination") + ); +} diff --git a/runledger-postgres/tests/job_summary_plans.rs b/runledger-postgres/tests/job_summary_plans.rs new file mode 100644 index 0000000..02878ca --- /dev/null +++ b/runledger-postgres/tests/job_summary_plans.rs @@ -0,0 +1,171 @@ +use chrono::{TimeZone, Utc}; +use runledger_core::jobs::{JobStatus, JobType}; +use runledger_postgres::jobs::*; +use runledger_test_support::{setup_ephemeral_pool, teardown_ephemeral_pool}; +use serde_json::Value; +use sqlx::{Connection, types::Uuid}; + +mod support; + +fn has_cursor_index_condition(value: &Value, require_id: bool) -> bool { + match value { + Value::Object(object) => { + object + .get("Index Cond") + .and_then(Value::as_str) + .is_some_and(|condition| { + condition.contains("created_at") + && (!require_id || condition.contains("id")) + && condition.contains('<') + }) + || object + .values() + .any(|value| has_cursor_index_condition(value, require_id)) + } + Value::Array(values) => values + .iter() + .any(|value| has_cursor_index_condition(value, require_id)), + _ => false, + } +} + +#[tokio::test] +async fn actual_summary_cursor_constrains_index_with_custom_and_generic_plans() { + let (pool, database) = setup_ephemeral_pool("summary_plans", 1).await; + let version: String = sqlx::query_scalar("SHOW server_version") + .fetch_one(&pool) + .await + .expect("version"); + eprintln!("summary plans PostgreSQL {version}"); + assert!(version.starts_with("18.")); + support::register_test_job_definition(&pool, "summary.job").await; + support::register_test_job_definition(&pool, "summary.sparse").await; + sqlx::raw_sql( + "INSERT INTO job_queue (job_type, max_attempts, organization_id, created_at, status) + SELECT CASE WHEN n % 101 = 0 THEN 'summary.sparse' ELSE 'summary.job' END, 3, CASE WHEN n % 3 = 0 THEN NULL ELSE md5((n % 3)::text)::uuid END, + '2026-01-01'::timestamptz + n * interval '1 second', + CASE WHEN n % 97 = 0 THEN 'CANCELED' ELSE 'PENDING' END::job_status + FROM generate_series(1, 30000) n; ANALYZE job_queue;", + ) + .execute(&pool) + .await + .expect("seed"); + let tenant: Uuid = sqlx::query_scalar("SELECT md5('1')::uuid") + .fetch_one(&pool) + .await + .expect("tenant"); + let created_at = Utc + .with_ymd_and_hms(2026, 1, 1, 0, 16, 40) + .single() + .expect("timestamp"); + for mode in ["force_custom_plan", "force_generic_plan"] { + sqlx::raw_sql(&format!("SET plan_cache_mode = {mode}")) + .execute(&pool) + .await + .expect("plan mode"); + for scope in [ + JobReadScope::Global, + JobReadScope::Organization(tenant), + JobReadScope::Admin, + ] { + for (status, job_type) in [ + (None, None), + (Some(JobStatus::Canceled), None), + (None, Some(JobType::new("summary.sparse"))), + ( + Some(JobStatus::Canceled), + Some(JobType::new("summary.sparse")), + ), + ] { + assert_summary_plan(&pool, mode, scope, created_at, status, job_type).await; + } + } + } + teardown_ephemeral_pool(pool, database).await; +} + +async fn assert_summary_plan( + pool: &runledger_postgres::DbPool, + mode: &str, + scope: JobReadScope, + created_at: chrono::DateTime, + status: Option, + job_type: Option>, +) { + pool.acquire() + .await + .expect("connection") + .clear_cached_statements() + .await + .expect("clear statements"); + let rows = list_job_summaries( + pool, + &JobSummaryFilter { + scope, + status, + job_type, + limit: 20, + after: Some(JobSummaryCursor { + created_at, + id: Uuid::nil(), + }), + }, + ) + .await + .expect("page"); + let expected: Vec<_> = (1_i64..1000) + .rev() + .filter(|n| { + let scope_matches = match scope { + JobReadScope::Global => n % 3 == 0, + JobReadScope::Organization(_) => n % 3 == 1, + JobReadScope::Admin => true, + }; + scope_matches + && (status.is_none() || n % 97 == 0) + && (job_type.is_none() || n % 101 == 0) + }) + .take(20) + .collect(); + let epoch = Utc + .with_ymd_and_hms(2026, 1, 1, 0, 0, 0) + .single() + .expect("epoch"); + assert_eq!( + rows.iter() + .map(|r| (r.created_at - epoch).num_seconds()) + .collect::>(), + expected + ); + assert!(rows.iter().all(|r| status.is_none_or(|s| r.status == s) + && job_type.is_none_or(|t| r.job_type.as_str() == t.as_str()))); + let name: String = sqlx::query_scalar( + "SELECT name FROM pg_prepared_statements + WHERE statement LIKE 'SELECT id, job_type,%' + AND statement LIKE '%AND (created_at, id) < ($5, $6)%' + AND statement LIKE '%FROM job_queue WHERE%' LIMIT 1", + ) + .fetch_one(pool) + .await + .expect("actual public query"); + let organization = match scope { + JobReadScope::Organization(id) => format!("'{id}'::uuid"), + _ => "NULL".into(), + }; + let status_sql = status.map_or_else(|| "NULL".into(), |s| format!("'{}'", s.as_db_value())); + let type_sql = job_type.map_or_else(|| "NULL".into(), |t| format!("'{}'", t.as_str())); + let plan: Value = sqlx::query_scalar(&format!("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON, TIMING OFF) + EXECUTE \"{}\" ({organization}, {status_sql}, {type_sql}, 20, '2026-01-01 00:16:40+00', '00000000-0000-0000-0000-000000000000')", name.replace('"', "\"\""))) + .fetch_one(pool).await.expect("explain"); + // A selective custom plan may use the existing type/status/time index, + // applying the UUID tie-break as a residual filter. That is valid bounded + // access too; the independent expected rows above verify full semantics. + assert!( + has_cursor_index_condition(&plan, status.is_none() && job_type.is_none()), + "{mode} {scope:?}: {plan}" + ); + eprintln!( + "{mode} {scope:?} status={status:?} type={job_type:?}: execution={} ms, shared buffers={}", + plan[0]["Execution Time"], plan[0]["Plan"]["Shared Hit Blocks"] + ); +} diff --git a/runledger-postgres/tests/legacy_read_contracts.rs b/runledger-postgres/tests/legacy_read_contracts.rs new file mode 100644 index 0000000..c15e1d4 --- /dev/null +++ b/runledger-postgres/tests/legacy_read_contracts.rs @@ -0,0 +1,434 @@ +use runledger_core::jobs::JobType; +use runledger_postgres::prelude::*; +use runledger_test_support::{setup_ephemeral_pool, teardown_ephemeral_pool}; +use serde_json::json; +use sqlx::types::Uuid; + +mod support; + +const JOB_TYPE: &str = "jobs.test.legacy_reads"; + +#[tokio::test] +async fn metrics_none_aggregates_global_and_tenant_rows_while_payload_keys_are_tenant_local() { + let (pool, database) = setup_ephemeral_pool("legacy_read_contracts", 3).await; + support::register_test_job_definition(&pool, JOB_TYPE).await; + let tenants = [Uuid::now_v7(), Uuid::now_v7()]; + let run_id = Uuid::now_v7(); + let mut rows = Vec::new(); + for organization_id in [None, Some(tenants[0]), Some(tenants[1])] { + let payload = json!({"run_id":run_id, "organization_id":organization_id}); + let id = enqueue_job( + &pool, + &JobEnqueue { + job_type: JobType::new(JOB_TYPE), + organization_id, + payload: &payload, + idempotency_key: Some("same-key"), + priority: None, + max_attempts: None, + timeout_seconds: None, + next_run_at: None, + stage: None, + }, + ) + .await + .expect("same key in independent scopes"); + let intent = JobEnqueueIntent::new(JobType::new(JOB_TYPE), &payload, "same-key"); + let intent = match organization_id { + Some(id) => intent.with_organization_id(id), + None => intent, + }; + let recorded = record_job_enqueue_intent(&pool, &intent) + .await + .expect("intent"); + assert_eq!(recorded.status(), JobEnqueueIntentStatus::Pending); + rows.push((id, payload)); + } + for (organization_id, count) in [ + (None, 3), + (Some(tenants[0]), 1), + (Some(tenants[1]), 1), + (Some(Uuid::now_v7()), 0), + ] { + let metrics = get_job_metrics(&pool, organization_id, Some(JOB_TYPE)) + .await + .expect("job metrics"); + assert_eq!( + metrics.len(), + 1, + "job definitions remain visible with zero counts" + ); + assert_eq!(metrics[0].pending_count, count); + let filter = + JobEnqueueIntentMetricsFilter::new(10, 0).with_job_type(JobType::new(JOB_TYPE)); + let filter = match organization_id { + Some(id) => filter.with_organization_id(id), + None => filter, + }; + let metrics = get_job_enqueue_intent_metrics(&pool, &filter) + .await + .expect("intent metrics"); + assert_eq!( + metrics.iter().map(|row| row.pending_count).sum::(), + count + ); + assert_eq!(metrics.len(), usize::from(count != 0)); + } + for (index, tenant) in tenants.into_iter().enumerate() { + assert_eq!( + get_job_payload_by_idempotency_key(&pool, tenant, JobType::new(JOB_TYPE), "same-key") + .await + .expect("key lookup"), + Some(rows[index + 1].clone()) + ); + assert_eq!( + get_latest_job_payload_for_run(&pool, tenant, JobType::new(JOB_TYPE), run_id) + .await + .expect("run lookup"), + Some(rows[index + 1].clone()) + ); + } + // Neither legacy payload helper interprets a sentinel UUID as global/admin. + assert_eq!( + get_job_payload_by_idempotency_key(&pool, Uuid::nil(), JobType::new(JOB_TYPE), "same-key") + .await + .expect("absent tenant"), + None + ); + assert_eq!( + get_latest_job_payload_for_run(&pool, Uuid::nil(), JobType::new(JOB_TYPE), run_id) + .await + .expect("absent tenant"), + None + ); + let global = list_jobs_with_scope( + &pool, + &JobReadListFilter { + scope: JobReadScope::Global, + status: None, + job_type: Some(JOB_TYPE), + limit: 10, + offset: 0, + }, + ) + .await + .expect("exact global inspection"); + assert_eq!(global.len(), 1); + assert_eq!(global[0].id, rows[0].0); + teardown_ephemeral_pool(pool, database).await; +} + +async fn enqueue_payload( + pool: &DbPool, + scope: JobScope, + job_type: &str, + key: &str, + payload: &serde_json::Value, +) -> Uuid { + enqueue_job( + pool, + &JobEnqueue { + job_type: JobType::new(job_type), + organization_id: scope.organization_id(), + payload, + idempotency_key: Some(key), + priority: None, + max_attempts: None, + timeout_seconds: None, + next_run_at: None, + stage: None, + }, + ) + .await + .expect("enqueue scoped payload") +} + +#[tokio::test] +async fn exact_payload_scopes_preserve_duplicate_keys_and_latest_ordering() { + let (pool, database) = setup_ephemeral_pool("exact_payload_scopes", 3).await; + support::register_test_job_definition(&pool, JOB_TYPE).await; + let other_type = "jobs.test.legacy_reads.other"; + support::register_test_job_definition(&pool, other_type).await; + let scopes = [ + JobScope::Global, + JobScope::Organization(Uuid::now_v7()), + JobScope::Organization(Uuid::now_v7()), + ]; + let run_id = Uuid::now_v7(); + let mut expected = Vec::new(); + for (index, scope) in scopes.into_iter().enumerate() { + let payload = json!({"run_id": run_id, "scope": index, "version": "keyed"}); + let id = enqueue_payload(&pool, scope, JOB_TYPE, "shared", &payload).await; + let mut candidates = Vec::new(); + for version in ["tie-a", "tie-b", "older-high-id"] { + let value = json!({"run_id": run_id, "scope": index, "version": version}); + let id = enqueue_payload(&pool, scope, JOB_TYPE, version, &value).await; + candidates.push((id, value)); + } + // The larger UUID wins a timestamp tie, but a larger UUID cannot beat a newer timestamp. + candidates.sort_by_key(|row| row.0); + for (position, row) in candidates.iter().enumerate() { + sqlx::query("UPDATE job_queue SET created_at = '2026-01-01'::timestamptz + make_interval(days => $2) WHERE id = $1") + .bind(row.0).bind(if position == 2 { 0 } else { 10 + index as i32 }).execute(&pool).await.expect("set deterministic order"); + } + sqlx::query("UPDATE job_queue SET created_at = '2025-01-01'::timestamptz WHERE id = $1") + .bind(id) + .execute(&pool) + .await + .expect("age keyed row"); + // Same scope and run, wrong job type, newer than every matching row. + enqueue_payload( + &pool, + scope, + other_type, + "shared", + &json!({"run_id":run_id}), + ) + .await; + expected.push(((id, payload), candidates[1].clone())); + } + enqueue_payload( + &pool, + JobScope::Organization(Uuid::now_v7()), + JOB_TYPE, + "shared", + &json!({"run_id":run_id, "scope":"newer-unrelated"}), + ) + .await; + for (scope, (keyed, latest)) in scopes.into_iter().zip(expected) { + assert_eq!( + get_job_payload_by_idempotency_key_with_scope( + &pool, + scope, + JobType::new(JOB_TYPE), + "shared" + ) + .await + .expect("scoped payload or metrics read succeeds"), + Some(keyed.clone()) + ); + assert_eq!( + get_latest_job_payload_for_run_with_scope(&pool, scope, JobType::new(JOB_TYPE), run_id) + .await + .expect("scoped payload or metrics read succeeds"), + Some(latest.clone()) + ); + if let JobScope::Organization(tenant) = scope { + assert_eq!( + get_job_payload_by_idempotency_key(&pool, tenant, JobType::new(JOB_TYPE), "shared") + .await + .expect("scoped payload or metrics read succeeds"), + Some(keyed) + ); + assert_eq!( + get_latest_job_payload_for_run(&pool, tenant, JobType::new(JOB_TYPE), run_id) + .await + .expect("scoped payload or metrics read succeeds"), + Some(latest) + ); + } + assert_eq!( + get_job_payload_by_idempotency_key_with_scope( + &pool, + scope, + JobType::new(JOB_TYPE), + "missing" + ) + .await + .expect("scoped payload or metrics read succeeds"), + None + ); + assert_eq!( + get_latest_job_payload_for_run_with_scope( + &pool, + scope, + JobType::new(JOB_TYPE), + Uuid::nil() + ) + .await + .expect("scoped payload or metrics read succeeds"), + None + ); + } + for tenant in [Uuid::now_v7(), Uuid::nil()] { + assert_eq!( + get_job_payload_by_idempotency_key_with_scope( + &pool, + JobScope::Organization(tenant), + JobType::new(JOB_TYPE), + "shared" + ) + .await + .expect("scoped payload or metrics read succeeds"), + None + ); + assert_eq!( + get_latest_job_payload_for_run_with_scope( + &pool, + JobScope::Organization(tenant), + JobType::new(JOB_TYPE), + run_id + ) + .await + .expect("scoped payload or metrics read succeeds"), + None + ); + assert_eq!( + get_job_payload_by_idempotency_key(&pool, tenant, JobType::new(JOB_TYPE), "shared") + .await + .expect("scoped payload or metrics read succeeds"), + None + ); + assert_eq!( + get_latest_job_payload_for_run(&pool, tenant, JobType::new(JOB_TYPE), run_id) + .await + .expect("scoped payload or metrics read succeeds"), + None + ); + } + assert_nil_payload_scope(&pool).await; + teardown_ephemeral_pool(pool, database).await; +} + +fn metric_values(row: &JobMetricsRecord) -> ([i64; 9], [Option; 2]) { + ( + [ + row.pending_count, + row.leased_count, + row.stale_leases, + row.succeeded_24h, + row.retryable_24h, + row.terminal_24h, + row.panicked_24h, + row.timeout_24h, + row.dead_lettered_24h, + ], + [row.p50_duration_ms_24h, row.p95_duration_ms_24h], + ) +} + +#[tokio::test] +async fn exact_job_metrics_preserve_counts_durations_and_empty_definitions() { + let (pool, database) = setup_ephemeral_pool("exact_job_metrics", 3).await; + support::register_test_job_definition(&pool, JOB_TYPE).await; + let empty_type = "jobs.test.legacy_reads.empty"; + support::register_test_job_definition(&pool, empty_type).await; + let tenants = [Uuid::now_v7(), Uuid::now_v7()]; + for (scope, count) in [ + (JobScope::Global, 1), + (JobScope::Organization(tenants[0]), 2), + (JobScope::Organization(tenants[1]), 4), + ] { + for index in 0..count { + let id = enqueue_payload( + &pool, + scope, + JOB_TYPE, + &format!("pending-{index}"), + &json!({}), + ) + .await; + for (attempt, outcome) in ["RETRYABLE", "TERMINAL", "PANICKED", "TIMEOUT"] + .into_iter() + .enumerate() + { + sqlx::query("INSERT INTO job_attempts (job_id, attempt, worker_id, leased_at, started_at, finished_at, outcome) VALUES ($1, $2, 'metrics', now(), now(), now() + make_interval(secs => $3), $4::job_failure_kind)") + .bind(id).bind(attempt as i32 + 1).bind(f64::from(count)).bind(outcome).execute(&pool).await.expect("attempt fixture"); + } + sqlx::query("INSERT INTO job_events (job_id, event_type) VALUES ($1, 'SUCCEEDED')") + .bind(id) + .execute(&pool) + .await + .expect("success fixture"); + sqlx::query("INSERT INTO job_dead_letters (job_id, job_type, organization_id, attempt, payload_snapshot, failed_at) VALUES ($1, $2, $3, 1, '{}'::jsonb, now())") + .bind(id).bind(JOB_TYPE).bind(scope.organization_id()).execute(&pool).await.expect("dead letter fixture"); + let leased = enqueue_payload( + &pool, + scope, + JOB_TYPE, + &format!("leased-{index}"), + &json!({}), + ) + .await; + sqlx::query("UPDATE job_queue SET status = 'LEASED', lease_expires_at = now() - interval '1 hour' WHERE id = $1").bind(leased).execute(&pool).await.expect("stale lease fixture"); + } + } + for (scope, count, duration) in [ + (JobReadScope::Global, 1, Some(1000.0)), + (JobReadScope::Organization(tenants[0]), 2, Some(2000.0)), + (JobReadScope::Organization(tenants[1]), 4, Some(4000.0)), + // Preserve averaging per-scope percentiles, rather than pooling all attempts. + (JobReadScope::Admin, 7, Some(7000.0 / 3.0)), + (JobReadScope::Organization(Uuid::now_v7()), 0, None), + ] { + let rows = get_job_metrics_with_scope(&pool, scope, Some(JOB_TYPE)) + .await + .expect("scoped payload or metrics read succeeds"); + assert_eq!(rows.len(), 1); + assert_eq!(metric_values(&rows[0]), ([count; 9], [duration, duration])); + let all = get_job_metrics_with_scope(&pool, scope, None) + .await + .expect("scoped payload or metrics read succeeds"); + assert_eq!(all.len(), 2); + assert_eq!(all[0].job_type.as_str(), JOB_TYPE); + assert_eq!(all[1].job_type.as_str(), empty_type); + assert_eq!(metric_values(&all[1]), ([0; 9], [None; 2])); + assert!( + get_job_metrics_with_scope(&pool, scope, Some("legacy_reads")) + .await + .expect("scoped payload or metrics read succeeds") + .is_empty(), + "job type filter is exact" + ); + let legacy_scope = match scope { + JobReadScope::Global => continue, + JobReadScope::Organization(id) => Some(id), + JobReadScope::Admin => None, + }; + let legacy = get_job_metrics(&pool, legacy_scope, Some(JOB_TYPE)) + .await + .expect("scoped payload or metrics read succeeds"); + assert_eq!(legacy.len(), 1); + assert_eq!(metric_values(&legacy[0]), metric_values(&rows[0])); + } + teardown_ephemeral_pool(pool, database).await; +} + +async fn assert_nil_payload_scope(pool: &DbPool) { + // Nil UUID is an exact tenant/run value when it actually has data, never a sentinel. + let payload = json!({"run_id":Uuid::nil(), "scope":"nil-tenant"}); + let id = enqueue_payload( + pool, + JobScope::Organization(Uuid::nil()), + JOB_TYPE, + "shared", + &payload, + ) + .await; + for result in [ + get_job_payload_by_idempotency_key_with_scope( + pool, + JobScope::Organization(Uuid::nil()), + JobType::new(JOB_TYPE), + "shared", + ) + .await + .expect("scoped payload or metrics read succeeds"), + get_latest_job_payload_for_run_with_scope( + pool, + JobScope::Organization(Uuid::nil()), + JobType::new(JOB_TYPE), + Uuid::nil(), + ) + .await + .expect("scoped payload or metrics read succeeds"), + get_job_payload_by_idempotency_key(pool, Uuid::nil(), JobType::new(JOB_TYPE), "shared") + .await + .expect("scoped payload or metrics read succeeds"), + get_latest_job_payload_for_run(pool, Uuid::nil(), JobType::new(JOB_TYPE), Uuid::nil()) + .await + .expect("scoped payload or metrics read succeeds"), + ] { + assert_eq!(result, Some((id, payload.clone()))); + } +} diff --git a/runledger-postgres/tests/migrations.rs b/runledger-postgres/tests/migrations.rs index 0e3948e..d8bf10b 100644 --- a/runledger-postgres/tests/migrations.rs +++ b/runledger-postgres/tests/migrations.rs @@ -34,6 +34,7 @@ const JOB_ENQUEUE_INTENTS_MIGRATION_VERSION: i64 = 202608180001; const CONTINUATION_METRICS_CTE_MIGRATION_VERSION: i64 = 202608230001; const WORKFLOW_STEP_JOB_LINK_EXPAND_MIGRATION_VERSION: i64 = 202608240001; const WORKFLOW_STEP_JOB_LINK_CONTRACT_MIGRATION_VERSION: i64 = 202608240002; +const JOB_SUMMARY_PAGINATION_MIGRATION_VERSION: i64 = 202609050001; const COMPATIBILITY_FENCE_EXEMPT_MIGRATION_VERSIONS: &[i64] = &[ // Adds replay lineage and a read-only metrics view without changing legacy writes. REPLAY_METRICS_MIGRATION_VERSION, @@ -60,9 +61,47 @@ const COMPATIBILITY_FENCE_EXEMPT_MIGRATION_VERSIONS: &[i64] = &[ // Keeps the legacy reciprocal column as a trigger-maintained projection so // old and new binaries can coexist before the fenced contract migration. WORKFLOW_STEP_JOB_LINK_EXPAND_MIGRATION_VERSION, + JOB_SUMMARY_PAGINATION_MIGRATION_VERSION, // Additive pagination indexes. ]; const TEST_HARNESS_POOL_CONNECTIONS: u32 = 4; +#[tokio::test] +async fn summary_indexes_are_required_by_startup_but_not_the_custom_compatibility_fence() { + let harness = TestHarness::fresh("summary_index_startup").await; + record_postgres_18_server_version(&harness.pool, "summary index startup guard").await; + apply_runledger_migrations_through( + &harness.pool, + WORKFLOW_STEP_JOB_LINK_CONTRACT_MIGRATION_VERSION, + ) + .await; + let error = ensure_schema_compatible_after_idempotency_cutover(&harness.pool) + .await + .expect_err("current startup must require the additive index migration"); + assert!( + matches!( + error, + SchemaCompatibilityError::Incompatible(sqlx::migrate::MigrateError::VersionTooNew( + JOB_SUMMARY_PAGINATION_MIGRATION_VERSION, + _ + )) + ), + "unexpected startup error: {error}" + ); + apply_runledger_migration(&harness.pool, JOB_SUMMARY_PAGINATION_MIGRATION_VERSION).await; + ensure_schema_compatible_after_idempotency_cutover(&harness.pool) + .await + .expect("startup accepts SQLx history without a custom fence entry"); + let recorded: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM runledger_migration_history WHERE version = $1)", + ) + .bind(JOB_SUMMARY_PAGINATION_MIGRATION_VERSION) + .fetch_one(&harness.pool) + .await + .expect("custom fence"); + assert!(!recorded); + harness.teardown().await; +} + struct TestHarness { pool: PgPool, database: EphemeralDatabase, @@ -933,6 +972,9 @@ async fn workflow_step_job_link_rollout_backfills_and_supports_mixed_writers() { .await; let workflow_run_id = seed_and_expand_workflow_job_links(&harness.pool).await; + // Apply later additive indexes while deliberately retaining expand-schema + // triggers and mixed writers; the contract migration remains unapplied. + apply_runledger_migration(&harness.pool, JOB_SUMMARY_PAGINATION_MIGRATION_VERSION).await; let new_job_id = assert_expand_mixed_writers(&harness.pool, workflow_run_id).await; assert_expand_trigger_guard_rejections(&harness.pool).await; assert_expand_trigger_modes(&harness.pool).await; diff --git a/runledger-postgres/tests/operational_costs.rs b/runledger-postgres/tests/operational_costs.rs new file mode 100644 index 0000000..0f5787e --- /dev/null +++ b/runledger-postgres/tests/operational_costs.rs @@ -0,0 +1,381 @@ +//! Reproducible diagnostic, deliberately excluded from timing-sensitive CI. +use std::time::Instant; + +use runledger_core::jobs::{ + JobType, StepKey, WorkflowRunEnqueueBuilder, WorkflowStepEnqueueBuilder, WorkflowType, +}; +use runledger_postgres::{DbPool, jobs::*}; +use runledger_test_support::{setup_ephemeral_pool, teardown_ephemeral_pool}; +use serde_json::json; + +mod support; + +fn report(label: &str, mut samples: Vec) { + samples.sort_by(f64::total_cmp); + println!( + "{label}: median_ms={:.3} p95_ms={:.3}", + samples[samples.len() / 2], + samples[(samples.len() * 95 / 100).min(samples.len() - 1)] + ); +} + +async fn reads(pool: &DbPool) { + // Deterministic, individually varied text; payload/checkpoint/output are deliberately wide. + sqlx::raw_sql( + "INSERT INTO job_queue (job_type, max_attempts, payload, checkpoint, output, created_at) + SELECT 'cost.job', 3, jsonb_build_object('data', data), jsonb_build_object('data', data), + jsonb_build_object('data', data), '2026-01-01'::timestamptz + n * interval '1 second' + FROM generate_series(1, 10000) n + CROSS JOIN LATERAL (SELECT string_agg(md5(n::text || ':' || k::text), '') AS data + FROM generate_series(1, 128) k) wide; + ANALYZE job_queue;", + ) + .execute(pool) + .await + .expect("seed reads"); + for offset in [0, 9000] { + let mut samples = Vec::new(); + let mut json_bytes = 0; + for sample in 0..32 { + let start = Instant::now(); + let rows = list_jobs_with_scope( + pool, + &JobReadListFilter { + scope: JobReadScope::Global, + status: None, + job_type: None, + limit: 100, + offset, + }, + ) + .await + .expect("full page"); + let elapsed = start.elapsed().as_secs_f64() * 1000.; + assert_eq!(rows.len(), 100); + if sample > 0 { + samples.push(elapsed); + } + json_bytes = rows + .iter() + .map(|r| { + r.payload.to_string().len() + + r.checkpoint.as_ref().map_or(0, |v| v.to_string().len()) + + r.output.as_ref().map_or(0, |v| v.to_string().len()) + }) + .sum::(); + } + report( + &format!("full offset={offset} json_bytes={json_bytes}"), + samples, + ); + } + compact_reads(pool).await; +} + +async fn compact_reads(pool: &DbPool) { + let (created_at, id) = sqlx::query_as( + "SELECT created_at, id FROM job_queue + ORDER BY created_at DESC, id DESC OFFSET 8999 LIMIT 1", + ) + .fetch_one(pool) + .await + .expect("cursor outside measured region"); + for after in [None, Some(JobSummaryCursor { created_at, id })] { + let request = JobSummaryFilter { + scope: JobReadScope::Global, + status: None, + job_type: None, + limit: 100, + after, + }; + let mut samples = Vec::new(); + for sample in 0..32 { + let start = Instant::now(); + let rows = list_job_summaries(pool, &request) + .await + .expect("compact page"); + let elapsed = start.elapsed().as_secs_f64() * 1000.; + assert_eq!(rows.len(), 100); + if sample > 0 { + samples.push(elapsed); + } + } + report( + &format!("compact after={} json_bytes=0", after.is_some()), + samples, + ); + } + // Compare equal projections/decoding. Derive the offset counterpart from + // the actual prepared public cursor query, changing pagination only. + let cursor_sql: String = sqlx::query_scalar( + "SELECT statement FROM pg_prepared_statements + WHERE statement LIKE 'SELECT id, job_type,%' + AND statement LIKE '%AND (created_at, id) < ($5, $6)%' + AND statement LIKE '%organization_id IS NULL%' LIMIT 1", + ) + .fetch_one(pool) + .await + .expect("public cursor statement"); + let offset_sql = cursor_sql.replace("AND (created_at, id) < ($5, $6)", "") + " OFFSET $5"; + let mut cursor_samples = Vec::new(); + let mut offset_samples = Vec::new(); + for sample in 0..32 { + let start = Instant::now(); + let cursor_rows = sqlx::query(&cursor_sql) + .bind(None::) + .bind(None::) + .bind(None::) + .bind(100_i64) + .bind(created_at) + .bind(id) + .fetch_all(pool) + .await + .expect("cursor raw"); + let cursor_elapsed = start.elapsed().as_secs_f64() * 1000.; + let start = Instant::now(); + let offset_rows = sqlx::query(&offset_sql) + .bind(None::) + .bind(None::) + .bind(None::) + .bind(100_i64) + .bind(9000_i64) + .fetch_all(pool) + .await + .expect("offset raw"); + let offset_elapsed = start.elapsed().as_secs_f64() * 1000.; + use sqlx::Row; + assert_eq!( + cursor_rows + .iter() + .map(|r| r.get::("id")) + .collect::>(), + offset_rows + .iter() + .map(|r| r.get::("id")) + .collect::>() + ); + if sample > 0 { + cursor_samples.push(cursor_elapsed); + offset_samples.push(offset_elapsed); + } + } + report("compact raw cursor at depth 9000", cursor_samples); + report("compact raw offset=9000", offset_samples); +} + +async fn workflows(pool: &DbPool) { + let payload = json!({"sample": true}); + for count in [10_usize, 100, 600] { + for width in [1, 4] { + let keys = (0..count).map(|i| format!("s{i}")).collect::>(); + let mut builder = + WorkflowRunEnqueueBuilder::new(WorkflowType::new("cost.workflow"), &payload); + let mut edges = 0; + for i in 0..count { + let deps = keys[i.saturating_sub(width)..i] + .iter() + .map(|s| StepKey::new(s)) + .collect::>(); + edges += deps.len(); + builder = builder.step( + WorkflowStepEnqueueBuilder::new( + StepKey::new(&keys[i]), + JobType::new("cost.job"), + &payload, + ) + .depends_on_success(&deps) + .try_build() + .expect("step"), + ); + } + let run = builder.try_build().expect("run"); + let mut samples = Vec::new(); + sqlx::query("SELECT pg_stat_statements_reset()") + .execute(pool) + .await + .expect("reset counters"); + for sample in 0..12 { + let start = Instant::now(); + enqueue_workflow_run(pool, &run) + .await + .expect("enqueue workflow"); + if sample > 0 { + samples.push(start.elapsed().as_secs_f64() * 1000.); + } + } + report(&format!("workflow V={count} E={edges}"), samples); + let counts: Vec<(String, i64)> = sqlx::query_as("SELECT CASE WHEN query ILIKE 'INSERT INTO workflow_steps %' THEN 'step inserts' + WHEN query ILIKE 'INSERT INTO workflow_step_dependencies %' THEN 'edge inserts' ELSE 'other statements' END, + sum(calls)::bigint FROM pg_stat_statements WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND query NOT ILIKE '%pg_stat_statements%' GROUP BY 1 ORDER BY 1") + .fetch_all(pool).await.expect("statement counts"); + println!("12 enqueues: {counts:?}"); + } + } +} + +async fn direct_jobs(pool: &DbPool) { + let payload = json!({"sample": true}); + let request = JobEnqueue { + job_type: JobType::new("cost.job"), + organization_id: None, + payload: &payload, + priority: None, + max_attempts: None, + timeout_seconds: None, + next_run_at: None, + idempotency_key: None, + stage: None, + }; + for own_transaction in [true, false] { + let mut samples = Vec::new(); + sqlx::query("SELECT pg_stat_statements_reset()") + .execute(pool) + .await + .expect("reset counters"); + for sample in 0..12 { + let start = Instant::now(); + if own_transaction { + for _ in 0..100 { + enqueue_job_with_outcome(pool, &request) + .await + .expect("direct enqueue"); + } + } else { + let mut tx = pool.begin().await.expect("begin"); + for _ in 0..100 { + enqueue_job_with_outcome_tx(&mut tx, &request) + .await + .expect("direct enqueue tx"); + } + tx.commit().await.expect("commit"); + } + if sample > 0 { + samples.push(start.elapsed().as_secs_f64() * 1000.); + } + } + report( + &format!("100 direct jobs own_transaction={own_transaction}"), + samples, + ); + let calls: i64 = sqlx::query_scalar( + "SELECT sum(calls)::bigint FROM pg_stat_statements + WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND query NOT ILIKE '%pg_stat_statements%'", + ) + .fetch_one(pool) + .await + .expect("direct statement count"); + println!("12 direct groups: total statements={calls}"); + } +} + +#[tokio::test] +#[ignore = "manual PostgreSQL 18 measurement; requires shared_preload_libraries=pg_stat_statements"] +async fn measure_operational_costs() { + let (pool, database) = setup_ephemeral_pool("operational_costs", 1).await; + let version: String = sqlx::query_scalar("SHOW server_version") + .fetch_one(&pool) + .await + .expect("version"); + let version_num: String = sqlx::query_scalar("SHOW server_version_num") + .fetch_one(&pool) + .await + .expect("version num"); + assert!(version_num.starts_with("18")); + println!("PostgreSQL {version}; server_version_num={version_num}; pool max=1"); + sqlx::raw_sql("CREATE EXTENSION pg_stat_statements") + .execute(&pool) + .await + .expect("extension"); + support::register_test_job_definition(&pool, "cost.job").await; + reads(&pool).await; + workflows(&pool).await; + direct_jobs(&pool).await; + println!("connections={}, idle={}", pool.size(), pool.num_idle()); + teardown_ephemeral_pool(pool, database).await; +} + +#[tokio::test] +#[ignore = "manual PostgreSQL 18 measurement; requires shared_preload_libraries=pg_stat_statements"] +async fn measure_progress_costs() { + let (pool, database) = setup_ephemeral_pool("progress_costs", 1).await; + let version: String = sqlx::query_scalar("SHOW server_version") + .fetch_one(&pool) + .await + .expect("version"); + assert!(version.starts_with("18.")); + println!("progress costs PostgreSQL {version}; pool max=1"); + sqlx::raw_sql("CREATE EXTENSION pg_stat_statements") + .execute(&pool) + .await + .expect("extension"); + support::register_test_job_definition(&pool, "cost.progress").await; + let id = support::enqueue_test_job(&pool, "cost.progress", None, &json!({})).await; + let job = support::claim_one_job(&pool, "progress-cost-worker").await; + let identity = JobLeaseIdentity::new(id, job.run_number, job.attempt, "progress-cost-worker"); + let checkpoint = json!({"page": 1}); + let update = JobOrdinaryProgressUpdate { + progress_done: Some(1), + progress_total: Some(10), + checkpoint: Some(&checkpoint), + }; + update_job_ordinary_progress_for_lease(&pool, identity, &update) + .await + .expect("warm up"); + sqlx::query("SELECT pg_stat_statements_reset(0, (SELECT oid FROM pg_database WHERE datname = current_database()))") + .execute(&pool).await.expect("reset this database's statistics"); + let mut samples = Vec::new(); + for _ in 0..64 { + let start = Instant::now(); + update_job_ordinary_progress_for_lease(&pool, identity, &update) + .await + .expect("progress commits"); + samples.push(start.elapsed().as_secs_f64() * 1000.); + } + let statements: Vec<(String, i64)> = sqlx::query_as( + "SELECT query, sum(calls)::bigint FROM pg_stat_statements + WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND toplevel AND query NOT ILIKE '%pg_stat_statements%' + GROUP BY query ORDER BY query", + ) + .fetch_all(&pool) + .await + .expect("actual progress statements"); + let calls: i64 = statements.iter().map(|(_, calls)| calls).sum(); + println!( + "progress: writes=64, total_statements={calls}, statements_per_write={}", + calls / 64 + ); + assert!( + calls <= 64 * 6, + "ordinary progress exceeded its six-statement budget: {statements:?}" + ); + for (query, calls) in &statements { + println!( + "calls={calls}: {}", + query.split_whitespace().collect::>().join(" ") + ); + } + report("ordinary progress with checkpoint and audit event", samples); + let saved = get_job_by_id(&pool, None, id) + .await + .expect("read") + .expect("job"); + assert_eq!(saved.checkpoint, Some(checkpoint)); + assert_eq!( + (saved.progress_done, saved.progress_total), + (Some(1), Some(10)) + ); + let events = list_job_events(&pool, None, id, 100, None) + .await + .expect("events"); + assert_eq!( + events + .iter() + .filter(|event| event.event_type == runledger_core::jobs::JobEventType::Progress) + .count(), + 65 + ); + teardown_ephemeral_pool(pool, database).await; +} diff --git a/runledger-postgres/tests/progress_validation.rs b/runledger-postgres/tests/progress_validation.rs new file mode 100644 index 0000000..8fee592 --- /dev/null +++ b/runledger-postgres/tests/progress_validation.rs @@ -0,0 +1,186 @@ +use runledger_core::jobs::JobProgressValidationError; +use runledger_postgres::Error; +use runledger_postgres::prelude::*; +use runledger_test_support::{setup_ephemeral_pool, teardown_ephemeral_pool}; +use serde_json::json; + +mod support; + +#[tokio::test] +async fn competing_partial_progress_updates_validate_against_the_locked_row() { + let (pool, database) = setup_ephemeral_pool("competing_progress", 4).await; + support::register_test_job_definition(&pool, "jobs.test.progress_validation").await; + let id = + support::enqueue_test_job(&pool, "jobs.test.progress_validation", None, &json!({})).await; + let job = support::claim_one_job(&pool, "progress-worker").await; + let identity = JobLeaseIdentity::new(id, job.run_number, job.attempt, "progress-worker"); + update_job_ordinary_progress_for_lease( + &pool, + identity, + &JobOrdinaryProgressUpdate { + progress_done: Some(1), + progress_total: Some(3), + checkpoint: Some(&json!("initial")), + }, + ) + .await + .expect("initial progress"); + let done_checkpoint = json!("done"); + let total_checkpoint = json!("total"); + let done = JobOrdinaryProgressUpdate { + progress_done: Some(3), + progress_total: None, + checkpoint: Some(&done_checkpoint), + }; + let total = JobOrdinaryProgressUpdate { + progress_done: None, + progress_total: Some(2), + checkpoint: Some(&total_checkpoint), + }; + // Both are valid against the starting row; only one remains valid once the + // other commits. The loser must return a typed validation error, not a CHECK. + let (done_result, total_result) = tokio::join!( + update_job_ordinary_progress_for_lease(&pool, identity, &done), + update_job_ordinary_progress_for_lease(&pool, identity, &total), + ); + assert_ne!(done_result.is_ok(), total_result.is_ok()); + let saved = get_job_by_id(&pool, None, id) + .await + .expect("read") + .expect("job"); + let error = if done_result.is_ok() { + assert_eq!( + (saved.progress_done, saved.progress_total), + (Some(3), Some(3)) + ); + assert_eq!(saved.checkpoint, Some(done_checkpoint)); + total_result.expect_err("lower total loses") + } else { + assert_eq!( + (saved.progress_done, saved.progress_total), + (Some(1), Some(2)) + ); + assert_eq!(saved.checkpoint, Some(total_checkpoint)); + done_result.expect_err("higher done loses") + }; + let Error::QueryError(error) = error else { + panic!("expected domain query error") + }; + assert_eq!(error.category(), QueryErrorCategory::Validation); + assert_eq!(error.code(), "job.invalid_progress"); + assert_eq!( + error.progress_validation_error(), + Some(JobProgressValidationError::DoneExceedsTotal { done: 3, total: 2 }) + ); + assert_eq!( + error.sqlstate(), + None, + "validation happens before the CHECK constraint" + ); + let events = list_job_events(&pool, None, id, 100, None) + .await + .expect("events"); + assert_eq!( + events + .iter() + .filter(|event| event.event_type == runledger_core::jobs::JobEventType::Progress) + .count(), + 2 + ); + // A rejected write must release its transaction and leave the lease usable. + update_job_ordinary_progress_for_lease( + &pool, + identity, + &JobOrdinaryProgressUpdate { + progress_done: Some(3), + progress_total: Some(4), + checkpoint: None, + }, + ) + .await + .expect("later valid update"); + teardown_ephemeral_pool(pool, database).await; +} + +#[tokio::test] +async fn progress_cannot_write_after_its_lease_expires_while_waiting_for_the_row() { + let (pool, database) = setup_ephemeral_pool("progress_expiry_during_lock", 3).await; + support::register_test_job_definition(&pool, "jobs.test.progress_expiry").await; + let id = support::enqueue_test_job(&pool, "jobs.test.progress_expiry", None, &json!({})).await; + let job = support::claim_one_job(&pool, "expiring-progress-worker").await; + sqlx::query("UPDATE job_queue SET lease_expires_at = clock_timestamp() + interval '1 second' WHERE id = $1") + .bind(id).execute(&pool).await.expect("short live lease"); + let mut holder = pool.begin().await.expect("holder transaction"); + sqlx::query("SELECT id FROM job_queue WHERE id = $1 FOR UPDATE") + .bind(id) + .fetch_one(&mut *holder) + .await + .expect("hold job row"); + let writer_pool = pool.clone(); + let writer = tokio::spawn(async move { + update_job_ordinary_progress_for_lease( + &writer_pool, + JobLeaseIdentity::new(id, job.run_number, job.attempt, "expiring-progress-worker"), + &JobOrdinaryProgressUpdate { + progress_done: Some(1), + progress_total: Some(2), + checkpoint: Some(&json!("too late")), + }, + ) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(3), async { + loop { + let blocked: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity + WHERE datname = current_database() AND wait_event_type = 'Lock' + AND query LIKE '%FROM job_queue%')", + ) + .fetch_one(&pool) + .await + .expect("observe lock waiter"); + if blocked { + break; + } + assert!(!writer.is_finished(), "writer must wait on the live row"); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + loop { + let expired: bool = sqlx::query_scalar( + "SELECT lease_expires_at <= clock_timestamp() FROM job_queue WHERE id = $1", + ) + .bind(id) + .fetch_one(&pool) + .await + .expect("database clock expiry"); + if expired { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .expect("writer blocks before database lease expiry"); + holder.rollback().await.expect("release expired row"); + let error = writer + .await + .expect("writer joins") + .expect_err("expired lease cannot write"); + assert!( + matches!(error, Error::QueryError(ref query) if query.code() == "job.lease_owner_mismatch") + ); + let saved = get_job_by_id(&pool, None, id) + .await + .expect("read") + .expect("job"); + assert_eq!(saved.checkpoint, None); + assert_eq!(saved.progress_done, None); + assert!( + !list_job_events(&pool, None, id, 100, None) + .await + .expect("events") + .iter() + .any(|event| event.event_type == runledger_core::jobs::JobEventType::Progress) + ); + teardown_ephemeral_pool(pool, database).await; +} diff --git a/runledger-postgres/tests/shared_specs.rs b/runledger-postgres/tests/shared_specs.rs new file mode 100644 index 0000000..3e79794 --- /dev/null +++ b/runledger-postgres/tests/shared_specs.rs @@ -0,0 +1,158 @@ +//! API-only acceptance: this target imports no runtime or provider clients. +use runledger_core::jobs::{JobContract, JobDefinitionSettings, JobSpec, JobSpecs, JobType}; +use runledger_postgres::jobs::{ + JobDefinitionCatalogSyncMode, JobDefinitionUpsert, JobEnqueue, JobEnqueueDisposition, + sync_catalog_job_definitions_exact_tx, sync_catalog_job_definitions_tx, +}; +use runledger_postgres::prelude::enqueue_job_with_outcome; +use runledger_test_support::{setup_ephemeral_pool, teardown_ephemeral_pool}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +#[derive(Serialize, Deserialize)] +struct Payload { + request_id: String, +} +struct Send; +impl JobContract for Send { + type Payload = Payload; + fn spec() -> JobSpec { + JobSpec::new(JobType::new("producer.send")) + .expect("static spec") + .with_settings( + JobDefinitionSettings::new() + .max_attempts(5) + .timeout_seconds(60), + ) + .expect("settings") + } +} + +async fn sync(pool: &runledger_postgres::DbPool, spec: JobSpec) { + let specs = JobSpecs::new([spec]).expect("producer specs"); + let definitions: Vec<_> = specs.iter().map(JobDefinitionUpsert::from).collect(); + let mut tx = pool.begin().await.expect("begin"); + sync_catalog_job_definitions_tx( + &mut tx, + &definitions, + JobDefinitionCatalogSyncMode::PreserveExistingEnabledForEnabledDefinitions, + ) + .await + .expect("sync"); + tx.commit().await.expect("commit"); +} + +#[tokio::test] +async fn api_only_submission_preserves_snapshot_outcomes_and_operator_disables() { + let (pool, database) = setup_ephemeral_pool("shared_specs", 3).await; + let version: String = sqlx::query_scalar("SHOW server_version") + .fetch_one(&pool) + .await + .expect("version"); + eprintln!("shared specs server_version={version}"); + assert!( + version.starts_with("18."), + "repository baseline is PostgreSQL 18: {version}" + ); + sync(&pool, Send::spec()).await; + let request = Send::submit(&Payload { + request_id: "legacy-id".into(), + }) + .expect("typed submit") + .idempotency_key("same-request"); + let inserted = enqueue_job_with_outcome(&pool, &JobEnqueue::from(&request)) + .await + .expect("insert"); + assert_eq!(inserted.disposition, JobEnqueueDisposition::Inserted); + let stored: (Value, i32, i32) = sqlx::query_as( + "SELECT enqueue_request, max_attempts, timeout_seconds FROM job_queue WHERE id = $1", + ) + .bind(inserted.job_id) + .fetch_one(&pool) + .await + .expect("stored row"); + assert_eq!( + stored, + ( + json!({"payload":{"request_id":"legacy-id"},"priority":null,"max_attempts":null,"timeout_seconds":null,"next_run_at":null,"stage":"queued"}), + 5, + 60 + ) + ); + let updated = Send::spec() + .with_settings( + JobDefinitionSettings::new() + .version(9) + .max_attempts(8) + .timeout_seconds(90), + ) + .expect("updated definition"); + sync(&pool, updated).await; + let existing = enqueue_job_with_outcome(&pool, &JobEnqueue::from(&request)) + .await + .expect("retry across settings change"); + assert_eq!(existing.disposition, JobEnqueueDisposition::Existing); + assert_eq!(existing.job_id, inserted.job_id); + let mut changed = request.clone(); + changed.payload = json!({"request_id":"different"}); + let error = enqueue_job_with_outcome(&pool, &JobEnqueue::from(&changed)) + .await + .expect_err("strict conflict"); + assert!( + matches!(error, runledger_postgres::Error::QueryError(ref error) if error.code() == "job.idempotency_conflict"), + "{error:?}" + ); + let overridden = request.clone().max_attempts(5); + assert!( + enqueue_job_with_outcome(&pool, &JobEnqueue::from(&overridden)) + .await + .is_err() + ); + sqlx::query("UPDATE job_definitions SET is_enabled = false WHERE job_type = 'producer.send'") + .execute(&pool) + .await + .expect("operator disable"); + sync(&pool, updated).await; + let enabled: bool = sqlx::query_scalar( + "SELECT is_enabled FROM job_definitions WHERE job_type = 'producer.send'", + ) + .fetch_one(&pool) + .await + .expect("enabled state"); + assert!(!enabled, "additive sync must preserve operator disable"); + let fresh = request.clone().idempotency_key("fresh"); + assert!( + enqueue_job_with_outcome(&pool, &JobEnqueue::from(&fresh)) + .await + .is_err() + ); + // Explicit exact mode restores code-owned enabled state, just as the legacy catalog does. + let mut tx = pool.begin().await.expect("begin exact"); + sync_catalog_job_definitions_exact_tx( + &mut tx, + &[JobDefinitionUpsert::from(&updated)], + &[runledger_core::jobs::JobTypeName::new(updated.job_type().as_str()).expect("scope")], + ) + .await + .expect("exact sync"); + tx.commit().await.expect("commit exact"); + assert_eq!( + enqueue_job_with_outcome(&pool, &JobEnqueue::from(&fresh)) + .await + .expect("restored") + .disposition, + JobEnqueueDisposition::Inserted + ); + let events: i64 = sqlx::query_scalar( + "SELECT count(*) FROM job_events WHERE job_id = $1 AND event_type = 'ENQUEUED'", + ) + .bind(inserted.job_id) + .fetch_one(&pool) + .await + .expect("events"); + assert_eq!( + events, 1, + "retries and rejected conflicts must not add enqueue events" + ); + teardown_ephemeral_pool(pool, database).await; +} diff --git a/runledger-postgres/tests/workflow_batch_inserts.rs b/runledger-postgres/tests/workflow_batch_inserts.rs new file mode 100644 index 0000000..1033085 --- /dev/null +++ b/runledger-postgres/tests/workflow_batch_inserts.rs @@ -0,0 +1,246 @@ +use runledger_core::jobs::{ + JobStage, JobType, StepKey, WorkflowRunEnqueueBuilder, WorkflowStepEnqueue, + WorkflowStepEnqueueBuilder, WorkflowType, +}; +use runledger_postgres::{DbPool, jobs::*}; +use runledger_test_support::{setup_ephemeral_pool, teardown_ephemeral_pool}; +use serde_json::{Value, json}; +use sqlx::types::Uuid; + +mod support; + +#[tokio::test] +async fn batch_preserves_json_null_payloads_for_job_and_external_steps() { + let (pool, database) = setup_ephemeral_pool("workflow_batch_null", 1).await; + support::register_test_job_definition(&pool, "batch.job").await; + let payload = Value::Null; + let job = + WorkflowStepEnqueueBuilder::new(StepKey::new("job"), JobType::new("batch.job"), &payload) + .try_build() + .expect("null job payload"); + let gate = WorkflowStepEnqueueBuilder::new_external(StepKey::new("gate"), &payload) + .try_build() + .expect("null gate payload"); + let request = WorkflowRunEnqueueBuilder::new(WorkflowType::new("batch.null"), &payload) + .step(job) + .step(gate) + .try_build() + .expect("run"); + enqueue_workflow_run(&pool, &request) + .await + .expect("enqueue null payloads"); + let payloads: Vec = + sqlx::query_scalar("SELECT payload FROM workflow_steps ORDER BY step_key") + .fetch_all(&pool) + .await + .expect("persisted JSON nulls"); + assert_eq!(payloads, vec![Value::Null, Value::Null]); + let queued_payload: Value = sqlx::query_scalar("SELECT payload FROM job_queue") + .fetch_one(&pool) + .await + .expect("queued JSON null"); + assert_eq!(queued_payload, Value::Null); + teardown_ephemeral_pool(pool, database).await; +} + +fn graph_steps<'a>(keys: &'a [String], payload: &'a Value) -> Vec> { + keys.iter() + .enumerate() + .map(|(i, key)| { + let dependencies = keys[i.saturating_sub(2)..i] + .iter() + .map(|k| StepKey::new(k)) + .collect::>(); + WorkflowStepEnqueueBuilder::new(StepKey::new(key), JobType::new("batch.job"), payload) + .depends_on_success(&dependencies) + .priority(-7) + .max_attempts(5) + .timeout_seconds(43) + .stage(JobStage::Scheduled) + .allow_handler_continuation() + .execution_resource("batch-resource") + .try_build() + .expect("step") + }) + .collect() +} + +async fn counts(pool: &DbPool) -> (i64, i64, i64, i64, i64) { + sqlx::query_as( + "SELECT (SELECT count(*) FROM workflow_runs), (SELECT count(*) FROM workflow_steps), + (SELECT count(*) FROM workflow_step_dependencies), (SELECT count(*) FROM job_queue), + (SELECT count(*) FROM job_events)", + ) + .fetch_one(pool) + .await + .expect("durable counts") +} + +#[tokio::test] +async fn multi_chunk_graph_preserves_fields_edges_audit_and_snapshot_idempotency() { + let (pool, database) = setup_ephemeral_pool("workflow_batch", 1).await; + support::register_test_job_definition(&pool, "batch.job").await; + let keys = (0..270).map(|i| format!("s{i:03}")).collect::>(); + let payload = json!({"nested": [null, {"unicode": "žluťoučký", "text": "x".repeat(4000)}]}); + let tenant = Uuid::now_v7(); + let mut builder = WorkflowRunEnqueueBuilder::new(WorkflowType::new("batch.workflow"), &payload) + .organization_id(tenant) + .idempotency_key("batch-key"); + for step in graph_steps(&keys, &payload) { + builder = builder.step(step); + } + let request = builder.try_build().expect("run"); + let run = enqueue_workflow_run(&pool, &request) + .await + .expect("enqueue"); + assert_eq!(counts(&pool).await, (1, 270, 537, 1, 1)); + let wrong_fields: i64 = sqlx::query_scalar("SELECT count(*) FROM workflow_steps WHERE + organization_id IS DISTINCT FROM $1 OR payload <> $2 OR priority <> -7 OR max_attempts <> 5 + OR timeout_seconds <> 43 OR stage <> 'scheduled' OR NOT allow_handler_continuation + OR execution_resource_key <> 'batch-resource' OR dependency_count_unsatisfied <> 0 + OR dependency_count_total <> CASE WHEN step_key='s000' THEN 0 WHEN step_key='s001' THEN 1 ELSE 2 END + OR dependency_count_pending <> dependency_count_total") + .bind(tenant).bind(&payload).fetch_one(&pool).await.expect("persisted fields"); + assert_eq!(wrong_fields, 0); + let edges: Vec<(String, String, String)> = sqlx::query_as( + "SELECT p.step_key, d.step_key, e.release_mode::text + FROM workflow_step_dependencies e JOIN workflow_steps p ON p.id=e.prerequisite_step_id + JOIN workflow_steps d ON d.id=e.dependent_step_id ORDER BY d.step_key, p.step_key", + ) + .fetch_all(&pool) + .await + .expect("edges"); + let expected = (0..270_usize) + .flat_map(|i| { + (i.saturating_sub(2)..i).map(move |j| { + ( + format!("s{j:03}"), + format!("s{i:03}"), + "ON_SUCCESS".to_owned(), + ) + }) + }) + .collect::>(); + assert_eq!(edges, expected); + assert_eq!( + enqueue_workflow_run(&pool, &request) + .await + .expect("identical retry") + .id, + run.id + ); + assert_eq!(counts(&pool).await, (1, 270, 537, 1, 1)); + // The single initial root still has one ordinary ENQUEUED audit event. + let event: String = sqlx::query_scalar("SELECT event_type::text FROM job_events") + .fetch_one(&pool) + .await + .expect("event"); + assert_eq!(event, "ENQUEUED"); + teardown_ephemeral_pool(pool, database).await; +} + +#[tokio::test] +async fn failure_in_later_step_or_edge_chunk_rolls_back_the_owned_transaction() { + let (pool, database) = setup_ephemeral_pool("workflow_batch_rollback", 1).await; + support::register_test_job_definition(&pool, "batch.job").await; + let keys = (0..270).map(|i| format!("s{i:03}")).collect::>(); + let payload = json!({}); + let mut builder = WorkflowRunEnqueueBuilder::new(WorkflowType::new("batch.rollback"), &payload); + for step in graph_steps(&keys, &payload) { + builder = builder.step(step); + } + let request = builder.try_build().expect("run"); + for (table, predicate) in [ + ("workflow_steps", "NEW.step_key = 's269'"), + ( + "workflow_step_dependencies", + "EXISTS (SELECT 1 FROM workflow_steps WHERE id=NEW.dependent_step_id AND step_key='s269')", + ), + ] { + // Fail after earlier chunks have succeeded, exercising the actual transaction boundary. + sqlx::raw_sql(&format!("CREATE FUNCTION reject_late_row() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF {predicate} THEN RAISE EXCEPTION 'injected late row failure'; END IF; RETURN NEW; END $$; + CREATE TRIGGER reject_late BEFORE INSERT ON {table} FOR EACH ROW EXECUTE FUNCTION reject_late_row();")) + .execute(&pool).await.expect("install fault"); + enqueue_workflow_run(&pool, &request) + .await + .expect_err("late insertion fails"); + assert_eq!(counts(&pool).await, (0, 0, 0, 0, 0)); + sqlx::raw_sql(&format!( + "DROP TRIGGER reject_late ON {table}; DROP FUNCTION reject_late_row();" + )) + .execute(&pool) + .await + .expect("remove fault"); + } + teardown_ephemeral_pool(pool, database).await; +} + +#[tokio::test] +async fn multi_chunk_append_preserves_input_order_and_idempotent_outcome() { + let (pool, database) = setup_ephemeral_pool("workflow_batch_append", 1).await; + let payload = json!({}); + let gate = WorkflowStepEnqueueBuilder::new_external(StepKey::new("gate"), &payload) + .try_build() + .expect("gate"); + let run = enqueue_workflow_run( + &pool, + &WorkflowRunEnqueueBuilder::new(WorkflowType::new("batch.append"), &payload) + .step(gate) + .try_build() + .expect("run"), + ) + .await + .expect("enqueue"); + let keys = (0..270) + .rev() + .map(|i| format!("s{i:03}")) + .collect::>(); + let steps = keys + .iter() + .map(|key| { + WorkflowStepEnqueueBuilder::new_external(StepKey::new(key), &payload) + .depends_on_terminal(&[StepKey::new("gate")]) + .try_build() + .expect("external step") + }) + .collect(); + let request = AppendWorkflowStepsInput { + workflow_run_id: run.id, + organization_id: None, + mutation_key: "append", + mutation_metadata: &payload, + append_window_step_key: StepKey::new("gate"), + steps, + }; + let first = append_workflow_steps(&pool, &request) + .await + .expect("append"); + assert_eq!(first.outcome, AppendWorkflowStepsOutcome::Appended); + assert_eq!( + first + .appended_steps + .iter() + .map(|s| s.step_key.as_str()) + .collect::>(), + keys + ); + let retry = append_workflow_steps(&pool, &request) + .await + .expect("retry append"); + assert_eq!(retry.outcome, AppendWorkflowStepsOutcome::AlreadyApplied); + assert_eq!( + first + .appended_steps + .iter() + .map(|s| s.id) + .collect::>(), + retry + .appended_steps + .iter() + .map(|s| s.id) + .collect::>() + ); + assert_eq!(counts(&pool).await, (1, 271, 270, 0, 0)); + teardown_ephemeral_pool(pool, database).await; +} diff --git a/runledger-runtime/.sqlx/query-07c64366e2b4e12cd65cad0e26ff8e2442eafaa21bc515a90063c7750ed0a070.json b/runledger-runtime/.sqlx/query-07c64366e2b4e12cd65cad0e26ff8e2442eafaa21bc515a90063c7750ed0a070.json new file mode 100644 index 0000000..7da9f8b --- /dev/null +++ b/runledger-runtime/.sqlx/query-07c64366e2b4e12cd65cad0e26ff8e2442eafaa21bc515a90063c7750ed0a070.json @@ -0,0 +1,105 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, job_type, organization_id, status::text AS \"status!\",\n priority, run_number, attempt, max_attempts, next_run_at, stage,\n progress_done, progress_total, created_at, updated_at\n FROM job_queue WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type = $3) AND (created_at, id) < ($5, $6) ORDER BY created_at DESC, id DESC LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Timestamptz", + "Uuid" + ] + }, + "nullable": [ + false, + false, + true, + null, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + }, + "hash": "07c64366e2b4e12cd65cad0e26ff8e2442eafaa21bc515a90063c7750ed0a070" +} diff --git a/runledger-runtime/.sqlx/query-10269f3f8cd79707daaba3c26771d5aa97e7c9c08801cbfd18f70bbd8670e17a.json b/runledger-runtime/.sqlx/query-10269f3f8cd79707daaba3c26771d5aa97e7c9c08801cbfd18f70bbd8670e17a.json new file mode 100644 index 0000000..704da5f --- /dev/null +++ b/runledger-runtime/.sqlx/query-10269f3f8cd79707daaba3c26771d5aa97e7c9c08801cbfd18f70bbd8670e17a.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, payload FROM job_queue WHERE organization_id = $1 AND job_type = $2\n AND payload->>'run_id' = $3\n ORDER BY created_at DESC, id DESC\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "payload", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "10269f3f8cd79707daaba3c26771d5aa97e7c9c08801cbfd18f70bbd8670e17a" +} diff --git a/runledger-postgres/.sqlx/query-dbcca04872cf174ca1a23862f2f632fcf5da4599e75c8f21b1c134b34b1a876a.json b/runledger-runtime/.sqlx/query-1059ac0fa26c3192fe4160b03fe6f8fadf3727d67e102953a0793af534de602a.json similarity index 83% rename from runledger-postgres/.sqlx/query-dbcca04872cf174ca1a23862f2f632fcf5da4599e75c8f21b1c134b34b1a876a.json rename to runledger-runtime/.sqlx/query-1059ac0fa26c3192fe4160b03fe6f8fadf3727d67e102953a0793af534de602a.json index b05c115..db05c86 100644 --- a/runledger-postgres/.sqlx/query-dbcca04872cf174ca1a23862f2f632fcf5da4599e75c8f21b1c134b34b1a876a.json +++ b/runledger-runtime/.sqlx/query-1059ac0fa26c3192fe4160b03fe6f8fadf3727d67e102953a0793af534de602a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n je.id,\n je.job_id,\n je.run_number,\n je.attempt,\n je.event_type::text AS \"event_type!\",\n je.stage,\n je.progress_done,\n je.progress_total,\n je.payload,\n je.occurred_at\n FROM job_events je\n JOIN job_queue jq ON jq.id = je.job_id\n WHERE je.job_id = $1\n AND ($2::uuid IS NULL OR jq.organization_id = $2)\n AND ($3::bigint IS NULL OR je.id > $3)\n ORDER BY je.id ASC\n LIMIT $4", + "query": "SELECT\n je.id,\n je.job_id,\n je.run_number,\n je.attempt,\n je.event_type::text AS \"event_type!\",\n je.stage,\n je.progress_done,\n je.progress_total,\n je.payload,\n je.occurred_at\n FROM job_events je\n JOIN job_queue jq ON jq.id = je.job_id\n WHERE je.job_id = $1\n AND ($5::bool OR jq.organization_id IS NOT DISTINCT FROM $2::uuid)\n AND ($3::bigint IS NULL OR je.id > $3)\n ORDER BY je.id ASC\n LIMIT $4", "describe": { "columns": [ { @@ -59,7 +59,8 @@ "Uuid", "Uuid", "Int8", - "Int8" + "Int8", + "Bool" ] }, "nullable": [ @@ -75,5 +76,5 @@ false ] }, - "hash": "dbcca04872cf174ca1a23862f2f632fcf5da4599e75c8f21b1c134b34b1a876a" + "hash": "1059ac0fa26c3192fe4160b03fe6f8fadf3727d67e102953a0793af534de602a" } diff --git a/runledger-runtime/.sqlx/query-290943331ceeb17e0d9e726a3bb5a2afb203b501dccdf13baa3b2390800515f7.json b/runledger-runtime/.sqlx/query-290943331ceeb17e0d9e726a3bb5a2afb203b501dccdf13baa3b2390800515f7.json new file mode 100644 index 0000000..399cc9a --- /dev/null +++ b/runledger-runtime/.sqlx/query-290943331ceeb17e0d9e726a3bb5a2afb203b501dccdf13baa3b2390800515f7.json @@ -0,0 +1,158 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n next_run_at,\n idempotency_key,\n stage,\n enqueue_request_version,\n execution_resource_key,\n promotion_attempts,\n next_promotion_at,\n last_attempted_at,\n status,\n promoted_job_id,\n promoted_at,\n conflicted_at,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_enqueue_intents\n WHERE $1::uuid IS NULL AND ($2::text IS NULL OR status = $2)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "payload", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "timeout_seconds", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 8, + "name": "idempotency_key", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "enqueue_request_version", + "type_info": "Int2" + }, + { + "ordinal": 11, + "name": "execution_resource_key", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "promotion_attempts", + "type_info": "Int4" + }, + { + "ordinal": 13, + "name": "next_promotion_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 14, + "name": "last_attempted_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 15, + "name": "status", + "type_info": "Text" + }, + { + "ordinal": 16, + "name": "promoted_job_id", + "type_info": "Uuid" + }, + { + "ordinal": 17, + "name": "promoted_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 18, + "name": "conflicted_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 19, + "name": "last_error_code", + "type_info": "Text" + }, + { + "ordinal": 20, + "name": "last_error_message", + "type_info": "Text" + }, + { + "ordinal": 21, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 22, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false, + true, + true, + true, + true, + false, + false, + false, + true, + false, + false, + true, + false, + true, + true, + true, + true, + true, + false, + false + ] + }, + "hash": "290943331ceeb17e0d9e726a3bb5a2afb203b501dccdf13baa3b2390800515f7" +} diff --git a/.sqlx/query-c2350f3a1f05bb5bf576e20872873d7135e8dfff31412ed42bda795dbeee7586.json b/runledger-runtime/.sqlx/query-2b0c48cf0a53ca60dfb663b242b1b65b0833bf0b32fe4cacb637e18df8ae77bb.json similarity index 92% rename from .sqlx/query-c2350f3a1f05bb5bf576e20872873d7135e8dfff31412ed42bda795dbeee7586.json rename to runledger-runtime/.sqlx/query-2b0c48cf0a53ca60dfb663b242b1b65b0833bf0b32fe4cacb637e18df8ae77bb.json index 9150683..b9f7f28 100644 --- a/.sqlx/query-c2350f3a1f05bb5bf576e20872873d7135e8dfff31412ed42bda795dbeee7586.json +++ b/runledger-runtime/.sqlx/query-2b0c48cf0a53ca60dfb663b242b1b65b0833bf0b32fe4cacb637e18df8ae77bb.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n status::text AS \"status!\",\n priority,\n run_number,\n attempt,\n max_attempts,\n timeout_seconds,\n next_run_at,\n lease_expires_at,\n last_heartbeat_at,\n worker_id,\n started_at,\n finished_at,\n stage,\n progress_done,\n progress_total,\n progress_pct::float8 AS progress_pct,\n checkpoint,\n output,\n idempotency_key,\n status_reason,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_queue\n WHERE ($1::uuid IS NULL OR organization_id = $1)\n AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n status::text AS \"status!\",\n priority,\n run_number,\n attempt,\n max_attempts,\n timeout_seconds,\n next_run_at,\n lease_expires_at,\n last_heartbeat_at,\n worker_id,\n started_at,\n finished_at,\n stage,\n progress_done,\n progress_total,\n progress_pct::float8 AS progress_pct,\n checkpoint,\n output,\n idempotency_key,\n status_reason,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_queue\n WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", "describe": { "columns": [ { @@ -184,5 +184,5 @@ false ] }, - "hash": "c2350f3a1f05bb5bf576e20872873d7135e8dfff31412ed42bda795dbeee7586" + "hash": "2b0c48cf0a53ca60dfb663b242b1b65b0833bf0b32fe4cacb637e18df8ae77bb" } diff --git a/runledger-runtime/.sqlx/query-2ca138d0f77ca8b5d61381edcdb32a7f98c016c7214a13f69be251c78058b0e7.json b/runledger-runtime/.sqlx/query-2ca138d0f77ca8b5d61381edcdb32a7f98c016c7214a13f69be251c78058b0e7.json new file mode 100644 index 0000000..d18086b --- /dev/null +++ b/runledger-runtime/.sqlx/query-2ca138d0f77ca8b5d61381edcdb32a7f98c016c7214a13f69be251c78058b0e7.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, status::text AS \"status!\", run_number, attempt, updated_at\n FROM job_queue WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND id = ANY($2::uuid[]) ORDER BY id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "UuidArray" + ] + }, + "nullable": [ + false, + null, + false, + false, + false + ] + }, + "hash": "2ca138d0f77ca8b5d61381edcdb32a7f98c016c7214a13f69be251c78058b0e7" +} diff --git a/runledger-runtime/.sqlx/query-2f5d35c92e35e69d3d0260eeb2bac01332025306d05cb43392686d9cc73d5eba.json b/runledger-runtime/.sqlx/query-2f5d35c92e35e69d3d0260eeb2bac01332025306d05cb43392686d9cc73d5eba.json new file mode 100644 index 0000000..6abf7e3 --- /dev/null +++ b/runledger-runtime/.sqlx/query-2f5d35c92e35e69d3d0260eeb2bac01332025306d05cb43392686d9cc73d5eba.json @@ -0,0 +1,158 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n next_run_at,\n idempotency_key,\n stage,\n enqueue_request_version,\n execution_resource_key,\n promotion_attempts,\n next_promotion_at,\n last_attempted_at,\n status,\n promoted_job_id,\n promoted_at,\n conflicted_at,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_enqueue_intents\n WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND ($2::text IS NULL OR status = $2)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "payload", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "timeout_seconds", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 8, + "name": "idempotency_key", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "enqueue_request_version", + "type_info": "Int2" + }, + { + "ordinal": 11, + "name": "execution_resource_key", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "promotion_attempts", + "type_info": "Int4" + }, + { + "ordinal": 13, + "name": "next_promotion_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 14, + "name": "last_attempted_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 15, + "name": "status", + "type_info": "Text" + }, + { + "ordinal": 16, + "name": "promoted_job_id", + "type_info": "Uuid" + }, + { + "ordinal": 17, + "name": "promoted_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 18, + "name": "conflicted_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 19, + "name": "last_error_code", + "type_info": "Text" + }, + { + "ordinal": 20, + "name": "last_error_message", + "type_info": "Text" + }, + { + "ordinal": 21, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 22, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false, + true, + true, + true, + true, + false, + false, + false, + true, + false, + false, + true, + false, + true, + true, + true, + true, + true, + false, + false + ] + }, + "hash": "2f5d35c92e35e69d3d0260eeb2bac01332025306d05cb43392686d9cc73d5eba" +} diff --git a/runledger-postgres/.sqlx/query-9f9c494b36d03026a351373e70483c382d2e53386fac1d737a5f2e20818007d8.json b/runledger-runtime/.sqlx/query-321da426e59fb877a1e48430f03013433f919b3f4edc06e322f195e682e7be0f.json similarity index 88% rename from runledger-postgres/.sqlx/query-9f9c494b36d03026a351373e70483c382d2e53386fac1d737a5f2e20818007d8.json rename to runledger-runtime/.sqlx/query-321da426e59fb877a1e48430f03013433f919b3f4edc06e322f195e682e7be0f.json index 9b99123..fb67a3d 100644 --- a/runledger-postgres/.sqlx/query-9f9c494b36d03026a351373e70483c382d2e53386fac1d737a5f2e20818007d8.json +++ b/runledger-runtime/.sqlx/query-321da426e59fb877a1e48430f03013433f919b3f4edc06e322f195e682e7be0f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n jd.job_type AS \"job_type!\",\n COALESCE(SUM(jmr.pending_count), 0)::bigint AS \"pending_count!\",\n COALESCE(SUM(jmr.leased_count), 0)::bigint AS \"leased_count!\",\n COALESCE(SUM(jmr.stale_leases), 0)::bigint AS \"stale_leases!\",\n COALESCE(SUM(jmr.succeeded_24h), 0)::bigint AS \"succeeded_24h!\",\n COALESCE(SUM(jmr.retryable_24h), 0)::bigint AS \"retryable_24h!\",\n COALESCE(SUM(jmr.terminal_24h), 0)::bigint AS \"terminal_24h!\",\n COALESCE(SUM(jmr.panicked_24h), 0)::bigint AS \"panicked_24h!\",\n COALESCE(SUM(jmr.timeout_24h), 0)::bigint AS \"timeout_24h!\",\n COALESCE(SUM(jmr.dead_lettered_24h), 0)::bigint AS \"dead_lettered_24h!\",\n AVG(jmr.p50_duration_ms_24h) AS p50_duration_ms_24h,\n AVG(jmr.p95_duration_ms_24h) AS p95_duration_ms_24h\n FROM job_definitions jd\n LEFT JOIN job_metrics_rollup jmr\n ON jmr.job_type = jd.job_type\n AND ($1::uuid IS NULL OR jmr.organization_id = $1)\n WHERE ($2::text IS NULL OR jd.job_type = $2)\n GROUP BY jd.job_type\n ORDER BY jd.job_type ASC", + "query": "SELECT\n jd.job_type AS \"job_type!\",\n COALESCE(SUM(jmr.pending_count), 0)::bigint AS \"pending_count!\",\n COALESCE(SUM(jmr.leased_count), 0)::bigint AS \"leased_count!\",\n COALESCE(SUM(jmr.stale_leases), 0)::bigint AS \"stale_leases!\",\n COALESCE(SUM(jmr.succeeded_24h), 0)::bigint AS \"succeeded_24h!\",\n COALESCE(SUM(jmr.retryable_24h), 0)::bigint AS \"retryable_24h!\",\n COALESCE(SUM(jmr.terminal_24h), 0)::bigint AS \"terminal_24h!\",\n COALESCE(SUM(jmr.panicked_24h), 0)::bigint AS \"panicked_24h!\",\n COALESCE(SUM(jmr.timeout_24h), 0)::bigint AS \"timeout_24h!\",\n COALESCE(SUM(jmr.dead_lettered_24h), 0)::bigint AS \"dead_lettered_24h!\",\n AVG(jmr.p50_duration_ms_24h) AS p50_duration_ms_24h,\n AVG(jmr.p95_duration_ms_24h) AS p95_duration_ms_24h\n FROM job_definitions jd\n LEFT JOIN job_metrics_rollup jmr\n ON jmr.job_type = jd.job_type\n AND ($3::boolean OR (jmr.organization_id = $1 OR ($1::uuid IS NULL AND jmr.organization_id IS NULL)))\n WHERE ($2::text IS NULL OR jd.job_type = $2)\n GROUP BY jd.job_type\n ORDER BY jd.job_type ASC", "describe": { "columns": [ { @@ -67,7 +67,8 @@ "parameters": { "Left": [ "Uuid", - "Text" + "Text", + "Bool" ] }, "nullable": [ @@ -85,5 +86,5 @@ null ] }, - "hash": "9f9c494b36d03026a351373e70483c382d2e53386fac1d737a5f2e20818007d8" + "hash": "321da426e59fb877a1e48430f03013433f919b3f4edc06e322f195e682e7be0f" } diff --git a/runledger-runtime/.sqlx/query-48789cd60eb5ace54e41f24bafc110420c7e67fe6175f5bcb786579a66f9af8e.json b/runledger-runtime/.sqlx/query-48789cd60eb5ace54e41f24bafc110420c7e67fe6175f5bcb786579a66f9af8e.json new file mode 100644 index 0000000..cc80fa9 --- /dev/null +++ b/runledger-runtime/.sqlx/query-48789cd60eb5ace54e41f24bafc110420c7e67fe6175f5bcb786579a66f9af8e.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, payload FROM job_queue WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND job_type = $2\n AND payload->>'run_id' = $3\n ORDER BY created_at DESC, id DESC\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "payload", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "48789cd60eb5ace54e41f24bafc110420c7e67fe6175f5bcb786579a66f9af8e" +} diff --git a/runledger-runtime/.sqlx/query-809f796d49a1830f8338380883e2ee58cce341f4e12b85bb982b01bf09d6cc50.json b/runledger-runtime/.sqlx/query-51b23cabe44bba220eb95d76cfcc135768be65ed533785bea2d9d668e9305ebd.json similarity index 82% rename from runledger-runtime/.sqlx/query-809f796d49a1830f8338380883e2ee58cce341f4e12b85bb982b01bf09d6cc50.json rename to runledger-runtime/.sqlx/query-51b23cabe44bba220eb95d76cfcc135768be65ed533785bea2d9d668e9305ebd.json index 2ccb7de..9ced603 100644 --- a/runledger-runtime/.sqlx/query-809f796d49a1830f8338380883e2ee58cce341f4e12b85bb982b01bf09d6cc50.json +++ b/runledger-runtime/.sqlx/query-51b23cabe44bba220eb95d76cfcc135768be65ed533785bea2d9d668e9305ebd.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n jl.id,\n jl.job_id,\n jl.run_number,\n jl.attempt,\n jl.level,\n jl.message,\n jl.payload,\n jl.occurred_at\n FROM job_logs jl\n JOIN job_queue jq ON jq.id = jl.job_id\n WHERE jl.job_id = $1\n AND ($2::uuid IS NULL OR jq.organization_id = $2)\n AND ($3::bigint IS NULL OR jl.id > $3)\n ORDER BY jl.id ASC\n LIMIT $4", + "query": "SELECT\n jl.id,\n jl.job_id,\n jl.run_number,\n jl.attempt,\n jl.level,\n jl.message,\n jl.payload,\n jl.occurred_at\n FROM job_logs jl\n JOIN job_queue jq ON jq.id = jl.job_id\n WHERE jl.job_id = $1\n AND ($5::bool OR jq.organization_id IS NOT DISTINCT FROM $2::uuid)\n AND ($3::bigint IS NULL OR jl.id > $3)\n ORDER BY jl.id ASC\n LIMIT $4", "describe": { "columns": [ { @@ -49,7 +49,8 @@ "Uuid", "Uuid", "Int8", - "Int8" + "Int8", + "Bool" ] }, "nullable": [ @@ -63,5 +64,5 @@ false ] }, - "hash": "809f796d49a1830f8338380883e2ee58cce341f4e12b85bb982b01bf09d6cc50" + "hash": "51b23cabe44bba220eb95d76cfcc135768be65ed533785bea2d9d668e9305ebd" } diff --git a/runledger-runtime/.sqlx/query-6ab23505891fce132239b6cd02ab480be547302313aeb4be81f2c14bb20c0569.json b/runledger-runtime/.sqlx/query-6ab23505891fce132239b6cd02ab480be547302313aeb4be81f2c14bb20c0569.json new file mode 100644 index 0000000..0f09710 --- /dev/null +++ b/runledger-runtime/.sqlx/query-6ab23505891fce132239b6cd02ab480be547302313aeb4be81f2c14bb20c0569.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE job_queue\n SET stage = COALESCE($5, stage),\n progress_done = COALESCE($6, progress_done),\n progress_total = COALESCE($7, progress_total),\n checkpoint = COALESCE($8::jsonb, checkpoint),\n updated_at = now()\n WHERE id = $1\n AND run_number = $2\n AND attempt = $3\n AND worker_id = $4\n AND status = 'LEASED'\n AND lease_expires_at > clock_timestamp()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int4", + "Int4", + "Text", + "Text", + "Int8", + "Int8", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "6ab23505891fce132239b6cd02ab480be547302313aeb4be81f2c14bb20c0569" +} diff --git a/runledger-runtime/.sqlx/query-6c5d8cb4f663423a38334a1418faf9050dee8d8fd943820afd0bcb6e0bc32565.json b/runledger-runtime/.sqlx/query-6c5d8cb4f663423a38334a1418faf9050dee8d8fd943820afd0bcb6e0bc32565.json deleted file mode 100644 index d56ae1f..0000000 --- a/runledger-runtime/.sqlx/query-6c5d8cb4f663423a38334a1418faf9050dee8d8fd943820afd0bcb6e0bc32565.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workflow_steps (\n workflow_run_id,\n step_key,\n execution_kind,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n stage,\n allow_handler_continuation,\n execution_resource_key,\n status,\n dependency_count_total,\n dependency_count_pending,\n dependency_count_unsatisfied\n )\n VALUES (\n $1,\n $2,\n $3::text::workflow_step_execution_kind,\n $4,\n $5,\n $6::jsonb,\n $7,\n $8,\n $9,\n $10,\n $11,\n $12,\n 'BLOCKED',\n $13,\n $14,\n $15\n )\n RETURNING id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Text", - "Text", - "Uuid", - "Jsonb", - "Int4", - "Int4", - "Int4", - "Text", - "Bool", - "Text", - "Int4", - "Int4", - "Int4" - ] - }, - "nullable": [ - false - ] - }, - "hash": "6c5d8cb4f663423a38334a1418faf9050dee8d8fd943820afd0bcb6e0bc32565" -} diff --git a/runledger-runtime/.sqlx/query-76e0bb60c9fe984d8b52e294bc657bc1aece09a76f064220c406b146277929cc.json b/runledger-runtime/.sqlx/query-76e0bb60c9fe984d8b52e294bc657bc1aece09a76f064220c406b146277929cc.json new file mode 100644 index 0000000..49786b4 --- /dev/null +++ b/runledger-runtime/.sqlx/query-76e0bb60c9fe984d8b52e294bc657bc1aece09a76f064220c406b146277929cc.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, payload FROM job_queue WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND job_type = $2\n AND idempotency_key = $3\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "payload", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "76e0bb60c9fe984d8b52e294bc657bc1aece09a76f064220c406b146277929cc" +} diff --git a/runledger-postgres/.sqlx/query-118f48ae11853ca23401d3569a8c10677a04ac4114f2fbc5d832d866a46e778c.json b/runledger-runtime/.sqlx/query-7e287b60fb932a4795c61e25e247e4313adf268188c94931c8f9ebb995c8812e.json similarity index 94% rename from runledger-postgres/.sqlx/query-118f48ae11853ca23401d3569a8c10677a04ac4114f2fbc5d832d866a46e778c.json rename to runledger-runtime/.sqlx/query-7e287b60fb932a4795c61e25e247e4313adf268188c94931c8f9ebb995c8812e.json index b0eef8b..39f343e 100644 --- a/runledger-postgres/.sqlx/query-118f48ae11853ca23401d3569a8c10677a04ac4114f2fbc5d832d866a46e778c.json +++ b/runledger-runtime/.sqlx/query-7e287b60fb932a4795c61e25e247e4313adf268188c94931c8f9ebb995c8812e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n next_run_at,\n idempotency_key,\n stage,\n enqueue_request_version,\n execution_resource_key,\n promotion_attempts,\n next_promotion_at,\n last_attempted_at,\n status,\n promoted_job_id,\n promoted_at,\n conflicted_at,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_enqueue_intents\n WHERE id = $1\n AND ($2::uuid IS NULL OR organization_id = $2)\n LIMIT 1", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n next_run_at,\n idempotency_key,\n stage,\n enqueue_request_version,\n execution_resource_key,\n promotion_attempts,\n next_promotion_at,\n last_attempted_at,\n status,\n promoted_job_id,\n promoted_at,\n conflicted_at,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_enqueue_intents\n WHERE id = $1\n AND ($3::bool OR organization_id IS NOT DISTINCT FROM $2::uuid)\n LIMIT 1", "describe": { "columns": [ { @@ -122,7 +122,8 @@ "parameters": { "Left": [ "Uuid", - "Uuid" + "Uuid", + "Bool" ] }, "nullable": [ @@ -151,5 +152,5 @@ false ] }, - "hash": "118f48ae11853ca23401d3569a8c10677a04ac4114f2fbc5d832d866a46e778c" + "hash": "7e287b60fb932a4795c61e25e247e4313adf268188c94931c8f9ebb995c8812e" } diff --git a/runledger-runtime/.sqlx/query-80abaede7d6da749be3c1a5a6f5ff2da49f9b39c07bb5b2a0bfb35d86d7b7c8e.json b/runledger-runtime/.sqlx/query-80abaede7d6da749be3c1a5a6f5ff2da49f9b39c07bb5b2a0bfb35d86d7b7c8e.json new file mode 100644 index 0000000..0fcf450 --- /dev/null +++ b/runledger-runtime/.sqlx/query-80abaede7d6da749be3c1a5a6f5ff2da49f9b39c07bb5b2a0bfb35d86d7b7c8e.json @@ -0,0 +1,188 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n status::text AS \"status!\",\n priority,\n run_number,\n attempt,\n max_attempts,\n timeout_seconds,\n next_run_at,\n lease_expires_at,\n last_heartbeat_at,\n worker_id,\n started_at,\n finished_at,\n stage,\n progress_done,\n progress_total,\n progress_pct::float8 AS progress_pct,\n checkpoint,\n output,\n idempotency_key,\n status_reason,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_queue\n WHERE organization_id = $1 AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "payload", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 9, + "name": "timeout_seconds", + "type_info": "Int4" + }, + { + "ordinal": 10, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 11, + "name": "lease_expires_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 12, + "name": "last_heartbeat_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "worker_id", + "type_info": "Text" + }, + { + "ordinal": 14, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 15, + "name": "finished_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 16, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 17, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 18, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 19, + "name": "progress_pct", + "type_info": "Float8" + }, + { + "ordinal": 20, + "name": "checkpoint", + "type_info": "Jsonb" + }, + { + "ordinal": 21, + "name": "output", + "type_info": "Jsonb" + }, + { + "ordinal": 22, + "name": "idempotency_key", + "type_info": "Text" + }, + { + "ordinal": 23, + "name": "status_reason", + "type_info": "Text" + }, + { + "ordinal": 24, + "name": "last_error_code", + "type_info": "Text" + }, + { + "ordinal": 25, + "name": "last_error_message", + "type_info": "Text" + }, + { + "ordinal": 26, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 27, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false, + null, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + true, + false, + true, + true, + null, + true, + true, + true, + true, + true, + true, + false, + false + ] + }, + "hash": "80abaede7d6da749be3c1a5a6f5ff2da49f9b39c07bb5b2a0bfb35d86d7b7c8e" +} diff --git a/runledger-runtime/.sqlx/query-822a3544ce03dde10c9f57947ccbb25546b94f29cfdcf2151d28361d3a04d461.json b/runledger-runtime/.sqlx/query-822a3544ce03dde10c9f57947ccbb25546b94f29cfdcf2151d28361d3a04d461.json new file mode 100644 index 0000000..464adbd --- /dev/null +++ b/runledger-runtime/.sqlx/query-822a3544ce03dde10c9f57947ccbb25546b94f29cfdcf2151d28361d3a04d461.json @@ -0,0 +1,105 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, job_type, organization_id, status::text AS \"status!\",\n priority, run_number, attempt, max_attempts, next_run_at, stage,\n progress_done, progress_total, created_at, updated_at\n FROM job_queue WHERE $1::uuid IS NULL AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type = $3) AND (created_at, id) < ($5, $6) ORDER BY created_at DESC, id DESC LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Timestamptz", + "Uuid" + ] + }, + "nullable": [ + false, + false, + true, + null, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + }, + "hash": "822a3544ce03dde10c9f57947ccbb25546b94f29cfdcf2151d28361d3a04d461" +} diff --git a/runledger-runtime/.sqlx/query-85eca198e5990ee9fa5ec1abef7c0abaf3750dfb0d4e56e6a242e2921a46d995.json b/runledger-runtime/.sqlx/query-85eca198e5990ee9fa5ec1abef7c0abaf3750dfb0d4e56e6a242e2921a46d995.json new file mode 100644 index 0000000..9b9f66e --- /dev/null +++ b/runledger-runtime/.sqlx/query-85eca198e5990ee9fa5ec1abef7c0abaf3750dfb0d4e56e6a242e2921a46d995.json @@ -0,0 +1,105 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, job_type, organization_id, status::text AS \"status!\",\n priority, run_number, attempt, max_attempts, next_run_at, stage,\n progress_done, progress_total, created_at, updated_at\n FROM job_queue WHERE organization_id = $1 AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type = $3) AND (created_at, id) < ($5, $6) ORDER BY created_at DESC, id DESC LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Timestamptz", + "Uuid" + ] + }, + "nullable": [ + false, + false, + true, + null, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + }, + "hash": "85eca198e5990ee9fa5ec1abef7c0abaf3750dfb0d4e56e6a242e2921a46d995" +} diff --git a/runledger-runtime/.sqlx/query-89faf4374de8090be4461c194856d94ddc63813894ceaf9fef0c0a00cb2598e7.json b/runledger-runtime/.sqlx/query-89faf4374de8090be4461c194856d94ddc63813894ceaf9fef0c0a00cb2598e7.json new file mode 100644 index 0000000..9ac167b --- /dev/null +++ b/runledger-runtime/.sqlx/query-89faf4374de8090be4461c194856d94ddc63813894ceaf9fef0c0a00cb2598e7.json @@ -0,0 +1,62 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH status_metrics AS (\n SELECT\n job_type,\n COUNT(*)::bigint AS pending_count,\n COUNT(*) FILTER (WHERE promotion_attempts > 0)::bigint AS retrying_count,\n MAX(promotion_attempts)::integer AS max_promotion_attempts,\n 0::bigint AS conflicted_24h,\n 0::bigint AS promoted_24h,\n MIN(created_at) AS oldest_pending_at\n FROM job_enqueue_intents\n WHERE status = 'PENDING'\n AND ($5::boolean OR (organization_id = $1 OR ($1::uuid IS NULL AND organization_id IS NULL)))\n AND ($2::text IS NULL OR job_type = $2)\n GROUP BY job_type\n\n UNION ALL\n\n SELECT\n job_type,\n 0::bigint AS pending_count,\n 0::bigint AS retrying_count,\n 0::integer AS max_promotion_attempts,\n COUNT(*)::bigint AS conflicted_24h,\n 0::bigint AS promoted_24h,\n NULL::timestamptz AS oldest_pending_at\n FROM job_enqueue_intents\n WHERE status = 'CONFLICTED'\n AND conflicted_at >= now() - interval '24 hours'\n AND ($5::boolean OR (organization_id = $1 OR ($1::uuid IS NULL AND organization_id IS NULL)))\n AND ($2::text IS NULL OR job_type = $2)\n GROUP BY job_type\n\n UNION ALL\n\n SELECT\n job_type,\n 0::bigint AS pending_count,\n 0::bigint AS retrying_count,\n 0::integer AS max_promotion_attempts,\n 0::bigint AS conflicted_24h,\n COUNT(*)::bigint AS promoted_24h,\n NULL::timestamptz AS oldest_pending_at\n FROM job_enqueue_intents\n WHERE status = 'PROMOTED'\n AND promoted_at >= now() - interval '24 hours'\n AND ($5::boolean OR (organization_id = $1 OR ($1::uuid IS NULL AND organization_id IS NULL)))\n AND ($2::text IS NULL OR job_type = $2)\n GROUP BY job_type\n )\n SELECT\n job_type AS \"job_type!\",\n MAX(pending_count)::bigint AS \"pending_count!\",\n MAX(retrying_count)::bigint AS \"retrying_count!\",\n MAX(max_promotion_attempts)::integer AS \"max_promotion_attempts!\",\n MAX(conflicted_24h)::bigint AS \"conflicted_24h!\",\n MAX(promoted_24h)::bigint AS \"promoted_24h!\",\n MIN(oldest_pending_at) AS oldest_pending_at\n FROM status_metrics\n GROUP BY job_type\n ORDER BY job_type\n LIMIT $3\n OFFSET $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_type!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "pending_count!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "retrying_count!", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "max_promotion_attempts!", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "conflicted_24h!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "promoted_24h!", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "oldest_pending_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Int8", + "Int8", + "Bool" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "89faf4374de8090be4461c194856d94ddc63813894ceaf9fef0c0a00cb2598e7" +} diff --git a/runledger-runtime/.sqlx/query-8a04ed3b1b3497fbb7ec16ef976e9be2b03e4d64d5448bd90ce15cf8a9e5acc0.json b/runledger-runtime/.sqlx/query-8a04ed3b1b3497fbb7ec16ef976e9be2b03e4d64d5448bd90ce15cf8a9e5acc0.json new file mode 100644 index 0000000..a47e779 --- /dev/null +++ b/runledger-runtime/.sqlx/query-8a04ed3b1b3497fbb7ec16ef976e9be2b03e4d64d5448bd90ce15cf8a9e5acc0.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, payload FROM job_queue WHERE organization_id = $1 AND job_type = $2\n AND idempotency_key = $3\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "payload", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "8a04ed3b1b3497fbb7ec16ef976e9be2b03e4d64d5448bd90ce15cf8a9e5acc0" +} diff --git a/runledger-runtime/.sqlx/query-8de6d5625581aff24f7bb689361c8dcd9319042f9883e4cc748bcd771b06c09e.json b/runledger-runtime/.sqlx/query-8de6d5625581aff24f7bb689361c8dcd9319042f9883e4cc748bcd771b06c09e.json new file mode 100644 index 0000000..c506de2 --- /dev/null +++ b/runledger-runtime/.sqlx/query-8de6d5625581aff24f7bb689361c8dcd9319042f9883e4cc748bcd771b06c09e.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, status::text AS \"status!\", run_number, attempt, updated_at\n FROM job_queue WHERE organization_id = $1 AND id = ANY($2::uuid[]) ORDER BY id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "UuidArray" + ] + }, + "nullable": [ + false, + null, + false, + false, + false + ] + }, + "hash": "8de6d5625581aff24f7bb689361c8dcd9319042f9883e4cc748bcd771b06c09e" +} diff --git a/runledger-runtime/.sqlx/query-90841ab00a885ed00fd2c3a6b49c90b549a262963d281b863f4283af861ea5e5.json b/runledger-runtime/.sqlx/query-90841ab00a885ed00fd2c3a6b49c90b549a262963d281b863f4283af861ea5e5.json deleted file mode 100644 index bf1cfb5..0000000 --- a/runledger-runtime/.sqlx/query-90841ab00a885ed00fd2c3a6b49c90b549a262963d281b863f4283af861ea5e5.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH status_metrics AS (\n SELECT\n job_type,\n COUNT(*)::bigint AS pending_count,\n COUNT(*) FILTER (WHERE promotion_attempts > 0)::bigint AS retrying_count,\n MAX(promotion_attempts)::integer AS max_promotion_attempts,\n 0::bigint AS conflicted_24h,\n 0::bigint AS promoted_24h,\n MIN(created_at) AS oldest_pending_at\n FROM job_enqueue_intents\n WHERE status = 'PENDING'\n AND ($1::uuid IS NULL OR organization_id = $1)\n AND ($2::text IS NULL OR job_type = $2)\n GROUP BY job_type\n\n UNION ALL\n\n SELECT\n job_type,\n 0::bigint AS pending_count,\n 0::bigint AS retrying_count,\n 0::integer AS max_promotion_attempts,\n COUNT(*)::bigint AS conflicted_24h,\n 0::bigint AS promoted_24h,\n NULL::timestamptz AS oldest_pending_at\n FROM job_enqueue_intents\n WHERE status = 'CONFLICTED'\n AND conflicted_at >= now() - interval '24 hours'\n AND ($1::uuid IS NULL OR organization_id = $1)\n AND ($2::text IS NULL OR job_type = $2)\n GROUP BY job_type\n\n UNION ALL\n\n SELECT\n job_type,\n 0::bigint AS pending_count,\n 0::bigint AS retrying_count,\n 0::integer AS max_promotion_attempts,\n 0::bigint AS conflicted_24h,\n COUNT(*)::bigint AS promoted_24h,\n NULL::timestamptz AS oldest_pending_at\n FROM job_enqueue_intents\n WHERE status = 'PROMOTED'\n AND promoted_at >= now() - interval '24 hours'\n AND ($1::uuid IS NULL OR organization_id = $1)\n AND ($2::text IS NULL OR job_type = $2)\n GROUP BY job_type\n )\n SELECT\n job_type AS \"job_type!\",\n MAX(pending_count)::bigint AS \"pending_count!\",\n MAX(retrying_count)::bigint AS \"retrying_count!\",\n MAX(max_promotion_attempts)::integer AS \"max_promotion_attempts!\",\n MAX(conflicted_24h)::bigint AS \"conflicted_24h!\",\n MAX(promoted_24h)::bigint AS \"promoted_24h!\",\n MIN(oldest_pending_at) AS oldest_pending_at\n FROM status_metrics\n GROUP BY job_type\n ORDER BY job_type\n LIMIT $3\n OFFSET $4", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "job_type!", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "pending_count!", - "type_info": "Int8" - }, - { - "ordinal": 2, - "name": "retrying_count!", - "type_info": "Int8" - }, - { - "ordinal": 3, - "name": "max_promotion_attempts!", - "type_info": "Int4" - }, - { - "ordinal": 4, - "name": "conflicted_24h!", - "type_info": "Int8" - }, - { - "ordinal": 5, - "name": "promoted_24h!", - "type_info": "Int8" - }, - { - "ordinal": 6, - "name": "oldest_pending_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Int8", - "Int8" - ] - }, - "nullable": [ - null, - null, - null, - null, - null, - null, - null - ] - }, - "hash": "90841ab00a885ed00fd2c3a6b49c90b549a262963d281b863f4283af861ea5e5" -} diff --git a/runledger-runtime/.sqlx/query-92a41242e40c65c1eaad5aec4b2bd0b868b3e1fe7a7ba1ab015bf4c93c6fd431.json b/runledger-runtime/.sqlx/query-92a41242e40c65c1eaad5aec4b2bd0b868b3e1fe7a7ba1ab015bf4c93c6fd431.json new file mode 100644 index 0000000..b7ae055 --- /dev/null +++ b/runledger-runtime/.sqlx/query-92a41242e40c65c1eaad5aec4b2bd0b868b3e1fe7a7ba1ab015bf4c93c6fd431.json @@ -0,0 +1,103 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, job_type, organization_id, status::text AS \"status!\",\n priority, run_number, attempt, max_attempts, next_run_at, stage,\n progress_done, progress_total, created_at, updated_at\n FROM job_queue WHERE organization_id = $1 AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type = $3) ORDER BY created_at DESC, id DESC LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + null, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + }, + "hash": "92a41242e40c65c1eaad5aec4b2bd0b868b3e1fe7a7ba1ab015bf4c93c6fd431" +} diff --git a/runledger-runtime/.sqlx/query-a0383524b739ee8a06fba3e337e07e608049f1a18f43a548859d9683d6bae1f6.json b/runledger-runtime/.sqlx/query-a0383524b739ee8a06fba3e337e07e608049f1a18f43a548859d9683d6bae1f6.json new file mode 100644 index 0000000..c515cda --- /dev/null +++ b/runledger-runtime/.sqlx/query-a0383524b739ee8a06fba3e337e07e608049f1a18f43a548859d9683d6bae1f6.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, status::text AS \"status!\", run_number, attempt, updated_at\n FROM job_queue WHERE $1::uuid IS NULL AND id = ANY($2::uuid[]) ORDER BY id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "UuidArray" + ] + }, + "nullable": [ + false, + null, + false, + false, + false + ] + }, + "hash": "a0383524b739ee8a06fba3e337e07e608049f1a18f43a548859d9683d6bae1f6" +} diff --git a/runledger-runtime/.sqlx/query-a0514692991ed79f5c48f8b8758234202b016da9c5c40cac67f72e000455b9b8.json b/runledger-runtime/.sqlx/query-a0514692991ed79f5c48f8b8758234202b016da9c5c40cac67f72e000455b9b8.json deleted file mode 100644 index 9fe67e1..0000000 --- a/runledger-runtime/.sqlx/query-a0514692991ed79f5c48f8b8758234202b016da9c5c40cac67f72e000455b9b8.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT id, payload\n FROM job_queue\n WHERE organization_id = $1\n AND job_type = $2\n AND payload->>'run_id' = $3\n ORDER BY created_at DESC, id DESC\n LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "payload", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Text" - ] - }, - "nullable": [ - false, - false - ] - }, - "hash": "a0514692991ed79f5c48f8b8758234202b016da9c5c40cac67f72e000455b9b8" -} diff --git a/runledger-runtime/.sqlx/query-a611f047bb82ca7e1794ff283b09aee3ef1d9656babed381e73eb12626f3648c.json b/runledger-runtime/.sqlx/query-a611f047bb82ca7e1794ff283b09aee3ef1d9656babed381e73eb12626f3648c.json new file mode 100644 index 0000000..6d0321f --- /dev/null +++ b/runledger-runtime/.sqlx/query-a611f047bb82ca7e1794ff283b09aee3ef1d9656babed381e73eb12626f3648c.json @@ -0,0 +1,103 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, job_type, organization_id, status::text AS \"status!\",\n priority, run_number, attempt, max_attempts, next_run_at, stage,\n progress_done, progress_total, created_at, updated_at\n FROM job_queue WHERE ($1::uuid IS NULL AND organization_id IS NULL) AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type = $3) ORDER BY created_at DESC, id DESC LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + null, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + }, + "hash": "a611f047bb82ca7e1794ff283b09aee3ef1d9656babed381e73eb12626f3648c" +} diff --git a/runledger-runtime/.sqlx/query-a74a899a375ff31b61d663d73cc4a13e467c246b649d4658484f183501c5d0c8.json b/runledger-runtime/.sqlx/query-a74a899a375ff31b61d663d73cc4a13e467c246b649d4658484f183501c5d0c8.json new file mode 100644 index 0000000..c375c3f --- /dev/null +++ b/runledger-runtime/.sqlx/query-a74a899a375ff31b61d663d73cc4a13e467c246b649d4658484f183501c5d0c8.json @@ -0,0 +1,158 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n priority,\n max_attempts,\n timeout_seconds,\n next_run_at,\n idempotency_key,\n stage,\n enqueue_request_version,\n execution_resource_key,\n promotion_attempts,\n next_promotion_at,\n last_attempted_at,\n status,\n promoted_job_id,\n promoted_at,\n conflicted_at,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_enqueue_intents\n WHERE organization_id = $1 AND ($2::text IS NULL OR status = $2)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "payload", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "timeout_seconds", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 8, + "name": "idempotency_key", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "enqueue_request_version", + "type_info": "Int2" + }, + { + "ordinal": 11, + "name": "execution_resource_key", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "promotion_attempts", + "type_info": "Int4" + }, + { + "ordinal": 13, + "name": "next_promotion_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 14, + "name": "last_attempted_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 15, + "name": "status", + "type_info": "Text" + }, + { + "ordinal": 16, + "name": "promoted_job_id", + "type_info": "Uuid" + }, + { + "ordinal": 17, + "name": "promoted_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 18, + "name": "conflicted_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 19, + "name": "last_error_code", + "type_info": "Text" + }, + { + "ordinal": 20, + "name": "last_error_message", + "type_info": "Text" + }, + { + "ordinal": 21, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 22, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false, + true, + true, + true, + true, + false, + false, + false, + true, + false, + false, + true, + false, + true, + true, + true, + true, + true, + false, + false + ] + }, + "hash": "a74a899a375ff31b61d663d73cc4a13e467c246b649d4658484f183501c5d0c8" +} diff --git a/runledger-runtime/.sqlx/query-ba0f6f24f4ce9dcb1d40cb75eb26228c07a691e183d68a876e0e8f778120d137.json b/runledger-runtime/.sqlx/query-ba0f6f24f4ce9dcb1d40cb75eb26228c07a691e183d68a876e0e8f778120d137.json new file mode 100644 index 0000000..c8ad8b2 --- /dev/null +++ b/runledger-runtime/.sqlx/query-ba0f6f24f4ce9dcb1d40cb75eb26228c07a691e183d68a876e0e8f778120d137.json @@ -0,0 +1,188 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n status::text AS \"status!\",\n priority,\n run_number,\n attempt,\n max_attempts,\n timeout_seconds,\n next_run_at,\n lease_expires_at,\n last_heartbeat_at,\n worker_id,\n started_at,\n finished_at,\n stage,\n progress_done,\n progress_total,\n progress_pct::float8 AS progress_pct,\n checkpoint,\n output,\n idempotency_key,\n status_reason,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_queue\n WHERE $1::uuid IS NULL AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type ILIKE '%' || $3 || '%')\n ORDER BY created_at DESC, id DESC\n LIMIT $4\n OFFSET $5", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "payload", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 9, + "name": "timeout_seconds", + "type_info": "Int4" + }, + { + "ordinal": 10, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 11, + "name": "lease_expires_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 12, + "name": "last_heartbeat_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "worker_id", + "type_info": "Text" + }, + { + "ordinal": 14, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 15, + "name": "finished_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 16, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 17, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 18, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 19, + "name": "progress_pct", + "type_info": "Float8" + }, + { + "ordinal": 20, + "name": "checkpoint", + "type_info": "Jsonb" + }, + { + "ordinal": 21, + "name": "output", + "type_info": "Jsonb" + }, + { + "ordinal": 22, + "name": "idempotency_key", + "type_info": "Text" + }, + { + "ordinal": 23, + "name": "status_reason", + "type_info": "Text" + }, + { + "ordinal": 24, + "name": "last_error_code", + "type_info": "Text" + }, + { + "ordinal": 25, + "name": "last_error_message", + "type_info": "Text" + }, + { + "ordinal": 26, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 27, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false, + null, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + true, + false, + true, + true, + null, + true, + true, + true, + true, + true, + true, + false, + false + ] + }, + "hash": "ba0f6f24f4ce9dcb1d40cb75eb26228c07a691e183d68a876e0e8f778120d137" +} diff --git a/runledger-runtime/.sqlx/query-ba9acdf8f6c1427f2488aec9ab85ac3eddf200e7f69494de198b3560888d03f7.json b/runledger-runtime/.sqlx/query-ba9acdf8f6c1427f2488aec9ab85ac3eddf200e7f69494de198b3560888d03f7.json new file mode 100644 index 0000000..fa50003 --- /dev/null +++ b/runledger-runtime/.sqlx/query-ba9acdf8f6c1427f2488aec9ab85ac3eddf200e7f69494de198b3560888d03f7.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workflow_step_dependencies (\n workflow_run_id, prerequisite_step_id, dependent_step_id, release_mode\n ) SELECT $1, r.prerequisite_step_id, r.dependent_step_id,\n r.release_mode::workflow_dependency_release_mode\n FROM jsonb_to_recordset($2::jsonb) AS r(\n prerequisite_step_id uuid, dependent_step_id uuid, release_mode text)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "ba9acdf8f6c1427f2488aec9ab85ac3eddf200e7f69494de198b3560888d03f7" +} diff --git a/runledger-runtime/.sqlx/query-bd7c8cceca8d42541d5816afd514778bd665cb721f5a44bcc700a08a8057fd95.json b/runledger-runtime/.sqlx/query-bd7c8cceca8d42541d5816afd514778bd665cb721f5a44bcc700a08a8057fd95.json deleted file mode 100644 index f427afe..0000000 --- a/runledger-runtime/.sqlx/query-bd7c8cceca8d42541d5816afd514778bd665cb721f5a44bcc700a08a8057fd95.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workflow_step_dependencies (\n workflow_run_id,\n prerequisite_step_id,\n dependent_step_id,\n release_mode\n )\n VALUES ($1, $2, $3, $4::text::workflow_dependency_release_mode)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Uuid", - "Uuid", - "Text" - ] - }, - "nullable": [] - }, - "hash": "bd7c8cceca8d42541d5816afd514778bd665cb721f5a44bcc700a08a8057fd95" -} diff --git a/.sqlx/query-af553a066c6cd1b24770d60492216a8a192bebc3257165496ee1523797b2df5d.json b/runledger-runtime/.sqlx/query-d3fdc5ee034100f4b1d0feca12b9e0cca4d8a97b4d1700f4f1cd4b2380e6ba43.json similarity index 74% rename from .sqlx/query-af553a066c6cd1b24770d60492216a8a192bebc3257165496ee1523797b2df5d.json rename to runledger-runtime/.sqlx/query-d3fdc5ee034100f4b1d0feca12b9e0cca4d8a97b4d1700f4f1cd4b2380e6ba43.json index 871d4e7..232afe9 100644 --- a/.sqlx/query-af553a066c6cd1b24770d60492216a8a192bebc3257165496ee1523797b2df5d.json +++ b/runledger-runtime/.sqlx/query-d3fdc5ee034100f4b1d0feca12b9e0cca4d8a97b4d1700f4f1cd4b2380e6ba43.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n jd.job_type AS \"job_type!\",\n COALESCE(SUM(jcmr.continued_24h), 0)::bigint AS \"continued_24h!\",\n COALESCE(SUM(jcmr.active_continued_count), 0)::bigint AS \"active_continued_count!\",\n COALESCE(MAX(jcmr.max_active_run_number), 0)::int4 AS \"max_active_run_number!\"\n FROM job_definitions jd\n LEFT JOIN job_continuation_metrics_rollup jcmr\n ON jcmr.job_type = jd.job_type\n AND ($1::uuid IS NULL OR jcmr.organization_id = $1)\n WHERE ($2::text IS NULL OR jd.job_type = $2)\n GROUP BY jd.job_type\n ORDER BY jd.job_type ASC", + "query": "SELECT\n jd.job_type AS \"job_type!\",\n COALESCE(SUM(jcmr.continued_24h), 0)::bigint AS \"continued_24h!\",\n COALESCE(SUM(jcmr.active_continued_count), 0)::bigint AS \"active_continued_count!\",\n COALESCE(MAX(jcmr.max_active_run_number), 0)::int4 AS \"max_active_run_number!\"\n FROM job_definitions jd\n LEFT JOIN job_continuation_metrics_rollup jcmr\n ON jcmr.job_type = jd.job_type\n AND ($3::boolean OR (jcmr.organization_id = $1 OR ($1::uuid IS NULL AND jcmr.organization_id IS NULL)))\n WHERE ($2::text IS NULL OR jd.job_type = $2)\n GROUP BY jd.job_type\n ORDER BY jd.job_type ASC", "describe": { "columns": [ { @@ -27,7 +27,8 @@ "parameters": { "Left": [ "Uuid", - "Text" + "Text", + "Bool" ] }, "nullable": [ @@ -37,5 +38,5 @@ null ] }, - "hash": "af553a066c6cd1b24770d60492216a8a192bebc3257165496ee1523797b2df5d" + "hash": "d3fdc5ee034100f4b1d0feca12b9e0cca4d8a97b4d1700f4f1cd4b2380e6ba43" } diff --git a/runledger-runtime/.sqlx/query-d72de6e5b2099c445d943a72a081987f9b1cc6ca13c877861d41fcf86466d6e6.json b/runledger-runtime/.sqlx/query-d72de6e5b2099c445d943a72a081987f9b1cc6ca13c877861d41fcf86466d6e6.json new file mode 100644 index 0000000..4783ff9 --- /dev/null +++ b/runledger-runtime/.sqlx/query-d72de6e5b2099c445d943a72a081987f9b1cc6ca13c877861d41fcf86466d6e6.json @@ -0,0 +1,103 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, job_type, organization_id, status::text AS \"status!\",\n priority, run_number, attempt, max_attempts, next_run_at, stage,\n progress_done, progress_total, created_at, updated_at\n FROM job_queue WHERE $1::uuid IS NULL AND ($2::text::job_status IS NULL OR status = $2::text::job_status)\n AND ($3::text IS NULL OR job_type = $3) ORDER BY created_at DESC, id DESC LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "job_type", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "run_number", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "attempt", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "max_attempts", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "next_run_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "stage", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "progress_done", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "progress_total", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + null, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + }, + "hash": "d72de6e5b2099c445d943a72a081987f9b1cc6ca13c877861d41fcf86466d6e6" +} diff --git a/runledger-runtime/.sqlx/query-d74db24f58285ef5796dcfe06b49136fea475553bcf65423c59c3067a2ef49b1.json b/runledger-runtime/.sqlx/query-d74db24f58285ef5796dcfe06b49136fea475553bcf65423c59c3067a2ef49b1.json new file mode 100644 index 0000000..b67bac0 --- /dev/null +++ b/runledger-runtime/.sqlx/query-d74db24f58285ef5796dcfe06b49136fea475553bcf65423c59c3067a2ef49b1.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workflow_steps (\n workflow_run_id, step_key, execution_kind, job_type, organization_id, payload,\n priority, max_attempts, timeout_seconds, stage, allow_handler_continuation,\n execution_resource_key, status, dependency_count_total,\n dependency_count_pending, dependency_count_unsatisfied\n ) SELECT $1, r.step_key, r.execution_kind::workflow_step_execution_kind,\n r.job_type, r.organization_id, COALESCE(r.payload, 'null'::jsonb), r.priority, r.max_attempts,\n r.timeout_seconds, r.stage, r.allow_handler_continuation, r.execution_resource_key,\n 'BLOCKED', r.dependency_count_total, r.dependency_count_pending, r.dependency_count_unsatisfied\n FROM jsonb_to_recordset($2::jsonb) AS r(\n step_key text, execution_kind text, job_type text, organization_id uuid, payload jsonb,\n priority int, max_attempts int, timeout_seconds int, stage text,\n allow_handler_continuation bool, execution_resource_key text,\n dependency_count_total int, dependency_count_pending int, dependency_count_unsatisfied int)\n RETURNING id, step_key", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "step_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Jsonb" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "d74db24f58285ef5796dcfe06b49136fea475553bcf65423c59c3067a2ef49b1" +} diff --git a/runledger-runtime/.sqlx/query-26acb4ac4804ccb2f1231f1e2bfe017f40f2e13c61b74158d5e09a66aac6c06d.json b/runledger-runtime/.sqlx/query-ddd4e90290e03da64cf22463877941b79baa32e938e95d66339a91fc3756d599.json similarity index 95% rename from runledger-runtime/.sqlx/query-26acb4ac4804ccb2f1231f1e2bfe017f40f2e13c61b74158d5e09a66aac6c06d.json rename to runledger-runtime/.sqlx/query-ddd4e90290e03da64cf22463877941b79baa32e938e95d66339a91fc3756d599.json index e8d6762..28b299f 100644 --- a/runledger-runtime/.sqlx/query-26acb4ac4804ccb2f1231f1e2bfe017f40f2e13c61b74158d5e09a66aac6c06d.json +++ b/runledger-runtime/.sqlx/query-ddd4e90290e03da64cf22463877941b79baa32e938e95d66339a91fc3756d599.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n status::text AS \"status!\",\n priority,\n run_number,\n attempt,\n max_attempts,\n timeout_seconds,\n next_run_at,\n lease_expires_at,\n last_heartbeat_at,\n worker_id,\n started_at,\n finished_at,\n stage,\n progress_done,\n progress_total,\n progress_pct::float8 AS progress_pct,\n checkpoint,\n output,\n idempotency_key,\n status_reason,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_queue\n WHERE id = $1\n AND ($2::uuid IS NULL OR organization_id = $2)\n LIMIT 1", + "query": "SELECT\n id,\n job_type,\n organization_id,\n payload,\n status::text AS \"status!\",\n priority,\n run_number,\n attempt,\n max_attempts,\n timeout_seconds,\n next_run_at,\n lease_expires_at,\n last_heartbeat_at,\n worker_id,\n started_at,\n finished_at,\n stage,\n progress_done,\n progress_total,\n progress_pct::float8 AS progress_pct,\n checkpoint,\n output,\n idempotency_key,\n status_reason,\n last_error_code,\n last_error_message,\n created_at,\n updated_at\n FROM job_queue\n WHERE id = $1\n AND ($3::bool OR organization_id IS NOT DISTINCT FROM $2::uuid)\n LIMIT 1", "describe": { "columns": [ { @@ -147,7 +147,8 @@ "parameters": { "Left": [ "Uuid", - "Uuid" + "Uuid", + "Bool" ] }, "nullable": [ @@ -181,5 +182,5 @@ false ] }, - "hash": "26acb4ac4804ccb2f1231f1e2bfe017f40f2e13c61b74158d5e09a66aac6c06d" + "hash": "ddd4e90290e03da64cf22463877941b79baa32e938e95d66339a91fc3756d599" } diff --git a/runledger-runtime/.sqlx/query-f6b58ff379b8af9e2ab6346743ad1da85494d295268da6627d59c2bcc70b9fb0.json b/runledger-runtime/.sqlx/query-f6b58ff379b8af9e2ab6346743ad1da85494d295268da6627d59c2bcc70b9fb0.json deleted file mode 100644 index ab6c006..0000000 --- a/runledger-runtime/.sqlx/query-f6b58ff379b8af9e2ab6346743ad1da85494d295268da6627d59c2bcc70b9fb0.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH locked_job AS MATERIALIZED (\n SELECT id\n FROM job_queue\n WHERE id = $1\n AND run_number = $2\n AND attempt = $3\n AND worker_id = $4\n AND status = 'LEASED'\n AND lease_expires_at IS NOT NULL\n FOR UPDATE\n )\n UPDATE job_queue\n SET stage = COALESCE($5, stage),\n progress_done = COALESCE($6, progress_done),\n progress_total = COALESCE($7, progress_total),\n checkpoint = COALESCE($8::jsonb, checkpoint),\n updated_at = now()\n FROM locked_job\n WHERE job_queue.id = locked_job.id\n AND job_queue.lease_expires_at > clock_timestamp()", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Int4", - "Int4", - "Text", - "Text", - "Int8", - "Int8", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "f6b58ff379b8af9e2ab6346743ad1da85494d295268da6627d59c2bcc70b9fb0" -} diff --git a/runledger-runtime/Cargo.toml b/runledger-runtime/Cargo.toml index 8e5c026..e9a52f2 100644 --- a/runledger-runtime/Cargo.toml +++ b/runledger-runtime/Cargo.toml @@ -31,6 +31,17 @@ tracing = { workspace = true } uuid = { version = "1", features = ["v7", "serde"] } [dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } +serde = { version = "1", features = ["derive"] } runledger-postgres = { workspace = true, features = ["test-support"] } runledger-test-support = { workspace = true } tracing-subscriber = "0.3" + +[[example]] +name = "producer" +path = "examples/producer_worker/producer.rs" + +[[example]] +name = "worker" +path = "examples/producer_worker/worker.rs" +test = true diff --git a/runledger-runtime/examples/checkpointed_counter.rs b/runledger-runtime/examples/checkpointed_counter.rs new file mode 100644 index 0000000..e1977b7 --- /dev/null +++ b/runledger-runtime/examples/checkpointed_counter.rs @@ -0,0 +1,48 @@ +use runledger_core::prelude::*; +use serde_json::{Value, json}; +use std::time::Duration; + +struct CheckpointedCounter; + +#[async_trait] +impl JobExecutionHandler for CheckpointedCounter { + fn job_type(&self) -> JobType<'static> { + JobType::new("jobs.example.counter") + } + + async fn execute( + &self, + execution: JobExecution<'_>, + _payload: Value, + ) -> Result { + let mut cursor = execution + .checkpoint::() + .map_err(|_| { + JobFailure::terminal("counter.invalid_checkpoint", "Invalid counter checkpoint.") + })? + .unwrap_or(0); + while cursor < 10 { + if execution + .remaining_work_budget(Duration::from_secs(1)) + .is_zero() + { + return Ok(JobCompletion::continue_now()); + } + cursor += 1; + let checkpoint = json!(cursor); + execution + .persist_progress(JobExecutionUpdate { + progress_done: Some(cursor as i64), + progress_total: Some(10), + checkpoint: Some(&checkpoint), + }) + .await?; + } + Ok(JobCompletion::success()) + } +} + +fn main() { + let _catalog = runledger_runtime::catalog::JobCatalog::new() + .handler(CheckpointedCounter.into_job_handler()); +} diff --git a/runledger-runtime/examples/producer_worker/producer.rs b/runledger-runtime/examples/producer_worker/producer.rs new file mode 100644 index 0000000..44c687e --- /dev/null +++ b/runledger-runtime/examples/producer_worker/producer.rs @@ -0,0 +1,26 @@ +pub mod shared; + +use runledger_postgres::jobs::enqueue_job_tx; +use shared::{Greeting, request}; +use sqlx::postgres::PgPoolOptions; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let name = std::env::args() + .nth(1) + .ok_or("usage: producer ")?; + let key = std::env::args().nth(2).ok_or("missing request-key")?; + let pool = PgPoolOptions::new() + .connect(&std::env::var("DATABASE_URL")?) + .await?; + runledger_postgres::ensure_schema_compatible_after_idempotency_cutover(&pool).await?; + + let payload = serde_json::to_value(Greeting { name })?; + let mut tx = pool.begin().await?; + // Persist application changes with this same transaction when needed. + let job_id = enqueue_job_tx(&mut tx, &request(&payload, &key)).await?; + tx.commit().await?; + println!("enqueued {job_id}"); + pool.close().await; + Ok(()) +} diff --git a/runledger-runtime/examples/producer_worker/shared.rs b/runledger-runtime/examples/producer_worker/shared.rs new file mode 100644 index 0000000..e42ed4a --- /dev/null +++ b/runledger-runtime/examples/producer_worker/shared.rs @@ -0,0 +1,25 @@ +use runledger_core::jobs::JobType; +use runledger_postgres::jobs::JobEnqueue; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +pub const GREETING_JOB: JobType<'static> = JobType::new("jobs.greeting.print"); + +#[derive(Serialize, Deserialize)] +pub struct Greeting { + pub name: String, +} + +pub fn request<'a>(payload: &'a Value, key: &'a str) -> JobEnqueue<'a> { + JobEnqueue { + job_type: GREETING_JOB, + organization_id: None, + payload, + priority: None, + max_attempts: None, + timeout_seconds: None, + next_run_at: None, + idempotency_key: Some(key), + stage: None, + } +} diff --git a/runledger-runtime/examples/producer_worker/tests.rs b/runledger-runtime/examples/producer_worker/tests.rs new file mode 100644 index 0000000..a645e40 --- /dev/null +++ b/runledger-runtime/examples/producer_worker/tests.rs @@ -0,0 +1,88 @@ +use super::*; +use runledger_core::jobs::JobStatus; +use runledger_postgres::jobs::{JobReadScope, enqueue_job_tx, get_job_by_id_with_scope}; +use runledger_runtime::config::JobsConfig; +use runledger_test_support::{setup_ephemeral_pool, teardown_ephemeral_pool}; +use shared::request; + +#[tokio::test] +async fn shared_contract_transaction_and_worker_round_trip() { + let (pool, database) = setup_ephemeral_pool("producer_worker_example", 5).await; + let version: String = sqlx::query_scalar("SHOW server_version") + .fetch_one(&pool) + .await + .expect("server version"); + eprintln!("producer/worker example PostgreSQL {version}"); + let catalog = JobCatalog::new().handler(PrintGreeting); + catalog.sync_definitions(&pool).await.expect("definitions"); + let payload = serde_json::to_value(Greeting { name: "Ada".into() }).expect("payload"); + + let mut tx = pool.begin().await.expect("transaction"); + let rolled_back = enqueue_job_tx(&mut tx, &request(&payload, "rolled-back")) + .await + .expect("enqueue before rollback"); + tx.rollback().await.expect("rollback"); + assert!( + get_job_by_id_with_scope(&pool, JobReadScope::Global, rolled_back) + .await + .expect("read rolled back job") + .is_none() + ); + + let mut tx = pool.begin().await.expect("transaction"); + let job_id = enqueue_job_tx(&mut tx, &request(&payload, "greeting:1")) + .await + .expect("enqueue"); + tx.commit().await.expect("commit"); + let mut tx = pool.begin().await.expect("retry transaction"); + let retry_id = enqueue_job_tx(&mut tx, &request(&payload, "greeting:1")) + .await + .expect("idempotent retry"); + tx.commit().await.expect("retry commit"); + assert_eq!(retry_id, job_id); + + let config = JobsConfig { + worker_id: "example-test-worker".into(), + poll_interval: Duration::from_millis(25), + claim_batch_size: 1, + lease_ttl_seconds: 30, + max_global_concurrency: 1, + reaper_interval: Duration::from_secs(1), + schedule_poll_interval: Duration::from_secs(1), + reaper_retry_delay_ms: 100, + }; + let supervisor = Supervisor::builder(&pool, config) + .expect("builder") + .with_catalog(&catalog) + .build() + .expect("supervisor"); + let (stop_tx, stop_rx) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(supervisor.run_until_shutdown( + async { + let _ = stop_rx.await; + }, + Duration::from_secs(10), + )); + let completed = tokio::time::timeout(Duration::from_secs(20), async { + loop { + let job = get_job_by_id_with_scope(&pool, JobReadScope::Global, job_id) + .await + .expect("read job") + .expect("committed job"); + if job.status == JobStatus::Succeeded { + break job; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await; + stop_tx.send(()).expect("request shutdown"); + task.await + .expect("supervisor task") + .expect("graceful shutdown"); + let job = completed.expect("job completes"); + assert_eq!(job.payload, payload); + assert_eq!(job.progress_done, Some(1)); + assert_eq!(job.progress_total, Some(1)); + teardown_ephemeral_pool(pool, database).await; +} diff --git a/runledger-runtime/examples/producer_worker/worker.rs b/runledger-runtime/examples/producer_worker/worker.rs new file mode 100644 index 0000000..1366905 --- /dev/null +++ b/runledger-runtime/examples/producer_worker/worker.rs @@ -0,0 +1,65 @@ +pub mod shared; + +use std::time::Duration; + +use runledger_core::jobs::{JobCompletion, JobContext, JobFailure, JobType}; +use runledger_core::prelude::async_trait; +use runledger_runtime::{Supervisor, catalog::JobCatalog, registry::JobHandler}; +use serde_json::Value; +use shared::{GREETING_JOB, Greeting}; +use sqlx::postgres::PgPoolOptions; + +struct PrintGreeting; + +#[async_trait] +impl JobHandler for PrintGreeting { + fn job_type(&self) -> JobType<'static> { + GREETING_JOB + } + + async fn execute( + &self, + _context: JobContext, + payload: Value, + ) -> Result { + let greeting: Greeting = serde_json::from_value(payload) + .map_err(|_| JobFailure::terminal("greeting.invalid_payload", "Expected a name."))?; + println!("Hello, {}!", greeting.name); + JobCompletion::success().progress(1, 1).map_err(|_| { + JobFailure::terminal("greeting.invalid_progress", "Invalid completion counts.") + }) + } +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let pool = PgPoolOptions::new() + .connect(&std::env::var("DATABASE_URL")?) + .await?; + // For a fresh database. Existing deployments must follow the migration runbook. + runledger_postgres::migrate_after_idempotency_cutover(&pool).await?; + let catalog = JobCatalog::new().handler(PrintGreeting); + catalog.sync_definitions(&pool).await?; + println!("worker ready; producers can now enqueue greetings"); + + let supervisor = Supervisor::builder_from_env(&pool)? + .with_catalog(&catalog) + .build()?; + let shutdown_result = supervisor + .run_until_shutdown( + async { + if let Err(error) = tokio::signal::ctrl_c().await { + eprintln!("failed to listen for shutdown signal: {error}"); + } + }, + Duration::from_secs(30), + ) + .await; + pool.close().await; + shutdown_result?; + Ok(()) +} + +#[cfg(test)] +#[path = "tests.rs"] +mod tests; diff --git a/runledger-runtime/src/catalog.rs b/runledger-runtime/src/catalog.rs index a4ba657..33aa5a3 100644 --- a/runledger-runtime/src/catalog.rs +++ b/runledger-runtime/src/catalog.rs @@ -41,6 +41,7 @@ //! and scope for [`JobCatalog::sync_schedules_exact_with`]; Runledger does not //! provide an implicit union helper because that can hide ownership mistakes. +mod bindings; mod error; mod inputs; mod registration; @@ -50,6 +51,7 @@ mod sync_schedules; mod types; mod workflow; +pub use bindings::JobBindingError; pub use error::CatalogError; pub use inputs::{CatalogJobEnqueueInput, CatalogJobScheduleInput}; pub use schedule_spec::CatalogJobScheduleSpec; diff --git a/runledger-runtime/src/catalog/bindings.rs b/runledger-runtime/src/catalog/bindings.rs new file mode 100644 index 0000000..d2a1ce3 --- /dev/null +++ b/runledger-runtime/src/catalog/bindings.rs @@ -0,0 +1,100 @@ +use std::{collections::BTreeSet, sync::Arc}; + +use runledger_core::jobs::{JobHandler, JobSpec, JobSpecs}; + +use super::{CatalogError, JobCatalog, JobCatalogDefinitionOverrides}; + +/// Worker startup failure when bindings do not match the shared specifications. +#[derive(Debug)] +pub enum JobBindingError { + Catalog(CatalogError), + MissingHandler(String), + UnknownHandler(String), +} + +impl std::fmt::Display for JobBindingError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Catalog(error) => error.fmt(f), + Self::MissingHandler(name) => { + write!(f, "missing handler for job specification: {name}") + } + Self::UnknownHandler(name) => write!(f, "handler has no job specification: {name}"), + } + } +} + +impl std::error::Error for JobBindingError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Catalog(error) => Some(error), + _ => None, + } + } +} + +impl JobCatalog { + /// Adds a legacy or typed-adapted handler using shared operational metadata. + pub fn try_handler_for_spec( + self, + spec: &JobSpec, + handler: H, + ) -> Result { + self.insert_spec_handler(spec, Arc::new(handler)) + } + + fn insert_spec_handler( + self, + spec: &JobSpec, + handler: Arc, + ) -> Result { + let job_type = spec.job_type(); + let handler_type = handler.job_type(); + if handler_type != job_type { + return Err(CatalogError::HandlerJobTypeMismatch { + declared: job_type.as_str().to_owned(), + handler: handler_type.as_str().to_owned(), + }); + } + let settings = spec.settings(); + self.insert_handler(job_type, handler)? + .try_definition_overrides( + job_type.as_str(), + JobCatalogDefinitionOverrides::new() + .version(settings.version) + .max_attempts(settings.max_attempts) + .timeout_seconds(settings.default_timeout_seconds) + .priority(settings.default_priority) + .enabled(settings.is_enabled), + ) + } + + /// Binds exactly one handler per shared spec. Disabled specs still require + /// handlers to service already queued work and terminal cleanup. + /// No partially bound catalog is returned on missing or duplicate handlers. + pub fn from_specs( + specs: &JobSpecs, + handlers: impl IntoIterator>, + ) -> Result { + let mut catalog = Self::new(); + let mut bound = BTreeSet::new(); + for handler in handlers { + let job_type = handler.job_type(); + let spec = specs + .get(job_type) + .ok_or_else(|| JobBindingError::UnknownHandler(job_type.as_str().to_owned()))?; + catalog = catalog + .insert_spec_handler(spec, handler) + .map_err(JobBindingError::Catalog)?; + bound.insert(job_type); + } + for spec in specs.iter() { + if !bound.contains(&spec.job_type()) { + return Err(JobBindingError::MissingHandler( + spec.job_type().as_str().to_owned(), + )); + } + } + Ok(catalog) + } +} diff --git a/runledger-runtime/src/catalog/registration.rs b/runledger-runtime/src/catalog/registration.rs index 38d7584..e695234 100644 --- a/runledger-runtime/src/catalog/registration.rs +++ b/runledger-runtime/src/catalog/registration.rs @@ -473,7 +473,7 @@ impl JobCatalog { }) } - fn insert_handler( + pub(super) fn insert_handler( mut self, job_type: JobType<'static>, handler: Arc, diff --git a/runledger-runtime/src/catalog/types.rs b/runledger-runtime/src/catalog/types.rs index 06556de..ab3f239 100644 --- a/runledger-runtime/src/catalog/types.rs +++ b/runledger-runtime/src/catalog/types.rs @@ -6,90 +6,7 @@ use runledger_core::jobs::{JobHandler, JobType, JobTypeName}; use super::CatalogError; -/// Default values applied when syncing catalog jobs to `job_definitions`. -#[non_exhaustive] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct JobCatalogDefaults { - /// Definition version written for catalog jobs. - pub version: i32, - /// Default maximum attempts written for catalog jobs. - pub max_attempts: i32, - /// Default execution timeout, in seconds, written for catalog jobs. - pub default_timeout_seconds: i32, - /// Default queue priority written for catalog jobs. - pub default_priority: i32, - /// Whether catalog jobs should be synced as enabled. - pub is_enabled: bool, -} - -impl Default for JobCatalogDefaults { - fn default() -> Self { - Self { - version: 1, - max_attempts: 3, - default_timeout_seconds: 300, - default_priority: 0, - is_enabled: true, - } - } -} - -impl JobCatalogDefaults { - /// Creates the default catalog definition values. - #[must_use] - pub fn new() -> Self { - Self::default() - } - - /// Sets the definition version written for catalog jobs. - #[must_use] - pub fn version(mut self, version: i32) -> Self { - self.version = version; - self - } - - /// Sets the default maximum attempts written for catalog jobs. - #[must_use] - pub fn max_attempts(mut self, max_attempts: i32) -> Self { - self.max_attempts = max_attempts; - self - } - - /// Sets the default execution timeout, in seconds, written for catalog jobs. - #[must_use] - pub fn timeout_seconds(mut self, default_timeout_seconds: i32) -> Self { - self.default_timeout_seconds = default_timeout_seconds; - self - } - - /// Sets the default queue priority written for catalog jobs. - #[must_use] - pub fn priority(mut self, default_priority: i32) -> Self { - self.default_priority = default_priority; - self - } - - /// Sets whether catalog jobs should be synced as enabled. - #[must_use] - pub fn enabled(mut self, is_enabled: bool) -> Self { - self.is_enabled = is_enabled; - self - } - - pub(super) fn validate(self) -> Result<(), &'static str> { - if self.version <= 0 { - return Err("version"); - } - if self.max_attempts <= 0 { - return Err("max_attempts"); - } - if self.default_timeout_seconds <= 0 { - return Err("default_timeout_seconds"); - } - // default_priority intentionally accepts zero and negative values. - Ok(()) - } -} +pub use runledger_core::jobs::JobDefinitionSettings as JobCatalogDefaults; /// Per-job definition values that override [`JobCatalogDefaults`]. #[non_exhaustive] @@ -167,15 +84,15 @@ impl JobCatalogDefinitionOverrides { } pub(super) fn apply_to(self, defaults: JobCatalogDefaults) -> JobCatalogDefaults { - JobCatalogDefaults { - version: self.version.unwrap_or(defaults.version), - max_attempts: self.max_attempts.unwrap_or(defaults.max_attempts), - default_timeout_seconds: self - .default_timeout_seconds - .unwrap_or(defaults.default_timeout_seconds), - default_priority: self.default_priority.unwrap_or(defaults.default_priority), - is_enabled: self.is_enabled.unwrap_or(defaults.is_enabled), - } + defaults + .version(self.version.unwrap_or(defaults.version)) + .max_attempts(self.max_attempts.unwrap_or(defaults.max_attempts)) + .timeout_seconds( + self.default_timeout_seconds + .unwrap_or(defaults.default_timeout_seconds), + ) + .priority(self.default_priority.unwrap_or(defaults.default_priority)) + .enabled(self.is_enabled.unwrap_or(defaults.is_enabled)) } } diff --git a/runledger-runtime/src/catalog/workflow.rs b/runledger-runtime/src/catalog/workflow.rs index b11fa23..eb2d4af 100644 --- a/runledger-runtime/src/catalog/workflow.rs +++ b/runledger-runtime/src/catalog/workflow.rs @@ -1,10 +1,32 @@ -use runledger_core::jobs::{WorkflowDagBuilder, WorkflowRunEnqueue}; +use runledger_core::jobs::{WorkflowDagBuilder, WorkflowRunEnqueue, WorkflowStepEnqueue}; use serde_json::Value; use uuid::Uuid; use super::{CatalogError, JobCatalog}; /// Workflow DAG builder that validates job types against a [`JobCatalog`]. +/// +/// # Examples +/// ```rust +/// use runledger_core::jobs::WorkflowStepEnqueueBuilder; +/// use runledger_runtime::catalog::JobCatalog; +/// use uuid::Uuid; +/// +/// let catalog = JobCatalog::new(); +/// let payload = serde_json::json!({"ticket": "review"}); +/// let approval = WorkflowStepEnqueueBuilder::try_new_external("approval", &payload)? +/// .organization_id(Uuid::nil()) +/// .try_build()?; +/// let run = catalog.workflow_dag("review", &payload) +/// .active_key("review:active") +/// .step(approval)? +/// .external("receipt", &payload)? +/// .after_success("receipt", ["approval"])? +/// .result_step("receipt")? +/// .build()?; +/// assert_eq!(run.steps().len(), 2); +/// # Ok::<_, runledger_runtime::catalog::CatalogError>(()) +/// ``` #[derive(Debug, Clone)] pub struct CatalogWorkflowDagBuilder<'a, 'catalog> { pub(super) catalog: &'catalog JobCatalog, @@ -55,6 +77,24 @@ impl<'a, 'catalog> CatalogWorkflowDagBuilder<'a, 'catalog> { self } + /// Sets a reusable active-cycle key, independent of request idempotency. + /// + /// Shared across workflow types in the same organization/global scope. + /// Checked for non-blank content and a maximum of 512 bytes at build time. + /// Use the request with `enqueue_or_get_active_workflow`. + #[must_use] + pub fn active_key(mut self, active_key: &'a str) -> Self { + self.inner = self.inner.active_key(active_key); + self + } + + /// Clears the reusable active workflow key. + #[must_use] + pub fn clear_active_key(mut self) -> Self { + self.inner = self.inner.clear_active_key(); + self + } + /// Declares the step whose successful output becomes the workflow result. /// /// # Errors @@ -95,6 +135,36 @@ impl<'a, 'catalog> CatalogWorkflowDagBuilder<'a, 'catalog> { Ok(self) } + /// Adds a configured step, checking job steps against this catalog. + /// + /// Use [`JobCatalog::workflow_step`] or the core step builder to configure + /// step policies, then pass its `try_build()` result here. Even steps from + /// another catalog must be enabled in this catalog. External steps require + /// no job registration. All settings and dependencies are preserved. + /// + /// # Errors + /// Returns [`CatalogError`] for unknown/disabled job types or duplicate keys. + pub fn step(mut self, step: WorkflowStepEnqueue<'a>) -> Result { + if let Some(job_type) = step.job_type() { + self.catalog + .require_catalog_enabled_job_type(job_type.as_str())?; + } + self.inner = self.inner.step(step).map_err(CatalogError::WorkflowBuild)?; + Ok(self) + } + + /// Adds a step completed by an external actor; no catalog job is required. + /// + /// # Errors + /// Returns [`CatalogError::WorkflowBuild`] for a blank or duplicate key. + pub fn external(mut self, step_key: &'a str, payload: &'a Value) -> Result { + self.inner = self + .inner + .external(step_key, payload) + .map_err(CatalogError::WorkflowBuild)?; + Ok(self) + } + /// Adds success dependencies to an existing workflow step. /// /// # Errors diff --git a/runledger-runtime/src/worker.rs b/runledger-runtime/src/worker.rs index 62419bf..85dd972 100644 --- a/runledger-runtime/src/worker.rs +++ b/runledger-runtime/src/worker.rs @@ -10,6 +10,7 @@ use tracing::{error, info, warn}; mod completion; mod dead_letter; mod execution; +mod execution_services; mod observers; use self::execution::ClaimedJobExecution; diff --git a/runledger-runtime/src/worker/execution.rs b/runledger-runtime/src/worker/execution.rs index 8588389..7a87265 100644 --- a/runledger-runtime/src/worker/execution.rs +++ b/runledger-runtime/src/worker/execution.rs @@ -3,7 +3,7 @@ use std::panic::AssertUnwindSafe; use std::sync::Arc; use futures_util::{FutureExt, StreamExt, stream::FuturesUnordered}; -use runledger_core::jobs::{JobCompletion, JobContext, JobFailure}; +use runledger_core::jobs::{JobCompletion, JobContext, JobExecution, JobFailure}; use runledger_postgres::QueryErrorKind; use runledger_postgres::jobs::{self, JobLeaseIdentity, JobRunningUpdate}; use tokio::time::{Duration, Instant, MissedTickBehavior, sleep_until}; @@ -13,6 +13,7 @@ use super::completion::{ CompletionContext, CompletionObservation, complete_job_after_handler, complete_job_failure_after_handler, }; +use super::execution_services::LeaseExecutionServices; use super::observers::{JobRunningNotification, TerminalJobObserverEvent, TerminalObserverTasks}; use crate::WorkerError; use crate::observer::{JobLeaseLostEvent, JobLifecycleObservers, ObservedJob}; @@ -286,11 +287,18 @@ impl ClaimedJobExecution { context: &JobContext, ) -> Result { let registry = Arc::clone(&self.registry); - let mut execution = Box::pin( - AssertUnwindSafe(execute_job_handler(registry, context, &self.job)).catch_unwind(), - ); let timeout_deadline = Instant::now() + Duration::from_secs(self.job.timeout_seconds.max(1) as u64); + let services = + LeaseExecutionServices::new(&self.pool, self.lease_identity(), timeout_deadline); + let mut execution = Box::pin( + AssertUnwindSafe(execute_job_handler( + registry, + JobExecution::new(context, &services), + &self.job, + )) + .catch_unwind(), + ); let mut timeout = Box::pin(sleep_until(timeout_deadline)); let heartbeat_budget = heartbeat_maintenance_budget(self.lease_ttl_seconds); @@ -302,7 +310,13 @@ impl ClaimedJobExecution { loop { tokio::select! { result = &mut execution => { - return map_handler_join(result); + if services.lease_was_lost() { + return Err(JobExecutionFailure::LeaseMaintenance(lease_owner_mismatch_failure())); + } + return map_handler_join(result, timeout_deadline); + } + _ = services.wait_for_lease_loss() => { + return Err(JobExecutionFailure::LeaseMaintenance(lease_owner_mismatch_failure())); } _ = &mut timeout => { return Err(JobExecutionFailure::Handler(JobFailure::timeout( @@ -379,7 +393,16 @@ impl ClaimedJobExecution { fn map_handler_join( result: Result, Box>, + timeout_deadline: Instant, ) -> Result { + // Deadline precedence is independent of select!'s randomized polling order, + // including a non-yielding handler that returns after crossing the cutoff. + if Instant::now() >= timeout_deadline { + return Err(JobExecutionFailure::Handler(JobFailure::timeout( + "job.timeout_exceeded", + "Job exceeded the configured timeout.", + ))); + } match result { Ok(result) => result.map_err(JobExecutionFailure::Handler), Err(panic_payload) => Err(JobExecutionFailure::Handler(handler_panic_failure( @@ -489,7 +512,7 @@ async fn heartbeat_within_maintenance_budget( async fn execute_job_handler( registry: Arc, - context: &JobContext, + execution: JobExecution<'_>, job: &jobs::JobQueueRecord, ) -> Result { let Some(handler) = registry.get(job.job_type.as_borrowed()) else { @@ -499,7 +522,9 @@ async fn execute_job_handler( )); }; - handler.execute(context.clone(), job.payload.clone()).await + handler + .execute_with_services(execution, job.payload.clone()) + .await } pub(super) fn lease_owner_mismatch_failure() -> JobFailure { @@ -564,3 +589,36 @@ mod tests { assert_eq!(heartbeat_maintenance_budget(60), Duration::from_secs(20)); } } + +#[cfg(test)] +mod deadline_tests { + use super::*; + + #[tokio::test(start_paused = true)] + async fn success_and_continuation_must_be_observed_strictly_before_the_deadline() { + let deadline = Instant::now() + Duration::from_secs(1); + for completion in [ + JobCompletion::with_output(serde_json::json!({"result": "kept"})), + JobCompletion::continue_after(Duration::from_secs(7)), + ] { + let Ok(accepted) = map_handler_join(Ok(Ok(completion.clone())), deadline) else { + panic!("result before the cutoff must be accepted"); + }; + assert_eq!(accepted.disposition(), completion.disposition()); + assert_eq!(accepted.output(), completion.output()); + } + tokio::time::advance(Duration::from_secs(1)).await; + assert_eq!(Instant::now(), deadline); + for lateness in [Duration::ZERO, Duration::from_nanos(1)] { + tokio::time::advance(lateness).await; + for completion in [JobCompletion::success(), JobCompletion::continue_now()] { + let Err(JobExecutionFailure::Handler(failure)) = + map_handler_join(Ok(Ok(completion)), deadline) + else { + panic!("completion observed at or after the cutoff must time out"); + }; + assert_eq!(failure.code, "job.timeout_exceeded"); + } + } + } +} diff --git a/runledger-runtime/src/worker/execution_services.rs b/runledger-runtime/src/worker/execution_services.rs new file mode 100644 index 0000000..fed78bd --- /dev/null +++ b/runledger-runtime/src/worker/execution_services.rs @@ -0,0 +1,101 @@ +use std::sync::atomic::{AtomicBool, Ordering}; + +use async_trait::async_trait; +use runledger_core::jobs::{JobExecutionError, JobExecutionServices, JobExecutionUpdate}; +use runledger_postgres::jobs::{ + JobLeaseIdentity, JobOrdinaryProgressUpdate, update_job_ordinary_progress_for_lease, +}; +use runledger_postgres::{DbPool, Error}; +use tokio::sync::Notify; +use tokio::time::{Duration, Instant, timeout_at}; + +use super::execution::is_lease_owner_mismatch_error; + +/// Borrowed only for the worker-owned handler future; it cannot outlive it. +pub(super) struct LeaseExecutionServices<'a> { + pool: &'a DbPool, + identity: JobLeaseIdentity<'a>, + deadline: Instant, + lease_lost: AtomicBool, + lease_lost_notification: Notify, +} + +impl<'a> LeaseExecutionServices<'a> { + pub(super) fn new(pool: &'a DbPool, identity: JobLeaseIdentity<'a>, deadline: Instant) -> Self { + Self { + pool, + identity, + deadline, + lease_lost: AtomicBool::new(false), + lease_lost_notification: Notify::new(), + } + } + + pub(super) fn lease_was_lost(&self) -> bool { + self.lease_lost.load(Ordering::Acquire) + } + + pub(super) async fn wait_for_lease_loss(&self) { + self.lease_lost_notification.notified().await; + } +} + +#[async_trait] +impl JobExecutionServices for LeaseExecutionServices<'_> { + fn deadline(&self) -> std::time::Instant { + self.deadline.into_std() + } + + fn remaining_budget(&self) -> Duration { + self.deadline.saturating_duration_since(Instant::now()) + } + + async fn persist_progress( + &self, + update: JobExecutionUpdate<'_>, + ) -> Result<(), JobExecutionError> { + if self.lease_was_lost() { + return Err(JobExecutionError::LeaseLost); + } + if Instant::now() >= self.deadline { + return Err(JobExecutionError::DeadlineElapsed); + } + let result = timeout_at( + self.deadline, + update_job_ordinary_progress_for_lease( + self.pool, + self.identity, + &JobOrdinaryProgressUpdate { + progress_done: update.progress_done, + progress_total: update.progress_total, + checkpoint: update.checkpoint, + }, + ), + ) + .await; + match result { + Ok(Ok(())) => Ok(()), + Err(_) => Err(JobExecutionError::DeadlineElapsed), + Ok(Err(error)) if is_lease_owner_mismatch_error(&error) => { + self.lease_lost.store(true, Ordering::Release); + self.lease_lost_notification.notify_one(); + Err(JobExecutionError::LeaseLost) + } + Ok(Err(error)) => { + if let Error::QueryError(query_error) = &error + && let Some(progress_error) = query_error.progress_validation_error() + { + return Err(JobExecutionError::InvalidProgress(progress_error)); + } + tracing::warn!( + job_id = %self.identity.job_id, + run_number = self.identity.run_number, + attempt = self.identity.attempt, + %error, + "handler progress persistence failed" + ); + Err(JobExecutionError::PersistenceFailed) + } + } + } +} diff --git a/runledger-runtime/src/worker/tests/execution_services.rs b/runledger-runtime/src/worker/tests/execution_services.rs new file mode 100644 index 0000000..0853efc --- /dev/null +++ b/runledger-runtime/src/worker/tests/execution_services.rs @@ -0,0 +1,505 @@ +use runledger_core::jobs::{ + JobExecution, JobExecutionError, JobExecutionHandler, JobExecutionUpdate, +}; + +use super::*; + +const JOB_TYPE: JobType<'static> = JobType::new("jobs.test.execution_services"); + +struct CheckpointHandler { + committed: Arc, + finish: Arc, + pending: bool, + dead_checkpoint: Arc>>, +} + +#[async_trait::async_trait] +impl JobExecutionHandler for CheckpointHandler { + fn job_type(&self) -> JobType<'static> { + JOB_TYPE + } + + async fn execute( + &self, + execution: JobExecution<'_>, + _payload: Value, + ) -> Result { + if execution + .checkpoint::() + .expect("decode durable cursor") + == Some(1) + { + return Ok(JobCompletion::success()); + } + let configured_timeout = Duration::from_secs(if self.pending { 1 } else { 30 }); + assert!(execution.remaining_budget() <= configured_timeout); + assert!( + execution + .deadline() + .saturating_duration_since(std::time::Instant::now()) + <= configured_timeout + ); + assert!(matches!( + execution + .persist_progress(JobExecutionUpdate { + progress_done: Some(-1), + ..Default::default() + }) + .await, + Err(JobExecutionError::InvalidProgress(_)) + )); + let before = execution.remaining_budget(); + let deadline = execution.deadline(); + assert!(before > Duration::ZERO); + assert_eq!( + execution.remaining_work_budget(Duration::from_secs(60)), + Duration::ZERO + ); + let checkpoint = json!(1); + execution + .persist_progress(JobExecutionUpdate { + progress_done: Some(1), + progress_total: Some(3), + checkpoint: Some(&checkpoint), + }) + .await?; + assert_eq!( + execution.deadline(), + deadline, + "writes cannot reset the deadline" + ); + assert!(execution.remaining_budget() < before); + assert_eq!( + execution.checkpoint::().expect("initial snapshot"), + None + ); + self.committed.notify_one(); + if self.pending { + pending::<()>().await; + } + self.finish.notified().await; + Ok(JobCompletion::continue_now()) + } + + async fn on_dead_letter( + &self, + context: JobContext, + _payload: Value, + _dead_letter: JobDeadLetterInfo, + ) { + *self + .dead_checkpoint + .lock() + .expect("dead-letter checkpoint lock") = context.checkpoint; + } +} + +fn checkpoint_handler(pending: bool) -> CheckpointHandler { + CheckpointHandler { + committed: Arc::new(Notify::new()), + finish: Arc::new(Notify::new()), + pending, + dead_checkpoint: Arc::new(Mutex::new(None)), + } +} + +#[tokio::test] +async fn awaited_progress_commits_before_return_and_continuation_resumes_checkpoint() { + let (pool, database) = setup_ephemeral_pool("execution_services_checkpoint", 4).await; + record_postgres_server_version(&pool, "execution-service checkpoint regression").await; + let (job_id, job) = + enqueue_and_claim_job(&pool, JOB_TYPE, 3, json!({}), "services-worker").await; + let handler = checkpoint_handler(false); + let committed = handler.committed.clone(); + let finish = handler.finish.clone(); + let mut registry = JobRegistry::new(); + registry + .try_register(handler.into_job_handler()) + .expect("register opt-in handler"); + let registry = Arc::new(registry); + let mut task = tokio::spawn(process_claimed_job(pool.clone(), registry.clone(), job, 30)); + timeout(Duration::from_secs(3), committed.notified()) + .await + .expect("progress acknowledged"); + let saved = get_job_by_id(&pool, None, job_id) + .await + .expect("read durable job") + .expect("job"); + assert_eq!(saved.status, JobStatus::Leased); + assert_eq!(saved.checkpoint, Some(json!(1))); + assert_eq!( + (saved.progress_done, saved.progress_total), + (Some(1), Some(3)) + ); + assert!( + list_job_events(&pool, None, job_id, 100, None) + .await + .expect("events") + .iter() + .any(|event| event.event_type == JobEventType::Progress + && event.progress_done == Some(1)) + ); + finish.notify_one(); + await_spawned_task( + &mut task, + Duration::from_secs(3), + "continuation completes", + "worker joins", + ) + .await; + let next = claim_prestart_jobs(&pool, "services-worker-next", 30, 1) + .await + .expect("claim next run") + .pop() + .expect("continued job"); + assert_eq!(next.run_number, 2); + process_claimed_job(pool.clone(), registry, next, 30).await; + let done = get_job_by_id(&pool, None, job_id) + .await + .expect("read result") + .expect("job"); + assert_eq!(done.status, JobStatus::Succeeded); + assert_eq!(done.checkpoint, Some(json!(1))); + teardown_ephemeral_pool(pool, database).await; +} + +#[tokio::test] +async fn runtime_deadline_cancels_handler_and_dead_letter_gets_committed_checkpoint() { + let (pool, database) = setup_ephemeral_pool("execution_services_timeout", 4).await; + record_postgres_server_version(&pool, "execution-service timeout regression").await; + let (job_id, mut job) = + enqueue_and_claim_job(&pool, JOB_TYPE, 1, json!({}), "services-worker").await; + sqlx::query("UPDATE job_queue SET timeout_seconds = 1 WHERE id = $1") + .bind(job_id) + .execute(&pool) + .await + .expect("set short timeout"); + job.timeout_seconds = 1; + let handler = checkpoint_handler(true); + let committed = handler.committed.clone(); + let dead_checkpoint = handler.dead_checkpoint.clone(); + let mut registry = JobRegistry::new(); + registry.register(handler.into_job_handler()); + let started = Instant::now(); + let mut task = tokio::spawn(process_claimed_job( + pool.clone(), + Arc::new(registry), + job, + 30, + )); + timeout(Duration::from_secs(3), committed.notified()) + .await + .expect("progress acknowledged"); + await_spawned_task( + &mut task, + Duration::from_secs(3), + "deadline terminates execution", + "worker joins", + ) + .await; + assert!(started.elapsed() >= Duration::from_secs(1)); + let saved = get_job_by_id(&pool, None, job_id) + .await + .expect("read timed-out job") + .expect("job"); + assert_eq!(saved.status, JobStatus::DeadLettered); + assert_eq!( + saved.last_error_code.as_deref(), + Some("job.timeout_exceeded") + ); + assert_eq!(saved.checkpoint, Some(json!(1))); + assert_eq!( + *dead_checkpoint.lock().expect("hook checkpoint"), + Some(json!(1)) + ); + teardown_ephemeral_pool(pool, database).await; +} + +struct ControlledWriteHandler { + started: Arc, + write: Arc, + outcome: Arc>>, + returned: Arc, +} + +#[async_trait::async_trait] +impl JobExecutionHandler for ControlledWriteHandler { + fn job_type(&self) -> JobType<'static> { + JOB_TYPE + } + async fn execute( + &self, + execution: JobExecution<'_>, + _payload: Value, + ) -> Result { + self.started.notify_one(); + self.write.notified().await; + let result = execution.save_checkpoint(&7_u64).await; + let code = match result { + Ok(()) => "committed", + Err(JobExecutionError::LeaseLost) => "lease_lost", + Err(JobExecutionError::PersistenceFailed) => "persistence_failed", + Err(JobExecutionError::DeadlineElapsed) => "deadline_elapsed", + Err(error) => panic!("unexpected progress error: {error}"), + }; + *self.outcome.lock().expect("write outcome") = Some(code); + self.returned.notify_one(); + // Deliberately swallow the failure: the runtime must still stop a handler + // when its progress operation discovers lease loss. + pending::<()>().await; + Ok(JobCompletion::success()) + } +} + +fn controlled_handler() -> ControlledWriteHandler { + ControlledWriteHandler { + started: Arc::new(Notify::new()), + write: Arc::new(Notify::new()), + outcome: Arc::new(Mutex::new(None)), + returned: Arc::new(Notify::new()), + } +} + +#[tokio::test] +async fn progress_detects_expired_and_replaced_leases_and_aborts_even_if_error_is_swallowed() { + let (pool, database) = setup_ephemeral_pool("execution_services_lease_loss", 4).await; + record_postgres_server_version(&pool, "execution-service lease-loss regression").await; + for mutation in [ + "lease_expires_at = clock_timestamp() - interval '1 second'", + "worker_id = 'replacement-worker'", + "attempt = attempt + 1", + "run_number = run_number + 1", + ] { + let (job_id, job) = + enqueue_and_claim_job(&pool, JOB_TYPE, 3, json!({}), "services-worker").await; + let handler = controlled_handler(); + let started = handler.started.clone(); + let write = handler.write.clone(); + let outcome = handler.outcome.clone(); + let observer = RecordingObserver::default(); + let mut registry = JobRegistry::new(); + registry.register(handler.into_job_handler()); + let mut task = tokio::spawn(process_claimed_job_with_observer( + pool.clone(), + Arc::new(registry), + job, + 30, + observer.lifecycle_observers(), + )); + timeout(Duration::from_secs(3), started.notified()) + .await + .expect("handler starts"); + sqlx::query(&format!("UPDATE job_queue SET {mutation} WHERE id = $1")) + .bind(job_id) + .execute(&pool) + .await + .expect("invalidate lease"); + write.notify_one(); + await_spawned_task( + &mut task, + Duration::from_secs(3), + "progress lease loss aborts", + "worker joins", + ) + .await; + assert_eq!(*outcome.lock().expect("write outcome"), Some("lease_lost")); + wait_for_observer_count(|| observer.lease_lost().len(), 1, Duration::from_secs(2)).await; + let saved = get_job_by_id(&pool, None, job_id) + .await + .expect("read abandoned job") + .expect("job"); + assert_eq!(saved.checkpoint, None); + assert_eq!(saved.status, JobStatus::Leased); + assert!(saved.finished_at.is_none()); + } + teardown_ephemeral_pool(pool, database).await; +} + +#[tokio::test] +async fn failed_progress_transaction_does_not_acknowledge_or_persist_checkpoint() { + let (pool, database) = setup_ephemeral_pool("execution_services_failure", 4).await; + let (job_id, mut job) = + enqueue_and_claim_job(&pool, JOB_TYPE, 1, json!({}), "services-worker").await; + job.timeout_seconds = 1; + let handler = controlled_handler(); + let started = handler.started.clone(); + let write = handler.write.clone(); + let returned = handler.returned.clone(); + let outcome = handler.outcome.clone(); + let mut registry = JobRegistry::new(); + registry.register(handler.into_job_handler()); + let mut task = tokio::spawn(process_claimed_job( + pool.clone(), + Arc::new(registry), + job, + 30, + )); + timeout(Duration::from_secs(3), started.notified()) + .await + .expect("handler starts"); + sqlx::raw_sql("CREATE FUNCTION reject_checkpoint() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.checkpoint IS NOT NULL THEN RAISE EXCEPTION 'injected checkpoint failure'; END IF; RETURN NEW; END $$; + CREATE TRIGGER reject_checkpoint BEFORE UPDATE ON job_queue FOR EACH ROW EXECUTE FUNCTION reject_checkpoint();") + .execute(&pool).await.expect("inject persistence failure"); + write.notify_one(); + timeout(Duration::from_secs(3), returned.notified()) + .await + .expect("write returns error"); + assert_eq!( + *outcome.lock().expect("outcome"), + Some("persistence_failed") + ); + let saved = get_job_by_id(&pool, None, job_id) + .await + .expect("read after failed update") + .expect("job"); + assert_eq!(saved.checkpoint, None); + await_spawned_task( + &mut task, + Duration::from_secs(3), + "timeout completes", + "worker joins", + ) + .await; + teardown_ephemeral_pool(pool, database).await; +} + +#[tokio::test] +async fn handler_deadline_bounds_a_progress_write_waiting_for_a_row_lock() { + for connections in [1, 2, 4] { + assert_deadline_bounds_blocked_progress(connections).await; + } +} + +async fn assert_deadline_bounds_blocked_progress(connections: u32) { + let (pool, database) = + setup_ephemeral_pool("execution_services_blocked_write", connections).await; + record_postgres_server_version(&pool, "blocked progress with a small pool").await; + // An independent connection holds the contended row. Even a one-connection + // worker pool must reclaim its cancelled progress transaction for completion. + let lock_pool = PgPoolOptions::new() + .max_connections(1) + .connect_with(pool.connect_options().as_ref().clone()) + .await + .expect("lock pool"); + let (job_id, mut job) = + enqueue_and_claim_job(&pool, JOB_TYPE, 1, json!({}), "services-worker").await; + job.timeout_seconds = 1; + let handler = controlled_handler(); + let started = handler.started.clone(); + let write = handler.write.clone(); + let outcome = handler.outcome.clone(); + let mut registry = JobRegistry::new(); + registry.register(handler.into_job_handler()); + let mut task = tokio::spawn(process_claimed_job( + pool.clone(), + Arc::new(registry), + job, + 30, + )); + timeout(Duration::from_secs(3), started.notified()) + .await + .expect("handler starts"); + let mut tx = lock_pool.begin().await.expect("begin row lock"); + sqlx::query("SELECT id FROM job_queue WHERE id = $1 FOR UPDATE") + .bind(job_id) + .fetch_one(&mut *tx) + .await + .expect("hold row lock"); + write.notify_one(); + sleep(Duration::from_millis(1200)).await; + assert_ne!( + *outcome.lock().expect("unacknowledged write"), + Some("committed") + ); + tx.rollback() + .await + .expect("release row lock after deadline"); + await_spawned_task( + &mut task, + Duration::from_secs(4), + "timeout completes after releasing row lock", + "worker joins", + ) + .await; + let saved = get_job_by_id(&pool, None, job_id) + .await + .expect("read timed-out write") + .expect("job"); + assert_eq!( + saved.checkpoint, None, + "cancelled locked update must not commit later" + ); + assert_eq!( + saved.last_error_code.as_deref(), + Some("job.timeout_exceeded") + ); + lock_pool.close().await; + teardown_ephemeral_pool(pool, database).await; +} + +struct LateCompletionHandler { + completion: JobCompletion, + returned: Arc, +} + +#[async_trait::async_trait] +impl JobExecutionHandler for LateCompletionHandler { + fn job_type(&self) -> JobType<'static> { + JOB_TYPE + } + + async fn execute( + &self, + execution: JobExecution<'_>, + _payload: Value, + ) -> Result { + // Deliberately cross the cutoff without yielding. The execution branch + // returns Ready in this poll, so the timer branch cannot enforce policy. + // This models a slow synchronous handler section or delayed worker poll. + while execution.remaining_budget() > Duration::ZERO { + std::thread::sleep(execution.remaining_budget()); + } + self.returned.fetch_add(1, Ordering::SeqCst); + Ok(self.completion.clone()) + } +} + +#[tokio::test] +async fn a_handler_returning_success_or_continuation_after_the_cutoff_is_timed_out() { + let (pool, database) = setup_ephemeral_pool("execution_services_late_result", 1).await; + for completion in [JobCompletion::success(), JobCompletion::continue_now()] { + let (job_id, mut job) = + enqueue_and_claim_job(&pool, JOB_TYPE, 1, json!({}), "late-worker").await; + job.timeout_seconds = 1; + let returned = Arc::new(AtomicUsize::new(0)); + let mut registry = JobRegistry::new(); + registry.register( + LateCompletionHandler { + completion, + returned: returned.clone(), + } + .into_job_handler(), + ); + process_claimed_job(pool.clone(), Arc::new(registry), job, 30).await; + assert_eq!( + returned.load(Ordering::SeqCst), + 1, + "handler actually returned a result" + ); + let saved = get_job_by_id(&pool, None, job_id) + .await + .expect("read") + .expect("job"); + assert_eq!(saved.status, JobStatus::DeadLettered); + assert_eq!( + saved.run_number, 1, + "late continuation must not schedule a run" + ); + assert_eq!( + saved.last_error_code.as_deref(), + Some("job.timeout_exceeded") + ); + } + teardown_ephemeral_pool(pool, database).await; +} diff --git a/runledger-runtime/src/worker/tests/mod.rs b/runledger-runtime/src/worker/tests/mod.rs index 8a4be33..e9d97c5 100644 --- a/runledger-runtime/src/worker/tests/mod.rs +++ b/runledger-runtime/src/worker/tests/mod.rs @@ -48,12 +48,14 @@ use crate::registry::{JobHandler, JobRegistry}; mod capacity; mod completion_and_retry; +mod execution_services; mod heartbeat_progress; mod lease_fencing; mod observer_tasks; mod prestart_recovery; mod support; mod terminal_hooks; +mod typed_execution; struct CountingHandler { runs: Arc, diff --git a/runledger-runtime/src/worker/tests/typed_execution.rs b/runledger-runtime/src/worker/tests/typed_execution.rs new file mode 100644 index 0000000..8caeaf1 --- /dev/null +++ b/runledger-runtime/src/worker/tests/typed_execution.rs @@ -0,0 +1,180 @@ +use runledger_core::jobs::{ + JobContract, JobExecution, JobExecutionUpdate, JobSpec, TypedJobHandler, +}; +use serde::{Deserialize, Serialize}; + +use super::*; + +const JOB_TYPE: JobType<'static> = JobType::new("jobs.test.typed_execution"); + +#[derive(Deserialize, Serialize)] +struct ProgressPayload { + done: Option, + total: Option, +} + +struct ProgressContract; + +impl JobContract for ProgressContract { + type Payload = ProgressPayload; + + fn spec() -> JobSpec { + JobSpec::new(JOB_TYPE).expect("static spec") + } +} + +struct ProgressHandler { + dead_payload: Arc>>, +} + +#[async_trait::async_trait] +impl TypedJobHandler for ProgressHandler { + type Contract = ProgressContract; + + async fn execute( + &self, + _: JobContext, + _: ProgressPayload, + ) -> Result { + panic!("worker must dispatch typed execution with live services") + } + + async fn execute_with_services( + &self, + execution: JobExecution<'_>, + payload: ProgressPayload, + ) -> Result { + assert!(execution.remaining_budget() > Duration::ZERO); + if execution.checkpoint::().expect("resume checkpoint") == Some(2) { + return Ok(JobCompletion::success()); + } + execution + .persist_progress(JobExecutionUpdate { + progress_done: Some(1), + progress_total: Some(3), + checkpoint: Some(&json!(1)), + }) + .await?; + execution + .persist_progress(JobExecutionUpdate { + progress_done: payload.done, + progress_total: payload.total, + checkpoint: Some(&json!(2)), + }) + .await?; + Ok(JobCompletion::continue_now()) + } + + async fn on_dead_letter(&self, _: JobContext, payload: Value, _: JobDeadLetterInfo) { + *self.dead_payload.lock().expect("dead payload") = Some(payload); + } +} + +fn registry(dead_payload: Arc>>) -> Arc { + let mut registry = JobRegistry::new(); + registry.register(ProgressHandler { dead_payload }.into_job_handler()); + Arc::new(registry) +} + +#[tokio::test] +async fn invalid_partial_progress_is_terminal_and_preserves_the_last_commit() { + let (pool, database) = setup_ephemeral_pool("typed_invalid_progress", 4).await; + record_postgres_server_version(&pool, "typed partial progress validation").await; + for payload in [json!({"done":4}), json!({"total":0})] { + let (id, job) = + enqueue_and_claim_job(&pool, JOB_TYPE, 3, payload.clone(), "typed-worker").await; + let dead_payload = Arc::new(Mutex::new(None)); + process_claimed_job(pool.clone(), registry(dead_payload.clone()), job, 30).await; + let saved = get_job_by_id(&pool, None, id) + .await + .expect("read") + .expect("job"); + assert_eq!( + saved.status, + JobStatus::DeadLettered, + "must not retry invalid progress" + ); + assert_eq!(saved.attempt, 1); + assert_eq!( + saved.last_error_code.as_deref(), + Some("job.invalid_progress") + ); + assert_eq!( + (saved.progress_done, saved.progress_total), + (Some(1), Some(3)) + ); + assert_eq!( + saved.checkpoint, + Some(json!(1)), + "rejected checkpoint is atomic with progress" + ); + assert_eq!(*dead_payload.lock().expect("dead payload"), Some(payload)); + let events = list_job_events(&pool, None, id, 100, None) + .await + .expect("events"); + assert_eq!( + events + .iter() + .filter(|event| event.event_type == JobEventType::Progress) + .count(), + 1 + ); + } + teardown_ephemeral_pool(pool, database).await; +} + +#[tokio::test] +async fn typed_worker_persists_partial_progress_and_resumes_a_continuation() { + let (pool, database) = setup_ephemeral_pool("typed_continuation", 4).await; + let (id, job) = + enqueue_and_claim_job(&pool, JOB_TYPE, 3, json!({"done":2}), "typed-worker").await; + let registry = registry(Arc::new(Mutex::new(None))); + process_claimed_job(pool.clone(), registry.clone(), job, 30).await; + let next = claim_prestart_jobs(&pool, "typed-next", 30, 1) + .await + .expect("claim") + .pop() + .expect("continuation"); + assert_eq!(next.run_number, 2); + assert_eq!( + (next.progress_done, next.progress_total), + (Some(2), Some(3)) + ); + assert_eq!(next.checkpoint, Some(json!(2))); + process_claimed_job(pool.clone(), registry, next, 30).await; + assert_eq!( + get_job_by_id(&pool, None, id) + .await + .expect("read") + .expect("job") + .status, + JobStatus::Succeeded + ); + teardown_ephemeral_pool(pool, database).await; +} + +#[tokio::test] +async fn typed_worker_rejects_malformed_payload_before_progress_and_passes_raw_json_to_cleanup() { + let (pool, database) = setup_ephemeral_pool("typed_malformed", 4).await; + let payload = json!({"done":"private-input"}); + let (id, job) = + enqueue_and_claim_job(&pool, JOB_TYPE, 3, payload.clone(), "typed-worker").await; + let dead_payload = Arc::new(Mutex::new(None)); + process_claimed_job(pool.clone(), registry(dead_payload.clone()), job, 30).await; + let saved = get_job_by_id(&pool, None, id) + .await + .expect("read") + .expect("job"); + assert_eq!(saved.status, JobStatus::DeadLettered); + assert_eq!( + saved.last_error_code.as_deref(), + Some("job.invalid_payload") + ); + assert_eq!( + saved.last_error_message.as_deref(), + Some("Job payload has an invalid shape.") + ); + assert_eq!(saved.checkpoint, None); + assert_eq!(*dead_payload.lock().expect("dead payload"), Some(payload)); + teardown_ephemeral_pool(pool, database).await; +} diff --git a/runledger-runtime/tests/fluent_workflow.rs b/runledger-runtime/tests/fluent_workflow.rs new file mode 100644 index 0000000..8cc1a6a --- /dev/null +++ b/runledger-runtime/tests/fluent_workflow.rs @@ -0,0 +1,375 @@ +use runledger_core::jobs::{ + JobCompletion, JobContext, JobFailure, JobStage, JobType, StepKey, WorkflowBuildError, + WorkflowDagBuilder, WorkflowDependencyReleaseMode, WorkflowRunEnqueue, + WorkflowRunEnqueueBuilder, WorkflowStepEnqueue, WorkflowStepEnqueueBuilder, + WorkflowStepExecution, WorkflowType, +}; +use runledger_runtime::catalog::{CatalogError, JobCatalog, JobCatalogDefaults}; +use runledger_runtime::registry::JobHandler; +use serde_json::{Value, json}; +use uuid::Uuid; + +const ENRICH: &str = "buyer.enrich"; + +struct EnrichHandler; + +#[async_trait::async_trait] +impl JobHandler for EnrichHandler { + fn job_type(&self) -> JobType<'static> { + JobType::new(ENRICH) + } + + async fn execute(&self, _: JobContext, _: Value) -> Result { + unreachable!("builder-only fixture must never execute jobs") + } +} + +fn catalog() -> JobCatalog { + JobCatalog::new().handler(EnrichHandler) +} + +// OneSales buyer_enrichment/launcher.rs: account inputs retain dynamic keys and +// payloads, each account has its own tenant, all accounts share one provider +// resource, and each account releases the next on terminal completion. +struct AccountInput { + key: String, + organization: Uuid, + payload: Value, +} + +fn account_step(input: &AccountInput) -> WorkflowStepEnqueueBuilder<'_> { + WorkflowStepEnqueueBuilder::try_new(&input.key, ENRICH, &input.payload) + .expect("account identifiers") + .organization_id(input.organization) + .allow_handler_continuation() + .execution_resource("buyer:provider") +} + +fn low_level_buyer<'a>( + inputs: &'a [AccountInput], + metadata: &'a Value, + organization: Option, + active_key: &'a str, +) -> WorkflowRunEnqueue<'a> { + let mut run = WorkflowRunEnqueueBuilder::new(WorkflowType::new("buyer.enrichment"), metadata) + .idempotency_key("launcher:request") + .active_key(active_key); + if let Some(organization) = organization { + run = run.organization_id(organization); + } + for (index, input) in inputs.iter().enumerate() { + let mut step = account_step(input); + if index > 0 { + step = step.depends_on_terminal(&[StepKey::new(&inputs[index - 1].key)]); + } + run = run.step(step.try_build().expect("account step")); + } + run.try_build().expect("low-level buyer graph") +} + +fn assert_same_request(actual: &WorkflowRunEnqueue<'_>, expected: &WorkflowRunEnqueue<'_>) { + assert_eq!(actual.workflow_type(), expected.workflow_type()); + assert_eq!(actual.organization_id(), expected.organization_id()); + assert_eq!(actual.metadata(), expected.metadata()); + assert_eq!(actual.idempotency_key(), expected.idempotency_key()); + assert_eq!(actual.active_key(), expected.active_key()); + assert_eq!(actual.result_step_key(), expected.result_step_key()); + assert_eq!(actual.steps().len(), expected.steps().len()); + for (actual, expected) in actual.steps().iter().zip(expected.steps()) { + assert_eq!(actual.step_key(), expected.step_key()); + assert_eq!(actual.execution(), expected.execution()); + assert_eq!(actual.organization_id(), expected.organization_id()); + assert_eq!(actual.payload(), expected.payload()); + let dependencies = |step: &WorkflowStepEnqueue<'_>| { + step.dependencies() + .iter() + .map(|edge| { + ( + edge.prerequisite_step_key.as_str().to_owned(), + edge.release_mode, + ) + }) + .collect::>() + }; + assert_eq!(dependencies(actual), dependencies(expected)); + } +} + +#[test] +fn buyer_enrichment_has_identical_policies_through_both_fluent_builders() { + let inputs = [ + AccountInput { + key: "account:000000:a".into(), + organization: Uuid::from_u128(1), + payload: json!({"account_id": "a", "version": 1}), + }, + AccountInput { + key: "account:000001:b".into(), + organization: Uuid::from_u128(2), + payload: json!({"account_id": "b", "version": 1}), + }, + ]; + let catalog = catalog(); + for (organization, active_key) in [ + (None, "buyer:active"), + (Some(inputs[0].organization), "buyer:active"), + (Some(inputs[0].organization), "buyer:active:account:a"), + ] { + let inputs = if organization.is_some() { + &inputs[..1] + } else { + &inputs[..] + }; + let metadata = json!({"account_count": inputs.len()}); + let expected = low_level_buyer(inputs, &metadata, organization, active_key); + let mut core = WorkflowDagBuilder::new("buyer.enrichment", &metadata) + .idempotency_key("launcher:request") + .active_key(active_key); + let mut checked = catalog + .workflow_dag("buyer.enrichment", &metadata) + .idempotency_key("launcher:request") + .active_key(active_key); + if let Some(organization) = organization { + core = core.organization_id(organization); + checked = checked.organization_id(organization); + } + for (index, input) in inputs.iter().enumerate() { + core = core + .step(account_step(input).try_build().expect("account step")) + .expect("core step"); + checked = checked + .step( + catalog + .workflow_step(&input.key, ENRICH, &input.payload) + .expect("catalog account") + .organization_id(input.organization) + .allow_handler_continuation() + .execution_resource("buyer:provider") + .try_build() + .expect("catalog step"), + ) + .expect("checked step"); + if index > 0 { + core = core + .after_terminal(&input.key, [inputs[index - 1].key.as_str()]) + .expect("core edge"); + checked = checked + .after_terminal(&input.key, [inputs[index - 1].key.as_str()]) + .expect("catalog edge"); + } + } + for actual in [ + core.build().expect("core graph"), + checked.build().expect("catalog graph"), + ] { + assert_same_request(&actual, &expected); + for (index, step) in actual.steps().iter().enumerate() { + assert_eq!(step.organization_id(), Some(inputs[index].organization)); + assert!(step.allows_handler_continuation()); + assert_eq!(step.execution_resource_key(), Some("buyer:provider")); + } + assert!(actual.steps()[0].dependencies().is_empty()); + if inputs.len() > 1 { + assert_eq!( + actual.steps()[1].dependencies()[0].release_mode, + Some(WorkflowDependencyReleaseMode::OnTerminal) + ); + } + } + } +} + +#[test] +fn receiving_catalog_rejects_unknown_and_disabled_configured_jobs() { + let payload = json!({}); + let source = catalog(); + let step = source + .workflow_step("work", ENRICH, &payload) + .expect("source catalog step") + .try_build() + .expect("valid source step"); + let unknown = JobCatalog::new(); + let disabled = catalog().defaults(JobCatalogDefaults::new().enabled(false)); + assert!(matches!( + unknown + .workflow_dag("workflow", &payload) + .step(step.clone()), + Err(CatalogError::UnknownJobType { .. }) + )); + assert!(matches!( + disabled + .workflow_dag("workflow", &payload) + .step(step.clone()), + Err(CatalogError::DisabledJobType { .. }) + )); + assert!( + source + .workflow_dag("workflow", &payload) + .step(step) + .expect("own catalog accepts step") + .build() + .is_ok() + ); + assert!(matches!( + disabled + .workflow_dag("workflow", &payload) + .job("work", ENRICH, &payload), + Err(CatalogError::DisabledJobType { .. }) + )); +} + +#[test] +fn catalog_preserves_queue_settings_and_mixed_external_graph() { + let payload = json!({"ticket": 4}); + let catalog = catalog(); + let configured = catalog + .workflow_step("work", ENRICH, &payload) + .expect("catalog step") + .priority(9) + .max_attempts(4) + .timeout_seconds(120) + .stage(JobStage::Scheduled) + .depends_on_success(&[StepKey::new("approval")]) + .try_build() + .expect("configured step"); + let external = WorkflowStepEnqueueBuilder::try_new_external("approval", &payload) + .expect("external step") + .organization_id(Uuid::from_u128(9)) + .try_build() + .expect("scoped external"); + let run = catalog + .workflow_dag("workflow", &payload) + .step(configured) + .expect("job") + .step(external) + .expect("external") + .external("done", &payload) + .expect("external shorthand") + .after_terminal("done", ["work"]) + .expect("terminal edge") + .result_step("done") + .expect("result") + .build() + .expect("mixed graph"); + assert_eq!(run.steps()[0].priority(), Some(9)); + assert_eq!(run.steps()[0].max_attempts(), Some(4)); + assert_eq!(run.steps()[0].timeout_seconds(), Some(120)); + assert_eq!(run.steps()[0].stage(), Some(JobStage::Scheduled)); + assert_eq!( + run.steps()[0].dependencies()[0].release_mode, + Some(WorkflowDependencyReleaseMode::OnSuccess) + ); + assert_eq!(run.steps()[1].execution(), WorkflowStepExecution::External); + assert_eq!(run.steps()[1].organization_id(), Some(Uuid::from_u128(9))); + assert_eq!( + run.steps()[2].dependencies()[0].release_mode, + Some(WorkflowDependencyReleaseMode::OnTerminal) + ); + assert_eq!(run.result_step_key(), Some(StepKey::new("done"))); + let empty_catalog = JobCatalog::new(); + assert!( + empty_catalog + .workflow_dag("external-only", &payload) + .external("approval", &payload) + .expect("no handler required") + .build() + .is_ok() + ); +} + +#[test] +fn catalog_forwards_active_key_validation_and_clear() { + let payload = json!({}); + let catalog = catalog(); + let base = catalog + .workflow_dag("workflow", &payload) + .idempotency_key("request") + .external("approval", &payload) + .expect("external"); + assert!(matches!( + base.clone().active_key(" ").build(), + Err(CatalogError::WorkflowBuild( + WorkflowBuildError::BlankActiveKey + )) + )); + assert!(matches!( + base.clone().active_key(&"x".repeat(513)).build(), + Err(CatalogError::WorkflowBuild( + WorkflowBuildError::ActiveKeyTooLong + )) + )); + let run = base + .active_key("invalid") + .clear_active_key() + .build() + .expect("cleared key"); + assert_eq!(run.active_key(), None); + assert_eq!(run.idempotency_key(), Some("request")); +} + +#[test] +fn catalog_configured_external_steps_keep_core_graph_validation() { + let payload = json!({}); + let catalog = JobCatalog::new(); + let step = WorkflowStepEnqueueBuilder::try_new_external("approval", &payload) + .expect("external step") + .depends_on_success(&[StepKey::new("start")]) + .try_build() + .expect("external shape"); + let base = catalog + .workflow_dag("workflow", &payload) + .step(step.clone()) + .expect("configured external requires no handler"); + assert!(matches!( + base.clone().build(), + Err(CatalogError::WorkflowBuild( + WorkflowBuildError::MissingDependency { .. } + )) + )); + assert!(matches!( + base.clone().step(step), + Err(CatalogError::WorkflowBuild( + WorkflowBuildError::DuplicateStepKey { .. } + )) + )); + assert!(matches!( + base.clone().external("approval", &payload), + Err(CatalogError::WorkflowBuild( + WorkflowBuildError::DuplicateStepKey { .. } + )) + )); + assert!(matches!( + base.clone().external(" ", &payload), + Err(CatalogError::WorkflowBuild( + WorkflowBuildError::BlankStepKey { .. } + )) + )); + let base = base.external("start", &payload).expect("prerequisite"); + assert!(base.clone().build().is_ok()); + assert!(matches!( + base.clone() + .after_terminal("approval", ["start"]) + .expect("deferred duplicate") + .build(), + Err(CatalogError::WorkflowBuild( + WorkflowBuildError::DuplicateDependency { .. } + )) + )); + assert!(matches!( + base.clone() + .after_success("start", ["start"]) + .expect("deferred self edge") + .build(), + Err(CatalogError::WorkflowBuild( + WorkflowBuildError::SelfDependency { .. } + )) + )); + assert!(matches!( + base.after_success("start", ["approval"]) + .expect("deferred cycle") + .build(), + Err(CatalogError::WorkflowBuild( + WorkflowBuildError::CycleDetected + )) + )); +} diff --git a/runledger-runtime/tests/shared_specs.rs b/runledger-runtime/tests/shared_specs.rs new file mode 100644 index 0000000..49670a0 --- /dev/null +++ b/runledger-runtime/tests/shared_specs.rs @@ -0,0 +1,286 @@ +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; + +use async_trait::async_trait; +use runledger_core::jobs::{ + JobCompletion, JobContext, JobContract, JobFailure, JobHandler, JobSpec, JobSpecs, JobType, + TypedJobHandler, +}; +use runledger_runtime::catalog::{CatalogError, JobBindingError, JobCatalog}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use uuid::Uuid; + +#[derive(Serialize, Deserialize)] +struct Payload { + request_id: Uuid, + #[serde(default)] + revision: u32, +} +struct Delivery; +impl JobContract for Delivery { + type Payload = Payload; + fn spec() -> JobSpec { + JobSpec::new(JobType::new("delivery.send")).expect("static spec") + } +} +struct Handler { + calls: Arc, +} +#[async_trait] +impl TypedJobHandler for Handler { + type Contract = Delivery; + async fn execute( + &self, + context: JobContext, + payload: Payload, + ) -> Result { + assert_eq!(context.organization_id, Some(payload.request_id)); + assert_eq!(context.checkpoint, Some(json!({"resume": 2}))); + assert_eq!(payload.revision, 0); + self.calls.fetch_add(1, Ordering::SeqCst); + Ok(JobCompletion::success()) + } +} +fn context(id: Uuid) -> JobContext { + JobContext { + job_id: Uuid::nil(), + run_number: 1, + attempt: 1, + organization_id: Some(id), + worker_id: "test".into(), + checkpoint: Some(json!({"resume":2})), + } +} +fn handler() -> Arc { + Arc::new( + Handler { + calls: Arc::new(AtomicUsize::new(0)), + } + .into_job_handler(), + ) +} + +#[test] +fn complete_bindings_are_required_and_metadata_matches_producer() { + let specs = JobSpecs::new([Delivery::spec()]).expect("specs"); + assert!(matches!( + JobCatalog::from_specs(&specs, []), + Err(JobBindingError::MissingHandler(_)) + )); + assert!(matches!( + JobCatalog::from_specs(&specs, [handler(), handler()]), + Err(JobBindingError::Catalog( + CatalogError::DuplicateJobType { .. } + )) + )); + assert!(matches!( + JobCatalog::from_specs(&JobSpecs::default(), [handler()]), + Err(JobBindingError::UnknownHandler(_)) + )); + let catalog = JobCatalog::from_specs(&specs, [handler()]).expect("bound catalog"); + assert!( + catalog + .to_registry() + .get(Delivery::spec().job_type()) + .is_some() + ); +} + +#[tokio::test] +async fn decodes_old_json_rows_without_changing_context_or_running_invalid_payloads() { + let calls = Arc::new(AtomicUsize::new(0)); + let handler = Handler { + calls: calls.clone(), + } + .into_job_handler(); + let id = Uuid::now_v7(); + // Unknown legacy fields are accepted unless the application opts out via serde. + handler + .execute(context(id), json!({"request_id":id, "legacy":true})) + .await + .expect("old row"); + let failure = handler + .execute(context(id), json!({"request_id":"private-attacker-value"})) + .await + .expect_err("bad UUID"); + assert_eq!(failure.code, "job.invalid_payload"); + assert_eq!(failure.message, "Job payload has an invalid shape."); + assert_eq!(calls.load(Ordering::SeqCst), 1); +} + +struct ApplicationFailure; +#[async_trait] +impl TypedJobHandler for ApplicationFailure { + type Contract = Delivery; + async fn execute(&self, _: JobContext, _: Payload) -> Result { + panic!("invalid payload must not reach business logic"); + } + fn malformed_payload(&self, source: &serde_json::Error) -> JobFailure { + assert!(source.is_data()); + JobFailure::terminal("app.invalid_delivery", "Invalid delivery payload.") + } +} + +#[tokio::test] +async fn application_can_customize_static_failure_and_legacy_json_handlers_still_bind() { + let error = ApplicationFailure + .into_job_handler() + .execute(context(Uuid::nil()), json!({})) + .await + .expect_err("missing field"); + assert_eq!(error.code, "app.invalid_delivery"); + assert_eq!(error.message, "Invalid delivery payload."); + struct Legacy; + #[async_trait] + impl JobHandler for Legacy { + fn job_type(&self) -> JobType<'static> { + Delivery::spec().job_type() + } + async fn execute( + &self, + _: JobContext, + payload: Value, + ) -> Result { + assert_eq!(payload, json!({"legacy":true})); + Ok(JobCompletion::success()) + } + } + let specs = JobSpecs::new([Delivery::spec()]).expect("specs"); + let catalog = JobCatalog::from_specs(&specs, [Arc::new(Legacy) as Arc]) + .expect("legacy binding"); + catalog + .to_registry() + .get(Delivery::spec().job_type()) + .expect("legacy handler") + .execute(context(Uuid::nil()), json!({"legacy":true})) + .await + .expect("legacy execute"); +} + +#[tokio::test] +async fn shared_settings_survive_catalog_defaults_and_disabled_specs_keep_handlers() { + use runledger_core::jobs::JobDefinitionSettings; + use runledger_postgres::jobs::get_job_definition_by_type; + use runledger_runtime::catalog::JobCatalogDefaults; + use runledger_test_support::{setup_ephemeral_pool, teardown_ephemeral_pool}; + + let spec = Delivery::spec() + .with_settings( + JobDefinitionSettings::new() + .version(8) + .max_attempts(7) + .timeout_seconds(91) + .priority(-2) + .enabled(false), + ) + .expect("settings"); + let specs = JobSpecs::new([spec]).expect("specs"); + assert!(matches!( + JobCatalog::from_specs(&specs, []), + Err(JobBindingError::MissingHandler(_)) + )); + let catalog = JobCatalog::from_specs(&specs, [handler()]) + .expect("disabled binding") + .defaults(JobCatalogDefaults::new().max_attempts(20).enabled(true)); + assert!(catalog.to_registry().get(spec.job_type()).is_some()); + assert!(matches!( + catalog.require_catalog_enabled_job_type(spec.job_type().as_str()), + Err(CatalogError::DisabledJobType { .. }) + )); + let (pool, database) = setup_ephemeral_pool("shared_worker_specs", 2).await; + catalog.sync_definitions(&pool).await.expect("worker sync"); + let row = get_job_definition_by_type(&pool, spec.job_type()) + .await + .expect("read") + .expect("definition"); + assert_eq!(row.version, 8); + assert_eq!(row.max_attempts, 7); + assert_eq!(row.default_timeout_seconds, 91); + assert_eq!(row.default_priority, -2); + assert!(!row.is_enabled); + teardown_ephemeral_pool(pool, database).await; +} + +#[tokio::test] +async fn typed_dispatch_retains_execution_services_and_raw_terminal_cleanup() { + use runledger_core::jobs::{ + JobDeadLetterInfo, JobDeadLetterReason, JobExecution, JobExecutionError, + JobExecutionServices, JobExecutionUpdate, + }; + use std::time::{Duration, Instant}; + struct Services(Instant); + #[async_trait] + impl JobExecutionServices for Services { + fn deadline(&self) -> Instant { + self.0 + } + fn remaining_budget(&self) -> Duration { + Duration::from_secs(10) + } + async fn persist_progress( + &self, + _: JobExecutionUpdate<'_>, + ) -> Result<(), JobExecutionError> { + Ok(()) + } + } + struct WithServices(Arc); + #[async_trait] + impl TypedJobHandler for WithServices { + type Contract = Delivery; + async fn execute(&self, _: JobContext, _: Payload) -> Result { + panic!("runtime dispatch must retain services"); + } + async fn execute_with_services( + &self, + execution: JobExecution<'_>, + payload: Payload, + ) -> Result { + assert_eq!(execution.remaining_budget(), Duration::from_secs(10)); + assert_eq!( + execution.context().organization_id, + Some(payload.request_id) + ); + assert_eq!( + execution.checkpoint::().expect("typed checkpoint"), + Some(json!({"resume":2})) + ); + self.0.fetch_add(1, Ordering::SeqCst); + Ok(JobCompletion::success()) + } + async fn on_dead_letter(&self, _: JobContext, payload: Value, _: JobDeadLetterInfo) { + assert_eq!(payload, json!({"invalid":"raw"})); + self.0.fetch_add(1, Ordering::SeqCst); + } + } + let calls = Arc::new(AtomicUsize::new(0)); + let handler = WithServices(calls.clone()).into_job_handler(); + let id = Uuid::now_v7(); + let context = context(id); + let services = Services(Instant::now()); + handler + .execute_with_services( + JobExecution::new(&context, &services), + json!({"request_id":id}), + ) + .await + .expect("typed service dispatch"); + let error = handler + .execute_with_services( + JobExecution::new(&context, &services), + json!({"invalid":"raw"}), + ) + .await + .expect_err("malformed service payload"); + handler + .on_dead_letter( + context, + json!({"invalid":"raw"}), + JobDeadLetterInfo::new(error, JobDeadLetterReason::FailureKindNonRetryable, None), + ) + .await; + assert_eq!(calls.load(Ordering::SeqCst), 2); +} diff --git a/runledger-test-support/migrations/202609050001_job_summary_pagination.down.sql b/runledger-test-support/migrations/202609050001_job_summary_pagination.down.sql new file mode 100644 index 0000000..113842c --- /dev/null +++ b/runledger-test-support/migrations/202609050001_job_summary_pagination.down.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS idx_job_queue_created_id; +DROP INDEX IF EXISTS idx_job_queue_scope_created_id; diff --git a/runledger-test-support/migrations/202609050001_job_summary_pagination.up.sql b/runledger-test-support/migrations/202609050001_job_summary_pagination.up.sql new file mode 100644 index 0000000..597676f --- /dev/null +++ b/runledger-test-support/migrations/202609050001_job_summary_pagination.up.sql @@ -0,0 +1,9 @@ +-- Support compact pages with and without an exact organization scope. +-- Include the UUID tie-breaker so deep cursors constrain the full ordering key. +CREATE INDEX idx_job_queue_scope_created_id + ON job_queue (organization_id, created_at DESC, id DESC); +CREATE INDEX idx_job_queue_created_id + ON job_queue (created_at DESC, id DESC); + +-- Additive indexes do not change the persisted contract. Omit this migration +-- from runledger_migration_history so older filtered startup helpers coexist. diff --git a/runledger-test-support/src/postgres_container.rs b/runledger-test-support/src/postgres_container.rs index bb8f713..a4c28e8 100644 --- a/runledger-test-support/src/postgres_container.rs +++ b/runledger-test-support/src/postgres_container.rs @@ -15,7 +15,9 @@ const POSTGRES_PASSWORD: &str = "runledger"; const POSTGRES_DB: &str = "postgres"; const TEST_ADMIN_DATABASE_URL_ENV: &str = "RUNLEDGER_TEST_ADMIN_DATABASE_URL"; const TEST_PG_IMAGE_ENV: &str = "RUNLEDGER_TEST_PG_IMAGE"; -const MAX_POSTGRES_BOOTSTRAP_ATTEMPTS: u8 = 40; +const POSTGRES_BOOTSTRAP_TIMEOUT: Duration = Duration::from_secs(30); +const POSTGRES_CONNECT_TIMEOUT: Duration = Duration::from_secs(2); +const POSTGRES_RETRY_INTERVAL: Duration = Duration::from_millis(250); const MAX_PORT_RESOLVE_ATTEMPTS: u8 = 10; static SHARED_POSTGRES: tokio::sync::OnceCell = tokio::sync::OnceCell::const_new(); @@ -153,47 +155,60 @@ fn postgres_admin_url(port: u16) -> String { format!("postgres://{POSTGRES_USER}:{POSTGRES_PASSWORD}@127.0.0.1:{port}/{POSTGRES_DB}") } -async fn wait_for_postgres(admin_url: &str) { - for attempt in 1..=MAX_POSTGRES_BOOTSTRAP_ATTEMPTS { - if let Ok(pool) = PgPoolOptions::new() +async fn connect_postgres(admin_url: &str, timeout: Duration) -> Result { + let deadline = tokio::time::Instant::now() + timeout; + loop { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + match PgPoolOptions::new() .max_connections(1) + .acquire_timeout(remaining.min(POSTGRES_CONNECT_TIMEOUT)) .connect(admin_url) .await { - let server_version_num = - sqlx::query_scalar::<_, i32>("SELECT current_setting('server_version_num')::int") - .fetch_one(&pool) - .await - .unwrap_or_else(|error| { - panic!("failed to read PostgreSQL server_version_num: {error}") - }); - assert!( - server_version_num >= 180_000, - "Runledger requires PostgreSQL 18 or later; connected server_version_num was {server_version_num}" - ); - let uuidv7_check = sqlx::query_scalar::<_, String>("SELECT uuidv7()::text") - .fetch_one(&pool) - .await; - pool.close().await; - - if let Err(err) = uuidv7_check { - panic!( - "postgres is reachable but `uuidv7()` failed ({err}). Runledger requires PostgreSQL 18 or later; ensure RUNLEDGER_TEST_PG_IMAGE or RUNLEDGER_TEST_ADMIN_DATABASE_URL points to PostgreSQL 18+." - ); + Ok(pool) => return Ok(pool), + Err(error) => { + // Fast connection resets must not exhaust readiness while + // initdb is still running. Bound slow handshakes and backoff + // by the same overall deadline. + let next_attempt = tokio::time::Instant::now() + POSTGRES_RETRY_INTERVAL; + tokio::time::sleep_until(next_attempt.min(deadline)).await; + if tokio::time::Instant::now() >= deadline { + return Err(error); + } } - return; } + } +} - tokio::time::sleep(Duration::from_millis(250)).await; - - if attempt == MAX_POSTGRES_BOOTSTRAP_ATTEMPTS { +async fn wait_for_postgres(admin_url: &str) { + let pool = connect_postgres(admin_url, POSTGRES_BOOTSTRAP_TIMEOUT) + .await + .unwrap_or_else(|error| { panic!( - "failed to connect to postgres test container after {attempt} attempts ({admin_url})" - ); - } + "failed to connect to PostgreSQL within {POSTGRES_BOOTSTRAP_TIMEOUT:?}; last connection error: {error}" + ) + }); + let server_version_num = + sqlx::query_scalar::<_, i32>("SELECT current_setting('server_version_num')::int") + .fetch_one(&pool) + .await + .unwrap_or_else(|error| { + panic!("failed to read PostgreSQL server_version_num: {error}") + }); + assert!( + server_version_num >= 180_000, + "Runledger requires PostgreSQL 18 or later; connected server_version_num was {server_version_num}" + ); + let uuidv7_check = sqlx::query_scalar::<_, String>("SELECT uuidv7()::text") + .fetch_one(&pool) + .await; + pool.close().await; + + if let Err(err) = uuidv7_check { + panic!( + "postgres is reachable but `uuidv7()` failed ({err}). Runledger requires PostgreSQL 18 or later; ensure RUNLEDGER_TEST_PG_IMAGE or RUNLEDGER_TEST_ADMIN_DATABASE_URL points to PostgreSQL 18+." + ); } - - panic!("unexpected postgres bootstrap loop termination"); } #[cfg(test)] @@ -206,6 +221,67 @@ mod tests { use super::*; const CONCURRENT_CALLERS: usize = 8; + #[tokio::test] + async fn bootstrap_deadline_bounds_a_stalled_handshake() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind stalled PostgreSQL listener"); + let admin_url = postgres_admin_url(listener.local_addr().expect("listener address").port()); + let (result, accepted) = tokio::time::timeout(Duration::from_secs(2), async { + // Keep the accepted socket open without replying to SQLx. A + // connection attempt must respect the shorter bootstrap budget, + // rather than SQLx's default 30-second acquisition timeout. + tokio::join!( + connect_postgres(&admin_url, Duration::from_millis(100)), + listener.accept(), + ) + }) + .await + .expect("bootstrap must finish within its deadline despite a stalled handshake"); + accepted.expect("accept PostgreSQL connection"); + assert!(matches!(result, Err(sqlx::Error::PoolTimedOut))); + } + + #[tokio::test] + async fn bootstrap_waits_for_delayed_postgres_18_startup() { + if std::env::var_os(TEST_ADMIN_DATABASE_URL_ENV).is_some() { + eprintln!( + "skipping Docker-only delayed startup test: external PostgreSQL is configured" + ); + return; + } + // Exceed the former 40 x 250 ms retry window without depending on + // host load or Docker's image cache to make initialization slow. + let (repository, tag) = parse_image_ref( + &std::env::var(TEST_PG_IMAGE_ENV).unwrap_or_else(|_| DEFAULT_POSTGRES_IMAGE.to_owned()), + ); + let container = GenericImage::new(repository, tag) + .with_exposed_port(ContainerPort::Tcp(5432)) + .with_env_var("POSTGRES_USER", POSTGRES_USER) + .with_env_var("POSTGRES_PASSWORD", POSTGRES_PASSWORD) + .with_env_var("POSTGRES_DB", POSTGRES_DB) + .with_cmd(["sh", "-c", "sleep 12; exec docker-entrypoint.sh postgres"]) + .start() + .await + .expect("start delayed PostgreSQL container"); + let port = resolve_host_port(&container, 5432).await; + let admin_url = postgres_admin_url(port); + + wait_for_postgres(&admin_url).await; + + let pool = PgPoolOptions::new() + .max_connections(1) + .connect(&admin_url) + .await + .expect("connect after delayed PostgreSQL startup"); + let server_version = sqlx::query_scalar::<_, String>("SHOW server_version") + .fetch_one(&pool) + .await + .expect("read delayed PostgreSQL server version"); + eprintln!("delayed startup: PostgreSQL server_version={server_version}"); + pool.close().await; + } + async fn postgres_18_admin_url(test_name: &str) -> String { let admin_url = admin_database_url().await.to_owned(); let pool = PgPoolOptions::new() diff --git a/scripts/check-readme.py b/scripts/check-readme.py new file mode 100644 index 0000000..89a2d63 --- /dev/null +++ b/scripts/check-readme.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Check release versions and compiled quick-start sources; --write updates versions.""" + +import argparse +from pathlib import Path +import re +import sys +import tomllib + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def check(root: Path, write: bool) -> None: + manifest = tomllib.loads((root / "Cargo.toml").read_text()) + version = manifest["workspace"]["package"]["version"] + crates = { + name for name in manifest["workspace"]["dependencies"] + if name.startswith("runledger-") + } + path = root / "README.md" + readme = path.read_text() + if write: + # Only current dependency recommendations and release commands change. + # Historical upgrade guidance and compatibility examples remain intact. + installation_start = readme.index("## Installation\n") + installation_end = readme.index("\n## ", installation_start) + installation = re.sub( + r'^(runledger-[\w-]+ = ")[^"]+("\s*)$', + lambda match: match[1] + version + match[2], + readme[installation_start:installation_end], flags=re.MULTILINE, + ) + readme = readme[:installation_start] + installation + readme[installation_end:] + readme = re.sub( + r'^(\./scripts/(?:prepare|publish)-release\.sh )\S+$', + lambda match: match[1] + version, + readme, flags=re.MULTILINE, + ) + + installation = readme.split("## Installation\n", 1)[1].split("\n## ", 1)[0] + block = re.search(r"```toml\n(.*?)\n```", installation, re.DOTALL) + require(block is not None, "missing installation manifest") + dependencies = tomllib.loads(block[1]) + versions = { + name: value if isinstance(value, str) else value.get("version") + for section in ("dependencies", "dev-dependencies") + for name, value in dependencies.get(section, {}).items() + if name.startswith("runledger-") + } + require( + versions == dict.fromkeys(crates, version), + f"installation must recommend {version} for {sorted(crates)}; got {versions}" + ) + for command in ("prepare", "publish"): + versions = re.findall( + rf"^\./scripts/{command}-release\.sh (\S+)$", readme, re.MULTILINE, + ) + require(versions == [version], f"{command}-release command must use {version}") + + snippets = re.findall( + r"\n```rust\n(.*?)\n```", + readme, re.DOTALL, + ) + expected = { + f"runledger-runtime/examples/producer_worker/{name}.rs" + for name in ("shared", "producer", "worker") + } + require( + len(snippets) == len(expected) and {p for p, _ in snippets} == expected, + "quick start must include shared, producer, and worker source blocks" + ) + for source, snippet in snippets: + code = (root / source).read_text().split("\n#[cfg(test)]", 1)[0].rstrip() + require(snippet == code, f"quick-start snippet differs from {source}") + + if write: + path.write_text(readme) + print(f"README versions ({version}) and quick-start sources verified.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--write", action="store_true") + args = parser.parse_args() + try: + check(Path(__file__).resolve().parent.parent, args.write) + except (KeyError, IndexError, ValueError) as error: + sys.exit(f"error: {error}") diff --git a/scripts/lint.sh b/scripts/lint.sh index 62f09a8..e607324 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -7,6 +7,8 @@ SMOKE_MANIFEST="$ROOT_DIR/smoke/external-consumer/Cargo.toml" cd "$ROOT_DIR" +python3 scripts/check-readme.py + cargo fmt --all -- --check cargo fmt --manifest-path "$SMOKE_MANIFEST" -- --check diff --git a/scripts/prepare-release.sh b/scripts/prepare-release.sh index 8c34d3e..edf0f0a 100755 --- a/scripts/prepare-release.sh +++ b/scripts/prepare-release.sh @@ -36,6 +36,7 @@ release_generated_path() { case "$path" in Cargo.toml | \ + README.md | \ Cargo.lock | \ smoke/external-consumer/Cargo.lock | \ .sqlx/* | \ @@ -224,6 +225,7 @@ cd "$ROOT_DIR" require_command cargo require_command curl require_command git +require_command python3 validate_version "$VERSION" require_clean_or_resumable_worktree "$VERSION" @@ -241,6 +243,8 @@ done require_manifest_versions "$VERSION" +python3 scripts/check-readme.py --write + cargo update -w cargo update \ --manifest-path "$ROOT_DIR/smoke/external-consumer/Cargo.toml" \ diff --git a/scripts/publish-release.sh b/scripts/publish-release.sh index 1f9e87a..0bce990 100755 --- a/scripts/publish-release.sh +++ b/scripts/publish-release.sh @@ -165,9 +165,11 @@ require_command cargo require_command curl require_command gh require_command git +require_command python3 require_clean_worktree validate_version "$VERSION" require_manifest_versions "$VERSION" +python3 scripts/check-readme.py current_branch="$(git rev-parse --abbrev-ref HEAD)" if [[ "$current_branch" == "HEAD" ]]; then diff --git a/smoke/external-consumer/Cargo.lock b/smoke/external-consumer/Cargo.lock index 1c66399..de20801 100644 --- a/smoke/external-consumer/Cargo.lock +++ b/smoke/external-consumer/Cargo.lock @@ -181,6 +181,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bollard" version = "0.21.1" @@ -320,6 +329,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "conquer-once" version = "0.4.0" @@ -341,6 +356,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "core-foundation" version = "0.10.1" @@ -398,7 +419,7 @@ checksum = "089df96cf6a25253b4b6b6744d86f91150a3d4df546f31a95def47976b8cba97" dependencies = [ "chrono", "once_cell", - "phf", + "phf 0.11.3", "winnow", ] @@ -427,6 +448,24 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "darling" version = "0.23.0" @@ -498,7 +537,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "pem-rfc7468", "zeroize", ] @@ -518,12 +557,24 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "displaydoc" version = "0.2.7" @@ -614,6 +665,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "ferroid" version = "2.0.0" @@ -780,7 +837,7 @@ checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", ] [[package]] @@ -864,7 +921,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", ] [[package]] @@ -873,7 +930,16 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", ] [[package]] @@ -930,6 +996,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.11.0" @@ -1348,7 +1423,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", ] [[package]] @@ -1370,7 +1455,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] @@ -1469,6 +1554,24 @@ dependencies = [ "libm", ] +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1557,7 +1660,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ "phf_macros", - "phf_shared", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared 0.13.1", + "serde", ] [[package]] @@ -1566,7 +1679,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ - "phf_shared", + "phf_shared 0.11.3", "rand 0.8.7", ] @@ -1577,7 +1690,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" dependencies = [ "phf_generator", - "phf_shared", + "phf_shared 0.11.3", "proc-macro2", "quote", "syn 2.0.119", @@ -1592,6 +1705,15 @@ dependencies = [ "siphasher", ] +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.13" @@ -1666,6 +1788,49 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "postgres-protocol" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" +dependencies = [ + "base64", + "byteorder", + "bytes", + "fallible-iterator", + "hmac 0.13.0", + "md-5 0.11.0", + "memchr", + "rand 0.10.2", + "sha2 0.11.0", + "stringprep", +] + +[[package]] +name = "postgres-test-harness" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae16463f0ea9790d102559acc5dda876b3dfe577ed8d338b2b8023cc7fdf2456" +dependencies = [ + "sha2 0.11.0", + "thiserror", + "tokio", + "tokio-postgres", + "url", + "uuid", +] + +[[package]] +name = "postgres-types" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol", +] + [[package]] name = "potential_utf" version = "0.1.6" @@ -1880,8 +2045,8 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", - "digest", + "const-oid 0.9.6", + "digest 0.10.7", "num-bigint-dig", "num-integer", "num-traits", @@ -1912,6 +2077,7 @@ version = "0.1.0" dependencies = [ "async-trait", "chrono", + "postgres-test-harness", "runledger-core", "runledger-postgres", "runledger-runtime", @@ -1930,6 +2096,7 @@ dependencies = [ "runledger-core", "serde", "serde_json", + "sha2 0.10.9", "sqlx", "tokio", "tracing", @@ -2208,7 +2375,7 @@ checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", ] [[package]] @@ -2219,7 +2386,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -2254,7 +2432,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core 0.6.4", ] @@ -2347,7 +2525,7 @@ dependencies = [ "percent-encoding", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "thiserror", "tokio", @@ -2385,7 +2563,7 @@ dependencies = [ "quote", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "sqlx-core", "sqlx-mysql", "sqlx-postgres", @@ -2408,7 +2586,7 @@ dependencies = [ "bytes", "chrono", "crc", - "digest", + "digest 0.10.7", "dotenvy", "either", "futures-channel", @@ -2418,10 +2596,10 @@ dependencies = [ "generic-array", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "once_cell", "percent-encoding", @@ -2429,14 +2607,14 @@ dependencies = [ "rsa", "serde", "sha1", - "sha2", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", "thiserror", "tracing", "uuid", - "whoami", + "whoami 1.6.1", ] [[package]] @@ -2458,24 +2636,24 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "home", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "once_cell", "rand 0.8.7", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", "thiserror", "tracing", "uuid", - "whoami", + "whoami 1.6.1", ] [[package]] @@ -2730,6 +2908,32 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "tokio-postgres" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf 0.13.1", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand 0.10.2", + "socket2", + "tokio", + "tokio-util", + "whoami 2.1.3", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -2971,12 +3175,39 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasite" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + [[package]] name = "wasm-bindgen" version = "0.2.127" @@ -3022,6 +3253,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "web-time" version = "1.1.0" @@ -3039,7 +3280,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" dependencies = [ "libredox", - "wasite", + "wasite 0.1.0", +] + +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite 1.0.2", + "web-sys", ] [[package]] @@ -3280,6 +3534,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "writeable" version = "0.6.4" diff --git a/smoke/external-consumer/Cargo.toml b/smoke/external-consumer/Cargo.toml index 2b148b0..9348c46 100644 --- a/smoke/external-consumer/Cargo.toml +++ b/smoke/external-consumer/Cargo.toml @@ -30,6 +30,7 @@ workspace = true [dev-dependencies] async-trait = "0.1" chrono = { version = "0.4", default-features = false, features = ["clock", "std"] } +postgres-test-harness = { version = "=0.2.0", default-features = false } # Direct `cargo test --manifest-path smoke/external-consumer/Cargo.toml` uses # local paths so contributors are not blocked before the next crates.io release. # scripts/run-external-consumer-smoke.sh rewrites these to packaged version diff --git a/smoke/external-consumer/tests/fixtures/hocr-runledger-0.5.0/202603280001_runledger_baseline.sql b/smoke/external-consumer/tests/fixtures/hocr-runledger-0.5.0/202603280001_runledger_baseline.sql new file mode 100644 index 0000000..91ddcb8 --- /dev/null +++ b/smoke/external-consumer/tests/fixtures/hocr-runledger-0.5.0/202603280001_runledger_baseline.sql @@ -0,0 +1,759 @@ +CREATE FUNCTION set_updated_at_timestamp() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + NEW.updated_at = now(); + RETURN NEW; +END; +$$; + +CREATE TYPE job_status AS ENUM ( + 'PENDING', + 'LEASED', + 'SUCCEEDED', + 'DEAD_LETTERED', + 'CANCELED' +); + +CREATE TYPE job_event_type AS ENUM ( + 'ENQUEUED', + 'LEASED', + 'HEARTBEAT', + 'STAGE_CHANGED', + 'PROGRESS', + 'RETRY_SCHEDULED', + 'SUCCEEDED', + 'FAILED', + 'DEAD_LETTERED', + 'CANCELED', + 'REQUEUED' +); + +CREATE TYPE job_failure_kind AS ENUM ( + 'RETRYABLE', + 'TERMINAL', + 'PANICKED', + 'TIMEOUT', + 'LEASE_EXPIRED' +); + +CREATE TYPE workflow_run_status AS ENUM ( + 'RUNNING', + 'WAITING_FOR_EXTERNAL', + 'SUCCEEDED', + 'COMPLETED_WITH_ERRORS', + 'CANCELED' +); + +CREATE TYPE workflow_step_status AS ENUM ( + 'BLOCKED', + 'WAITING_FOR_EXTERNAL', + 'ENQUEUED', + 'RUNNING', + 'SUCCEEDED', + 'FAILED', + 'CANCELED' +); + +CREATE TYPE workflow_dependency_release_mode AS ENUM ( + 'ON_TERMINAL', + 'ON_SUCCESS' +); + +CREATE TYPE workflow_step_execution_kind AS ENUM ( + 'JOB', + 'EXTERNAL' +); + +CREATE TABLE job_definitions ( + job_type text PRIMARY KEY, + version integer NOT NULL DEFAULT 1, + max_attempts integer NOT NULL DEFAULT 8, + default_timeout_seconds integer NOT NULL DEFAULT 900, + default_priority integer NOT NULL DEFAULT 100, + is_enabled boolean NOT NULL DEFAULT true, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT chk_job_definitions_type_not_blank + CHECK (length(trim(job_type)) > 0), + CONSTRAINT chk_job_definitions_version_positive + CHECK (version > 0), + CONSTRAINT chk_job_definitions_max_attempts_positive + CHECK (max_attempts > 0), + CONSTRAINT chk_job_definitions_timeout_positive + CHECK (default_timeout_seconds > 0) +); + +CREATE TRIGGER trg_job_definitions_set_updated_at + BEFORE UPDATE ON job_definitions + FOR EACH ROW + EXECUTE FUNCTION set_updated_at_timestamp(); + +CREATE TABLE job_queue ( + id uuid PRIMARY KEY DEFAULT uuidv7(), + job_type text NOT NULL, + organization_id uuid, + payload jsonb NOT NULL DEFAULT '{}'::jsonb, + status job_status NOT NULL DEFAULT 'PENDING', + priority integer NOT NULL DEFAULT 100, + run_number integer NOT NULL DEFAULT 1, + attempt integer NOT NULL DEFAULT 0, + max_attempts integer NOT NULL, + timeout_seconds integer NOT NULL DEFAULT 900, + next_run_at timestamptz NOT NULL DEFAULT now(), + lease_expires_at timestamptz, + last_heartbeat_at timestamptz, + worker_id text, + started_at timestamptz, + finished_at timestamptz, + stage text NOT NULL DEFAULT 'queued', + progress_done bigint, + progress_total bigint, + progress_pct numeric GENERATED ALWAYS AS ( + CASE + WHEN progress_done IS NULL + OR progress_total IS NULL + OR progress_total <= 0 THEN NULL + ELSE round((progress_done::numeric * 100.0) / progress_total::numeric, 2) + END + ) STORED, + checkpoint jsonb, + idempotency_key text, + status_reason text, + last_error_code text, + last_error_message text, + workflow_step_id uuid, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT fk_job_queue_job_type + FOREIGN KEY (job_type) + REFERENCES job_definitions (job_type) ON DELETE RESTRICT, + CONSTRAINT chk_job_queue_attempt_nonnegative + CHECK (attempt >= 0), + CONSTRAINT chk_job_queue_max_attempts_positive + CHECK (max_attempts > 0), + CONSTRAINT chk_job_queue_run_number_positive + CHECK (run_number > 0), + CONSTRAINT chk_job_queue_timeout_positive + CHECK (timeout_seconds > 0), + CONSTRAINT chk_job_queue_progress_nonnegative + CHECK ( + (progress_done IS NULL OR progress_done >= 0) + AND (progress_total IS NULL OR progress_total >= 0) + AND ( + progress_done IS NULL + OR progress_total IS NULL + OR progress_done <= progress_total + ) + ), + CONSTRAINT chk_job_queue_stage_not_blank + CHECK (length(trim(stage)) > 0), + CONSTRAINT chk_job_queue_worker_id_not_blank + CHECK (worker_id IS NULL OR length(trim(worker_id)) > 0), + CONSTRAINT chk_job_queue_idempotency_key_not_blank + CHECK (idempotency_key IS NULL OR length(trim(idempotency_key)) > 0) +); + +CREATE TRIGGER trg_job_queue_set_updated_at + BEFORE UPDATE ON job_queue + FOR EACH ROW + EXECUTE FUNCTION set_updated_at_timestamp(); + +CREATE UNIQUE INDEX uq_job_queue_type_idempotency_org + ON job_queue (job_type, organization_id, idempotency_key) + WHERE idempotency_key IS NOT NULL + AND organization_id IS NOT NULL; + +CREATE UNIQUE INDEX uq_job_queue_type_idempotency_global + ON job_queue (job_type, idempotency_key) + WHERE idempotency_key IS NOT NULL + AND organization_id IS NULL; + +CREATE INDEX idx_job_queue_claim + ON job_queue (status, priority DESC, next_run_at, created_at) + WHERE status = 'PENDING'; + +CREATE INDEX idx_job_queue_leased_expiry + ON job_queue (lease_expires_at) + WHERE status = 'LEASED'; + +CREATE INDEX idx_job_queue_org_status_created + ON job_queue (organization_id, status, created_at DESC); + +CREATE INDEX idx_job_queue_type_status_created + ON job_queue (job_type, status, created_at DESC); + +CREATE TABLE job_attempts ( + id uuid PRIMARY KEY DEFAULT uuidv7(), + job_id uuid NOT NULL, + run_number integer NOT NULL DEFAULT 1, + attempt integer NOT NULL, + worker_id text NOT NULL, + leased_at timestamptz NOT NULL, + started_at timestamptz NOT NULL, + finished_at timestamptz, + outcome job_failure_kind, + error_code text, + error_message text, + retry_delay_ms integer, + claim_origin text NOT NULL DEFAULT 'DIRECT', + execution_started_persisted_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT fk_job_attempts_job + FOREIGN KEY (job_id) + REFERENCES job_queue (id) ON DELETE CASCADE, + CONSTRAINT uq_job_attempts_job_run_attempt + UNIQUE (job_id, run_number, attempt), + CONSTRAINT chk_job_attempts_attempt_positive + CHECK (attempt > 0), + CONSTRAINT chk_job_attempts_run_number_positive + CHECK (run_number > 0), + CONSTRAINT chk_job_attempts_worker_not_blank + CHECK (length(trim(worker_id)) > 0), + CONSTRAINT chk_job_attempts_retry_delay_nonnegative + CHECK (retry_delay_ms IS NULL OR retry_delay_ms >= 0), + CONSTRAINT chk_job_attempts_claim_origin + CHECK (claim_origin IN ('DIRECT', 'WORKER_PRESTART')) +); + +CREATE INDEX idx_job_attempts_job_run_created + ON job_attempts (job_id, run_number, created_at DESC); + +CREATE TABLE job_events ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL, + run_number integer NOT NULL DEFAULT 1, + attempt integer, + event_type job_event_type NOT NULL, + stage text, + progress_done bigint, + progress_total bigint, + payload jsonb NOT NULL DEFAULT '{}'::jsonb, + occurred_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT fk_job_events_job + FOREIGN KEY (job_id) + REFERENCES job_queue (id) ON DELETE CASCADE, + CONSTRAINT chk_job_events_attempt_nonnegative + CHECK (attempt IS NULL OR attempt > 0), + CONSTRAINT chk_job_events_run_number_positive + CHECK (run_number > 0), + CONSTRAINT chk_job_events_stage_not_blank + CHECK (stage IS NULL OR length(trim(stage)) > 0), + CONSTRAINT chk_job_events_progress_nonnegative + CHECK ( + (progress_done IS NULL OR progress_done >= 0) + AND (progress_total IS NULL OR progress_total >= 0) + AND ( + progress_done IS NULL + OR progress_total IS NULL + OR progress_done <= progress_total + ) + ) +); + +CREATE INDEX idx_job_events_job_run_id + ON job_events (job_id, run_number, id); + +CREATE INDEX idx_job_events_type_time + ON job_events (event_type, occurred_at DESC); + +CREATE TABLE job_dead_letters ( + job_id uuid PRIMARY KEY, + job_type text NOT NULL, + organization_id uuid, + run_number integer NOT NULL DEFAULT 1, + attempt integer NOT NULL, + error_code text, + error_message text, + payload_snapshot jsonb NOT NULL, + checkpoint_snapshot jsonb, + failed_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT fk_job_dead_letters_job + FOREIGN KEY (job_id) + REFERENCES job_queue (id) ON DELETE CASCADE, + CONSTRAINT fk_job_dead_letters_job_type + FOREIGN KEY (job_type) + REFERENCES job_definitions (job_type) ON DELETE RESTRICT, + CONSTRAINT chk_job_dead_letters_run_number_positive + CHECK (run_number > 0), + CONSTRAINT chk_job_dead_letters_attempt_positive + CHECK (attempt > 0) +); + +CREATE INDEX idx_job_dead_letters_failed_at + ON job_dead_letters (failed_at DESC); + +CREATE TABLE job_schedules ( + id uuid PRIMARY KEY DEFAULT uuidv7(), + name text NOT NULL UNIQUE, + job_type text NOT NULL, + organization_id uuid, + payload_template jsonb NOT NULL DEFAULT '{}'::jsonb, + cron_expr text NOT NULL, + timezone text NOT NULL DEFAULT 'UTC', + is_active boolean NOT NULL DEFAULT true, + next_fire_at timestamptz NOT NULL, + last_fired_at timestamptz, + max_jitter_seconds integer NOT NULL DEFAULT 0, + created_by_user_id uuid, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT fk_job_schedules_job_type + FOREIGN KEY (job_type) + REFERENCES job_definitions (job_type) ON DELETE RESTRICT, + CONSTRAINT chk_job_schedules_name_not_blank + CHECK (length(trim(name)) > 0), + CONSTRAINT chk_job_schedules_cron_not_blank + CHECK (length(trim(cron_expr)) > 0), + CONSTRAINT chk_job_schedules_timezone_not_blank + CHECK (length(trim(timezone)) > 0), + CONSTRAINT chk_job_schedules_timezone_utc + CHECK (timezone = 'UTC'), + CONSTRAINT chk_job_schedules_jitter_nonnegative + CHECK (max_jitter_seconds >= 0) +); + +CREATE TRIGGER trg_job_schedules_set_updated_at + BEFORE UPDATE ON job_schedules + FOR EACH ROW + EXECUTE FUNCTION set_updated_at_timestamp(); + +CREATE INDEX idx_job_schedules_next_fire + ON job_schedules (next_fire_at) + WHERE is_active; + +CREATE TABLE workflow_runs ( + id uuid PRIMARY KEY DEFAULT uuidv7(), + workflow_type text NOT NULL, + organization_id uuid, + status workflow_run_status NOT NULL DEFAULT 'RUNNING', + idempotency_key text, + metadata jsonb NOT NULL DEFAULT '{}'::jsonb, + started_at timestamptz NOT NULL DEFAULT now(), + finished_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT chk_workflow_runs_workflow_type_not_blank + CHECK (length(trim(workflow_type)) > 0), + CONSTRAINT chk_workflow_runs_idempotency_key_not_blank + CHECK (idempotency_key IS NULL OR length(trim(idempotency_key)) > 0) +); + +CREATE TRIGGER trg_workflow_runs_set_updated_at + BEFORE UPDATE ON workflow_runs + FOR EACH ROW + EXECUTE FUNCTION set_updated_at_timestamp(); + +CREATE UNIQUE INDEX uq_workflow_runs_type_idempotency_org + ON workflow_runs (workflow_type, organization_id, idempotency_key) + WHERE idempotency_key IS NOT NULL + AND organization_id IS NOT NULL; + +CREATE UNIQUE INDEX uq_workflow_runs_type_idempotency_global + ON workflow_runs (workflow_type, idempotency_key) + WHERE idempotency_key IS NOT NULL + AND organization_id IS NULL; + +CREATE INDEX idx_workflow_runs_status_created + ON workflow_runs (status, created_at DESC); + +CREATE TABLE workflow_steps ( + id uuid PRIMARY KEY DEFAULT uuidv7(), + workflow_run_id uuid NOT NULL, + step_key text NOT NULL, + job_type text, + organization_id uuid, + payload jsonb NOT NULL DEFAULT '{}'::jsonb, + priority integer, + max_attempts integer, + timeout_seconds integer, + stage text, + execution_kind workflow_step_execution_kind NOT NULL DEFAULT 'JOB', + status workflow_step_status NOT NULL DEFAULT 'BLOCKED', + job_id uuid, + released_at timestamptz, + started_at timestamptz, + finished_at timestamptz, + dependency_count_total integer NOT NULL DEFAULT 0, + dependency_count_pending integer NOT NULL DEFAULT 0, + dependency_count_unsatisfied integer NOT NULL DEFAULT 0, + status_reason text, + last_error_code text, + last_error_message text, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT fk_workflow_steps_workflow_run + FOREIGN KEY (workflow_run_id) + REFERENCES workflow_runs (id) ON DELETE CASCADE, + CONSTRAINT fk_workflow_steps_job_type + FOREIGN KEY (job_type) + REFERENCES job_definitions (job_type) ON DELETE RESTRICT, + CONSTRAINT fk_workflow_steps_job + FOREIGN KEY (job_id) + REFERENCES job_queue (id) ON DELETE SET NULL, + CONSTRAINT uq_workflow_steps_run_step_key + UNIQUE (workflow_run_id, step_key), + CONSTRAINT uq_workflow_steps_job_id + UNIQUE (job_id), + CONSTRAINT uq_workflow_steps_run_step_id + UNIQUE (workflow_run_id, id), + CONSTRAINT chk_workflow_steps_step_key_not_blank + CHECK (length(trim(step_key)) > 0), + CONSTRAINT chk_workflow_steps_job_type_not_blank + CHECK (job_type IS NULL OR length(trim(job_type)) > 0), + CONSTRAINT chk_workflow_steps_stage_not_blank + CHECK (stage IS NULL OR length(trim(stage)) > 0), + CONSTRAINT chk_workflow_steps_max_attempts_positive + CHECK (max_attempts IS NULL OR max_attempts > 0), + CONSTRAINT chk_workflow_steps_timeout_positive + CHECK (timeout_seconds IS NULL OR timeout_seconds > 0), + CONSTRAINT chk_workflow_steps_dependency_counts_nonnegative + CHECK ( + dependency_count_total >= 0 + AND dependency_count_pending >= 0 + AND dependency_count_unsatisfied >= 0 + AND dependency_count_pending <= dependency_count_total + AND dependency_count_unsatisfied <= dependency_count_total + ), + CONSTRAINT chk_workflow_steps_execution_shape + CHECK ( + ( + execution_kind = 'JOB' + AND job_type IS NOT NULL + AND priority IS NOT NULL + AND max_attempts IS NOT NULL + AND timeout_seconds IS NOT NULL + AND stage IS NOT NULL + ) + OR ( + execution_kind = 'EXTERNAL' + AND job_id IS NULL + AND job_type IS NULL + AND priority IS NULL + AND max_attempts IS NULL + AND timeout_seconds IS NULL + AND stage IS NULL + ) + ) +); + +CREATE TRIGGER trg_workflow_steps_set_updated_at + BEFORE UPDATE ON workflow_steps + FOR EACH ROW + EXECUTE FUNCTION set_updated_at_timestamp(); + +CREATE INDEX idx_workflow_steps_run_status_created + ON workflow_steps (workflow_run_id, status, created_at ASC); + +ALTER TABLE job_queue ADD CONSTRAINT fk_job_queue_workflow_step + FOREIGN KEY (workflow_step_id) + REFERENCES workflow_steps (id) ON DELETE SET NULL; + +ALTER TABLE job_queue ADD CONSTRAINT uq_job_queue_workflow_step_id + UNIQUE (workflow_step_id); + +CREATE TABLE workflow_step_dependencies ( + workflow_run_id uuid NOT NULL, + prerequisite_step_id uuid NOT NULL, + dependent_step_id uuid NOT NULL, + release_mode workflow_dependency_release_mode NOT NULL DEFAULT 'ON_TERMINAL', + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT pk_workflow_step_dependencies + PRIMARY KEY (workflow_run_id, prerequisite_step_id, dependent_step_id), + CONSTRAINT fk_workflow_step_dependencies_run + FOREIGN KEY (workflow_run_id) + REFERENCES workflow_runs (id) ON DELETE CASCADE, + CONSTRAINT fk_workflow_step_dependencies_prerequisite + FOREIGN KEY (workflow_run_id, prerequisite_step_id) + REFERENCES workflow_steps (workflow_run_id, id) ON DELETE CASCADE, + CONSTRAINT fk_workflow_step_dependencies_dependent + FOREIGN KEY (workflow_run_id, dependent_step_id) + REFERENCES workflow_steps (workflow_run_id, id) ON DELETE CASCADE, + CONSTRAINT chk_workflow_step_dependencies_no_self_reference + CHECK (prerequisite_step_id <> dependent_step_id) +); + +CREATE INDEX idx_workflow_step_dependencies_prerequisite + ON workflow_step_dependencies (prerequisite_step_id); + +CREATE INDEX idx_workflow_step_dependencies_dependent + ON workflow_step_dependencies (dependent_step_id); + +CREATE OR REPLACE FUNCTION enforce_workflow_job_linkage_symmetry() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + linked_job_id uuid; + linked_workflow_step_id uuid; + current_workflow_step_id uuid; + current_job_id uuid; +BEGIN + IF TG_TABLE_NAME = 'job_queue' THEN + SELECT jq.workflow_step_id + INTO current_workflow_step_id + FROM job_queue jq + WHERE jq.id = NEW.id; + + IF current_workflow_step_id IS NOT NULL THEN + SELECT ws.job_id + INTO linked_job_id + FROM workflow_steps ws + WHERE ws.id = current_workflow_step_id; + + IF linked_job_id IS DISTINCT FROM NEW.id THEN + RAISE EXCEPTION + 'workflow job linkage symmetry violation: job_queue.id=% job_queue.workflow_step_id=% workflow_steps.job_id=%', + NEW.id, + current_workflow_step_id, + linked_job_id + USING ERRCODE = '23514', + CONSTRAINT = 'os_workflow_job_linkage_symmetry'; + END IF; + ELSIF EXISTS ( + SELECT 1 + FROM workflow_steps ws + WHERE ws.job_id = NEW.id + ) THEN + RAISE EXCEPTION + 'workflow job linkage symmetry violation: job_queue.id=% has workflow_steps.job_id reference but job_queue.workflow_step_id is NULL', + NEW.id + USING ERRCODE = '23514', + CONSTRAINT = 'os_workflow_job_linkage_symmetry'; + END IF; + + RETURN NEW; + END IF; + + IF TG_TABLE_NAME = 'workflow_steps' THEN + SELECT ws.job_id + INTO current_job_id + FROM workflow_steps ws + WHERE ws.id = NEW.id; + + IF current_job_id IS NOT NULL THEN + SELECT jq.workflow_step_id + INTO linked_workflow_step_id + FROM job_queue jq + WHERE jq.id = current_job_id; + + IF linked_workflow_step_id IS DISTINCT FROM NEW.id THEN + RAISE EXCEPTION + 'workflow job linkage symmetry violation: workflow_steps.id=% workflow_steps.job_id=% job_queue.workflow_step_id=%', + NEW.id, + current_job_id, + linked_workflow_step_id + USING ERRCODE = '23514', + CONSTRAINT = 'os_workflow_job_linkage_symmetry'; + END IF; + ELSIF EXISTS ( + SELECT 1 + FROM job_queue jq + WHERE jq.workflow_step_id = NEW.id + ) THEN + RAISE EXCEPTION + 'workflow job linkage symmetry violation: workflow_steps.id=% has job_queue.workflow_step_id reference but workflow_steps.job_id is NULL', + NEW.id + USING ERRCODE = '23514', + CONSTRAINT = 'os_workflow_job_linkage_symmetry'; + END IF; + + RETURN NEW; + END IF; + + RAISE EXCEPTION + 'workflow job linkage symmetry trigger called by unsupported table: %', + TG_TABLE_NAME + USING ERRCODE = '23514', + CONSTRAINT = 'os_workflow_job_linkage_symmetry_trigger_table'; +END; +$$; + +CREATE CONSTRAINT TRIGGER trg_job_queue_workflow_step_linkage_symmetry +AFTER INSERT OR UPDATE OF workflow_step_id ON job_queue +DEFERRABLE INITIALLY DEFERRED +FOR EACH ROW +EXECUTE FUNCTION enforce_workflow_job_linkage_symmetry(); + +CREATE CONSTRAINT TRIGGER trg_workflow_steps_job_linkage_symmetry +AFTER INSERT OR UPDATE OF job_id ON workflow_steps +DEFERRABLE INITIALLY DEFERRED +FOR EACH ROW +EXECUTE FUNCTION enforce_workflow_job_linkage_symmetry(); + +CREATE TABLE job_logs ( + id bigint PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + job_id uuid NOT NULL, + run_number integer NOT NULL DEFAULT 1, + attempt integer, + level text NOT NULL, + message text NOT NULL, + payload jsonb NOT NULL DEFAULT '{}'::jsonb, + occurred_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT fk_job_logs_job + FOREIGN KEY (job_id) + REFERENCES job_queue (id) ON DELETE CASCADE, + CONSTRAINT chk_job_logs_attempt_nonnegative + CHECK (attempt IS NULL OR attempt >= 0), + CONSTRAINT chk_job_logs_run_number_positive + CHECK (run_number > 0), + CONSTRAINT chk_job_logs_level_not_blank + CHECK (length(trim(level)) > 0), + CONSTRAINT chk_job_logs_message_not_blank + CHECK (length(trim(message)) > 0) +); + +CREATE INDEX idx_job_logs_job_run_id + ON job_logs (job_id, run_number, id); + +CREATE INDEX idx_job_logs_level_time + ON job_logs (level, occurred_at DESC); + +CREATE TABLE job_runtime_configs ( + job_type text PRIMARY KEY, + schema_version integer NOT NULL, + config jsonb NOT NULL, + updated_by_user_id uuid, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT fk_job_runtime_configs_job_type + FOREIGN KEY (job_type) + REFERENCES job_definitions (job_type) ON DELETE CASCADE, + CONSTRAINT chk_job_runtime_configs_schema_version_positive + CHECK (schema_version > 0), + CONSTRAINT chk_job_runtime_configs_config_object + CHECK (jsonb_typeof(config) = 'object') +); + +CREATE TRIGGER trg_job_runtime_configs_set_updated_at + BEFORE UPDATE ON job_runtime_configs + FOR EACH ROW + EXECUTE FUNCTION set_updated_at_timestamp(); + +CREATE TABLE workflow_run_mutations ( + id uuid PRIMARY KEY DEFAULT uuidv7(), + workflow_run_id uuid NOT NULL, + mutation_key text NOT NULL, + metadata jsonb NOT NULL DEFAULT '{}'::jsonb, + request jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT fk_workflow_run_mutations_workflow_run + FOREIGN KEY (workflow_run_id) + REFERENCES workflow_runs (id) ON DELETE CASCADE, + CONSTRAINT chk_workflow_run_mutations_mutation_key_not_blank + CHECK (length(trim(mutation_key)) > 0) +); + +CREATE TRIGGER trg_workflow_run_mutations_set_updated_at + BEFORE UPDATE ON workflow_run_mutations + FOR EACH ROW + EXECUTE FUNCTION set_updated_at_timestamp(); + +CREATE UNIQUE INDEX uq_workflow_run_mutations_run_mutation_key + ON workflow_run_mutations (workflow_run_id, mutation_key); + +CREATE VIEW job_metrics_rollup AS +WITH queue_counts AS ( + SELECT + jq.organization_id, + jq.job_type, + COUNT(*) FILTER (WHERE jq.status = 'PENDING')::bigint AS pending_count, + COUNT(*) FILTER (WHERE jq.status = 'LEASED')::bigint AS leased_count, + COUNT(*) FILTER ( + WHERE jq.status = 'LEASED' + AND jq.lease_expires_at IS NOT NULL + AND jq.lease_expires_at < now() + )::bigint AS stale_leases + FROM job_queue jq + GROUP BY jq.organization_id, jq.job_type +), +recent_attempts AS ( + SELECT + jq.organization_id, + jq.job_type, + ja.outcome, + CASE + WHEN ja.finished_at IS NOT NULL THEN extract(epoch FROM (ja.finished_at - ja.started_at)) * 1000.0 + ELSE NULL + END AS duration_ms + FROM job_attempts ja + JOIN job_queue jq ON jq.id = ja.job_id + WHERE ja.created_at >= now() - interval '24 hours' +), +attempt_rollup AS ( + SELECT + ra.organization_id, + ra.job_type, + COUNT(*) FILTER (WHERE ra.outcome = 'RETRYABLE')::bigint AS retryable_24h, + COUNT(*) FILTER (WHERE ra.outcome = 'TERMINAL')::bigint AS terminal_24h, + COUNT(*) FILTER (WHERE ra.outcome = 'PANICKED')::bigint AS panicked_24h, + COUNT(*) FILTER (WHERE ra.outcome = 'TIMEOUT')::bigint AS timeout_24h, + percentile_cont(0.50) WITHIN GROUP (ORDER BY ra.duration_ms) AS p50_duration_ms_24h, + percentile_cont(0.95) WITHIN GROUP (ORDER BY ra.duration_ms) AS p95_duration_ms_24h + FROM recent_attempts ra + WHERE ra.duration_ms IS NOT NULL + GROUP BY ra.organization_id, ra.job_type +), +dead_letter_rollup AS ( + SELECT + jdl.organization_id, + jdl.job_type, + COUNT(*)::bigint AS dead_lettered_24h + FROM job_dead_letters jdl + WHERE jdl.failed_at >= now() - interval '24 hours' + GROUP BY jdl.organization_id, jdl.job_type +), +succeeded_rollup AS ( + SELECT + jq.organization_id, + jq.job_type, + COUNT(*)::bigint AS succeeded_24h + FROM job_events je + JOIN job_queue jq ON jq.id = je.job_id + WHERE je.event_type = 'SUCCEEDED' + AND je.occurred_at >= now() - interval '24 hours' + GROUP BY jq.organization_id, jq.job_type +), +job_keys AS ( + SELECT qc.organization_id, qc.job_type + FROM queue_counts qc + UNION + SELECT ar.organization_id, ar.job_type + FROM attempt_rollup ar + UNION + SELECT dl.organization_id, dl.job_type + FROM dead_letter_rollup dl + UNION + SELECT sr.organization_id, sr.job_type + FROM succeeded_rollup sr +) +SELECT + jk.organization_id, + jk.job_type, + COALESCE(qc.pending_count, 0) AS pending_count, + COALESCE(qc.leased_count, 0) AS leased_count, + COALESCE(qc.stale_leases, 0) AS stale_leases, + COALESCE(sr.succeeded_24h, 0) AS succeeded_24h, + COALESCE(ar.retryable_24h, 0) AS retryable_24h, + COALESCE(ar.terminal_24h, 0) AS terminal_24h, + COALESCE(ar.panicked_24h, 0) AS panicked_24h, + COALESCE(ar.timeout_24h, 0) AS timeout_24h, + COALESCE(dl.dead_lettered_24h, 0) AS dead_lettered_24h, + ar.p50_duration_ms_24h, + ar.p95_duration_ms_24h +FROM job_keys jk +LEFT JOIN queue_counts qc + ON qc.organization_id IS NOT DISTINCT FROM jk.organization_id + AND qc.job_type = jk.job_type +LEFT JOIN attempt_rollup ar + ON ar.organization_id IS NOT DISTINCT FROM jk.organization_id + AND ar.job_type = jk.job_type +LEFT JOIN dead_letter_rollup dl + ON dl.organization_id IS NOT DISTINCT FROM jk.organization_id + AND dl.job_type = jk.job_type +LEFT JOIN succeeded_rollup sr + ON sr.organization_id IS NOT DISTINCT FROM jk.organization_id + AND sr.job_type = jk.job_type; diff --git a/smoke/external-consumer/tests/fixtures/hocr-runledger-0.5.0/202604100001_runledger_migration_history.sql b/smoke/external-consumer/tests/fixtures/hocr-runledger-0.5.0/202604100001_runledger_migration_history.sql new file mode 100644 index 0000000..bd47f0a --- /dev/null +++ b/smoke/external-consumer/tests/fixtures/hocr-runledger-0.5.0/202604100001_runledger_migration_history.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS runledger_migration_history ( + version BIGINT PRIMARY KEY, + recorded_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +INSERT INTO runledger_migration_history (version) +VALUES (202603280001), (202604100001) +ON CONFLICT (version) DO NOTHING; diff --git a/smoke/external-consumer/tests/fixtures/hocr-runledger-0.5.0/202605180001_add_enqueue_request_snapshots.sql b/smoke/external-consumer/tests/fixtures/hocr-runledger-0.5.0/202605180001_add_enqueue_request_snapshots.sql new file mode 100644 index 0000000..00e2f7b --- /dev/null +++ b/smoke/external-consumer/tests/fixtures/hocr-runledger-0.5.0/202605180001_add_enqueue_request_snapshots.sql @@ -0,0 +1,12 @@ +-- Existing rows intentionally keep NULL snapshots. Their original enqueue +-- requests cannot be reconstructed safely after workflow steps or queue rows +-- may have been mutated, so runtime code uses explicit legacy fallback checks. +ALTER TABLE job_queue + ADD COLUMN IF NOT EXISTS enqueue_request jsonb; + +ALTER TABLE workflow_runs + ADD COLUMN IF NOT EXISTS enqueue_request jsonb; + +INSERT INTO runledger_migration_history (version) +VALUES (202605180001) +ON CONFLICT (version) DO NOTHING; diff --git a/smoke/external-consumer/tests/fixtures/hocr-runledger-0.5.0/202605220001_enforce_enqueue_request_snapshots.sql b/smoke/external-consumer/tests/fixtures/hocr-runledger-0.5.0/202605220001_enforce_enqueue_request_snapshots.sql new file mode 100644 index 0000000..a25e18e --- /dev/null +++ b/smoke/external-consumer/tests/fixtures/hocr-runledger-0.5.0/202605220001_enforce_enqueue_request_snapshots.sql @@ -0,0 +1,38 @@ +-- Block new keyed rows that do not carry the canonical enqueue request +-- snapshot, while allowing startup validation to report any pre-cutover rows. +-- These constraints start NOT VALID so the migration can be applied before +-- legacy rows are remediated; migrate_after_idempotency_cutover validates them +-- after the legacy-row check passes. +-- +-- The helper indexes are intentionally regular CREATE INDEX statements because +-- SQLx migrator runs migrations transactionally. Apply this migration during a +-- maintenance window sized for your write volume on job_queue/workflow_runs. +CREATE INDEX IF NOT EXISTS idx_job_queue_missing_enqueue_request_snapshot + ON job_queue (id) + WHERE idempotency_key IS NOT NULL + AND enqueue_request IS NULL; + +CREATE INDEX IF NOT EXISTS idx_workflow_runs_missing_enqueue_request_snapshot + ON workflow_runs (id) + WHERE idempotency_key IS NOT NULL + AND enqueue_request IS NULL; + +ALTER TABLE job_queue + DROP CONSTRAINT IF EXISTS ck_job_queue_idempotency_enqueue_request; + +ALTER TABLE job_queue + ADD CONSTRAINT ck_job_queue_idempotency_enqueue_request + CHECK (idempotency_key IS NULL OR enqueue_request IS NOT NULL) + NOT VALID; + +ALTER TABLE workflow_runs + DROP CONSTRAINT IF EXISTS ck_workflow_runs_idempotency_enqueue_request; + +ALTER TABLE workflow_runs + ADD CONSTRAINT ck_workflow_runs_idempotency_enqueue_request + CHECK (idempotency_key IS NULL OR enqueue_request IS NOT NULL) + NOT VALID; + +INSERT INTO runledger_migration_history (version) +VALUES (202605220001) +ON CONFLICT (version) DO NOTHING; diff --git a/smoke/external-consumer/tests/fixtures/hocr-runledger-0.5.0/202606030001_workflow_results.sql b/smoke/external-consumer/tests/fixtures/hocr-runledger-0.5.0/202606030001_workflow_results.sql new file mode 100644 index 0000000..d69cc3a --- /dev/null +++ b/smoke/external-consumer/tests/fixtures/hocr-runledger-0.5.0/202606030001_workflow_results.sql @@ -0,0 +1,31 @@ +ALTER TABLE job_queue + ADD COLUMN output jsonb; + +ALTER TABLE workflow_steps + ADD COLUMN output jsonb; + +ALTER TABLE workflow_runs + ADD COLUMN result_step_key text, + ADD COLUMN result jsonb; + +ALTER TABLE workflow_runs + ADD CONSTRAINT chk_workflow_runs_result_step_key_not_blank + CHECK (result_step_key IS NULL OR length(trim(result_step_key)) > 0) + NOT VALID; + +ALTER TABLE workflow_runs + VALIDATE CONSTRAINT chk_workflow_runs_result_step_key_not_blank; + +ALTER TABLE workflow_runs + ADD CONSTRAINT fk_workflow_runs_result_step + FOREIGN KEY (id, result_step_key) + REFERENCES workflow_steps (workflow_run_id, step_key) + DEFERRABLE INITIALLY DEFERRED + NOT VALID; + +ALTER TABLE workflow_runs + VALIDATE CONSTRAINT fk_workflow_runs_result_step; + +INSERT INTO runledger_migration_history (version) +VALUES (202606030001) +ON CONFLICT (version) DO NOTHING; diff --git a/smoke/external-consumer/tests/smoke.rs b/smoke/external-consumer/tests/smoke.rs index 97ac127..af23bfe 100644 --- a/smoke/external-consumer/tests/smoke.rs +++ b/smoke/external-consumer/tests/smoke.rs @@ -19,7 +19,8 @@ use runledger_postgres::jobs::{ get_job_enqueue_intent_by_id, record_job_enqueue_intent_tx, upsert_job_definition_tx, }; use runledger_postgres::prelude::{ - DbPool, DecodedJobEventPayload, DecodedRequeuedEventPayload, JobEventRecord, list_job_events, + DbPool, DecodedJobEventPayload, DecodedRequeuedEventPayload, JobEventRecord, + enqueue_job_with_outcome, list_job_events, }; use runledger_runtime::Supervisor; use runledger_runtime::catalog::JobCatalog; @@ -31,6 +32,9 @@ use sqlx::types::Uuid; use tokio::sync::{Mutex, Notify}; use tokio::time::{Instant, sleep, timeout}; +#[path = "support/migration_identity.rs"] +mod migration_identity; + const SMOKE_JOB_TYPE: &str = "jobs.external.smoke"; const SMOKE_POOL_MAX_CONNECTIONS: u32 = 12; const CONTINUATION_CHECKPOINT_VERSION: i64 = 1; @@ -208,19 +212,13 @@ async fn assert_keyed_recovery(pool: &DbPool, recovery_payload: &Value) -> Uuid idempotency_key: Some("external-smoke-recovery"), stage: None, }; - let mut recovery_enqueue_tx = pool.begin().await.expect("begin recovery enqueue"); - let inserted_recovery = - enqueue_job_with_outcome_tx(&mut recovery_enqueue_tx, &recovery_request) - .await - .expect("insert recovery job with outcome"); + let inserted_recovery = enqueue_job_with_outcome(pool, &recovery_request) + .await + .expect("insert recovery job with outcome"); assert_eq!( inserted_recovery.disposition, JobEnqueueDisposition::Inserted ); - recovery_enqueue_tx - .commit() - .await - .expect("commit recovery enqueue"); cancel_job_with_scope( pool, JobCancellationScope::Global, @@ -653,9 +651,7 @@ impl SmokeHandler { if slice < max_runs { Ok(JobCompletion::continue_after(Duration::from_millis(25)) .progress(slice, max_runs) - .map_err(|error| { - JobFailure::terminal("smoke.invalid_progress", error.to_string()) - })? + .map_err(|error| JobFailure::terminal("smoke.invalid_progress", error.to_string()))? .checkpoint(json!({ "version": CONTINUATION_CHECKPOINT_VERSION, "cursor": slice, @@ -663,9 +659,7 @@ impl SmokeHandler { } else { JobCompletion::success() .progress(slice, max_runs) - .map_err(|error| { - JobFailure::terminal("smoke.invalid_progress", error.to_string()) - }) + .map_err(|error| JobFailure::terminal("smoke.invalid_progress", error.to_string())) } } } @@ -1140,3 +1134,71 @@ fn payload_kind(payload: &Value) -> &str { .and_then(Value::as_str) .unwrap_or("unknown") } + +#[tokio::test] +async fn packaged_prelude_exports_explicit_metric_and_payload_scopes() { + use runledger_postgres::prelude::{ + JobEnqueueIntentReadMetricsFilter, JobReadScope, JobScope, + get_job_continuation_metrics_with_scope, get_job_enqueue_intent_metrics_with_scope, + get_job_metrics_with_scope, get_job_payload_by_idempotency_key_with_scope, + get_latest_job_payload_for_run_with_scope, + }; + let (pool, database) = setup_unmigrated_ephemeral_pool("consumer_explicit_scopes", 2).await; + runledger_postgres::migrate_after_idempotency_cutover(&pool) + .await + .expect("packaged explicit scope API succeeds"); + let tenant = Uuid::now_v7(); + for scope in [ + JobReadScope::Global, + JobReadScope::Organization(tenant), + JobReadScope::Admin, + ] { + assert!( + get_job_metrics_with_scope(&pool, scope, Some(SMOKE_JOB_TYPE)) + .await + .expect("packaged explicit scope API succeeds") + .is_empty() + ); + assert!( + get_job_continuation_metrics_with_scope(&pool, scope, Some(SMOKE_JOB_TYPE)) + .await + .expect("packaged explicit scope API succeeds") + .is_empty() + ); + assert!( + get_job_enqueue_intent_metrics_with_scope( + &pool, + &JobEnqueueIntentReadMetricsFilter::new(scope, 10, 0) + .with_job_type(JobType::new(SMOKE_JOB_TYPE)) + ) + .await + .expect("packaged explicit scope API succeeds") + .is_empty() + ); + } + for scope in [JobScope::Global, JobScope::Organization(tenant)] { + assert_eq!( + get_job_payload_by_idempotency_key_with_scope( + &pool, + scope, + JobType::new(SMOKE_JOB_TYPE), + "missing" + ) + .await + .expect("packaged explicit scope API succeeds"), + None + ); + assert_eq!( + get_latest_job_payload_for_run_with_scope( + &pool, + scope, + JobType::new(SMOKE_JOB_TYPE), + Uuid::nil() + ) + .await + .expect("packaged explicit scope API succeeds"), + None + ); + } + teardown_ephemeral_pool(pool, database).await; +} diff --git a/smoke/external-consumer/tests/support/migration_identity.rs b/smoke/external-consumer/tests/support/migration_identity.rs new file mode 100644 index 0000000..5ac4703 --- /dev/null +++ b/smoke/external-consumer/tests/support/migration_identity.rs @@ -0,0 +1,195 @@ +use postgres_test_harness::{FingerprintBuilder, TemplateFingerprint}; +use runledger_postgres::{RUNLEDGER_POSTGRES_VERSION, migration_bundle}; +use sqlx::migrate::{Migration, MigrationType}; + +const HISTORICAL_VERSIONS: [i64; 5] = [ + 202603280001, + 202604100001, + 202605180001, + 202605220001, + 202606030001, +]; + +// The exact adapter shape used by IdentityPro, with small host/Runlimit inputs. +// The real FingerprintBuilder is used, without starting its database harness. +fn add_migrator_inputs( + mut builder: FingerprintBuilder, + namespace: &str, + migrations: &[Migration], +) -> FingerprintBuilder { + for migration in migrations { + builder = builder + .add( + format!("{namespace}:version"), + migration.version.to_be_bytes(), + ) + .add( + format!("{namespace}:description"), + migration.description.as_bytes(), + ) + .add( + format!("{namespace}:type"), + migration.migration_type.suffix(), + ) + .add(format!("{namespace}:checksum"), &migration.checksum) + .add(format!("{namespace}:no_tx"), [u8::from(migration.no_tx)]); + } + builder +} + +fn composed_identity( + host_domain: &str, + owner_order: &[&str], + host_sql: &'static str, + runlimit_sql: &'static str, + runlimit_version: &str, +) -> TemplateFingerprint { + let mut builder = FingerprintBuilder::new(host_domain); + for &owner in owner_order { + builder = match owner { + "runledger" => builder.add( + "runledger-postgres:pipeline-fingerprint", + migration_bundle().pipeline_fingerprint(), + ), + "runlimit" => add_migrator_inputs( + builder.add("runlimit-postgres:pipeline-version", runlimit_version), + owner, + &[Migration::new( + 1, + "limit".into(), + MigrationType::Simple, + runlimit_sql.into(), + false, + )], + ), + "identitypro" => add_migrator_inputs( + builder, + owner, + &[Migration::new( + 2, + "host".into(), + MigrationType::Simple, + host_sql.into(), + false, + )], + ), + _ => panic!("unexpected fixture owner"), + }; + } + builder.finish() +} + +#[test] +fn identitypro_composition_retains_host_and_other_library_inputs() { + let order = ["runledger", "runlimit", "identitypro"]; + let fingerprint = composed_identity("host-v16", &order, "SELECT 1", "SELECT 2", "0.3.0"); + assert_eq!( + fingerprint, + composed_identity("host-v16", &order, "SELECT 1", "SELECT 2", "0.3.0") + ); + let changed = [ + composed_identity("host-v17", &order, "SELECT 1", "SELECT 2", "0.3.0"), + composed_identity("host-v16", &order, "SELECT 3", "SELECT 2", "0.3.0"), + composed_identity("host-v16", &order, "SELECT 1", "SELECT 3", "0.3.0"), + composed_identity("host-v16", &order, "SELECT 1", "SELECT 2", "0.3.1"), + composed_identity( + "host-v16", + &["runlimit", "runledger", "identitypro"], + "SELECT 1", + "SELECT 2", + "0.3.0", + ), + composed_identity( + "host-v16", + &["runlimit", "identitypro"], + "SELECT 1", + "SELECT 2", + "0.3.0", + ), + ]; + for candidate in changed { + assert_ne!(fingerprint, candidate); + } + assert_eq!( + migration_bundle().library_version(), + RUNLEDGER_POSTGRES_VERSION + ); + assert_ne!(RUNLEDGER_POSTGRES_VERSION, env!("CARGO_PKG_VERSION")); +} + +fn verify_historical_bundle( + vendored: &[Migration], + upstream: &[&Migration], +) -> Result<(), &'static str> { + if vendored + .iter() + .map(|entry| entry.version) + .collect::>() + != HISTORICAL_VERSIONS + || upstream + .iter() + .map(|entry| entry.version) + .collect::>() + != HISTORICAL_VERSIONS + { + return Err("historical version set differs"); + } + for (vendored, upstream) in vendored.iter().zip(upstream) { + if vendored.checksum != upstream.checksum || vendored.sql != upstream.sql { + return Err("historical SQL or checksum differs"); + } + } + Ok(()) +} + +#[test] +fn hocr_historical_bundle_matches_published_manifest_and_detects_drift() { + // Independent snapshot copied from HOCR, not read from the Runledger tree. + let vendored = sqlx::migrate!("tests/fixtures/hocr-runledger-0.5.0"); + let originals: Vec<_> = vendored.iter().cloned().collect(); + // This is the historical prefix check during an upgrade. The current bundle + // also has newer entries: it must never be described as the entire 0.5 bundle. + let upstream: Vec<_> = migration_bundle() + .migrations() + .filter(|entry| entry.migration_type.is_up_migration() && entry.version <= 202606030001) + .collect(); + assert!( + migration_bundle() + .migrations() + .any(|entry| entry.version > 202606030001) + ); + assert_eq!(verify_historical_bundle(&originals, &upstream), Ok(())); + + let mut changed = originals.clone(); + changed[0] = Migration::new( + changed[0].version, + changed[0].description.clone(), + changed[0].migration_type, + "SELECT 'modified historical SQL'".into(), + changed[0].no_tx, + ); + assert_eq!( + verify_historical_bundle(&changed, &upstream), + Err("historical SQL or checksum differs") + ); + changed = originals.clone(); + changed[0].sql = "SELECT 'tampered without updating checksum'".into(); + assert_eq!( + verify_historical_bundle(&changed, &upstream), + Err("historical SQL or checksum differs") + ); + changed = originals.clone(); + changed[0].checksum = vec![0; 48].into(); + assert_eq!( + verify_historical_bundle(&changed, &upstream), + Err("historical SQL or checksum differs") + ); + assert_eq!( + verify_historical_bundle(&originals[1..], &upstream), + Err("historical version set differs") + ); + assert_eq!( + verify_historical_bundle(&originals, &upstream[1..]), + Err("historical version set differs") + ); +}