From a366b4549bf0b7e8811bf5636a2da35bed6e2eac Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Tue, 10 Nov 2026 09:49:16 +0600 Subject: [PATCH 1/4] feat: workflow counts + task times (contract v0.9.0) - WorkflowCounts (contract.WorkflowCounter): GROUP BY state, same rows as WorkflowKeys - index __sqliteq_wf_workflows_state (state, created_at DESC, id) - task times on TaskRecord + Undo: EnqueuedAt at release, StartedAt in Runnable on true, FinishedAt at end/cancel/drop/fail-fast - Runnable: one write tx, check then stamp; unknown workflow makes no row - __sqliteq_wf_tasks: six INTEGER DEFAULT 0 time columns, guarded ALTER at open - on-demand purge store runs init: old tables migrate before Purge loads them - tests: counts per state incl. purge + sweep, task times, v0.8 schema migration --- CHANGELOG.md | 5 ++ README.md | 2 +- migration_test.go | 198 +++++++++++++++++++++++++++++++++++++++++++ queue_test.go | 205 +++++++++++++++++++++++++++++++++++++++++++++ wfmachine.go | 16 ++-- workflow.go | 149 +++++++++++++++++++++++--------- workflow_finish.go | 10 ++- 7 files changed, 537 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d98ee7..38d8a87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `WithSingleFile`: the end, the pushes and the ack are one transaction. (#25) - A redelivered job, or a push repeated after the job was removed, is safe: the pre-run check skips a task that already ended. (#25) - `Workflows` (`contract.WorkflowInspector`) and `WorkflowKeys` (`contract.WorkflowKeyLister`) list workflows newest first. (#25) +- `WorkflowCounts` (`contract.WorkflowCounter`, contract v0.9.0): stored workflows per state, same rows as `WorkflowKeys`. +- Task times on `TaskRecord` and its `Undo`: `EnqueuedAt` at push, `StartedAt` when `Runnable` last answered true, `FinishedAt` at end. Unix ms, store clock. +- `Runnable` is one write transaction: check, then stamp on a true answer. Unknown workflow: `ErrWorkflowNotFound`, no row made. +- `__sqliteq_wf_tasks` columns `enqueued_at`, `started_at`, `finished_at`, `undo_enqueued_at`, `undo_started_at`, `undo_finished_at` (INTEGER, default 0). Added at open to older tables; old rows read 0. +- Index `__sqliteq_wf_workflows_state` on `(state, created_at DESC, id)`. - A cancel mark never drops a valid workflow item; the workflow decides its end. (#25) - Task JSON whose keys differ only in case (`WF` next to `wf`) is a plain job. (#25) - `Purge` ends each purged workflow task as cancelled, reason `queue purged`; its workflow goes on. (#25) diff --git a/README.md b/README.md index de17d58..a65ddf3 100644 --- a/README.md +++ b/README.md @@ -293,7 +293,7 @@ Options of `NewWorkflowStore`: Every queue of a workflow must have the same scope. The scope of a queue is `ScopeID()`: `sqliteq:` and the absolute path of the store folder, or of the file under `WithSingleFile`, with symlinks resolved. A file already open under one scope cannot be opened under another, for example as a single file and as a queue file of a folder. `NewQueue` returns an error then. `WithSingleFile` refuses the file name `__sqliteq_workflows.db`. One process holds one store for each scope. A second `NewWorkflowStore` for the same scope returns `contract.ErrScopeRegistered`. -The store implements `contract.WorkflowStore`, `WorkflowInspector` and `WorkflowKeyLister`. So VarMQ's REST API lists and cancels sqliteq workflows. +The store implements `contract.WorkflowStore`, `WorkflowInspector`, `WorkflowKeyLister` and `WorkflowCounter`. So VarMQ's REST API lists, counts and cancels sqliteq workflows. Each task record carries its times: enqueued, started, finished (Unix ms, 0 when not reached). **Where the state lives.** In the folder layout the store uses its own file, `__sqliteq_workflows.db`. Under `WithSingleFile` it uses three tables of the store file. The tables are `__sqliteq_wf_workflows`, `__sqliteq_wf_tasks` and `__sqliteq_wf_outbox`. A queue name must not contain `__sqliteq_`, so a queue never takes these names. diff --git a/migration_test.go b/migration_test.go index be706f8..8e70673 100644 --- a/migration_test.go +++ b/migration_test.go @@ -1,11 +1,15 @@ package sqliteq import ( + "context" "database/sql" "fmt" + "maps" "path/filepath" "sort" "testing" + + "github.com/goptics/varmq/contract" ) // The v0.2.3 schema, pasted from tag v0.2.3's queue.go initTable and @@ -247,3 +251,197 @@ func TestPendingIndexNameFreeForQueues(t *testing.T) { }) } } + +// The workflow tables as #25 (contract v0.8.0) made them: no task times, no +// state index. Pasted from that commit's workflow.go init. +const v08WorkflowTables = ` + CREATE TABLE "__sqliteq_wf_workflows" ( + id TEXT PRIMARY KEY, + hash TEXT NOT NULL, + state TEXT NOT NULL, + created_at INTEGER NOT NULL, + finished_at INTEGER, + fail_fast INTEGER NOT NULL, + undo_on_cancel INTEGER NOT NULL, + failed INTEGER NOT NULL DEFAULT 0, + cancelled INTEGER NOT NULL DEFAULT 0 + ); + CREATE INDEX "__sqliteq_wf_workflows_created" ON "__sqliteq_wf_workflows" (created_at DESC, id); + CREATE INDEX "__sqliteq_wf_workflows_finished" ON "__sqliteq_wf_workflows" (finished_at) WHERE finished_at IS NOT NULL; + CREATE TABLE "__sqliteq_wf_tasks" ( + workflow_id TEXT NOT NULL, + name TEXT NOT NULL, + ord INTEGER NOT NULL, + state TEXT NOT NULL, + reason TEXT NOT NULL DEFAULT '', + result BLOB, + open INTEGER NOT NULL DEFAULT 0, + deps TEXT NOT NULL, + always INTEGER NOT NULL, + queue_id TEXT NOT NULL, + job_id TEXT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + item BLOB, + has_undo INTEGER NOT NULL DEFAULT 0, + undo_queue_id TEXT NOT NULL DEFAULT '', + undo_job_id TEXT NOT NULL DEFAULT '', + undo_item BLOB, + undo_state TEXT NOT NULL DEFAULT '', + undo_open INTEGER NOT NULL DEFAULT 0, + undo_reason TEXT NOT NULL DEFAULT '', + PRIMARY KEY (workflow_id, name) + ) WITHOUT ROWID; + CREATE TABLE "__sqliteq_wf_outbox" ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + workflow_id TEXT NOT NULL, + queue_id TEXT NOT NULL, + job_id TEXT NOT NULL, + item BLOB NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL + ); + CREATE TABLE "__sqliteq_wf_purged" ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + queue_id TEXT NOT NULL, + workflow_id TEXT NOT NULL, + task TEXT NOT NULL, + undo INTEGER NOT NULL, + created_at INTEGER NOT NULL + ); + CREATE INDEX "__sqliteq_wf_outbox_workflow" ON "__sqliteq_wf_outbox" (workflow_id); +` + +// writeV08Workflow builds the v0.8.0 workflow tables in the file at path, +// with running workflow "old" on queue queueID: a completed, b pushed, c +// waiting for b. +func writeV08Workflow(t *testing.T, path, queueID string) { + t.Helper() + db, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatalf("Failed to open database: %v", err) + } + defer db.Close() + + exec := func(query string, args ...any) { + t.Helper() + if _, err := db.Exec(query, args...); err != nil { + t.Fatalf("Fixture statement failed: %v\n%s", err, query) + } + } + exec(v08WorkflowTables) + exec(`INSERT INTO "__sqliteq_wf_workflows" (id, hash, state, created_at, fail_fast, undo_on_cancel) VALUES ('old', 'h', 'running', 1000, 0, 0)`) + for i, r := range []struct { + name, state, deps string + open int + item []byte + }{ + {"a", "completed", `[]`, 0, nil}, + {"b", "pending", `["a"]`, 0, nil}, + {"c", "waiting", `["b"]`, 1, wfItem("old", "c", false)}, + } { + exec(`INSERT INTO "__sqliteq_wf_tasks" (workflow_id, name, ord, state, open, deps, always, queue_id, job_id, item) + VALUES ('old', ?, ?, ?, ?, ?, 0, ?, ?, ?)`, r.name, i, r.state, r.open, r.deps, queueID, "wf:old:"+r.name, r.item) + } +} + +// Workflow tables of contract v0.8.0 open: the task time columns are added +// once, old rows read 0, new pushes, starts and ends are stamped. A Purge +// with no store open migrates the tables too. +func TestMigrateWorkflowTaskTimes(t *testing.T) { + ctx := context.Background() + open := func(t *testing.T, single bool) (Queues, *Queue) { + t.Helper() + dir := t.TempDir() + root, wfPath, opts := dir, filepath.Join(dir, workflowsFile), []StoreOption(nil) + if single { + root = filepath.Join(dir, "all.db") + wfPath, opts = root, []StoreOption{WithSingleFile()} + } + qs := mustNew(t, root, opts...) + t.Cleanup(func() { qs.Close() }) + a, err := qs.NewQueue("a") + if err != nil { + t.Fatal(err) + } + writeV08Workflow(t, wfPath, a.QueueID()) + if !a.EnqueueOnce(wfItem("old", "b", false), "wf:old:b") { + t.Fatal("Failed to queue the old workflow's task") + } + return qs, a + } + store := func(t *testing.T, qs Queues) *WorkflowStore { + t.Helper() + st, err := NewWorkflowStore(qs) + if err != nil { + t.Fatalf("Failed to open the v0.8.0 workflow tables: %v", err) + } + t.Cleanup(func() { st.Close() }) + for _, col := range taskTimeCols { + if ok, err := hasColumn(st.f.db, "__sqliteq_wf_tasks", col); err != nil || !ok { + t.Errorf("Expected column %s, got %t %v", col, ok, err) + } + } + return st + } + + t.Run("open", func(t *testing.T) { + eachLayout(t, func(t *testing.T, single bool) { + qs, a := open(t, single) + st := store(t, qs) + r, err := st.Record(ctx, "old") + if err != nil { + t.Fatal(err) + } + if got := taskStates(r); got != "a=completed b=pending c=waiting" { + t.Errorf("got %s", got) + } + for _, tr := range r.Tasks { + if times(tr) != [3]int64{} { + t.Errorf("Expected no times on old task %s, got %v", tr.Name, times(tr)) + } + } + counts, err := st.WorkflowCounts(ctx) + if err != nil || !maps.Equal(counts, map[contract.WorkflowState]int64{contract.WorkflowRunning: 1}) { + t.Errorf("counts %v %v", counts, err) + } + st.Close() + + // Reopened: nothing to add; the old workflow goes on with times + st = store(t, qs) + if ok, err := st.Runnable(ctx, "old", "b", false); err != nil || !ok { + t.Fatalf("Runnable = %t %v", ok, err) + } + _, ack := take(t, a) + if _, ok := st.FinishTask(a.QueueID(), ack, done("old", "b", "")); !ok { + t.Fatal("FinishTask failed") + } + if r, err = st.Record(ctx, "old"); err != nil { + t.Fatal(err) + } + ta, tb, tc := r.Tasks[0], r.Tasks[1], r.Tasks[2] + if times(ta) != [3]int64{} || tb.EnqueuedAt != 0 || tb.StartedAt == 0 || tb.FinishedAt == 0 || tc.EnqueuedAt == 0 { + t.Errorf("Expected old times 0 and new ones set, got a %v b %v c %v", times(ta), times(tb), times(tc)) + } + if got := taskStates(r); got != "a=completed b=completed c=pending" || a.Len() != 1 { + t.Errorf("got %s, a %d", got, a.Len()) + } + }) + }) + + t.Run("purge", func(t *testing.T) { + eachLayout(t, func(t *testing.T, single bool) { + qs, a := open(t, single) + a.Purge() // no store open: the on-demand one works on the old tables + r, err := store(t, qs).Record(ctx, "old") + if err != nil { + t.Fatal(err) + } + if got := taskStates(r); got != "a=completed b=cancelled c=cancelled" || r.State != contract.WorkflowFailed { + t.Errorf("got %s %s", got, r.State) + } + if r.Tasks[1].FinishedAt == 0 || r.Tasks[2].FinishedAt == 0 || r.Tasks[1].EnqueuedAt != 0 { + t.Errorf("Expected the purge's ends stamped, got b %v c %v", times(r.Tasks[1]), times(r.Tasks[2])) + } + }) + }) +} diff --git a/queue_test.go b/queue_test.go index 2b04bc0..1abdb05 100644 --- a/queue_test.go +++ b/queue_test.go @@ -3297,3 +3297,208 @@ func TestWorkflowPurgeIntentWaitsForDelete(t *testing.T) { t.Errorf("Expected every intent cleared, got %d", intents()) } } + +// WorkflowCounts counts the stored workflows per state, as the listing sees +// them: an end moves a workflow, a purge fails it, the sweep drops it. +func TestWorkflowCounts(t *testing.T) { + eachLayout(t, func(t *testing.T, single bool) { + e := newWFEnv(t, single, WithWorkflowRetention(time.Hour)) + ctx := context.Background() + var clock atomic.Int64 + clock.Store(time.Now().UnixMilli()) + e.st.setClock(func() time.Time { return time.UnixMilli(clock.Load()) }) + check := func(what string, want map[contract.WorkflowState]int64) { + t.Helper() + got, err := e.st.WorkflowCounts(ctx) + if err != nil { + t.Fatal(err) + } + if !maps.Equal(got, want) { + t.Errorf("%s: counts %v, want %v", what, got, want) + } + for _, state := range []contract.WorkflowState{contract.WorkflowRunning, contract.WorkflowCompleted, contract.WorkflowFailed, + contract.WorkflowCompensating, contract.WorkflowCompensated, contract.WorkflowCompensationFailed, contract.WorkflowCancelled} { + keys, err := e.st.WorkflowKeys(ctx, state, 0, 100) + if err != nil || int64(len(keys)) != got[state] { + t.Errorf("%s: %s lists %d (%v), counts %d", what, state, len(keys), err, got[state]) + } + } + } + check("empty store", map[contract.WorkflowState]int64{}) + + for _, id := range []string{"run", "done", "done2", "fail", "cancel"} { + if err := e.st.Submit(ctx, planWF(id, planTask(id, "a", e.a))); err != nil { + t.Fatal(err) + } + } + if err := e.st.Submit(ctx, planWF("undo", withUndo(planTask("undo", "a", e.a), "undo", e.b), planTask("undo", "z", e.a, "a"))); err != nil { + t.Fatal(err) + } + check("submitted", map[contract.WorkflowState]int64{contract.WorkflowRunning: 6}) + + take(t, e.a) // run: taken, never finished + for _, end := range []contract.TaskEnd{ + done("done", "a", ""), + done("done2", "a", ""), + {Workflow: "fail", Task: "a", Outcome: contract.OutcomeFailed}, + {Workflow: "cancel", Task: "a", Outcome: contract.OutcomeCancelled}, + done("undo", "a", ""), + {Workflow: "undo", Task: "z", Outcome: contract.OutcomeFailed}, + } { + if end.Workflow == "cancel" { + if err := e.st.Cancel(ctx, "cancel", nil); err != nil { + t.Fatal(err) + } + } + _, ack := take(t, e.a) + e.finish(t, e.a, ack, end) + } + check("ended", map[contract.WorkflowState]int64{contract.WorkflowRunning: 1, contract.WorkflowCompleted: 2, + contract.WorkflowFailed: 1, contract.WorkflowCancelled: 1, contract.WorkflowCompensating: 1}) + + _, ack := take(t, e.b) + e.finish(t, e.b, ack, contract.TaskEnd{Workflow: "undo", Task: "a", Undo: true, Outcome: contract.OutcomeCompleted}) + check("compensated", map[contract.WorkflowState]int64{contract.WorkflowRunning: 1, contract.WorkflowCompleted: 2, + contract.WorkflowFailed: 1, contract.WorkflowCancelled: 1, contract.WorkflowCompensated: 1}) + + e.a.Purge() // ends run's task cancelled: the workflow fails + check("purged", map[contract.WorkflowState]int64{contract.WorkflowCompleted: 2, + contract.WorkflowFailed: 2, contract.WorkflowCancelled: 1, contract.WorkflowCompensated: 1}) + + if err := e.st.Submit(ctx, planWF("late", planTask("late", "a", e.a))); err != nil { + t.Fatal(err) + } + clock.Add(2 * time.Hour.Milliseconds()) + if err := e.st.sweep(ctx); err != nil { + t.Fatal(err) + } + check("swept", map[contract.WorkflowState]int64{contract.WorkflowRunning: 1}) + }) +} + +// times is r's time fields: enqueued, started, finished. +func times(r contract.TaskRecord) [3]int64 { + return [3]int64{r.EnqueuedAt, r.StartedAt, r.FinishedAt} +} + +// A task record carries when it was pushed, when the pre-run check last let +// it run and when it ended; so does its undo. A refused or unknown check +// writes nothing. +func TestWorkflowTaskTimes(t *testing.T) { + eachLayout(t, func(t *testing.T, single bool) { + e := newWFEnv(t, single) + ctx := context.Background() + var clock atomic.Int64 + clock.Store(1_000_000) + e.st.setClock(func() time.Time { return time.UnixMilli(clock.Load()) }) + tick := func() int64 { return clock.Add(1000) } + task := func(wf, name string) contract.TaskRecord { + t.Helper() + for _, tr := range e.state(t, wf).Tasks { + if tr.Name == name { + return tr + } + } + t.Fatalf("no task %s/%s", wf, name) + return contract.TaskRecord{} + } + want := func(what string, tr contract.TaskRecord, enqueued, started, finished int64) { + t.Helper() + if got, want := times(tr), [3]int64{enqueued, started, finished}; got != want { + t.Errorf("%s: times %v, want %v", what, got, want) + } + } + runnable := func(wf, name string, undo, want bool) { + t.Helper() + if ok, err := e.st.Runnable(ctx, wf, name, undo); err != nil || ok != want { + t.Fatalf("Runnable %s/%s undo %t = %t %v, want %t", wf, name, undo, ok, err, want) + } + } + + t0 := clock.Load() + plan := planWF("w", withUndo(planTask("w", "a", e.a), "w", e.b), planTask("w", "b", e.a, "a"), planTask("w", "c", e.a, "b")) + if err := e.st.Submit(ctx, plan); err != nil { + t.Fatal(err) + } + want("root pushed at submit", task("w", "a"), t0, 0, 0) + want("waiting", task("w", "b"), 0, 0, 0) + want("undo not reached", *task("w", "a").Undo, 0, 0, 0) + + tick() + runnable("w", "a", false, true) + t1 := tick() + runnable("w", "a", false, true) + runnable("w", "b", false, false) + want("last yes wins", task("w", "a"), t0, t1, 0) + want("a refused check stamps nothing", task("w", "b"), 0, 0, 0) + + t2 := tick() + _, ack := take(t, e.a) + e.finish(t, e.a, ack, done("w", "a", "")) + want("completed", task("w", "a"), t0, t1, t2) + want("released by a's end", task("w", "b"), t2, 0, 0) + + t3 := tick() + _, ack = take(t, e.a) + e.finish(t, e.a, ack, contract.TaskEnd{Workflow: "w", Task: "b", Outcome: contract.OutcomeFailed}) + want("failed without a check", task("w", "b"), t2, 0, t3) + want("cancelled while waiting", task("w", "c"), 0, 0, t3) + want("undo pushed", *task("w", "a").Undo, t3, 0, 0) + + t4 := tick() + runnable("w", "a", false, false) + runnable("w", "a", true, true) + t5 := tick() + _, ack = take(t, e.b) + e.finish(t, e.b, ack, contract.TaskEnd{Workflow: "w", Task: "a", Undo: true, Outcome: contract.OutcomeCompleted}) + want("undo ended", *task("w", "a").Undo, t3, t4, t5) + want("the undo leaves the task's own times", task("w", "a"), t0, t1, t2) + + // Cancel ends the waiting task now, the queued one at its finish + if err := e.st.Submit(ctx, planWF("x", planTask("x", "a", e.a), planTask("x", "b", e.a, "a"))); err != nil { + t.Fatal(err) + } + t6 := tick() + if err := e.st.Cancel(ctx, "x", nil); err != nil { + t.Fatal(err) + } + runnable("x", "a", false, false) + want("cancelled by Cancel", task("x", "b"), 0, 0, t6) + want("queued under a cancel", task("x", "a"), t5, 0, 0) + t7 := tick() + _, ack = take(t, e.a) + e.finish(t, e.a, ack, contract.TaskEnd{Workflow: "x", Task: "a", Outcome: contract.OutcomeCancelled}) + want("cancelled at its pre-run check", task("x", "a"), t5, 0, t7) + + // FailFast ends what waits; the queued task keeps only its push + ff := planWF("f", planTask("f", "a", e.a), planTask("f", "b", e.a), planTask("f", "c", e.a, "b")) + ff.FailFast = true + if err := e.st.Submit(ctx, ff); err != nil { + t.Fatal(err) + } + t8 := tick() + _, ack = take(t, e.a) + e.finish(t, e.a, ack, contract.TaskEnd{Workflow: "f", Task: "a", Outcome: contract.OutcomeFailed}) + want("dropped by FailFast", task("f", "c"), 0, 0, t8) + want("queued, stopped", task("f", "b"), t7, 0, 0) + + // A check of an unknown workflow or task creates no row + rows := func() (n int) { + t.Helper() + if err := e.st.f.db.QueryRow(`SELECT (SELECT COUNT(*) FROM ` + wfWorkflows + `) + (SELECT COUNT(*) FROM ` + wfTasks + `)`).Scan(&n); err != nil { + t.Fatal(err) + } + return n + } + before := rows() + for _, undo := range []bool{false, true} { + if ok, err := e.st.Runnable(ctx, "ghost", "a", undo); ok || !errors.Is(err, contract.ErrWorkflowNotFound) { + t.Errorf("Expected ErrWorkflowNotFound for an unknown workflow, got %t %v", ok, err) + } + runnable("f", "ghost", undo, false) + } + if n := rows(); n != before { + t.Errorf("Expected no row made by a check, got %d rows, had %d", n, before) + } + }) +} diff --git a/wfmachine.go b/wfmachine.go index b2695fc..9e8fc84 100644 --- a/wfmachine.go +++ b/wfmachine.go @@ -27,6 +27,11 @@ type wfTask struct { undoState contract.JobState undoOpen int // undos of completed descendants not ended yet undoReason string + + // Unix ms, 0 when not reached (TaskRecord). The machine stamps the + // finishes; the store stamps pushes and starts. + enqueued, started, finished int64 + undoEnqueued, undoStarted, undoFinished int64 } // wfUndo is a task's compensation job. @@ -37,6 +42,7 @@ type wfFlow struct { state contract.WorkflowState created int64 finished int64 + at int64 // now, Unix ms: the store sets it before finish and cancel failFast bool undoOnCancel bool failed bool // an unabsorbed failure happened before any cancel @@ -77,7 +83,7 @@ func (s *wfFlow) finish(e contract.TaskEnd) ([]wfEffect, error) { return nil, nil } - t.reason, t.result = e.Reason, e.Result + t.reason, t.result, t.finished = e.Reason, e.Result, s.at push := map[string]bool{} if e.Outcome == contract.OutcomeCompleted { t.state = contract.JobCompleted @@ -110,7 +116,7 @@ func (s *wfFlow) cancel(undo bool) []wfEffect { s.cancelled, s.undoOnCancel = true, undo for _, name := range s.order { if t := s.tasks[name]; t.state == contract.JobWaiting { - t.state = contract.JobCancelled + t.state, t.finished = contract.JobCancelled, s.at } } return s.settle(nil) @@ -133,7 +139,7 @@ func (s *wfFlow) finishUndo(t *wfTask, e contract.TaskEnd) []wfEffect { if s.state != contract.WorkflowCompensating || !active(t.undoState) { return nil } - t.undoState, t.undoReason = contract.JobFailed, e.Reason + t.undoState, t.undoReason, t.undoFinished = contract.JobFailed, e.Reason, s.at if e.Outcome == contract.OutcomeCompleted { t.undoState, t.undoReason = contract.JobCompleted, "" } @@ -191,7 +197,7 @@ func (s *wfFlow) drop(name string, push map[string]bool) { s.release(d, push) continue } - t.state = contract.JobCancelled + t.state, t.finished = contract.JobCancelled, s.at s.drop(d, push) } } @@ -212,7 +218,7 @@ func (s *wfFlow) failFast0(failed string, push map[string]bool) { var cancelled []string for _, name := range s.order { if t := s.tasks[name]; t.state == contract.JobWaiting && !keep[name] { - t.state = contract.JobCancelled + t.state, t.finished = contract.JobCancelled, s.at cancelled = append(cancelled, name) } } diff --git a/workflow.go b/workflow.go index 7be012c..f53877a 100644 --- a/workflow.go +++ b/workflow.go @@ -36,6 +36,7 @@ var ( _ contract.WorkflowStore = (*WorkflowStore)(nil) _ contract.WorkflowInspector = (*WorkflowStore)(nil) _ contract.WorkflowKeyLister = (*WorkflowStore)(nil) + _ contract.WorkflowCounter = (*WorkflowStore)(nil) ) const ( @@ -62,6 +63,11 @@ const ( purgeReason = "queue purged" ) +// taskTimeCols are the task time columns (contract v0.9.0), Unix ms, 0 when +// not reached. init adds them to a table made before them. +var taskTimeCols = [...]string{"enqueued_at", "started_at", "finished_at", + "undo_enqueued_at", "undo_started_at", "undo_finished_at"} + // recoverEvery is how often the recovery loop pushes leftover outbox rows; // those younger than it are left to the writer that is about to push them. var recoverEvery = time.Second @@ -174,7 +180,8 @@ func NewWorkflowStore(qs Queues, opts ...WorkflowOption) (*WorkflowStore, error) return s, nil } -// init creates the tables under the write mutex. +// init creates the tables under the write mutex, and adds the task time +// columns to a tasks table made before them. func (s *WorkflowStore) init() error { s.f.mu.Lock() defer s.f.mu.Unlock() @@ -193,6 +200,8 @@ func (s *WorkflowStore) init() error { )`, `CREATE INDEX IF NOT EXISTS "__sqliteq_wf_workflows_created" ON ` + wfWorkflows + ` (created_at DESC, id)`, `CREATE INDEX IF NOT EXISTS "__sqliteq_wf_workflows_finished" ON ` + wfWorkflows + ` (finished_at) WHERE finished_at IS NOT NULL`, + // state filter of the listing in order, and the counts + `CREATE INDEX IF NOT EXISTS "__sqliteq_wf_workflows_state" ON ` + wfWorkflows + ` (state, created_at DESC, id)`, `CREATE TABLE IF NOT EXISTS ` + wfTasks + ` ( workflow_id TEXT NOT NULL, name TEXT NOT NULL, @@ -214,6 +223,12 @@ func (s *WorkflowStore) init() error { undo_state TEXT NOT NULL DEFAULT '', undo_open INTEGER NOT NULL DEFAULT 0, undo_reason TEXT NOT NULL DEFAULT '', + enqueued_at INTEGER NOT NULL DEFAULT 0, + started_at INTEGER NOT NULL DEFAULT 0, + finished_at INTEGER NOT NULL DEFAULT 0, + undo_enqueued_at INTEGER NOT NULL DEFAULT 0, + undo_started_at INTEGER NOT NULL DEFAULT 0, + undo_finished_at INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (workflow_id, name) ) WITHOUT ROWID`, `CREATE TABLE IF NOT EXISTS ` + wfOutbox + ` ( @@ -239,6 +254,15 @@ func (s *WorkflowStore) init() error { return fmt.Errorf("sqliteq: failed to create workflow tables: %w", err) } } + for _, col := range taskTimeCols { + exists, err := hasColumn(s.f.db, strings.Trim(wfTasks, `"`), col) + if err == nil && !exists { + _, err = s.f.db.Exec(`ALTER TABLE ` + wfTasks + ` ADD COLUMN ` + col + ` INTEGER NOT NULL DEFAULT 0`) + } + if err != nil { + return fmt.Errorf("sqliteq: failed to add workflow task column %s: %w", col, err) + } + } return nil } @@ -651,20 +675,21 @@ func (s *WorkflowStore) stamp(w *wfFlow, fx []wfEffect) { } // release turns the pushes of fx into rows (reading and clearing each task's -// stored item) and delivers them in tx. +// stored item, stamping its enqueue time) and delivers them in tx. // Folder layout: outbox rows, minSeq the first one's seq (0 when none). // Single file: straight into the queues (ids), except those this client // cannot reach, which get outbox rows. func (s *WorkflowStore) release(ctx context.Context, tx *sql.Tx, w *wfFlow, fx []wfEffect) (minSeq int64, ids []string, err error) { var rows []outRow + now := s.now().UnixMilli() for _, e := range fx { if e.push == "" { continue } t := w.tasks[e.push] - col, row := "item", outRow{queueID: t.queueID, jobID: t.jobID, priority: t.priority} + col, set, row := "item", "item = NULL, enqueued_at = ?", outRow{queueID: t.queueID, jobID: t.jobID, priority: t.priority} if e.undo { - col, row = "undo_item", outRow{queueID: t.undo.queueID, jobID: t.undo.jobID} + col, set, row = "undo_item", "undo_enqueued_at = ?", outRow{queueID: t.undo.queueID, jobID: t.undo.jobID} } where := ` WHERE workflow_id = ? AND name = ?` if err = tx.QueryRowContext(ctx, `SELECT `+col+` FROM `+wfTasks+where, w.id, e.push).Scan(&row.item); err != nil { @@ -673,10 +698,8 @@ func (s *WorkflowStore) release(ctx context.Context, tx *sql.Tx, w *wfFlow, fx [ if row.item == nil { continue } - if !e.undo { - if _, err = tx.ExecContext(ctx, `UPDATE `+wfTasks+` SET item = NULL`+where, w.id, e.push); err != nil { - return 0, nil, err - } + if _, err = tx.ExecContext(ctx, `UPDATE `+wfTasks+` SET `+set+where, now, w.id, e.push); err != nil { + return 0, nil, err } rows = append(rows, row) } @@ -699,7 +722,7 @@ func (s *WorkflowStore) release(ctx context.Context, tx *sql.Tx, w *wfFlow, fx [ } res, err := tx.ExecContext(ctx, `INSERT INTO `+wfOutbox+ ` (workflow_id, queue_id, job_id, item, priority, created_at) VALUES (?, ?, ?, ?, ?, ?)`, - w.id, r.queueID, r.jobID, r.item, r.priority, s.now().UnixMilli()) + w.id, r.queueID, r.jobID, r.item, r.priority, now) if err != nil { return 0, nil, err } @@ -870,7 +893,8 @@ func (s *WorkflowStore) load(ctx context.Context, q querier, id string) (*wfFlow w.failFast, w.undoOnCancel, w.failed, w.cancelled = ff == 1, uoc == 1, f == 1, cn == 1 rows, err := q.QueryContext(ctx, `SELECT name, state, reason, open, deps, always, queue_id, job_id, priority, COALESCE(length(result), 0), - has_undo, undo_queue_id, undo_job_id, undo_state, undo_open, undo_reason + has_undo, undo_queue_id, undo_job_id, undo_state, undo_open, undo_reason, + enqueued_at, started_at, finished_at, undo_enqueued_at, undo_started_at, undo_finished_at FROM `+wfTasks+` WHERE workflow_id = ? ORDER BY ord`, id) if err != nil { return nil, err @@ -887,7 +911,8 @@ func (s *WorkflowStore) load(ctx context.Context, q querier, id string) (*wfFlow u wfUndo ) if err := rows.Scan(&t.name, &t.state, &t.reason, &t.open, &deps, &always, &t.queueID, &t.jobID, &t.priority, &t.resultSize, - &hasUndo, &u.queueID, &u.jobID, &t.undoState, &t.undoOpen, &t.undoReason); err != nil { + &hasUndo, &u.queueID, &u.jobID, &t.undoState, &t.undoOpen, &t.undoReason, + &t.enqueued, &t.started, &t.finished, &t.undoEnqueued, &t.undoStarted, &t.undoFinished); err != nil { return nil, err } if err := json.Unmarshal([]byte(deps), &t.deps); err != nil { @@ -940,7 +965,8 @@ func (w *wfFlow) dirty(orig map[string]mut) (tasks []string, flow bool) { return tasks, orig[""] != mut{wfState: w.state, failed: w.failed, cancelled: w.cancelled, undoCancel: w.undoOnCancel} } -// save writes what changed since orig; result is the finishing task's +// save writes what changed since orig, with the finish times the machine +// stamped (an end always changes the state); result is the finishing task's // result, written when that task is task. func (s *WorkflowStore) save(ctx context.Context, tx *sql.Tx, w *wfFlow, orig map[string]mut, task string, result []byte) error { tasks, flow := w.dirty(orig) @@ -956,9 +982,9 @@ func (s *WorkflowStore) save(ctx context.Context, tx *sql.Tx, w *wfFlow, orig ma } for _, name := range tasks { t := w.tasks[name] - if _, err := tx.ExecContext(ctx, `UPDATE `+wfTasks+` SET state = ?, open = ?, reason = ?, undo_state = ?, undo_open = ?, undo_reason = ? - WHERE workflow_id = ? AND name = ?`, - string(t.state), t.open, t.reason, string(t.undoState), t.undoOpen, t.undoReason, w.id, name); err != nil { + if _, err := tx.ExecContext(ctx, `UPDATE `+wfTasks+` SET state = ?, open = ?, reason = ?, finished_at = ?, + undo_state = ?, undo_open = ?, undo_reason = ?, undo_finished_at = ? WHERE workflow_id = ? AND name = ?`, + string(t.state), t.open, t.reason, t.finished, string(t.undoState), t.undoOpen, t.undoReason, t.undoFinished, w.id, name); err != nil { return err } if name == task && len(result) > 0 && t.state == contract.JobCompleted { @@ -1008,6 +1034,7 @@ func (s *WorkflowStore) Cancel(ctx context.Context, id string, undo *bool) error on = *undo } orig := w.snapshot() + w.at = s.now().UnixMilli() fx := w.cancel(on) s.stamp(w, fx) if err := s.save(ctx, tx, w, orig, "", nil); err != nil { @@ -1025,34 +1052,48 @@ func (s *WorkflowStore) Cancel(ctx context.Context, id string, undo *bool) error return nil } -// Runnable is the pre-run check (spec 6.12). +// Runnable is the pre-run check (spec 6.12). A yes stamps the start time of +// the task (undo: of its undo) in the same transaction: the job runs next. func (s *WorkflowStore) Runnable(ctx context.Context, workflowID, task string, undo bool) (bool, error) { if !validWorkflowID(workflowID) { return false, s.notFound(workflowID) } - var ( - state contract.WorkflowState - ff, failed, cancelled int - tstate, ustate contract.JobState - always, hasUndo, taskFound int - ) - err := s.f.db.QueryRowContext(ctx, `SELECT w.state, w.fail_fast, w.failed, w.cancelled, - COALESCE(t.state, ''), COALESCE(t.undo_state, ''), COALESCE(t.always, 0), COALESCE(t.has_undo, 0), t.name IS NOT NULL - FROM `+wfWorkflows+` w LEFT JOIN `+wfTasks+` t ON t.workflow_id = w.id AND t.name = ? WHERE w.id = ?`, task, workflowID). - Scan(&state, &ff, &failed, &cancelled, &tstate, &ustate, &always, &hasUndo, &taskFound) - switch { - case errors.Is(err, sql.ErrNoRows): - return false, s.notFound(workflowID) - case err != nil: - return false, err - case taskFound == 0: - return false, nil - } - if undo { - return state == contract.WorkflowCompensating && hasUndo == 1 && ustate == contract.JobPending, nil - } - stopped := ff == 1 && failed == 1 && always == 0 - return state == contract.WorkflowRunning && cancelled == 0 && !stopped && tstate == contract.JobPending, nil + var ok bool + err := s.write(ctx, func(tx *sql.Tx) error { + var ( + state contract.WorkflowState + ff, failed, cancelled int + tstate, ustate contract.JobState + always, hasUndo, taskFound int + ) + err := tx.QueryRowContext(ctx, `SELECT w.state, w.fail_fast, w.failed, w.cancelled, + COALESCE(t.state, ''), COALESCE(t.undo_state, ''), COALESCE(t.always, 0), COALESCE(t.has_undo, 0), t.name IS NOT NULL + FROM `+wfWorkflows+` w LEFT JOIN `+wfTasks+` t ON t.workflow_id = w.id AND t.name = ? WHERE w.id = ?`, task, workflowID). + Scan(&state, &ff, &failed, &cancelled, &tstate, &ustate, &always, &hasUndo, &taskFound) + switch { + case errors.Is(err, sql.ErrNoRows): + return s.notFound(workflowID) + case err != nil: + return err + case taskFound == 0: + return nil + } + col := "started_at" + if undo { + col = "undo_started_at" + ok = state == contract.WorkflowCompensating && hasUndo == 1 && ustate == contract.JobPending + } else { + stopped := ff == 1 && failed == 1 && always == 0 + ok = state == contract.WorkflowRunning && cancelled == 0 && !stopped && tstate == contract.JobPending + } + if !ok { + return nil + } + _, err = tx.ExecContext(ctx, `UPDATE `+wfTasks+` SET `+col+` = ? WHERE workflow_id = ? AND name = ?`, + s.now().UnixMilli(), workflowID, task) + return err + }) + return ok && err == nil, err } // DepResult returns task's stored result: ErrDepFailed when it failed or was @@ -1104,13 +1145,15 @@ func (w *wfFlow) record() contract.WorkflowRecord { for _, name := range w.order { t := w.tasks[name] tr := contract.TaskRecord{Name: name, State: t.state, Deps: slices.Clone(t.deps), QueueID: t.queueID, - JobID: t.jobID, Always: t.always, Reason: t.reason, ResultSize: t.resultSize} + JobID: t.jobID, Always: t.always, Reason: t.reason, ResultSize: t.resultSize, + EnqueuedAt: t.enqueued, StartedAt: t.started, FinishedAt: t.finished} if t.undo != nil { st := t.undoState if st == "" { st = contract.JobWaiting } - tr.Undo = &contract.TaskRecord{Name: name, State: st, QueueID: t.undo.queueID, JobID: t.undo.jobID, Reason: t.undoReason} + tr.Undo = &contract.TaskRecord{Name: name, State: st, QueueID: t.undo.queueID, JobID: t.undo.jobID, Reason: t.undoReason, + EnqueuedAt: t.undoEnqueued, StartedAt: t.undoStarted, FinishedAt: t.undoFinished} } r.Tasks = append(r.Tasks, tr) } @@ -1166,5 +1209,29 @@ func (s *WorkflowStore) WorkflowKeys(ctx context.Context, state contract.Workflo return out, rows.Err() } +// WorkflowCounts counts the stored workflows per state: the rows Workflows +// lists. A state with none is absent. +// ponytail: scans the state index; a counter row per state if stores get huge. +func (s *WorkflowStore) WorkflowCounts(ctx context.Context) (map[contract.WorkflowState]int64, error) { + rows, err := s.f.db.QueryContext(ctx, `SELECT state, COUNT(*) FROM `+wfWorkflows+` GROUP BY state`) + if err != nil { + return nil, err + } + defer rows.Close() + + out := map[contract.WorkflowState]int64{} + for rows.Next() { + var ( + state contract.WorkflowState + n int64 + ) + if err := rows.Scan(&state, &n); err != nil { + return nil, err + } + out[state] = n + } + return out, rows.Err() +} + // setClock moves the store's clock; tests only. func (s *WorkflowStore) setClock(f func() time.Time) { s.clock.Store(&f) } diff --git a/workflow_finish.go b/workflow_finish.go index d04e094..6f75df3 100644 --- a/workflow_finish.go +++ b/workflow_finish.go @@ -137,6 +137,7 @@ func (s *WorkflowStore) endTx(ctx context.Context, tx *sql.Tx, queueID string, e } orig := w.snapshot() + w.at = s.now().UnixMilli() fx, err := w.finish(e) if err != nil { return r, err @@ -287,5 +288,12 @@ func (f *file) purgeStore() (ws *WorkflowStore, done func()) { } qs := &queues{cfg: storeConfig{singleFile: single, checkpointPages: f.pages}, root: root, files: map[*file]struct{}{}} qs.byID.Store(&map[string]*handle{}) - return &WorkflowStore{qs: qs, f: wf, single: single, scope: f.scope, cfg: workflowConfig{resultLimit: defaultResultLimit, retention: defaultRetention}}, release + ws = &WorkflowStore{qs: qs, f: wf, single: single, scope: f.scope, cfg: workflowConfig{resultLimit: defaultResultLimit, retention: defaultRetention}} + // tables of an older version get the columns this one reads + if err := ws.init(); err != nil { + log.Printf("sqliteq: purge: %v", err) + release() + return nil, func() {} + } + return ws, release } From bb3b3bbd3b5e3681b2672f613e8331b02d64d01f Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Tue, 10 Nov 2026 10:19:18 +0600 Subject: [PATCH 2/4] fix: lock-free Runnable, Purge keeps jobs when store not ready - Runnable: answer from lock-free read; yes -> stamp started_at in own write, guarded on pending state. Stamp error logged, answer stays true. Foreign write lock no longer gives "database is locked" (worker ran task on that error). - purgeStore returns err when workflow tables may exist but store cannot be prepared (check, open, init fail); Purge logs, deletes nothing. - HostLocal marker; guarded ALTER for task time columns (concurrent open). - Doc: enqueued_at = release time, kept while push waits in outbox. - Tests: Runnable under foreign lock, stamp guard, count query plan uses state index, Purge keeps jobs on corrupt workflow file. --- CHANGELOG.md | 6 +- queue.go | 8 ++- queue_test.go | 172 +++++++++++++++++++++++++++++++++++++++++++++ workflow.go | 94 +++++++++++++++---------- workflow_finish.go | 35 +++++---- 5 files changed, 263 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38d8a87..8306082 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,8 +33,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A redelivered job, or a push repeated after the job was removed, is safe: the pre-run check skips a task that already ended. (#25) - `Workflows` (`contract.WorkflowInspector`) and `WorkflowKeys` (`contract.WorkflowKeyLister`) list workflows newest first. (#25) - `WorkflowCounts` (`contract.WorkflowCounter`, contract v0.9.0): stored workflows per state, same rows as `WorkflowKeys`. -- Task times on `TaskRecord` and its `Undo`: `EnqueuedAt` at push, `StartedAt` when `Runnable` last answered true, `FinishedAt` at end. Unix ms, store clock. -- `Runnable` is one write transaction: check, then stamp on a true answer. Unknown workflow: `ErrWorkflowNotFound`, no row made. +- `HostLocal` marker (`contract.HostLocal`): VarMQ puts host name into the store's ID in `GET /workflow-counts`, so same path on two hosts = two stores. +- Task times on `TaskRecord` and its `Undo`: `EnqueuedAt` at push (written to queue or outbox; a push that waits in the outbox keeps it), `StartedAt` when `Runnable` last answered true, `FinishedAt` at end. Unix ms, store clock. +- `Runnable` answers from a lock-free read: a foreign write lock gives no error. A true answer then stamps `StartedAt`, only on a still-pending task; a failed stamp is logged, answer stays true. Unknown workflow: `ErrWorkflowNotFound`, no row made. +- `Purge` keeps the jobs (logs the cause) when the file has workflow tables but no store is ready (check, open or init failed): no task vanishes under a workflow left `running`. - `__sqliteq_wf_tasks` columns `enqueued_at`, `started_at`, `finished_at`, `undo_enqueued_at`, `undo_started_at`, `undo_finished_at` (INTEGER, default 0). Added at open to older tables; old rows read 0. - Index `__sqliteq_wf_workflows_state` on `(state, created_at DESC, id)`. - A cancel mark never drops a valid workflow item; the workflow decides its end. (#25) diff --git a/queue.go b/queue.go index 0c7d582..6f1cbeb 100644 --- a/queue.go +++ b/queue.go @@ -867,9 +867,15 @@ func (q *Queue) Values() []any { // demand when the file has workflow tables. Under WithSingleFile the delete // and the ends are one transaction. Folder layout: an intent is stored in // the workflow file first, and the recovery loop applies one a crash left. +// Workflow tables there but no store ready (check, open or init failed): +// nothing is deleted, the cause is logged; a later Purge can retry. func (q *Queue) Purge() { - ws, done := q.f.purgeStore() + ws, done, err := q.f.purgeStore() defer done() + if err != nil { + log.Printf("sqliteq: purge of %s: jobs kept, workflow store not ready: %v", q.name, err) + return + } if ws == nil { q.purge(nil) return diff --git a/queue_test.go b/queue_test.go index 1abdb05..7839f7a 100644 --- a/queue_test.go +++ b/queue_test.go @@ -10,6 +10,7 @@ import ( "maps" "os" "path/filepath" + "runtime" "slices" "strconv" "strings" @@ -3373,6 +3374,25 @@ func TestWorkflowCounts(t *testing.T) { t.Fatal(err) } check("swept", map[contract.WorkflowState]int64{contract.WorkflowRunning: 1}) + + // the count is an index scan, not a table scan + rows, err := e.st.f.db.Query(`EXPLAIN QUERY PLAN SELECT state, COUNT(*) FROM ` + wfWorkflows + ` GROUP BY state`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var plan []string + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + plan = append(plan, detail) + } + if !slices.ContainsFunc(plan, func(d string) bool { return strings.Contains(d, "__sqliteq_wf_workflows_state") }) { + t.Errorf("Expected the count to use index __sqliteq_wf_workflows_state, plan %q", plan) + } }) } @@ -3502,3 +3522,155 @@ func TestWorkflowTaskTimes(t *testing.T) { } }) } + +// Another connection (as another process) holds the write lock: the check +// still answers from a read, fast; a yes stays yes when its stamp cannot be +// written. +func TestRunnableUnderForeignLock(t *testing.T) { + eachLayout(t, func(t *testing.T, single bool) { + e := newWFEnv(t, single) + ctx := context.Background() + if err := e.st.Submit(ctx, planWF("x", planTask("x", "a", e.a), planTask("x", "b", e.a, "a"))); err != nil { + t.Fatal(err) + } + if err := e.st.Cancel(ctx, "x", nil); err != nil { + t.Fatal(err) + } + if err := e.st.Submit(ctx, planWF("y", planTask("y", "a", e.a))); err != nil { + t.Fatal(err) + } + other, err := sql.Open("sqlite3", e.st.f.key+"?_txlock=immediate&_busy_timeout=100") + if err != nil { + t.Fatal(err) + } + defer other.Close() + tx, err := other.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + if _, err := tx.Exec(`UPDATE ` + wfWorkflows + ` SET hash = hash`); err != nil { + t.Fatal(err) + } + + start := time.Now() + if ok, err := e.st.Runnable(ctx, "x", "a", false); ok || err != nil { + t.Errorf("Expected false, nil for a cancelled workflow's task, got %t %v", ok, err) + } + if d := time.Since(start); d > time.Second { + t.Errorf("Expected a no without waiting for the lock, took %v", d) + } + // ctx ends between answer and stamp (else the stamp waits the 5s + // busy timeout): the stamp fails, the yes stays + yctx, cancel := context.WithCancel(ctx) + defer cancel() + e.st.setClock(func() time.Time { + if inRunnable() { + cancel() + } + return time.Now() + }) + ok, err := e.st.Runnable(yctx, "y", "a", false) + e.st.setClock(time.Now) + if !ok || err != nil { + t.Errorf("Expected true, nil with the stamp failed, got %t %v", ok, err) + } + }) +} + +// inRunnable: the caller runs inside WorkflowStore.Runnable (a clock hook +// that must not fire for the store's loop). +func inRunnable() bool { + buf := make([]byte, 8<<10) + return strings.Contains(string(buf[:runtime.Stack(buf, false)]), "(*WorkflowStore).Runnable") +} + +// A task that ends between the check's read and its stamp keeps no start +// time; a later check answers no and stamps nothing. +func TestRunnableStampGuard(t *testing.T) { + eachLayout(t, func(t *testing.T, single bool) { + e := newWFEnv(t, single) + ctx := context.Background() + if err := e.st.Submit(ctx, planWF("w", planTask("w", "a", e.a))); err != nil { + t.Fatal(err) + } + end := func() { + if _, err := e.st.f.db.Exec(`UPDATE ` + wfTasks + ` SET state = 'completed' WHERE workflow_id = 'w' AND name = 'a'`); err != nil { + t.Error(err) + } + } + started := func() (n int64) { + t.Helper() + if err := e.st.f.db.QueryRow(`SELECT started_at FROM ` + wfTasks + ` WHERE workflow_id = 'w' AND name = 'a'`).Scan(&n); err != nil { + t.Fatal(err) + } + return n + } + // the clock is read after the answer, before the stamp: end the task there + e.st.setClock(func() time.Time { + if inRunnable() { + end() + } + return time.Now() + }) + ok, err := e.st.Runnable(ctx, "w", "a", false) + e.st.setClock(time.Now) + if !ok || err != nil { + t.Fatalf("Expected true, nil from the read before the end, got %t %v", ok, err) + } + if n := started(); n != 0 { + t.Errorf("Expected no start time on an ended task, got %d", n) + } + if ok, err := e.st.Runnable(ctx, "w", "a", false); ok || err != nil { + t.Errorf("Expected false, nil after the end, got %t %v", ok, err) + } + if n := started(); n != 0 { + t.Errorf("Expected no start time from a late check, got %d", n) + } + }) +} + +// Folder layout, workflow file there but no store can open it: Purge keeps +// the jobs (else their workflows stay running for ever). No workflow file, +// or a single file with no workflow tables: a plain purge. +func TestPurgeKeepsJobsWithoutStore(t *testing.T) { + t.Run("folder", func(t *testing.T) { + dir := t.TempDir() + qs := mustNew(t, dir) + defer qs.Close() + a, err := qs.NewQueue("a") + if err != nil { + t.Fatal(err) + } + a.Enqueue("j1") + a.Enqueue("j2") + wfPath := filepath.Join(dir, workflowsFile) + if err := os.WriteFile(wfPath, []byte(strings.Repeat("not a database ", 512)), 0o600); err != nil { + t.Fatal(err) + } + a.Purge() + if n := a.Len(); n != 2 { + t.Errorf("Expected the jobs kept when the workflow file cannot open, got len %d", n) + } + if err := os.Remove(wfPath); err != nil { + t.Fatal(err) + } + a.Purge() + if n := a.Len(); n != 0 { + t.Errorf("Expected a plain purge with no workflow file, got len %d", n) + } + }) + t.Run("single", func(t *testing.T) { + qs := mustNew(t, filepath.Join(t.TempDir(), "all.db"), WithSingleFile()) + defer qs.Close() + a, err := qs.NewQueue("a") + if err != nil { + t.Fatal(err) + } + a.Enqueue("j1") + a.Purge() + if n := a.Len(); n != 0 { + t.Errorf("Expected a plain purge with no workflow tables, got len %d", n) + } + }) +} diff --git a/workflow.go b/workflow.go index f53877a..294823c 100644 --- a/workflow.go +++ b/workflow.go @@ -37,6 +37,7 @@ var ( _ contract.WorkflowInspector = (*WorkflowStore)(nil) _ contract.WorkflowKeyLister = (*WorkflowStore)(nil) _ contract.WorkflowCounter = (*WorkflowStore)(nil) + _ contract.HostLocal = (*WorkflowStore)(nil) ) const ( @@ -255,9 +256,15 @@ func (s *WorkflowStore) init() error { } } for _, col := range taskTimeCols { - exists, err := hasColumn(s.f.db, strings.Trim(wfTasks, `"`), col) + table := strings.Trim(wfTasks, `"`) + exists, err := hasColumn(s.f.db, table, col) if err == nil && !exists { - _, err = s.f.db.Exec(`ALTER TABLE ` + wfTasks + ` ADD COLUMN ` + col + ` INTEGER NOT NULL DEFAULT 0`) + if _, err = s.f.db.Exec(`ALTER TABLE ` + wfTasks + ` ADD COLUMN ` + col + ` INTEGER NOT NULL DEFAULT 0`); err != nil { + // Other process added it between check and ALTER: fine. + if added, _ := hasColumn(s.f.db, table, col); added { + err = nil + } + } } if err != nil { return fmt.Errorf("sqliteq: failed to add workflow task column %s: %w", col, err) @@ -276,6 +283,10 @@ func (s *WorkflowStore) now() time.Time { // ScopeID implements contract.Scoped. func (s *WorkflowStore) ScopeID() string { return s.scope } +// HostLocal implements contract.HostLocal: the scope is a path, so the same +// path on another host is another store. +func (s *WorkflowStore) HostLocal() {} + // Close unregisters the store, stops its loop and releases its file. Queues // stay open. Calling it again does nothing. func (s *WorkflowStore) Close() error { @@ -679,6 +690,8 @@ func (s *WorkflowStore) stamp(w *wfFlow, fx []wfEffect) { // Folder layout: outbox rows, minSeq the first one's seq (0 when none). // Single file: straight into the queues (ids), except those this client // cannot reach, which get outbox rows. +// enqueued_at is this release time, when the push is written (queue or +// outbox): a push that waits in the outbox keeps it, no restamp at drain. func (s *WorkflowStore) release(ctx context.Context, tx *sql.Tx, w *wfFlow, fx []wfEffect) (minSeq int64, ids []string, err error) { var rows []outRow now := s.now().UnixMilli() @@ -1052,48 +1065,55 @@ func (s *WorkflowStore) Cancel(ctx context.Context, id string, undo *bool) error return nil } -// Runnable is the pre-run check (spec 6.12). A yes stamps the start time of -// the task (undo: of its undo) in the same transaction: the job runs next. +// Runnable is the pre-run check (spec 6.12). Answer: lock-free read, so a +// foreign write lock cannot turn it into an error. A yes then stamps the +// start time of the task (undo: of its undo), only while it is still +// pending: an ended task gets no stamp. A failed stamp is logged, the answer +// stays yes: the time is display data, the run must not depend on it. func (s *WorkflowStore) Runnable(ctx context.Context, workflowID, task string, undo bool) (bool, error) { if !validWorkflowID(workflowID) { return false, s.notFound(workflowID) } + var ( + state contract.WorkflowState + ff, failed, cancelled int + tstate, ustate contract.JobState + always, hasUndo, taskFound int + ) + err := s.f.db.QueryRowContext(ctx, `SELECT w.state, w.fail_fast, w.failed, w.cancelled, + COALESCE(t.state, ''), COALESCE(t.undo_state, ''), COALESCE(t.always, 0), COALESCE(t.has_undo, 0), t.name IS NOT NULL + FROM `+wfWorkflows+` w LEFT JOIN `+wfTasks+` t ON t.workflow_id = w.id AND t.name = ? WHERE w.id = ?`, task, workflowID). + Scan(&state, &ff, &failed, &cancelled, &tstate, &ustate, &always, &hasUndo, &taskFound) + switch { + case errors.Is(err, sql.ErrNoRows): + return false, s.notFound(workflowID) + case err != nil: + return false, err + case taskFound == 0: + return false, nil + } + col, stateCol := "started_at", "state" var ok bool - err := s.write(ctx, func(tx *sql.Tx) error { - var ( - state contract.WorkflowState - ff, failed, cancelled int - tstate, ustate contract.JobState - always, hasUndo, taskFound int - ) - err := tx.QueryRowContext(ctx, `SELECT w.state, w.fail_fast, w.failed, w.cancelled, - COALESCE(t.state, ''), COALESCE(t.undo_state, ''), COALESCE(t.always, 0), COALESCE(t.has_undo, 0), t.name IS NOT NULL - FROM `+wfWorkflows+` w LEFT JOIN `+wfTasks+` t ON t.workflow_id = w.id AND t.name = ? WHERE w.id = ?`, task, workflowID). - Scan(&state, &ff, &failed, &cancelled, &tstate, &ustate, &always, &hasUndo, &taskFound) - switch { - case errors.Is(err, sql.ErrNoRows): - return s.notFound(workflowID) - case err != nil: - return err - case taskFound == 0: - return nil - } - col := "started_at" - if undo { - col = "undo_started_at" - ok = state == contract.WorkflowCompensating && hasUndo == 1 && ustate == contract.JobPending - } else { - stopped := ff == 1 && failed == 1 && always == 0 - ok = state == contract.WorkflowRunning && cancelled == 0 && !stopped && tstate == contract.JobPending - } - if !ok { - return nil - } - _, err = tx.ExecContext(ctx, `UPDATE `+wfTasks+` SET `+col+` = ? WHERE workflow_id = ? AND name = ?`, - s.now().UnixMilli(), workflowID, task) + if undo { + col, stateCol = "undo_started_at", "undo_state" + ok = state == contract.WorkflowCompensating && hasUndo == 1 && ustate == contract.JobPending + } else { + stopped := ff == 1 && failed == 1 && always == 0 + ok = state == contract.WorkflowRunning && cancelled == 0 && !stopped && tstate == contract.JobPending + } + if !ok { + return false, nil + } + now := s.now().UnixMilli() + err = s.write(ctx, func(tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, `UPDATE `+wfTasks+` SET `+col+` = ? WHERE workflow_id = ? AND name = ? AND `+stateCol+` = ?`, + now, workflowID, task, contract.JobPending) return err }) - return ok && err == nil, err + if err != nil { + log.Printf("sqliteq: stamping start of workflow %s task %s: %v", workflowID, task, err) + } + return true, nil } // DepResult returns task's stored result: ErrDepFailed when it failed or was diff --git a/workflow_finish.go b/workflow_finish.go index 6f75df3..a0c01f1 100644 --- a/workflow_finish.go +++ b/workflow_finish.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "log" "os" "path/filepath" @@ -261,28 +262,39 @@ func (s *WorkflowStore) applyIntents() { // purgeStore returns the store whose workflows a Purge of the file must end: // the registered one, else one opened on demand when the file's workflow // tables exist (nil when they do not). done releases an on-demand store. -func (f *file) purgeStore() (ws *WorkflowStore, done func()) { +// err: tables may exist but no store is ready (check, open or init failed); +// the Purge must keep the jobs, or their workflows stay running for ever. +func (f *file) purgeStore() (ws *WorkflowStore, done func(), err error) { + none := func() {} if st, ok := contract.LookupWorkflowStore(f.scope); ok { if ws, ok := st.(*WorkflowStore); ok { - return ws, func() {} + return ws, none, nil } } root := strings.TrimPrefix(f.scope, "sqliteq:") single := root == f.key - wf, release := f, func() {} + wf, release := f, none if single { var n int - if f.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name = '__sqliteq_wf_workflows'`).Scan(&n) != nil || n == 0 { - return nil, func() {} + if err := f.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name = '__sqliteq_wf_workflows'`).Scan(&n); err != nil { + return nil, none, fmt.Errorf("checking for workflow tables: %w", err) + } + if n == 0 { + return nil, none, nil } } else { path, err := resolve(filepath.Join(root, workflowsFile)) - if _, serr := os.Stat(path); err != nil || serr != nil { - return nil, func() {} + if err == nil { + _, err = os.Stat(path) + } + switch { + case errors.Is(err, fs.ErrNotExist): + return nil, none, nil + case err != nil: + return nil, none, fmt.Errorf("checking for the workflow file: %w", err) } if wf, err = acquire(path, root, f.pages); err != nil { - log.Printf("sqliteq: purge: opening the workflow file: %v", err) - return nil, func() {} + return nil, none, fmt.Errorf("opening the workflow file: %w", err) } release = func() { wf.release() } } @@ -291,9 +303,8 @@ func (f *file) purgeStore() (ws *WorkflowStore, done func()) { ws = &WorkflowStore{qs: qs, f: wf, single: single, scope: f.scope, cfg: workflowConfig{resultLimit: defaultResultLimit, retention: defaultRetention}} // tables of an older version get the columns this one reads if err := ws.init(); err != nil { - log.Printf("sqliteq: purge: %v", err) release() - return nil, func() {} + return nil, none, err } - return ws, release + return ws, release, nil } From 7c39b3b15818da3018daba8983abddfef7f55335 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Tue, 10 Nov 2026 11:40:36 +0600 Subject: [PATCH 3/4] chore: bump contract to v0.9.0, varmq to 8057d16 --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index d979276..0c44fe9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/goptics/sqliteq go 1.27 require ( - github.com/goptics/varmq v1.10.1-0.20261109165008-c75ed52b3b0a - github.com/goptics/varmq/contract v0.8.0 + github.com/goptics/varmq v1.10.1-0.20261110052717-8057d16e4b55 + github.com/goptics/varmq/contract v0.9.0 github.com/mattn/go-sqlite3 v1.14.28 ) diff --git a/go.sum b/go.sum index d21bc24..854f05d 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,9 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/goptics/varmq v1.10.1-0.20261109165008-c75ed52b3b0a h1:+UjHIuTdKROtu6zLVk8b/efeP6uTmQqIy7TgshIHx6U= -github.com/goptics/varmq v1.10.1-0.20261109165008-c75ed52b3b0a/go.mod h1:4hso4XnUSTh5sjr/h2//MYh0FPGRvebLCx5AaoL4qII= -github.com/goptics/varmq/contract v0.8.0 h1:U87RzQITqs3TPNuzNXLCl9Fp9ebUa6Eiw3cDNd2cb5Q= -github.com/goptics/varmq/contract v0.8.0/go.mod h1:55tOUywUsZhBgyo8oJhNmKMknETVHL2OuMrITztanXg= +github.com/goptics/varmq v1.10.1-0.20261110052717-8057d16e4b55 h1:aRBQi8UhqtTVfl20meNmvHq+v8SKGLc22wZTfhnh7p8= +github.com/goptics/varmq v1.10.1-0.20261110052717-8057d16e4b55/go.mod h1:wYDkTj4w50s08bqsWnsEg/pooFyBQrPHGzht31CDkE0= +github.com/goptics/varmq/contract v0.9.0 h1:Z/0pJNi6cviSIwavCyJn43fCK3ZLk19IXc3D7BQfBXE= +github.com/goptics/varmq/contract v0.9.0/go.mod h1:55tOUywUsZhBgyo8oJhNmKMknETVHL2OuMrITztanXg= github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= From c4aa6637c25dc124bfb9b4972f9ccb2708b406f3 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Tue, 10 Nov 2026 15:09:19 +0600 Subject: [PATCH 4/4] test: cover WorkflowCounts error path and HostLocal --- queue_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/queue_test.go b/queue_test.go index 7839f7a..9881d5c 100644 --- a/queue_test.go +++ b/queue_test.go @@ -3375,6 +3375,17 @@ func TestWorkflowCounts(t *testing.T) { } check("swept", map[contract.WorkflowState]int64{contract.WorkflowRunning: 1}) + // a failed count is an error, never a partial map + cctx, cancel := context.WithCancel(ctx) + cancel() + if got, err := e.st.WorkflowCounts(cctx); err == nil || got != nil { + t.Errorf("Expected an error and no counts from a cancelled count, got %v, %v", got, err) + } + + // the scope is a path: a store on another host is another store + var hl contract.HostLocal = e.st + hl.HostLocal() + // the count is an index scan, not a table scan rows, err := e.st.f.db.Query(`EXPLAIN QUERY PLAN SELECT state, COUNT(*) FROM ` + wfWorkflows + ` GROUP BY state`) if err != nil {