From 726580377e1e77ecf10aca6cd1a48caf9c522b73 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Mon, 9 Nov 2026 21:41:02 +0600 Subject: [PATCH 1/8] feat: workflow store with outbox finish, scope IDs, workflow-aware pop and purge --- cancel.go | 12 +- queue.go | 106 ++++- queue_test.go | 819 ++++++++++++++++++++++++++++++++++- queues.go | 23 +- registry.go | 4 +- wfmachine.go | 315 ++++++++++++++ workflow.go | 1016 ++++++++++++++++++++++++++++++++++++++++++++ workflow_finish.go | 166 ++++++++ 8 files changed, 2439 insertions(+), 22 deletions(-) create mode 100644 wfmachine.go create mode 100644 workflow.go create mode 100644 workflow_finish.go diff --git a/cancel.go b/cancel.go index 6b4445e..46edfa8 100644 --- a/cancel.go +++ b/cancel.go @@ -14,7 +14,8 @@ import ( // Every pop drops a marked waiting job with outcome cancelled; OnCancel // learns of marks by polling seq; AcknowledgeCancelled finishes a running // job that stopped for it. Any finish, and an EnqueueOnce that stores the -// id, clears the mark. sqliteq has no workflow store, so no item is exempt. +// id, clears the mark. A valid workflow item (wfTag) is never dropped: the +// workflow decides its end, and its mark stays. var ( _ contract.JobCanceller = (*Queue)(nil) @@ -190,3 +191,12 @@ func marked(isMarked *sql.Stmt, id string, now time.Time) (bool, error) { err := isMarked.QueryRow(id, now.UnixMilli()).Scan(&n) return n == 1, err } + +// markedDrop is marked for a pop: a valid workflow item is never dropped. +func markedDrop(isMarked *sql.Stmt, id string, data []byte, now time.Time) (bool, error) { + drop, err := marked(isMarked, id, now) + if err != nil || !drop { + return false, err + } + return !isWorkflowItem(data), nil +} diff --git a/queue.go b/queue.go index 1ae55ed..bb20e39 100644 --- a/queue.go +++ b/queue.go @@ -332,6 +332,13 @@ func parseAckID(s string) (int64, bool) { return id, err == nil && id > 0 } +// ScopeID implements contract.Scoped: "sqliteq:" + the store root, the +// folder (or the file under WithSingleFile). Queues of one root share a +// workflow store. +func (q *Queue) ScopeID() string { + return q.f.scope +} + // Enqueue adds an item to the queue // Returns true if the operation was successful func (q *Queue) Enqueue(item any) bool { @@ -385,6 +392,19 @@ func (q *Queue) enqueue(item any, priority int, id string) bool { }) } +// enqueueTx is enqueue in the caller's tx, which holds the file's write +// mutex: one pending row under id, none when a row with that id exists. +func (q *Queue) enqueueTx(tx *sql.Tx, item []byte, priority int, id string) error { + now := time.Now().UTC() + insert := tx.Stmt(q.insert) + if q.priority { + _, err := insert.Exec(item, now, now, id, priority) + return err + } + _, err := insert.Exec(item, now, now, id) + return err +} + // claimN moves up to n pending rows to processing in one statement and // returns them in queue order; refused reports a pause stopped it func (q *Queue) claimN(n int) (out []claimed, capped, refused bool, err error) { @@ -452,7 +472,7 @@ func (q *Queue) claimGated(n int) (out []claimed, capped, refused bool, err erro } for _, c := range rows { id := jobIDOf(c.once, c.data) - drop, err := marked(isMarked, id, now) + drop, err := markedDrop(isMarked, id, c.data, now) if err != nil { return nil, false, false, err } @@ -567,7 +587,7 @@ func (q *Queue) popGated(now time.Time) (data []byte, capped, refused bool, err return nil, false, false, err } id := jobIDOf(once, data) - drop, err := marked(isMarked, id, now) + drop, err := markedDrop(isMarked, id, data, now) if err != nil { return nil, false, false, err } @@ -682,20 +702,25 @@ func (q *Queue) finish(ackID string, state contract.JobState, reason string, res ok = ok && err == nil }() } - // No row is an outcome, not an error: it must not fail the round - if ok, err = q.end(tx, id, "", state, reason, result, now); !ok || err != nil { - return ok, err - } - // Totals: cancelled counts as failed (spec 6.14) - total := "failed" - if state == contract.JobCompleted { - total = "completed" - } - _, err = tx.Stmt(q.tally).Exec(total) - return err == nil, err + return q.finishTx(tx, id, state, reason, result, now) }) } +// finishTx is finish in the caller's tx, with the row id parsed. +func (q *Queue) finishTx(tx *sql.Tx, id int64, state contract.JobState, reason string, result []byte, now time.Time) (bool, error) { + // No row is an outcome, not an error: it must not fail the round + if ok, err := q.end(tx, id, "", state, reason, result, now); !ok || err != nil { + return ok, err + } + // Totals: cancelled counts as failed (spec 6.14) + total := "failed" + if state == contract.JobCompleted { + total = "completed" + } + _, err := tx.Stmt(q.tally).Exec(total) + return err == nil, err +} + // end finishes processing row id with state, in tx: the row is deleted or // kept with state as status, and a job with a Job ID gets its outcome // record. jobID is the row's, when the caller knows it. Reports whether @@ -759,25 +784,70 @@ func (q *Queue) Values() []any { } // Purge removes all items from the queue, with its outcome records, cancel -// marks and totals; a pause stays (spec 6.6, 6.7, 6.14) +// marks and totals; a pause stays (spec 6.6, 6.7, 6.14). A purged workflow +// task or undo ends cancelled (reason "queue purged"), so its workflow goes +// on: dependents follow their policy, and what that releases is pushed after +// the delete. Without a workflow store for the queue's scope, no workflow +// ends. func (q *Queue) Purge() { + store, _ := contract.LookupWorkflowStore(q.f.scope) + ws, _ := store.(*WorkflowStore) + if tags := q.purge(ws != nil); ws != nil { + ws.endPurged(q, tags) + } +} + +// purge deletes the rows under the write mutex; with workflows it first +// reads the tags of the pending and processing workflow items. +func (q *Queue) purge(workflows bool) (tags []wfItemTag) { q.f.mu.Lock() defer q.f.mu.Unlock() tx, err := q.f.db.Begin() if err != nil { - return + return nil } defer tx.Rollback() + if workflows { + if tags, err = q.workflowTags(tx); err != nil { + return nil + } + } for _, t := range []string{q.table, q.jobs, q.cancelled} { if _, err := tx.Exec("DELETE FROM " + t); err != nil { - return + return nil } } if _, err := tx.Exec("UPDATE " + q.meta + " SET value = 0 WHERE key <> 'paused_at'"); err != nil { - return + return nil + } + if tx.Commit() != nil { + return nil + } + return tags +} + +// workflowTags reads the tags of the queued and running workflow items, in +// queue order. +func (q *Queue) workflowTags(tx *sql.Tx) ([]wfItemTag, error) { + rows, err := tx.Query(fmt.Sprintf(`SELECT data FROM %s WHERE status IN ('pending', 'processing') + AND (instr(CAST(data AS TEXT), '"wf"') > 0 OR instr(CAST(data AS TEXT), char(92)) > 0) ORDER BY %s`, q.table, q.order)) + if err != nil { + return nil, err + } + defer rows.Close() + + var tags []wfItemTag + for rows.Next() { + var data []byte + if err := rows.Scan(&data); err != nil { + return nil, err + } + if tag, ok := workflowTag(data); ok { + tags = append(tags, tag) + } } - tx.Commit() + return tags, rows.Err() } // Close closes the queue and drops its outcome watches and OnCancel hooks diff --git a/queue_test.go b/queue_test.go index a8ab137..a5ad706 100644 --- a/queue_test.go +++ b/queue_test.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "errors" "fmt" + "os" "path/filepath" "slices" "strconv" @@ -2086,7 +2087,7 @@ func TestVarmqPause(t *testing.T) { ran := make(chan int, 4) w := varmq.NewWorker(func(j varmq.Job[int]) { ran <- j.Data() }) defer w.Stop() - handle := w.WithPersistentQueue(q) + handle := w.WithQueue(q) if err := handle.Pause(); err != nil { t.Fatal(err) } @@ -2281,3 +2282,819 @@ func TestTotalsIgnoreNoopFinish(t *testing.T) { }) } } + +// wfEnv is a workflow store over fifo queues "a" and "b" and priority queue +// "p" in the folder layout (own workflow file) or, with single, one file. +type wfEnv struct { + qs Queues + st *WorkflowStore + a, b *Queue + p *PriorityQueue + dir string + single bool +} + +func newWFEnv(t *testing.T, single bool, opts ...WorkflowOption) *wfEnv { + t.Helper() + e := &wfEnv{single: single, dir: t.TempDir()} + if single { + e.qs = mustNew(t, filepath.Join(e.dir, "all.db"), WithSingleFile()) + } else { + e.qs = mustNew(t, e.dir) + } + var err error + if e.a, err = e.qs.NewQueue("a"); err != nil { + t.Fatal(err) + } + if e.b, err = e.qs.NewQueue("b"); err != nil { + t.Fatal(err) + } + if e.p, err = e.qs.NewPriorityQueue("p"); err != nil { + t.Fatal(err) + } + if e.st, err = NewWorkflowStore(e.qs, opts...); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + e.st.Close() + e.qs.Close() + }) + return e +} + +func eachLayout(t *testing.T, fn func(t *testing.T, single bool)) { + for _, single := range []bool{false, true} { + name := "folder" + if single { + name = "single" + } + t.Run(name, func(t *testing.T) { fn(t, single) }) + } +} + +// wfItem is the envelope of task (or its undo) of workflow wf. +func wfItem(wf, task string, undo bool) []byte { + id, extra := "wf:"+wf+":"+task, "" + if undo { + id, extra = id+":undo", `,"undo":true` + } + return []byte(`{"id":"` + id + `","status":"Created","data":1,"wf":"` + wf + `","task":"` + task + `"` + extra + `}`) +} + +// planTask plans task name of workflow wf on queue q (a *Queue or *PriorityQueue). +func planTask(wf, name string, q any, deps ...string) contract.TaskPlan { + return contract.TaskPlan{Name: name, JobID: "wf:" + wf + ":" + name, Queue: q, Deps: deps, Item: wfItem(wf, name, false)} +} + +func withUndo(tp contract.TaskPlan, wf string, q any) contract.TaskPlan { + tp.Undo = &contract.UndoPlan{JobID: tp.JobID + ":undo", Queue: q, Item: wfItem(wf, tp.Name, true)} + return tp +} + +func wfPlan(id string, tasks ...contract.TaskPlan) contract.WorkflowPlan { + return contract.WorkflowPlan{ID: id, Hash: id + "-hash", Tasks: tasks} +} + +// take pops the next job of q and returns its ack ID. +func take(t *testing.T, q interface { + DequeueWithAckId() (any, bool, string) +}) (task string, ack string) { + t.Helper() + v, ok, ack := q.DequeueWithAckId() + if !ok { + t.Fatal("Expected a job to take") + } + tag, ok := workflowTag(v.([]byte)) + if !ok { + t.Fatalf("Expected a workflow item, got %s", v) + } + return tag.task, ack +} + +func (e *wfEnv) finish(t *testing.T, q *Queue, ack string, end contract.TaskEnd) []string { + t.Helper() + pushed, ok := e.st.FinishTask(q.QueueID(), ack, end) + if !ok { + t.Fatalf("FinishTask %+v failed", end) + } + return pushed +} + +func (e *wfEnv) state(t *testing.T, id string) contract.WorkflowRecord { + t.Helper() + r, err := e.st.Record(context.Background(), id) + if err != nil { + t.Fatal(err) + } + return r +} + +func taskStates(r contract.WorkflowRecord) string { + var s []string + for _, tr := range r.Tasks { + s = append(s, tr.Name+"="+string(tr.State)) + } + return strings.Join(s, " ") +} + +func (e *wfEnv) outbox(t *testing.T) int { + t.Helper() + var n int + if err := e.st.f.db.QueryRow(`SELECT COUNT(*) FROM ` + wfOutbox).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +func done(wf, task string, result string) contract.TaskEnd { + return contract.TaskEnd{Workflow: wf, Task: task, Outcome: contract.OutcomeCompleted, Result: []byte(result)} +} + +// The store registers for the queues' scope, one per scope; Close unregisters. +func TestWorkflowStoreRegisterClose(t *testing.T) { + eachLayout(t, func(t *testing.T, single bool) { + e := newWFEnv(t, single) + scope := e.a.ScopeID() + if !strings.HasPrefix(scope, "sqliteq:/") || e.p.ScopeID() != scope || e.st.ScopeID() != scope { + t.Fatalf("Expected one absolute sqliteq scope, got %q %q %q", scope, e.p.ScopeID(), e.st.ScopeID()) + } + if single != strings.HasSuffix(scope, "all.db") { + t.Errorf("Expected the file as the scope root only under WithSingleFile, got %q", scope) + } + if got, ok := contract.LookupWorkflowStore(scope); !ok || got != contract.WorkflowStore(e.st) { + t.Fatal("Expected the store to be registered") + } + if _, err := NewWorkflowStore(e.qs); !errors.Is(err, contract.ErrScopeRegistered) { + t.Errorf("Expected ErrScopeRegistered for a second store, got %v", err) + } + if _, err := os.Stat(filepath.Join(e.dir, workflowsFile)); single == (err == nil) { + t.Errorf("Expected a workflow file only in the folder layout, stat err %v", err) + } + if err := e.st.Close(); err != nil { + t.Fatal(err) + } + if _, ok := contract.LookupWorkflowStore(scope); ok { + t.Error("Expected Close to unregister") + } + if err := e.st.Close(); err != nil { + t.Errorf("Expected a second Close to do nothing, got %v", err) + } + st2, err := NewWorkflowStore(e.qs) + if err != nil { + t.Fatalf("Expected a store after Close, got %v", err) + } + st2.Close() + }) +} + +func TestWorkflowStoreOptions(t *testing.T) { + qs := mustNew(t, t.TempDir()) + defer qs.Close() + for name, opt := range map[string]WorkflowOption{ + "limit": WithWorkflowResultLimit(0), + "retention": WithWorkflowRetention(0), + "nil": nil, + } { + if st, err := NewWorkflowStore(qs, opt); err == nil { + st.Close() + t.Errorf("Expected an error for the %s option", name) + } + } + if len(contract.WorkflowStores()) != 0 { + for _, s := range contract.WorkflowStores() { + t.Errorf("Expected no leftover store, got %s", s.ScopeID()) + } + } +} + +// A chain across fifo and priority queues: Submit pushes the root, each +// finish pushes the next task and returns its queue, results flow through +// DepResult, a re-finish returns every push target. +func TestWorkflowHappyPath(t *testing.T) { + eachLayout(t, func(t *testing.T, single bool) { + e := newWFEnv(t, single) + ctx := context.Background() + plan := wfPlan("w1", planTask("w1", "t1", e.a), planTask("w1", "t2", e.p, "t1"), planTask("w1", "t3", e.b, "t2")) + plan.Tasks[1].Priority = 5 + if err := e.st.Submit(ctx, plan); err != nil { + t.Fatal(err) + } + if err := e.st.Submit(ctx, plan); err != nil { + t.Errorf("Expected a repeated Submit of the same plan to succeed, got %v", err) + } + other := plan + other.Hash = "other" + if err := e.st.Submit(ctx, other); !errors.Is(err, contract.ErrWorkflowExists) { + t.Errorf("Expected ErrWorkflowExists for another plan, got %v", err) + } + if e.a.Len() != 1 || e.p.Len() != 0 || e.b.Len() != 0 { + t.Fatalf("Expected only the root pushed, got %d %d %d", e.a.Len(), e.p.Len(), e.b.Len()) + } + if got := taskStates(e.state(t, "w1")); got != "t1=pending t2=waiting t3=waiting" { + t.Fatalf("got %s", got) + } + for task, want := range map[string]bool{"t1": true, "t2": false, "nope": false} { + if ok, err := e.st.Runnable(ctx, "w1", task, false); err != nil || ok != want { + t.Errorf("Runnable %s = %t %v, want %t", task, ok, err, want) + } + } + if _, err := e.st.Runnable(ctx, "missing", "t1", false); !errors.Is(err, contract.ErrWorkflowNotFound) { + t.Errorf("Expected ErrWorkflowNotFound, got %v", err) + } + if _, err := e.st.DepResult(ctx, "w1", "t1"); !errors.Is(err, contract.ErrNoResult) { + t.Errorf("Expected ErrNoResult before the end, got %v", err) + } + + task, ack := take(t, e.a) + if task != "t1" { + t.Fatalf("got task %s", task) + } + if pushed := e.finish(t, e.a, ack, done("w1", "t1", `{"n":1}`)); !slices.Equal(pushed, []string{e.p.QueueID()}) { + t.Fatalf("Expected a push into p, got %v", pushed) + } + if res, err := e.st.DepResult(ctx, "w1", "t1"); err != nil || string(res) != `{"n":1}` { + t.Errorf("DepResult = %q %v", res, err) + } + if r := e.state(t, "w1"); r.Tasks[0].ResultSize != 7 || taskStates(r) != "t1=completed t2=pending t3=waiting" { + t.Errorf("got %s size %d", taskStates(r), r.Tasks[0].ResultSize) + } + if e.a.Len() != 0 { + t.Errorf("Expected the job acknowledged") + } + if n := stats(t, e.a).Counts.Processing; n != 0 { + t.Errorf("Expected the job settled, got %d processing", n) + } + if n := e.outbox(t); n != 0 { + t.Errorf("Expected an empty outbox, got %d", n) + } + + task, ack = take(t, e.p) + e.finish(t, e.p.Queue, ack, done("w1", task, "")) + task, ack = take(t, e.b) + e.finish(t, e.b, ack, done("w1", task, "not json")) + r := e.state(t, "w1") + if r.State != contract.WorkflowCompleted || r.FinishedAt < r.CreatedAt || r.FinishedAt == 0 { + t.Fatalf("Expected a completed workflow, got %s at %d..%d", r.State, r.CreatedAt, r.FinishedAt) + } + if _, err := e.st.DepResult(ctx, "w1", "t3"); !errors.Is(err, contract.ErrNoResult) { + t.Errorf("Expected a non-JSON result dropped, got %v", err) + } + + // A re-finish changes nothing and names every push target + pushed := e.finish(t, e.a, "999", done("w1", "t1", `{"n":2}`)) + want := []string{e.a.QueueID(), e.p.QueueID(), e.b.QueueID()} + slices.Sort(pushed) + slices.Sort(want) + if !slices.Equal(pushed, want) { + t.Errorf("Expected every target %v, got %v", want, pushed) + } + if res, _ := e.st.DepResult(ctx, "w1", "t1"); string(res) != `{"n":1}` { + t.Errorf("Expected the first result kept, got %s", res) + } + }) +} + +func TestWorkflowSubmitChecks(t *testing.T) { + e := newWFEnv(t, false) + ctx := context.Background() + other := mustNew(t, t.TempDir()) + defer other.Close() + oq, _ := other.NewQueue("a") + + bad := map[string]contract.WorkflowPlan{ + "empty": {ID: "x", Hash: "h"}, + "bad id": wfPlan("a b", planTask("a b", "t", e.a)), + "index": wfPlan("index", planTask("index", "t", e.a)), + "bad task": wfPlan("x", planTask("x", "t t", e.a)), + "other scope": wfPlan("x", planTask("x", "t", oq)), + "not a queue": wfPlan("x", planTask("x", "t", "q")), + "bad item": wfPlan("x", func() contract.TaskPlan { tp := planTask("x", "t", e.a); tp.Item = []byte("[1]"); return tp }()), + "dup name": wfPlan("x", planTask("x", "t", e.a), planTask("x", "t", e.a)), + } + for name, plan := range bad { + if err := e.st.Submit(ctx, plan); err == nil { + t.Errorf("Expected an error for %s", name) + } + } + if err := e.st.Submit(ctx, bad["other scope"]); !errors.Is(err, contract.ErrScopeMismatch) { + t.Errorf("Expected ErrScopeMismatch, got %v", err) + } + if _, err := e.st.Record(ctx, "x"); !errors.Is(err, contract.ErrWorkflowNotFound) { + t.Errorf("Expected nothing stored by a refused Submit, got %v", err) + } + if e.a.Len() != 0 { + t.Errorf("Expected nothing pushed by a refused Submit, got %d", e.a.Len()) + } +} + +// Failure follows the dependency policy; FailFast cancels waiting tasks; an +// undo runs in reverse order once the run is over. +func TestWorkflowFailureAndCompensation(t *testing.T) { + eachLayout(t, func(t *testing.T, single bool) { + e := newWFEnv(t, single) + ctx := context.Background() + plan := wfPlan("w2", + withUndo(planTask("w2", "a", e.a), "w2", e.b), + withUndo(planTask("w2", "b", e.a, "a"), "w2", e.b), + planTask("w2", "c", e.a, "b")) + if err := e.st.Submit(ctx, plan); err != nil { + t.Fatal(err) + } + _, ack := take(t, e.a) + e.finish(t, e.a, ack, done("w2", "a", "")) + _, ack = take(t, e.a) + e.finish(t, e.a, ack, done("w2", "b", "")) + _, ack = take(t, e.a) + pushed := e.finish(t, e.a, ack, contract.TaskEnd{Workflow: "w2", Task: "c", Outcome: contract.OutcomeFailed, Reason: "boom"}) + r := e.state(t, "w2") + if r.State != contract.WorkflowCompensating || r.Tasks[2].Reason != "boom" || !slices.Equal(pushed, []string{e.b.QueueID()}) { + t.Fatalf("Expected compensation started in b, got %s %s %v", r.State, taskStates(r), pushed) + } + // b's undo first: a's waits for it + if e.b.Len() != 1 { + t.Fatalf("Expected one undo queued, got %d", e.b.Len()) + } + task, ack := take(t, e.b) + if task != "b" { + t.Fatalf("Expected the undo of b first, got %s", task) + } + if ok, _ := e.st.Runnable(ctx, "w2", "b", true); !ok { + t.Error("Expected the undo runnable") + } + if ok, _ := e.st.Runnable(ctx, "w2", "a", true); ok { + t.Error("Expected a's undo to wait") + } + e.finish(t, e.b, ack, contract.TaskEnd{Workflow: "w2", Task: "b", Undo: true, Outcome: contract.OutcomeCompleted}) + task, ack = take(t, e.b) + if task != "a" { + t.Fatalf("got %s", task) + } + e.finish(t, e.b, ack, contract.TaskEnd{Workflow: "w2", Task: "a", Undo: true, Outcome: contract.OutcomeFailed, Reason: "undo broke"}) + r = e.state(t, "w2") + if r.State != contract.WorkflowCompensationFailed || r.Tasks[0].Undo.State != contract.JobFailed || r.Tasks[0].Undo.Reason != "undo broke" { + t.Errorf("got %s undo %+v", r.State, r.Tasks[0].Undo) + } + + // FailFast: the first failure cancels what waits + ff := wfPlan("w3", planTask("w3", "a", e.a), planTask("w3", "b", e.a), planTask("w3", "c", e.a, "a")) + ff.FailFast = true + if err := e.st.Submit(ctx, ff); err != nil { + t.Fatal(err) + } + _, ack = take(t, e.a) + e.finish(t, e.a, ack, contract.TaskEnd{Workflow: "w3", Task: "a", Outcome: contract.OutcomeFailed}) + if got := taskStates(e.state(t, "w3")); got != "a=failed b=pending c=cancelled" { + t.Errorf("got %s", got) + } + if ok, _ := e.st.Runnable(ctx, "w3", "b", false); ok { + t.Error("Expected b stopped by FailFast") + } + }) +} + +func TestWorkflowCancel(t *testing.T) { + eachLayout(t, func(t *testing.T, single bool) { + e := newWFEnv(t, single) + ctx := context.Background() + if err := e.st.Cancel(ctx, "missing", nil); !errors.Is(err, contract.ErrWorkflowNotFound) { + t.Errorf("Expected ErrWorkflowNotFound, got %v", err) + } + plan := wfPlan("w1", withUndo(planTask("w1", "a", e.a), "w1", e.b), planTask("w1", "b", e.a, "a")) + if err := e.st.Submit(ctx, plan); err != nil { + t.Fatal(err) + } + yes := true + if err := e.st.Cancel(ctx, "w1", &yes); err != nil { + t.Fatal(err) + } + r := e.state(t, "w1") + if taskStates(r) != "a=pending b=cancelled" || !r.UndoOnCancel || r.State != contract.WorkflowRunning { + t.Fatalf("got %s %t %s", taskStates(r), r.UndoOnCancel, r.State) + } + if ok, _ := e.st.Runnable(ctx, "w1", "a", false); ok { + t.Error("Expected the queued task stopped by the cancel") + } + if err := e.st.Cancel(ctx, "w1", nil); err != nil { + t.Errorf("Expected a second Cancel to do nothing, got %v", err) + } + // The pre-run check finishes it cancelled; no task completed, so no undo + _, ack := take(t, e.a) + e.finish(t, e.a, ack, contract.TaskEnd{Workflow: "w1", Task: "a", Outcome: contract.OutcomeCancelled, Reason: "workflow cancelled"}) + if r := e.state(t, "w1"); r.State != contract.WorkflowCancelled || r.FinishedAt == 0 { + t.Errorf("got %s", r.State) + } + if _, err := e.st.DepResult(ctx, "w1", "a"); !errors.Is(err, contract.ErrDepFailed) { + t.Errorf("Expected ErrDepFailed, got %v", err) + } + }) +} + +// A completed result over the limit fails the task. +func TestWorkflowResultLimit(t *testing.T) { + e := newWFEnv(t, false, WithWorkflowResultLimit(8)) + ctx := context.Background() + if err := e.st.Submit(ctx, wfPlan("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.a, "a"))); err != nil { + t.Fatal(err) + } + _, ack := take(t, e.a) + e.finish(t, e.a, ack, done("w1", "a", `{"big":"0123456789"}`)) + r := e.state(t, "w1") + if r.Tasks[0].State != contract.JobFailed || r.Tasks[0].Reason != ErrResultTooLarge.Error() || r.Tasks[0].ResultSize != 0 { + t.Errorf("got %s %q", taskStates(r), r.Tasks[0].Reason) + } +} + +// Crash after step a: the end and its pushes are in the workflow file's +// outbox only; recovery pushes them. +func TestWorkflowCrashAfterRecord(t *testing.T) { + e := newWFEnv(t, false) + ctx := context.Background() + if err := e.st.Submit(ctx, wfPlan("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.b, "a"))); err != nil { + t.Fatal(err) + } + _, ack := take(t, e.a) + end := done("w1", "a", `1`) + var r endResult + if err := e.st.write(ctx, func(tx *sql.Tx) (err error) { + r, err = e.st.endTx(ctx, tx, e.a.QueueID(), end) + return err + }); err != nil { + t.Fatal(err) + } + if r.minSeq == 0 || e.outbox(t) != 1 || e.b.Len() != 0 { + t.Fatalf("Expected one outbox row and nothing pushed, got %d rows, b %d", e.outbox(t), e.b.Len()) + } + e.st.recoverOutbox(0) + if e.outbox(t) != 0 || e.b.Len() != 1 { + t.Fatalf("Expected recovery to push the row, got %d rows, b %d", e.outbox(t), e.b.Len()) + } + // The job is redelivered after the restart, skipped by the pre-run check + e.a.requeueNoAck() + task, ack2 := take(t, e.a) + if ok, _ := e.st.Runnable(ctx, "w1", task, false); ok { + t.Fatal("Expected the redelivered job to be skipped") + } + _ = ack + e.finish(t, e.a, ack2, contract.TaskEnd{Workflow: "w1", Task: "a", Outcome: contract.OutcomeCancelled, Reason: "workflow cancelled"}) + if got := taskStates(e.state(t, "w1")); got != "a=completed b=pending" || e.b.Len() != 1 { + t.Errorf("Expected the end unchanged and one push, got %s, b %d", got, e.b.Len()) + } +} + +// Leftover outbox rows are pushed when a store opens. +func TestWorkflowRecoverOnOpen(t *testing.T) { + e := newWFEnv(t, false) + ctx := context.Background() + if err := e.st.Submit(ctx, wfPlan("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.b, "a"))); err != nil { + t.Fatal(err) + } + _, _ = take(t, e.a) + if err := e.st.write(ctx, func(tx *sql.Tx) error { + _, err := e.st.endTx(ctx, tx, e.a.QueueID(), done("w1", "a", "")) + return err + }); err != nil { + t.Fatal(err) + } + e.st.Close() + st2, err := NewWorkflowStore(e.qs) + if err != nil { + t.Fatal(err) + } + defer st2.Close() + if e.b.Len() != 1 { + t.Errorf("Expected the leftover row pushed at open, got %d", e.b.Len()) + } +} + +// The loop pushes rows left by a writer, once they are older than the +// recovery interval. +func TestWorkflowRecoveryLoop(t *testing.T) { + old := recoverEvery + recoverEvery = 20 * time.Millisecond + defer func() { recoverEvery = old }() + e := newWFEnv(t, false) + ctx := context.Background() + if err := e.st.Submit(ctx, wfPlan("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.b, "a"))); err != nil { + t.Fatal(err) + } + _, _ = take(t, e.a) + if err := e.st.write(ctx, func(tx *sql.Tx) error { + _, err := e.st.endTx(ctx, tx, e.a.QueueID(), done("w1", "a", "")) + return err + }); err != nil { + t.Fatal(err) + } + for range 100 { + if e.b.Len() == 1 { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatal("Expected the loop to push the leftover row") +} + +// Crash after step b: a duplicate push of the same job ID stores nothing. +func TestWorkflowDuplicatePush(t *testing.T) { + e := newWFEnv(t, false) + ctx := context.Background() + if err := e.st.Submit(ctx, wfPlan("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.b, "a"))); err != nil { + t.Fatal(err) + } + _, ack := take(t, e.a) + var r endResult + if err := e.st.write(ctx, func(tx *sql.Tx) (err error) { + r, err = e.st.endTx(ctx, tx, e.a.QueueID(), done("w1", "a", "")) + return err + }); err != nil { + t.Fatal(err) + } + where, args := `workflow_id = ? AND seq >= ?`, []any{"w1", r.minSeq} + rows, err := e.st.f.db.Query(`SELECT queue_id, job_id, item, priority FROM `+wfOutbox+` WHERE `+where, args...) + if err != nil { + t.Fatal(err) + } + var row outRow + for rows.Next() { + rows.Scan(&row.queueID, &row.jobID, &row.item, &row.priority) + } + rows.Close() + e.st.drain(ctx, where, args...) // step b done, the crash comes before the ack + if !pushRow(&handle{q: e.b}, row) || e.b.Len() != 1 { + t.Fatalf("Expected the duplicate push to store nothing, got %d rows", e.b.Len()) + } + // Redelivery of a: skipped, then finished; b is still queued once + e.a.requeueNoAck() + _, ack = take(t, e.a) + e.finish(t, e.a, ack, contract.TaskEnd{Workflow: "w1", Task: "a", Outcome: contract.OutcomeCancelled}) + if e.b.Len() != 1 || e.a.Len() != 0 { + t.Errorf("Expected b once and a settled, got b %d a %d", e.b.Len(), e.a.Len()) + } +} + +// Single file: the end, the pushes and the ack are one transaction, so a +// failed push (an unopened queue) leaves everything as it was. +func TestWorkflowSingleFileAtomic(t *testing.T) { + e := newWFEnv(t, true) + ctx := context.Background() + if err := e.st.Submit(ctx, wfPlan("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.b, "a"))); err != nil { + t.Fatal(err) + } + if n := e.outbox(t); n != 0 { + t.Fatalf("Expected no outbox rows in one file, got %d", n) + } + _, ack := take(t, e.a) + e.qs.(*queues).mu.Lock() + delete(e.qs.(*queues).byID, e.b.QueueID()) // b is not open in this client + e.qs.(*queues).mu.Unlock() + if _, ok := e.st.FinishTask(e.a.QueueID(), ack, done("w1", "a", "")); ok { + t.Fatal("Expected FinishTask to fail when a push target is unknown") + } + if got := taskStates(e.state(t, "w1")); got != "a=pending b=waiting" { + t.Errorf("Expected the workflow untouched, got %s", got) + } + if n := stats(t, e.a).Counts.Processing; n != 1 { + t.Errorf("Expected the job still processing, got %d processing", n) + } +} + +// Ended workflows are dropped after the retention, running ones are kept. +func TestWorkflowRetention(t *testing.T) { + eachLayout(t, func(t *testing.T, single bool) { + e := newWFEnv(t, single, WithWorkflowRetention(time.Hour)) + ctx := context.Background() + var clock atomic.Int64 + clock.Store(time.Now().UnixMilli()) + e.st.setClock(func() time.Time { return time.UnixMilli(clock.Load()) }) + for _, id := range []string{"ended", "running"} { + if err := e.st.Submit(ctx, wfPlan(id, planTask(id, "a", e.a))); err != nil { + t.Fatal(err) + } + } + _, ack := take(t, e.a) + e.finish(t, e.a, ack, done("ended", "a", `1`)) + + clock.Add(30 * time.Minute.Milliseconds()) + if err := e.st.sweep(ctx); err != nil { + t.Fatal(err) + } + if _, err := e.st.Record(ctx, "ended"); err != nil { + t.Fatalf("Expected the workflow kept inside the retention, got %v", err) + } + clock.Add(2 * time.Hour.Milliseconds()) + if err := e.st.sweep(ctx); err != nil { + t.Fatal(err) + } + if _, err := e.st.Record(ctx, "ended"); !errors.Is(err, contract.ErrWorkflowNotFound) { + t.Errorf("Expected the ended workflow swept, got %v", err) + } + if _, err := e.st.Record(ctx, "running"); err != nil { + t.Errorf("Expected the running workflow kept, got %v", err) + } + var n int + e.st.f.db.QueryRow(`SELECT COUNT(*) FROM ` + wfTasks + ` WHERE workflow_id = 'ended'`).Scan(&n) + if n != 0 { + t.Errorf("Expected the tasks swept too, got %d", n) + } + // A finish of a swept workflow settles the job and returns ok + if err := e.st.Submit(ctx, wfPlan("gone", planTask("gone", "a", e.a))); err != nil { + t.Fatal(err) + } + _, ack = take(t, e.a) + e.st.f.db.Exec(`DELETE FROM ` + wfWorkflows + ` WHERE id = 'gone'`) + if pushed, ok := e.st.FinishTask(e.a.QueueID(), ack, done("gone", "a", "")); !ok || len(pushed) != 0 { + t.Errorf("Expected ok with no IDs for a gone workflow, got %v %t", pushed, ok) + } + }) +} + +func TestWorkflowListing(t *testing.T) { + e := newWFEnv(t, false) + ctx := context.Background() + var clock atomic.Int64 + clock.Store(1_000_000) + e.st.setClock(func() time.Time { return time.UnixMilli(clock.Load()) }) + for _, id := range []string{"b", "a", "c"} { + if err := e.st.Submit(ctx, wfPlan(id, planTask(id, "t", e.a))); err != nil { + t.Fatal(err) + } + if id == "a" { + clock.Add(1) // c is newer; a and b share a ms + } + } + _, ack := take(t, e.a) + e.finish(t, e.a, ack, done("b", "t", "")) + + ids := func(state contract.WorkflowState, off, lim int) string { + keys, err := e.st.WorkflowKeys(ctx, state, off, lim) + if err != nil { + t.Fatal(err) + } + recs, err := e.st.Workflows(ctx, state, off, lim) + if err != nil || len(recs) != len(keys) { + t.Fatalf("Workflows %v %d vs keys %d", err, len(recs), len(keys)) + } + var s []string + for i, k := range keys { + if recs[i].ID != k.ID || recs[i].CreatedAt != k.CreatedAt { + t.Errorf("record %+v != key %+v", recs[i], k) + } + s = append(s, k.ID) + } + return strings.Join(s, ",") + } + if got := ids("", 0, 10); got != "c,a,b" { + t.Errorf("Expected newest first then by ID, got %s", got) + } + if got := ids("", 1, 1); got != "a" { + t.Errorf("got %s", got) + } + if got := ids(contract.WorkflowCompleted, 0, 10); got != "b" { + t.Errorf("got %s", got) + } + if got := ids("", 0, 0); got != "" { + t.Errorf("got %s", got) + } + if got := ids("", -3, -1); got != "" { + t.Errorf("got %s", got) + } +} + +func TestWorkflowTag(t *testing.T) { + for _, c := range []struct { + name string + item string + ok bool + }{ + {"task", `{"id":"wf:w:t","wf":"w","task":"t"}`, true}, + {"undo", `{"id":"wf:w:t:undo","wf":"w","task":"t","undo":true}`, true}, + {"undo false", `{"id":"wf:w:t","wf":"w","task":"t","undo":false}`, true}, + {"undo null", `{"id":"wf:w:t","wf":"w","task":"t","undo":null}`, true}, + {"undo not bool", `{"id":"wf:w:t","wf":"w","task":"t","undo":1}`, false}, + {"plain", `{"id":"x","data":1}`, false}, + {"wrong id", `{"id":"wf:w:u","wf":"w","task":"t"}`, false}, + {"missing :undo", `{"id":"wf:w:t","wf":"w","task":"t","undo":true}`, false}, + {"bad names", `{"id":"wf:w w:t","wf":"w w","task":"t"}`, false}, + {"index", `{"id":"wf:index:t","wf":"index","task":"t"}`, false}, + {"upper WF", `{"id":"wf:w:t","wf":"w","WF":"x","task":"t"}`, false}, + {"upper Task", `{"id":"wf:w:t","wf":"w","Task":"t"}`, false}, + {"Id", `{"Id":"wf:w:t","id":"wf:w:t","wf":"w","task":"t"}`, false}, + {"long s", `{"id":"wf:w:t","wf":"w","taſk":"t","task":"t"}`, false}, + {"kelvin", `{"id":"wf:w:t","wf":"w","tasK":"t","task":"t"}`, false}, + {"not object", `[1]`, false}, + {"wf number", `{"id":"wf:w:t","wf":1,"task":"t"}`, false}, + } { + if _, ok := workflowTag([]byte(c.item)); ok != c.ok { + t.Errorf("%s: got %t, want %t", c.name, ok, c.ok) + } + } +} + +// A cancel mark never drops a valid workflow item, whichever pop takes it; +// task JSON whose keys differ in case is a plain job and is dropped. +func TestWorkflowItemSurvivesCancelMark(t *testing.T) { + eachLayout(t, func(t *testing.T, single bool) { + e := newWFEnv(t, single) + ctx := context.Background() + if err := e.st.Submit(ctx, wfPlan("w1", planTask("w1", "a", e.a))); err != nil { + t.Fatal(err) + } + if err := e.a.CancelJob("wf:w1:a"); err != nil { + t.Fatal(err) + } + if _, ok := e.a.Dequeue(); !ok { + t.Fatal("Expected the marked workflow item to be handed out") + } + // An envelope whose id does not match its tag is a plain job + e.a.EnqueueOnce([]byte(`{"id":"wf:w1:x","status":"Created","data":1,"wf":"w1","task":"y"}`), "wf:w1:x") + e.a.CancelJob("wf:w1:x") + items, acks := make([]any, 2), make([]string, 2) + if n, _, err := e.a.DequeueBatchWithAckIds(items, acks); err != nil || n != 0 { + t.Fatalf("Expected the mismatched item dropped as a plain job, got %d %v", n, err) + } + + upper := []byte(`{"id":"wf:w1:b","status":"Created","data":1,"WF":"w1","wf":"w1","task":"b"}`) + e.a.EnqueueOnce(upper, "wf:w1:b") + e.a.CancelJob("wf:w1:b") + if _, ok := e.a.Dequeue(); ok { + t.Error("Expected the case-variant item dropped as a plain job") + } + valid := wfItem("w1", "c", false) + e.a.EnqueueOnce(valid, "wf:w1:c") + e.a.CancelJob("wf:w1:c") + n, _, err := e.a.DequeueBatchWithAckIds(items, acks) + if err != nil || n != 1 { + t.Errorf("Expected the batch pop to hand out the marked workflow item, got %d %v", n, err) + } + }) +} + +// A purged workflow task ends cancelled; its workflow goes on and what that +// releases is pushed after the delete. +func TestWorkflowPurge(t *testing.T) { + eachLayout(t, func(t *testing.T, single bool) { + e := newWFEnv(t, single) + ctx := context.Background() + plan := wfPlan("w1", planTask("w1", "a", e.a), planTask("w1", "n", e.a, "a"), planTask("w1", "z", e.b, "a")) + plan.Tasks[2].Always = true + if err := e.st.Submit(ctx, plan); err != nil { + t.Fatal(err) + } + e.a.Purge() + r := e.state(t, "w1") + if got := taskStates(r); got != "a=cancelled n=cancelled z=pending" || r.Tasks[0].Reason != purgeReason { + t.Fatalf("got %s reason %q", got, r.Tasks[0].Reason) + } + if e.b.Len() != 1 || e.a.Len() != 0 { + t.Errorf("Expected the always task pushed into b, got b %d a %d", e.b.Len(), e.a.Len()) + } + task, ack := take(t, e.b) + e.finish(t, e.b, ack, done("w1", task, "")) + if r := e.state(t, "w1"); r.State != contract.WorkflowFailed { + t.Errorf("Expected failed (a cancelled end before any cancel is a failure), got %s", r.State) + } + // A purge with no workflow items, or an already ended task, changes nothing + e.b.Purge() + if r := e.state(t, "w1"); r.State != contract.WorkflowFailed { + t.Errorf("got %s", r.State) + } + }) +} + +// End to end with varmq workers on two queues: the workflow runs through the +// registered store, in both layouts. +func TestWorkflowWithVarmqWorkers(t *testing.T) { + eachLayout(t, func(t *testing.T, single bool) { + e := newWFEnv(t, single) + var mu sync.Mutex + var order []string + run := func(name string) func(varmq.Job[int]) error { + return func(j varmq.Job[int]) error { + mu.Lock() + order = append(order, name) + mu.Unlock() + return nil + } + } + wa := varmq.NewErrWorker(run("a")) + wb := varmq.NewErrWorker(run("b")) + defer wa.Stop() + defer wb.Stop() + qa, qb := wa.WithQueue(e.a), wb.WithQueue(e.b) + + wf := varmq.NewWorkflow(e.st, varmq.WithWorkflowID("w1")) + ta := wf.Add("a", qa, 1) + wf.Add("b", qb, 2).After(ta) + if err := wf.Submit(context.Background()); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + r, err := wf.WaitContext(ctx) + if err != nil { + t.Fatal(err) + } + if r.State != contract.WorkflowCompleted || !slices.Equal(order, []string{"a", "b"}) { + t.Errorf("got %s %v", r.State, order) + } + }) +} diff --git a/queues.go b/queues.go index 14836db..53dc6f3 100644 --- a/queues.go +++ b/queues.go @@ -12,6 +12,7 @@ type queues struct { cfg storeConfig root string // folder, or the resolved file under WithSingleFile files map[*file]struct{} // one reference held per file + byID map[string]*handle // queues this client opened, by QueueID: FinishTask's lookup closed bool mu sync.Mutex } @@ -48,6 +49,7 @@ func New(path string, opts ...StoreOption) (Queues, error) { cfg: cfg, root: root, files: map[*file]struct{}{}, + byID: map[string]*handle{}, }, nil } @@ -98,6 +100,23 @@ func (qs *queues) filePath(name string) (string, error) { return resolve(filepath.Join(qs.root, foldName(name)+".db")) } +// rootOf is the scope root of a database file path: the file under +// WithSingleFile, else its folder, resolved. +func (qs *queues) rootOf(path string) string { + if qs.cfg.singleFile { + return path + } + return filepath.Dir(path) +} + +// handleOf returns the queue this client opened under id. +func (qs *queues) handleOf(id string) (*handle, bool) { + qs.mu.Lock() + defer qs.mu.Unlock() + h, ok := qs.byID[id] + return h, ok +} + // open finds or creates the handle for name, taking a reference on its file // the first time this Queues reaches that file. func (qs *queues) open(name string, priority bool, opts []Option) (*handle, error) { @@ -123,7 +142,7 @@ func (qs *queues) open(name string, priority bool, opts []Option) (*handle, erro fresh := f == nil if fresh { - if f, err = acquire(path, qs.cfg.checkpointPages); err != nil { + if f, err = acquire(path, qs.rootOf(path), qs.cfg.checkpointPages); err != nil { return nil, err } } @@ -137,6 +156,7 @@ func (qs *queues) open(name string, priority bool, opts []Option) (*handle, erro } qs.files[f] = struct{}{} + qs.byID[h.queue().id] = h return h, nil } @@ -172,5 +192,6 @@ func (qs *queues) Close() error { errs = append(errs, f.release()) } qs.files = nil + qs.byID = nil return errors.Join(errs...) } diff --git a/registry.go b/registry.go index c04e8d1..3af7dea 100644 --- a/registry.go +++ b/registry.go @@ -15,6 +15,7 @@ import ( // process that reaches it. type file struct { key string // resolved path, the registry key + scope string // "sqliteq:" + store root: the folder, or the file under WithSingleFile db *sql.DB pages int // checkpoint pages it was opened with refs int // Queues values holding it; under registry.mu @@ -79,7 +80,7 @@ var registry = struct { // acquire returns the registry entry for the resolved path, opening the pool // on first use, and takes one reference on it. -func acquire(path string, pages int) (*file, error) { +func acquire(path, root string, pages int) (*file, error) { registry.mu.Lock() defer registry.mu.Unlock() @@ -102,6 +103,7 @@ func acquire(path string, pages int) (*file, error) { f := &file{ key: path, + scope: "sqliteq:" + root, db: db, pages: pages, refs: 1, diff --git a/wfmachine.go b/wfmachine.go new file mode 100644 index 0000000..b2695fc --- /dev/null +++ b/wfmachine.go @@ -0,0 +1,315 @@ +package sqliteq + +import ( + "fmt" + "slices" + + "github.com/goptics/varmq/contract" +) + +// Workflow state machine: varmq's wfmachine.go over a workflow loaded from +// the tables. Pure: no I/O. The caller holds the write tx. Every walk goes +// in order, never map order, so pushes come out in task order. + +type wfTask struct { + name string + deps []string + always bool + queueID string + jobID string + priority int + undo *wfUndo + state contract.JobState + open int // dependencies not ended yet + reason string + result []byte // set by a completed finish only; not loaded + resultSize int // length of the stored result + undoState contract.JobState + undoOpen int // undos of completed descendants not ended yet + undoReason string +} + +// wfUndo is a task's compensation job. +type wfUndo struct{ queueID, jobID string } + +type wfFlow struct { + id string + state contract.WorkflowState + created int64 + finished int64 + failFast bool + undoOnCancel bool + failed bool // an unabsorbed failure happened before any cancel + cancelled bool // cancel was called + order []string + tasks map[string]*wfTask + next map[string][]string // dependents per task, in order +} + +// wfEffect is one thing storage must do: push a task (or its undo), and/or +// end the workflow. +type wfEffect struct { + push string + undo bool + end contract.WorkflowState +} + +func (s *wfFlow) index() { + s.next = make(map[string][]string, len(s.order)) + for _, name := range s.order { + for _, d := range s.tasks[name].deps { + s.next[d] = append(s.next[d], name) + } + } +} + +// finish applies a task's (or undo's) outcome. A finish for a task that is +// not pending or processing, or for an ended workflow, does nothing. +func (s *wfFlow) finish(e contract.TaskEnd) ([]wfEffect, error) { + t, ok := s.tasks[e.Task] + if !ok { + return nil, fmt.Errorf("varmq: workflow %q has no task %q", e.Workflow, e.Task) + } + if e.Undo { + return s.finishUndo(t, e), nil + } + if s.state != contract.WorkflowRunning || !active(t.state) { + return nil, nil + } + + t.reason, t.result = e.Reason, e.Result + push := map[string]bool{} + if e.Outcome == contract.OutcomeCompleted { + t.state = contract.JobCompleted + for _, d := range s.next[e.Task] { + s.release(d, push) + } + return s.settle(push), nil + } + + t.state = contract.JobFailed + if e.Outcome == contract.OutcomeCancelled { + t.state = contract.JobCancelled + } + // After cancel, ends (pre-run cancels included) are not failures + if !s.cancelled && !s.absorbed(e.Task) { + if s.failFast && !s.failed { // first unabsorbed failure only + s.failFast0(e.Task, push) + } + s.failed = true + } + s.drop(e.Task, push) + return s.settle(push), nil +} + +// cancel cancels waiting tasks now; pending ones stay for the pre-run check. +func (s *wfFlow) cancel(undo bool) []wfEffect { + if s.state != contract.WorkflowRunning || s.cancelled { + return nil + } + s.cancelled, s.undoOnCancel = true, undo + for _, name := range s.order { + if t := s.tasks[name]; t.state == contract.JobWaiting { + t.state = contract.JobCancelled + } + } + return s.settle(nil) +} + +// runnable is the pre-run check. +func (s *wfFlow) runnable(task string, undo bool) bool { + t, ok := s.tasks[task] + if !ok { + return false + } + if undo { + return s.state == contract.WorkflowCompensating && t.undo != nil && t.undoState == contract.JobPending + } + stopped := s.failFast && s.failed && !t.always + return s.state == contract.WorkflowRunning && !s.cancelled && !stopped && t.state == contract.JobPending +} + +func (s *wfFlow) finishUndo(t *wfTask, e contract.TaskEnd) []wfEffect { + if s.state != contract.WorkflowCompensating || !active(t.undoState) { + return nil + } + t.undoState, t.undoReason = contract.JobFailed, e.Reason + if e.Outcome == contract.OutcomeCompleted { + t.undoState, t.undoReason = contract.JobCompleted, "" + } + push := map[string]bool{} + up := s.reach(e.Task, true) + for _, name := range s.order { + x := s.tasks[name] + if !up[name] || x.undoState != contract.JobWaiting { + continue + } + if x.undoOpen--; x.undoOpen == 0 { + x.undoState = contract.JobPending + push[name] = true + } + } + fx := s.emit(push, true) + + failed := false + for _, name := range s.order { + switch s.tasks[name].undoState { + case contract.JobWaiting, contract.JobPending, contract.JobProcessing: + return fx + case contract.JobFailed: + failed = true + } + } + end := contract.WorkflowCompensated + if failed { + end = contract.WorkflowCompensationFailed + } + return s.endWith(fx, end) +} + +// release counts one ended dependency of a waiting task; pushes it at zero. +func (s *wfFlow) release(name string, push map[string]bool) { + t := s.tasks[name] + if t.state != contract.JobWaiting { + return + } + if t.open--; t.open == 0 { + t.state = contract.JobPending + push[name] = true + } +} + +// drop applies a failed or cancelled task to its dependents: always ones +// count it as ended, others are cancelled, recursively. +func (s *wfFlow) drop(name string, push map[string]bool) { + for _, d := range s.next[name] { + t := s.tasks[d] + if t.state != contract.JobWaiting { + continue + } + if t.always { + s.release(d, push) + continue + } + t.state = contract.JobCancelled + s.drop(d, push) + } +} + +// absorbed: the task has dependents and every one is always. +func (s *wfFlow) absorbed(name string) bool { + next := s.next[name] + return len(next) > 0 && !slices.ContainsFunc(next, func(d string) bool { return !s.tasks[d].always }) +} + +// failFast0 cancels every waiting task on an unabsorbed failure, except the +// failed task's always dependents; each cancelled task is dropped. +func (s *wfFlow) failFast0(failed string, push map[string]bool) { + keep := map[string]bool{} + for _, d := range s.next[failed] { + keep[d] = s.tasks[d].always + } + var cancelled []string + for _, name := range s.order { + if t := s.tasks[name]; t.state == contract.JobWaiting && !keep[name] { + t.state = contract.JobCancelled + cancelled = append(cancelled, name) + } + } + for _, name := range cancelled { + s.drop(name, push) + } +} + +// reach returns every task reachable from name: ancestors when up, else +// descendants. +func (s *wfFlow) reach(name string, up bool) map[string]bool { + seen := map[string]bool{} + stack := []string{name} + for len(stack) > 0 { + n := stack[len(stack)-1] + stack = stack[:len(stack)-1] + edges := s.next[n] + if up { + edges = s.tasks[n].deps + } + for _, m := range edges { + if !seen[m] { + seen[m] = true + stack = append(stack, m) + } + } + } + return seen +} + +// settle emits the pushes and, once the forward run is over, ends the +// workflow or starts compensation. +func (s *wfFlow) settle(push map[string]bool) []wfEffect { + fx := s.emit(push, false) + for _, name := range s.order { + switch s.tasks[name].state { + case contract.JobWaiting, contract.JobPending, contract.JobProcessing: + return fx + } + } + if s.failed || s.cancelled && s.undoOnCancel { + if undos := s.compensate(); len(undos) > 0 { + s.state = contract.WorkflowCompensating + return append(fx, undos...) + } + } + switch { + case s.failed: + return s.endWith(fx, contract.WorkflowFailed) + case s.cancelled: + return s.endWith(fx, contract.WorkflowCancelled) + } + return s.endWith(fx, contract.WorkflowCompleted) +} + +// compensate marks the undo of every completed task waiting on the undos of +// all its completed descendants that have one. Returns the undos ready now. +func (s *wfFlow) compensate() []wfEffect { + push := map[string]bool{} + for _, name := range s.order { + t := s.tasks[name] + if t.state != contract.JobCompleted || t.undo == nil { + continue + } + t.undoState, t.undoOpen = contract.JobWaiting, 0 + for d := range s.reach(name, false) { + if dt := s.tasks[d]; dt.state == contract.JobCompleted && dt.undo != nil { + t.undoOpen++ + } + } + if t.undoOpen == 0 { + t.undoState = contract.JobPending + push[name] = true + } + } + return s.emit(push, true) +} + +func (s *wfFlow) endWith(fx []wfEffect, state contract.WorkflowState) []wfEffect { + s.state = state + return append(fx, wfEffect{end: state}) +} + +// emit turns a push set into effects in task order. +func (s *wfFlow) emit(push map[string]bool, undo bool) []wfEffect { + if len(push) == 0 { + return nil + } + fx := make([]wfEffect, 0, len(push)) + for _, name := range s.order { + if push[name] { + fx = append(fx, wfEffect{push: name, undo: undo}) + } + } + return fx +} + +func active(st contract.JobState) bool { + return st == contract.JobPending || st == contract.JobProcessing +} diff --git a/workflow.go b/workflow.go new file mode 100644 index 0000000..7e05e8c --- /dev/null +++ b/workflow.go @@ -0,0 +1,1016 @@ +package sqliteq + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "log" + "path/filepath" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/goptics/varmq/contract" +) + +// Workflows (varmq spec/redis-storage.md 6.9 to 6.12), on SQLite. The store +// keeps workflows, tasks and an outbox in three tables: +// __sqliteq_wf_workflows, __sqliteq_wf_tasks, __sqliteq_wf_outbox. In folder +// layout they live in their own file, /__sqliteq_workflows.db; under +// WithSingleFile, in the store's file. Each end runs the state machine +// (wfmachine.go) over the workflow loaded in one write transaction. +// +// Separate files cannot share one atomic transaction, so folder layout pushes +// what an end releases through the outbox: the release is written in the +// workflow transaction, then pushed into the queue with EnqueueOnce, then +// deleted from the outbox. A recovery loop pushes leftovers. A duplicate +// push or a redelivered job is safe: the pre-run check (Runnable) skips a +// task that already ended. Under WithSingleFile the end, the pushes and the +// ack are one transaction. + +var ( + _ contract.WorkflowStore = (*WorkflowStore)(nil) + _ contract.WorkflowInspector = (*WorkflowStore)(nil) + _ contract.WorkflowKeyLister = (*WorkflowStore)(nil) +) + +const ( + defaultResultLimit = 64 << 10 + defaultRetention = 7 * 24 * time.Hour + + // workflowsFile is the folder layout's workflow database. + workflowsFile = "__sqliteq_workflows.db" + + wfWorkflows = `"__sqliteq_wf_workflows"` + wfTasks = `"__sqliteq_wf_tasks"` + wfOutbox = `"__sqliteq_wf_outbox"` + + // sweepWorkflowsEvery is how often the loop drops expired workflows. + sweepWorkflowsEvery = time.Minute + // drainBatch is how many outbox rows one read takes. + drainBatch = 256 + // purgeReason is the reason of a task its queue's Purge ended. + purgeReason = "queue purged" +) + +// recoverEvery is how often the recovery loop pushes leftover outbox rows; +// those younger than it are left to the writer that is about to push them. +var recoverEvery = time.Second + +// ErrResultTooLarge is contract.ErrResultTooLarge: a completed task whose +// result is over the store's limit fails with it. +var ErrResultTooLarge = contract.ErrResultTooLarge + +type workflowConfig struct { + resultLimit int + retention time.Duration +} + +// WorkflowOption configures NewWorkflowStore. +type WorkflowOption func(*workflowConfig) error + +// WithWorkflowResultLimit caps a workflow task's result at n bytes (default +// 64 KiB). A bigger one fails the task with ErrResultTooLarge. +func WithWorkflowResultLimit(n int) WorkflowOption { + return func(c *workflowConfig) error { + if n <= 0 { + return fmt.Errorf("sqliteq: workflow result limit %d must be above 0", n) + } + c.resultLimit = n + return nil + } +} + +// WithWorkflowRetention sets how long an ended workflow is kept (default 7 +// days). A running one is kept until it ends. +func WithWorkflowRetention(d time.Duration) WorkflowOption { + return func(c *workflowConfig) error { + if d < time.Millisecond { + return fmt.Errorf("sqliteq: workflow retention %v must be at least 1ms", d) + } + c.retention = d + return nil + } +} + +// WorkflowStore is the workflow store of one Queues root. NewWorkflowStore +// registers it for its scope, so varmq finds it from the queues' ScopeID. +type WorkflowStore struct { + qs *queues + f *file + single bool + scope string + cfg workflowConfig + clock atomic.Pointer[func() time.Time] // tests move it + + stop chan struct{} + done chan struct{} + closeOnce sync.Once + closeErr error +} + +// NewWorkflowStore opens the workflow store of qs and registers it with +// contract.RegisterWorkflowStore, so every queue of the same root can run +// workflow tasks; a process that only runs workers needs this one call. In +// folder layout it uses its own file, __sqliteq_workflows.db in the folder; +// under WithSingleFile, tables of the store's file. A second store for the +// same root is contract.ErrScopeRegistered. Close it before qs. +func NewWorkflowStore(qs Queues, opts ...WorkflowOption) (*WorkflowStore, error) { + q, ok := qs.(*queues) + if !ok { + return nil, fmt.Errorf("sqliteq: workflow store needs the Queues made by New, got %T", qs) + } + cfg := workflowConfig{resultLimit: defaultResultLimit, retention: defaultRetention} + for _, opt := range opts { + if opt == nil { + return nil, errNilOption + } + if err := opt(&cfg); err != nil { + return nil, err + } + } + + path := q.root + if !q.cfg.singleFile { + var err error + if path, err = resolve(filepath.Join(q.root, workflowsFile)); err != nil { + return nil, err + } + } + + q.mu.Lock() + closed := q.closed + q.mu.Unlock() + if closed { + return nil, errors.New("sqliteq: queues are closed") + } + f, err := acquire(path, q.root, q.cfg.checkpointPages) + if err != nil { + return nil, err + } + + s := &WorkflowStore{qs: q, f: f, single: q.cfg.singleFile, scope: "sqliteq:" + q.root, cfg: cfg, + stop: make(chan struct{}), done: make(chan struct{})} + if err := s.init(); err != nil { + return nil, errors.Join(err, f.release()) + } + if err := contract.RegisterWorkflowStore(s); err != nil { + return nil, errors.Join(err, f.release()) + } + + s.recoverOutbox(0) // leftovers of an earlier run + go s.loop() + return s, nil +} + +// init creates the tables under the write mutex. +func (s *WorkflowStore) init() error { + s.f.mu.Lock() + defer s.f.mu.Unlock() + + for _, stmt := range []string{ + `CREATE TABLE IF NOT EXISTS ` + wfWorkflows + ` ( + id TEXT PRIMARY KEY, + hash TEXT NOT NULL, + state TEXT NOT NULL, + created_at INTEGER NOT NULL, + finished_at INTEGER, + fail_fast INTEGER NOT NULL, + undo_on_cancel INTEGER NOT NULL, + failed INTEGER NOT NULL DEFAULT 0, + cancelled INTEGER NOT NULL DEFAULT 0 + )`, + `CREATE INDEX IF NOT EXISTS "__sqliteq_wf_workflows_created" ON ` + wfWorkflows + ` (created_at DESC, id)`, + `CREATE INDEX IF NOT EXISTS "__sqliteq_wf_workflows_finished" ON ` + wfWorkflows + ` (finished_at) WHERE finished_at IS NOT NULL`, + `CREATE TABLE IF NOT EXISTS ` + wfTasks + ` ( + workflow_id TEXT NOT NULL, + name TEXT NOT NULL, + ord INTEGER NOT NULL, + state TEXT NOT NULL, + reason TEXT NOT NULL DEFAULT '', + result BLOB, + open INTEGER NOT NULL DEFAULT 0, + deps TEXT NOT NULL, + always INTEGER NOT NULL, + queue_id TEXT NOT NULL, + job_id TEXT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + item BLOB, + has_undo INTEGER NOT NULL DEFAULT 0, + undo_queue_id TEXT NOT NULL DEFAULT '', + undo_job_id TEXT NOT NULL DEFAULT '', + undo_item BLOB, + undo_state TEXT NOT NULL DEFAULT '', + undo_open INTEGER NOT NULL DEFAULT 0, + undo_reason TEXT NOT NULL DEFAULT '', + PRIMARY KEY (workflow_id, name) + ) WITHOUT ROWID`, + `CREATE TABLE IF NOT EXISTS ` + wfOutbox + ` ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + workflow_id TEXT NOT NULL, + queue_id TEXT NOT NULL, + job_id TEXT NOT NULL, + item BLOB NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS "__sqliteq_wf_outbox_workflow" ON ` + wfOutbox + ` (workflow_id)`, + } { + if _, err := s.f.db.Exec(stmt); err != nil { + return fmt.Errorf("sqliteq: failed to create workflow tables: %w", err) + } + } + return nil +} + +func (s *WorkflowStore) now() time.Time { + if f := s.clock.Load(); f != nil { + return (*f)() + } + return time.Now() +} + +// ScopeID implements contract.Scoped. +func (s *WorkflowStore) ScopeID() string { return s.scope } + +// Close unregisters the store, stops its loop and releases its file. Queues +// stay open. Calling it again does nothing. +func (s *WorkflowStore) Close() error { + s.closeOnce.Do(func() { + close(s.stop) + <-s.done + contract.UnregisterWorkflowStore(s) + s.closeErr = s.f.release() + }) + return s.closeErr +} + +// loop pushes leftover outbox rows and sweeps expired workflows until Close. +func (s *WorkflowStore) loop() { + defer close(s.done) + + tick := time.NewTicker(recoverEvery) + defer tick.Stop() + lastSweep := s.now() + for { + select { + case <-s.stop: + return + case <-tick.C: + } + s.recoverOutbox(recoverEvery) + if now := s.now(); now.Sub(lastSweep) >= sweepWorkflowsEvery { + lastSweep = now + if err := s.sweep(context.Background()); err != nil { + log.Printf("sqliteq: sweeping workflows: %v", err) + } + } + } +} + +func (s *WorkflowStore) notFound(id string) error { + return fmt.Errorf("%w: %s", contract.ErrWorkflowNotFound, id) +} + +// validWFName: 1 to n characters of [A-Za-z0-9_.-] (spec 1.2). +func validWFName(s string, n int) bool { + if len(s) == 0 || len(s) > n { + return false + } + for _, c := range []byte(s) { + if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_' || c == '.' || c == '-') { + return false + } + } + return true +} + +func validWorkflowID(id string) bool { return validWFName(id, 128) && id != "index" } +func validTaskName(n string) bool { return validWFName(n, 64) } + +// wfFold lower-cases s the way encoding/json matches keys: ASCII case, and +// U+017F and U+212A fold to s and k. +func wfFold(s string) string { + return strings.ToLower(strings.NewReplacer("ſ", "s", "K", "k").Replace(s)) +} + +// wfItemTag is the workflow fields of an item. +type wfItemTag struct { + wf, task string + undo bool +} + +// workflowTag reports the workflow fields of item data when they could come +// from a Submit (spec 3, wftag): wf and task valid names, undo absent, null +// or a boolean, and id "wf::" (":undo" appended for undo true). A +// key that differs from id, wf, task or undo only in case makes it a plain +// job, as encoding/json would read it as that field. +func workflowTag(data []byte) (wfItemTag, bool) { + var t wfItemTag + if len(data) == 0 || data[0] != '{' { + return t, false + } + var env map[string]json.RawMessage + if json.Unmarshal(data, &env) != nil { + return t, false + } + for k := range env { + switch f := wfFold(k); f { + case "id", "wf", "task", "undo": + if k != f { + return t, false + } + } + } + var id string + if json.Unmarshal(env["wf"], &t.wf) != nil || json.Unmarshal(env["task"], &t.task) != nil || + json.Unmarshal(env["id"], &id) != nil { + return t, false + } + if raw, ok := env["undo"]; ok && string(raw) != "null" { + if json.Unmarshal(raw, &t.undo) != nil { + return t, false + } + } + if !validWorkflowID(t.wf) || !validTaskName(t.task) { + return t, false + } + want := "wf:" + t.wf + ":" + t.task + if t.undo { + want += ":undo" + } + return t, id == want +} + +func isWorkflowItem(data []byte) bool { + _, ok := workflowTag(data) + return ok +} + +// querier is what a *sql.DB and a *sql.Tx share. +type querier interface { + QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row + ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) +} + +// write runs fn in one write transaction under the file's write mutex. +func (s *WorkflowStore) write(ctx context.Context, fn func(tx *sql.Tx) error) error { + s.f.mu.Lock() + defer s.f.mu.Unlock() + + tx, err := s.f.db.BeginTx(ctx, nil) + if err != nil { + return err + } + if err := fn(tx); err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + +// planQueue is the queue handle of a plan's task or undo, in this scope. +func (s *WorkflowStore) planQueue(name string, q, item any) (*handle, []byte, error) { + var h *handle + switch v := q.(type) { + case *Queue: + if v != nil { + h = &handle{q: v} + } + case *PriorityQueue: + if v != nil { + h = &handle{pq: v} + } + } + if h == nil || h.queue().ScopeID() != s.scope { + return nil, nil, fmt.Errorf("%w: task %q queue is not in sqliteq scope %s", contract.ErrScopeMismatch, name, s.scope) + } + b, ok := item.([]byte) + if !ok || !json.Valid(b) || len(b) == 0 || b[0] != '{' { + return nil, nil, fmt.Errorf("varmq: task %q: item is not a JSON envelope", name) + } + return h, b, nil +} + +// learn records handles a plan names, so a worker that only finishes tasks +// still finds queues made by another Queues of the same root. +func (qs *queues) learn(h *handle) { + qs.mu.Lock() + defer qs.mu.Unlock() + if qs.byID == nil { + return + } + if _, ok := qs.byID[h.queue().id]; !ok { + qs.byID[h.queue().id] = h + } +} + +func btoi(b bool) int { + if b { + return 1 + } + return 0 +} + +// Submit stores plan p and pushes its roots. A stored ID is success when its +// plan hash is equal, else ErrWorkflowExists. +func (s *WorkflowStore) Submit(ctx context.Context, p contract.WorkflowPlan) error { + if len(p.Tasks) == 0 { + return fmt.Errorf("varmq: workflow %s has no tasks", p.ID) + } + if !validWorkflowID(p.ID) { + return fmt.Errorf("varmq: invalid workflow id %q", p.ID) + } + + type planned struct { + tp contract.TaskPlan + item []byte + uitem []byte + deps []byte + uq *handle + qh *handle + } + plan := make([]planned, len(p.Tasks)) + for i, tp := range p.Tasks { + if !validTaskName(tp.Name) { + return fmt.Errorf("varmq: invalid task name %q", tp.Name) + } + h, item, err := s.planQueue(tp.Name, tp.Queue, tp.Item) + if err != nil { + return err + } + pl := planned{tp: tp, item: item, qh: h} + if u := tp.Undo; u != nil { + if pl.uq, pl.uitem, err = s.planQueue(tp.Name+" undo", u.Queue, u.Item); err != nil { + return err + } + } + deps := tp.Deps + if deps == nil { + deps = []string{} + } + pl.deps, _ = json.Marshal(deps) // strings: Marshal can't fail + plan[i] = pl + } + for _, pl := range plan { + s.qs.learn(pl.qh) + if pl.uq != nil { + s.qs.learn(pl.uq) + } + } + + var minSeq int64 + var exists bool + err := s.write(ctx, func(tx *sql.Tx) error { + var hash string + switch err := tx.QueryRowContext(ctx, `SELECT hash FROM `+wfWorkflows+` WHERE id = ?`, p.ID).Scan(&hash); { + case err == nil: + if hash != p.Hash { + return fmt.Errorf("%w: %s", contract.ErrWorkflowExists, p.ID) + } + exists = true + return nil + case !errors.Is(err, sql.ErrNoRows): + return err + } + + now := s.now().UnixMilli() + if _, err := tx.ExecContext(ctx, `INSERT INTO `+wfWorkflows+ + ` (id, hash, state, created_at, fail_fast, undo_on_cancel) VALUES (?, ?, 'running', ?, ?, ?)`, + p.ID, p.Hash, now, btoi(p.FailFast), btoi(p.UndoOnCancel)); err != nil { + return err + } + + w := &wfFlow{id: p.ID, state: contract.WorkflowRunning, created: now, failFast: p.FailFast, + undoOnCancel: p.UndoOnCancel, tasks: make(map[string]*wfTask, len(plan))} + var roots []wfEffect + for i, pl := range plan { + tp := pl.tp + if _, dup := w.tasks[tp.Name]; dup { + return fmt.Errorf("varmq: duplicate task name %q", tp.Name) + } + t := &wfTask{name: tp.Name, deps: tp.Deps, always: tp.Always, queueID: pl.qh.queue().id, jobID: tp.JobID, + priority: tp.Priority, state: contract.JobPending, open: len(tp.Deps)} + if t.open > 0 { + t.state = contract.JobWaiting + } else { + roots = append(roots, wfEffect{push: tp.Name}) + } + var uq, uj string + var uitem []byte + if tp.Undo != nil { + t.undo = &wfUndo{queueID: pl.uq.queue().id, jobID: tp.Undo.JobID} + uq, uj, uitem = t.undo.queueID, t.undo.jobID, pl.uitem + } + w.tasks[tp.Name] = t + w.order = append(w.order, tp.Name) + if _, err := tx.ExecContext(ctx, `INSERT INTO `+wfTasks+ + ` (workflow_id, name, ord, state, open, deps, always, queue_id, job_id, priority, item, + has_undo, undo_queue_id, undo_job_id, undo_item) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + p.ID, tp.Name, i, string(t.state), t.open, string(pl.deps), btoi(tp.Always), t.queueID, t.jobID, + tp.Priority, pl.item, btoi(tp.Undo != nil), uq, uj, uitem); err != nil { + return err + } + } + w.index() + + var err error + minSeq, _, err = s.release(ctx, tx, w, roots) + return err + }) + if err != nil { + return err + } + if !exists { + if minSeq != 0 { + s.drain(ctx, `workflow_id = ? AND seq >= ?`, p.ID, minSeq) + } + if err := s.sweep(ctx); err != nil { + log.Printf("sqliteq: sweeping workflows: %v", err) + } + } + return nil +} + +// outRow is one push: an item bound for a queue under its job ID. +type outRow struct { + queueID, jobID string + item []byte + priority int +} + +// stamp sets the finish time when fx ends the workflow; call it before save. +func (s *WorkflowStore) stamp(w *wfFlow, fx []wfEffect) { + for _, e := range fx { + if e.end != "" { + w.finished = max(s.now().UnixMilli(), w.created) // never before the start + } + } +} + +// release turns the pushes of fx into rows (reading and clearing each task's +// stored item) and delivers them in tx. +// Folder layout: outbox rows, minSeq the first one's seq (0 when none). +// Single file: straight into the queues, ids the queue IDs it pushed into. +func (s *WorkflowStore) release(ctx context.Context, tx *sql.Tx, w *wfFlow, fx []wfEffect) (minSeq int64, ids []string, err error) { + var rows []outRow + for _, e := range fx { + if e.push == "" { + continue + } + t := w.tasks[e.push] + col, row := "item", outRow{queueID: t.queueID, jobID: t.jobID, priority: t.priority} + if e.undo { + col, row = "undo_item", outRow{queueID: t.undo.queueID, jobID: t.undo.jobID} + } + where := ` WHERE workflow_id = ? AND name = ?` + if err = tx.QueryRowContext(ctx, `SELECT `+col+` FROM `+wfTasks+where, w.id, e.push).Scan(&row.item); err != nil { + return 0, nil, err + } + if row.item == nil { + continue + } + if !e.undo { + if _, err = tx.ExecContext(ctx, `UPDATE `+wfTasks+` SET item = NULL`+where, w.id, e.push); err != nil { + return 0, nil, err + } + } + rows = append(rows, row) + } + + for _, r := range rows { + if s.single { + h, ok := s.qs.handleOf(r.queueID) + if !ok { + return 0, nil, fmt.Errorf("sqliteq: workflow %s: queue %s is not open in this client", w.id, r.queueID) + } + if err = h.queue().enqueueTx(tx, r.item, r.priority, r.jobID); err != nil { + return 0, nil, err + } + if !slices.Contains(ids, r.queueID) { + ids = append(ids, r.queueID) + } + continue + } + res, err := tx.ExecContext(ctx, `INSERT INTO `+wfOutbox+ + ` (workflow_id, queue_id, job_id, item, priority, created_at) VALUES (?, ?, ?, ?, ?, ?)`, + w.id, r.queueID, r.jobID, r.item, r.priority, s.now().UnixMilli()) + if err != nil { + return 0, nil, err + } + if minSeq == 0 { + if minSeq, err = res.LastInsertId(); err != nil { + return 0, nil, err + } + } + } + return minSeq, ids, nil +} + +// drain pushes the outbox rows matching where into their queues and deletes +// each pushed one. A queue this client did not open, or a closed one, keeps +// its rows for the next recovery. It returns the queue IDs pushed into. +func (s *WorkflowStore) drain(ctx context.Context, where string, args ...any) []string { + if s.single { + return nil + } + var ids []string + after := int64(0) + for { + rows, err := s.f.db.QueryContext(ctx, `SELECT seq, queue_id, job_id, item, priority FROM `+wfOutbox+ + ` WHERE seq > ? AND (`+where+`) ORDER BY seq LIMIT `+fmt.Sprint(drainBatch), append([]any{after}, args...)...) + if err != nil { + log.Printf("sqliteq: reading workflow outbox: %v", err) + return ids + } + type row struct { + seq int64 + outRow + } + var batch []row + for rows.Next() { + var r row + if err := rows.Scan(&r.seq, &r.queueID, &r.jobID, &r.item, &r.priority); err != nil { + rows.Close() + return ids + } + batch = append(batch, r) + } + err = errors.Join(rows.Err(), rows.Close()) + if err != nil || len(batch) == 0 { + return ids + } + + for _, r := range batch { + after = r.seq + h, ok := s.qs.handleOf(r.queueID) + if !ok || !pushRow(h, r.outRow) { + continue + } + s.f.mu.Lock() + _, err := s.f.db.ExecContext(ctx, `DELETE FROM `+wfOutbox+` WHERE seq = ?`, r.seq) + s.f.mu.Unlock() + if err != nil { + log.Printf("sqliteq: deleting workflow outbox row: %v", err) + continue + } + if !slices.Contains(ids, r.queueID) { + ids = append(ids, r.queueID) + } + } + if len(batch) < drainBatch { + return ids + } + } +} + +// pushRow adds the row's item to its queue once per job ID. +func pushRow(h *handle, r outRow) bool { + if h.pq != nil { + return h.pq.EnqueueOnce(r.item, r.priority, r.jobID) + } + return h.q.EnqueueOnce(r.item, r.jobID) +} + +// recoverOutbox pushes outbox rows at least minAge old. +func (s *WorkflowStore) recoverOutbox(minAge time.Duration) { + s.drain(context.Background(), `created_at <= ?`, s.now().Add(-minAge).UnixMilli()) +} + +// sweep drops workflows ended more than the retention ago, with their tasks +// and outbox rows. +func (s *WorkflowStore) sweep(ctx context.Context) error { + cut := s.now().Add(-s.cfg.retention).UnixMilli() + return s.write(ctx, func(tx *sql.Tx) error { + gone := `workflow_id IN (SELECT id FROM ` + wfWorkflows + ` WHERE finished_at IS NOT NULL AND finished_at <= ?)` + for _, t := range []string{wfTasks, wfOutbox} { + if _, err := tx.ExecContext(ctx, `DELETE FROM `+t+` WHERE `+gone, cut); err != nil { + return err + } + } + _, err := tx.ExecContext(ctx, `DELETE FROM `+wfWorkflows+` WHERE finished_at IS NOT NULL AND finished_at <= ?`, cut) + return err + }) +} + +// load reads workflow id with its tasks (no items, no results). +func (s *WorkflowStore) load(ctx context.Context, q querier, id string) (*wfFlow, error) { + w := &wfFlow{id: id} + var ( + finished sql.NullInt64 + ff, uoc, f, cn int + ) + switch err := q.QueryRowContext(ctx, `SELECT state, created_at, finished_at, fail_fast, undo_on_cancel, failed, cancelled FROM `+ + wfWorkflows+` WHERE id = ?`, id).Scan(&w.state, &w.created, &finished, &ff, &uoc, &f, &cn); { + case errors.Is(err, sql.ErrNoRows): + return nil, s.notFound(id) + case err != nil: + return nil, err + } + w.finished = finished.Int64 + w.failFast, w.undoOnCancel, w.failed, w.cancelled = ff == 1, uoc == 1, f == 1, cn == 1 + + rows, err := q.QueryContext(ctx, `SELECT name, state, reason, open, deps, always, queue_id, job_id, priority, COALESCE(length(result), 0), + has_undo, undo_queue_id, undo_job_id, undo_state, undo_open, undo_reason + FROM `+wfTasks+` WHERE workflow_id = ? ORDER BY ord`, id) + if err != nil { + return nil, err + } + defer rows.Close() + + w.tasks = map[string]*wfTask{} + for rows.Next() { + var ( + t wfTask + deps string + always int + hasUndo int + u wfUndo + ) + if err := rows.Scan(&t.name, &t.state, &t.reason, &t.open, &deps, &always, &t.queueID, &t.jobID, &t.priority, &t.resultSize, + &hasUndo, &u.queueID, &u.jobID, &t.undoState, &t.undoOpen, &t.undoReason); err != nil { + return nil, err + } + if err := json.Unmarshal([]byte(deps), &t.deps); err != nil { + return nil, fmt.Errorf("sqliteq: workflow %s task %s: bad deps: %w", id, t.name, err) + } + if len(t.deps) == 0 { + t.deps = nil + } + t.always = always == 1 + if hasUndo == 1 { + t.undo = &u + } + w.tasks[t.name] = &t + w.order = append(w.order, t.name) + } + if err := rows.Err(); err != nil { + return nil, err + } + w.index() + return w, nil +} + +// mut is the part of a task (and of the workflow) the machine changes. +type mut struct { + state, undoState contract.JobState + open, undoOpen int + reason, undoReason string + wfState contract.WorkflowState + failed, cancelled, undoCance bool +} + +func (w *wfFlow) snapshot() map[string]mut { + m := make(map[string]mut, len(w.tasks)+1) + for name, t := range w.tasks { + m[name] = mut{state: t.state, undoState: t.undoState, open: t.open, undoOpen: t.undoOpen, reason: t.reason, undoReason: t.undoReason} + } + m[""] = mut{wfState: w.state, failed: w.failed, cancelled: w.cancelled, undoCance: w.undoOnCancel} + return m +} + +// dirty lists the tasks the machine changed since snapshot orig, and whether +// the workflow row changed. +func (w *wfFlow) dirty(orig map[string]mut) (tasks []string, flow bool) { + for _, name := range w.order { + t := w.tasks[name] + if orig[name] != (mut{state: t.state, undoState: t.undoState, open: t.open, undoOpen: t.undoOpen, reason: t.reason, undoReason: t.undoReason}) { + tasks = append(tasks, name) + } + } + return tasks, orig[""] != mut{wfState: w.state, failed: w.failed, cancelled: w.cancelled, undoCance: w.undoOnCancel} +} + +// save writes what changed since orig; result is the finishing task's +// result, written when that task is task. +func (s *WorkflowStore) save(ctx context.Context, tx *sql.Tx, w *wfFlow, orig map[string]mut, task string, result []byte) error { + tasks, flow := w.dirty(orig) + if flow { + var fin any + if w.finished > 0 { + fin = w.finished + } + if _, err := tx.ExecContext(ctx, `UPDATE `+wfWorkflows+` SET state = ?, finished_at = ?, undo_on_cancel = ?, failed = ?, cancelled = ? WHERE id = ?`, + string(w.state), fin, btoi(w.undoOnCancel), btoi(w.failed), btoi(w.cancelled), w.id); err != nil { + return err + } + } + for _, name := range tasks { + t := w.tasks[name] + if _, err := tx.ExecContext(ctx, `UPDATE `+wfTasks+` SET state = ?, open = ?, reason = ?, undo_state = ?, undo_open = ?, undo_reason = ? + WHERE workflow_id = ? AND name = ?`, + string(t.state), t.open, t.reason, string(t.undoState), t.undoOpen, t.undoReason, w.id, name); err != nil { + return err + } + if name == task && len(result) > 0 && t.state == contract.JobCompleted { + if _, err := tx.ExecContext(ctx, `UPDATE `+wfTasks+` SET result = ? WHERE workflow_id = ? AND name = ?`, result, w.id, name); err != nil { + return err + } + } + } + return nil +} + +// targets are the queue IDs a workflow can push into: every task's and undo's, +// once each, in task order. +func (w *wfFlow) targets() []string { + var ids []string + add := func(id string) { + if !slices.Contains(ids, id) { + ids = append(ids, id) + } + } + for _, name := range w.order { + t := w.tasks[name] + add(t.queueID) + if t.undo != nil { + add(t.undo.queueID) + } + } + return ids +} + +// Cancel cancels workflow id: waiting tasks now, queued ones at their +// pre-run check. undo nil keeps the workflow's undo_on_cancel. Cancelling an +// ended or cancelled workflow does nothing. +func (s *WorkflowStore) Cancel(ctx context.Context, id string, undo *bool) error { + if !validWorkflowID(id) { + return s.notFound(id) + } + var minSeq int64 + err := s.write(ctx, func(tx *sql.Tx) error { + w, err := s.load(ctx, tx, id) + if err != nil { + return err + } + on := w.undoOnCancel + if undo != nil { + on = *undo + } + orig := w.snapshot() + fx := w.cancel(on) + s.stamp(w, fx) + if err := s.save(ctx, tx, w, orig, "", nil); err != nil { + return err + } + minSeq, _, err = s.release(ctx, tx, w, fx) + return err + }) + if err != nil { + return err + } + if minSeq != 0 { + s.drain(ctx, `workflow_id = ? AND seq >= ?`, id, minSeq) + } + return nil +} + +// Runnable is the pre-run check (spec 6.12). +func (s *WorkflowStore) Runnable(ctx context.Context, workflowID, task string, undo bool) (bool, error) { + if !validWorkflowID(workflowID) { + return false, s.notFound(workflowID) + } + var ( + state contract.WorkflowState + ff, failed, cancelled int + tstate, ustate contract.JobState + always, hasUndo, taskFound int + ) + err := s.f.db.QueryRowContext(ctx, `SELECT w.state, w.fail_fast, w.failed, w.cancelled, + COALESCE(t.state, ''), COALESCE(t.undo_state, ''), COALESCE(t.always, 0), COALESCE(t.has_undo, 0), t.name IS NOT NULL + FROM `+wfWorkflows+` w LEFT JOIN `+wfTasks+` t ON t.workflow_id = w.id AND t.name = ? WHERE w.id = ?`, task, workflowID). + Scan(&state, &ff, &failed, &cancelled, &tstate, &ustate, &always, &hasUndo, &taskFound) + switch { + case errors.Is(err, sql.ErrNoRows): + return false, s.notFound(workflowID) + case err != nil: + return false, err + case taskFound == 0: + return false, nil + } + if undo { + return state == contract.WorkflowCompensating && hasUndo == 1 && ustate == contract.JobPending, nil + } + stopped := ff == 1 && failed == 1 && always == 0 + return state == contract.WorkflowRunning && cancelled == 0 && !stopped && tstate == contract.JobPending, nil +} + +// DepResult returns task's stored result: ErrDepFailed when it failed or was +// cancelled, ErrNoResult when it left none, has not ended or is not a task. +func (s *WorkflowStore) DepResult(ctx context.Context, workflowID, task string) ([]byte, error) { + if !validWorkflowID(workflowID) { + return nil, s.notFound(workflowID) + } + var ( + state sql.NullString + result []byte + ) + err := s.f.db.QueryRowContext(ctx, `SELECT t.state, t.result FROM `+wfWorkflows+` w + LEFT JOIN `+wfTasks+` t ON t.workflow_id = w.id AND t.name = ? WHERE w.id = ?`, task, workflowID).Scan(&state, &result) + switch { + case errors.Is(err, sql.ErrNoRows): + return nil, s.notFound(workflowID) + case err != nil: + return nil, err + } + switch contract.JobState(state.String) { + case contract.JobFailed, contract.JobCancelled: + return nil, contract.ErrDepFailed + case contract.JobCompleted: + if len(result) > 0 { + return result, nil + } + } + return nil, contract.ErrNoResult +} + +// Record returns workflow id as stored. +func (s *WorkflowStore) Record(ctx context.Context, id string) (contract.WorkflowRecord, error) { + if !validWorkflowID(id) { + return contract.WorkflowRecord{}, s.notFound(id) + } + w, err := s.load(ctx, s.f.db, id) + if err != nil { + return contract.WorkflowRecord{}, err + } + return w.record(), nil +} + +// record builds the WorkflowRecord: tasks in Add order. An undo is waiting +// until compensation reaches it. +func (w *wfFlow) record() contract.WorkflowRecord { + r := contract.WorkflowRecord{ID: w.id, State: w.state, CreatedAt: w.created, FinishedAt: w.finished, + UndoOnCancel: w.undoOnCancel, Tasks: make([]contract.TaskRecord, 0, len(w.order))} + for _, name := range w.order { + t := w.tasks[name] + tr := contract.TaskRecord{Name: name, State: t.state, Deps: slices.Clone(t.deps), QueueID: t.queueID, + JobID: t.jobID, Always: t.always, Reason: t.reason, ResultSize: t.resultSize} + if t.undo != nil { + st := t.undoState + if st == "" { + st = contract.JobWaiting + } + tr.Undo = &contract.TaskRecord{Name: name, State: st, QueueID: t.undo.queueID, JobID: t.undo.jobID, Reason: t.undoReason} + } + r.Tasks = append(r.Tasks, tr) + } + return r +} + +// Workflows lists up to limit workflows in state ("" any), newest first then +// by ID, skipping offset. +func (s *WorkflowStore) Workflows(ctx context.Context, state contract.WorkflowState, offset, limit int) ([]contract.WorkflowRecord, error) { + keys, err := s.WorkflowKeys(ctx, state, offset, limit) + if err != nil { + return nil, err + } + out := make([]contract.WorkflowRecord, 0, len(keys)) + for _, k := range keys { + r, err := s.Record(ctx, k.ID) + if errors.Is(err, contract.ErrWorkflowNotFound) { + continue // swept since the listing + } + if err != nil { + return nil, err + } + out = append(out, r) + } + return out, nil +} + +// WorkflowKeys is Workflows without the records. +func (s *WorkflowStore) WorkflowKeys(ctx context.Context, state contract.WorkflowState, offset, limit int) ([]contract.WorkflowKey, error) { + offset, limit = max(offset, 0), min(max(limit, 0), 1<<31-1) + if limit == 0 { + return []contract.WorkflowKey{}, nil + } + where, args := "", []any{} + if state != "" { + where, args = ` WHERE state = ?`, append(args, string(state)) + } + rows, err := s.f.db.QueryContext(ctx, `SELECT id, created_at FROM `+wfWorkflows+where+ + ` ORDER BY created_at DESC, id LIMIT ? OFFSET ?`, append(args, limit, offset)...) + if err != nil { + return nil, err + } + defer rows.Close() + + out := []contract.WorkflowKey{} + for rows.Next() { + var k contract.WorkflowKey + if err := rows.Scan(&k.ID, &k.CreatedAt); err != nil { + return nil, err + } + out = append(out, k) + } + return out, rows.Err() +} + +// setClock moves the store's clock; tests only. +func (s *WorkflowStore) setClock(f func() time.Time) { s.clock.Store(&f) } diff --git a/workflow_finish.go b/workflow_finish.go new file mode 100644 index 0000000..0126ed4 --- /dev/null +++ b/workflow_finish.go @@ -0,0 +1,166 @@ +package sqliteq + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "log" + "time" + + "github.com/goptics/varmq/contract" +) + +// endResult is what one task end did to its workflow. +type endResult struct { + gone bool // no such workflow (expired): the item settles alone + skip bool // the end does not name this queue's task: the item settles alone + refinish bool // the task had ended already: nothing changed + targets []string // refinish: every push-target queue of the workflow + minSeq int64 // folder layout: first outbox row written + pushed []string // single file: queues pushed into +} + +// FinishTask settles the job (queueID, ackID) and applies the task end to +// its workflow. Folder layout: one transaction in the workflow file records +// the end and writes what it releases to the outbox; then each row is pushed +// into its queue and deleted; then the job is acknowledged. A crash between +// the steps leaves rows the recovery loop pushes, or a job that is +// redelivered and skipped by the pre-run check (Runnable). Single file: all +// three steps are one transaction. It returns the queue IDs it pushed into, +// or, for a task that had ended already, every push-target queue of the +// workflow. A gone workflow, or an end that does not name this queue's task, +// settles the job and returns ok with no IDs. A completed result over the +// limit fails the task (ErrResultTooLarge); one that is not JSON is dropped. +func (s *WorkflowStore) FinishTask(queueID, ackID string, e contract.TaskEnd) ([]string, bool) { + h, ok := s.qs.handleOf(queueID) + if !ok { + log.Printf("sqliteq: finish task %s/%s: queue %s is not open in this client", e.Workflow, e.Task, queueID) + return nil, false + } + q := h.queue() + + state, reason, result := contract.JobCompleted, e.Reason, e.Result + switch e.Outcome { + case contract.OutcomeCompleted: + reason = "" + switch { + case e.Undo: + result = nil + case len(result) > s.cfg.resultLimit: + e.Outcome, state, reason, result = contract.OutcomeFailed, contract.JobFailed, contract.ErrResultTooLarge.Error(), nil + case !json.Valid(result): + result = nil + } + case contract.OutcomeFailed: + state, result = contract.JobFailed, nil + case contract.OutcomeCancelled: + state, result = contract.JobCancelled, nil + default: + log.Printf("sqliteq: task %s/%s: unknown outcome %q", e.Workflow, e.Task, e.Outcome) + return nil, false + } + e.Reason, e.Result = reason, result + + if !validWorkflowID(e.Workflow) || !validTaskName(e.Task) { + return nil, q.finish(ackID, state, reason, result) + } + + ctx := context.Background() + var r endResult + var acked bool + err := s.write(ctx, func(tx *sql.Tx) (err error) { + if r, err = s.endTx(ctx, tx, queueID, e); err != nil { + return err + } + if !s.single { + return nil + } + // One file: the end, its pushes and the ack commit together + id, ok := parseAckID(ackID) + if !ok { + return nil + } + acked = true + _, err = q.finishTx(tx, id, state, reason, result, time.Now()) + return err + }) + if err != nil { + log.Printf("sqliteq: finishing workflow %s task %s: %v", e.Workflow, e.Task, err) + return nil, false + } + + pushed := r.pushed + if !s.single { + pushed = nil + if r.minSeq != 0 { + pushed = s.drain(ctx, `workflow_id = ? AND seq >= ?`, e.Workflow, r.minSeq) + } + } + if !acked { + q.finish(ackID, state, reason, result) + } + if r.refinish { + pushed = r.targets + } + return pushed, true +} + +// endTx applies end e of the task in queue queueID to its workflow, in tx: +// the state machine, the changed rows, and the release of what it makes ready. +func (s *WorkflowStore) endTx(ctx context.Context, tx *sql.Tx, queueID string, e contract.TaskEnd) (endResult, error) { + var r endResult + w, err := s.load(ctx, tx, e.Workflow) + if errors.Is(err, contract.ErrWorkflowNotFound) { + r.gone = true + return r, nil + } + if err != nil { + return r, err + } + + t := w.tasks[e.Task] + if t == nil || (e.Undo && (t.undo == nil || t.undo.queueID != queueID)) || (!e.Undo && t.queueID != queueID) { + r.skip = true + return r, nil + } + + orig := w.snapshot() + fx, err := w.finish(e) + if err != nil { + return r, err + } + if tasks, flow := w.dirty(orig); len(tasks) == 0 && !flow && len(fx) == 0 { + r.refinish, r.targets = true, w.targets() + return r, nil + } + s.stamp(w, fx) + if err := s.save(ctx, tx, w, orig, e.Task, e.Result); err != nil { + return r, err + } + r.minSeq, r.pushed, err = s.release(ctx, tx, w, fx) + return r, err +} + +// endPurged ends the workflow tasks of items a Purge of queue q removed as +// cancelled (reason "queue purged"), so their workflows go on. There is no +// job to settle: Purge deleted it. A task not in q, or one that had ended, +// changes nothing. +func (s *WorkflowStore) endPurged(q *Queue, tags []wfItemTag) { + ctx := context.Background() + for _, tag := range tags { + e := contract.TaskEnd{Workflow: tag.wf, Task: tag.task, Undo: tag.undo, Outcome: contract.OutcomeCancelled, Reason: purgeReason} + var r endResult + err := s.write(ctx, func(tx *sql.Tx) (err error) { + r, err = s.endTx(ctx, tx, q.id, e) + return err + }) + if err != nil { + log.Printf("sqliteq: purge of %s: ending workflow %s task %s: %v", q.name, tag.wf, tag.task, err) + continue + } + if r.minSeq != 0 { + s.drain(ctx, `workflow_id = ? AND seq >= ?`, tag.wf, r.minSeq) + } + } +} From 52a807080b5813ec9a5256696cd7096ab59f11e6 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Mon, 9 Nov 2026 21:41:02 +0600 Subject: [PATCH 2/8] docs: workflows section, changelog and example --- CHANGELOG.md | 14 ++++++++ README.md | 57 +++++++++++++++++++++++++++++- example/README.md | 6 ++++ example/main.go | 3 ++ example/workflow_example.go | 70 +++++++++++++++++++++++++++++++++++++ 5 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 example/workflow_example.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 3601f9f..9bb62a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Workflows (VarMQ spec 6.9 to 6.12): `NewWorkflowStore(qs, ...WorkflowOption)` returns the `contract.WorkflowStore` of a store root. (#NN) +- The store registers with `contract.RegisterWorkflowStore`; `Close` unregisters it and releases its file. (#NN) +- `WithWorkflowResultLimit` (default 64 KiB, `ErrResultTooLarge`) and `WithWorkflowRetention` (default 7 days). (#NN) +- `ScopeID()` on `Queue` and `PriorityQueue`: `sqliteq:` and the absolute store folder, or the file under `WithSingleFile`. (#NN) +- Folder layout: the store keeps its tables in `__sqliteq_workflows.db`. `WithSingleFile`: tables in the same file. (#NN) +- Tables `__sqliteq_wf_workflows`, `__sqliteq_wf_tasks`, `__sqliteq_wf_outbox`. (#NN) +- `FinishTask` in three steps in the folder layout: record the end and write the outbox, push each row with `EnqueueOnce`, acknowledge the job. (#NN) +- A recovery loop pushes leftover outbox rows at open and every second. (#NN) +- `WithSingleFile`: the end, the pushes and the ack are one transaction. (#NN) +- A duplicate push or a redelivered job is safe: the pre-run check skips a task that already ended. (#NN) +- `Workflows` (`contract.WorkflowInspector`) and `WorkflowKeys` (`contract.WorkflowKeyLister`) list workflows newest first. (#NN) +- A cancel mark never drops a valid workflow item; the workflow decides its end. (#NN) +- Task JSON whose keys differ only in case (`WF` next to `wf`) is a plain job. (#NN) +- `Purge` ends each purged workflow task as cancelled, reason `queue purged`; its workflow goes on. (#NN) - `Reject(ackID, reason)` on `Queue` and `PriorityQueue`, for VarMQ's `IAcknowledgeable` (varmq#173). (#21) - `Reject` finishes an item that failed for good: never requeued. (#21) - Rejected item deleted like an ack; with `WithRemoveOnComplete(false)` kept with `status` `failed`. (#21) diff --git a/README.md b/README.md index 46fc59e..3882e28 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ sqliteq imports VarMQ's `contract` module, and it imports VarMQ itself only in t A worker whose only source is the sqliteq queue claims several jobs in one statement through `DequeueBatchWithAckIds`. A worker bound to several queues claims one job per call. A job added with an ID (`WithJobId`, or one from `WithJobIdGenerator`) goes through `EnqueueOnce`. -sqliteq has no workflow store and no `JobReader`. So VarMQ's one-job read (`GET /queues/{id}/jobs/{jobId}`) answers 501 for a sqliteq queue. +sqliteq has no `JobReader`. So VarMQ's one-job read (`GET /queues/{id}/jobs/{jobId}`) answers 501 for a sqliteq queue. For workflows, see [Workflows](#workflows). ### Job outcome and cancel @@ -255,6 +255,61 @@ Both queue types implement VarMQ's `QueuePauser` and `PauseNotifier`. So VarMQ's `Paused()` reads this queue object's cache, and a pop refused for the pause sets it. While it is set, the queue object polls the row every 50 ms. So a `Resume` from another process wakes this one's workers within that time. `OnPauseChange(fn)` calls `fn` each time the cache changes, and `PauseRefusals()` counts the pops refused for a pause. +### Workflows + +A workflow is a graph of VarMQ tasks. A task runs after the tasks it depends on. sqliteq keeps the workflow state in SQLite, so a workflow survives a restart and spans several queues of one store. + +Create the store once, after the queues, and keep it open while workers run. `NewWorkflowStore` registers the store for the scope of its queues. VarMQ finds it from there. A process that only runs workers needs this one call too. + +```go +qs, _ := sqliteq.New("./data") +defer qs.Close() + +store, err := sqliteq.NewWorkflowStore(qs) +if err != nil { + log.Fatal(err) +} +defer store.Close() // before qs.Close + +resize, _ := qs.NewQueue("resize") +publish, _ := qs.NewQueue("publish") + +wf := varmq.NewWorkflow(store) +a := wf.Add("resize", worker.WithQueue(resize), "photo.jpg") +wf.Add("publish", worker.WithQueue(publish), "photo.jpg").After(a) + +wf.Submit(ctx) +record, _ := wf.Wait() +``` + +Options of `NewWorkflowStore`: + +- `WithWorkflowResultLimit(n)` caps a task result at `n` bytes. The default is 64 KiB. A bigger result fails the task with `ErrResultTooLarge`. +- `WithWorkflowRetention(d)` sets how long an ended workflow is kept. The default is 7 days. A running workflow is kept until it ends. + +Every queue of a workflow must have the same scope. The scope of a queue is `ScopeID()`: `sqliteq:` and the absolute path of the store folder, or of the file under `WithSingleFile`. One process holds one store for each scope. A second `NewWorkflowStore` for the same scope returns `contract.ErrScopeRegistered`. + +The store implements `contract.WorkflowStore`, `WorkflowInspector` and `WorkflowKeyLister`. So VarMQ's REST API lists and cancels sqliteq workflows. + +**Where the state lives.** In the folder layout the store uses its own file, `__sqliteq_workflows.db`. Under `WithSingleFile` it uses three tables of the store file. The tables are `__sqliteq_wf_workflows`, `__sqliteq_wf_tasks` and `__sqliteq_wf_outbox`. A queue name must not contain `__sqliteq_`, so a queue never takes these names. + +**How a task ends.** In the folder layout, two files cannot share one transaction. So `FinishTask` works in three steps: + +1. One transaction in the workflow file records the end and the result. It also writes each task the end releases into the outbox table. +2. Each outbox row is pushed into its queue with `EnqueueOnce` and the job ID `wf::`. Then the row is deleted. +3. The job is acknowledged in its queue. + +A recovery loop pushes leftover outbox rows when the store opens and every second while it is open. A crash between the steps is safe. A row pushed twice stores one job. A job that was redelivered finds its task already ended: the pre-run check (`Runnable`) skips it. Under `WithSingleFile` all three steps are one transaction. + +**Limits.** + +- The recovery loop cannot wake the workers of a process. A task pushed by the loop starts when a worker next polls its queue. +- The recovery loop pushes only into queues that this `Queues` created, or that a submitted plan named. +- A crash inside `Purge` of a queue with workflow tasks, after the delete and before the workflow update, leaves those tasks pending with no job. Cancel the workflow to end it. +- The stored items of waiting tasks and the undo items stay in the workflow file until the task is pushed or the workflow expires. + +A valid workflow item is never dropped by a cancel mark: the workflow decides how its task ends. A `Purge` of a queue ends each purged workflow task as cancelled with the reason `queue purged`. The workflow goes on. An item whose JSON has keys that differ only in case, such as `WF` next to `wf`, is a plain job. + ### Storage #### Layout on disk diff --git a/example/README.md b/example/README.md index dde010a..b137bc3 100644 --- a/example/README.md +++ b/example/README.md @@ -18,6 +18,11 @@ The example program provides two demonstrations: - Adding items with different priorities (lower number = higher priority) - Observing how items are dequeued in priority order +3. **Workflow Example** - Runs a two-step VarMQ workflow over two queues: + - Creating a workflow store with `NewWorkflowStore` + - Adding tasks to queues and ordering them with `After` + - Waiting for the workflow to complete + ## Running the Examples To run the examples: @@ -34,6 +39,7 @@ SQLiteQ Examples --------------- 1. Regular Queue Example 2. Priority Queue Example +3. Workflow Example Choose an example (default is 1): ``` diff --git a/example/main.go b/example/main.go index 686afc1..2d30399 100644 --- a/example/main.go +++ b/example/main.go @@ -90,6 +90,7 @@ func main() { fmt.Println("---------------") fmt.Println("1. Regular Queue Example") fmt.Println("2. Priority Queue Example") + fmt.Println("3. Workflow Example") // Default to regular queue example var choice string @@ -101,6 +102,8 @@ func main() { switch choice { case "2": priorityQueueExample() + case "3": + workflowExample() default: regularQueueExample() } diff --git a/example/workflow_example.go b/example/workflow_example.go new file mode 100644 index 0000000..a1ee91a --- /dev/null +++ b/example/workflow_example.go @@ -0,0 +1,70 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "time" + + "github.com/goptics/sqliteq" + "github.com/goptics/varmq" +) + +// workflowExample runs a two-step workflow over two queues. The state lives +// in __sqliteq_workflows.db next to the queue files. +func workflowExample() { + dir, err := os.MkdirTemp("", "sqliteq-workflow-*") + if err != nil { + log.Fatalf("Failed to create temp folder: %v", err) + } + defer os.RemoveAll(dir) + + qs, err := sqliteq.New(dir) + if err != nil { + log.Fatalf("Failed to open queues: %v", err) + } + defer qs.Close() + + resize, err := qs.NewQueue("resize") + if err != nil { + log.Fatalf("Failed to create queue: %v", err) + } + publish, err := qs.NewQueue("publish") + if err != nil { + log.Fatalf("Failed to create queue: %v", err) + } + + // Create the store after the queues; it registers itself for their scope + store, err := sqliteq.NewWorkflowStore(qs) + if err != nil { + log.Fatalf("Failed to create workflow store: %v", err) + } + defer store.Close() + + step := func(name string) func(varmq.Job[string]) error { + return func(j varmq.Job[string]) error { + fmt.Printf("%s %s\n", name, j.Data()) + return nil + } + } + resizer := varmq.NewErrWorker(step("resize")) + defer resizer.WaitAndStop() + publisher := varmq.NewErrWorker(step("publish")) + defer publisher.WaitAndStop() + + wf := varmq.NewWorkflow(store) + a := wf.Add("resize", resizer.WithQueue(resize), "photo.jpg") + wf.Add("publish", publisher.WithQueue(publish), "photo.jpg").After(a) + + if err := wf.Submit(context.Background()); err != nil { + log.Fatalf("Failed to submit workflow: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + record, err := wf.WaitContext(ctx) + if err != nil { + log.Fatalf("Workflow did not finish: %v", err) + } + fmt.Println("workflow", record.ID, record.State) +} From e739372b5ee943fe851fd558e6d0ebac935ed7fb Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Mon, 9 Nov 2026 21:46:20 +0600 Subject: [PATCH 3/8] fix: lock-free queue lookup and canonical scope root The workflow write holds the file mutex, so FinishTask must not take queues.mu: byID is now a copy-on-write atomic map. The folder root is resolved, so queue and store scopes agree through symlinks. --- README.md | 3 +++ layout_test.go | 51 +++++++++++++++++++++++++++++++++++++++++++++++++ queue_test.go | 13 ++++++++----- queues.go | 52 ++++++++++++++++++++++++++++---------------------- registry.go | 2 ++ workflow.go | 21 +++++++++----------- 6 files changed, 102 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 3882e28..cb36cd8 100644 --- a/README.md +++ b/README.md @@ -275,6 +275,9 @@ resize, _ := qs.NewQueue("resize") publish, _ := qs.NewQueue("publish") wf := varmq.NewWorkflow(store) +worker := varmq.NewErrWorker(func(j varmq.Job[string]) error { return nil }) +defer worker.WaitAndStop() + a := wf.Add("resize", worker.WithQueue(resize), "photo.jpg") wf.Add("publish", worker.WithQueue(publish), "photo.jpg").After(a) diff --git a/layout_test.go b/layout_test.go index 1b3bd0e..3b72bd9 100644 --- a/layout_test.go +++ b/layout_test.go @@ -10,6 +10,8 @@ import ( "strings" "sync" "testing" + + "github.com/goptics/varmq/contract" ) // mustNew opens a store at path and fails the test on error. @@ -698,3 +700,52 @@ func TestReopenAfterCloseNoRecovery(t *testing.T) { t.Error("Acknowledge on the closed handle failed after the reopen") } } + +// The scope is the canonical store root: a symlinked folder, and a queue +// file that is a symlink inside it, give the queues and the store one scope. +func TestScopeThroughSymlinks(t *testing.T) { + real := t.TempDir() + link := filepath.Join(t.TempDir(), "link") + if err := os.Symlink(real, link); err != nil { + t.Skipf("Cannot create a symlink: %v", err) + } + target := filepath.Join(t.TempDir(), "elsewhere.db") + if err := os.WriteFile(target, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, filepath.Join(real, "a.db")); err != nil { + t.Skipf("Cannot create a symlink: %v", err) + } + + qs := mustNew(t, link) + defer qs.Close() + a, err := qs.NewQueue("a") + if err != nil { + t.Fatal(err) + } + b, err := qs.NewQueue("b") + if err != nil { + t.Fatal(err) + } + st, err := NewWorkflowStore(qs) + if err != nil { + t.Fatal(err) + } + defer st.Close() + + want, err := filepath.EvalSymlinks(real) + if err != nil { + t.Fatal(err) + } + want = "sqliteq:" + want + if a.ScopeID() != want || b.ScopeID() != want || st.ScopeID() != want { + t.Fatalf("Expected scope %q, got %q %q %q", want, a.ScopeID(), b.ScopeID(), st.ScopeID()) + } + if got, ok := contract.LookupWorkflowStore(a.ScopeID()); !ok || got != contract.WorkflowStore(st) { + t.Error("Expected the queue's scope to find the store") + } + plan := wfPlan("w1", planTask("w1", "x", a), planTask("w1", "y", b, "x")) + if err := st.Submit(context.Background(), plan); err != nil { + t.Fatalf("Expected Submit across the symlinked queues to succeed, got %v", err) + } +} diff --git a/queue_test.go b/queue_test.go index a5ad706..4e1b170 100644 --- a/queue_test.go +++ b/queue_test.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "errors" "fmt" + "maps" "os" "path/filepath" "slices" @@ -2712,7 +2713,7 @@ func TestWorkflowCrashAfterRecord(t *testing.T) { if err := e.st.Submit(ctx, wfPlan("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.b, "a"))); err != nil { t.Fatal(err) } - _, ack := take(t, e.a) + _, _ = take(t, e.a) end := done("w1", "a", `1`) var r endResult if err := e.st.write(ctx, func(tx *sql.Tx) (err error) { @@ -2734,7 +2735,6 @@ func TestWorkflowCrashAfterRecord(t *testing.T) { if ok, _ := e.st.Runnable(ctx, "w1", task, false); ok { t.Fatal("Expected the redelivered job to be skipped") } - _ = ack e.finish(t, e.a, ack2, contract.TaskEnd{Workflow: "w1", Task: "a", Outcome: contract.OutcomeCancelled, Reason: "workflow cancelled"}) if got := taskStates(e.state(t, "w1")); got != "a=completed b=pending" || e.b.Len() != 1 { t.Errorf("Expected the end unchanged and one push, got %s, b %d", got, e.b.Len()) @@ -2843,9 +2843,12 @@ func TestWorkflowSingleFileAtomic(t *testing.T) { t.Fatalf("Expected no outbox rows in one file, got %d", n) } _, ack := take(t, e.a) - e.qs.(*queues).mu.Lock() - delete(e.qs.(*queues).byID, e.b.QueueID()) // b is not open in this client - e.qs.(*queues).mu.Unlock() + qs := e.qs.(*queues) + qs.mu.Lock() + unknown := maps.Clone(*qs.byID.Load()) + delete(unknown, e.b.QueueID()) // b is not open in this client + qs.byID.Store(&unknown) + qs.mu.Unlock() if _, ok := e.st.FinishTask(e.a.QueueID(), ack, done("w1", "a", "")); ok { t.Fatal("Expected FinishTask to fail when a push target is unknown") } diff --git a/queues.go b/queues.go index 53dc6f3..398215f 100644 --- a/queues.go +++ b/queues.go @@ -3,16 +3,18 @@ package sqliteq import ( "errors" "fmt" + "maps" "os" "path/filepath" "sync" + "sync/atomic" ) type queues struct { cfg storeConfig - root string // folder, or the resolved file under WithSingleFile - files map[*file]struct{} // one reference held per file - byID map[string]*handle // queues this client opened, by QueueID: FinishTask's lookup + root string // folder, or the resolved file under WithSingleFile + files map[*file]struct{} // one reference held per file + byID atomic.Pointer[map[string]*handle] // queues by QueueID, copy on write under mu; read lock-free closed bool mu sync.Mutex } @@ -45,12 +47,13 @@ func New(path string, opts ...StoreOption) (Queues, error) { return nil, err } - return &queues{ + qs := &queues{ cfg: cfg, root: root, files: map[*file]struct{}{}, - byID: map[string]*handle{}, - }, nil + } + qs.byID.Store(&map[string]*handle{}) + return qs, nil } // layout checks path against the chosen layout and returns the store root. @@ -86,7 +89,7 @@ func layout(path string, singleFile bool) (string, error) { if _, err := checkDSN(real); err != nil { return "", err } - return abs, nil + return real, nil } // filePath returns the resolved database file for a queue name. @@ -100,23 +103,26 @@ func (qs *queues) filePath(name string) (string, error) { return resolve(filepath.Join(qs.root, foldName(name)+".db")) } -// rootOf is the scope root of a database file path: the file under -// WithSingleFile, else its folder, resolved. -func (qs *queues) rootOf(path string) string { - if qs.cfg.singleFile { - return path - } - return filepath.Dir(path) -} - -// handleOf returns the queue this client opened under id. +// handleOf returns the queue this client opened under id. It takes no lock: +// a workflow write holds the file's write mutex, which comes after mu. func (qs *queues) handleOf(id string) (*handle, bool) { - qs.mu.Lock() - defer qs.mu.Unlock() - h, ok := qs.byID[id] + h, ok := (*qs.byID.Load())[id] return h, ok } +// setHandle records h under its QueueID, replacing a held one when replace; +// the caller holds mu. +func (qs *queues) setHandle(h *handle, replace bool) { + old := *qs.byID.Load() + id := h.queue().id + if _, ok := old[id]; ok && !replace { + return + } + next := maps.Clone(old) + next[id] = h + qs.byID.Store(&next) +} + // open finds or creates the handle for name, taking a reference on its file // the first time this Queues reaches that file. func (qs *queues) open(name string, priority bool, opts []Option) (*handle, error) { @@ -142,7 +148,7 @@ func (qs *queues) open(name string, priority bool, opts []Option) (*handle, erro fresh := f == nil if fresh { - if f, err = acquire(path, qs.rootOf(path), qs.cfg.checkpointPages); err != nil { + if f, err = acquire(path, qs.root, qs.cfg.checkpointPages); err != nil { return nil, err } } @@ -156,7 +162,7 @@ func (qs *queues) open(name string, priority bool, opts []Option) (*handle, erro } qs.files[f] = struct{}{} - qs.byID[h.queue().id] = h + qs.setHandle(h, true) return h, nil } @@ -192,6 +198,6 @@ func (qs *queues) Close() error { errs = append(errs, f.release()) } qs.files = nil - qs.byID = nil + qs.byID.Store(&map[string]*handle{}) return errors.Join(errs...) } diff --git a/registry.go b/registry.go index 3af7dea..cdfd603 100644 --- a/registry.go +++ b/registry.go @@ -73,6 +73,8 @@ func dsn(path string) string { // registry holds every open file in the process, keyed by resolved path. // Lock order is queues.mu, then registry.mu, then file.mu. A file's // batcher.mu is never held across a commit, so never together with file.mu. +// A workflow store's write transaction holds file.mu, so it never takes +// queues.mu: queues.handleOf reads an atomic copy-on-write map. var registry = struct { mu sync.Mutex files map[string]*file diff --git a/workflow.go b/workflow.go index 7e05e8c..af56602 100644 --- a/workflow.go +++ b/workflow.go @@ -402,11 +402,8 @@ func (s *WorkflowStore) planQueue(name string, q, item any) (*handle, []byte, er func (qs *queues) learn(h *handle) { qs.mu.Lock() defer qs.mu.Unlock() - if qs.byID == nil { - return - } - if _, ok := qs.byID[h.queue().id]; !ok { - qs.byID[h.queue().id] = h + if !qs.closed { + qs.setHandle(h, false) } } @@ -759,11 +756,11 @@ func (s *WorkflowStore) load(ctx context.Context, q querier, id string) (*wfFlow // mut is the part of a task (and of the workflow) the machine changes. type mut struct { - state, undoState contract.JobState - open, undoOpen int - reason, undoReason string - wfState contract.WorkflowState - failed, cancelled, undoCance bool + state, undoState contract.JobState + open, undoOpen int + reason, undoReason string + wfState contract.WorkflowState + failed, cancelled, undoCancel bool } func (w *wfFlow) snapshot() map[string]mut { @@ -771,7 +768,7 @@ func (w *wfFlow) snapshot() map[string]mut { for name, t := range w.tasks { m[name] = mut{state: t.state, undoState: t.undoState, open: t.open, undoOpen: t.undoOpen, reason: t.reason, undoReason: t.undoReason} } - m[""] = mut{wfState: w.state, failed: w.failed, cancelled: w.cancelled, undoCance: w.undoOnCancel} + m[""] = mut{wfState: w.state, failed: w.failed, cancelled: w.cancelled, undoCancel: w.undoOnCancel} return m } @@ -784,7 +781,7 @@ func (w *wfFlow) dirty(orig map[string]mut) (tasks []string, flow bool) { tasks = append(tasks, name) } } - return tasks, orig[""] != mut{wfState: w.state, failed: w.failed, cancelled: w.cancelled, undoCance: w.undoOnCancel} + return tasks, orig[""] != mut{wfState: w.state, failed: w.failed, cancelled: w.cancelled, undoCancel: w.undoOnCancel} } // save writes what changed since orig; result is the finishing task's From 16416c3cba9f549dfb59e17d4bf099bdc6ebf02c Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Mon, 9 Nov 2026 22:06:19 +0600 Subject: [PATCH 4/8] fix: workflow store review fixes Outbox row for unreachable push targets in both layouts, queue lookup across clients of a scope, purge without a registered store (one transaction in a single file, stored intents in folders), job ID guard for wf: IDs, json-style key folding, scope checks, deps check, expired resubmit, sweep throttle. --- CHANGELOG.md | 12 +- README.md | 13 +- cancel.go | 5 +- example/workflow_example.go | 2 +- layout_test.go | 21 ++-- queue.go | 142 +++++++++++++++++++--- queue_test.go | 233 ++++++++++++++++++++++++++++++------ queues.go | 3 + registry.go | 3 + workflow.go | 231 +++++++++++++++++++++++++++++------ workflow_finish.go | 154 ++++++++++++++++++++---- 11 files changed, 688 insertions(+), 131 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bb62a9..a53196e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,13 +18,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Workflows (VarMQ spec 6.9 to 6.12): `NewWorkflowStore(qs, ...WorkflowOption)` returns the `contract.WorkflowStore` of a store root. (#NN) - The store registers with `contract.RegisterWorkflowStore`; `Close` unregisters it and releases its file. (#NN) - `WithWorkflowResultLimit` (default 64 KiB, `ErrResultTooLarge`) and `WithWorkflowRetention` (default 7 days). (#NN) -- `ScopeID()` on `Queue` and `PriorityQueue`: `sqliteq:` and the absolute store folder, or the file under `WithSingleFile`. (#NN) +- `ScopeID()` on `Queue` and `PriorityQueue`: `sqliteq:` and the absolute store folder (symlinks resolved), or the file under `WithSingleFile`. (#NN) +- A file open under one scope is refused under another; `WithSingleFile` refuses the name `__sqliteq_workflows.db`. (#NN) +- `FinishTask` finds the queue for any client of the scope in this process. (#NN) +- `WithSingleFile`: a push into a queue no client has open becomes an outbox row in the same transaction. (#NN) +- A plain job cannot take a job ID that starts with `wf:`; a held ID keeps the push in the outbox. (#NN) +- `Purge` ends purged workflow tasks without a registered store; one transaction under `WithSingleFile`, a stored intent in the folder layout. (#NN) +- `Submit` refuses a dependency that names no task, and replaces an expired, unswept workflow of the same ID. (#NN) - Folder layout: the store keeps its tables in `__sqliteq_workflows.db`. `WithSingleFile`: tables in the same file. (#NN) - Tables `__sqliteq_wf_workflows`, `__sqliteq_wf_tasks`, `__sqliteq_wf_outbox`. (#NN) - `FinishTask` in three steps in the folder layout: record the end and write the outbox, push each row with `EnqueueOnce`, acknowledge the job. (#NN) -- A recovery loop pushes leftover outbox rows at open and every second. (#NN) +- A recovery loop pushes leftover outbox rows at open and every second, in both layouts. (#NN) - `WithSingleFile`: the end, the pushes and the ack are one transaction. (#NN) -- A duplicate push or a redelivered job is safe: the pre-run check skips a task that already ended. (#NN) +- A redelivered job, or a push repeated after the job was removed, is safe: the pre-run check skips a task that already ended. (#NN) - `Workflows` (`contract.WorkflowInspector`) and `WorkflowKeys` (`contract.WorkflowKeyLister`) list workflows newest first. (#NN) - A cancel mark never drops a valid workflow item; the workflow decides its end. (#NN) - Task JSON whose keys differ only in case (`WF` next to `wf`) is a plain job. (#NN) diff --git a/README.md b/README.md index cb36cd8..10e0520 100644 --- a/README.md +++ b/README.md @@ -259,7 +259,7 @@ Both queue types implement VarMQ's `QueuePauser` and `PauseNotifier`. So VarMQ's A workflow is a graph of VarMQ tasks. A task runs after the tasks it depends on. sqliteq keeps the workflow state in SQLite, so a workflow survives a restart and spans several queues of one store. -Create the store once, after the queues, and keep it open while workers run. `NewWorkflowStore` registers the store for the scope of its queues. VarMQ finds it from there. A process that only runs workers needs this one call too. +Create the store once and keep it open while workers run. The order of the store and the queues does not matter. `NewWorkflowStore` registers the store for the scope of its queues. VarMQ finds it from there. A process that only runs workers needs this one call too. ```go qs, _ := sqliteq.New("./data") @@ -290,7 +290,7 @@ Options of `NewWorkflowStore`: - `WithWorkflowResultLimit(n)` caps a task result at `n` bytes. The default is 64 KiB. A bigger result fails the task with `ErrResultTooLarge`. - `WithWorkflowRetention(d)` sets how long an ended workflow is kept. The default is 7 days. A running workflow is kept until it ends. -Every queue of a workflow must have the same scope. The scope of a queue is `ScopeID()`: `sqliteq:` and the absolute path of the store folder, or of the file under `WithSingleFile`. One process holds one store for each scope. A second `NewWorkflowStore` for the same scope returns `contract.ErrScopeRegistered`. +Every queue of a workflow must have the same scope. The scope of a queue is `ScopeID()`: `sqliteq:` and the absolute path of the store folder, or of the file under `WithSingleFile`, with symlinks resolved. A file already open under one scope cannot be opened under another, for example as a single file and as a queue file of a folder. `NewQueue` returns an error then. `WithSingleFile` refuses the file name `__sqliteq_workflows.db`. One process holds one store for each scope. A second `NewWorkflowStore` for the same scope returns `contract.ErrScopeRegistered`. The store implements `contract.WorkflowStore`, `WorkflowInspector` and `WorkflowKeyLister`. So VarMQ's REST API lists and cancels sqliteq workflows. @@ -302,16 +302,17 @@ The store implements `contract.WorkflowStore`, `WorkflowInspector` and `Workflow 2. Each outbox row is pushed into its queue with `EnqueueOnce` and the job ID `wf::`. Then the row is deleted. 3. The job is acknowledged in its queue. -A recovery loop pushes leftover outbox rows when the store opens and every second while it is open. A crash between the steps is safe. A row pushed twice stores one job. A job that was redelivered finds its task already ended: the pre-run check (`Runnable`) skips it. Under `WithSingleFile` all three steps are one transaction. +A recovery loop pushes leftover outbox rows when the store opens and every second while it is open. A crash between the steps is safe. A row pushed twice stores one job while the first job is still in the queue. Once a worker has finished and removed that job, a second push would queue it again. The pre-run check (`Runnable`) then skips it, because its task already ended. A job that was redelivered is skipped the same way. Under `WithSingleFile` the end, the pushes and the ack are one transaction. A push into a queue that no client in the process has open becomes an outbox row in that transaction, and the loop pushes it later. **Limits.** - The recovery loop cannot wake the workers of a process. A task pushed by the loop starts when a worker next polls its queue. -- The recovery loop pushes only into queues that this `Queues` created, or that a submitted plan named. -- A crash inside `Purge` of a queue with workflow tasks, after the delete and before the workflow update, leaves those tasks pending with no job. Cancel the workflow to end it. +- The loop pushes only into queues that some client in this process has open. Rows for other queues wait, and the store logs each such queue once. +- A job ID that starts with `wf:` is for workflow items. A plain job cannot take one. If an older row already holds the ID, the push stays in the outbox and the store logs it once. - The stored items of waiting tasks and the undo items stay in the workflow file until the task is pushed or the workflow expires. +- Folder layout: `Purge` stores an intent in the workflow file before it deletes the rows. If the process crashes between them, the recovery loop ends the purged tasks later. -A valid workflow item is never dropped by a cancel mark: the workflow decides how its task ends. A `Purge` of a queue ends each purged workflow task as cancelled with the reason `queue purged`. The workflow goes on. An item whose JSON has keys that differ only in case, such as `WF` next to `wf`, is a plain job. +A valid workflow item is never dropped by a cancel mark: the workflow decides how its task ends. A `Purge` of a queue ends each purged workflow task as cancelled with the reason `queue purged`. The workflow goes on. This works also when no store is open in the process, as long as the workflow tables exist. Under `WithSingleFile` the delete and the ends are one transaction. An item whose JSON has keys that differ only in case, such as `WF` next to `wf`, is a plain job. ### Storage diff --git a/cancel.go b/cancel.go index 46edfa8..36570bb 100644 --- a/cancel.go +++ b/cancel.go @@ -14,7 +14,7 @@ import ( // Every pop drops a marked waiting job with outcome cancelled; OnCancel // learns of marks by polling seq; AcknowledgeCancelled finishes a running // job that stopped for it. Any finish, and an EnqueueOnce that stores the -// id, clears the mark. A valid workflow item (wfTag) is never dropped: the +// id, clears the mark. A valid workflow item (workflowTag) whose job ID is its tag's is never dropped: the // workflow decides its end, and its mark stays. var ( @@ -198,5 +198,6 @@ func markedDrop(isMarked *sql.Stmt, id string, data []byte, now time.Time) (bool if err != nil || !drop { return false, err } - return !isWorkflowItem(data), nil + tag, ok := workflowTag(data) + return !(ok && id == tag.jobID()), nil } diff --git a/example/workflow_example.go b/example/workflow_example.go index a1ee91a..2a15f6a 100644 --- a/example/workflow_example.go +++ b/example/workflow_example.go @@ -35,7 +35,7 @@ func workflowExample() { log.Fatalf("Failed to create queue: %v", err) } - // Create the store after the queues; it registers itself for their scope + // The store registers itself for the scope of the queues store, err := sqliteq.NewWorkflowStore(qs) if err != nil { log.Fatalf("Failed to create workflow store: %v", err) diff --git a/layout_test.go b/layout_test.go index 3b72bd9..6cad5cb 100644 --- a/layout_test.go +++ b/layout_test.go @@ -415,12 +415,12 @@ func TestSingleFileAndFolderOneEntry(t *testing.T) { if err != nil { t.Fatalf("Failed to create queue: %v", err) } - b, err := single.NewQueue("JOBS") - if err != nil { - t.Fatalf("Failed to create queue: %v", err) + // One file under two roots would have two scopes: refused + if _, err := single.NewQueue("JOBS"); err == nil || a == nil { + t.Errorf("Expected an error for a file open under another scope, got %v", err) } - if a != b { - t.Error("Expected the folder and single-file stores to share the handle") + if _, err := New(filepath.Join(dir, workflowsFile), WithSingleFile()); err == nil { + t.Error("Expected the workflow file name refused for WithSingleFile") } } @@ -655,7 +655,7 @@ func TestSymlinkedFileOneEntry(t *testing.T) { for _, store := range []struct { path string opts []StoreOption - }{{real, []StoreOption{WithSingleFile()}}, {link, []StoreOption{WithSingleFile()}}, {folder, nil}} { + }{{real, []StoreOption{WithSingleFile()}}, {link, []StoreOption{WithSingleFile()}}} { qs := mustNew(t, store.path, store.opts...) defer qs.Close() q, err := qs.NewQueue("jobs") @@ -670,6 +670,13 @@ func TestSymlinkedFileOneEntry(t *testing.T) { t.Errorf("Path %d reached a different file entry", i+1) } } + + // The same file under a folder root would have another scope: refused + qs := mustNew(t, folder) + defer qs.Close() + if _, err := qs.NewQueue("jobs"); err == nil { + t.Error("Expected a folder symlinking to an open single file refused") + } } func TestReopenAfterCloseNoRecovery(t *testing.T) { @@ -744,7 +751,7 @@ func TestScopeThroughSymlinks(t *testing.T) { if got, ok := contract.LookupWorkflowStore(a.ScopeID()); !ok || got != contract.WorkflowStore(st) { t.Error("Expected the queue's scope to find the store") } - plan := wfPlan("w1", planTask("w1", "x", a), planTask("w1", "y", b, "x")) + plan := planWF("w1", planTask("w1", "x", a), planTask("w1", "y", b, "x")) if err := st.Submit(context.Background(), plan); err != nil { t.Fatalf("Expected Submit across the symlinked queues to succeed, got %v", err) } diff --git a/queue.go b/queue.go index bb20e39..0c7d582 100644 --- a/queue.go +++ b/queue.go @@ -2,10 +2,12 @@ package sqliteq import ( "cmp" + "context" "database/sql" "database/sql/driver" "errors" "fmt" + "log" "slices" "strconv" "strings" @@ -339,6 +341,27 @@ func (q *Queue) ScopeID() string { return q.f.scope } +// finishDirect is finish in a transaction of its own, outside the group +// commit: found is false when no processing row had the ack ID, which is no +// error (a re-finish). +func (q *Queue) finishDirect(ackID string, state contract.JobState, reason string, result []byte) (found bool, err error) { + id, ok := parseAckID(ackID) + if !ok { + return false, nil + } + q.f.mu.Lock() + defer q.f.mu.Unlock() + tx, err := q.f.db.Begin() + if err != nil { + return false, err + } + defer tx.Rollback() + if found, err = q.finishTx(tx, id, state, reason, result, time.Now()); err != nil { + return false, err + } + return found, tx.Commit() +} + // Enqueue adds an item to the queue // Returns true if the operation was successful func (q *Queue) Enqueue(item any) bool { @@ -368,6 +391,20 @@ func (q *Queue) enqueue(item any, priority int, id string) bool { return false } + // A wf: ID belongs to workflow items; a plain job must not hold one + if strings.HasPrefix(id, "wf:") { + var b []byte + switch v := data.(type) { + case []byte: + b = v + case string: + b = []byte(v) + } + if tag, ok := workflowTag(b); !ok || tag.jobID() != id { + return false + } + } + // No id is NULL, which the job_id index leaves out; "" would be one id var jobID any if id != "" { @@ -392,17 +429,56 @@ func (q *Queue) enqueue(item any, priority int, id string) bool { }) } +// errJobIDTaken: the job ID belongs to a row that is not the same workflow task. +var errJobIDTaken = errors.New("sqliteq: job id is held by another job") + // enqueueTx is enqueue in the caller's tx, which holds the file's write -// mutex: one pending row under id, none when a row with that id exists. +// mutex: one pending row under id. A repeat is fine only when the stored row +// is the same workflow item (same wf, task and undo); else errJobIDTaken. func (q *Queue) enqueueTx(tx *sql.Tx, item []byte, priority int, id string) error { now := time.Now().UTC() insert := tx.Stmt(q.insert) + var res sql.Result + var err error if q.priority { - _, err := insert.Exec(item, now, now, id, priority) + res, err = insert.Exec(item, now, now, id, priority) + } else { + res, err = insert.Exec(item, now, now, id) + } + if err != nil { return err } - _, err := insert.Exec(item, now, now, id) - return err + if n, err := res.RowsAffected(); err != nil || n > 0 { + return err + } + var data []byte + if err := tx.QueryRow(fmt.Sprintf("SELECT data FROM %s WHERE job_id = ?", q.table), id).Scan(&data); err != nil { + return err + } + want, ok := workflowTag(item) + if have, hok := workflowTag(data); !ok || !hok || have != want { + return errJobIDTaken + } + return nil +} + +// pushWorkflow adds a workflow item under its job ID in a transaction of its +// own (see enqueueTx). +func (q *Queue) pushWorkflow(item []byte, priority int, id string) error { + if q.closed.Load() { + return errClosed + } + q.f.mu.Lock() + defer q.f.mu.Unlock() + tx, err := q.f.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if err := q.enqueueTx(tx, item, priority, id); err != nil { + return err + } + return tx.Commit() } // claimN moves up to n pending rows to processing in one statement and @@ -787,19 +863,47 @@ func (q *Queue) Values() []any { // marks and totals; a pause stays (spec 6.6, 6.7, 6.14). A purged workflow // task or undo ends cancelled (reason "queue purged"), so its workflow goes // on: dependents follow their policy, and what that releases is pushed after -// the delete. Without a workflow store for the queue's scope, no workflow -// ends. +// the delete. The workflow store is the registered one, or one opened on +// demand when the file has workflow tables. Under WithSingleFile the delete +// and the ends are one transaction. Folder layout: an intent is stored in +// the workflow file first, and the recovery loop applies one a crash left. func (q *Queue) Purge() { - store, _ := contract.LookupWorkflowStore(q.f.scope) - ws, _ := store.(*WorkflowStore) - if tags := q.purge(ws != nil); ws != nil { - ws.endPurged(q, tags) + ws, done := q.f.purgeStore() + defer done() + if ws == nil { + q.purge(nil) + return } + ctx := context.Background() + pending := q.pendingTags() + if ws.single { + for _, tag := range pending { + ws.preResolve(ctx, tag.wf) + } + for _, tag := range q.purge(ws) { + ws.drain(ctx, `workflow_id = ?`, tag.wf) + } + return + } + intents := ws.recordPurge(ctx, q, pending) + ws.endPurged(q, intents, q.purge(ws)) +} + +// pendingTags reads the workflow tags of the queued and running items. +func (q *Queue) pendingTags() []wfItemTag { + tx, err := q.f.db.Begin() + if err != nil { + return nil + } + defer tx.Rollback() + tags, _ := q.workflowTags(tx) + return tags } -// purge deletes the rows under the write mutex; with workflows it first -// reads the tags of the pending and processing workflow items. -func (q *Queue) purge(workflows bool) (tags []wfItemTag) { +// purge deletes the rows under the write mutex and returns the workflow tags +// it found in them (read in the same transaction); with a single-file store +// it also ends those tasks in the transaction, and returns them. +func (q *Queue) purge(ws *WorkflowStore) (tags []wfItemTag) { q.f.mu.Lock() defer q.f.mu.Unlock() @@ -808,7 +912,7 @@ func (q *Queue) purge(workflows bool) (tags []wfItemTag) { return nil } defer tx.Rollback() - if workflows { + if ws != nil { if tags, err = q.workflowTags(tx); err != nil { return nil } @@ -821,6 +925,16 @@ func (q *Queue) purge(workflows bool) (tags []wfItemTag) { if _, err := tx.Exec("UPDATE " + q.meta + " SET value = 0 WHERE key <> 'paused_at'"); err != nil { return nil } + if ws != nil && ws.single { + ctx := context.Background() + for _, tag := range tags { + e := contract.TaskEnd{Workflow: tag.wf, Task: tag.task, Undo: tag.undo, Outcome: contract.OutcomeCancelled, Reason: purgeReason} + if _, err := ws.endTx(ctx, tx, q.id, e); err != nil { + log.Printf("sqliteq: purge of %s: ending workflow %s task %s: %v", q.name, tag.wf, tag.task, err) + return nil + } + } + } if tx.Commit() != nil { return nil } diff --git a/queue_test.go b/queue_test.go index 4e1b170..18cd790 100644 --- a/queue_test.go +++ b/queue_test.go @@ -2352,7 +2352,7 @@ func withUndo(tp contract.TaskPlan, wf string, q any) contract.TaskPlan { return tp } -func wfPlan(id string, tasks ...contract.TaskPlan) contract.WorkflowPlan { +func planWF(id string, tasks ...contract.TaskPlan) contract.WorkflowPlan { return contract.WorkflowPlan{ID: id, Hash: id + "-hash", Tasks: tasks} } @@ -2475,7 +2475,7 @@ func TestWorkflowHappyPath(t *testing.T) { eachLayout(t, func(t *testing.T, single bool) { e := newWFEnv(t, single) ctx := context.Background() - plan := wfPlan("w1", planTask("w1", "t1", e.a), planTask("w1", "t2", e.p, "t1"), planTask("w1", "t3", e.b, "t2")) + plan := planWF("w1", planTask("w1", "t1", e.a), planTask("w1", "t2", e.p, "t1"), planTask("w1", "t3", e.b, "t2")) plan.Tasks[1].Priority = 5 if err := e.st.Submit(ctx, plan); err != nil { t.Fatal(err) @@ -2564,13 +2564,13 @@ func TestWorkflowSubmitChecks(t *testing.T) { bad := map[string]contract.WorkflowPlan{ "empty": {ID: "x", Hash: "h"}, - "bad id": wfPlan("a b", planTask("a b", "t", e.a)), - "index": wfPlan("index", planTask("index", "t", e.a)), - "bad task": wfPlan("x", planTask("x", "t t", e.a)), - "other scope": wfPlan("x", planTask("x", "t", oq)), - "not a queue": wfPlan("x", planTask("x", "t", "q")), - "bad item": wfPlan("x", func() contract.TaskPlan { tp := planTask("x", "t", e.a); tp.Item = []byte("[1]"); return tp }()), - "dup name": wfPlan("x", planTask("x", "t", e.a), planTask("x", "t", e.a)), + "bad id": planWF("a b", planTask("a b", "t", e.a)), + "index": planWF("index", planTask("index", "t", e.a)), + "bad task": planWF("x", planTask("x", "t t", e.a)), + "other scope": planWF("x", planTask("x", "t", oq)), + "not a queue": planWF("x", planTask("x", "t", "q")), + "bad item": planWF("x", func() contract.TaskPlan { tp := planTask("x", "t", e.a); tp.Item = []byte("[1]"); return tp }()), + "dup name": planWF("x", planTask("x", "t", e.a), planTask("x", "t", e.a)), } for name, plan := range bad { if err := e.st.Submit(ctx, plan); err == nil { @@ -2594,7 +2594,7 @@ func TestWorkflowFailureAndCompensation(t *testing.T) { eachLayout(t, func(t *testing.T, single bool) { e := newWFEnv(t, single) ctx := context.Background() - plan := wfPlan("w2", + plan := planWF("w2", withUndo(planTask("w2", "a", e.a), "w2", e.b), withUndo(planTask("w2", "b", e.a, "a"), "w2", e.b), planTask("w2", "c", e.a, "b")) @@ -2637,7 +2637,7 @@ func TestWorkflowFailureAndCompensation(t *testing.T) { } // FailFast: the first failure cancels what waits - ff := wfPlan("w3", planTask("w3", "a", e.a), planTask("w3", "b", e.a), planTask("w3", "c", e.a, "a")) + ff := planWF("w3", planTask("w3", "a", e.a), planTask("w3", "b", e.a), planTask("w3", "c", e.a, "a")) ff.FailFast = true if err := e.st.Submit(ctx, ff); err != nil { t.Fatal(err) @@ -2660,7 +2660,7 @@ func TestWorkflowCancel(t *testing.T) { if err := e.st.Cancel(ctx, "missing", nil); !errors.Is(err, contract.ErrWorkflowNotFound) { t.Errorf("Expected ErrWorkflowNotFound, got %v", err) } - plan := wfPlan("w1", withUndo(planTask("w1", "a", e.a), "w1", e.b), planTask("w1", "b", e.a, "a")) + plan := planWF("w1", withUndo(planTask("w1", "a", e.a), "w1", e.b), planTask("w1", "b", e.a, "a")) if err := e.st.Submit(ctx, plan); err != nil { t.Fatal(err) } @@ -2694,7 +2694,7 @@ func TestWorkflowCancel(t *testing.T) { func TestWorkflowResultLimit(t *testing.T) { e := newWFEnv(t, false, WithWorkflowResultLimit(8)) ctx := context.Background() - if err := e.st.Submit(ctx, wfPlan("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.a, "a"))); err != nil { + if err := e.st.Submit(ctx, planWF("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.a, "a"))); err != nil { t.Fatal(err) } _, ack := take(t, e.a) @@ -2710,7 +2710,7 @@ func TestWorkflowResultLimit(t *testing.T) { func TestWorkflowCrashAfterRecord(t *testing.T) { e := newWFEnv(t, false) ctx := context.Background() - if err := e.st.Submit(ctx, wfPlan("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.b, "a"))); err != nil { + if err := e.st.Submit(ctx, planWF("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.b, "a"))); err != nil { t.Fatal(err) } _, _ = take(t, e.a) @@ -2745,7 +2745,7 @@ func TestWorkflowCrashAfterRecord(t *testing.T) { func TestWorkflowRecoverOnOpen(t *testing.T) { e := newWFEnv(t, false) ctx := context.Background() - if err := e.st.Submit(ctx, wfPlan("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.b, "a"))); err != nil { + if err := e.st.Submit(ctx, planWF("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.b, "a"))); err != nil { t.Fatal(err) } _, _ = take(t, e.a) @@ -2774,7 +2774,7 @@ func TestWorkflowRecoveryLoop(t *testing.T) { defer func() { recoverEvery = old }() e := newWFEnv(t, false) ctx := context.Background() - if err := e.st.Submit(ctx, wfPlan("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.b, "a"))); err != nil { + if err := e.st.Submit(ctx, planWF("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.b, "a"))); err != nil { t.Fatal(err) } _, _ = take(t, e.a) @@ -2797,7 +2797,7 @@ func TestWorkflowRecoveryLoop(t *testing.T) { func TestWorkflowDuplicatePush(t *testing.T) { e := newWFEnv(t, false) ctx := context.Background() - if err := e.st.Submit(ctx, wfPlan("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.b, "a"))); err != nil { + if err := e.st.Submit(ctx, planWF("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.b, "a"))); err != nil { t.Fatal(err) } _, ack := take(t, e.a) @@ -2819,7 +2819,7 @@ func TestWorkflowDuplicatePush(t *testing.T) { } rows.Close() e.st.drain(ctx, where, args...) // step b done, the crash comes before the ack - if !pushRow(&handle{q: e.b}, row) || e.b.Len() != 1 { + if pushRow(&handle{q: e.b}, row) != nil || e.b.Len() != 1 { t.Fatalf("Expected the duplicate push to store nothing, got %d rows", e.b.Len()) } // Redelivery of a: skipped, then finished; b is still queued once @@ -2831,32 +2831,186 @@ func TestWorkflowDuplicatePush(t *testing.T) { } } -// Single file: the end, the pushes and the ack are one transaction, so a -// failed push (an unopened queue) leaves everything as it was. +// Single file: the end and the ack are one transaction. A push target this +// client cannot reach gets an outbox row in that transaction; the recovery +// loop pushes it once a client opens the queue. func TestWorkflowSingleFileAtomic(t *testing.T) { e := newWFEnv(t, true) ctx := context.Background() - if err := e.st.Submit(ctx, wfPlan("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.b, "a"))); err != nil { + if err := e.st.Submit(ctx, planWF("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.b, "a"))); err != nil { t.Fatal(err) } if n := e.outbox(t); n != 0 { - t.Fatalf("Expected no outbox rows in one file, got %d", n) + t.Fatalf("Expected no outbox rows while every queue is reachable, got %d", n) } _, ack := take(t, e.a) - qs := e.qs.(*queues) - qs.mu.Lock() - unknown := maps.Clone(*qs.byID.Load()) - delete(unknown, e.b.QueueID()) // b is not open in this client - qs.byID.Store(&unknown) - qs.mu.Unlock() - if _, ok := e.st.FinishTask(e.a.QueueID(), ack, done("w1", "a", "")); ok { - t.Fatal("Expected FinishTask to fail when a push target is unknown") + hide := func(id string) { + qs := e.qs.(*queues) + qs.mu.Lock() + m := maps.Clone(*qs.byID.Load()) + delete(m, id) + qs.byID.Store(&m) + qs.mu.Unlock() + } + e.b.Close() // closed: the registry still lists it, the table does not + hide(e.b.QueueID()) + registry.mu.Lock() + delete(e.a.f.handles, "b") + registry.mu.Unlock() + pushed, ok := e.st.FinishTask(e.a.QueueID(), ack, done("w1", "a", "")) + if !ok || len(pushed) != 0 { + t.Fatalf("Expected ok with no push, got %v %t", pushed, ok) + } + if got := taskStates(e.state(t, "w1")); got != "a=completed b=pending" || e.outbox(t) != 1 { + t.Errorf("Expected the end recorded and one outbox row, got %s, %d rows", got, e.outbox(t)) + } + if n := stats(t, e.a).Counts.Processing; n != 0 { + t.Errorf("Expected the job acknowledged, got %d processing", n) + } + // Opened later (here by a second client): the loop pushes the row + other := mustNew(t, e.qs.(*queues).root, WithSingleFile()) + defer other.Close() + b2, err := other.NewQueue("b") + if err != nil { + t.Fatal(err) + } + e.st.recoverOutbox(0) + if e.outbox(t) != 0 || b2.Len() != 1 { + t.Errorf("Expected the row pushed, got %d rows, b %d", e.outbox(t), b2.Len()) + } +} + +// FinishTask finds a queue opened by any client of the scope in this process. +func TestWorkflowQueueOfAnotherClient(t *testing.T) { + eachLayout(t, func(t *testing.T, single bool) { + e := newWFEnv(t, single) + var other Queues + if single { + other = mustNew(t, e.qs.(*queues).root, WithSingleFile()) + } else { + other = mustNew(t, e.dir) + } + defer other.Close() + c, err := other.NewQueue("c") // known to the second client only + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + if err := e.st.Submit(ctx, planWF("w1", planTask("w1", "a", e.a), planTask("w1", "c", c, "a"))); err != nil { + t.Fatal(err) + } + _, ack := take(t, e.a) + // Forget what Submit learned: the lookup goes through the registry + qs := e.qs.(*queues) + qs.mu.Lock() + qs.byID.Store(&map[string]*handle{}) + qs.mu.Unlock() + if _, ok := e.st.FinishTask(e.a.QueueID(), ack, done("w1", "a", "")); !ok { + t.Fatal("FinishTask failed") + } + if c.Len() != 1 { + t.Errorf("Expected the task pushed into the other client's queue, got %d", c.Len()) + } + }) +} + +// A plain job cannot hold a wf: job ID, and a row that already holds one is +// not taken for the workflow's push. +func TestWorkflowJobIDHeldByPlainJob(t *testing.T) { + eachLayout(t, func(t *testing.T, single bool) { + e := newWFEnv(t, single) + if e.a.EnqueueOnce(job("wf:w1:a"), "wf:w1:a") || e.a.EnqueueOnce(wfItem("w1", "other", false), "wf:w1:a") { + t.Error("Expected a plain job refused a wf: job ID") + } + // A row planted by an older version, before the guard + if _, err := e.a.f.db.Exec(`INSERT INTO "a" (data, status, ack, created_at, updated_at, job_id) VALUES (?, 'pending', 0, 0, 0, 'wf:w1:a')`, job("wf:w1:a")); err != nil { + t.Fatal(err) + } + if err := e.st.Submit(context.Background(), planWF("w1", planTask("w1", "a", e.a))); err != nil { + t.Fatal(err) + } + if e.outbox(t) != 1 || e.a.Len() != 1 { + t.Errorf("Expected the push kept in the outbox, got %d rows, a %d", e.outbox(t), e.a.Len()) + } + e.st.recoverOutbox(0) + if e.outbox(t) != 1 { + t.Error("Expected the row to stay while the job ID is held") + } + }) +} + +// A Deps entry that names no task is refused; an ended, expired, unswept +// workflow does not block a Submit of its ID. +func TestWorkflowSubmitDepsAndExpired(t *testing.T) { + e := newWFEnv(t, false, WithWorkflowRetention(time.Hour)) + ctx := context.Background() + if err := e.st.Submit(ctx, planWF("w1", planTask("w1", "a", e.a, "ghost"))); err == nil { + t.Error("Expected a dependency on no task refused") + } + var clock atomic.Int64 + clock.Store(time.Now().UnixMilli()) + e.st.setClock(func() time.Time { return time.UnixMilli(clock.Load()) }) + if err := e.st.Submit(ctx, planWF("w2", planTask("w2", "a", e.a))); err != nil { + t.Fatal(err) + } + _, ack := take(t, e.a) + e.finish(t, e.a, ack, done("w2", "a", "")) + clock.Add(2 * time.Hour.Milliseconds()) + again := planWF("w2", planTask("w2", "a", e.a)) + again.Hash = "new" + if err := e.st.Submit(ctx, again); err != nil { + t.Fatalf("Expected a new workflow under the expired ID, got %v", err) + } + if got := taskStates(e.state(t, "w2")); got != "a=pending" { + t.Errorf("got %s", got) + } +} + +// Purge with no store registered still ends the purged workflow tasks, from +// the workflow tables on disk; under WithSingleFile in one transaction. +func TestWorkflowPurgeWithoutStore(t *testing.T) { + eachLayout(t, func(t *testing.T, single bool) { + e := newWFEnv(t, single) + ctx := context.Background() + plan := planWF("w1", planTask("w1", "a", e.a), planTask("w1", "z", e.b, "a")) + plan.Tasks[1].Always = true + if err := e.st.Submit(ctx, plan); err != nil { + t.Fatal(err) + } + e.st.Close() // unregisters; the tables stay + e.a.Purge() + st, err := NewWorkflowStore(e.qs) + if err != nil { + t.Fatal(err) + } + defer st.Close() + r, err := st.Record(ctx, "w1") + if err != nil { + t.Fatal(err) + } + if got := taskStates(r); got != "a=cancelled z=pending" || e.b.Len() != 1 { + t.Errorf("got %s, b %d", got, e.b.Len()) + } + }) +} + +// Folder layout: a purge intent a crash left is applied by the recovery. +func TestWorkflowPurgeIntentRecovery(t *testing.T) { + e := newWFEnv(t, false) + ctx := context.Background() + if err := e.st.Submit(ctx, planWF("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.b, "a"))); err != nil { + t.Fatal(err) } - if got := taskStates(e.state(t, "w1")); got != "a=pending b=waiting" { - t.Errorf("Expected the workflow untouched, got %s", got) + e.st.recordPurge(ctx, e.a, []wfItemTag{{wf: "w1", task: "a"}}) // the crash comes after this + e.a.purge(nil) + e.st.recoverOutbox(0) + if got := taskStates(e.state(t, "w1")); got != "a=cancelled b=cancelled" { + t.Errorf("got %s", got) } - if n := stats(t, e.a).Counts.Processing; n != 1 { - t.Errorf("Expected the job still processing, got %d processing", n) + var n int + e.st.f.db.QueryRow(`SELECT COUNT(*) FROM ` + wfPurged).Scan(&n) + if n != 0 { + t.Errorf("Expected the intent deleted, got %d", n) } } @@ -2869,7 +3023,7 @@ func TestWorkflowRetention(t *testing.T) { clock.Store(time.Now().UnixMilli()) e.st.setClock(func() time.Time { return time.UnixMilli(clock.Load()) }) for _, id := range []string{"ended", "running"} { - if err := e.st.Submit(ctx, wfPlan(id, planTask(id, "a", e.a))); err != nil { + if err := e.st.Submit(ctx, planWF(id, planTask(id, "a", e.a))); err != nil { t.Fatal(err) } } @@ -2899,7 +3053,7 @@ func TestWorkflowRetention(t *testing.T) { t.Errorf("Expected the tasks swept too, got %d", n) } // A finish of a swept workflow settles the job and returns ok - if err := e.st.Submit(ctx, wfPlan("gone", planTask("gone", "a", e.a))); err != nil { + if err := e.st.Submit(ctx, planWF("gone", planTask("gone", "a", e.a))); err != nil { t.Fatal(err) } _, ack = take(t, e.a) @@ -2917,7 +3071,7 @@ func TestWorkflowListing(t *testing.T) { clock.Store(1_000_000) e.st.setClock(func() time.Time { return time.UnixMilli(clock.Load()) }) for _, id := range []string{"b", "a", "c"} { - if err := e.st.Submit(ctx, wfPlan(id, planTask(id, "t", e.a))); err != nil { + if err := e.st.Submit(ctx, planWF(id, planTask(id, "t", e.a))); err != nil { t.Fatal(err) } if id == "a" { @@ -2980,6 +3134,7 @@ func TestWorkflowTag(t *testing.T) { {"index", `{"id":"wf:index:t","wf":"index","task":"t"}`, false}, {"upper WF", `{"id":"wf:w:t","wf":"w","WF":"x","task":"t"}`, false}, {"upper Task", `{"id":"wf:w:t","wf":"w","Task":"t"}`, false}, + {"dotted I", `{"\u0130d":"x","id":"wf:w:t","wf":"w","task":"t"}`, true}, {"Id", `{"Id":"wf:w:t","id":"wf:w:t","wf":"w","task":"t"}`, false}, {"long s", `{"id":"wf:w:t","wf":"w","taſk":"t","task":"t"}`, false}, {"kelvin", `{"id":"wf:w:t","wf":"w","tasK":"t","task":"t"}`, false}, @@ -2998,7 +3153,7 @@ func TestWorkflowItemSurvivesCancelMark(t *testing.T) { eachLayout(t, func(t *testing.T, single bool) { e := newWFEnv(t, single) ctx := context.Background() - if err := e.st.Submit(ctx, wfPlan("w1", planTask("w1", "a", e.a))); err != nil { + if err := e.st.Submit(ctx, planWF("w1", planTask("w1", "a", e.a))); err != nil { t.Fatal(err) } if err := e.a.CancelJob("wf:w1:a"); err != nil { @@ -3037,7 +3192,7 @@ func TestWorkflowPurge(t *testing.T) { eachLayout(t, func(t *testing.T, single bool) { e := newWFEnv(t, single) ctx := context.Background() - plan := wfPlan("w1", planTask("w1", "a", e.a), planTask("w1", "n", e.a, "a"), planTask("w1", "z", e.b, "a")) + plan := planWF("w1", planTask("w1", "a", e.a), planTask("w1", "n", e.a, "a"), planTask("w1", "z", e.b, "a")) plan.Tasks[2].Always = true if err := e.st.Submit(ctx, plan); err != nil { t.Fatal(err) diff --git a/queues.go b/queues.go index 398215f..92c4e62 100644 --- a/queues.go +++ b/queues.go @@ -65,6 +65,9 @@ func layout(path string, singleFile bool) (string, error) { if exists && info.IsDir() { return "", fmt.Errorf("sqliteq: %s is a folder; WithSingleFile needs a file path", path) } + if filepath.Base(path) == workflowsFile { + return "", fmt.Errorf("sqliteq: %s is the workflow store's file name; choose another", path) + } return resolve(path) } diff --git a/registry.go b/registry.go index cdfd603..9960d49 100644 --- a/registry.go +++ b/registry.go @@ -87,6 +87,9 @@ func acquire(path, root string, pages int) (*file, error) { defer registry.mu.Unlock() if f, ok := registry.files[path]; ok { + if f.scope != "sqliteq:"+root { + return nil, fmt.Errorf("sqliteq: %s is already open under scope %s, not sqliteq:%s", path, f.scope, root) + } if f.pages != pages { return nil, fmt.Errorf("sqliteq: %s is already open with checkpoint pages %d, not %d", path, f.pages, pages) } diff --git a/workflow.go b/workflow.go index af56602..c27d73c 100644 --- a/workflow.go +++ b/workflow.go @@ -48,11 +48,16 @@ const ( wfWorkflows = `"__sqliteq_wf_workflows"` wfTasks = `"__sqliteq_wf_tasks"` wfOutbox = `"__sqliteq_wf_outbox"` + wfPurged = `"__sqliteq_wf_purged"` // sweepWorkflowsEvery is how often the loop drops expired workflows. sweepWorkflowsEvery = time.Minute + // sweepOnSubmit is the least time between the sweeps Submit starts. + sweepOnSubmit = 30 * time.Second // drainBatch is how many outbox rows one read takes. drainBatch = 256 + // drainCap is the most rows one drain pushes: a backlog goes on next tick. + drainCap = 2048 // purgeReason is the reason of a task its queue's Purge ended. purgeReason = "queue purged" ) @@ -109,6 +114,8 @@ type WorkflowStore struct { stop chan struct{} done chan struct{} + warned sync.Map // keys logged once + sweptAt atomic.Int64 // Unix ms of the last sweep Submit started closeOnce sync.Once closeErr error } @@ -218,6 +225,14 @@ func (s *WorkflowStore) init() error { priority INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL )`, + `CREATE TABLE IF NOT EXISTS ` + wfPurged + ` ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + queue_id TEXT NOT NULL, + workflow_id TEXT NOT NULL, + task TEXT NOT NULL, + undo INTEGER NOT NULL, + created_at INTEGER NOT NULL + )`, `CREATE INDEX IF NOT EXISTS "__sqliteq_wf_outbox_workflow" ON ` + wfOutbox + ` (workflow_id)`, } { if _, err := s.f.db.Exec(stmt); err != nil { @@ -292,12 +307,6 @@ func validWFName(s string, n int) bool { func validWorkflowID(id string) bool { return validWFName(id, 128) && id != "index" } func validTaskName(n string) bool { return validWFName(n, 64) } -// wfFold lower-cases s the way encoding/json matches keys: ASCII case, and -// U+017F and U+212A fold to s and k. -func wfFold(s string) string { - return strings.ToLower(strings.NewReplacer("ſ", "s", "K", "k").Replace(s)) -} - // wfItemTag is the workflow fields of an item. type wfItemTag struct { wf, task string @@ -318,10 +327,10 @@ func workflowTag(data []byte) (wfItemTag, bool) { if json.Unmarshal(data, &env) != nil { return t, false } + // encoding/json would read such a key as that field for k := range env { - switch f := wfFold(k); f { - case "id", "wf", "task", "undo": - if k != f { + for _, name := range [...]string{"id", "wf", "task", "undo"} { + if k != name && strings.EqualFold(k, name) { return t, false } } @@ -346,9 +355,13 @@ func workflowTag(data []byte) (wfItemTag, bool) { return t, id == want } -func isWorkflowItem(data []byte) bool { - _, ok := workflowTag(data) - return ok +// jobID is the job ID a Submit gives the task or undo. +func (t wfItemTag) jobID() string { + id := "wf:" + t.wf + ":" + t.task + if t.undo { + id += ":undo" + } + return id } // querier is what a *sql.DB and a *sql.Tx share. @@ -407,6 +420,78 @@ func (qs *queues) learn(h *handle) { } } +// lookup finds queue id for any client of the scope in this process: this +// client's table first, then the file registry; a hit is learned. Call it +// outside the file's write mutex: the registry lock comes before it. +func (s *WorkflowStore) lookup(id string) (*handle, bool) { + if h, ok := s.qs.handleOf(id); ok { + return h, true + } + var found *handle + registry.mu.Lock() + for _, f := range registry.files { + if f.scope != s.scope { + continue + } + for _, h := range f.handles { + if h.queue().id == id { + found = h + } + } + } + registry.mu.Unlock() + if found == nil { + return nil, false + } + s.qs.learn(found) + return found, true +} + +// openIDs learns every queue of the scope open in this process and returns +// their IDs. +func (s *WorkflowStore) openIDs() []string { + var hs []*handle + registry.mu.Lock() + for _, f := range registry.files { + if f.scope == s.scope { + for _, h := range f.handles { + hs = append(hs, h) + } + } + } + registry.mu.Unlock() + for _, h := range hs { + s.qs.learn(h) + } + m := *s.qs.byID.Load() + ids := make([]string, 0, len(m)) + for id := range m { + ids = append(ids, id) + } + return ids +} + +// preResolve looks up the queues of workflow id before a write, so the +// transaction (which holds the file mutex) only reads the lock-free table. +func (s *WorkflowStore) preResolve(ctx context.Context, id string) { + rows, err := s.f.db.QueryContext(ctx, `SELECT queue_id FROM `+wfTasks+` WHERE workflow_id = ? + UNION SELECT undo_queue_id FROM `+wfTasks+` WHERE workflow_id = ? AND has_undo = 1`, id, id) + if err != nil { + return + } + var ids []string + for rows.Next() { + var q string + if rows.Scan(&q) == nil { + ids = append(ids, q) + } + } + rows.Close() + for _, q := range ids { + s.lookup(q) + } +} + func btoi(b bool) int { if b { return 1 @@ -433,7 +518,16 @@ func (s *WorkflowStore) Submit(ctx context.Context, p contract.WorkflowPlan) err qh *handle } plan := make([]planned, len(p.Tasks)) + names := make(map[string]bool, len(p.Tasks)) + for _, tp := range p.Tasks { + names[tp.Name] = true + } for i, tp := range p.Tasks { + for _, d := range tp.Deps { + if !names[d] { + return fmt.Errorf("varmq: task %q depends on %q, which is not in the workflow", tp.Name, d) + } + } if !validTaskName(tp.Name) { return fmt.Errorf("varmq: invalid task name %q", tp.Name) } @@ -464,6 +558,9 @@ func (s *WorkflowStore) Submit(ctx context.Context, p contract.WorkflowPlan) err var minSeq int64 var exists bool err := s.write(ctx, func(tx *sql.Tx) error { + if err := s.dropExpired(ctx, tx, p.ID); err != nil { + return err + } var hash string switch err := tx.QueryRowContext(ctx, `SELECT hash FROM `+wfWorkflows+` WHERE id = ?`, p.ID).Scan(&hash); { case err == nil: @@ -528,8 +625,10 @@ func (s *WorkflowStore) Submit(ctx context.Context, p contract.WorkflowPlan) err if minSeq != 0 { s.drain(ctx, `workflow_id = ? AND seq >= ?`, p.ID, minSeq) } - if err := s.sweep(ctx); err != nil { - log.Printf("sqliteq: sweeping workflows: %v", err) + if last := s.sweptAt.Load(); s.now().UnixMilli()-last >= sweepOnSubmit.Milliseconds() && s.sweptAt.CompareAndSwap(last, s.now().UnixMilli()) { + if err := s.sweep(ctx); err != nil { + log.Printf("sqliteq: sweeping workflows: %v", err) + } } } return nil @@ -554,7 +653,8 @@ func (s *WorkflowStore) stamp(w *wfFlow, fx []wfEffect) { // release turns the pushes of fx into rows (reading and clearing each task's // stored item) and delivers them in tx. // Folder layout: outbox rows, minSeq the first one's seq (0 when none). -// Single file: straight into the queues, ids the queue IDs it pushed into. +// Single file: straight into the queues (ids), except those this client +// cannot reach, which get outbox rows. func (s *WorkflowStore) release(ctx context.Context, tx *sql.Tx, w *wfFlow, fx []wfEffect) (minSeq int64, ids []string, err error) { var rows []outRow for _, e := range fx { @@ -582,18 +682,20 @@ func (s *WorkflowStore) release(ctx context.Context, tx *sql.Tx, w *wfFlow, fx [ } for _, r := range rows { - if s.single { - h, ok := s.qs.handleOf(r.queueID) - if !ok { - return 0, nil, fmt.Errorf("sqliteq: workflow %s: queue %s is not open in this client", w.id, r.queueID) - } - if err = h.queue().enqueueTx(tx, r.item, r.priority, r.jobID); err != nil { + // A queue this client cannot reach (opened later, or by another + // client) gets an outbox row, in either layout: the push waits + if h, ok := s.qs.handleOf(r.queueID); ok && s.single { + switch err = h.queue().enqueueTx(tx, r.item, r.priority, r.jobID); { + case err == nil: + if !slices.Contains(ids, r.queueID) { + ids = append(ids, r.queueID) + } + continue + case errors.Is(err, errJobIDTaken): + s.warnOnce(r.jobID, "job id is held by another job; the push stays in the outbox") + default: return 0, nil, err } - if !slices.Contains(ids, r.queueID) { - ids = append(ids, r.queueID) - } - continue } res, err := tx.ExecContext(ctx, `INSERT INTO `+wfOutbox+ ` (workflow_id, queue_id, job_id, item, priority, created_at) VALUES (?, ?, ?, ?, ?, ?)`, @@ -611,17 +713,24 @@ func (s *WorkflowStore) release(ctx context.Context, tx *sql.Tx, w *wfFlow, fx [ } // drain pushes the outbox rows matching where into their queues and deletes -// each pushed one. A queue this client did not open, or a closed one, keeps -// its rows for the next recovery. It returns the queue IDs pushed into. +// each pushed one. Only rows of queues open in this process are read; a +// closed queue keeps its rows for the next recovery. It returns the queue IDs pushed into. func (s *WorkflowStore) drain(ctx context.Context, where string, args ...any) []string { - if s.single { + open := s.openIDs() + if len(open) == 0 { return nil } + in := strings.TrimSuffix(strings.Repeat("?,", len(open)), ",") var ids []string - after := int64(0) + after, total := int64(0), 0 for { + qargs := []any{after} + qargs = append(qargs, args...) + for _, id := range open { + qargs = append(qargs, id) + } rows, err := s.f.db.QueryContext(ctx, `SELECT seq, queue_id, job_id, item, priority FROM `+wfOutbox+ - ` WHERE seq > ? AND (`+where+`) ORDER BY seq LIMIT `+fmt.Sprint(drainBatch), append([]any{after}, args...)...) + ` WHERE seq > ? AND (`+where+`) AND queue_id IN (`+in+`) ORDER BY seq LIMIT `+fmt.Sprint(drainBatch), qargs...) if err != nil { log.Printf("sqliteq: reading workflow outbox: %v", err) return ids @@ -644,10 +753,17 @@ func (s *WorkflowStore) drain(ctx context.Context, where string, args ...any) [] return ids } + total += len(batch) for _, r := range batch { after = r.seq - h, ok := s.qs.handleOf(r.queueID) - if !ok || !pushRow(h, r.outRow) { + h, ok := s.lookup(r.queueID) + if !ok { + continue + } + if err := pushRow(h, r.outRow); err != nil { + if errors.Is(err, errJobIDTaken) { + s.warnOnce(r.jobID, "job id is held by another job; the row stays") + } continue } s.f.mu.Lock() @@ -661,25 +777,65 @@ func (s *WorkflowStore) drain(ctx context.Context, where string, args ...any) [] ids = append(ids, r.queueID) } } - if len(batch) < drainBatch { + if len(batch) < drainBatch || total >= drainCap { return ids } } } // pushRow adds the row's item to its queue once per job ID. -func pushRow(h *handle, r outRow) bool { - if h.pq != nil { - return h.pq.EnqueueOnce(r.item, r.priority, r.jobID) +func pushRow(h *handle, r outRow) error { + return h.queue().pushWorkflow(r.item, r.priority, r.jobID) +} + +// warnOnce logs msg for key once per store. +func (s *WorkflowStore) warnOnce(key, msg string) { + if _, seen := s.warned.LoadOrStore(key, struct{}{}); !seen { + log.Printf("sqliteq: workflow outbox %s: %s", key, msg) } - return h.q.EnqueueOnce(r.item, r.jobID) } // recoverOutbox pushes outbox rows at least minAge old. func (s *WorkflowStore) recoverOutbox(minAge time.Duration) { + s.logUnreachable() + s.applyIntents(minAge) s.drain(context.Background(), `created_at <= ?`, s.now().Add(-minAge).UnixMilli()) } +// dropExpired deletes workflow id when it ended more than the retention ago +// and no sweep has run yet, so a Submit of the ID starts a new workflow. +func (s *WorkflowStore) dropExpired(ctx context.Context, tx *sql.Tx, id string) error { + cut := s.now().Add(-s.cfg.retention).UnixMilli() + for _, t := range []string{wfTasks, wfOutbox} { + if _, err := tx.ExecContext(ctx, `DELETE FROM `+t+` WHERE workflow_id = ? AND EXISTS + (SELECT 1 FROM `+wfWorkflows+` WHERE id = ? AND finished_at IS NOT NULL AND finished_at <= ?)`, id, id, cut); err != nil { + return err + } + } + _, err := tx.ExecContext(ctx, `DELETE FROM `+wfWorkflows+` WHERE id = ? AND finished_at IS NOT NULL AND finished_at <= ?`, id, cut) + return err +} + +// logUnreachable names, once each, the queues of outbox rows that no client in +// this process has open. +func (s *WorkflowStore) logUnreachable() { + open := map[string]bool{} + for _, id := range s.openIDs() { + open[id] = true + } + rows, err := s.f.db.Query(`SELECT DISTINCT queue_id FROM ` + wfOutbox) + if err != nil { + return + } + defer rows.Close() + for rows.Next() { + var id string + if rows.Scan(&id) == nil && !open[id] { + s.warnOnce(id, "queue is not open in this process; its rows wait") + } + } +} + // sweep drops workflows ended more than the retention ago, with their tasks // and outbox rows. func (s *WorkflowStore) sweep(ctx context.Context) error { @@ -840,6 +996,7 @@ func (s *WorkflowStore) Cancel(ctx context.Context, id string, undo *bool) error if !validWorkflowID(id) { return s.notFound(id) } + s.preResolve(ctx, id) var minSeq int64 err := s.write(ctx, func(tx *sql.Tx) error { w, err := s.load(ctx, tx, id) diff --git a/workflow_finish.go b/workflow_finish.go index 0126ed4..6a7c5cf 100644 --- a/workflow_finish.go +++ b/workflow_finish.go @@ -6,6 +6,10 @@ import ( "encoding/json" "errors" "log" + "os" + "path/filepath" + "slices" + "strings" "time" "github.com/goptics/varmq/contract" @@ -33,7 +37,7 @@ type endResult struct { // settles the job and returns ok with no IDs. A completed result over the // limit fails the task (ErrResultTooLarge); one that is not JSON is dropped. func (s *WorkflowStore) FinishTask(queueID, ackID string, e contract.TaskEnd) ([]string, bool) { - h, ok := s.qs.handleOf(queueID) + h, ok := s.lookup(queueID) if !ok { log.Printf("sqliteq: finish task %s/%s: queue %s is not open in this client", e.Workflow, e.Task, queueID) return nil, false @@ -63,10 +67,12 @@ func (s *WorkflowStore) FinishTask(queueID, ackID string, e contract.TaskEnd) ([ e.Reason, e.Result = reason, result if !validWorkflowID(e.Workflow) || !validTaskName(e.Task) { - return nil, q.finish(ackID, state, reason, result) + _, err := q.finishDirect(ackID, state, reason, result) + return nil, err == nil } ctx := context.Background() + s.preResolve(ctx, e.Workflow) var r endResult var acked bool err := s.write(ctx, func(tx *sql.Tx) (err error) { @@ -91,14 +97,18 @@ func (s *WorkflowStore) FinishTask(queueID, ackID string, e contract.TaskEnd) ([ } pushed := r.pushed - if !s.single { - pushed = nil - if r.minSeq != 0 { - pushed = s.drain(ctx, `workflow_id = ? AND seq >= ?`, e.Workflow, r.minSeq) + if r.minSeq != 0 { + for _, id := range s.drain(ctx, `workflow_id = ? AND seq >= ?`, e.Workflow, r.minSeq) { + if !slices.Contains(pushed, id) { + pushed = append(pushed, id) + } } } if !acked { - q.finish(ackID, state, reason, result) + if _, err := q.finishDirect(ackID, state, reason, result); err != nil { + log.Printf("sqliteq: finishing workflow %s task %s: ack: %v", e.Workflow, e.Task, err) + return nil, false + } } if r.refinish { pushed = r.targets @@ -142,25 +152,125 @@ func (s *WorkflowStore) endTx(ctx context.Context, tx *sql.Tx, queueID string, e return r, err } -// endPurged ends the workflow tasks of items a Purge of queue q removed as -// cancelled (reason "queue purged"), so their workflows go on. There is no -// job to settle: Purge deleted it. A task not in q, or one that had ended, -// changes nothing. -func (s *WorkflowStore) endPurged(q *Queue, tags []wfItemTag) { - ctx := context.Background() +// purgeIntent is a purged workflow item whose end is still to be applied. +type purgeIntent struct { + seq int64 // 0: not stored + queueID string + tag wfItemTag + storedAt int64 +} + +// recordPurge stores an intent per tag in the workflow file before the +// Purge deletes the rows, so a crash after the delete loses no end: the +// recovery loop applies leftovers. +func (s *WorkflowStore) recordPurge(ctx context.Context, q *Queue, tags []wfItemTag) []purgeIntent { + var out []purgeIntent for _, tag := range tags { - e := contract.TaskEnd{Workflow: tag.wf, Task: tag.task, Undo: tag.undo, Outcome: contract.OutcomeCancelled, Reason: purgeReason} - var r endResult - err := s.write(ctx, func(tx *sql.Tx) (err error) { - r, err = s.endTx(ctx, tx, q.id, e) - return err - }) + s.f.mu.Lock() + res, err := s.f.db.ExecContext(ctx, `INSERT INTO `+wfPurged+` (queue_id, workflow_id, task, undo, created_at) VALUES (?, ?, ?, ?, ?)`, + q.id, tag.wf, tag.task, btoi(tag.undo), s.now().UnixMilli()) + s.f.mu.Unlock() if err != nil { - log.Printf("sqliteq: purge of %s: ending workflow %s task %s: %v", q.name, tag.wf, tag.task, err) + log.Printf("sqliteq: purge of %s: recording workflow %s task %s: %v", q.name, tag.wf, tag.task, err) continue } - if r.minSeq != 0 { - s.drain(ctx, `workflow_id = ? AND seq >= ?`, tag.wf, r.minSeq) + seq, _ := res.LastInsertId() + out = append(out, purgeIntent{seq: seq, queueID: q.id, tag: tag}) + } + return out +} + +// endPurged applies the ends of items a Purge of queue q removed: each +// intent, then the tags the delete's own read found that no intent covers. +func (s *WorkflowStore) endPurged(q *Queue, intents []purgeIntent, tags []wfItemTag) { + seen := map[wfItemTag]bool{} + for _, in := range intents { + seen[in.tag] = true + } + for _, tag := range tags { + if !seen[tag] { + intents = append(intents, purgeIntent{queueID: q.id, tag: tag}) + } + } + for _, in := range intents { + s.applyIntent(context.Background(), in) + } +} + +// applyIntent ends the purged task as cancelled (reason "queue purged") and +// deletes the intent, in one transaction. A task not in the queue, or one +// that had ended, changes nothing. +func (s *WorkflowStore) applyIntent(ctx context.Context, in purgeIntent) { + s.preResolve(ctx, in.tag.wf) + e := contract.TaskEnd{Workflow: in.tag.wf, Task: in.tag.task, Undo: in.tag.undo, Outcome: contract.OutcomeCancelled, Reason: purgeReason} + var r endResult + err := s.write(ctx, func(tx *sql.Tx) (err error) { + if r, err = s.endTx(ctx, tx, in.queueID, e); err != nil || in.seq == 0 { + return err + } + _, err = tx.ExecContext(ctx, `DELETE FROM `+wfPurged+` WHERE seq = ?`, in.seq) + return err + }) + if err != nil { + log.Printf("sqliteq: purge: ending workflow %s task %s: %v", in.tag.wf, in.tag.task, err) + return + } + if r.minSeq != 0 { + s.drain(ctx, `workflow_id = ? AND seq >= ?`, in.tag.wf, r.minSeq) + } +} + +// applyIntents applies the intents at least minAge old. +func (s *WorkflowStore) applyIntents(minAge time.Duration) { + rows, err := s.f.db.Query(`SELECT seq, queue_id, workflow_id, task, undo FROM `+wfPurged+` WHERE created_at <= ? ORDER BY seq LIMIT ?`, + s.now().Add(-minAge).UnixMilli(), drainBatch) + if err != nil { + return + } + var list []purgeIntent + for rows.Next() { + var in purgeIntent + var undo int + if rows.Scan(&in.seq, &in.queueID, &in.tag.wf, &in.tag.task, &undo) == nil { + in.tag.undo = undo == 1 + list = append(list, in) + } + } + rows.Close() + for _, in := range list { + s.applyIntent(context.Background(), in) + } +} + +// purgeStore returns the store whose workflows a Purge of the file must end: +// the registered one, else one opened on demand when the file's workflow +// tables exist (nil when they do not). done releases an on-demand store. +func (f *file) purgeStore() (ws *WorkflowStore, done func()) { + if st, ok := contract.LookupWorkflowStore(f.scope); ok { + if ws, ok := st.(*WorkflowStore); ok { + return ws, func() {} + } + } + root := strings.TrimPrefix(f.scope, "sqliteq:") + single := root == f.key + wf, release := f, func() {} + if single { + var n int + if f.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name = '__sqliteq_wf_workflows'`).Scan(&n) != nil || n == 0 { + return nil, func() {} + } + } else { + path, err := resolve(filepath.Join(root, workflowsFile)) + if _, serr := os.Stat(path); err != nil || serr != nil { + return nil, func() {} + } + if wf, err = acquire(path, root, f.pages); err != nil { + log.Printf("sqliteq: purge: opening the workflow file: %v", err) + return nil, func() {} } + release = func() { wf.release() } } + qs := &queues{cfg: storeConfig{singleFile: single, checkpointPages: f.pages}, root: root, files: map[*file]struct{}{}} + qs.byID.Store(&map[string]*handle{}) + return &WorkflowStore{qs: qs, f: wf, single: single, scope: f.scope, cfg: workflowConfig{resultLimit: defaultResultLimit, retention: defaultRetention}}, release } From 1eaacba5c6298c561233766aecbaa53746980e51 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Mon, 9 Nov 2026 22:07:55 +0600 Subject: [PATCH 5/8] fix: apply purge intents only after the job row is gone --- README.md | 2 +- queue_test.go | 41 +++++++++++++++++++++++++++++++++++++++++ workflow.go | 2 +- workflow_finish.go | 23 +++++++++++++++++++---- 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 10e0520..dcfcb95 100644 --- a/README.md +++ b/README.md @@ -310,7 +310,7 @@ A recovery loop pushes leftover outbox rows when the store opens and every secon - The loop pushes only into queues that some client in this process has open. Rows for other queues wait, and the store logs each such queue once. - A job ID that starts with `wf:` is for workflow items. A plain job cannot take one. If an older row already holds the ID, the push stays in the outbox and the store logs it once. - The stored items of waiting tasks and the undo items stay in the workflow file until the task is pushed or the workflow expires. -- Folder layout: `Purge` stores an intent in the workflow file before it deletes the rows. If the process crashes between them, the recovery loop ends the purged tasks later. +- Folder layout: `Purge` stores an intent in the workflow file before it deletes the rows. If the process crashes between them, the recovery loop ends the purged tasks later. It applies an intent only once the job's row is gone from its queue, so a slow or aborted purge never cancels a live task. A valid workflow item is never dropped by a cancel mark: the workflow decides how its task ends. A `Purge` of a queue ends each purged workflow task as cancelled with the reason `queue purged`. The workflow goes on. This works also when no store is open in the process, as long as the workflow tables exist. Under `WithSingleFile` the delete and the ends are one transaction. An item whose JSON has keys that differ only in case, such as `WF` next to `wf`, is a plain job. diff --git a/queue_test.go b/queue_test.go index 18cd790..2b04bc0 100644 --- a/queue_test.go +++ b/queue_test.go @@ -3256,3 +3256,44 @@ func TestWorkflowWithVarmqWorkers(t *testing.T) { } }) } + +// The loop applies a purge intent only once the job row is gone from its +// queue: a purge slower than the tick waits, and an aborted purge whose job +// then ran leaves the task alone. +func TestWorkflowPurgeIntentWaitsForDelete(t *testing.T) { + e := newWFEnv(t, false) + ctx := context.Background() + if err := e.st.Submit(ctx, planWF("w1", planTask("w1", "a", e.a), planTask("w1", "b", e.b, "a"))); err != nil { + t.Fatal(err) + } + if err := e.st.Submit(ctx, planWF("w2", planTask("w2", "a", e.a))); err != nil { + t.Fatal(err) + } + intents := func() int { + var n int + e.st.f.db.QueryRow(`SELECT COUNT(*) FROM ` + wfPurged).Scan(&n) + return n + } + e.st.recordPurge(ctx, e.a, []wfItemTag{{wf: "w1", task: "a"}, {wf: "w2", task: "a"}}) + + // The purge is still running: its delete has not committed + e.st.recoverOutbox(0) + if got := taskStates(e.state(t, "w1")); got != "a=pending b=waiting" || intents() != 2 { + t.Fatalf("Expected the intents to wait, got %s, %d intents", got, intents()) + } + + // w2's purge aborted and its job ran; w1's delete commits + e.a.f.db.Exec(`DELETE FROM "a" WHERE job_id = 'wf:w1:a'`) + _, ack := take(t, e.a) + e.finish(t, e.a, ack, done("w2", "a", "")) + e.st.recoverOutbox(0) + if got := taskStates(e.state(t, "w1")); got != "a=cancelled b=cancelled" { + t.Errorf("Expected the delete's intent applied, got %s", got) + } + if r := e.state(t, "w2"); r.State != contract.WorkflowCompleted { + t.Errorf("Expected the aborted purge's task left alone, got %s", r.State) + } + if intents() != 0 { + t.Errorf("Expected every intent cleared, got %d", intents()) + } +} diff --git a/workflow.go b/workflow.go index c27d73c..7be012c 100644 --- a/workflow.go +++ b/workflow.go @@ -798,7 +798,7 @@ func (s *WorkflowStore) warnOnce(key, msg string) { // recoverOutbox pushes outbox rows at least minAge old. func (s *WorkflowStore) recoverOutbox(minAge time.Duration) { s.logUnreachable() - s.applyIntents(minAge) + s.applyIntents() s.drain(context.Background(), `created_at <= ?`, s.now().Add(-minAge).UnixMilli()) } diff --git a/workflow_finish.go b/workflow_finish.go index 6a7c5cf..d04e094 100644 --- a/workflow_finish.go +++ b/workflow_finish.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "errors" + "fmt" "log" "os" "path/filepath" @@ -220,10 +221,13 @@ func (s *WorkflowStore) applyIntent(ctx context.Context, in purgeIntent) { } } -// applyIntents applies the intents at least minAge old. -func (s *WorkflowStore) applyIntents(minAge time.Duration) { - rows, err := s.f.db.Query(`SELECT seq, queue_id, workflow_id, task, undo FROM `+wfPurged+` WHERE created_at <= ? ORDER BY seq LIMIT ?`, - s.now().Add(-minAge).UnixMilli(), drainBatch) +// applyIntents applies the stored intents whose job row is gone from its +// queue: the purge's delete has committed. While the row is still there the +// delete has not committed yet, or the purge aborted and the job lives on; +// the intent waits. Once that job ends normally, its task has ended, and +// applying the intent changes nothing but deleting it. +func (s *WorkflowStore) applyIntents() { + rows, err := s.f.db.Query(`SELECT seq, queue_id, workflow_id, task, undo FROM `+wfPurged+` ORDER BY seq LIMIT ?`, drainBatch) if err != nil { return } @@ -238,6 +242,17 @@ func (s *WorkflowStore) applyIntents(minAge time.Duration) { } rows.Close() for _, in := range list { + h, ok := s.lookup(in.queueID) + if !ok { + continue + } + q := h.queue() + var n int + err := q.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE job_id = ? AND status IN ('pending', 'processing')", q.table), + in.tag.jobID()).Scan(&n) + if err != nil || n > 0 { + continue + } s.applyIntent(context.Background(), in) } } From ee6ca408acbc5c66fa2f2c05f8b601d810dab0d8 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Mon, 9 Nov 2026 22:15:50 +0600 Subject: [PATCH 6/8] docs: list workflows in README features --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index dcfcb95..de17d58 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ - **Job outcomes**: `WatchOutcome` tells any process on the file when a job completes, fails or is cancelled, with the worker's result. - **Job cancel**: `CancelJob` drops a waiting job, and `OnCancel` tells every process on the file to stop a running one. - **Job records and inspection**: counts, totals and job lists for VarMQ's REST API and vartop. +- **[Workflows](#workflows)**: `NewWorkflowStore` keeps VarMQ workflows (a DAG of jobs across queues) in SQLite, so they survive a restart. - **Durable queue pause**: `Pause` stops the pops of every process on the file, and the pause outlives restarts. - **Automatic WAL checkpoints**, tunable with `WithCheckpointPages`. - **Thread-safe**: share one queue object across goroutines. From 35341f2756b0241c8d405d562d0a1347200e0648 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Mon, 9 Nov 2026 23:09:52 +0600 Subject: [PATCH 7/8] build: require contract v0.8.0 and varmq main (c75ed52) --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index d59e374..d979276 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/goptics/sqliteq go 1.27 require ( - github.com/goptics/varmq v1.10.1-0.20261108161945-6bc553db29c6 - github.com/goptics/varmq/contract v0.6.0 + github.com/goptics/varmq v1.10.1-0.20261109165008-c75ed52b3b0a + github.com/goptics/varmq/contract v0.8.0 github.com/mattn/go-sqlite3 v1.14.28 ) diff --git a/go.sum b/go.sum index f800d0f..d21bc24 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,9 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/goptics/varmq v1.10.1-0.20261108161945-6bc553db29c6 h1:0GooGJC22Z/9WxAz0f8aVzWyPkBDJhHF8JI4oYTIW+w= -github.com/goptics/varmq v1.10.1-0.20261108161945-6bc553db29c6/go.mod h1:RLnozQPouu3sq2tHoCixCXSoCZMPjB90MNEHvlpFz+E= -github.com/goptics/varmq/contract v0.6.0 h1://rpDlzSwsqvPDS98siuPDN1LyyBLFhiYeAmB7MZfbQ= -github.com/goptics/varmq/contract v0.6.0/go.mod h1:55tOUywUsZhBgyo8oJhNmKMknETVHL2OuMrITztanXg= +github.com/goptics/varmq v1.10.1-0.20261109165008-c75ed52b3b0a h1:+UjHIuTdKROtu6zLVk8b/efeP6uTmQqIy7TgshIHx6U= +github.com/goptics/varmq v1.10.1-0.20261109165008-c75ed52b3b0a/go.mod h1:4hso4XnUSTh5sjr/h2//MYh0FPGRvebLCx5AaoL4qII= +github.com/goptics/varmq/contract v0.8.0 h1:U87RzQITqs3TPNuzNXLCl9Fp9ebUa6Eiw3cDNd2cb5Q= +github.com/goptics/varmq/contract v0.8.0/go.mod h1:55tOUywUsZhBgyo8oJhNmKMknETVHL2OuMrITztanXg= github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= From cf41dc60515db45738e3d961dc70923d7f2f67f7 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Mon, 9 Nov 2026 23:10:26 +0600 Subject: [PATCH 8/8] docs: changelog PR number #25 --- CHANGELOG.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a53196e..4d98ee7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,26 +15,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Workflows (VarMQ spec 6.9 to 6.12): `NewWorkflowStore(qs, ...WorkflowOption)` returns the `contract.WorkflowStore` of a store root. (#NN) -- The store registers with `contract.RegisterWorkflowStore`; `Close` unregisters it and releases its file. (#NN) -- `WithWorkflowResultLimit` (default 64 KiB, `ErrResultTooLarge`) and `WithWorkflowRetention` (default 7 days). (#NN) -- `ScopeID()` on `Queue` and `PriorityQueue`: `sqliteq:` and the absolute store folder (symlinks resolved), or the file under `WithSingleFile`. (#NN) -- A file open under one scope is refused under another; `WithSingleFile` refuses the name `__sqliteq_workflows.db`. (#NN) -- `FinishTask` finds the queue for any client of the scope in this process. (#NN) -- `WithSingleFile`: a push into a queue no client has open becomes an outbox row in the same transaction. (#NN) -- A plain job cannot take a job ID that starts with `wf:`; a held ID keeps the push in the outbox. (#NN) -- `Purge` ends purged workflow tasks without a registered store; one transaction under `WithSingleFile`, a stored intent in the folder layout. (#NN) -- `Submit` refuses a dependency that names no task, and replaces an expired, unswept workflow of the same ID. (#NN) -- Folder layout: the store keeps its tables in `__sqliteq_workflows.db`. `WithSingleFile`: tables in the same file. (#NN) -- Tables `__sqliteq_wf_workflows`, `__sqliteq_wf_tasks`, `__sqliteq_wf_outbox`. (#NN) -- `FinishTask` in three steps in the folder layout: record the end and write the outbox, push each row with `EnqueueOnce`, acknowledge the job. (#NN) -- A recovery loop pushes leftover outbox rows at open and every second, in both layouts. (#NN) -- `WithSingleFile`: the end, the pushes and the ack are one transaction. (#NN) -- A redelivered job, or a push repeated after the job was removed, is safe: the pre-run check skips a task that already ended. (#NN) -- `Workflows` (`contract.WorkflowInspector`) and `WorkflowKeys` (`contract.WorkflowKeyLister`) list workflows newest first. (#NN) -- A cancel mark never drops a valid workflow item; the workflow decides its end. (#NN) -- Task JSON whose keys differ only in case (`WF` next to `wf`) is a plain job. (#NN) -- `Purge` ends each purged workflow task as cancelled, reason `queue purged`; its workflow goes on. (#NN) +- Workflows (VarMQ spec 6.9 to 6.12): `NewWorkflowStore(qs, ...WorkflowOption)` returns the `contract.WorkflowStore` of a store root. (#25) +- The store registers with `contract.RegisterWorkflowStore`; `Close` unregisters it and releases its file. (#25) +- `WithWorkflowResultLimit` (default 64 KiB, `ErrResultTooLarge`) and `WithWorkflowRetention` (default 7 days). (#25) +- `ScopeID()` on `Queue` and `PriorityQueue`: `sqliteq:` and the absolute store folder (symlinks resolved), or the file under `WithSingleFile`. (#25) +- A file open under one scope is refused under another; `WithSingleFile` refuses the name `__sqliteq_workflows.db`. (#25) +- `FinishTask` finds the queue for any client of the scope in this process. (#25) +- `WithSingleFile`: a push into a queue no client has open becomes an outbox row in the same transaction. (#25) +- A plain job cannot take a job ID that starts with `wf:`; a held ID keeps the push in the outbox. (#25) +- `Purge` ends purged workflow tasks without a registered store; one transaction under `WithSingleFile`, a stored intent in the folder layout. (#25) +- `Submit` refuses a dependency that names no task, and replaces an expired, unswept workflow of the same ID. (#25) +- Folder layout: the store keeps its tables in `__sqliteq_workflows.db`. `WithSingleFile`: tables in the same file. (#25) +- Tables `__sqliteq_wf_workflows`, `__sqliteq_wf_tasks`, `__sqliteq_wf_outbox`. (#25) +- `FinishTask` in three steps in the folder layout: record the end and write the outbox, push each row with `EnqueueOnce`, acknowledge the job. (#25) +- A recovery loop pushes leftover outbox rows at open and every second, in both layouts. (#25) +- `WithSingleFile`: the end, the pushes and the ack are one transaction. (#25) +- A redelivered job, or a push repeated after the job was removed, is safe: the pre-run check skips a task that already ended. (#25) +- `Workflows` (`contract.WorkflowInspector`) and `WorkflowKeys` (`contract.WorkflowKeyLister`) list workflows newest first. (#25) +- A cancel mark never drops a valid workflow item; the workflow decides its end. (#25) +- Task JSON whose keys differ only in case (`WF` next to `wf`) is a plain job. (#25) +- `Purge` ends each purged workflow task as cancelled, reason `queue purged`; its workflow goes on. (#25) - `Reject(ackID, reason)` on `Queue` and `PriorityQueue`, for VarMQ's `IAcknowledgeable` (varmq#173). (#21) - `Reject` finishes an item that failed for good: never requeued. (#21) - Rejected item deleted like an ack; with `WithRemoveOnComplete(false)` kept with `status` `failed`. (#21)