From 8478bd9163773901b903ad36fcd9a02cde22bd30 Mon Sep 17 00:00:00 2001 From: ashprice Date: Sun, 14 Jun 2026 17:40:13 +0100 Subject: [PATCH 01/14] Cache pending and completed task vectors instead of copying Signed-off-by: ashprice Typo Signed-off-by: ashprice Added note Signed-off-by: ashprice --- src/TDB2.cpp | 14 ++++++++++---- src/TDB2.h | 6 ++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/TDB2.cpp b/src/TDB2.cpp index a9434c367..56fb6d881 100644 --- a/src/TDB2.cpp +++ b/src/TDB2.cpp @@ -253,12 +253,15 @@ const std::vector TDB2::all_tasks() { } //////////////////////////////////////////////////////////////////////////////// -const std::vector TDB2::pending_tasks() { +const std::vector& TDB2::pending_tasks() { if (!_pending_tasks) { Timer timer; auto pending_tctasks = replica()->pending_task_data(); std::vector result; + + result.reserve(pending_tctasks.size()); + for (auto& maybe_tctask : pending_tctasks) { auto tctask = maybe_tctask.take(); result.push_back(Task(std::move(tctask))); @@ -267,19 +270,22 @@ const std::vector TDB2::pending_tasks() { dependency_scan(result); Context::getContext().time_load_us += timer.total_us(); - _pending_tasks = result; + _pending_tasks = std::move(result); } return *_pending_tasks; } //////////////////////////////////////////////////////////////////////////////// -const std::vector TDB2::completed_tasks() { +const std::vector& TDB2::completed_tasks() { if (!_completed_tasks) { auto all_tctasks = replica()->all_task_data(); auto& ws = working_set(); std::vector result; + + result.reserve(all_tctasks.size()); + for (auto& maybe_tctask : all_tctasks) { auto tctask = maybe_tctask.take(); // if this task is _not_ in the working set, return it. @@ -287,7 +293,7 @@ const std::vector TDB2::completed_tasks() { result.push_back(Task(std::move(tctask))); } } - _completed_tasks = result; + _completed_tasks = std::move(result); } return *_completed_tasks; } diff --git a/src/TDB2.h b/src/TDB2.h index 048765258..d91573ead 100644 --- a/src/TDB2.h +++ b/src/TDB2.h @@ -54,9 +54,11 @@ class TDB2 { int latest_id(); // Generalized task accessors. + // We don't cache the all_tasks vector because no command accesses it more than once. + // Caching it would cause higher memory use and would require additional rewrites to make tests pass. const std::vector all_tasks(); - const std::vector pending_tasks(); - const std::vector completed_tasks(); + const std::vector& pending_tasks(); + const std::vector& completed_tasks(); bool get(int, Task&); bool get(const std::string&, Task&); bool has(const std::string&); From d40ad3e57198f6b8b7f13beb8b6828de9cae8f19 Mon Sep 17 00:00:00 2001 From: ashprice Date: Mon, 15 Jun 2026 13:29:41 +0100 Subject: [PATCH 02/14] Add a lazily-loaded UUID index for the pending tasks vector. Signed-off-by: ashprice --- src/TDB2.cpp | 69 +++++++++++++++++++++++++++++++++++++++------------- src/TDB2.h | 7 ++++++ 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/src/TDB2.cpp b/src/TDB2.cpp index 56fb6d881..38e558ce7 100644 --- a/src/TDB2.cpp +++ b/src/TDB2.cpp @@ -38,6 +38,7 @@ #include #include +#include #include #include @@ -267,6 +268,13 @@ const std::vector& TDB2::pending_tasks() { result.push_back(Task(std::move(tctask))); } + // Build a UUID map for use with get()/modify() and dependency_scan() + // while the pending vector is already in memory. + _pending_index.emplace(); + _pending_index->reserve(result.size()); + for (size_t i = 0, n = result.size(); i < n; ++i) + _pending_index->emplace(result[i].get_ref("uuid"), i); + dependency_scan(result); Context::getContext().time_load_us += timer.total_us(); @@ -303,6 +311,27 @@ void TDB2::invalidate_cached_info() { _pending_tasks = std::nullopt; _completed_tasks = std::nullopt; _working_set = std::nullopt; + _pending_index = std::nullopt; +} + +///////////////////////////////////////////////////////////////////////////////// +// This builds and returns the UUID map if it is missing. +const std::unordered_map& TDB2::pending_index() { + if (!_pending_index) { + _pending_index.emplace(); + _pending_index->reserve(_pending_tasks->size()); + for (size_t i = 0, n = _pending_tasks->size(); i < n; ++i) + _pending_index->emplace((*_pending_tasks)[i].get_ref("uuid"), i); + } + + return *_pending_index; +} + +// Finds the UUID in the index. Returns SIZE_MAX if it isn't present. +size_t TDB2::pending_index_of(const std::string& uuid) { + auto& idx = pending_index(); + auto it = idx.find(uuid); + return it != idx.end() ? it->second : SIZE_MAX; } //////////////////////////////////////////////////////////////////////////////// @@ -312,14 +341,13 @@ bool TDB2::get(int id, Task& task) { const auto tcuuid = ws->by_index(id); if (!tcuuid.is_nil()) { std::string uuid = static_cast(tcuuid.to_string()); - // Load all pending tasks in order to get dependency data, and in particular - // `task.is_blocking` and `task.is_blocked`, set correctly. - std::vector pending = pending_tasks(); - for (auto& pending_task : pending) { - if (pending_task.get("uuid") == uuid) { - task = pending_task; - return true; - } + // Load index of pending tasks. + pending_tasks(); + // Lookup the UUID in the index instead of scanning the vector. + auto idx = pending_index_of(uuid); + if (idx != SIZE_MAX) { + task = (*_pending_tasks)[idx]; + return true; } } @@ -329,15 +357,22 @@ bool TDB2::get(int id, Task& task) { //////////////////////////////////////////////////////////////////////////////// // Locate task by UUID, including by partial ID, wherever it is. bool TDB2::get(const std::string& uuid, Task& task) { - // Load all pending tasks in order to get dependency data, and in particular - // `task.is_blocking` and `task.is_blocked`, set correctly. - std::vector pending = pending_tasks(); - - // try by raw uuid, if the length is right - for (auto& pending_task : pending) { - if (closeEnough(pending_task.get("uuid"), uuid, uuid.length())) { - task = pending_task; - return true; + pending_tasks(); + + // Try to match exact UUID within the index. + auto idx = pending_index_of(uuid); + if (idx != SIZE_MAX) { + task = (*_pending_tasks)[idx]; + return true; + } + + // try a partial match + if (uuid.length() < 36) { + for (const auto& pending_task : *_pending_tasks) { + if (closeEnough(pending_task.get_ref("uuid"), uuid, uuid.length())) { + task = pending_task; + return true; + } } } diff --git a/src/TDB2.h b/src/TDB2.h index d91573ead..bb0b72e70 100644 --- a/src/TDB2.h +++ b/src/TDB2.h @@ -59,6 +59,8 @@ class TDB2 { const std::vector all_tasks(); const std::vector& pending_tasks(); const std::vector& completed_tasks(); + // Index of pending tasks by UUID. + size_t pending_index_of(const std::string& uuid); bool get(int, Task&); bool get(const std::string&, Task&); bool has(const std::string&); @@ -81,8 +83,13 @@ class TDB2 { std::optional> _working_set; std::optional> _pending_tasks; std::optional> _completed_tasks; + // Lazily cache UUIDs within the pending set. + std::optional> _pending_index; void invalidate_cached_info(); + // Return the full pending UUID map. + const std::unordered_map& pending_index(); + // UUID -> Task containing all tasks modified in this invocation. std::map changes; From 45f1a0c0f911ada74c6c2efea17f11d15fda8716 Mon Sep 17 00:00:00 2001 From: ashprice Date: Mon, 15 Jun 2026 16:35:19 +0100 Subject: [PATCH 03/14] Create a dependency map/graph for all pending tasks (and an ephemeral one to resolve references within all_tasks) Signed-off-by: ashprice --- src/TDB2.cpp | 90 ++++++++++++++++++++++++++++++++++++++-------------- src/TDB2.h | 11 +++++++ 2 files changed, 78 insertions(+), 23 deletions(-) diff --git a/src/TDB2.cpp b/src/TDB2.cpp index 38e558ce7..a8761f9f3 100644 --- a/src/TDB2.cpp +++ b/src/TDB2.cpp @@ -43,7 +43,10 @@ #include bool TDB2::debug_mode = false; -static void dependency_scan(std::vector&); +static void dependency_scan(std::vector&, const std::unordered_map&); + +// Build maps for dependency queries. +static DependencyGraph build_dependency_graph(const std::vector&, const std::unordered_map&); //////////////////////////////////////////////////////////////////////////////// void TDB2::open_replica(const std::string& location, bool create_if_missing, bool read_write) { @@ -247,7 +250,14 @@ const std::vector TDB2::all_tasks() { all.push_back(Task(std::move(tctask))); } - dependency_scan(all); + // Build a temporary map so that dependency_scan can resolve references + // inside all_tasks. + std::unordered_map all_index; + all_index.reserve(all.size()); + for (size_t i = 0; i < all.size(); ++i) + all_index[all[i].get_ref("uuid")] = i; + + dependency_scan(all, all_index); Context::getContext().time_load_us += timer.total_us(); return all; @@ -275,7 +285,7 @@ const std::vector& TDB2::pending_tasks() { for (size_t i = 0, n = result.size(); i < n; ++i) _pending_index->emplace(result[i].get_ref("uuid"), i); - dependency_scan(result); + dependency_scan(result, *_pending_index); Context::getContext().time_load_us += timer.total_us(); _pending_tasks = std::move(result); @@ -306,11 +316,24 @@ const std::vector& TDB2::completed_tasks() { return *_completed_tasks; } +///////////////////////////////////////////////////////////////////////////////// +// Build and return the dependency map for pending tasks. +const DependencyGraph& TDB2::dependency_graph() { + if (!_dependency_graph) { + pending_tasks(); + // reuse the UUID index + _dependency_graph = build_dependency_graph(*_pending_tasks, pending_index()); + } + + return *_dependency_graph; +} + //////////////////////////////////////////////////////////////////////////////// void TDB2::invalidate_cached_info() { _pending_tasks = std::nullopt; _completed_tasks = std::nullopt; _working_set = std::nullopt; + _dependency_graph = std::nullopt; _pending_index = std::nullopt; } @@ -479,30 +502,51 @@ int TDB2::num_local_changes() { return (int)replica()->num_local_operations(); } int TDB2::num_reverts_possible() { return (int)replica()->num_undo_points(); } //////////////////////////////////////////////////////////////////////////////// -// For any task that has depenencies, follow the chain of dependencies until the -// end. Along the way, update the Task::is_blocked and Task::is_blocking data -// cache. -static void dependency_scan(std::vector& tasks) { - for (auto& left : tasks) { - for (auto& dep : left.getDependencyUUIDs()) { - for (auto& right : tasks) { - if (right.get("uuid") == dep) { - // GC hasn't run yet, check both tasks for their current status - Task::status lstatus = left.getStatus(); - Task::status rstatus = right.getStatus(); - if (lstatus != Task::completed && lstatus != Task::deleted && - rstatus != Task::completed && rstatus != Task::deleted) { - left.is_blocked = true; - right.is_blocking = true; - } - - // Only want to break out of the "right" loop. - break; - } +// Set Task::is_blocked / Task::is_blocking flags using the pre-built UUID map +static void dependency_scan(std::vector& tasks, const std::unordered_map& uuid_index) { + for (size_t i = 0; i< tasks.size(); ++i) { + auto lstatus = tasks[i].getStatus(); + for (const auto& dep : tasks[i].getDependencyUUIDs()) { + auto it = uuid_index.find(dep); + if (it == uuid_index.end()) + continue; + + size_t j = it->second; + auto rstatus = tasks[j].getStatus(); + if (lstatus != Task::completed && lstatus != Task::deleted && + rstatus != Task::completed && rstatus != Task::deleted) { + tasks[i].is_blocked = true; + tasks[j].is_blocking = true; } } } } +///////////////////////////////////////////////////////////////////////////////// +// Build the full dependency map from the task vector. +static DependencyGraph build_dependency_graph(const std::vector& tasks, + const std::unordered_map& uuid_index) { + DependencyGraph graph; + + for (size_t i = 0; i < tasks.size(); ++i) { + const auto& deps = tasks[i].getDependencyUUIDs(); + if (deps.empty()) + continue; + + const auto& uuid = tasks[i].get_ref("uuid"); + + for (const auto& dep : deps) { + auto it = uuid_index.find(dep); + if (it == uuid_index.end()) + continue; + + graph.dependencies[uuid].push_back(it->second); + graph.dependents[dep].push_back(i); + } + } + + return graph; +} + //////////////////////////////////////////////////////////////////////////////// // vim: ts=2 et sw=2 diff --git a/src/TDB2.h b/src/TDB2.h index bb0b72e70..aa48aa35d 100644 --- a/src/TDB2.h +++ b/src/TDB2.h @@ -37,6 +37,13 @@ #include #include +// Adjacency maps for the dependency graph. +// Index refers to the pending_tasks() vector. +struct DependencyGraph { + std::unordered_map> dependents; + std::unordered_map> dependencies; +}; + // TDB2 Class represents all the files in the task database. class TDB2 { public: @@ -59,6 +66,8 @@ class TDB2 { const std::vector all_tasks(); const std::vector& pending_tasks(); const std::vector& completed_tasks(); + // dependency_graph is built on first use from pending_tasks() and reused. + const DependencyGraph& dependency_graph(); // Index of pending tasks by UUID. size_t pending_index_of(const std::string& uuid); bool get(int, Task&); @@ -83,6 +92,8 @@ class TDB2 { std::optional> _working_set; std::optional> _pending_tasks; std::optional> _completed_tasks; + // Dependency cache. + std::optional _dependency_graph; // Lazily cache UUIDs within the pending set. std::optional> _pending_index; void invalidate_cached_info(); From 3bf596109e038684a73b7c6e742506262992a35c Mon Sep 17 00:00:00 2001 From: ashprice Date: Mon, 15 Jun 2026 16:53:16 +0100 Subject: [PATCH 04/14] modify the pending tasks vector in-place with cache invalidation. Signed-off-by: ashprice --- src/TDB2.cpp | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/TDB2.cpp b/src/TDB2.cpp index a8761f9f3..426291117 100644 --- a/src/TDB2.cpp +++ b/src/TDB2.cpp @@ -122,7 +122,7 @@ void TDB2::modify(Task& task) { // invoke the hook and allow it to modify the task before updating Task original; - get(uuid, original); + bool found_original = get(uuid, original); Context::getContext().hooks.onModify(original, task); tc::Uuid tcuuid = tc::uuid_from_string(uuid); @@ -170,7 +170,24 @@ void TDB2::modify(Task& task) { replica()->commit_operations(std::move(ops)); - invalidate_cached_info(); + // If the task entered or left the pending set, we must invalidate the cache. + bool was_pending = found_original && (original.getStatus() == Task::pending); + bool now_pending = task.getStatus() == Task::pending; + if (was_pending != now_pending) { + invalidate_cached_info(); + return; + } + + // If the task stayed in the set, we can edit the vector in-place. + if (_pending_tasks) { + auto idx = pending_index_of(uuid); + if (idx != SIZE_MAX) + (*_pending_tasks)[idx] = task; + } + // We have to drop the dependency map, in case modifications were made to those. + _dependency_graph = std::nullopt; + // We also have to drop _completed_tasks. + _completed_tasks = std::nullopt; } //////////////////////////////////////////////////////////////////////////////// From 76b04d2b4cfbbac15b266c45cf99c90e3d7f8ec9 Mon Sep 17 00:00:00 2001 From: ashprice Date: Mon, 15 Jun 2026 21:07:52 +0100 Subject: [PATCH 05/14] Use get_ref for UUIDs during add/modify/purge Signed-off-by: ashprice --- src/TDB2.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/TDB2.cpp b/src/TDB2.cpp index 426291117..60accf167 100644 --- a/src/TDB2.cpp +++ b/src/TDB2.cpp @@ -64,7 +64,7 @@ void TDB2::add(Task& task) { rust::Vec ops; maybe_add_undo_point(ops); - auto uuid = task.get("uuid"); + auto uuid = task.get_ref("uuid"); changes[uuid] = task; tc::Uuid tcuuid = tc::uuid_from_string(uuid); @@ -113,7 +113,7 @@ void TDB2::modify(Task& task) { // changes the user or hooks tried to apply to the "modified" attribute. task.setAsNow("modified"); task.validate(false); - auto uuid = task.get("uuid"); + auto uuid = task.get_ref("uuid"); rust::Vec ops; maybe_add_undo_point(ops); @@ -192,7 +192,7 @@ void TDB2::modify(Task& task) { //////////////////////////////////////////////////////////////////////////////// void TDB2::purge(Task& task) { - auto uuid = tc::uuid_from_string(task.get("uuid")); + auto uuid = tc::uuid_from_string(task.get_ref("uuid")); rust::Vec ops; auto maybe_tctask = replica()->get_task_data(uuid); if (maybe_tctask.is_some()) { From efece7a898f97021cdc0129538d1f6b45f01d199 Mon Sep 17 00:00:00 2001 From: ashprice Date: Mon, 15 Jun 2026 21:28:28 +0100 Subject: [PATCH 06/14] Code organization and comments. Signed-off-by: ashprice --- src/TDB2.cpp | 30 ++++++++++++++++++++---------- src/TDB2.h | 18 ++++++++++++++++-- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/TDB2.cpp b/src/TDB2.cpp index 60accf167..a277a75b2 100644 --- a/src/TDB2.cpp +++ b/src/TDB2.cpp @@ -43,6 +43,7 @@ #include bool TDB2::debug_mode = false; +// This functions main job is to set Task::is_blocked / Task::is_blocking flags. static void dependency_scan(std::vector&, const std::unordered_map&); // Build maps for dependency queries. @@ -179,6 +180,7 @@ void TDB2::modify(Task& task) { } // If the task stayed in the set, we can edit the vector in-place. + // This speeds up modifications a lot relative to reloading and parsing from rust. if (_pending_tasks) { auto idx = pending_index_of(uuid); if (idx != SIZE_MAX) @@ -186,7 +188,8 @@ void TDB2::modify(Task& task) { } // We have to drop the dependency map, in case modifications were made to those. _dependency_graph = std::nullopt; - // We also have to drop _completed_tasks. + // We also have to drop _completed_tasks. This probably isn't strictly necessary + // but it seems sensible for correctness reasons. _completed_tasks = std::nullopt; } @@ -258,6 +261,7 @@ int TDB2::latest_id() { } //////////////////////////////////////////////////////////////////////////////// +// Not cached: callers read this once per process, and it can be very large. const std::vector TDB2::all_tasks() { Timer timer; auto all_tctasks = replica()->all_task_data(); @@ -281,6 +285,8 @@ const std::vector TDB2::all_tasks() { } //////////////////////////////////////////////////////////////////////////////// +// Load and cache pending tasks. The first call will build the UUID index, +// after which it is reused. const std::vector& TDB2::pending_tasks() { if (!_pending_tasks) { Timer timer; @@ -312,6 +318,9 @@ const std::vector& TDB2::pending_tasks() { } //////////////////////////////////////////////////////////////////////////////// +// Load and cache all completed tests by scanning all tasks, +// and excluding those in the working set. We cache it to speed up those reports +// which involve completed tasks. const std::vector& TDB2::completed_tasks() { if (!_completed_tasks) { auto all_tctasks = replica()->all_task_data(); @@ -335,6 +344,7 @@ const std::vector& TDB2::completed_tasks() { ///////////////////////////////////////////////////////////////////////////////// // Build and return the dependency map for pending tasks. +// We invalidate it whenever pending_tasks may have changed. const DependencyGraph& TDB2::dependency_graph() { if (!_dependency_graph) { pending_tasks(); @@ -345,15 +355,6 @@ const DependencyGraph& TDB2::dependency_graph() { return *_dependency_graph; } -//////////////////////////////////////////////////////////////////////////////// -void TDB2::invalidate_cached_info() { - _pending_tasks = std::nullopt; - _completed_tasks = std::nullopt; - _working_set = std::nullopt; - _dependency_graph = std::nullopt; - _pending_index = std::nullopt; -} - ///////////////////////////////////////////////////////////////////////////////// // This builds and returns the UUID map if it is missing. const std::unordered_map& TDB2::pending_index() { @@ -374,6 +375,15 @@ size_t TDB2::pending_index_of(const std::string& uuid) { return it != idx.end() ? it->second : SIZE_MAX; } +//////////////////////////////////////////////////////////////////////////////// +void TDB2::invalidate_cached_info() { + _pending_tasks = std::nullopt; + _completed_tasks = std::nullopt; + _working_set = std::nullopt; + _dependency_graph = std::nullopt; + _pending_index = std::nullopt; +} + //////////////////////////////////////////////////////////////////////////////// // Locate task by ID, wherever it is. bool TDB2::get(int id, Task& task) { diff --git a/src/TDB2.h b/src/TDB2.h index aa48aa35d..1f4d1562c 100644 --- a/src/TDB2.h +++ b/src/TDB2.h @@ -43,6 +43,11 @@ struct DependencyGraph { std::unordered_map> dependents; std::unordered_map> dependencies; }; +// We adopt a caching strategy which lazily builds _pending_tasks, +// _completed_tasks, _working_set, _dependency_graph from the Rust +// replica whwnever its state may have changed. Callers receive const +// refs to avoid silent copies of the whole vectors. + // TDB2 Class represents all the files in the task database. class TDB2 { @@ -64,12 +69,14 @@ class TDB2 { // We don't cache the all_tasks vector because no command accesses it more than once. // Caching it would cause higher memory use and would require additional rewrites to make tests pass. const std::vector all_tasks(); + // pending and completed tasks are cached after first use. const std::vector& pending_tasks(); const std::vector& completed_tasks(); // dependency_graph is built on first use from pending_tasks() and reused. + // functions that use it are Task::getDependencyTasks(), getBlockedTasks() + // and urgency_inherit(). const DependencyGraph& dependency_graph(); - // Index of pending tasks by UUID. - size_t pending_index_of(const std::string& uuid); + bool get(int, Task&); bool get(const std::string&, Task&); bool has(const std::string&); @@ -80,6 +87,10 @@ class TDB2 { std::string uuid(int); int id(const std::string&); + // Index of pending tasks by UUID. + // Used by get() and modify(). + size_t pending_index_of(const std::string& uuid); + int num_local_changes(); int num_reverts_possible(); @@ -89,12 +100,15 @@ class TDB2 { std::optional> _replica; // Cached information from the replica + // All caches are invalidated when state may have changed + // in the replica. std::optional> _working_set; std::optional> _pending_tasks; std::optional> _completed_tasks; // Dependency cache. std::optional _dependency_graph; // Lazily cache UUIDs within the pending set. + // Avoids scans of the vectors with get/modify.. std::optional> _pending_index; void invalidate_cached_info(); From 7738b45b977ae64228ab194a986aab8f12da3703 Mon Sep 17 00:00:00 2001 From: ashprice Date: Wed, 17 Jun 2026 03:08:49 +0100 Subject: [PATCH 07/14] Use the _pending_tasks cache in children(); use get_ref() in siblings()/children() Signed-off-by: ashprice --- src/TDB2.cpp | 40 ++++++++-------------------------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/src/TDB2.cpp b/src/TDB2.cpp index 7fa7a4759..18047f919 100644 --- a/src/TDB2.cpp +++ b/src/TDB2.cpp @@ -448,9 +448,9 @@ bool TDB2::has(const std::string& uuid) { const std::vector TDB2::siblings(Task& task) { std::vector results; if (task.has("parent")) { - std::string parent = task.get("parent"); + const auto& parent = task.get_ref("parent"); - for (auto& i : this->pending_tasks()) { + for (const auto& i : pending_tasks()) { // Do not include self in results. if (i.id != task.id) { // Do not include completed or deleted tasks. @@ -468,39 +468,15 @@ const std::vector TDB2::siblings(Task& task) { } //////////////////////////////////////////////////////////////////////////////// +// Return the child tasks of a parent. Uses the _pending_tasks cache, to avoid +// having to fetch this info from the Rust replica. const std::vector TDB2::children(Task& parent) { - // scan _pending_ tasks for those with `parent` equal to this task std::vector results; - std::string this_uuid = parent.get("uuid"); + const auto& this_uuid = parent.get_ref("uuid"); - auto& ws = working_set(); - size_t end_idx = ws->largest_index(); - - for (size_t i = 0; i <= end_idx; i++) { - auto uuid = ws->by_index(i); - if (uuid.is_nil()) { - continue; - } - - // skip self-references - if (uuid.to_string() == this_uuid) { - continue; - } - - auto task_opt = replica()->get_task_data(uuid); - if (task_opt.is_none()) { - continue; - } - auto task = task_opt.take(); - - std::string parent_uuid; - if (!task->get("parent", parent_uuid)) { - continue; - } - - if (parent_uuid == this_uuid) { - results.push_back(Task(std::move(task))); - } + for (const auto& i : pending_tasks()) { + if (i.get_ref("uuid") == this_uuid) continue; + if (i.has("parent") && i.get("parent") == this_uuid) results.push_back(i); } return results; } From 08a2482fea6ff15a6baae4dad6eee6793a972cbb Mon Sep 17 00:00:00 2001 From: ashprice Date: Mon, 15 Jun 2026 22:06:43 +0100 Subject: [PATCH 08/14] Code organization and comments. Signed-off-by: ashprice Clang-format Signed-off-by: ashprice --- src/TDB2.cpp | 31 ++++++++++++++----------------- src/TDB2.h | 8 ++++---- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/src/TDB2.cpp b/src/TDB2.cpp index a277a75b2..7fa7a4759 100644 --- a/src/TDB2.cpp +++ b/src/TDB2.cpp @@ -47,7 +47,8 @@ bool TDB2::debug_mode = false; static void dependency_scan(std::vector&, const std::unordered_map&); // Build maps for dependency queries. -static DependencyGraph build_dependency_graph(const std::vector&, const std::unordered_map&); +static DependencyGraph build_dependency_graph(const std::vector&, + const std::unordered_map&); //////////////////////////////////////////////////////////////////////////////// void TDB2::open_replica(const std::string& location, bool create_if_missing, bool read_write) { @@ -183,8 +184,7 @@ void TDB2::modify(Task& task) { // This speeds up modifications a lot relative to reloading and parsing from rust. if (_pending_tasks) { auto idx = pending_index_of(uuid); - if (idx != SIZE_MAX) - (*_pending_tasks)[idx] = task; + if (idx != SIZE_MAX) (*_pending_tasks)[idx] = task; } // We have to drop the dependency map, in case modifications were made to those. _dependency_graph = std::nullopt; @@ -275,8 +275,7 @@ const std::vector TDB2::all_tasks() { // inside all_tasks. std::unordered_map all_index; all_index.reserve(all.size()); - for (size_t i = 0; i < all.size(); ++i) - all_index[all[i].get_ref("uuid")] = i; + for (size_t i = 0; i < all.size(); ++i) all_index[all[i].get_ref("uuid")] = i; dependency_scan(all, all_index); @@ -530,18 +529,18 @@ int TDB2::num_reverts_possible() { return (int)replica()->num_undo_points(); } //////////////////////////////////////////////////////////////////////////////// // Set Task::is_blocked / Task::is_blocking flags using the pre-built UUID map -static void dependency_scan(std::vector& tasks, const std::unordered_map& uuid_index) { - for (size_t i = 0; i< tasks.size(); ++i) { +static void dependency_scan(std::vector& tasks, + const std::unordered_map& uuid_index) { + for (size_t i = 0; i < tasks.size(); ++i) { auto lstatus = tasks[i].getStatus(); for (const auto& dep : tasks[i].getDependencyUUIDs()) { auto it = uuid_index.find(dep); - if (it == uuid_index.end()) - continue; + if (it == uuid_index.end()) continue; size_t j = it->second; auto rstatus = tasks[j].getStatus(); - if (lstatus != Task::completed && lstatus != Task::deleted && - rstatus != Task::completed && rstatus != Task::deleted) { + if (lstatus != Task::completed && lstatus != Task::deleted && rstatus != Task::completed && + rstatus != Task::deleted) { tasks[i].is_blocked = true; tasks[j].is_blocking = true; } @@ -551,21 +550,19 @@ static void dependency_scan(std::vector& tasks, const std::unordered_map& tasks, - const std::unordered_map& uuid_index) { +static DependencyGraph build_dependency_graph( + const std::vector& tasks, const std::unordered_map& uuid_index) { DependencyGraph graph; for (size_t i = 0; i < tasks.size(); ++i) { const auto& deps = tasks[i].getDependencyUUIDs(); - if (deps.empty()) - continue; + if (deps.empty()) continue; const auto& uuid = tasks[i].get_ref("uuid"); for (const auto& dep : deps) { auto it = uuid_index.find(dep); - if (it == uuid_index.end()) - continue; + if (it == uuid_index.end()) continue; graph.dependencies[uuid].push_back(it->second); graph.dependents[dep].push_back(i); diff --git a/src/TDB2.h b/src/TDB2.h index 1f4d1562c..afb78738d 100644 --- a/src/TDB2.h +++ b/src/TDB2.h @@ -40,15 +40,14 @@ // Adjacency maps for the dependency graph. // Index refers to the pending_tasks() vector. struct DependencyGraph { - std::unordered_map> dependents; - std::unordered_map> dependencies; + std::unordered_map> dependents; + std::unordered_map> dependencies; }; // We adopt a caching strategy which lazily builds _pending_tasks, // _completed_tasks, _working_set, _dependency_graph from the Rust // replica whwnever its state may have changed. Callers receive const // refs to avoid silent copies of the whole vectors. - // TDB2 Class represents all the files in the task database. class TDB2 { public: @@ -67,7 +66,8 @@ class TDB2 { // Generalized task accessors. // We don't cache the all_tasks vector because no command accesses it more than once. - // Caching it would cause higher memory use and would require additional rewrites to make tests pass. + // Caching it would cause higher memory use and would require additional rewrites to make tests + // pass. const std::vector all_tasks(); // pending and completed tasks are cached after first use. const std::vector& pending_tasks(); From 0c62e85bc3fe59df03e403be624f5acf1eee1bba Mon Sep 17 00:00:00 2001 From: ashprice Date: Tue, 23 Jun 2026 22:55:04 +0100 Subject: [PATCH 09/14] Replace SIZE_MAX with Task* ptr for clearer semantics. Signed-off-by: ashprice --- src/TDB2.cpp | 25 ++++++++++++++----------- src/TDB2.h | 5 +++-- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/TDB2.cpp b/src/TDB2.cpp index 18047f919..a84f890c4 100644 --- a/src/TDB2.cpp +++ b/src/TDB2.cpp @@ -183,8 +183,9 @@ void TDB2::modify(Task& task) { // If the task stayed in the set, we can edit the vector in-place. // This speeds up modifications a lot relative to reloading and parsing from rust. if (_pending_tasks) { - auto idx = pending_index_of(uuid); - if (idx != SIZE_MAX) (*_pending_tasks)[idx] = task; + auto* pt = find_pending(uuid); + if (pt) + *pt = task; } // We have to drop the dependency map, in case modifications were made to those. _dependency_graph = std::nullopt; @@ -367,11 +368,13 @@ const std::unordered_map& TDB2::pending_index() { return *_pending_index; } -// Finds the UUID in the index. Returns SIZE_MAX if it isn't present. -size_t TDB2::pending_index_of(const std::string& uuid) { +// Finds the UUID in the index. Returns nullptr if the task is not in the pending +// set. +Task* TDB2::find_pending(const std::string& uuid) { auto& idx = pending_index(); auto it = idx.find(uuid); - return it != idx.end() ? it->second : SIZE_MAX; + if (it != idx.end()) return &(*_pending_tasks)[it->second]; + return nullptr; } //////////////////////////////////////////////////////////////////////////////// @@ -393,9 +396,9 @@ bool TDB2::get(int id, Task& task) { // Load index of pending tasks. pending_tasks(); // Lookup the UUID in the index instead of scanning the vector. - auto idx = pending_index_of(uuid); - if (idx != SIZE_MAX) { - task = (*_pending_tasks)[idx]; + auto* pt = find_pending(uuid); + if (pt) { + task = *pt; return true; } } @@ -409,9 +412,9 @@ bool TDB2::get(const std::string& uuid, Task& task) { pending_tasks(); // Try to match exact UUID within the index. - auto idx = pending_index_of(uuid); - if (idx != SIZE_MAX) { - task = (*_pending_tasks)[idx]; + auto* pt = find_pending(uuid); + if (pt) { + task = *pt; return true; } diff --git a/src/TDB2.h b/src/TDB2.h index afb78738d..a3d671c59 100644 --- a/src/TDB2.h +++ b/src/TDB2.h @@ -87,9 +87,10 @@ class TDB2 { std::string uuid(int); int id(const std::string&); - // Index of pending tasks by UUID. + // Index of pending tasks by UUID. The ptr is stable until the + // cache is invalidated. // Used by get() and modify(). - size_t pending_index_of(const std::string& uuid); + Task* find_pending(const std::string& uuid); int num_local_changes(); int num_reverts_possible(); From c5abceaa9d38dcb1a286228ae3d62487816a14bc Mon Sep 17 00:00:00 2001 From: ashprice Date: Tue, 23 Jun 2026 23:21:12 +0100 Subject: [PATCH 10/14] Fixed a bug which would lead to incorrect +BLOCKING reports. Signed-off-by: ashprice --- src/TDB2.cpp | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/TDB2.cpp b/src/TDB2.cpp index a84f890c4..0b68fbf63 100644 --- a/src/TDB2.cpp +++ b/src/TDB2.cpp @@ -182,13 +182,22 @@ void TDB2::modify(Task& task) { // If the task stayed in the set, we can edit the vector in-place. // This speeds up modifications a lot relative to reloading and parsing from rust. + bool deps_changed = false; if (_pending_tasks) { auto* pt = find_pending(uuid); - if (pt) + if (pt) { + auto old_deps = pt->getDependencyUUIDs(); *pt = task; + auto new_deps = task.getDependencyUUIDs(); + if (old_deps != new_deps) { + deps_changed = true; + dependency_scan(*_pending_tasks, pending_index()); + } + } } // We have to drop the dependency map, in case modifications were made to those. - _dependency_graph = std::nullopt; + // We only drop it if they actually changed. + if (deps_changed) _dependency_graph = std::nullopt; // We also have to drop _completed_tasks. This probably isn't strictly necessary // but it seems sensible for correctness reasons. _completed_tasks = std::nullopt; @@ -479,7 +488,7 @@ const std::vector TDB2::children(Task& parent) { for (const auto& i : pending_tasks()) { if (i.get_ref("uuid") == this_uuid) continue; - if (i.has("parent") && i.get("parent") == this_uuid) results.push_back(i); + if (i.has("parent") && i.get_ref("parent") == this_uuid) results.push_back(i); } return results; } @@ -510,6 +519,16 @@ int TDB2::num_reverts_possible() { return (int)replica()->num_undo_points(); } // Set Task::is_blocked / Task::is_blocking flags using the pre-built UUID map static void dependency_scan(std::vector& tasks, const std::unordered_map& uuid_index) { + // Reset all flags first. This is for safety reasons - if we don't do this + // dependency_scan() only sets them to true, so it can stay true (within the cache) + // even after we have changed a task's dependencies after a modify when + // dependency_scan() runs the second time - the flag will still be set to true! + // In many cases, this isn't user-visible - it's only visible in reports + // or filters that explicitly filter for +BLOCKING. + for (auto& task : tasks) { + task.is_blocking = false; + task.is_blocked = false; + } for (size_t i = 0; i < tasks.size(); ++i) { auto lstatus = tasks[i].getStatus(); for (const auto& dep : tasks[i].getDependencyUUIDs()) { From ec2322898fcfb1278ae1b9a5801db604afa228e6 Mon Sep 17 00:00:00 2001 From: ashprice Date: Tue, 23 Jun 2026 23:36:31 +0100 Subject: [PATCH 11/14] Add cache invalidation if a modified pending task is no longer found. Co-authored-by: Dustin J. Mitchell --- src/TDB2.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TDB2.cpp b/src/TDB2.cpp index 0b68fbf63..a26d4924d 100644 --- a/src/TDB2.cpp +++ b/src/TDB2.cpp @@ -175,7 +175,7 @@ void TDB2::modify(Task& task) { // If the task entered or left the pending set, we must invalidate the cache. bool was_pending = found_original && (original.getStatus() == Task::pending); bool now_pending = task.getStatus() == Task::pending; - if (was_pending != now_pending) { + if (was_pending != now_pending || !found_original) { invalidate_cached_info(); return; } From 60c991fa821b789d4d697a08025620205df27903 Mon Sep 17 00:00:00 2001 From: ashprice Date: Wed, 17 Jun 2026 03:08:49 +0100 Subject: [PATCH 12/14] Use the _pending_tasks cache in children(); use get_ref() in siblings()/children() Signed-off-by: ashprice --- src/TDB2.cpp | 42 +++++++++--------------------------------- 1 file changed, 9 insertions(+), 33 deletions(-) diff --git a/src/TDB2.cpp b/src/TDB2.cpp index 7fa7a4759..79d71968f 100644 --- a/src/TDB2.cpp +++ b/src/TDB2.cpp @@ -448,15 +448,15 @@ bool TDB2::has(const std::string& uuid) { const std::vector TDB2::siblings(Task& task) { std::vector results; if (task.has("parent")) { - std::string parent = task.get("parent"); + const auto& parent = task.get_ref("parent"); - for (auto& i : this->pending_tasks()) { + for (const auto& i : pending_tasks()) { // Do not include self in results. if (i.id != task.id) { // Do not include completed or deleted tasks. if (i.getStatus() != Task::completed && i.getStatus() != Task::deleted) { // If task has the same parent, it is a sibling. - if (i.has("parent") && i.get("parent") == parent) { + if (i.has("parent") && i.get_ref("parent") == parent) { results.push_back(i); } } @@ -468,39 +468,15 @@ const std::vector TDB2::siblings(Task& task) { } //////////////////////////////////////////////////////////////////////////////// +// Return the child tasks of a parent. Uses the _pending_tasks cache, to avoid +// having to fetch this info from the Rust replica. const std::vector TDB2::children(Task& parent) { - // scan _pending_ tasks for those with `parent` equal to this task std::vector results; - std::string this_uuid = parent.get("uuid"); + const auto& this_uuid = parent.get_ref("uuid"); - auto& ws = working_set(); - size_t end_idx = ws->largest_index(); - - for (size_t i = 0; i <= end_idx; i++) { - auto uuid = ws->by_index(i); - if (uuid.is_nil()) { - continue; - } - - // skip self-references - if (uuid.to_string() == this_uuid) { - continue; - } - - auto task_opt = replica()->get_task_data(uuid); - if (task_opt.is_none()) { - continue; - } - auto task = task_opt.take(); - - std::string parent_uuid; - if (!task->get("parent", parent_uuid)) { - continue; - } - - if (parent_uuid == this_uuid) { - results.push_back(Task(std::move(task))); - } + for (const auto& i : pending_tasks()) { + if (i.get_ref("uuid") == this_uuid) continue; + if (i.has("parent") && i.get_ref("parent") == this_uuid) results.push_back(i); } return results; } From a7585f820a0adbcc13bff63a83616ec57d52d933 Mon Sep 17 00:00:00 2001 From: ashprice Date: Tue, 23 Jun 2026 22:55:04 +0100 Subject: [PATCH 13/14] Replace SIZE_MAX with Task* ptr for clearer semantics. Signed-off-by: ashprice --- src/TDB2.cpp | 25 ++++++++++++++----------- src/TDB2.h | 5 +++-- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/TDB2.cpp b/src/TDB2.cpp index 79d71968f..e8ed5d074 100644 --- a/src/TDB2.cpp +++ b/src/TDB2.cpp @@ -183,8 +183,9 @@ void TDB2::modify(Task& task) { // If the task stayed in the set, we can edit the vector in-place. // This speeds up modifications a lot relative to reloading and parsing from rust. if (_pending_tasks) { - auto idx = pending_index_of(uuid); - if (idx != SIZE_MAX) (*_pending_tasks)[idx] = task; + auto* pt = find_pending(uuid); + if (pt) + *pt = task; } // We have to drop the dependency map, in case modifications were made to those. _dependency_graph = std::nullopt; @@ -367,11 +368,13 @@ const std::unordered_map& TDB2::pending_index() { return *_pending_index; } -// Finds the UUID in the index. Returns SIZE_MAX if it isn't present. -size_t TDB2::pending_index_of(const std::string& uuid) { +// Finds the UUID in the index. Returns nullptr if the task is not in the pending +// set. +Task* TDB2::find_pending(const std::string& uuid) { auto& idx = pending_index(); auto it = idx.find(uuid); - return it != idx.end() ? it->second : SIZE_MAX; + if (it != idx.end()) return &(*_pending_tasks)[it->second]; + return nullptr; } //////////////////////////////////////////////////////////////////////////////// @@ -393,9 +396,9 @@ bool TDB2::get(int id, Task& task) { // Load index of pending tasks. pending_tasks(); // Lookup the UUID in the index instead of scanning the vector. - auto idx = pending_index_of(uuid); - if (idx != SIZE_MAX) { - task = (*_pending_tasks)[idx]; + auto* pt = find_pending(uuid); + if (pt) { + task = *pt; return true; } } @@ -409,9 +412,9 @@ bool TDB2::get(const std::string& uuid, Task& task) { pending_tasks(); // Try to match exact UUID within the index. - auto idx = pending_index_of(uuid); - if (idx != SIZE_MAX) { - task = (*_pending_tasks)[idx]; + auto* pt = find_pending(uuid); + if (pt) { + task = *pt; return true; } diff --git a/src/TDB2.h b/src/TDB2.h index afb78738d..a3d671c59 100644 --- a/src/TDB2.h +++ b/src/TDB2.h @@ -87,9 +87,10 @@ class TDB2 { std::string uuid(int); int id(const std::string&); - // Index of pending tasks by UUID. + // Index of pending tasks by UUID. The ptr is stable until the + // cache is invalidated. // Used by get() and modify(). - size_t pending_index_of(const std::string& uuid); + Task* find_pending(const std::string& uuid); int num_local_changes(); int num_reverts_possible(); From 058f3df6d97f695c6dc196f2ce72f1c0839a5495 Mon Sep 17 00:00:00 2001 From: ashprice Date: Tue, 23 Jun 2026 23:49:08 +0100 Subject: [PATCH 14/14] Fixed a bug which would lead to incorrect +BLOCKING reports. Signed-off-by: ashprice --- src/TDB2.cpp | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/TDB2.cpp b/src/TDB2.cpp index e8ed5d074..afb566b0d 100644 --- a/src/TDB2.cpp +++ b/src/TDB2.cpp @@ -182,13 +182,22 @@ void TDB2::modify(Task& task) { // If the task stayed in the set, we can edit the vector in-place. // This speeds up modifications a lot relative to reloading and parsing from rust. + bool deps_changed = false; if (_pending_tasks) { auto* pt = find_pending(uuid); - if (pt) + if (pt) { + auto old_deps = pt->getDependencyUUIDs(); *pt = task; + auto new_deps = task.getDependencyUUIDs(); + if (old_deps != new_deps) { + deps_changed = true; + dependency_scan(*_pending_tasks, pending_index()); + } + } } // We have to drop the dependency map, in case modifications were made to those. - _dependency_graph = std::nullopt; + // We only drop it if they actually changed. + if (deps_changed) _dependency_graph = std::nullopt; // We also have to drop _completed_tasks. This probably isn't strictly necessary // but it seems sensible for correctness reasons. _completed_tasks = std::nullopt; @@ -510,6 +519,16 @@ int TDB2::num_reverts_possible() { return (int)replica()->num_undo_points(); } // Set Task::is_blocked / Task::is_blocking flags using the pre-built UUID map static void dependency_scan(std::vector& tasks, const std::unordered_map& uuid_index) { + // Reset all flags first. This is for safety reasons - if we don't do this + // dependency_scan() only sets them to true, so it can stay true (within the cache) + // even after we have changed a task's dependencies after a modify when + // dependency_scan() runs the second time - the flag will still be set to true! + // In many cases, this isn't user-visible - it's only visible in reports + // or filters that explicitly filter for +BLOCKING. + for (auto& task : tasks) { + task.is_blocking = false; + task.is_blocked = false; + } for (size_t i = 0; i < tasks.size(); ++i) { auto lstatus = tasks[i].getStatus(); for (const auto& dep : tasks[i].getDependencyUUIDs()) {