examples: add dynamic data project

Author: Neiko2002

cpp/bench/include/build.h

@@ -160,17 +160,25 @@ inline void create_graph(const deglib::StaticFeatureRepository& repository,
 
     if (data_stream_type == DataStreamType::AddHalfRemoveAndAddOneAtATime) {
         auto base_size_half = base_size / 2;
-        auto base_size_fourth = base_size / 4;
-        for (uint32_t i = 0; i < base_size_fourth; i++) {
+        auto base_size_quarter = base_size / 4;
+        // First loop: add base_size_quarter pairs from [0, base_size_quarter) and [base_size_half, base_size_half + base_size_quarter)
+        for (uint32_t i = 0; i < base_size_quarter; i++) {
             addEntry(0 + i);
             addEntry(base_size_half + i);
         }
-        for (uint32_t i = 0; i < base_size_fourth; i++) {
-            addEntry(base_size_fourth + i);
-            addEntry(base_size_half + base_size_fourth + i);
+        // Second loop: add base_size_quarter pairs from [base_size_quarter, base_size_half) and remove the same number
+        // to keep exactly base_size_half vertices in the graph
+        for (uint32_t i = 0; i < base_size_quarter; i++) {
+            addEntry(base_size_quarter + i);
+            addEntry(base_size_half + base_size_quarter + i);
             builder.removeEntry(base_size_half + (i * 2) + 0);
             builder.removeEntry(base_size_half + (i * 2) + 1);
         }
+        // If base_size is not divisible by 4, add the remaining entries to reach exactly base_size_half vertices
+        auto remainder = base_size_half - (base_size_quarter * 2);
+        for (uint32_t i = 0; i < remainder; i++) {
+            addEntry(base_size_quarter * 2 + i);
+        }
     } else {
         base_size /= (data_stream_type == DataStreamType::AddHalf) ? 2 : 1;
         for (uint32_t i = 0; i < base_size; i++) addEntry(i);

cpp/bench/src/bench_dynamic_data.cpp

@@ -25,7 +25,6 @@ struct DynamicConfig {
     uint8_t k_ext = 60;
     float eps_ext = 0.1f;
     deglib::builder::OptimizationTarget lid = deglib::builder::OptimizationTarget::StreamingData;
-    uint32_t build_threads = std::thread::hardware_concurrency() / 2;
     uint32_t anns_k = 100;
     uint32_t anns_repeat = 1;
     uint32_t anns_threads = 1;
@@ -91,7 +90,6 @@ void print_help(const char* program_name) {
     log("Options:\n");
     log("  --force-rebuild       Force rebuilding the graphs even if graph files already exist\n");
     log("  --instruction <inst>  Select distance instruction set (auto, avx512, avx2, scalar)\n");
-    log("  --threads <count>     Number of threads used for building the graph (default: hardware_concurrency / 2)\n");
     log("  --help, -h            Display this detailed help message\n\n");
     log("Data Path:\n");
     log("  Data is loaded from / saved to DATA_PATH defined at build time (e.g. \"{}\")\n", DATA_PATH);
@@ -106,10 +104,9 @@ static deglib::cpu::InstructionSet parse_instruction_set(const std::string& str)
     return deglib::cpu::InstructionSet::Auto;
 }
 
-void run_dynamic_benchmark(const DatasetName& ds_name, const std::filesystem::path& data_path, bool force_rebuild, deglib::cpu::InstructionSet instruction, uint32_t build_threads) {
+void run_dynamic_benchmark(const DatasetName& ds_name, const std::filesystem::path& data_path, bool force_rebuild, deglib::cpu::InstructionSet instruction) {
     Dataset ds(ds_name, data_path);
     auto config = get_dataset_config(ds_name);
-    config.build_threads = build_threads;
 
     log("\n=== Benchmarking Dynamic Data Streams for Dataset: {} ===\n", ds.name());
 
@@ -149,7 +146,7 @@ void run_dynamic_benchmark(const DatasetName& ds_name, const std::filesystem::pa
                          config.k_ext,
                          config.eps_ext,
                          0, 0, 0,
-                         config.build_threads,
+                         1,
                          true,
                          ds.info().scale,
                          false,
@@ -190,10 +187,9 @@ void run_dynamic_benchmark(const DatasetName& ds_name, const std::filesystem::pa
 
 int main(int argc, char* argv[]) {
     const auto data_path = std::filesystem::path(DATA_PATH);
-    DatasetName ds_name = DatasetName::SIFT1M;
+    DatasetName ds_name = DatasetName::AUDIO;
     bool force_rebuild = false;
-    deglib::cpu::InstructionSet instruction = deglib::cpu::InstructionSet::Auto;
-    uint32_t build_threads = std::thread::hardware_concurrency() / 2;
+    deglib::cpu::InstructionSet instruction = deglib::cpu::InstructionSet::AVX2;
 
     for (int i = 1; i < argc; ++i) {
         std::string arg = argv[i];
@@ -204,8 +200,6 @@ int main(int argc, char* argv[]) {
             force_rebuild = true;
         } else if (arg == "--instruction" && i + 1 < argc) {
             instruction = parse_instruction_set(argv[++i]);
-        } else if (arg == "--threads" && i + 1 < argc) {
-            build_threads = (uint32_t)std::stoul(argv[++i]);
         } else {
             auto parsed = DatasetName::from_string(arg);
             if (parsed.is_valid()) {
@@ -228,7 +222,7 @@ int main(int argc, char* argv[]) {
     }
 
     for (const auto& current_ds : datasets_to_run) {
-        run_dynamic_benchmark(current_ds, data_path, force_rebuild, instruction, build_threads);
+        run_dynamic_benchmark(current_ds, data_path, force_rebuild, instruction);
     }
 
     log("\nDynamic Benchmark Finished Successfully.\n");

examples/dynamic_data/README.md

@@ -0,0 +1,68 @@
+# DEG Dynamic Data Benchmark (Python)
+
+Python mirror of `cpp/bench/src/bench_dynamic_data.cpp`.
+
+Builds and evaluates DEG graphs under three dynamic data streaming conditions and measures ANNS recall vs. QPS performance against the **half-dataset ground truth** (first half of base vectors).
+
+## Tested Data Stream Patterns
+
+| Stream Type | Description |
+|---|---|
+| `AddHalf` | Insert only the first half of base vectors |
+| `AddHalfRemoveAndAddOneAtATime` | Interleaved insertion and deletion operations |
+| `AddAllRemoveHalf` | Insert all vectors, then remove the second half |
+
+All three use `OptimizationTarget.StreamingData`.
+
+## Usage
+
+```bash
+uv run python main.py [dataset] [options]
+```
+
+### Datasets
+- `sift1m` — SIFT1M (1M vectors, 128D, default)
+- `deep1m` — DEEP1M (1M vectors, 96D)
+- `glove` / `glove-100` — GloVe (1.18M vectors, 100D)
+- `audio` — Audio (53.3k vectors, 192D)
+- `enron` — Enron (94.9k vectors, 1369D)
+- `all` — Run all datasets sequentially
+
+### Options
+
+| Option | Description |
+|---|---|
+| `--graph-dir <path>` | Directory to save/load graph files. Filenames follow C++ naming: `{dims}D_K{k}_{stream_type}.deg` |
+| `--force-rebuild` | Rebuild graphs even if files already exist |
+| `--instruction <inst>` | Distance instruction set: `auto`, `avx512`, `avx2`, `scalar` |
+| `--threads <n>` | Build threads (default: 1) |
+| `--cache-dir <path>` | Dataset cache directory (default: `~/.cache/deg_datasets`) |
+| `--no-show` | Do not display interactive plots |
+| `--max-base-vecs <n>` | Limit base vectors for quick testing |
+
+### Examples
+
+```bash
+# Quick test with audio dataset
+uv run python main.py audio --max-base-vecs 5000 --no-show
+
+# Full SIFT1M benchmark, saving graphs to disk
+uv run python main.py sift1m --graph-dir /data/deg_graphs/sift1m/dynamic
+
+# All datasets
+uv run python main.py all --graph-dir /data/deg_graphs --threads 4
+```
+
+## Setup
+
+```bash
+uv sync
+```
+
+## Key Differences from `static_data`
+
+- Iterates over 3 `DataStreamType`s instead of a single `AddAll`
+- Uses `StreamingData` optimization target for all datasets
+- ANNS test uses **half-dataset ground truth** (queries evaluated against first `base_count/2` vectors)
+- No exploration test
+- Graph files named `{dims}D_K{k}_{stream_type}.deg` (C++ compatible)