Skip to content

Commit a5f682e

Browse files
committed
examples: add search example project
1 parent ebe5bf0 commit a5f682e

24 files changed

Lines changed: 2327 additions & 1020 deletions

.gitignore

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,3 +57,11 @@ Thumbs.db
5757

5858
# Agent rules
5959
.agents/
60+
61+
# Python generated files & virtual environments
62+
__pycache__/
63+
*.py[cod]
64+
*$py.class
65+
.venv/
66+
*.png
67+

cpp/deglib/include/graph/sizebounded_graph.h

Lines changed: 9 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -681,44 +681,13 @@ class SizeBoundedGraph : public deglib::graph::MutableGraph {
681681

682682
// add the entry vertex index to the vertices which gets checked next and ignore it for further checks
683683
checked_ids[entry_vertex_index] = checked_ids_tag;
684-
next_vertices.emplace(entry_vertex_index, 0);
684+
next_vertices.emplace(entry_vertex_index, 0.0f);
685685
if(include_entry)
686-
results.emplace(entry_vertex_index, 0);
686+
results.emplace(entry_vertex_index, 0.0f);
687687
const auto query = this->feature_by_index(entry_vertex_index);
688688

689-
// add the neighbors of the entry vertex to the next vertices queue
690-
{
691-
const auto neighbor_indices = this->neighbors_by_index(entry_vertex_index);
692-
const auto neighbor_weights = this->weights_by_index(entry_vertex_index);
693-
memory::prefetch(reinterpret_cast<const char*>(neighbor_indices));
694-
memory::prefetch(reinterpret_cast<const char*>(neighbor_weights));
695-
for (uint8_t i = 0; i < this->edges_per_vertex_; i++) {
696-
const auto neighbor_index = neighbor_indices[i];
697-
698-
if (checked_ids[neighbor_index] != checked_ids_tag) {
699-
checked_ids[neighbor_index] = checked_ids_tag;
700-
701-
const auto neighbor_distance = neighbor_weights[i];
702-
next_vertices.emplace(neighbor_index, neighbor_distance);
703-
results.emplace(neighbor_index, neighbor_distance);
704-
705-
// update the search radius
706-
if (results.size() > k)
707-
results.pop();
708-
709-
// early stop after to many computations
710-
if(max_distance_computation_count > 0 && ++distance_computation_count >= max_distance_computation_count)
711-
return results;
712-
}
713-
}
714-
}
715-
716689
// search radius
717-
auto radius = results.top().getDistance();
718-
719-
// experimental: eps replacement parameter
720-
const auto eps = (max_distance_computation_count > 0) ? std::log10(float(max_distance_computation_count)/k) : 0.0f;
721-
auto exploration_radius = radius * ((radius < 0) ? (1 - eps) : (1 + eps));
690+
auto radius = std::numeric_limits<float>::max();
722691

723692
// iterate as long as good elements are in the next_vertices queue and max_calcs is not yet reached
724693
auto good_neighbors = std::array<uint32_t, 256>(); // this limits the neighbor count to 256 using Variable Length Array wrapped in a macro
@@ -728,27 +697,13 @@ class SizeBoundedGraph : public deglib::graph::MutableGraph {
728697
const auto next_vertex = next_vertices.top();
729698
next_vertices.pop();
730699

731-
// if no weight of this neighbor would survive the distance estimation check, stop here
732-
if (next_vertex.getDistance() > exploration_radius)
733-
break;
734-
735700
uint8_t good_neighbor_count = 0;
736-
{
737-
const auto neighbor_indices = this->neighbors_by_index(next_vertex.getInternalIndex());
738-
const auto neighbor_weights = this->weights_by_index(next_vertex.getInternalIndex());
739-
memory::prefetch(reinterpret_cast<const char*>(neighbor_indices));
740-
memory::prefetch(reinterpret_cast<const char*>(neighbor_weights));
741-
for (uint8_t i = 0; i < this->edges_per_vertex_; i++) {
742-
const auto neighbor_index = neighbor_indices[i];
743-
744-
if (checked_ids[neighbor_index] != checked_ids_tag) {
745-
checked_ids[neighbor_index] = checked_ids_tag;
746-
747-
// distance estimation check: allow only edges with a worst case distance < r
748-
// this produces slighly better results and brings the sizebound graph on par with the readonly graph when comparing speed vs quality
749-
if(next_vertex.getDistance() + neighbor_weights[i] < exploration_radius)
750-
good_neighbors[good_neighbor_count++] = neighbor_index;
751-
}
701+
const auto neighbor_indices = this->neighbors_by_index(next_vertex.getInternalIndex());
702+
for (uint8_t i = 0; i < this->edges_per_vertex_; i++) {
703+
const auto neighbor_index = neighbor_indices[i];
704+
if (checked_ids[neighbor_index] != checked_ids_tag) {
705+
checked_ids[neighbor_index] = checked_ids_tag;
706+
good_neighbors[good_neighbor_count++] = neighbor_index;
752707
}
753708
}
754709

@@ -775,7 +730,6 @@ class SizeBoundedGraph : public deglib::graph::MutableGraph {
775730
if (results.size() > k) {
776731
results.pop();
777732
radius = results.top().getDistance();
778-
exploration_radius = radius * ((radius < 0) ? (1 - eps) : (1 + eps));
779733
}
780734
}
781735

cpp/test/CMakeLists.txt

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,10 @@ set(TEST_MAIN ${CMAKE_CURRENT_SOURCE_DIR}/test_main.cpp)
1010

1111
# Static library for GoogleTest to avoid compiling gmock-gtest-all.cc multiple times
1212
add_library(gtest STATIC ${GTEST_SOURCE})
13-
target_include_directories(gtest PUBLIC ${GTEST_INCLUDE})
13+
target_include_directories(gtest SYSTEM PUBLIC ${GTEST_INCLUDE})
14+
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU")
15+
target_compile_options(gtest PRIVATE -Wno-character-conversion)
16+
endif()
1417

1518
# Helper macro: creates a test executable with shared gtest + deglib setup
1619
macro(add_deglib_test name source)

docs/tutorials/installation.rst

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ This can be useful if you want to:
2929
python -m venv venv && . venv/bin/activate
3030
3131
# install build dependencies
32-
pip install setuptools==83.0.0 pybind11==3.0.4 build
32+
pip install setuptools==83.0.0 pybind11==3.0.4 build==1.5.0
3333
python setup.py copy_build_files # copy c++ library to ./lib/
3434
3535
# install

examples/.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
__pycache__/
2+
*.py[cod]
3+
.venv/
4+
*.png

examples/README.md

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# DEG Examples
2+
3+
This directory contains standalone Python projects demonstrating various features and benchmark workflows of the Dynamic Exploration Graph (DEG).
4+
5+
Each directory inside `examples/` is a self-contained [`uv`](https://docs.astral.sh/uv/) project with its own `pyproject.toml`.
6+
7+
## Prerequisites & UV Setup
8+
9+
We use [`uv`](https://docs.astral.sh/uv/) for fast Python environment and dependency management.
10+
11+
### 1. Install `uv`
12+
13+
- **macOS / Linux**:
14+
```bash
15+
curl -LsSf https://astral.sh/uv/install.sh | sh
16+
```
17+
- **Windows (PowerShell)**:
18+
```powershell
19+
powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
20+
```
21+
- **Via Pip**:
22+
```bash
23+
pip install uv
24+
```
25+
26+
### 2. Environment Setup
27+
28+
Navigating into any example directory and running `uv sync` or `uv run` will automatically set up a isolated Python virtual environment, build the local `deglib` C++ bindings, and install all required dependencies:
29+
30+
```bash
31+
cd examples/paper_reproduction
32+
uv sync
33+
```
34+
35+
## Projects
36+
37+
- [`paper_reproduction`](./paper_reproduction/): Reproduce DEG paper search benchmarks (Recall vs QPS) on datasets mentioned in the root `readme.md` (`Audio`, `Enron`, `SIFT1M`, `DEEP1M`, `GloVe-100`).

examples/static_data/README.md

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# DEG Paper Reproduction Benchmark
2+
3+
This example project demonstrates how to download paper datasets from `readme.md`, build a DEG graph with preset parameters, run search queries, measure Recall vs. QPS (Queries Per Second), and display the results plot interactively.
4+
5+
## Available Datasets
6+
7+
- `audio` (192D, 53k base vectors, L2 distance)
8+
- `enron` (1369D, 94k base vectors, L2 distance)
9+
- `sift1m` (128D, 1M base vectors, L2 distance)
10+
- `deep1m` (96D, 1M base vectors, L2 distance)
11+
- `glove-100` (100D, 1.18M base vectors, Angular / InnerProduct distance)
12+
13+
## Prerequisites & UV Setup
14+
15+
1. **Install `uv`** (if not already installed):
16+
- **Linux/macOS**: `curl -LsSf https://astral.sh/uv/install.sh | sh`
17+
- **Windows (PowerShell)**: `powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"`
18+
- **Pip**: `pip install uv`
19+
20+
2. **Sync Dependencies**:
21+
Inside this directory, initialize the environment and install dependencies (including building local `deglib` bindings):
22+
```bash
23+
uv sync
24+
```
25+
26+
## Running the Benchmark
27+
28+
`uv run` automatically uses the managed virtual environment.
29+
30+
### 1. Run Full Benchmark (e.g. SIFT1M)
31+
32+
```bash
33+
uv run main.py --dataset sift1m
34+
```
35+
36+
### 2. Fast Test Run (e.g. Audio dataset with 1,000 vectors)
37+
38+
```bash
39+
uv run main.py --dataset audio --max-base-vecs 1000
40+
```
41+
42+
## Options
43+
44+
- `--dataset`: Dataset name (`sift1m`, `audio`, `enron`, `deep1m`, `glove-100`). Default: `sift1m`.
45+
- `--cache-dir`: Persistent cache folder for datasets. Default: `~/.cache/deg_datasets`.
46+
- `--output-plot`: Optional path to also save the plot to a PNG image (e.g. `--output-plot recall_vs_qps.png`).
47+
- `--no-show`: Disable opening the interactive plot window (useful for headless CI environments).
48+
- `--max-base-vecs`: Limit base vectors for fast debugging / testing.

0 commit comments

Comments
 (0)