Skip to content

Commit d93d5b0

Browse files
GnaneshGnaniGnaneshGnani
committed
Merge main into sketchlib-kll for KLL sketchlib PR5
2 parents 0db700d + 11f348a commit d93d5b0

131 files changed

Lines changed: 15896 additions & 1733 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/correctness.yml‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
name: Correctness Tests
2+
3+
# Verifies cross-language parity between Python and Rust implementations of
4+
# PromQL pattern matching and sketch serialisation. Ephemeral GitHub Actions
5+
# VMs are well-suited for these deterministic correctness checks.
6+
7+
on:
8+
push:
9+
branches: [ main ]
10+
paths:
11+
- 'asap-common/tests/**'
12+
- 'asap-common/dependencies/**'
13+
- 'asap-common/sketch-core/**'
14+
- '.github/workflows/correctness.yml'
15+
pull_request:
16+
branches: [ main ]
17+
paths:
18+
- 'asap-common/tests/**'
19+
- 'asap-common/dependencies/**'
20+
- 'asap-common/sketch-core/**'
21+
- '.github/workflows/correctness.yml'
22+
workflow_dispatch:
23+
24+
jobs:
25+
# ── 1. Cross-language token matching (Python vs Rust) ──────────────────────
26+
cross-language-token-matching:
27+
name: Cross-language PromQL token matching
28+
runs-on: ubuntu-latest
29+
steps:
30+
- uses: actions/checkout@v4
31+
32+
- name: Set up Python
33+
uses: actions/setup-python@v5
34+
with:
35+
python-version: '3.11'
36+
37+
- name: Install Python dependencies
38+
run: |
39+
python -m pip install --upgrade pip
40+
if [ -f asap-common/dependencies/py/requirements.txt ]; then
41+
pip install -r asap-common/dependencies/py/requirements.txt
42+
fi
43+
pip install -e asap-common/dependencies/py/promql_utilities/
44+
45+
- name: Install Rust
46+
uses: dtolnay/rust-toolchain@stable
47+
48+
- name: Install protoc
49+
run: |
50+
sudo apt-get update -qq
51+
sudo apt-get install -y protobuf-compiler
52+
53+
- name: Run sccache
54+
uses: mozilla-actions/sccache-action@v0.0.4
55+
56+
- name: Cache cargo
57+
uses: actions/cache@v4
58+
with:
59+
path: |
60+
~/.cargo/registry
61+
~/.cargo/git
62+
target
63+
key: ${{ runner.os }}-cargo-correctness-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }}
64+
65+
- name: Run master test runner (Python + Rust + comparison)
66+
working-directory: asap-common/tests/compare_matched_tokens
67+
run: python utilities/master_test_runner.py
68+
env:
69+
RUSTC_WRAPPER: sccache
70+
71+
- name: Upload test artefacts
72+
if: always()
73+
uses: actions/upload-artifact@v4
74+
with:
75+
name: cross-language-token-results
76+
path: |
77+
asap-common/tests/compare_matched_tokens/python_tests/python_test_results.json
78+
asap-common/tests/compare_matched_tokens/rust_tests/rust_test_results.json
79+
asap-common/tests/compare_matched_tokens/comparison_tests/comparison_report.json
80+
asap-common/tests/compare_matched_tokens/test_summary.json
81+
if-no-files-found: warn
82+
83+
# ── 2. Cross-language serialised pattern comparison ────────────────────────
84+
cross-language-pattern-serialisation:
85+
name: Cross-language pattern serialisation
86+
runs-on: ubuntu-latest
87+
steps:
88+
- uses: actions/checkout@v4
89+
90+
- name: Set up Python
91+
uses: actions/setup-python@v5
92+
with:
93+
python-version: '3.11'
94+
95+
- name: Install Python dependencies
96+
run: |
97+
python -m pip install --upgrade pip
98+
if [ -f asap-common/dependencies/py/requirements.txt ]; then
99+
pip install -r asap-common/dependencies/py/requirements.txt
100+
fi
101+
pip install -e asap-common/dependencies/py/promql_utilities/
102+
103+
- name: Install Rust
104+
uses: dtolnay/rust-toolchain@stable
105+
106+
- name: Install protoc
107+
run: |
108+
sudo apt-get update -qq
109+
sudo apt-get install -y protobuf-compiler
110+
111+
- name: Run sccache
112+
uses: mozilla-actions/sccache-action@v0.0.4
113+
114+
- name: Cache cargo
115+
uses: actions/cache@v4
116+
with:
117+
path: |
118+
~/.cargo/registry
119+
~/.cargo/git
120+
target
121+
key: ${{ runner.os }}-cargo-correctness-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }}
122+
123+
- name: Generate Python patterns
124+
working-directory: asap-common/tests/compare_patterns
125+
run: python python_generate_patterns.py
126+
127+
- name: Build and run Rust pattern generator
128+
working-directory: asap-common/tests/compare_patterns
129+
run: cargo run --release
130+
env:
131+
RUSTC_WRAPPER: sccache
132+
133+
- name: Compare serialised patterns
134+
working-directory: asap-common/tests/compare_patterns
135+
run: python compare_serialized_patterns.py
136+
137+
# ── 3. Rust pattern-matching unit tests ────────────────────────────────────
138+
rust-pattern-matching:
139+
name: Rust pattern matching unit tests
140+
runs-on: ubuntu-latest
141+
steps:
142+
- uses: actions/checkout@v4
143+
144+
- name: Install Rust
145+
uses: dtolnay/rust-toolchain@stable
146+
147+
- name: Install protoc
148+
run: |
149+
sudo apt-get update -qq
150+
sudo apt-get install -y protobuf-compiler
151+
152+
- name: Run sccache
153+
uses: mozilla-actions/sccache-action@v0.0.4
154+
155+
- name: Cache cargo
156+
uses: actions/cache@v4
157+
with:
158+
path: |
159+
~/.cargo/registry
160+
~/.cargo/git
161+
target
162+
key: ${{ runner.os }}-cargo-correctness-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }}
163+
164+
- name: Run Rust pattern matching tests
165+
working-directory: asap-common/tests/rust_pattern_matching
166+
run: cargo test --release
167+
env:
168+
RUSTC_WRAPPER: sccache

‎.github/workflows/rust.yml‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,11 @@ jobs:
3838
toolchain: stable
3939
components: rustfmt, clippy
4040

41+
- name: Install protoc
42+
run: |
43+
sudo apt-get update
44+
sudo apt-get install -y protobuf-compiler
45+
4146
- name: Run sccache-cache
4247
uses: mozilla-actions/sccache-action@v0.0.4
4348

@@ -48,13 +53,13 @@ jobs:
4853
~/.cargo/registry
4954
~/.cargo/git
5055
target
51-
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
56+
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }}
5257

5358
- name: Check formatting
5459
run: cargo fmt --all -- --check
5560

5661
- name: Run clippy
57-
run: cargo clippy --workspace --all-targets --all-features -- -D warnings
62+
run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
5863
env:
5964
RUSTC_WRAPPER: sccache
6065

@@ -68,6 +73,11 @@ jobs:
6873
with:
6974
toolchain: stable
7075

76+
- name: Install protoc
77+
run: |
78+
sudo apt-get update
79+
sudo apt-get install -y protobuf-compiler
80+
7181
- name: Run sccache-cache
7282
uses: mozilla-actions/sccache-action@v0.0.4
7383

@@ -78,10 +88,10 @@ jobs:
7888
~/.cargo/registry
7989
~/.cargo/git
8090
target
81-
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
91+
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }}
8292

8393
- name: Run all workspace tests
84-
run: cargo test --workspace
94+
run: cargo test --workspace --locked
8595
env:
8696
RUSTC_WRAPPER: sccache
8797

@@ -98,4 +108,4 @@ jobs:
98108
uses: docker/setup-buildx-action@v3
99109

100110
- name: Build Docker image
101-
run: docker build -f asap-query-engine/Dockerfile -t sketchdb-queryengine-rust:latest .
111+
run: docker build -f asap-query-engine/Dockerfile -t sketchdb-queryengine-rust:latest .

0 commit comments

Comments
 (0)