updated scripts

Author: milindsrivastava1997

asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlparser_test.rs

@@ -496,6 +496,65 @@ mod tests {
         );
     }
 
+    // ── ClickHouse parametric syntax + explicit BETWEEN timestamps ────────────
+    // These verify that a fully ClickHouse-compatible query (no DATEADD, no NOW())
+    // is parseable by ASAP: quantile(q)(col) + BETWEEN 'start' AND 'end'.
+
+    #[test]
+    fn test_clickhouse_explicit_datetime_temporal_quantile() {
+        check_query(
+            "SELECT quantile(0.95)(value) FROM cpu_usage WHERE time BETWEEN '2025-10-01 00:00:00' AND '2025-10-01 00:00:10' GROUP BY L1, L2, L3, L4",
+            vec![QueryType::TemporalQuantile],
+            None,
+        );
+    }
+
+    #[test]
+    // ASAP-only: parse_datetime accepts the Z suffix (interprets as UTC), but ClickHouse
+    // rejects it with TYPE_MISMATCH when comparing against a DateTime column.
+    // Do not use Z-suffix strings in queries intended for both systems.
+    fn test_asap_only_iso_z_temporal_quantile() {
+        check_query(
+            "SELECT quantile(0.95)(value) FROM cpu_usage WHERE time BETWEEN '2025-10-01T00:00:00Z' AND '2025-10-01T00:00:10Z' GROUP BY L1, L2, L3, L4",
+            vec![QueryType::TemporalQuantile],
+            None,
+        );
+    }
+
+    #[test]
+    // Both ASAP (parse_datetime) and ClickHouse treat ISO-without-Z as local server time.
+    // They agree only when running in the same timezone; prefer 'YYYY-MM-DD HH:MM:SS'
+    // (space format) to avoid this implicit dependency.
+    fn test_iso_no_z_treated_as_local_time_temporal_quantile() {
+        check_query(
+            "SELECT quantile(0.95)(value) FROM cpu_usage WHERE time BETWEEN '2025-10-01T00:00:00' AND '2025-10-01T00:00:10' GROUP BY L1, L2, L3, L4",
+            vec![QueryType::TemporalQuantile],
+            None,
+        );
+    }
+
+    #[test]
+    fn test_clickhouse_explicit_datetime_spatial_quantile() {
+        check_query(
+            "SELECT quantile(0.95)(value) FROM cpu_usage WHERE time BETWEEN '2025-10-01 00:00:00' AND '2025-10-01 00:00:01' GROUP BY L1",
+            vec![QueryType::Spatial],
+            None,
+        );
+    }
+
+    #[test]
+    fn test_clickhouse_explicit_matches_now_template() {
+        // A ClickHouse-style query (explicit timestamps, parametric quantile) must
+        // match a stored DATEADD(NOW()) template of the same shape.
+        let template = parse_sql_query(
+            "SELECT quantile(0.95)(value) FROM cpu_usage WHERE time BETWEEN DATEADD(s, -10, NOW()) AND NOW() GROUP BY L1, L2, L3, L4"
+        ).unwrap();
+        let incoming = parse_sql_query(
+            "SELECT quantile(0.95)(value) FROM cpu_usage WHERE time BETWEEN '2025-10-01 00:00:00' AND '2025-10-01 00:00:10' GROUP BY L1, L2, L3, L4"
+        ).unwrap();
+        assert!(incoming.matches_sql_pattern(&template));
+    }
+
     // ── Error cases ──────────────────────────────────────────────────────────
 
     #[test]

asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlpattern_parser.rs

@@ -320,6 +320,11 @@ impl SQLPatternParser {
     }
 
     fn get_timestamp_from_datetime_str(datetime_str: &str) -> Option<f64> {
+        // parse_datetime treats timezone-naive strings (e.g. "2025-10-01 00:00:00",
+        // "2025-10-01T00:00:00") as local server time, matching ClickHouse's behavior —
+        // but only when both run in the same timezone. Z-suffix strings (e.g.
+        // "2025-10-01T00:00:00Z") are interpreted as UTC here but rejected by ClickHouse.
+        // Use space-format datetime strings ("YYYY-MM-DD HH:MM:SS") for portability.
         let parsed_datetime = parse_datetime(datetime_str).ok()?;
         Some(parsed_datetime.timestamp().as_second() as f64)
     }

asap-tools/execution-utilities/benchmark/README.md

@@ -35,6 +35,12 @@ pip3 install --user -r requirements.txt
 cd ~/ASAPQuery/asap-query-engine && cargo build --release
 ```
 
+> **UTC requirement:** Both ASAP and ClickHouse must run in UTC so that bare
+> datetime strings (`'YYYY-MM-DD HH:MM:SS'`) are interpreted identically by both
+> systems. Set `TZ=UTC` in the environment for ASAP processes and ensure
+> ClickHouse's `timezone` config is set to `UTC`. If the two systems run in
+> different timezones, queries will target different time windows on each side.
+
 ---
 
 ## ClickBench + ClickHouse End-to-End Example
@@ -115,8 +121,7 @@ python generate_queries.py \
 ```
 
 This writes:
-- `queries/clickbench_asap.sql` — ASAP queries (ISO timestamps)
-- `queries/clickbench_clickhouse.sql` — ClickHouse queries (datetime timestamps)
+- `queries/clickbench.sql` — shared query file for both ASAP and ClickHouse
 - `queries/clickbench_streaming.yaml` — Arroyo streaming config
 - `queries/clickbench_inference.yaml` — QueryEngineRust inference config
 
@@ -166,8 +171,8 @@ Verify: `$INSTALL_DIR/clickhouse client --query "SELECT count(*) FROM hits"`
 ```bash
 python run_benchmark.py \
     --mode both \
-    --asap-sql-file ./queries/clickbench_asap.sql \
-    --baseline-sql-file ./queries/clickbench_clickhouse.sql \
+    --asap-sql-file ./queries/clickbench.sql \
+    --baseline-sql-file ./queries/clickbench.sql \
     --asap-url "http://localhost:8088/api/v1/query" \
     --output-dir ./results \
     --output-prefix clickbench
@@ -258,8 +263,8 @@ python export_to_database.py \
 ```bash
 python run_benchmark.py \
     --mode both \
-    --asap-sql-file ./queries/h2o_asap.sql \
-    --baseline-sql-file ./queries/h2o_clickhouse.sql \
+    --asap-sql-file ./queries/h2o.sql \
+    --baseline-sql-file ./queries/h2o.sql \
     --asap-url "http://localhost:8088/api/v1/query" \
     --output-dir ./results \
     --output-prefix h2o
@@ -290,7 +295,7 @@ python export_to_arroyo.py \
 cd ~/ASAPQuery/asap-query-engine
 
 ./target/release/query_engine_rust \
-    --kafka-topic sketch_topic 
+    --kafka-topic sketch_topic
     --input-format json \
     --config ~/ASAPQuery/asap-tools/execution-utilities/benchmark/configs/h2o_inference.yaml \
     --streaming-config ~/ASAPQuery/asap-tools/execution-utilities/benchmark/configs/h2o_streaming.yaml \
@@ -303,25 +308,25 @@ cd ~/ASAPQuery/asap-query-engine
 ### Step 8 — Load data into Elasticsearch (baseline)
 
 ```bash
-python export_to_database.py 
-    --dataset h2o 
-    --file-path ./data/G1_1e7_1e2_0_0.csv 
-    --es-host localhost 
-    --es-port 9200 
-    --es-index h2o_groupby 
+python export_to_database.py
+    --dataset h2o
+    --file-path ./data/G1_1e7_1e2_0_0.csv
+    --es-host localhost
+    --es-port 9200
+    --es-index h2o_groupby
     --es-api-key your-api-key
     --es-bulk-size 5000
 ```
 
 ### Step 9 — Run benchmark
 
 ```bash
-python run_benchmark.py 
-    --mode asap 
-    --asap-sql-file ./queries/h2o_asap.sql 
-    --baseline-sql-file ./queries/h2o_elasticsearch.sql 
-    --elastic-host localhost 
-    --elastic-port 9200 
+python run_benchmark.py
+    --mode asap
+    --asap-sql-file ./queries/h2o.sql
+    --baseline-sql-file ./queries/h2o.sql
+    --elastic-host localhost
+    --elastic-port 9200
     --elastic-api-key your-api-key
     --output-dir ./results --output-prefix h2o
 ```
@@ -370,8 +375,8 @@ python export_to_database.py \
 # 6. Run benchmark
 python run_benchmark.py \
     --mode both \
-    --asap-sql-file ./queries/my_dataset_asap.sql \
-    --baseline-sql-file ./queries/my_dataset_clickhouse.sql \
+    --asap-sql-file ./queries/my_dataset.sql \
+    --baseline-sql-file ./queries/my_dataset.sql \
     --asap-url "http://localhost:8088/api/v1/query" \
     --output-dir ./results
 ```
@@ -407,6 +412,6 @@ $INSTALL_DIR/clickhouse client --query "TRUNCATE TABLE hits"
 | `prepare_data.py` | Convert raw data to Arroyo file source format (RFC3339, string columns) |
 | `export_to_arroyo.py` | Launch Arroyo sketch pipeline (file or kafka source) |
 | `export_to_database.py` | Load data into ClickHouse for baseline |
-| `generate_queries.py` | Generate paired ASAP + ClickHouse SQL query files and streaming/inference YAML configs |
+| `generate_queries.py` | Generate a shared SQL query file (ClickHouse-compatible syntax, used for both ASAP and ClickHouse) and optional streaming/inference YAML configs |
 | `run_benchmark.py` | Run queries and produce CSV results + plots |
 | `configs/` | ClickHouse init SQL (CREATE TABLE statements) |