From 57f195db658e9313ddd750bf51a425e7c445d4bb Mon Sep 17 00:00:00 2001 From: Yodan Bargida Date: Thu, 25 Jun 2026 12:03:01 +0300 Subject: [PATCH 01/11] feat: implement query execute endpoint and fix agent embedder url --- backend/app/infra_init.py | 12 ++--- backend/app/main.py | 2 + backend/app/routers/admin_auth.py | 4 +- backend/app/routers/audit.py | 5 +- backend/app/routers/feedback.py | 5 +- backend/app/routers/publish.py | 6 +-- backend/app/routers/query.py | 71 ++++++++++++++++++++++++++ backend/app/routers/questions.py | 5 +- backend/app/routers/scopes.py | 5 +- backend/app/services/auth.py | 5 +- backend/app/services/evaluator.py | 2 +- backend/app/services/join_detection.py | 3 +- backend/app/services/trino_client.py | 3 +- backend/tests/conftest.py | 4 +- backend/tests/test_api.py | 71 +++++++++++++++++++++++++- docker-compose.yml | 2 +- 16 files changed, 171 insertions(+), 34 deletions(-) create mode 100644 backend/app/routers/query.py diff --git a/backend/app/infra_init.py b/backend/app/infra_init.py index 16f2283..1ea1ff3 100644 --- a/backend/app/infra_init.py +++ b/backend/app/infra_init.py @@ -111,7 +111,7 @@ ('ORD-027','Alice Cohen','alice@example.com','Desk Lamp',1,45.0,45.0,'delivered',DATE '2024-03-12'), ('ORD-028','Bob Levi','bob@example.com','Laptop',1,1200.0,1200.0,'shipped',DATE '2024-03-15'), ('ORD-029','Carol Mizrahi','carol@example.com','Smartphone',1,800.0,800.0,'delivered',DATE '2024-03-18'), - ('ORD-030','Dan Shapiro','dan@example.com','Tablet',1,400.0,400.0,'delivered',DATE '2024-03-20')""" + ('ORD-030','Dan Shapiro','dan@example.com','Tablet',1,400.0,400.0,'delivered',DATE '2024-03-20')""", }, # ── complex_retail ───────────────────────────────────────────────────── { @@ -153,7 +153,7 @@ ('C22','Victor','Hugo','victor@example.com','France','Paris',TIMESTAMP '2024-01-02 10:00:00'), ('C23','Wendy','Darling','wendy@example.com','Canada','Toronto',TIMESTAMP '2024-01-10 15:45:00'), ('C24','Xavier','Charles','xavier@example.com','Canada','Vancouver',TIMESTAMP '2024-01-15 09:00:00'), - ('C25','Yasmine','Bleeth','yasmine@example.com','USA','Miami',TIMESTAMP '2024-01-22 13:15:00')""" + ('C25','Yasmine','Bleeth','yasmine@example.com','USA','Miami',TIMESTAMP '2024-01-22 13:15:00')""", }, { "fqn": "minio.complex_retail.products", @@ -183,7 +183,7 @@ ('P12','Standing Desk','Furniture','Tables',600.0,25), ('P13','Notebook','Office Supplies','Paper',5.0,500), ('P14','Gel Pens Pack','Office Supplies','Writing',12.0,400), - ('P15','Backpack','Office Supplies','Bags',80.0,100)""" + ('P15','Backpack','Office Supplies','Bags',80.0,100)""", }, { "fqn": "minio.complex_retail.orders", @@ -238,7 +238,7 @@ ('O37','C22',DATE '2024-03-02','delivered',180.0,'Paris, Rue de Rivoli 20'), ('O38','C23',DATE '2024-03-03','pending',100.0,'Toronto, Yonge St 100'), ('O39','C24',DATE '2024-03-04','delivered',75.0,'Vancouver, Georgia St 50'), - ('O40','C09',DATE '2024-03-05','delivered',60.0,'London, Baker St 221B')""" + ('O40','C09',DATE '2024-03-05','delivered',60.0,'London, Baker St 221B')""", }, { "fqn": "minio.complex_retail.order_items", @@ -280,7 +280,7 @@ ('I24','O22','P08',2,90.0,0.0), ('I25','O23','P02',4,25.0,0.0), ('I26','O24','P03',1,75.0,0.0), - ('I27','O25','P04',2,350.0,0.0)""" + ('I27','O25','P04',2,350.0,0.0)""", }, ] @@ -662,7 +662,7 @@ def _seed_trino_data() -> None: except Exception as e: # If DELETE is not supported (e.g. some Iceberg configs require specific formats), we ignore and fall back to count checks logger.debug("DELETE on %s failed: %s", table["fqn"], e) - + _trino_exec(table["seed_sql"]) logger.info("[InfraInit] Seeded sample data into '%s' ✓", table["fqn"]) except Exception as exc: diff --git a/backend/app/main.py b/backend/app/main.py index f46484a..0c1b27f 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -23,6 +23,7 @@ orchestration, profiling, publish, + query, questions, scopes, tables, @@ -123,6 +124,7 @@ async def audit_middleware(request: Request, call_next): api_router.include_router(admin_auth.router) api_router.include_router(admin_approval.router) api_router.include_router(agent.router) +api_router.include_router(query.router) app.include_router(api_router) diff --git a/backend/app/routers/admin_auth.py b/backend/app/routers/admin_auth.py index b2e0673..e9dadf1 100644 --- a/backend/app/routers/admin_auth.py +++ b/backend/app/routers/admin_auth.py @@ -1,9 +1,9 @@ +from core.db.engine import get_session +from core.models.models import SecurityUserRead from fastapi import APIRouter, Depends, HTTPException, status from pydantic import BaseModel from sqlmodel import Session -from core.db.engine import get_session -from core.models.models import SecurityUserRead from app.services.auth import get_user_by_email router = APIRouter(prefix="/admin", tags=["admin_auth"]) diff --git a/backend/app/routers/audit.py b/backend/app/routers/audit.py index 72e2c14..963e5f9 100644 --- a/backend/app/routers/audit.py +++ b/backend/app/routers/audit.py @@ -1,8 +1,7 @@ -from fastapi import APIRouter, Depends, Query -from sqlmodel import Session, select - from core.db.engine import get_session from core.models.models import AuditQuery, AuditQueryRead +from fastapi import APIRouter, Depends, Query +from sqlmodel import Session, select router = APIRouter(prefix="/audit", tags=["audit"]) diff --git a/backend/app/routers/feedback.py b/backend/app/routers/feedback.py index 3ee1967..162d22f 100644 --- a/backend/app/routers/feedback.py +++ b/backend/app/routers/feedback.py @@ -3,9 +3,6 @@ Feedback signals are consumed by the Table Health scoring engine. """ -from fastapi import APIRouter, Depends, HTTPException -from sqlmodel import Session, select - from core.db.engine import get_session from core.models.models import ( AuditQuery, @@ -13,6 +10,8 @@ QueryFeedbackCreate, QueryFeedbackRead, ) +from fastapi import APIRouter, Depends, HTTPException +from sqlmodel import Session, select router = APIRouter(tags=["feedback"]) diff --git a/backend/app/routers/publish.py b/backend/app/routers/publish.py index 772334a..c6a08a0 100644 --- a/backend/app/routers/publish.py +++ b/backend/app/routers/publish.py @@ -9,9 +9,6 @@ import uuid -from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException -from sqlmodel import Session, select - from core.db.engine import get_session from core.models.models import ( EnrichmentVersion, @@ -20,6 +17,9 @@ GoldenQuestion, Table, ) +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException +from sqlmodel import Session, select + from app.routers.evaluation import promote_table_to_production_workflow from app.services.scoring import REGRESSION_BLOCK diff --git a/backend/app/routers/query.py b/backend/app/routers/query.py new file mode 100644 index 0000000..1d88e12 --- /dev/null +++ b/backend/app/routers/query.py @@ -0,0 +1,71 @@ +import time +from typing import Any + +from core.trino import get_trino_connection +from fastapi import APIRouter +from pydantic import BaseModel + +from app.config import settings + +router = APIRouter(prefix="/query", tags=["query"]) + + +class QueryRequest(BaseModel): + sql: str + + +class QueryResponse(BaseModel): + success: bool + rows: list[list[Any]] + columns: list[str] + row_count: int + execution_time_ms: float + error: str | None + + +@router.post("/execute", response_model=QueryResponse) +def execute_query(request: QueryRequest) -> QueryResponse: + """ + Execute a SQL query against Trino and return the results. + """ + start_time = time.time() + + if not settings.TRINO_ENABLED: + return QueryResponse( + success=True, + rows=[], + columns=[], + row_count=0, + execution_time_ms=0.0, + error="Trino execution is disabled (TRINO_ENABLED=False)", + ) + + try: + conn = get_trino_connection() + cur = conn.cursor() + cur.execute(request.sql) + rows = cur.fetchall() + columns = [desc[0] for desc in cur.description] if cur.description else [] + execution_time_ms = (time.time() - start_time) * 1000 + + cur.close() + conn.close() + + return QueryResponse( + success=True, + rows=[list(row) for row in rows], + columns=columns, + row_count=len(rows), + execution_time_ms=execution_time_ms, + error=None, + ) + except Exception as exc: + execution_time_ms = (time.time() - start_time) * 1000 + return QueryResponse( + success=False, + rows=[], + columns=[], + row_count=0, + execution_time_ms=execution_time_ms, + error=str(exc), + ) diff --git a/backend/app/routers/questions.py b/backend/app/routers/questions.py index 29f5ade..5ec8039 100644 --- a/backend/app/routers/questions.py +++ b/backend/app/routers/questions.py @@ -2,9 +2,6 @@ import json import pandas as pd -from fastapi import APIRouter, Depends, File, HTTPException, UploadFile -from sqlmodel import Session, select - from core.db.engine import get_session from core.models.models import ( DifficultyLevel, @@ -14,6 +11,8 @@ QuestionType, Table, ) +from fastapi import APIRouter, Depends, File, HTTPException, UploadFile +from sqlmodel import Session, select router = APIRouter(prefix="/tables", tags=["golden-questions"]) diff --git a/backend/app/routers/scopes.py b/backend/app/routers/scopes.py index 257b6a4..201c206 100644 --- a/backend/app/routers/scopes.py +++ b/backend/app/routers/scopes.py @@ -1,8 +1,7 @@ -from fastapi import APIRouter, Depends, HTTPException -from sqlmodel import Session, select - from core.db.engine import get_session from core.models.models import UserScope, UserScopeCreate, UserScopeRead +from fastapi import APIRouter, Depends, HTTPException +from sqlmodel import Session, select router = APIRouter(prefix="/scopes", tags=["scopes"]) diff --git a/backend/app/services/auth.py b/backend/app/services/auth.py index 330cba7..240dc08 100644 --- a/backend/app/services/auth.py +++ b/backend/app/services/auth.py @@ -1,8 +1,7 @@ -from fastapi import Depends, HTTPException, status -from sqlmodel import Session, select - from core.db.engine import get_session from core.models.models import SecurityUser +from fastapi import Depends, HTTPException, status +from sqlmodel import Session, select def get_user_by_email(email: str, session: Session) -> SecurityUser | None: diff --git a/backend/app/services/evaluator.py b/backend/app/services/evaluator.py index 1a2332c..cf4d24f 100644 --- a/backend/app/services/evaluator.py +++ b/backend/app/services/evaluator.py @@ -22,10 +22,10 @@ from abc import ABC, abstractmethod from typing import Any +from core.models.models import EvalResult, GoldenQuestion from langfuse.decorators import langfuse_context, observe from sqlmodel import Session -from core.models.models import EvalResult, GoldenQuestion from app.services.langfuse_client import Evaluation, langfuse_client as _lf_client logger = logging.getLogger(__name__) diff --git a/backend/app/services/join_detection.py b/backend/app/services/join_detection.py index d88e623..d9e21e0 100644 --- a/backend/app/services/join_detection.py +++ b/backend/app/services/join_detection.py @@ -1,9 +1,8 @@ import logging -from sqlmodel import Session, select - from core.db.engine import engine from core.models.models import ColumnProfile, CrossTableProfile +from sqlmodel import Session, select logger = logging.getLogger(__name__) diff --git a/backend/app/services/trino_client.py b/backend/app/services/trino_client.py index 2312f56..77100ab 100644 --- a/backend/app/services/trino_client.py +++ b/backend/app/services/trino_client.py @@ -1,2 +1,3 @@ -from core import TrinoExecutionResult, get_trino_connection, execute_query_sync +from core.trino import TrinoExecutionResult, execute_query_sync, get_trino_connection +__all__ = ["TrinoExecutionResult", "execute_query_sync", "get_trino_connection"] diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 3c5f48d..f389304 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -1,14 +1,14 @@ import urllib.parse +import core.db.engine import psycopg2 import pytest +from core.db.engine import get_session from fastapi.testclient import TestClient from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT from sqlmodel import Session, SQLModel, create_engine -import core.db.engine from app.config import settings -from core.db.engine import get_session from app.main import app as fastapi_app # Parse the database URL from settings diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 92e4f11..eb5ff02 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -2,11 +2,11 @@ from core.models.models import ( AuditQuery, + EnrichmentVersion, EvalStatus, GoldenQuestion, Table, TableStatus, - EnrichmentVersion ) # ── Mock objects for testing ────────────────────────────────────────────────── @@ -212,3 +212,72 @@ def test_list_audit_queries(client, db_session): response = client.get("/api/audit/queries") assert response.status_code == 200 assert len(response.json()) >= 1 + + +# ── Query API Tests ──────────────────────────────────────────────────────────── +@patch("app.routers.query.settings") +def test_execute_query_disabled(mock_settings, client): + mock_settings.TRINO_ENABLED = False + payload = {"sql": "SELECT * FROM users LIMIT 10"} + response = client.post("/api/query/execute", json=payload) + assert response.status_code == 200 + data = response.json() + assert data["success"] is True + assert data["rows"] == [] + assert data["columns"] == [] + assert data["row_count"] == 0 + assert data["error"] is not None + assert "disabled" in data["error"] + + +@patch("app.routers.query.get_trino_connection") +@patch("app.routers.query.settings") +def test_execute_query_success(mock_settings, mock_get_conn, client): + mock_settings.TRINO_ENABLED = True + + mock_conn = MagicMock() + mock_cur = MagicMock() + mock_cur.description = [("id",), ("name",), ("email",)] + mock_cur.fetchall.return_value = [ + (1, "Alice", "alice@example.com"), + (2, "Bob", "bob@example.com"), + ] + mock_conn.cursor.return_value = mock_cur + mock_get_conn.return_value = mock_conn + + payload = {"sql": "SELECT id, name, email FROM users"} + response = client.post("/api/query/execute", json=payload) + assert response.status_code == 200 + data = response.json() + assert data["success"] is True + assert data["columns"] == ["id", "name", "email"] + assert data["rows"] == [ + [1, "Alice", "alice@example.com"], + [2, "Bob", "bob@example.com"], + ] + assert data["row_count"] == 2 + assert data["error"] is None + assert isinstance(data["execution_time_ms"], float) + + +@patch("app.routers.query.get_trino_connection") +@patch("app.routers.query.settings") +def test_execute_query_failure(mock_settings, mock_get_conn, client): + mock_settings.TRINO_ENABLED = True + + mock_conn = MagicMock() + mock_cur = MagicMock() + mock_cur.execute.side_effect = Exception("Syntax error near 'LIMIT'") + mock_conn.cursor.return_value = mock_cur + mock_get_conn.return_value = mock_conn + + payload = {"sql": "SELECT id FROM users LIMIT"} + response = client.post("/api/query/execute", json=payload) + assert response.status_code == 200 + data = response.json() + assert data["success"] is False + assert data["rows"] == [] + assert data["columns"] == [] + assert data["row_count"] == 0 + assert data["error"] is not None + assert "Syntax error" in data["error"] diff --git a/docker-compose.yml b/docker-compose.yml index b01c011..92bf7fb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -467,7 +467,7 @@ services: - LLM_BASE_URL=http://host.docker.internal:11434/v1 - LLM_MODEL=gemma4:e4b - LLM_API_KEY=ollama - - EMBEDDER_URL=http://host.docker.internal:11434 + - EMBEDDER_URL=http://host.docker.internal:11434/v1/embeddings - EMBEDDER_MODEL=nomic-embed-text:latest - ESCA_URL=http://host.docker.internal:7010 - MAX_PROFILES_TO_FETCH=4 From 30479fdc8d4429b514d46eff60c29356da99724b Mon Sep 17 00:00:00 2001 From: Yodan Bargida Date: Tue, 30 Jun 2026 14:54:14 +0300 Subject: [PATCH 02/11] fix(): handle timestams in finalizer and refiner --- agent/src/agent/nodes/finalizer.py | 8 +++++++- agent/src/agent/nodes/refiner.py | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/agent/src/agent/nodes/finalizer.py b/agent/src/agent/nodes/finalizer.py index 0c30660..dcd6765 100644 --- a/agent/src/agent/nodes/finalizer.py +++ b/agent/src/agent/nodes/finalizer.py @@ -29,13 +29,19 @@ async def get_esca_preview(esca_id: str, limit: int = 5) -> str: # Take a slice of the rows to avoid context overload preview_rows = rows[:limit] + import datetime + def json_serial(obj): + if isinstance(obj, (datetime.datetime, datetime.date)): + return obj.isoformat() + raise TypeError("Type %s not serializable" % type(obj)) + preview_info = { "columns": columns, "preview_rows": preview_rows, "preview_count": len(preview_rows), "total_rows": total_rows } - return json.dumps(preview_info, indent=2) + return json.dumps(preview_info, default=json_serial, indent=2) except Exception as e: return f"Error retrieving data preview from Esca: {e}" finally: diff --git a/agent/src/agent/nodes/refiner.py b/agent/src/agent/nodes/refiner.py index 041923c..7216e18 100644 --- a/agent/src/agent/nodes/refiner.py +++ b/agent/src/agent/nodes/refiner.py @@ -37,11 +37,17 @@ async def refiner_node(state: AgentState): else: # Success, save payload via Esca client = EscaClient(api_key=settings.ESCA_API_KEY, base_url=settings.ESCA_URL) + import datetime + def json_serial(obj): + if isinstance(obj, (datetime.datetime, datetime.date)): + return obj.isoformat() + raise TypeError("Type %s not serializable" % type(obj)) + payload_data = { "columns": result.columns, "rows": result.rows } - payload = json.dumps(payload_data).encode() + payload = json.dumps(payload_data, default=json_serial).encode() try: res = await client.save_data(payload) raw_ref = res.get("esca_id") From 270112edce7adc25b4f8f6c96cc3f0ae7921716e Mon Sep 17 00:00:00 2001 From: Yodan Bargida Date: Wed, 1 Jul 2026 17:13:40 +0300 Subject: [PATCH 03/11] feat: conect the new service --- backend/app/routers/agent.py | 19 +- backend/app/routers/evaluation.py | 367 ++++++++++++++++-------------- docker-compose.yml | 1 + frontend/nginx.conf | 6 +- infra/trino/etc/jvm.config | 2 +- 5 files changed, 215 insertions(+), 180 deletions(-) diff --git a/backend/app/routers/agent.py b/backend/app/routers/agent.py index 7dc5f05..4f6c4c6 100644 --- a/backend/app/routers/agent.py +++ b/backend/app/routers/agent.py @@ -11,10 +11,13 @@ import json import logging -from fastapi import APIRouter, HTTPException +from core.db.engine import get_session +from core.models.models import Table, TableRead, TableStatus +from fastapi import APIRouter, Depends, HTTPException from mcp.client.session import ClientSession from mcp.client.streamable_http import streamablehttp_client from pydantic import BaseModel +from sqlmodel import Session, select from app.config import settings @@ -166,3 +169,17 @@ async def chat(request: ChatRequest) -> ChatResponse: result = await _call_agent_mcp(tool_arguments) return ChatResponse(**result) + + +@router.get("/tables", response_model=list[TableRead]) +def get_agent_tables( + status: TableStatus | None = None, session: Session = Depends(get_session) +): + """ + Internal endpoint for the agent and evaluation service to fetch available tables + without requiring user SSO authentication. + """ + q = select(Table) + if status: + q = q.where(Table.status == status) + return session.exec(q).all() diff --git a/backend/app/routers/evaluation.py b/backend/app/routers/evaluation.py index 70f8c2b..fe7eb80 100644 --- a/backend/app/routers/evaluation.py +++ b/backend/app/routers/evaluation.py @@ -15,7 +15,6 @@ import logging from datetime import datetime -from typing import Literal import requests from core.db.engine import engine, get_session @@ -45,40 +44,80 @@ router = APIRouter(tags=["evaluation"]) -class EvalAPIRequest(BaseModel): - tables_names: list[str] - dataset_name: str - - -class EvalAPIQuestionMetrics(BaseModel): - exact_match: float - exact_execution_accuracy: float - contains_execution_accuracy: float - - -class EvalAPIQuestionResult(BaseModel): +class LatencyStatsDTO(BaseModel): + p50: float + p95: float + p99: float + average: float + minimum: float + maximum: float + total_samples: int + + +class AccuracyStatsDTO(BaseModel): + execution_accuracy: float + contains_accuracy: float + sql_exact_match: float + time_shift_score: float + component_match: float + schema_hallucination: float + dialect_error: float + composite_score: float + + +class FailureCategoryDTO(BaseModel): + category: str + count: int + rate: float + + +class FailureAnalysisDTO(BaseModel): + total_failures: int + failure_rate: float + agent_crash_count: int + agent_crash_rate: float + sql_execution_failure_count: int + sql_execution_failure_rate: float + trino_failure_count: int + trino_failure_rate: float + timeout_count: int + timeout_rate: float + validation_failure_count: int + validation_failure_rate: float + categories: list[FailureCategoryDTO] + + +class PerformanceStatsDTO(BaseModel): + average_total_execution_time_ms: float + average_time_to_first_row_ms: float + total_token_usage: int + average_token_usage: float + average_refiner_iterations: float + + +class RunDatasetCaseResultDTO(BaseModel): question_id: str generated_sql: str | None = None - metrics: EvalAPIQuestionMetrics - status: Literal["pass", "fail"] - error_message: str | None = None - row_count: int | None = None + expected_sql: str | None = None + succeeded: bool + error: str | None = None + scores: dict[str, float] -class EvalAPIOverallMetrics(BaseModel): - contains_execution_accuracy: float - exact_execution_accuracy: float - exact_match: float - total_questions: int - pass_rate: float - fail_rate: float - - -class EvalAPIResponse(BaseModel): +class RunDatasetResponse(BaseModel): + dataset_name: str run_id: str - status: Literal["completed", "failed"] - overall_metrics: EvalAPIOverallMetrics - results: list[EvalAPIQuestionResult] + total_cases: int + passed: int + failed: int + failure_rate: float + latency: LatencyStatsDTO + accuracy: AccuracyStatsDTO + failure_analysis: FailureAnalysisDTO + performance: PerformanceStatsDTO + langfuse_trace_id: str | None = None + duration_seconds: float + cases: list[RunDatasetCaseResultDTO] # Name of the single shared Langfuse dataset for all production table questions @@ -131,6 +170,65 @@ def _build_questions_payload(questions: list, table: Table) -> list: # ─── Core evaluation runner (single dataset) ─────────────────────────────────── +def _map_and_save_run_metrics( + run: EvalRun, eval_resp: RunDatasetResponse, session: Session, run_id: str +): + run.score = eval_resp.accuracy.contains_accuracy + run.pass_rate = 1.0 - eval_resp.failure_rate + run.fail_rate = eval_resp.failure_rate + run.total_questions = eval_resp.total_cases + run.duration_seconds = eval_resp.duration_seconds + run.status = EvalStatus.completed + run.completed_at = datetime.now() + + # Store failure breakdown details + failure_breakdown = { + c.category: c.count for c in eval_resp.failure_analysis.categories + } + failure_breakdown.update( + { + "agent_crash": eval_resp.failure_analysis.agent_crash_count, + "sql_execution_failure": eval_resp.failure_analysis.sql_execution_failure_count, + "trino_failure": eval_resp.failure_analysis.trino_failure_count, + "timeout": eval_resp.failure_analysis.timeout_count, + "validation_failure": eval_resp.failure_analysis.validation_failure_count, + } + ) + run.failure_breakdown = failure_breakdown + + run.dimension_averages = { + "contains_execution_accuracy": eval_resp.accuracy.contains_accuracy, + "exact_execution_accuracy": eval_resp.accuracy.execution_accuracy, + "exact_match": eval_resp.accuracy.sql_exact_match, + "time_shift_score": eval_resp.accuracy.time_shift_score, + "component_match": eval_resp.accuracy.component_match, + "schema_hallucination": eval_resp.accuracy.schema_hallucination, + "dialect_error": eval_resp.accuracy.dialect_error, + } + session.add(run) + + # Only insert EvalResult rows for question_ids that are valid FK references + # (i.e. exist in golden_questions). External benchmark cases (e.g. Spider2) won't match. + if eval_resp.cases: + case_ids = [c.question_id for c in eval_resp.cases] + valid_ids = set( + session.exec( + select(GoldenQuestion.id).where(GoldenQuestion.id.in_(case_ids)) + ).all() + ) + for case in eval_resp.cases: + if case.question_id in valid_ids: + session.add( + EvalResult( + run_id=run_id, + question_id=case.question_id, + score=case.scores.get("contains_accuracy", 0.0), + status="pass" if case.succeeded else "fail", + error_type=case.error, + ) + ) + + @observe(name="eval-single-table") def execute_single_table_eval(table_id: str, run_id: str, session: Session) -> float: run = session.get(EvalRun, run_id) @@ -143,10 +241,10 @@ def execute_single_table_eval(table_id: str, run_id: str, session: Session) -> f if not questions: run.status = EvalStatus.failed - run.score = -1.0 + run.score = 0.0 session.add(run) session.commit() - return -1.0 + return 0.0 langfuse_context.update_current_trace( metadata={"table_id": table_id, "run_id": run_id}, @@ -157,52 +255,36 @@ def execute_single_table_eval(table_id: str, run_id: str, session: Session) -> f dataset_name = f"text2sql_sandbox_{table_id}" if langfuse_client.enabled: - langfuse_client.ensure_dataset_synced( - dataset_name, _build_questions_payload(questions, table) - ) + try: + langfuse_client.ensure_dataset_synced( + dataset_name, _build_questions_payload(questions, table) + ) + except Exception as e: + logger.warning( + f"[Eval] Langfuse dataset sync failed (eval will continue): {e}" + ) try: - req = EvalAPIRequest(tables_names=[table.name], dataset_name=dataset_name) + req = { + "dataset_name": dataset_name, + "additional_tables": [table.name], + } resp = requests.post( f"{settings.EVALUATION_SERVICE_URL}/text-to-sql/evaluation/run-single-dataset", - json=req.model_dump(), + json=req, timeout=600, ) resp.raise_for_status() - eval_resp = EvalAPIResponse(**resp.json()) - if eval_resp.status == "failed": - raise Exception("API returned failed status") + eval_resp = RunDatasetResponse(**resp.json()) except Exception as e: logger.error(f"[Eval] Table {table_id} evaluation failed via API: {e}") run.status = EvalStatus.failed - run.score = -1.0 + run.score = 0.0 session.add(run) session.commit() - return -1.0 + return 0.0 - metrics = eval_resp.overall_metrics - run.score = metrics.contains_execution_accuracy - run.pass_rate = metrics.pass_rate - run.fail_rate = metrics.fail_rate - run.total_questions = metrics.total_questions - run.status = EvalStatus.completed - run.completed_at = datetime.now() - run.dimension_averages = { - "contains_execution_accuracy": metrics.contains_execution_accuracy, - "exact_execution_accuracy": metrics.exact_execution_accuracy, - "exact_match": metrics.exact_match, - } - session.add(run) - - for q_res in eval_resp.results: - session.add( - EvalResult( - run_id=run_id, - question_id=q_res.question_id, - score=q_res.metrics.contains_execution_accuracy, - status=q_res.status, - ) - ) + _map_and_save_run_metrics(run, eval_resp, session, run_id) # Lifecycle: draft → sandbox on first evaluation only if table and table.status == TableStatus.draft: @@ -212,17 +294,17 @@ def execute_single_table_eval(table_id: str, run_id: str, session: Session) -> f session.commit() logger.info( - f"[Eval] Table {table_id}: contains_exec_accuracy={metrics.contains_execution_accuracy} " - f"exact_exec_accuracy={metrics.exact_execution_accuracy} exact_match={metrics.exact_match} " - f"({metrics.total_questions} questions, pass_rate={metrics.pass_rate})" + f"[Eval] Table {table_id}: contains_accuracy={eval_resp.accuracy.contains_accuracy} " + f"exec_accuracy={eval_resp.accuracy.execution_accuracy} exact_match={eval_resp.accuracy.sql_exact_match} " + f"({eval_resp.total_cases} questions, pass_rate={1.0 - eval_resp.failure_rate})" ) langfuse_context.update_current_trace( output={ - "score": metrics.contains_execution_accuracy, - "pass_rate": metrics.pass_rate, + "score": eval_resp.accuracy.contains_accuracy, + "pass_rate": 1.0 - eval_resp.failure_rate, } ) - return metrics.contains_execution_accuracy + return eval_resp.accuracy.contains_accuracy # ─── Phase A: measure baseline score on production dataset ──────────────────── @@ -274,57 +356,34 @@ def _run_production_dataset_eval( table_names = [t.name for t in prod_tables] try: - req = EvalAPIRequest( - tables_names=table_names, dataset_name=PRODUCTION_DATASET_NAME - ) + req = { + "dataset_name": PRODUCTION_DATASET_NAME, + "additional_tables": table_names, + } resp = requests.post( f"{settings.EVALUATION_SERVICE_URL}/text-to-sql/evaluation/run-single-dataset", - json=req.model_dump(), + json=req, timeout=600, ) resp.raise_for_status() - eval_resp = EvalAPIResponse(**resp.json()) - if eval_resp.status == "failed": - raise Exception("API returned failed status") + eval_resp = RunDatasetResponse(**resp.json()) except Exception as e: logger.error(f"[Promotion/Phase-A] Baseline eval failed: {e}") run.status = EvalStatus.failed - run.score = -1.0 + run.score = 0.0 session.add(run) session.commit() - return -1.0 + return 0.0 - metrics = eval_resp.overall_metrics - run.score = metrics.contains_execution_accuracy - run.pass_rate = metrics.pass_rate - run.fail_rate = metrics.fail_rate - run.total_questions = metrics.total_questions - run.status = EvalStatus.completed - run.completed_at = datetime.now() - run.dimension_averages = { - "contains_execution_accuracy": metrics.contains_execution_accuracy, - "exact_execution_accuracy": metrics.exact_execution_accuracy, - "exact_match": metrics.exact_match, - } - session.add(run) - - for q_res in eval_resp.results: - session.add( - EvalResult( - run_id=run.id, - question_id=q_res.question_id, - score=q_res.metrics.contains_execution_accuracy, - status=q_res.status, - ) - ) + _map_and_save_run_metrics(run, eval_resp, session, run.id) session.commit() logger.info( - f"[Promotion/Phase-A] Baseline contains_exec_accuracy = {metrics.contains_execution_accuracy:.3f} " - f"exact_exec_accuracy = {metrics.exact_execution_accuracy:.3f} exact_match = {metrics.exact_match:.3f} " - f"({metrics.total_questions} questions)" + f"[Promotion/Phase-A] Baseline contains_exec_accuracy = {eval_resp.accuracy.contains_accuracy:.3f} " + f"exact_exec_accuracy = {eval_resp.accuracy.execution_accuracy:.3f} exact_match = {eval_resp.accuracy.sql_exact_match:.3f} " + f"({eval_resp.total_cases} questions)" ) - return metrics.contains_execution_accuracy + return eval_resp.accuracy.contains_accuracy # We no longer use a fixed dataset name to avoid soft-delete conflicts and question accumulation. @@ -359,60 +418,37 @@ def _run_candidate_eval( logger.error(f"[Promotion/Phase-B] Candidate eval prep failed: {e}") try: - req = EvalAPIRequest(tables_names=[table.name], dataset_name=dataset_name) + req = { + "dataset_name": dataset_name, + "additional_tables": [table.name], + } resp = requests.post( f"{settings.EVALUATION_SERVICE_URL}/text-to-sql/evaluation/run-single-dataset", - json=req.model_dump(), + json=req, timeout=600, ) resp.raise_for_status() - eval_resp = EvalAPIResponse(**resp.json()) - if eval_resp.status == "failed": - raise Exception("API returned failed status") + eval_resp = RunDatasetResponse(**resp.json()) except Exception as e: logger.error(f"[Promotion/Phase-B] Candidate eval failed: {e}") run.status = EvalStatus.failed - run.score = -1.0 + run.score = 0.0 session.add(run) session.commit() - return -1.0 + return 0.0 - metrics = eval_resp.overall_metrics - run.score = metrics.contains_execution_accuracy - run.pass_rate = metrics.pass_rate - run.fail_rate = metrics.fail_rate - run.total_questions = metrics.total_questions - run.status = EvalStatus.completed - run.completed_at = datetime.now() - run.dimension_averages = { - "contains_execution_accuracy": metrics.contains_execution_accuracy, - "exact_execution_accuracy": metrics.exact_execution_accuracy, - "exact_match": metrics.exact_match, - } - session.add(run) - - for q_res in eval_resp.results: - session.add( - EvalResult( - run_id=run.id, - question_id=q_res.question_id, - score=q_res.metrics.contains_execution_accuracy, - status=q_res.status, - ) - ) + _map_and_save_run_metrics(run, eval_resp, session, run.id) session.commit() logger.info( - f"[Promotion/Phase-B] Candidate '{table.name}' contains_score = {metrics.contains_execution_accuracy:.3f} " - f"exact_score = {metrics.exact_execution_accuracy:.3f} exact_match = {metrics.exact_match:.3f}" + f"[Promotion/Phase-B] Candidate '{table.name}' contains_score = {eval_resp.accuracy.contains_accuracy:.3f} " + f"exact_score = {eval_resp.accuracy.execution_accuracy:.3f} exact_match = {eval_resp.accuracy.sql_exact_match:.3f}" ) if langfuse_client.enabled: - # Since API might be async or Langfuse is async, we may still need to clear dataset - # Here we don't wait for traces, just clear it after evaluation finishes langfuse_client.clear_dataset(dataset_name) - return metrics.contains_execution_accuracy + return eval_resp.accuracy.contains_accuracy def _run_regression_eval( @@ -441,56 +477,33 @@ def _run_regression_eval( session.refresh(run) try: - req = EvalAPIRequest( - tables_names=table_names, dataset_name=PRODUCTION_DATASET_NAME - ) + req = { + "dataset_name": PRODUCTION_DATASET_NAME, + "additional_tables": table_names, + } resp = requests.post( f"{settings.EVALUATION_SERVICE_URL}/text-to-sql/evaluation/run-single-dataset", - json=req.model_dump(), + json=req, timeout=600, ) resp.raise_for_status() - eval_resp = EvalAPIResponse(**resp.json()) - if eval_resp.status == "failed": - raise Exception("API returned failed status") + eval_resp = RunDatasetResponse(**resp.json()) except Exception as e: logger.error(f"[Promotion/Phase-B] Regression eval failed: {e}") run.status = EvalStatus.failed - run.score = -1.0 + run.score = 0.0 session.add(run) session.commit() - return -1.0 + return 0.0 - metrics = eval_resp.overall_metrics - run.score = metrics.contains_execution_accuracy - run.pass_rate = metrics.pass_rate - run.fail_rate = metrics.fail_rate - run.total_questions = metrics.total_questions - run.status = EvalStatus.completed - run.completed_at = datetime.now() - run.dimension_averages = { - "contains_execution_accuracy": metrics.contains_execution_accuracy, - "exact_execution_accuracy": metrics.exact_execution_accuracy, - "exact_match": metrics.exact_match, - } - session.add(run) - - for q_res in eval_resp.results: - session.add( - EvalResult( - run_id=run.id, - question_id=q_res.question_id, - score=q_res.metrics.contains_execution_accuracy, - status=q_res.status, - ) - ) + _map_and_save_run_metrics(run, eval_resp, session, run.id) session.commit() logger.info( - f"[Promotion/Phase-B] Regression contains_score (with candidate) = {metrics.contains_execution_accuracy:.3f} " - f"exact_score = {metrics.exact_execution_accuracy:.3f} exact_match = {metrics.exact_match:.3f}" + f"[Promotion/Phase-B] Regression contains_score (with candidate) = {eval_resp.accuracy.contains_accuracy:.3f} " + f"exact_score = {eval_resp.accuracy.execution_accuracy:.3f} exact_match = {eval_resp.accuracy.sql_exact_match:.3f}" ) - return metrics.contains_execution_accuracy + return eval_resp.accuracy.contains_accuracy # ─── Main promotion workflow ─────────────────────────────────────────────────── diff --git a/docker-compose.yml b/docker-compose.yml index 7404e92..d5e58ab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -451,6 +451,7 @@ services: - KEYCLOAK_CLIENT_SECRET=${KEYCLOAK_CLIENT_SECRET} - FRONTEND_DEFAULT_REDIRECT=${FRONTEND_DEFAULT_REDIRECT} - SSO_REDIRECT_URI=${SSO_REDIRECT_URI} + - SESSION_SECRET_KEY=${SESSION_SECRET_KEY:-text2sql-dev-session-secret-change-in-prod} depends_on: db: condition: service_healthy diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 46b6809..2f0412b 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -2,6 +2,9 @@ server { listen 8080; server_name localhost; + # Use Docker's internal DNS so nginx re-resolves container IPs after restarts + resolver 127.0.0.11 valid=10s ipv6=off; + location / { root /usr/share/nginx/html; index index.html index.htm; @@ -10,7 +13,8 @@ server { # Proxy API requests to the backend configured via BACKEND_URL environment variable location /api/ { - proxy_pass ${BACKEND_URL}; + set $backend_url ${BACKEND_URL}; + proxy_pass $backend_url; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; diff --git a/infra/trino/etc/jvm.config b/infra/trino/etc/jvm.config index 17f7f39..71800bc 100644 --- a/infra/trino/etc/jvm.config +++ b/infra/trino/etc/jvm.config @@ -1,5 +1,5 @@ -server --Xmx1G +-Xmx2G -XX:+UseG1GC -XX:G1HeapRegionSize=32M -XX:+ExplicitGCInvokesConcurrent From 4eaa07a7b05792ab183396b2d97d04cbb4f82b72 Mon Sep 17 00:00:00 2001 From: Yodan Bargida Date: Thu, 2 Jul 2026 15:11:46 +0300 Subject: [PATCH 04/11] fix(): make langfuse result query --- backend/app/routers/evaluation.py | 275 ++++++++++++++++++++------- backend/app/routers/orchestration.py | 239 ++++++++++++----------- backend/app/services/evaluator.py | 5 +- 3 files changed, 334 insertions(+), 185 deletions(-) diff --git a/backend/app/routers/evaluation.py b/backend/app/routers/evaluation.py index 230aaed..73c3802 100644 --- a/backend/app/routers/evaluation.py +++ b/backend/app/routers/evaluation.py @@ -27,12 +27,13 @@ EvalRunRead, EvalStatus, EvaluationAlert, + EvaluationHistoryMetric, GoldenQuestion, Table, TableStatus, ) from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException -from langfuse import observe +from langfuse import observe, propagate_attributes from pydantic import BaseModel from sqlmodel import Session, desc, select @@ -218,97 +219,96 @@ def _map_and_save_run_metrics( ) for case in eval_resp.cases: if case.question_id in valid_ids: + score = case.scores.get("contains_accuracy", 0.0) + status = "pass" if score >= 0.5 else "fail" + error_type = None if status == "pass" else case.error session.add( EvalResult( run_id=run_id, question_id=case.question_id, - score=case.scores.get("contains_accuracy", 0.0), - status="pass" if case.succeeded else "fail", - error_type=case.error, + score=score, + status=status, + error_type=error_type, ) ) @observe(name="eval-single-table") def execute_single_table_eval(table_id: str, run_id: str, session: Session) -> float: - run = session.get(EvalRun, run_id) - if not run: - return -1.0 + with propagate_attributes( + metadata={"table_id": table_id, "run_id": run_id}, + tags=["eval-run", f"table:{table_id}"], + ): + run = session.get(EvalRun, run_id) + if not run: + return -1.0 - questions = session.exec( - select(GoldenQuestion).where(GoldenQuestion.table_id == table_id) - ).all() + questions = session.exec( + select(GoldenQuestion).where(GoldenQuestion.table_id == table_id) + ).all() - if not questions: - run.status = EvalStatus.failed - run.score = 0.0 - session.add(run) - session.commit() - return 0.0 + if not questions: + run.status = EvalStatus.failed + run.score = 0.0 + session.add(run) + session.commit() + return 0.0 - if langfuse_client.client and langfuse_client.client.get_current_trace_id(): - langfuse_client.client.trace( - id=langfuse_client.client.get_current_trace_id(), - metadata={"table_id": table_id, "run_id": run_id}, - tags=["eval-run", f"table:{table_id}"], - ) + table = session.get(Table, table_id) + dataset_name = f"text2sql_sandbox_{table_id}" - table = session.get(Table, table_id) - dataset_name = f"text2sql_sandbox_{table_id}" + if langfuse_client.enabled: + try: + langfuse_client.ensure_dataset_synced( + dataset_name, _build_questions_payload(questions, table) + ) + except Exception as e: + logger.warning( + f"[Eval] Langfuse dataset sync failed (eval will continue): {e}" + ) - if langfuse_client.enabled: try: - langfuse_client.ensure_dataset_synced( - dataset_name, _build_questions_payload(questions, table) + req = { + "dataset_name": dataset_name, + "additional_tables": [table.name], + } + resp = requests.post( + f"{settings.EVALUATION_SERVICE_URL}/text-to-sql/evaluation/run-single-dataset", + json=req, + timeout=600, ) + resp.raise_for_status() + eval_resp = RunDatasetResponse(**resp.json()) except Exception as e: - logger.warning( - f"[Eval] Langfuse dataset sync failed (eval will continue): {e}" - ) - - try: - req = { - "dataset_name": dataset_name, - "additional_tables": [table.name], - } - resp = requests.post( - f"{settings.EVALUATION_SERVICE_URL}/text-to-sql/evaluation/run-single-dataset", - json=req, - timeout=600, - ) - resp.raise_for_status() - eval_resp = RunDatasetResponse(**resp.json()) - except Exception as e: - logger.error(f"[Eval] Table {table_id} evaluation failed via API: {e}") - run.status = EvalStatus.failed - run.score = 0.0 - session.add(run) - session.commit() - return 0.0 + logger.error(f"[Eval] Table {table_id} evaluation failed via API: {e}") + run.status = EvalStatus.failed + run.score = 0.0 + session.add(run) + session.commit() + return 0.0 - _map_and_save_run_metrics(run, eval_resp, session, run_id) + _map_and_save_run_metrics(run, eval_resp, session, run_id) - # Lifecycle: draft → sandbox on first evaluation only - if table and table.status == TableStatus.draft: - table.status = TableStatus.sandbox - session.add(table) + # Lifecycle: draft → sandbox on first evaluation only + if table and table.status == TableStatus.draft: + table.status = TableStatus.sandbox + session.add(table) - session.commit() + session.commit() - logger.info( - f"[Eval] Table {table_id}: contains_accuracy={eval_resp.accuracy.contains_accuracy} " - f"exec_accuracy={eval_resp.accuracy.execution_accuracy} exact_match={eval_resp.accuracy.sql_exact_match} " - f"({eval_resp.total_cases} questions, pass_rate={1.0 - eval_resp.failure_rate})" - ) - if langfuse_client.client and langfuse_client.client.get_current_trace_id(): - langfuse_client.client.trace( - id=langfuse_client.client.get_current_trace_id(), - output={ - "score": eval_resp.accuracy.contains_accuracy, - "pass_rate": 1.0 - eval_resp.failure_rate, - }, + logger.info( + f"[Eval] Table {table_id}: contains_accuracy={eval_resp.accuracy.contains_accuracy} " + f"exec_accuracy={eval_resp.accuracy.execution_accuracy} exact_match={eval_resp.accuracy.sql_exact_match} " + f"({eval_resp.total_cases} questions, pass_rate={1.0 - eval_resp.failure_rate})" ) - return eval_resp.accuracy.contains_accuracy + if langfuse_client.client and langfuse_client.client.get_current_trace_id(): + langfuse_client.client.set_current_trace_io( + output={ + "score": eval_resp.accuracy.contains_accuracy, + "pass_rate": 1.0 - eval_resp.failure_rate, + }, + ) + return eval_resp.accuracy.contains_accuracy # ─── Phase A: measure baseline score on production dataset ──────────────────── @@ -627,10 +627,145 @@ def promote_table_to_production_workflow(table_id: str, run_id: str): logger.info(f"[Promotion] Done. Table '{table.name}' → {table.status}") +REGRESSION_BLOCK_DELTA = 0.10 +REGRESSION_WARNING_DELTA = 0.05 +LOW_SCORE_THRESHOLD = 0.70 + + +def _create_alert( + session: Session, + run_id: str | None, + table_id: str | None, + alert_type: str, + severity: AlertSeverity, + message: str, + details: dict | None = None, +): + alert = EvaluationAlert( + run_id=run_id, + table_id=table_id, + alert_type=alert_type, + severity=severity, + message=message, + details=details, + ) + session.add(alert) + session.commit() + + @observe(name="eval-run") def _run_evaluation_pipeline(table_id: str, run_id: str): with Session(engine) as session: - execute_single_table_eval(table_id, run_id, session) + run = session.get(EvalRun, run_id) + if not run: + return + + try: + score = execute_single_table_eval(table_id, run_id, session) + + # Fetch updated run state + session.refresh(run) + + # Detect regression vs previous run + prev_runs = session.exec( + select(EvalRun) + .where( + EvalRun.table_id == table_id, + EvalRun.status == EvalStatus.completed, + EvalRun.id != run_id, + ) + .order_by(EvalRun.created_at.desc()) + .limit(1) + ).first() + + regression_detected = False + regression_delta = None + + if prev_runs and prev_runs.score > 0: + delta = prev_runs.score - score + if delta > REGRESSION_BLOCK_DELTA: + regression_detected = True + regression_delta = round(delta, 4) + _create_alert( + session, + run_id, + table_id, + "regression", + AlertSeverity.critical, + f"Score dropped {delta:.1%} (from {prev_runs.score:.2f} → {score:.2f})", + { + "previous_score": prev_runs.score, + "current_score": score, + "delta": delta, + }, + ) + elif delta > REGRESSION_WARNING_DELTA: + regression_detected = True + regression_delta = round(delta, 4) + _create_alert( + session, + run_id, + table_id, + "regression", + AlertSeverity.warning, + f"Score warning: {delta:.1%} drop detected", + { + "previous_score": prev_runs.score, + "current_score": score, + "delta": delta, + }, + ) + + # Low score alert + if score < LOW_SCORE_THRESHOLD: + _create_alert( + session, + run_id, + table_id, + "low_score", + AlertSeverity.warning, + f"Table performance is low ({score:.1%})", + ) + + # Finalize run orchestration fields + run.regression_detected = regression_detected + run.regression_delta = regression_delta + session.add(run) + + # Persist metrics for analytics + metrics_to_save = [ + ("accuracy", score if score is not None else 0.0), + ] + if run.dimension_averages: + for dim_name, dim_val in run.dimension_averages.items(): + metrics_to_save.append( + (dim_name, dim_val if dim_val is not None else 0.0) + ) + if run.failure_breakdown: + for fail_type, count in run.failure_breakdown.items(): + metrics_to_save.append( + (fail_type, float(count) if count is not None else 0.0) + ) + + for metric_name, m_val in metrics_to_save: + metric = EvaluationHistoryMetric( + run_id=run_id, metric_name=metric_name, metric_value=m_val + ) + session.add(metric) + + session.commit() + except Exception as e: + logger.error( + f"[Eval] Error in evaluation pipeline run {run_id}: {e}", exc_info=True + ) + try: + session.refresh(run) + run.status = EvalStatus.failed + run.score = 0.0 + session.add(run) + session.commit() + except Exception as db_err: + logger.error(f"[Eval] Failed to mark run {run_id} as failed: {db_err}") # ─── Endpoints ──────────────────────────────────────────────────────────────── diff --git a/backend/app/routers/orchestration.py b/backend/app/routers/orchestration.py index 653a175..60024fd 100644 --- a/backend/app/routers/orchestration.py +++ b/backend/app/routers/orchestration.py @@ -10,6 +10,7 @@ System: GET /evaluations/system-health """ +import logging from datetime import datetime, timedelta from core.db.engine import engine, get_session @@ -31,11 +32,12 @@ Table, ) from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query -from langfuse import observe +from langfuse import observe, propagate_attributes from sqlmodel import Session, select from app.routers.evaluation import execute_single_table_eval -from app.services.langfuse_client import langfuse_client + +logger = logging.getLogger(__name__) router = APIRouter(prefix="/evaluations", tags=["evaluation-orchestration"]) @@ -56,117 +58,130 @@ def _run_full_pipeline( table_ids: list[str], run_ids: list[str], triggered_by: str = "user" ): """Run evaluation for multiple tables (one run per table).""" - if langfuse_client.client and langfuse_client.client.get_current_trace_id(): - langfuse_client.client.trace( - id=langfuse_client.client.get_current_trace_id(), - tags=["evaluation_run"], - metadata={ - "table_ids": table_ids, - "run_ids": run_ids, - "triggered_by": triggered_by, - }, - ) - - for table_id, run_id in zip(table_ids, run_ids, strict=False): - with Session(engine) as session: - run = session.get(EvalRun, run_id) - if not run: - continue - - # ── Delegate to shared core logic ─────────────────────────── - score = execute_single_table_eval(table_id, run_id, session) - - # Fetch updated run state - session.refresh(run) - - # Detect regression vs previous run - prev_runs = session.exec( - select(EvalRun) - .where( - EvalRun.table_id == table_id, - EvalRun.status == EvalStatus.completed, - EvalRun.id != run_id, - ) - .order_by(EvalRun.created_at.desc()) - .limit(1) - ).first() - - regression_detected = False - regression_delta = None - - if prev_runs and prev_runs.score > 0: - delta = prev_runs.score - score - if delta > REGRESSION_BLOCK_DELTA: - regression_detected = True - regression_delta = round(delta, 4) - _create_alert( - session, - run_id, - table_id, - "regression", - AlertSeverity.critical, - f"Score dropped {delta:.1%} (from {prev_runs.score:.2f} → {score:.2f})", - { - "previous_score": prev_runs.score, - "current_score": score, - "delta": delta, - }, - ) - elif delta > REGRESSION_WARNING_DELTA: - regression_detected = True - regression_delta = round(delta, 4) - _create_alert( - session, - run_id, - table_id, - "regression", - AlertSeverity.warning, - f"Score warning: {delta:.1%} drop detected", - { - "previous_score": prev_runs.score, - "current_score": score, - "delta": delta, - }, - ) - - # Low score alert - if score < LOW_SCORE_THRESHOLD: - _create_alert( - session, - run_id, - table_id, - "low_score", - AlertSeverity.warning, - f"Table performance is low ({score:.1%})", - ) - - # Finalize run orchestration fields - run.regression_detected = regression_detected - run.regression_delta = regression_delta - session.add(run) - - # Persist metrics for analytics - metrics_to_save = [ - ("accuracy", score if score is not None else 0.0), - ] - if run.dimension_averages: - for dim_name, dim_val in run.dimension_averages.items(): - metrics_to_save.append( - (dim_name, dim_val if dim_val is not None else 0.0) - ) - if run.failure_breakdown: - for fail_type, count in run.failure_breakdown.items(): - metrics_to_save.append( - (fail_type, float(count) if count is not None else 0.0) + with propagate_attributes( + tags=["evaluation_run"], + metadata={ + "table_ids": table_ids, + "run_ids": run_ids, + "triggered_by": triggered_by, + }, + ): + for table_id, run_id in zip(table_ids, run_ids, strict=False): + with Session(engine) as session: + run = session.get(EvalRun, run_id) + if not run: + continue + + try: + # ── Delegate to shared core logic ─────────────────────────── + score = execute_single_table_eval(table_id, run_id, session) + + # Fetch updated run state + session.refresh(run) + + # Detect regression vs previous run + prev_runs = session.exec( + select(EvalRun) + .where( + EvalRun.table_id == table_id, + EvalRun.status == EvalStatus.completed, + EvalRun.id != run_id, + ) + .order_by(EvalRun.created_at.desc()) + .limit(1) + ).first() + + regression_detected = False + regression_delta = None + + if prev_runs and prev_runs.score > 0: + delta = prev_runs.score - score + if delta > REGRESSION_BLOCK_DELTA: + regression_detected = True + regression_delta = round(delta, 4) + _create_alert( + session, + run_id, + table_id, + "regression", + AlertSeverity.critical, + f"Score dropped {delta:.1%} (from {prev_runs.score:.2f} → {score:.2f})", + { + "previous_score": prev_runs.score, + "current_score": score, + "delta": delta, + }, + ) + elif delta > REGRESSION_WARNING_DELTA: + regression_detected = True + regression_delta = round(delta, 4) + _create_alert( + session, + run_id, + table_id, + "regression", + AlertSeverity.warning, + f"Score warning: {delta:.1%} drop detected", + { + "previous_score": prev_runs.score, + "current_score": score, + "delta": delta, + }, + ) + + # Low score alert + if score < LOW_SCORE_THRESHOLD: + _create_alert( + session, + run_id, + table_id, + "low_score", + AlertSeverity.warning, + f"Table performance is low ({score:.1%})", + ) + + # Finalize run orchestration fields + run.regression_detected = regression_detected + run.regression_delta = regression_delta + session.add(run) + + # Persist metrics for analytics + metrics_to_save = [ + ("accuracy", score if score is not None else 0.0), + ] + if run.dimension_averages: + for dim_name, dim_val in run.dimension_averages.items(): + metrics_to_save.append( + (dim_name, dim_val if dim_val is not None else 0.0) + ) + if run.failure_breakdown: + for fail_type, count in run.failure_breakdown.items(): + metrics_to_save.append( + (fail_type, float(count) if count is not None else 0.0) + ) + + for metric_name, m_val in metrics_to_save: + metric = EvaluationHistoryMetric( + run_id=run_id, metric_name=metric_name, metric_value=m_val + ) + session.add(metric) + + session.commit() + except Exception as e: + logger.error( + f"[Orchestrator] Error evaluating table {table_id} in run {run_id}: {e}", + exc_info=True, ) - - for metric_name, m_val in metrics_to_save: - metric = EvaluationHistoryMetric( - run_id=run_id, metric_name=metric_name, metric_value=m_val - ) - session.add(metric) - - session.commit() + try: + session.refresh(run) + run.status = EvalStatus.failed + run.score = 0.0 + session.add(run) + session.commit() + except Exception as db_err: + logger.error( + f"[Orchestrator] Failed to mark run {run_id} as failed: {db_err}" + ) def _create_alert( diff --git a/backend/app/services/evaluator.py b/backend/app/services/evaluator.py index faff208..97c2fcc 100644 --- a/backend/app/services/evaluator.py +++ b/backend/app/services/evaluator.py @@ -149,8 +149,7 @@ def task(self, item) -> dict[str, Any]: ) if _lf_client.client and trace_id: - _lf_client.client.trace( - id=trace_id, + _lf_client.client.set_current_trace_io( input={ "query": question_obj.question, "databases": [question_obj.table_id], @@ -167,7 +166,7 @@ def task(self, item) -> dict[str, Any]: generated_sql = f"SELECT * FROM {question_obj.table_id} LIMIT 100" # STUB if _lf_client.client and trace_id: - _lf_client.client.trace(id=trace_id, output={"response": generated_sql}) + _lf_client.client.set_current_trace_io(output={"response": generated_sql}) # Persist EvalResult (score will be updated by evaluators after task returns) result_db = EvalResult( From 70fee4ffe1f7b4f8648f08660ec179bfc5fdb3e2 Mon Sep 17 00:00:00 2001 From: Yodan Bargida Date: Tue, 7 Jul 2026 16:17:39 +0300 Subject: [PATCH 05/11] add prompts --- agent/scripts/upload_all_prompts.py | 369 ++++++++++++++++++++++++---- 1 file changed, 316 insertions(+), 53 deletions(-) diff --git a/agent/scripts/upload_all_prompts.py b/agent/scripts/upload_all_prompts.py index deb2cdb..6fea9cc 100644 --- a/agent/scripts/upload_all_prompts.py +++ b/agent/scripts/upload_all_prompts.py @@ -26,26 +26,69 @@ def main(): { "role": "system", "content": ( - "You are a query enrichment assistant for a text-to-SQL system.\n\n" - "Your job is to read the user's natural-language query and add context that " - "makes ambiguous or implicit terms clearer for downstream processing.\n\n" - "Add enrichment entries for things like:\n" - " • Abbreviations or acronyms that have a specific meaning " - "(e.g. 'MDA' → 'Magen David Adom')\n" - " • Relative time expressions that can be resolved to absolute dates " - "(e.g. 'last quarter' → 'Q1 2025, Jan 1 – Mar 31 2025')\n" - " • Ambiguous proper nouns where context helps " - "(e.g. 'Jordan' used as a country vs. a person's name)\n" - " • Domain-specific shorthand the downstream system may not know\n\n" - "Do NOT try to identify which database table or column to use — that is handled by a " - "separate schema exploration phase.\n" - "Do NOT add enrichments for terms that are already fully clear from the query.\n" - "If the query needs no enrichment, return an empty enrichments list." + """ + You are a percise text extraction assistant. Specifically, Your role is to identify and translate location names from short Hebrew texts with absolute accuracy. You are decisive and follow formatting rules without deviation. + + **Your Goal**: Given an input of 1-2 Hebrew sentences, extract every location mentioned in the text and translate it to English. + + # Instructions: + 1. Analyze the given text and identify every location name that is mentioned in it (cities, countries, landmarks, streets or regions). + 2. Extract the mentioned locations from the text and translate them. + 3. Output a structured JSON object according to the **Output Format**. + + # Output Format: + Your output must strictly follow this JSON structure: + ```json + { + "hebrew_location_name1": , + "hebrew_location_name2": , + ... + } + ``` + + # Constraints: + * NEVER include introductory or concluding text (e.g "here is the json..."). + * ALWAYS output valid JSON format. + * For the "english" key, ALWAYS provide the standard, commonly accepted English spelling of the Hebrew location extracted. + * Empty results: If no locations are found, return empty lists for both keys. + """ ) }, { "role": "user", - "content": "{{user_query}}" + "content": ( + """ + Text: + "כמה חיילים נפצעו בחאן יונס וברפיח במאי 2025" + Output: + ```json + { + "חאן יונס": "khan_yunis", + "רפיח": "rafah" + } + ``` + + Text: + "מהם הסיפורים מהחטיבות המרכזיות שלהם עוקבות את האחריות המרבית האחרונה של שיגורים שקלטו בשטח עירון" + Output: + ```json + {{}} + ``` + + Text: + "איזה יחידות מפיקוד מרכז נמצאות ברגע במחנה נחשונים" + Output: + ```json + { + "מחנה נחשונים": "nahshonim_camp" + } + ``` + + Text: + "{{user_query}}" + Output: + """ + ) } ], "type": "chat" @@ -56,30 +99,28 @@ def main(): { "role": "system", "content": ( - "You are a Schema Explorer sub-agent. Your goal is to identify the most relevant tables " - "and inspect their column details to form a query plan for the user's question.\n\n" - "Candidate Tables found:\n{{tables_json}}\n\n" - "Detailed Profiles for top tables (with Esca Reference IDs):\n{{profiles_json}}\n\n" - "## Decision-Making Rules\n\n" - "You MUST make all planning decisions autonomously. This includes:\n" - "- Join strategy: If multiple tables are needed to answer the query, decide which tables to join and on which keys — do NOT ask the user.\n" - "- Column selection: Choose the most appropriate columns yourself.\n" - "- Filter strategy: Infer filters from the user's question.\n" - "- Table selection when one is clearly more appropriate: Pick the best match and proceed.\n\n" - "## When to Flag Ambiguity (ONLY these cases)\n\n" - "Set ambiguity_detected=true ONLY IF:\n" - "1. Two or more completely independent tables could each independently and fully answer the user's question, " - "and you have no way to determine which one the user wants (e.g., 'orders' vs 'orders_archive' with no " - "indication of time range, or two fact tables from different business domains that both seem equally relevant).\n" - "2. There is no table in the catalog that can answer the query at all.\n\n" - "Do NOT flag ambiguity for: join decisions, column choices, filter logic, or any decision you can make yourself.\n\n" - "## Output Format\n\n" - "Output MUST be a valid JSON object with the following keys:\n" - "- schema_plan: detailed query plan describing tables, columns, joins, and Esca reference IDs (empty string if ambiguity_detected is true)\n" - "- ambiguity_detected: boolean, true ONLY in the hard-blocker cases described above\n" - "- ambiguity_message: a concise question to ask the user to resolve the blocker (empty string if ambiguity_detected is false)\n" - "- candidate_options: list of strings (table names or options) for the user to choose from (empty list if ambiguity_detected is false)\n" - "Return only the raw JSON, no markdown formatting (no ```json code blocks)." + """ + You are a Schema Explorer sub-agent. Your goal is to identify the most relevant tables and inspect their column details to form a query plan for the user's question. + + Candidate Tables found: + {{tables_json}} + + Detailed Profiles for top tables (with Esca Reference IDs): + {{profiles_json}} + + ## Decision-Making Rules + + You MUST make all planning decisions autonomously. This includes: + - Join strategy: If multiple tables are needed to answer the query, decide which tables to join and on which keys — do NOT ask the user. + - Column selection: Choose the most appropriate columns yourself. + - Filter strategy: Infer filters from the user's question. + - Table selection: When one table is clearly more appropriate, pick the best match and proceed. + - When uncertain between two tables, pick the most semantically appropriate one and document your reasoning in schema_plan. + + ## Output Instructions + + Provide your output matching the requested schema. Ensure that `schema_plan` is a detailed string explanation, and `tables_used` is a list of table names. + """ ) }, { @@ -94,11 +135,57 @@ def main(): "prompt": [ { "role": "system", - "content": "You are a SQL expert who specializes in trino. Build a SQL query based on the plan and user query. Output ONLY the SQL query, nothing else." + "content": ( + """ + You are an expert Trino SQL Composer agent. Your primary function is to translate a user's request, into a single, efficient Trino SQL query for a specific database. + Your Goal: Generate a syntactically correct SQL query that retrieves data relevant to the user's request from a specified database. + You will receive the following inputs: + + 1. Database Schema: A detailed description of the tables, their columns, and their data types. + 2. User Request: A question/request from the user in Hebrew. + Instructions: + + 1. Decompose the user request. If possible, decompose the request into sub-questions that each can be answered by a simple SQL query. List the sub questions in the order they should be answered. + 2. Compose SQL queries: Write a single, executable Trino SQL statement for each sub-question in their order, re-using the result of previous queries so that at the end you have one final query that returns the answer to the original user request. + Final Output: Your final output must be only the final SQL query that answers the entire user request. + SQL Syntax Guidelines: + + 1. Use CTEs: when generating SQL for each sub question, ALWAYS define it as a named Common Table Expression using `WITH AS ()`. + 2. ST_GeometryFromText() function: when constructing geometries from WKT, queries MUST use this function. Never use ST_GeomFromText() as it is unsupported. + 3. contains() Function: when checking whether a value exists within an array, queries MUST use this function. Never use array_contains() as it is unsupported. + 4. When using ORDER BY ASC|DESC, ALWAYS add GROUP BY before to select distinct values. + 5. When querying for top (max/min) values, you must ALWAYS return ALL records that share the top value rather than using LIMIT 1, unless a secondary sorting is specifically requested. + 6. ALWAYS wrap each column used in an aggregation with COALESCE(column,0) (or a suitable default value) to avoid NULLs. + 7. ALWAYS work with ISO 8601 format in TIME columns, if a TIME column is not in ISO 8601 format, you must explicitly convert it (e.g. with CAST). + 8. When counting rows/entities, always apply DISTINCT on the identifier column (e.g. id or the column that uniquely represents the entity requested by the user) to ensure each entity is counted only once. + 9. When calculating distances using geographic coordinates in degrees (WGS84), use Trino's spherical geography engine (toSphericalGeography()). + 10. NEVER give variables names in Hebrew. + Constraints: + + * Strictly adhere to the provided database schema. NEVER invent or assume table or columns names. + * The query should be for the database you will be given by the user. + * In "select ", select only the needed columns in the User Request without any unnecessary column or value. + * FROM or JOIN
, never include unnecessary tables. + * NEVER use Unicode characters, only utf-8. + * Never use ';' + * NEVER include ANY inline comments (starting with '--' or '/*') in the SQL statement. + * NEVER give variables names in Hebrew. + * Never wrap column identifiers in single or double quoted string literals. + * Never include the literal token 'kill' (in any case) anywhere in a generated SQL query whether as a column alias, name, identifier, function name, or comment. + """ + ) }, { "role": "user", - "content": "Plan: {{schema_plan}}\nQuery: {{user_query}}{{feedback_str}}" + "content": ( + """ + Database Schema: + {{schema_plan}} + User Request + {{user_query}} + Decompoese the request into sub questions, and generate the SQL after thinking step by step: + """ + ) } ], "type": "chat" @@ -108,11 +195,123 @@ def main(): "prompt": [ { "role": "system", - "content": "You are a Trino SQL expert. Fix the SQL query based on the database error. Output ONLY the fixed SQL query, nothing else (no backticks, no explanation)." + "content": ( + """ + reasoning: high + + # **Role** + You are an expert Trino SQL Refiner and Validator agent. Your purpose is to take a proposed Trino SQL query and ensure it is syntactically correct and logically aligned with the user's request. + + Your Goal: To iteratively debug and validate a given SQL query until it executes successfully and its logic and results precisely fulfill the user's request. + + # **Input** + 1. **SQL Query:** The current SQL query to validate or refine. + 2. **Error:** The error message from the last execution attempt, if any. + 3. **Schema Context:** A detailed description of the available tables, their columns, data types, and any relevant notes or constraints. + 4. **Error History:** A record of previous errors encountered in this session, used to avoid repeating the same mistake. + 5. **User Query:** The user's original natural language request (Hebrew), asking to retrieve data. + 6. **Tool: 'Trino':** A tool you MUST use to run the query. Using the tool is available by outputting "TRINO", followed by the Trino SQL query, in an SQL code block, in your final output. Example of using the tool: + + TRINO + ```sql + + ``` + + # **Instructions** + You MUST follow this exact step-by-step reasoning process for every request: + + ## **Step 1. Initial Reception and Execution** + * Receive the user query and the SQL query. + * Action: immediately use the Trino Tool to run the provided query, to establish a baseline. NEVER attempt to guess if a query works, execute it first to establish a baseline. + + ## **Step 2. Examine tool response** + * If the response contains an error, go to **Step 2-A**. + * If the response is successful, go to **Step 2-B**. + + ### **Step 2-A. Error Driven Refinement** + 1. Analyse the error message: identify the syntax issue, type mismatch, etc. + 2. Review the error history, identify any recurring error patterns, and ensure your new revised query does not reintroduce those mistakes. + 3. Plan a fix that only touches the parts of the query that caused the error (do **not** add or remove clauses that are unrelated). + 4. Produce a revised query and immediately re-run it with the Trino tool, by finally outputting the TRINO block. + + NEVER perform the "Plain-English translation" or any logical comparison while the query is still failing. The only output in this branch is the refined query wrapped in the TRINO block. + + ### **Step 2-B. Logical Translation & Validation** + Now that the query runs without error, perform the logical audit. + + 1. Deconstruct the SQL: Break the query down clause-by-clause. + 2. Plain English Translation: Write a step-by-step translation of what the SQL is actually doing (e.g. "This query joins the 'Users' and 'Orders' tables, filters for orders over 50 dollars, and counts them by region"). + 3. Comparison: Compare this translation directly to the User Query. + 4. Cross Reference: Compare the translation against the Schema Context, specifically checking any notes that outline user assumptions, default values, and other table-specific constraints. Ensure the query aligns with those details. + 5. Zero Result Confirmation: If the query is syntactically correct and logically aligned with the User Query, and multiple refined attempts continue to return 0 rows, conclude that the requested data does not exist or is unavailable, and go to **Step 3**. + 6. Verification: Ask yourself, "Does this translation satisfy every constraint and request mentioned by the User Query and Schema Context?" + * If yes, go to **Step 3**. + * If no, go to **Step 3-A** (refinement for logical mismatch). + + ## **Step 3-A. Logical Mismatch Refinement** + 1. Identify the missing or extra requirement (e.g., missing 'ORDER BY', wrong aggregation, wrong join, etc.). + 2. Modify the query accordingly, without breaking the syntax. + 3. Re-run the updated query by finally outputting the TRINO block. + + ## **Step 3. Satisfied Output** + When the query both executes successfully and matches the User Query: + answer with "QUERY_SATISFIED" followed by the validated SQL code block, and then provide the final translation. + The final translation must be a Hebrew, step-by-step explanation of exactly what the SQL query does, written in simple language as if explaining to a 5-year-old. + + # **Final Output Format** + Your output MUST follow the following formats for each case: + + **Refinement and Tool Usage:** + + TRINO + ```sql + + ``` + + **Satisfied and Final Output:** + + QUERY_SATISFIED + ```sql + + ``` + TRANSLATION + + + # **Constraints** + * You are ONLY allowed to return two types of assistant-side messages - reasoning & final. NEVER return any other assistant-side message type. + * NEVER output 'QUERY_SATISFIED' if the previous tool call failed or returned an error. + * You MUST use the 'Trino' tool in every refinement turn. + * Executing a query is available by ONLY outputting the "refinement and tool usage" format. + * While analyzing the query and the User Query you MUST ignore details/requests that don't have a corresponding column in the Schema Context. When doing so you MUST mention it in your reasoning. + * Strictly adhere to the provided Schema Context. NEVER invent or assume a table's or columns' names. + * You MUST provide a detailed explanation of the way you think and reflect your steps in your responses. + * Never include the literal token 'kill' (in any case) anywhere in a generated SQL query, whether as a column/alias name, identifier, function name or comment. + * Never wrap column identifiers in single or double quoted string literals ('' or ""). + """ + ) }, { "role": "user", - "content": "SQL: {{sql}}\nError: {{error}}" + "content": ( + """ + User's question: + {{user_query}} + + SQL query: + {{sql}} + + Error (if any): + {{error}} + + Schema context: + {{schema_context}} + + Error history: + {{error_history}} + + Execute and possibly refine the query above. + """ + ) } ], "type": "chat" @@ -123,15 +322,54 @@ def main(): { "role": "system", "content": ( - "You are a helpful data assistant. Summarize the findings for the user nicely. " - "You are given the SQL query that was executed and a preview of the queried data (columns and first few rows) to help you understand the context of the results. " - "Note: If the columns contain single items or aliases like `_col0` with a numeric value, this is the result of an aggregation query (such as `COUNT(*)`). Use this direct result to answer the user's question.\n\n" - "Data Preview:\n{{data_preview}}" + """ + You are a Response Finalizer agent of a Text-To-SQL tool. Your job is to produce a clear and concise, human-friendly, high-quality Markdown answer, in Hebrew. + + # Input + 1. The user's original question. + 2. The final SQL query and its Hebrew explanation. + 3. A preview of the query results (columns and first few rows). + + # Output + A Hebrew natural language answer that summarizes the data preview and answers the user's request. + + * NEVER generate the table results, ONLY summarize them, since the user receives the full table separately. + * NEVER rewrite the SQL query or its explanation - the user already received them. + * NEVER ask or suggest a follow-up question. + + # Guidelines + 1. **Tone & style** - friendly, approachable, and concise. + 2. **Content** - Summarize the final result clearly. Explain the key reasoning steps so the user sees how the answer was derived. Provide the context needed for the user to grasp the implications of the result. Note: if the data preview contains a single value or a generic alias such as `_col0` with a numeric value, this is the result of an aggregation query (e.g. COUNT(*)) - use this value directly to answer the user's question. + 3. **Structure** - Use Markdown headings, bullet points, or code fences as needed to make the response easy to read. + 4. **Focus** - Emphasize the final result; all other details should support that conclusion. + + # Constraints + * You operate inside the IDF CTS (cloud top secret) network, so you don't have access to the public internet. Don't try to access common url addresses, only urls provided in context, descriptions, tool definitions, or tool results. + * NEVER generate any SQL - you ONLY generate a Hebrew answer based on the SQL query and its results. + * NEVER ask a follow-up question, as the system does not support it. + * All content MUST be written in Hebrew, except for technical identifiers such as table or column names. + """ ) }, { "role": "user", - "content": "User asked: {{user_query}}\nSQL Query: {{sql_query}}\nData Ref: {{raw_data_ref}}" + "content": """ + Based on the following information, please generate a human-friendly response to my query: + + User's question: + {{user_query}} + + SQL query: + {{sql_query}} + + SQL explanation: + {{data_preview}} + + Data reference: + {{raw_data_ref}} + + Please provide a clear and concise response that explains the solution and provides context for my question. + """ } ], "type": "chat" @@ -142,14 +380,39 @@ def main(): { "role": "system", "content": ( - "You are a database analyst assistant. Explain the following SQL query in clear, natural language. " - "Describe what fields and tables are queried, any filters, joins, groupings, or aggregations, and what the query accomplishes. " - "Keep the explanation concise and professional." + """ + You are a SQL Explanation agent of a Text-To-SQL tool. Your job is to translate a SQL query into a clear, concise, human-friendly explanation in Hebrew. + + # Input + The final SQL query that will be executed. + + # Output + A short Hebrew explanation of what the query does: which tables and columns are used, what filters, joins, groupings, or aggregations are applied, and what result the query is designed to produce. + + # Guidelines + 1. **Tone & style** - friendly, approachable, and concise. + 2. **Content** - Describe the query's logic step by step so a non-technical user understands what data is being retrieved and how. + 3. **Structure** - A short paragraph or a few bullet points; no need for headings or code fences. + 4. **Focus** - Explain only the query's logic. Do not comment on or guess at the results, since results are not available at this stage. + + # Constraints + * You operate inside the IDF CTS (cloud top secret) network, so you don't have access to the public internet. Don't try to access common url addresses, only urls provided in context, descriptions, tool definitions, or tool results. + * NEVER generate, modify, or suggest changes to SQL - ONLY explain the query you are given. + * NEVER ask a follow-up question, as the system does not support it. + * All content MUST be written in Hebrew, except for technical identifiers such as table or column names. + """ ) }, { "role": "user", - "content": "SQL Query:\n{{sql_query}}" + "content": ( + """ + Please explain the following SQL query in Hebrew: + +SQL query: +{{sql_query}} + """ + ) } ], "type": "chat" From 8cfff999f64c701eb783b9d025ee4970c25f5a1a Mon Sep 17 00:00:00 2001 From: Yodan Bargida Date: Tue, 7 Jul 2026 16:24:09 +0300 Subject: [PATCH 06/11] feat(): add spider questions --- agent/src/agent/nodes/query_builder.py | 2 +- agent/src/agent/nodes/refiner.py | 19 +++ backend/app/config.py | 2 +- backend/app/routers/agent.py | 2 +- backend/app/routers/orchestration.py | 125 ++++++++++++++- core/src/core/models/models.py | 4 +- frontend/src/api/orchestration.ts | 5 + .../components/monitoring/RunHistoryTable.css | 18 +++ .../components/monitoring/RunHistoryTable.tsx | 20 ++- frontend/src/components/tables/TableList.css | 18 +++ frontend/src/components/tables/TableList.tsx | 22 ++- frontend/src/hooks/useEvaluations.ts | 11 ++ frontend/src/pages/EvaluationsPage.css | 40 +++++ frontend/src/pages/EvaluationsPage.tsx | 143 +++++++++++++++++- 14 files changed, 410 insertions(+), 21 deletions(-) diff --git a/agent/src/agent/nodes/query_builder.py b/agent/src/agent/nodes/query_builder.py index 1e6884b..a2dc747 100644 --- a/agent/src/agent/nodes/query_builder.py +++ b/agent/src/agent/nodes/query_builder.py @@ -31,7 +31,7 @@ async def query_builder_node(state: AgentState, config: RunnableConfig | None = { "schema_plan": state.get("schema_plan"), "user_query": state.get("user_query"), - "feedback_str": feedback_str, + # "feedback_str": feedback_str, } ) content = response.content diff --git a/agent/src/agent/nodes/refiner.py b/agent/src/agent/nodes/refiner.py index 86524a3..d376866 100644 --- a/agent/src/agent/nodes/refiner.py +++ b/agent/src/agent/nodes/refiner.py @@ -1,3 +1,4 @@ +from six.moves.urllib import response import json import asyncio @@ -101,12 +102,15 @@ async def refiner_node(state: AgentState, config: RunnableConfig | None = None): tags=["schema_context_injected=True"], ) + schema_context = build_refiner_schema_context(state) + response = await chain.ainvoke( { "sql": sql, "error": trino_error, "schema_context": schema_context, "error_history": json.dumps(error_history), + "user_query": state.get("user_query", ""), } ) new_sql = clean_sql(response.content) @@ -160,3 +164,18 @@ def json_serial(obj): "error_history": error_history, "execution_path": execution_path + ["refiner"], } + +def build_refiner_schema_context(state: AgentState) -> str: + """Build schema context for the refiner, preferring enriched table + profiles over the raw schema plan when available.""" + table_profiles = state.get("table_profiles") + if table_profiles: + # Rich, structured schema info (columns, notes, assumptions, constraints) + return json.dumps(table_profiles, ensure_ascii=False, indent=2) + + schema_plan = state.get("schema_plan") + if schema_plan: + # Fallback: flat schema/plan string used by the composer + return schema_plan + + return "No schema context available." \ No newline at end of file diff --git a/backend/app/config.py b/backend/app/config.py index 22c58f9..dc3b921 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -39,7 +39,7 @@ class Settings(BaseSettings): OPENMETADATA_VERIFY_SSL: bool = False OPENMETADATA_SERVICE_NAME: str = "local_trino" RUN_SEED: bool = False - RUN_INFRA_INIT: bool = False + RUN_INFRA_INIT: bool = True # Trino connection TRINO_HOST: str = "localhost" TRINO_PORT: int = 8080 diff --git a/backend/app/routers/agent.py b/backend/app/routers/agent.py index 94256a4..388a541 100644 --- a/backend/app/routers/agent.py +++ b/backend/app/routers/agent.py @@ -102,7 +102,7 @@ async def _call_agent_mcp(tool_arguments: dict) -> dict: result = await session.call_tool( "chat_with_agent", arguments=tool_arguments, - read_timeout_seconds=timedelta(seconds=300.0), + read_timeout_seconds=timedelta(seconds=900.0), ) if not result.content: diff --git a/backend/app/routers/orchestration.py b/backend/app/routers/orchestration.py index 60024fd..653fc3e 100644 --- a/backend/app/routers/orchestration.py +++ b/backend/app/routers/orchestration.py @@ -278,6 +278,109 @@ def trigger_evaluation_run( return read_runs +def _run_dataset_pipeline(dataset_name: str, run_id: str): + import requests + + from app.config import settings + from app.routers.evaluation import RunDatasetResponse, _map_and_save_run_metrics + + with Session(engine) as session: + run = session.get(EvalRun, run_id) + if not run: + return + + try: + # Resolve all production table names from the DB to pass as additional_tables + from core.models.models import Table, TableStatus + + prod_tables = session.exec( + select(Table).where(Table.status == TableStatus.production) + ).all() + table_names = [t.name for t in prod_tables] + + req = { + "dataset_name": dataset_name, + "additional_tables": table_names, + } + resp = requests.post( + f"{settings.EVALUATION_SERVICE_URL}/text-to-sql/evaluation/run-single-dataset", + json=req, + timeout=600, + ) + resp.raise_for_status() + eval_resp = RunDatasetResponse(**resp.json()) + + _map_and_save_run_metrics(run, eval_resp, session, run_id) + session.commit() + except Exception as e: + logger.error( + f"[Eval] Dataset {dataset_name} evaluation failed: {e}", + exc_info=True, + ) + try: + session.refresh(run) + run.status = EvalStatus.failed + run.score = 0.0 + session.add(run) + session.commit() + except Exception as db_err: + logger.error(f"[Eval] Failed to mark run {run_id} as failed: {db_err}") + + +@router.post("/run-dataset", response_model=EvalRunRead, status_code=202) +def trigger_dataset_run( + dataset_name: str, + background_tasks: BackgroundTasks, + session: Session = Depends(get_session), +): + """Trigger evaluation for a specific dataset (e.g. 'spider2' or 'text2sql_production').""" + from core.models.models import Table, TableStatus + + # 1. Sync production dataset if requested + if dataset_name == "text2sql_production": + prod_tables = session.exec( + select(Table).where(Table.status == TableStatus.production) + ).all() + all_production_questions: list[GoldenQuestion] = [] + for table in prod_tables: + qs = session.exec( + select(GoldenQuestion).where(GoldenQuestion.table_id == table.id) + ).all() + all_production_questions.extend(qs) + + all_questions_payload = [] + from app.routers.evaluation import _build_questions_payload + + for table in prod_tables: + qs_for_table = [ + q for q in all_production_questions if q.table_id == table.id + ] + all_questions_payload.extend(_build_questions_payload(qs_for_table, table)) + + if all_questions_payload: + from app.services.langfuse_client import langfuse_client + + if langfuse_client.enabled: + try: + langfuse_client.sync_dataset( + "text2sql_production", all_questions_payload + ) + except Exception as e: + logger.warning(f"[Eval] Production dataset sync failed: {e}") + + # 2. Create the run record + run = EvalRun(table_id=None, triggered_by=dataset_name) + session.add(run) + session.commit() + session.refresh(run) + + # 3. Queue the task + background_tasks.add_task(_run_dataset_pipeline, dataset_name, run.id) + return EvalRunRead.model_validate( + run, update={"table_name": f"Dataset: {dataset_name}"} + ) + + @router.get("/runs", response_model=list[EvalRunRead]) def list_runs( limit: int = Query(default=50, le=200), @@ -301,7 +404,15 @@ def list_runs( results = session.exec(query).all() runs = [] for run, table_name in results: - t_name = table_name if table_name else "All prod tables" + t_name = ( + table_name + if table_name + else ( + f"Dataset: {run.triggered_by}" + if run.table_id is None + else "All prod tables" + ) + ) read = EvalRunRead.model_validate(run, update={"table_name": t_name}) runs.append(read) return runs @@ -311,7 +422,7 @@ def list_runs( def get_run(run_id: str, session: Session = Depends(get_session)): result = session.exec( select(EvalRun, Table.name) - .join(Table, EvalRun.table_id == Table.id) + .join(Table, EvalRun.table_id == Table.id, isouter=True) .where(EvalRun.id == run_id) ).first() @@ -319,7 +430,15 @@ def get_run(run_id: str, session: Session = Depends(get_session)): raise HTTPException(status_code=404, detail="Eval run not found") run, table_name = result - t_name = table_name if table_name else "All prod tables" + t_name = ( + table_name + if table_name + else ( + f"Dataset: {run.triggered_by}" + if run.table_id is None + else "All prod tables" + ) + ) return EvalRunRead.model_validate(run, update={"table_name": t_name}) diff --git a/core/src/core/models/models.py b/core/src/core/models/models.py index f82ccae..85a8103 100644 --- a/core/src/core/models/models.py +++ b/core/src/core/models/models.py @@ -149,7 +149,9 @@ class EvalRun(SQLModel, table=True): __tablename__ = "eval_runs" id: str = Field(default_factory=lambda: str(uuid.uuid4()), primary_key=True) - table_id: str = Field( + table_id: str | None = Field( + default=None, + nullable=True, sa_column_args=[ ForeignKey("tables.id", ondelete="CASCADE", onupdate="CASCADE") ], diff --git a/frontend/src/api/orchestration.ts b/frontend/src/api/orchestration.ts index ce768e2..b5baa5a 100644 --- a/frontend/src/api/orchestration.ts +++ b/frontend/src/api/orchestration.ts @@ -158,6 +158,11 @@ export const orchestrationApi = { .post('/evaluations/run', table_ids, { params: { triggered_by } }) .then((r) => r.data), + triggerDatasetRun: (dataset_name: string) => + api + .post('/evaluations/run-dataset', null, { params: { dataset_name } }) + .then((r) => r.data), + listRuns: (params?: { limit?: number; offset?: number; status?: string; table_id?: string }) => api.get('/evaluations/runs', { params }).then((r) => r.data), diff --git a/frontend/src/components/monitoring/RunHistoryTable.css b/frontend/src/components/monitoring/RunHistoryTable.css index fa84e35..a594645 100644 --- a/frontend/src/components/monitoring/RunHistoryTable.css +++ b/frontend/src/components/monitoring/RunHistoryTable.css @@ -305,3 +305,21 @@ display: flex; gap: 6px; } + +.run-table-name { + max-width: 180px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + display: inline-block; + vertical-align: middle; +} + +.run-triggered-by { + max-width: 140px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + display: inline-block; + vertical-align: middle; +} diff --git a/frontend/src/components/monitoring/RunHistoryTable.tsx b/frontend/src/components/monitoring/RunHistoryTable.tsx index 50000cd..5930420 100644 --- a/frontend/src/components/monitoring/RunHistoryTable.tsx +++ b/frontend/src/components/monitoring/RunHistoryTable.tsx @@ -327,7 +327,11 @@ export function RunHistoryTable({ tableId, limit = 50, compact = false }: RunHis } = useQuery({ queryKey: ['eval-runs', tableId, limit], queryFn: () => orchestrationApi.listRuns({ limit, table_id: tableId }), - refetchInterval: 15_000, + refetchInterval: (query) => { + const data = query.state.data as typeof runs; + const hasRunning = data?.some((r) => r.status === 'running'); + return hasRunning ? 5_000 : 15_000; + }, }); const paged = runs.slice(page * pageSize, (page + 1) * pageSize); @@ -381,7 +385,9 @@ export function RunHistoryTable({ tableId, limit = 50, compact = false }: RunHis setSelectedRunId(run.id)} className="run-history-row"> {!compact && ( )} {!tableId && ( @@ -390,12 +396,15 @@ export function RunHistoryTable({ tableId, limit = 50, compact = false }: RunHis e.stopPropagation()} - className="table-link hover:underline" + className="table-link hover:underline run-table-name" + title={run.table_name || run.table_id} > {run.table_name || run.table_id.slice(0, 8)} ) : ( - {run.table_name} + + {run.table_name} + )} )} @@ -423,7 +432,8 @@ export function RunHistoryTable({ tableId, limit = 50, compact = false }: RunHis - + diff --git a/frontend/src/hooks/useEvaluations.ts b/frontend/src/hooks/useEvaluations.ts index 2dac8e6..1d46cf9 100644 --- a/frontend/src/hooks/useEvaluations.ts +++ b/frontend/src/hooks/useEvaluations.ts @@ -29,6 +29,17 @@ export function useTriggerOrchestrationRun() { }); } +// ── Trigger dataset evaluation run ─────────────────────────────────────────── +export function useTriggerDatasetRun() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (datasetName: string) => orchestrationApi.triggerDatasetRun(datasetName), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: [QUERY_KEYS.EVAL_RUNS] }); + }, + }); +} + // ── List all evaluation runs ─────────────────────────────────────────────────── export function useAllEvalRuns() { return useQuery({ diff --git a/frontend/src/pages/EvaluationsPage.css b/frontend/src/pages/EvaluationsPage.css index eddbbf4..8bfc789 100644 --- a/frontend/src/pages/EvaluationsPage.css +++ b/frontend/src/pages/EvaluationsPage.css @@ -123,3 +123,43 @@ color: #ef4444; opacity: 0.9; } + +/* ── Dataset running notice ── */ + +.dataset-running-notice { + display: flex; + align-items: flex-start; + gap: 10px; + margin-top: 16px; + padding: 12px 16px; + border-radius: 10px; + background: rgba(99, 102, 241, 0.08); + border: 1px solid rgba(99, 102, 241, 0.3); + color: var(--text-primary); + font-size: 13px; + line-height: 1.5; + animation: pulse-border 2s ease-in-out infinite; +} + +.dataset-running-notice strong { + color: #818cf8; +} + +@keyframes pulse-border { + 0%, + 100% { + border-color: rgba(99, 102, 241, 0.3); + } + 50% { + border-color: rgba(99, 102, 241, 0.7); + } +} + +@keyframes spin { + from { + transform: rotate(0deg); + } + to { + transform: rotate(360deg); + } +} diff --git a/frontend/src/pages/EvaluationsPage.tsx b/frontend/src/pages/EvaluationsPage.tsx index bf42b88..c9c936e 100644 --- a/frontend/src/pages/EvaluationsPage.tsx +++ b/frontend/src/pages/EvaluationsPage.tsx @@ -1,11 +1,17 @@ import { useState } from 'react'; +import { useQuery } from '@tanstack/react-query'; import { App } from 'antd'; -import { CalendarClock, Check, Database, History, PlayCircle } from 'lucide-react'; +import { CalendarClock, Check, Database, History, Loader2, PlayCircle } from 'lucide-react'; +import { orchestrationApi } from '../api/orchestration'; import { EmptySlate, SectionHeader, Spinner } from '../components/common/EvalUI'; import { RunHistoryTable } from '../components/monitoring/RunHistoryTable'; import { ScheduleManager } from '../components/monitoring/ScheduleManager'; -import { useEvalReadiness, useTriggerOrchestrationRun } from '../hooks/useEvaluations'; +import { + useEvalReadiness, + useTriggerDatasetRun, + useTriggerOrchestrationRun, +} from '../hooks/useEvaluations'; import { useTables } from '../hooks/useTables'; import type { Table } from '../types'; @@ -15,7 +21,7 @@ import './EvaluationsPage.css'; type Tab = 'history' | 'schedules' | 'run'; // ── Run trigger panel ────────────────────────────────────────────────────────── -function RunTriggerPanel() { +function RunTriggerPanel({ onLaunch }: { onLaunch?: () => void }) { const [selectedTableIds, setSelectedTableIds] = useState([]); const [triggeredBy] = useState('user'); const [launched, setLaunched] = useState(false); @@ -34,6 +40,7 @@ function RunTriggerPanel() { onSuccess: () => { setLaunched(true); setTimeout(() => setLaunched(false), 4000); + onLaunch?.(); }, onError: (err: any) => { const detail = err?.response?.data?.detail; @@ -166,9 +173,110 @@ function RunTriggerPanel() { ); } +// ── DatasetRunPanel ───────────────────────────────────────────────────────────── +interface DatasetRunPanelProps { + runningDataset: string | null; + setRunningDataset: (val: string | null) => void; + setRunningRunId: (val: string | null) => void; + onLaunch?: () => void; +} + +function DatasetRunPanel({ + runningDataset, + setRunningDataset, + setRunningRunId, + onLaunch, +}: DatasetRunPanelProps) { + const triggerDatasetMut = useTriggerDatasetRun(); + + const handleLaunch = (datasetName: string) => { + setRunningDataset(datasetName); + triggerDatasetMut.mutate(datasetName, { + onSuccess: (run) => { + setRunningRunId(run.id); + // Switch to history tab so the user can watch progress + onLaunch?.(); + }, + onError: () => { + setRunningDataset(null); + setRunningRunId(null); + }, + }); + }; + + return ( +
+ + +
+ + + +
+
+ ); +} + // ── EvaluationsPage ──────────────────────────────────────────────────────────── export function EvaluationsPage() { const [activeTab, setActiveTab] = useState('history'); + const [runningDataset, setRunningDataset] = useState(null); + const [runningRunId, setRunningRunId] = useState(null); + + // Poll the specific run's details if we have a running run + useQuery({ + queryKey: ['running-run-status', runningRunId], + queryFn: () => { + if (!runningRunId) return null; + return orchestrationApi.getRun(runningRunId); + }, + enabled: !!runningRunId, + refetchInterval: (query) => { + const data = query.state.data as any; + if (data && (data.status === 'completed' || data.status === 'failed')) { + // Run has finished, clear states + setRunningDataset(null); + setRunningRunId(null); + return false; + } + return 3000; // poll every 3 seconds + }, + }); + + const switchToHistory = () => setActiveTab('history'); const TABS: { key: Tab; label: string; icon: React.ReactNode }[] = [ { key: 'history', label: 'Execution History', icon: }, @@ -176,6 +284,8 @@ export function EvaluationsPage() { { key: 'run', label: 'Run Controls', icon: }, ]; + const datasetLabel = runningDataset === 'text2sql_production' ? 'Production Dataset' : 'Spider2'; + return (
@@ -199,6 +309,21 @@ export function EvaluationsPage() { ))}
+ {/* Persistent running notice — stays visible at the top of the content across all tabs while the background job executes */} + {runningDataset && ( +
+ +
+ {datasetLabel} evaluation is currently running + + {' '} + — Results will appear in the History table below as the run completes (this may take + several minutes). + +
+
+ )} + {/* Tab content */} {activeTab === 'history' && (
@@ -212,7 +337,17 @@ export function EvaluationsPage() { {activeTab === 'schedules' && } - {activeTab === 'run' && } + {activeTab === 'run' && ( +
+ + +
+ )}
); } From 641a5cac04ae2e60bd4934c7183e1c2933a1e7b2 Mon Sep 17 00:00:00 2001 From: Yodan Bargida Date: Mon, 13 Jul 2026 09:33:32 +0300 Subject: [PATCH 07/11] fix(): change refiner prompt --- agent/scripts/upload_all_prompts.py | 133 ++++++++-------------------- agent/src/agent/nodes/refiner.py | 29 ++---- 2 files changed, 45 insertions(+), 117 deletions(-) diff --git a/agent/scripts/upload_all_prompts.py b/agent/scripts/upload_all_prompts.py index 6fea9cc..2220f2d 100644 --- a/agent/scripts/upload_all_prompts.py +++ b/agent/scripts/upload_all_prompts.py @@ -197,96 +197,43 @@ def main(): "role": "system", "content": ( """ - reasoning: high - - # **Role** - You are an expert Trino SQL Refiner and Validator agent. Your purpose is to take a proposed Trino SQL query and ensure it is syntactically correct and logically aligned with the user's request. - - Your Goal: To iteratively debug and validate a given SQL query until it executes successfully and its logic and results precisely fulfill the user's request. - - # **Input** - 1. **SQL Query:** The current SQL query to validate or refine. - 2. **Error:** The error message from the last execution attempt, if any. - 3. **Schema Context:** A detailed description of the available tables, their columns, data types, and any relevant notes or constraints. - 4. **Error History:** A record of previous errors encountered in this session, used to avoid repeating the same mistake. - 5. **User Query:** The user's original natural language request (Hebrew), asking to retrieve data. - 6. **Tool: 'Trino':** A tool you MUST use to run the query. Using the tool is available by outputting "TRINO", followed by the Trino SQL query, in an SQL code block, in your final output. Example of using the tool: - - TRINO - ```sql - - ``` - - # **Instructions** - You MUST follow this exact step-by-step reasoning process for every request: - - ## **Step 1. Initial Reception and Execution** - * Receive the user query and the SQL query. - * Action: immediately use the Trino Tool to run the provided query, to establish a baseline. NEVER attempt to guess if a query works, execute it first to establish a baseline. - - ## **Step 2. Examine tool response** - * If the response contains an error, go to **Step 2-A**. - * If the response is successful, go to **Step 2-B**. - - ### **Step 2-A. Error Driven Refinement** - 1. Analyse the error message: identify the syntax issue, type mismatch, etc. - 2. Review the error history, identify any recurring error patterns, and ensure your new revised query does not reintroduce those mistakes. - 3. Plan a fix that only touches the parts of the query that caused the error (do **not** add or remove clauses that are unrelated). - 4. Produce a revised query and immediately re-run it with the Trino tool, by finally outputting the TRINO block. - - NEVER perform the "Plain-English translation" or any logical comparison while the query is still failing. The only output in this branch is the refined query wrapped in the TRINO block. - - ### **Step 2-B. Logical Translation & Validation** - Now that the query runs without error, perform the logical audit. - - 1. Deconstruct the SQL: Break the query down clause-by-clause. - 2. Plain English Translation: Write a step-by-step translation of what the SQL is actually doing (e.g. "This query joins the 'Users' and 'Orders' tables, filters for orders over 50 dollars, and counts them by region"). - 3. Comparison: Compare this translation directly to the User Query. - 4. Cross Reference: Compare the translation against the Schema Context, specifically checking any notes that outline user assumptions, default values, and other table-specific constraints. Ensure the query aligns with those details. - 5. Zero Result Confirmation: If the query is syntactically correct and logically aligned with the User Query, and multiple refined attempts continue to return 0 rows, conclude that the requested data does not exist or is unavailable, and go to **Step 3**. - 6. Verification: Ask yourself, "Does this translation satisfy every constraint and request mentioned by the User Query and Schema Context?" - * If yes, go to **Step 3**. - * If no, go to **Step 3-A** (refinement for logical mismatch). - - ## **Step 3-A. Logical Mismatch Refinement** - 1. Identify the missing or extra requirement (e.g., missing 'ORDER BY', wrong aggregation, wrong join, etc.). - 2. Modify the query accordingly, without breaking the syntax. - 3. Re-run the updated query by finally outputting the TRINO block. - - ## **Step 3. Satisfied Output** - When the query both executes successfully and matches the User Query: - answer with "QUERY_SATISFIED" followed by the validated SQL code block, and then provide the final translation. - The final translation must be a Hebrew, step-by-step explanation of exactly what the SQL query does, written in simple language as if explaining to a 5-year-old. - - # **Final Output Format** - Your output MUST follow the following formats for each case: - - **Refinement and Tool Usage:** - - TRINO - ```sql - - ``` - - **Satisfied and Final Output:** - - QUERY_SATISFIED - ```sql - - ``` - TRANSLATION - - - # **Constraints** - * You are ONLY allowed to return two types of assistant-side messages - reasoning & final. NEVER return any other assistant-side message type. - * NEVER output 'QUERY_SATISFIED' if the previous tool call failed or returned an error. - * You MUST use the 'Trino' tool in every refinement turn. - * Executing a query is available by ONLY outputting the "refinement and tool usage" format. - * While analyzing the query and the User Query you MUST ignore details/requests that don't have a corresponding column in the Schema Context. When doing so you MUST mention it in your reasoning. - * Strictly adhere to the provided Schema Context. NEVER invent or assume a table's or columns' names. - * You MUST provide a detailed explanation of the way you think and reflect your steps in your responses. - * Never include the literal token 'kill' (in any case) anywhere in a generated SQL query, whether as a column/alias name, identifier, function name or comment. - * Never wrap column identifiers in single or double quoted string literals ('' or ""). + You are a Trino SQL correction assistant. + + You do NOT execute queries yourself — a separate system has already run the SQL query + against Trino and captured the resulting error. Your only job is to analyze that error + and the schema context, then output ONE corrected SQL query. + + # Inputs you will receive + - User's question (in Hebrew): what the user actually wants to know. + - SQL query: the query that was executed. + - Error: the exact error message Trino returned for that query. + - Schema context: descriptions of the available tables/columns, including notes on + assumptions, default values, and constraints. + - Error history: errors encountered on previous refinement attempts in this session. + + # Instructions + 1. Read the error message carefully and identify the precise cause (syntax error, + unknown column/table, type mismatch, aggregation issue, etc.). + 2. Check the error history — if a similar error occurred before, do not reintroduce + whatever caused it. + 3. Fix ONLY what is necessary to resolve the error. Do not add, remove, or rewrite + unrelated clauses. + 4. Strictly adhere to the schema context. NEVER invent or assume a table or column name + that isn't explicitly listed there. + 5. While fixing the query, keep it aligned with the user's original Hebrew question — + don't "fix" the error in a way that changes what the query is answering. + 6. Ignore any part of the user's request that has no corresponding column in the schema + context; do not attempt to hallucinate a column to satisfy it. + + # Output rules + - Output ONLY the corrected SQL query — nothing else. No reasoning, no explanation, + no labels like "TRINO" or "SQL:". + - You may optionally wrap the query in a ```sql ... ``` code block or output it raw; + either is fine, but include nothing besides the query itself either way. + - Do not include a trailing semicolon. + - Never wrap column identifiers in single or double quotes. + - Never include the literal token "kill" (any case) anywhere in the query — as an + identifier, alias, function name, or comment. """ ) }, @@ -296,20 +243,16 @@ def main(): """ User's question: {{user_query}} - SQL query: {{sql}} - Error (if any): {{error}} - Schema context: {{schema_context}} - Error history: {{error_history}} - Execute and possibly refine the query above. + Correct the query above. """ ) } diff --git a/agent/src/agent/nodes/refiner.py b/agent/src/agent/nodes/refiner.py index d376866..9b037a3 100644 --- a/agent/src/agent/nodes/refiner.py +++ b/agent/src/agent/nodes/refiner.py @@ -1,6 +1,4 @@ -from six.moves.urllib import response import json - import asyncio import logging from langchain_core.runnables.config import RunnableConfig @@ -17,18 +15,6 @@ llm = get_llm("refiner") -def build_refiner_schema_context(state: AgentState) -> str: - profiles = state.get("table_profiles") - if not profiles: - return "No schema context available." - - runtime_flags = state.get("runtime_flags") or {} - limit = int(runtime_flags.get("REFINER_SCHEMA_CONTEXT_TABLES", settings.REFINER_SCHEMA_CONTEXT_TABLES)) - - # Cap the context to REFINER_SCHEMA_CONTEXT_TABLES - capped_profiles = profiles[:limit] - return json.dumps(capped_profiles, indent=2) - async def refiner_node(state: AgentState, config: RunnableConfig | None = None): """Refine SQL against Trino.""" @@ -102,9 +88,7 @@ async def refiner_node(state: AgentState, config: RunnableConfig | None = None): tags=["schema_context_injected=True"], ) - schema_context = build_refiner_schema_context(state) - - response = await chain.ainvoke( + llm_response = await chain.ainvoke( { "sql": sql, "error": trino_error, @@ -113,7 +97,7 @@ async def refiner_node(state: AgentState, config: RunnableConfig | None = None): "user_query": state.get("user_query", ""), } ) - new_sql = clean_sql(response.content) + new_sql = clean_sql(llm_response.content) return { "sql_query": new_sql, "trino_error": trino_error, @@ -146,13 +130,14 @@ def json_serial(obj): raw_ref = res.get("esca_id") except Exception as e: esca_write_failed = True + error_msg = f"ESCA write failed: {e}" if langfuse_client and langfuse_client.get_current_trace_id(): langfuse_client.update_current_span( - level="ERROR", status_message=f"ESCA write failed: {e}" + level="WARNING", status_message=error_msg ) - else: - logging.error(f"ESCA write failed: {e}") - raise RuntimeError(f"Failed to write query result to ESCA: {e}") + logging.warning(error_msg) + # ESCA is an optional output store — do not crash the agent. + # The query result is still available as inline_result_rows/columns. return { "trino_error": None, From 4dacaaa47961316a13d23ee48507383f4807b597 Mon Sep 17 00:00:00 2001 From: Yodan Bargida Date: Sun, 19 Jul 2026 11:14:27 +0300 Subject: [PATCH 08/11] add infinate scroll and not show spider defaultly --- agent/src/agent/nodes/refiner.py | 21 +- backend/app/config.py | 2 +- backend/app/infra_init.py | 118 +- backend/app/routers/evaluation.py | 19 +- backend/app/routers/orchestration.py | 17 +- backend/app/spider2_questions.json | 1702 +++++++++++++++++ backend/app/sync_om_metadata.py | 1021 ++++++++++ backend/pyproject.toml | 1 + backend/uv.lock | 11 + docker-compose.yml | 9 +- .../components/monitoring/RunHistoryTable.tsx | 20 +- frontend/src/components/tables/TableList.css | 32 + frontend/src/components/tables/TableList.tsx | 76 +- frontend/src/pages/EvaluationsPage.css | 266 ++- frontend/src/pages/EvaluationsPage.tsx | 349 +++- frontend/tests/agent-testing.spec.ts | 2 +- frontend/tests/real-agent.spec.ts | 2 +- scripts/generate_trino_catalogs.py | 85 +- 18 files changed, 3623 insertions(+), 130 deletions(-) create mode 100644 backend/app/spider2_questions.json create mode 100644 backend/app/sync_om_metadata.py diff --git a/agent/src/agent/nodes/refiner.py b/agent/src/agent/nodes/refiner.py index 9b037a3..1c81aad 100644 --- a/agent/src/agent/nodes/refiner.py +++ b/agent/src/agent/nodes/refiner.py @@ -69,14 +69,21 @@ async def refiner_node(state: AgentState, config: RunnableConfig | None = None): "execution_path": execution_path + ["refiner"], } - langfuse_prompt = langfuse_client.get_prompt(settings.LANGFUSE_PROMPT_REFINER) - if langfuse_prompt is None: - raise RuntimeError( - f"Langfuse prompt '{settings.LANGFUSE_PROMPT_REFINER}' could not be retrieved." + try: + langfuse_prompt = langfuse_client.get_prompt(settings.LANGFUSE_PROMPT_REFINER) + except Exception as prompt_err: + logging.getLogger(__name__).warning(f"Failed to get refiner prompt from Langfuse: {prompt_err}. Using fallback.") + langfuse_prompt = None + + if langfuse_prompt is not None: + prompt = ChatPromptTemplate.from_messages( + langfuse_prompt.get_langchain_prompt() ) - prompt = ChatPromptTemplate.from_messages( - langfuse_prompt.get_langchain_prompt() - ) + else: + prompt = ChatPromptTemplate.from_messages([ + ("system", "You are a Trino SQL expert database assistant. Your task is to fix a Trino SQL query that failed with a syntax or schema error."), + ("user", "Original User Query: {user_query}\n\nFailed SQL Query: {sql}\n\nTrino Error: {error}\n\nError History: {error_history}\n\nDatabase Schema Context:\n{schema_context}\n\nPlease rewrite the SQL query to fix the error. Return ONLY the valid SQL query inside a ```sql ``` block.") + ]) _llm = get_llm("refiner", runtime_flags=runtime_flags) chain = prompt | _llm diff --git a/backend/app/config.py b/backend/app/config.py index dc3b921..6d803c6 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -38,7 +38,7 @@ class Settings(BaseSettings): OPENMETADATA_ADMIN_PASSWORD: str = "admin" OPENMETADATA_VERIFY_SSL: bool = False OPENMETADATA_SERVICE_NAME: str = "local_trino" - RUN_SEED: bool = False + RUN_SEED: bool = True RUN_INFRA_INIT: bool = True # Trino connection TRINO_HOST: str = "localhost" diff --git a/backend/app/infra_init.py b/backend/app/infra_init.py index 74538c8..11024f0 100644 --- a/backend/app/infra_init.py +++ b/backend/app/infra_init.py @@ -12,6 +12,12 @@ trigger a profiling job for each (idempotent). All steps are fully idempotent. Running this multiple times is safe. + +NOTE on Trino catalogs: this module does NOT create Snowflake catalog +.properties files -- that's scripts/generate_trino_catalogs.py. Trino only +loads file-based catalogs at container startup, so if new catalog files +were added after Trino was already running, restart Trino before this +module's OpenMetadata/Snowflake-related steps will see them. """ import base64 @@ -50,6 +56,14 @@ _TRINO_READY_RETRIES = 20 _TRINO_READY_INTERVAL = 5 # seconds between retries +# How long to wait after creating a brand-new Airflow ingestion pipeline +# before attempting to trigger it, and how many times to retry. Airflow +# needs a few seconds to pick up and register a newly-deployed DAG before +# it will accept a trigger call -- without this, a freshly created pipeline +# just sits untouched until its next scheduled run (which may be a day away). +_OM_PIPELINE_TRIGGER_DELAY = 20 # seconds before first retry attempt +_OM_PIPELINE_TRIGGER_RETRIES = 6 + # ── Airlines Snowflake catalog ───────────────────────────────────────────────── # System owner used for infrastructure-seeded tables _SYSTEM_OWNER_ID = "system" @@ -890,6 +904,53 @@ def _ensure_om_service(token: str) -> str | None: return None +def _trigger_pipeline_with_retries(pid: str, token: str, pipeline_name: str) -> None: + """ + Background worker: repeatedly attempt to trigger a just-created ingestion + pipeline. Airflow needs a short window after DAG deployment before it + will accept a trigger call for a brand-new DAG -- without this retry + loop, a freshly created pipeline just sits idle until its next scheduled + run (which may be a day away), leaving OpenMetadata (and everything + downstream of it, e.g. sync_om_metadata.py) looking stale/empty for + catalogs that were only just added. + """ + + def _worker() -> None: + for attempt in range(1, _OM_PIPELINE_TRIGGER_RETRIES + 1): + time.sleep(_OM_PIPELINE_TRIGGER_DELAY) + status, data = _om_post( + f"services/ingestionPipelines/trigger/{pid}", {}, token + ) + if status in ("200", "201"): + logger.info( + "[InfraInit] Triggered ingestion pipeline '%s' (attempt %d/%d) ✓", + pipeline_name, + attempt, + _OM_PIPELINE_TRIGGER_RETRIES, + ) + return + logger.debug( + "[InfraInit] Trigger attempt %d/%d for pipeline '%s' failed (HTTP %s): %s", + attempt, + _OM_PIPELINE_TRIGGER_RETRIES, + pipeline_name, + status, + data, + ) + logger.warning( + "[InfraInit] Could not trigger ingestion pipeline '%s' after %d attempts " + "over ~%ds; it will run on its next Airflow schedule instead. You can also " + "trigger it manually from the OpenMetadata UI.", + pipeline_name, + _OM_PIPELINE_TRIGGER_RETRIES, + _OM_PIPELINE_TRIGGER_DELAY * _OM_PIPELINE_TRIGGER_RETRIES, + ) + + threading.Thread( + target=_worker, daemon=True, name=f"om-pipeline-trigger-{pipeline_name}" + ).start() + + def _ensure_om_ingestion_pipeline(token: str, svc_id: str) -> None: pipeline_name = "local_trino_metadata" pipeline_fqn = f"{_OM_SERVICE_NAME}.{pipeline_name}" @@ -900,7 +961,8 @@ def _ensure_om_ingestion_pipeline(token: str, svc_id: str) -> None: "[InfraInit] OM ingestion pipeline '%s' already exists — OK", pipeline_name ) pid = data["id"] - # Trigger it on startup to ensure latest data + # Trigger it on startup to ensure latest data (this DAG already exists + # in Airflow, so no delay/retry needed here). _om_post(f"services/ingestionPipelines/trigger/{pid}", {}, token) return @@ -924,15 +986,15 @@ def _ensure_om_ingestion_pipeline(token: str, svc_id: str) -> None: pid = data["id"] # Deploy it to Airflow - status_deploy = _om_post(f"services/ingestionPipelines/deploy/{pid}", {}, token) status_deploy, _ = _om_post( f"services/ingestionPipelines/deploy/{pid}", {}, token ) logger.info("[InfraInit] Deployed pipeline: %s", status_deploy) - # We can't trigger it immediately because Airflow takes a few seconds to load the new DAG. - # But Airflow will pick it up and run it on schedule. - # Alternatively, the user can manually trigger it from the UI. + # Airflow needs a few seconds to register the newly-deployed DAG + # before it will accept a trigger call. Retry in the background + # instead of leaving it to wait for the next schedule. + _trigger_pipeline_with_retries(pid, token, pipeline_name) return logger.error( @@ -1034,6 +1096,12 @@ def _verify_custom_catalogs() -> None: """ Detect all non-default catalogs loaded in Trino (excluding system, minio, tpch) and verify their connectivity by running SHOW SCHEMAS in parallel. + + This is also the earliest place a "generated catalog files but forgot to + restart Trino" gap would show up as a LOW count -- if you just ran + scripts/generate_trino_catalogs.py and expected e.g. ~129 Snowflake + catalogs but this logs far fewer, Trino is very likely still running + with its pre-restart catalog set. Restart Trino and re-run. """ logger.info("[InfraInit] Scanning Trino for custom Snowflake/external catalogs...") try: @@ -1049,23 +1117,47 @@ def _verify_custom_catalogs() -> None: return logger.info( - "[InfraInit] Found %d custom catalog(s). Verifying connections in parallel...", + "[InfraInit] Found %d custom catalog(s) loaded in Trino. Verifying connections in parallel...", len(custom_catalogs), ) + failed_catalogs: list[str] = [] + lock = threading.Lock() + def verify_one(catalog: str) -> None: logger.info("[InfraInit] Verifying connection to catalog '%s'...", catalog) - schemas = _trino_exec(f"SHOW SCHEMAS FROM {catalog}") - logger.info( - "[InfraInit] Catalog '%s' connection verified successfully ✓ (%d schema(s) found: %s)", - catalog, - len(schemas), - [s[0] for s in schemas], - ) + try: + schemas = _trino_exec(f"SHOW SCHEMAS FROM {catalog}") + logger.info( + "[InfraInit] Catalog '%s' connection verified successfully ✓ (%d schema(s) found: %s)", + catalog, + len(schemas), + [s[0] for s in schemas], + ) + except Exception as exc: + with lock: + failed_catalogs.append(catalog) + logger.warning( + "[InfraInit] Catalog '%s' failed connectivity check: %s", + catalog, + exc, + ) with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor: list(executor.map(verify_one, custom_catalogs)) + logger.info( + "[InfraInit] Custom catalog scan complete: %d/%d catalog(s) verified OK.", + len(custom_catalogs) - len(failed_catalogs), + len(custom_catalogs), + ) + if failed_catalogs: + logger.warning( + "[InfraInit] %d catalog(s) failed connectivity: %s", + len(failed_catalogs), + sorted(failed_catalogs), + ) + except Exception as exc: logger.error("[InfraInit] Verification of custom catalogs failed: %s", exc) raise diff --git a/backend/app/routers/evaluation.py b/backend/app/routers/evaluation.py index 73c3802..748779b 100644 --- a/backend/app/routers/evaluation.py +++ b/backend/app/routers/evaluation.py @@ -318,7 +318,9 @@ def _run_production_dataset_eval( session: Session, run_name_prefix: str, promotion_run_id: str ) -> float: prod_tables = session.exec( - select(Table).where(Table.status == TableStatus.production) + select(Table) + .where(Table.status == TableStatus.production) + .where(Table.owner_id != "spider2") ).all() if not prod_tables: @@ -459,7 +461,9 @@ def _run_regression_eval( run_name_prefix: str, session: Session, promotion_run_id: str ) -> float: prod_tables = session.exec( - select(Table).where(Table.status == TableStatus.production) + select(Table) + .where(Table.status == TableStatus.production) + .where(Table.owner_id != "spider2") ).all() all_production_questions: list[GoldenQuestion] = [] table_names = [] @@ -913,12 +917,21 @@ def get_batch_runs(promotion_run_id: str, session: Session = Depends(get_session def get_run(run_id: str, session: Session = Depends(get_session)): result = session.exec( select(EvalRun, Table.name) - .join(Table, EvalRun.table_id == Table.id) + .join(Table, EvalRun.table_id == Table.id, isouter=True) .where(EvalRun.id == run_id) ).first() if not result: raise HTTPException(status_code=404, detail="Eval run not found") run, table_name = result + if not table_name: + if run.triggered_by == "promotion-baseline": + table_name = "Production Baseline" + elif run.triggered_by == "promotion-regression": + table_name = "Production Regression" + elif run.triggered_by: + table_name = f"Dataset: {run.triggered_by}" + else: + table_name = "Unknown" return EvalRunRead.model_validate(run, update={"table_name": table_name}) diff --git a/backend/app/routers/orchestration.py b/backend/app/routers/orchestration.py index 653fc3e..0eca9c9 100644 --- a/backend/app/routers/orchestration.py +++ b/backend/app/routers/orchestration.py @@ -293,9 +293,16 @@ def _run_dataset_pipeline(dataset_name: str, run_id: str): # Resolve all production table names from the DB to pass as additional_tables from core.models.models import Table, TableStatus - prod_tables = session.exec( - select(Table).where(Table.status == TableStatus.production) - ).all() + if dataset_name == "spider2": + prod_tables = session.exec( + select(Table).where(Table.owner_id == "spider2") + ).all() + else: + prod_tables = session.exec( + select(Table) + .where(Table.status == TableStatus.production) + .where(Table.owner_id != "spider2") + ).all() table_names = [t.name for t in prod_tables] req = { @@ -339,7 +346,9 @@ def trigger_dataset_run( # 1. Sync production dataset if requested if dataset_name == "text2sql_production": prod_tables = session.exec( - select(Table).where(Table.status == TableStatus.production) + select(Table) + .where(Table.status == TableStatus.production) + .where(Table.owner_id != "spider2") ).all() all_production_questions: list[GoldenQuestion] = [] for table in prod_tables: diff --git a/backend/app/spider2_questions.json b/backend/app/spider2_questions.json new file mode 100644 index 0000000..d2fe9d2 --- /dev/null +++ b/backend/app/spider2_questions.json @@ -0,0 +1,1702 @@ +[ + { + "id": "sf_bq091", + "input": { + "query": "In which year did the assignee with the most applications in the patent category 'A61' file the most?" + }, + "expected_output": { + "sql": "WITH AA AS (SELECT FIRST_VALUE(\"assignee_harmonized\") OVER (PARTITION BY \"application_number\" ORDER BY \"application_number\" ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS assignee_harmonized, FIRST_VALUE(\"filing_date\") OVER (PARTITION BY \"application_number\" ORDER BY \"application_number\" ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS filing_date, \"application_number\" FROM PATENTS.PATENTS.PUBLICATIONS AS pubs CROSS JOIN UNNEST(input => pubs.\"cpc\") AS c(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE JSON_EXTRACT(c.value, '$.code') LIKE 'A61%'), PatentApplications AS (SELECT ARBITRARY(assignee_harmonized) AS assignee_harmonized, ARBITRARY(filing_date) AS filing_date FROM AA GROUP BY \"application_number\"), AssigneeApplications AS (SELECT COUNT(*) AS total_applications, CAST(a.value AS VARCHAR) AS assignee_name, CAST(FLOOR(CAST(filing_date AS DOUBLE) / 10000) AS INTEGER) AS filing_year FROM PatentApplications CROSS JOIN UNNEST(input => assignee_harmonized) AS a(SEQ, KEY, PATH, INDEX, VALUE, THIS) GROUP BY CAST(a.value AS VARCHAR), filing_year), TotalApplicationsPerAssignee AS (SELECT assignee_name, SUM(total_applications) AS total_applications FROM AssigneeApplications GROUP BY assignee_name ORDER BY total_applications DESC NULLS FIRST LIMIT 1), MaxYearForTopAssignee AS (SELECT aa.assignee_name, aa.filing_year, aa.total_applications FROM AssigneeApplications AS aa INNER JOIN TotalApplicationsPerAssignee AS tapa ON aa.assignee_name = tapa.assignee_name ORDER BY aa.total_applications DESC NULLS FIRST LIMIT 1) SELECT filing_year FROM MaxYearForTopAssignee" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "PATENTS", + "catalog": "patents", + "external_knowledge": null + } + }, + { + "id": "sf_bq099", + "input": { + "query": "For patent class A01B3, I want to analyze the information of the top 3 assignees based on the total number of applications. Please provide the following five pieces of information: the name of this assignee, total number of applications, the year with the most applications, the number of applications in that year, and the country code with the most applications during that year." + }, + "expected_output": { + "sql": "WITH PatentApplications AS (SELECT \"assignee_harmonized\" AS assignee_harmonized, \"filing_date\" AS filing_date, \"country_code\" AS country_code, \"application_number\" AS application_number FROM PATENTS.PATENTS.PUBLICATIONS AS pubs CROSS JOIN UNNEST(input => pubs.\"cpc\") AS c(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE JSON_EXTRACT(c.value, '$.code') LIKE 'A01B3%'), AssigneeApplications AS (SELECT COUNT(*) AS year_country_cnt, JSON_EXTRACT(a.value, '$.name') AS assignee_name, CAST(FLOOR(CAST(filing_date AS DOUBLE) / 10000) AS INTEGER) AS filing_year, apps.country_code AS country_code FROM PatentApplications AS apps CROSS JOIN UNNEST(input => assignee_harmonized) AS a(SEQ, KEY, PATH, INDEX, VALUE, THIS) GROUP BY assignee_name, filing_year, country_code), RankedApplications AS (SELECT assignee_name, filing_year, country_code, year_country_cnt, SUM(year_country_cnt) OVER (PARTITION BY assignee_name, filing_year) AS total_cnt, ROW_NUMBER() OVER (PARTITION BY assignee_name, filing_year ORDER BY year_country_cnt DESC NULLS FIRST) AS rn FROM AssigneeApplications), AggregatedData AS (SELECT total_cnt AS year_cnt, assignee_name, filing_year, country_code FROM RankedApplications WHERE rn = 1) SELECT total_count, REPLACE(assignee_name, '\"', '') AS assignee_name, year_cnt, filing_year, country_code FROM (SELECT year_cnt, assignee_name, filing_year, country_code, SUM(year_cnt) OVER (PARTITION BY assignee_name) AS total_count, ROW_NUMBER() OVER (PARTITION BY assignee_name ORDER BY year_cnt DESC NULLS FIRST) AS rn FROM AggregatedData ORDER BY assignee_name) AS sub WHERE rn = 1 ORDER BY total_count DESC NULLS FIRST LIMIT 3" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "PATENTS", + "catalog": "patents", + "external_knowledge": null + } + }, + { + "id": "sf_bq033", + "input": { + "query": "How many U.S. publications related to IoT (where the abstract includes the phrase 'internet of things') were filed each month from 2008 to 2022, including months with no filings?" + }, + "expected_output": { + "sql": "WITH Patent_Matches AS (SELECT CAST(DATE_PARSE(CAST(ARBITRARY(patentsdb.\"filing_date\") AS VARCHAR), '%Y%m%d') AS DATE) AS Patent_Filing_Date, patentsdb.\"application_number\" AS Patent_Application_Number, MAX(JSON_EXTRACT(abstract_info.value, '$.text')) AS Patent_Title, MAX(JSON_EXTRACT(abstract_info.value, '$.language')) AS Patent_Title_Language FROM PATENTS.PATENTS.PUBLICATIONS AS patentsdb CROSS JOIN UNNEST(input => patentsdb.\"abstract_localized\") AS abstract_info(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE LOWER(JSON_EXTRACT(abstract_info.value, '$.text')) LIKE '%internet of things%' AND patentsdb.\"country_code\" = 'US' GROUP BY Patent_Application_Number), Date_Series_Table AS (SELECT DATE_ADD('DAY', CAST(SEQ4() AS BIGINT), CAST('2008-01-01' AS DATE)) AS day, 0 AS Number_of_Patents FROM TABLE(GENERATOR(5479)) ORDER BY day) SELECT DATE_FORMAT(Date_Series_Table.day, 'YYYY-MM') AS Patent_Date_YearMonth, COUNT(Patent_Matches.Patent_Application_Number) AS Number_of_Patent_Applications FROM Date_Series_Table LEFT JOIN Patent_Matches ON Date_Series_Table.day = Patent_Matches.Patent_Filing_Date WHERE Date_Series_Table.day < CAST('2023-01-01' AS DATE) GROUP BY DATE_FORMAT(Date_Series_Table.day, 'YYYY-MM') ORDER BY Patent_Date_YearMonth" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "PATENTS", + "catalog": "patents", + "external_knowledge": null + } + }, + { + "id": "sf_bq209", + "input": { + "query": "Can you calculate the number of utility patents that were granted in 2010 and have exactly one forward citation within a 10-year window following their application/filing date? For this analysis, forward citations should be counted as distinct citing application numbers that cited the patent within 10 years after the patent's own filing date." + }, + "expected_output": { + "sql": "WITH patents_sample AS (SELECT t1.\"publication_number\", t1.\"application_number\" FROM PATENTS.PATENTS.PUBLICATIONS AS t1 WHERE CAST(DATE_PARSE(CASE WHEN t1.\"grant_date\" <> 0 THEN DATE_FORMAT(t1.\"grant_date\") ELSE NULL END, '%Y%m%d') AS DATE) BETWEEN CAST(DATE_PARSE('20100101', '%Y%m%d') AS DATE) AND CAST(DATE_PARSE('20101231', '%Y%m%d') AS DATE)), forward_citation AS (SELECT patents_sample.\"publication_number\", COUNT(DISTINCT t3.\"citing_application_number\") AS \"forward_citations\" FROM patents_sample LEFT JOIN (SELECT x2.\"publication_number\", CAST(DATE_PARSE(CASE WHEN x2.\"filing_date\" <> 0 THEN DATE_FORMAT(x2.\"filing_date\") ELSE NULL END, '%Y%m%d') AS DATE) AS \"filing_date\" FROM PATENTS.PATENTS.PUBLICATIONS AS x2 WHERE x2.\"filing_date\" <> 0) AS t2 ON t2.\"publication_number\" = patents_sample.\"publication_number\" LEFT JOIN (SELECT x3.\"publication_number\" AS \"citing_publication_number\", x3.\"application_number\" AS \"citing_application_number\", CAST(DATE_PARSE(CASE WHEN x3.\"filing_date\" <> 0 THEN DATE_FORMAT(x3.\"filing_date\") ELSE NULL END, '%Y%m%d') AS DATE) AS \"joined_filing_date\", CAST(JSON_EXTRACT(cite.value, '$.publication_number') AS VARCHAR) AS \"cited_publication_number\" FROM PATENTS.PATENTS.PUBLICATIONS AS x3 CROSS JOIN UNNEST(INPUT => x3.\"citation\") AS cite(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE x3.\"filing_date\" <> 0) AS t3 ON patents_sample.\"publication_number\" = t3.\"cited_publication_number\" AND t3.\"joined_filing_date\" BETWEEN t2.\"filing_date\" AND DATE_ADD('YEAR', 10, t2.\"filing_date\") GROUP BY patents_sample.\"publication_number\") SELECT COUNT(*) FROM forward_citation WHERE \"forward_citations\" = 1" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "PATENTS", + "catalog": "patents", + "external_knowledge": null + } + }, + { + "id": "sf_bq210", + "input": { + "query": "How many US B2 patents granted between 2008 and 2018 contain claims that do not include the word 'claim'?" + }, + "expected_output": { + "sql": "WITH patents_sample AS (SELECT t1.\"publication_number\" AS publication_number, JSON_EXTRACT(claim.value, '$.text') AS claims_text FROM PATENTS.PATENTS.PUBLICATIONS AS t1 CROSS JOIN UNNEST(input => t1.\"claims_localized\") AS claim(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE t1.\"country_code\" = 'US' AND t1.\"grant_date\" BETWEEN 20080101 AND 20181231 AND t1.\"grant_date\" <> 0 AND t1.\"publication_number\" LIKE '%B2%'), Publication_data AS (SELECT publication_number, COUNT_IF(NOT claims_text LIKE '%claim%') AS nb_indep_claims FROM patents_sample GROUP BY publication_number) SELECT COUNT(nb_indep_claims) FROM Publication_data WHERE nb_indep_claims <> 0" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "PATENTS", + "catalog": "patents", + "external_knowledge": null + } + }, + { + "id": "sf_bq213", + "input": { + "query": "What is the most common 4-digit IPC code among US B2 utility patents granted from June to August in 2022?" + }, + "expected_output": { + "sql": "WITH interim_table AS (SELECT t1.\"publication_number\", SUBSTR(JSON_EXTRACT(ipc_u.value, '$.code'), 0, 4) AS ipc4 FROM PATENTS.PATENTS.PUBLICATIONS AS t1 CROSS JOIN UNNEST(input => t1.\"ipc\") AS ipc_u(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE \"country_code\" = 'US' AND \"grant_date\" BETWEEN 20220601 AND 20220831 AND \"grant_date\" <> 0 AND \"publication_number\" LIKE '%B2%' GROUP BY t1.\"publication_number\", ipc4) SELECT ipc4 FROM interim_table GROUP BY ipc4 ORDER BY COUNT(\"publication_number\") DESC NULLS FIRST LIMIT 1" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "PATENTS", + "catalog": "patents", + "external_knowledge": "patents_info.md" + } + }, + { + "id": "sf_bq216", + "input": { + "query": "Identify the top five patents filed in the same year as `US-9741766-B2` that are most similar to it based on technological similarities. Please provide the publication numbers." + }, + "expected_output": { + "sql": "WITH patents_sample AS (SELECT \"publication_number\", \"application_number\" FROM PATENTS_GOOGLE.PATENTS_GOOGLE.PUBLICATIONS WHERE \"publication_number\" = 'US-9741766-B2'), flattened_t5 AS (SELECT t5.\"publication_number\", f.value AS element_value, f.index AS pos FROM PATENTS_GOOGLE.PATENTS_GOOGLE.ABS_AND_EMB AS t5 CROSS JOIN UNNEST(input => t5.\"embedding_v1\") AS f(SEQ, KEY, PATH, INDEX, VALUE, THIS)), flattened_t6 AS (SELECT t6.\"publication_number\", f.value AS element_value, f.index AS pos FROM PATENTS_GOOGLE.PATENTS_GOOGLE.ABS_AND_EMB AS t6 CROSS JOIN UNNEST(input => t6.\"embedding_v1\") AS f(SEQ, KEY, PATH, INDEX, VALUE, THIS)), similarities AS (SELECT t1.\"publication_number\" AS base_publication_number, t4.\"publication_number\" AS similar_publication_number, SUM(ft5.element_value * ft6.element_value) AS similarity FROM (SELECT * FROM patents_sample LIMIT 1) AS t1 LEFT JOIN (SELECT x3.\"publication_number\", EXTRACT(YEAR FROM CAST(DATE_PARSE(CAST(x3.\"filing_date\" AS VARCHAR), '%Y%m%d') AS DATE)) AS focal_filing_year FROM PATENTS_GOOGLE.PATENTS_GOOGLE.PUBLICATIONS AS x3 WHERE x3.\"filing_date\" <> 0) AS t3 ON t3.\"publication_number\" = t1.\"publication_number\" LEFT JOIN (SELECT x4.\"publication_number\", EXTRACT(YEAR FROM CAST(DATE_PARSE(CAST(x4.\"filing_date\" AS VARCHAR), '%Y%m%d') AS DATE)) AS filing_year FROM PATENTS_GOOGLE.PATENTS_GOOGLE.PUBLICATIONS AS x4 WHERE x4.\"filing_date\" <> 0) AS t4 ON t4.\"publication_number\" <> t1.\"publication_number\" AND t3.focal_filing_year = t4.filing_year LEFT JOIN flattened_t5 AS ft5 ON ft5.\"publication_number\" = t1.\"publication_number\" LEFT JOIN flattened_t6 AS ft6 ON ft6.\"publication_number\" = t4.\"publication_number\" AND ft5.pos = ft6.pos /* Align vector positions */ GROUP BY t1.\"publication_number\", t4.\"publication_number\") SELECT s.similar_publication_number, s.similarity FROM (SELECT s.*, ROW_NUMBER() OVER (PARTITION BY s.base_publication_number ORDER BY s.similarity DESC NULLS FIRST) AS seqnum FROM similarities AS s) AS s WHERE seqnum <= 5" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "PATENTS_GOOGLE", + "catalog": "patents_google", + "external_knowledge": "patents_info.md" + } + }, + { + "id": "sf_bq127", + "input": { + "query": "For each publication family whose earliest publication was first published in January 2015, please provide the earliest publication date, the distinct publication numbers, their country codes, the distinct CPC and IPC codes, distinct families (namely, the ids) that cite and are cited by this publication family. Please present all lists as comma-separated values, sorted alphabetically" + }, + "expected_output": { + "sql": "WITH fam AS (SELECT DISTINCT \"family_id\" FROM \"PATENTS_GOOGLE\".\"PATENTS_GOOGLE\".\"PUBLICATIONS\"), crossover AS (SELECT \"publication_number\", \"family_id\" FROM \"PATENTS_GOOGLE\".\"PATENTS_GOOGLE\".\"PUBLICATIONS\"), pub AS (SELECT \"family_id\", MIN(\"publication_date\") AS \"publication_date\", LISTAGG(\"publication_number\", ',') WITHIN GROUP (ORDER BY \"publication_number\") AS \"publication_number\", LISTAGG(\"country_code\", ',') WITHIN GROUP (ORDER BY \"country_code\") AS \"country_code\" FROM \"PATENTS_GOOGLE\".\"PATENTS_GOOGLE\".\"PUBLICATIONS\" AS p GROUP BY \"family_id\"), tech_class AS (SELECT p.\"family_id\", LISTAGG(DISTINCT CAST(JSON_EXTRACT(cpc.value, '$.code') AS VARCHAR), ',') WITHIN GROUP (ORDER BY CAST(JSON_EXTRACT(cpc.value, '$.code') AS VARCHAR)) AS \"cpc\", LISTAGG(DISTINCT CAST(JSON_EXTRACT(ipc.value, '$.code') AS VARCHAR), ',') WITHIN GROUP (ORDER BY CAST(JSON_EXTRACT(ipc.value, '$.code') AS VARCHAR)) AS \"ipc\" FROM \"PATENTS_GOOGLE\".\"PATENTS_GOOGLE\".\"PUBLICATIONS\" AS p CROSS JOIN CROSS JOIN UNNEST(input => p.\"cpc\") AS cpc(SEQ, KEY, PATH, INDEX, VALUE, THIS) CROSS JOIN CROSS JOIN UNNEST(input => p.\"ipc\") AS ipc(SEQ, KEY, PATH, INDEX, VALUE, THIS) GROUP BY p.\"family_id\"), cit AS (SELECT p.\"family_id\", LISTAGG(crossover.\"family_id\", ',') WITHIN GROUP (ORDER BY crossover.\"family_id\" ASC) AS \"citation\" FROM \"PATENTS_GOOGLE\".\"PATENTS_GOOGLE\".\"PUBLICATIONS\" AS p CROSS JOIN CROSS JOIN UNNEST(input => p.\"citation\") AS citation(SEQ, KEY, PATH, INDEX, VALUE, THIS) LEFT JOIN crossover ON CAST(JSON_EXTRACT(citation.value, '$.publication_number') AS VARCHAR) = crossover.\"publication_number\" GROUP BY p.\"family_id\"), tmp_gpr AS (SELECT \"family_id\", LISTAGG(crossover.\"publication_number\", ',') AS \"cited_by_publication_number\" FROM \"PATENTS_GOOGLE\".\"PATENTS_GOOGLE\".\"ABS_AND_EMB\" AS p CROSS JOIN CROSS JOIN UNNEST(input => p.\"cited_by\") AS cited_by(SEQ, KEY, PATH, INDEX, VALUE, THIS) LEFT JOIN crossover ON CAST(JSON_EXTRACT(cited_by.value, '$.publication_number') AS VARCHAR) = crossover.\"publication_number\" GROUP BY \"family_id\"), gpr AS (SELECT tmp_gpr.\"family_id\", LISTAGG(crossover.\"family_id\", ',') WITHIN GROUP (ORDER BY crossover.\"family_id\" ASC) AS \"cited_by\" FROM tmp_gpr CROSS JOIN CROSS JOIN UNNEST(input => SPLIT(tmp_gpr.\"cited_by_publication_number\", ',')) AS cited_by_publication_number(SEQ, KEY, PATH, INDEX, VALUE, THIS) LEFT JOIN crossover ON CAST(cited_by_publication_number.value AS VARCHAR) = crossover.\"publication_number\" GROUP BY tmp_gpr.\"family_id\") SELECT fam.\"family_id\", pub.\"publication_date\", pub.\"publication_number\", pub.\"country_code\", tech_class.\"cpc\", tech_class.\"ipc\", cit.\"citation\", gpr.\"cited_by\" FROM fam LEFT JOIN pub ON fam.\"family_id\" = pub.\"family_id\" LEFT JOIN tech_class ON fam.\"family_id\" = tech_class.\"family_id\" LEFT JOIN cit ON fam.\"family_id\" = cit.\"family_id\" LEFT JOIN gpr ON fam.\"family_id\" = gpr.\"family_id\" WHERE pub.\"publication_date\" BETWEEN 20150101 AND 20150131" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "PATENTS_GOOGLE", + "catalog": "patents_google", + "external_knowledge": null + } + }, + { + "id": "sf_bq222", + "input": { + "query": "Find the CPC technology areas in Germany that had the highest exponential moving average (smoothing factor 0.1) of patent filings per year, specifically for patents granted in December 2016. For each CPC group at level 4, show the full title, CPC group, and the year with the highest exponential moving average of patent filings." + }, + "expected_output": { + "sql": "WITH patent_cpcs AS (SELECT cd.\"parents\", CAST(FLOOR(CAST(\"filing_date\" AS DOUBLE) / 10000) AS INTEGER) AS \"filing_year\" FROM (SELECT MAX(\"cpc\") AS \"cpc\", MAX(\"filing_date\") AS \"filing_date\" FROM \"PATENTS\".\"PATENTS\".\"PUBLICATIONS\" WHERE \"application_number\" <> '' AND \"country_code\" = 'DE' AND \"grant_date\" >= 20161201 AND \"grant_date\" <= 20161231 GROUP BY \"application_number\") CROSS JOIN UNNEST(INPUT => \"cpc\") AS cpcs(SEQ, KEY, PATH, INDEX, VALUE, THIS) JOIN \"PATENTS\".\"PATENTS\".\"CPC_DEFINITION\" AS cd ON cd.\"symbol\" = JSON_EXTRACT(cpcs.value, '$.code') WHERE JSON_EXTRACT(cpcs.value, '$.first') = TRUE AND \"filing_date\" > 0), yearly_counts AS (SELECT \"cpc_group\", \"filing_year\", COUNT(*) AS \"cnt\" FROM (SELECT cpc_parent.VALUE AS \"cpc_group\" /* Corrected reference to flattened \"parents\" */, \"filing_year\" FROM patent_cpcs CROSS JOIN UNNEST(INPUT => \"parents\") AS cpc_parent(SEQ, KEY, PATH, INDEX, VALUE, THIS) /* Corrected reference to flattened \"parents\" */) GROUP BY \"cpc_group\", \"filing_year\"), moving_avg AS (SELECT \"cpc_group\", \"filing_year\", \"cnt\", AVG(\"cnt\") OVER (PARTITION BY \"cpc_group\" ORDER BY \"filing_year\" ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS \"moving_avg\" FROM yearly_counts) SELECT c.\"titleFull\" /* Ensure correct column name (check case) */, REPLACE(\"cpc_group\", '\"', '') AS \"cpc_group\", MAX(\"filing_year\") AS \"best_filing_year\" FROM moving_avg JOIN \"PATENTS\".\"PATENTS\".\"CPC_DEFINITION\" AS c ON \"cpc_group\" = c.\"symbol\" WHERE c.\"level\" = 4 GROUP BY c.\"titleFull\", \"cpc_group\" ORDER BY c.\"titleFull\", \"cpc_group\" ASC" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "PATENTS", + "catalog": "patents", + "external_knowledge": "sliding_windows_calculation_cpc.md" + } + }, + { + "id": "sf_bq221", + "input": { + "query": "Identify the CPC technology areas with the highest exponential moving average of patent filings each year (with a smoothing factor of 0.2), considering only the first CPC code for each patent that has a valid filing date and a non-empty application number, and report the full CPC title along with the best year associated with the highest exponential moving average for each CPC group at level 5." + }, + "expected_output": { + "sql": "WITH patent_cpcs AS (SELECT cd.\"parents\", CAST(FLOOR(CAST(\"filing_date\" AS DOUBLE) / 10000) AS INTEGER) AS \"filing_year\" FROM (SELECT MAX(\"cpc\") AS \"cpc\", MAX(\"filing_date\") AS \"filing_date\" FROM PATENTS.PATENTS.PUBLICATIONS WHERE \"application_number\" <> '' GROUP BY \"application_number\") AS publications CROSS JOIN UNNEST(INPUT => \"cpc\") AS cpcs(SEQ, KEY, PATH, INDEX, VALUE, THIS) JOIN PATENTS.PATENTS.CPC_DEFINITION AS cd ON cd.\"symbol\" = JSON_EXTRACT(cpcs.value, '$.code') WHERE JSON_EXTRACT(cpcs.value, '$.first') = TRUE AND \"filing_date\" > 0), yearly_counts AS (SELECT \"cpc_group\", \"filing_year\", COUNT(*) AS \"cnt\" FROM (SELECT CAST(cpc_parent.value AS VARCHAR) AS \"cpc_group\", \"filing_year\" FROM patent_cpcs CROSS JOIN UNNEST(input => patent_cpcs.\"parents\") AS cpc_parent(SEQ, KEY, PATH, INDEX, VALUE, THIS)) GROUP BY \"cpc_group\", \"filing_year\"), ordered_counts AS (SELECT \"cpc_group\", \"filing_year\", \"cnt\", ROW_NUMBER() OVER (PARTITION BY \"cpc_group\" ORDER BY \"filing_year\" ASC) AS rn FROM yearly_counts), recursive_ema AS (/* Anchor member: first year per cpc_group */ SELECT \"cpc_group\", \"filing_year\", \"cnt\", \"cnt\" * 0.2 + 0 * 0.8 AS \"ema\", rn FROM ordered_counts WHERE rn = 1 UNION ALL /* Recursive member: subsequent years */ SELECT oc.\"cpc_group\", oc.\"filing_year\", oc.\"cnt\", oc.\"cnt\" * 0.2 + re.\"ema\" * 0.8 AS \"ema\", oc.rn FROM ordered_counts AS oc JOIN recursive_ema AS re ON oc.\"cpc_group\" = re.\"cpc_group\" AND oc.rn = re.rn + 1), max_ema AS (SELECT \"cpc_group\", \"filing_year\", \"ema\" FROM recursive_ema), ranked_ema AS (SELECT me.\"cpc_group\", me.\"filing_year\", me.\"ema\", ROW_NUMBER() OVER (PARTITION BY me.\"cpc_group\" ORDER BY me.\"ema\" DESC NULLS FIRST, me.\"filing_year\" DESC NULLS FIRST) AS rn_rank FROM max_ema AS me) SELECT c.\"titleFull\", REPLACE(r.\"cpc_group\", '\"', '') AS \"cpc_group\", r.\"filing_year\" AS \"best_filing_year\" FROM ranked_ema AS r JOIN \"PATENTS\".\"PATENTS\".\"CPC_DEFINITION\" AS c ON r.\"cpc_group\" = c.\"symbol\" WHERE c.\"level\" = 5 AND r.rn_rank = 1 ORDER BY c.\"titleFull\", \"cpc_group\" ASC" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "PATENTS", + "catalog": "patents", + "external_knowledge": "sliding_windows_calculation_cpc.md" + } + }, + { + "id": "sf_bq223", + "input": { + "query": "Which assignees, excluding DENSO CORP itself, have cited patents assigned to DENSO CORP, and what are the titles of the primary CPC subclasses associated with these citations? Provide the name of each citing assignee (excluding DENSO CORP), the full title of the primary CPC subclass (based on the first CPC code), and the count of citations grouped by the citing assignee and the CPC subclass title. Ensure that only citations of patents with valid filing dates are considered, and focus on the first CPC code for each citing patent. The results should specifically exclude DENSO CORP as a citing assignee." + }, + "expected_output": { + "sql": "SELECT REPLACE(citing_assignee, '\"', '') AS citing_assignee, cpcdef.\"titleFull\" AS cpc_title, COUNT(*) AS number FROM (SELECT pubs.\"publication_number\" AS citing_publication_number, JSON_EXTRACT(cite.value, '$.publication_number') AS cited_publication_number, JSON_EXTRACT(citing_assignee_s.value, '$.name') AS citing_assignee, SUBSTR(JSON_EXTRACT(cpcs.value, '$.code'), 1, 4) AS citing_cpc_subclass FROM PATENTS.PATENTS.PUBLICATIONS AS pubs CROSS JOIN UNNEST(input => pubs.\"citation\") AS cite(SEQ, KEY, PATH, INDEX, VALUE, THIS) CROSS JOIN UNNEST(input => pubs.\"assignee_harmonized\") AS citing_assignee_s(SEQ, KEY, PATH, INDEX, VALUE, THIS) CROSS JOIN UNNEST(input => pubs.\"cpc\") AS cpcs(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE JSON_EXTRACT(cpcs.value, '$.first') = TRUE) AS pubs JOIN (SELECT \"publication_number\" AS cited_publication_number, JSON_EXTRACT(cited_assignee_s.value, '$.name') AS cited_assignee FROM PATENTS.PATENTS.PUBLICATIONS CROSS JOIN UNNEST(input => \"assignee_harmonized\") AS cited_assignee_s(SEQ, KEY, PATH, INDEX, VALUE, THIS)) AS refs ON pubs.cited_publication_number = refs.cited_publication_number JOIN PATENTS.PATENTS.CPC_DEFINITION AS cpcdef ON cpcdef.\"symbol\" = pubs.citing_cpc_subclass WHERE refs.cited_assignee = 'DENSO CORP' AND pubs.citing_assignee <> 'DENSO CORP' GROUP BY citing_assignee, cpcdef.\"titleFull\"" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "PATENTS", + "catalog": "patents", + "external_knowledge": "patents_info.md" + } + }, + { + "id": "sf_bq128", + "input": { + "query": "Retrieve the following information for U.S. patents filed between January 1, 2014, and February 1, 2014. The patent title and abstract. The publication date of the patent. The number of backward citations for each patent (i.e., the number of patents cited by the current patent before its filing date). The number of forward citations for each patent within the first 5 years of its publication (i.e., the number of patents that cited the current patent within 5 years after its publication). For each patent, ensure the forward citations are counted only for citations within 5 years after the publication date, and backward citations are counted for citations before the filing date." + }, + "expected_output": { + "sql": "SELECT patent.\"title\", patent.\"abstract\", app.\"date\" AS publication_date, filterData.\"bkwdCitations\", filterData.\"fwrdCitations_5\" FROM \"PATENTSVIEW\".\"PATENTSVIEW\".\"PATENT\" AS patent JOIN \"PATENTSVIEW\".\"PATENTSVIEW\".\"APPLICATION\" AS app ON app.\"patent_id\" = patent.\"id\" JOIN (SELECT DISTINCT cpc.\"patent_id\", COALESCE(citation_5.\"bkwdCitations\", 0) AS \"bkwdCitations\", COALESCE(citation_5.\"fwrdCitations_5\", 0) AS \"fwrdCitations_5\" FROM \"PATENTSVIEW\".\"PATENTSVIEW\".\"CPC_CURRENT\" AS cpc LEFT JOIN (SELECT b.\"patent_id\", b.\"bkwdCitations\", f.\"fwrdCitations_5\" FROM (SELECT cited.\"citation_id\" AS \"patent_id\", COALESCE(COUNT(*), 0) AS \"fwrdCitations_5\" FROM \"PATENTSVIEW\".\"PATENTSVIEW\".\"USPATENTCITATION\" AS cited JOIN \"PATENTSVIEW\".\"PATENTSVIEW\".\"APPLICATION\" AS apps ON cited.\"citation_id\" = apps.\"patent_id\" WHERE apps.\"country\" = 'US' AND cited.\"date\" >= apps.\"date\" AND TRY_CAST(cited.\"date\" AS DATE) <= DATE_ADD('YEAR', 5, TRY_CAST(apps.\"date\" AS DATE)) /* 5-year citation window */ GROUP BY cited.\"citation_id\") AS f JOIN (SELECT cited.\"patent_id\", COALESCE(COUNT(*), 0) AS \"bkwdCitations\" FROM \"PATENTSVIEW\".\"PATENTSVIEW\".\"USPATENTCITATION\" AS cited JOIN \"PATENTSVIEW\".\"PATENTSVIEW\".\"APPLICATION\" AS apps ON cited.\"patent_id\" = apps.\"patent_id\" WHERE apps.\"country\" = 'US' AND cited.\"date\" < apps.\"date\" /* backward citation count */ GROUP BY cited.\"patent_id\") AS b ON b.\"patent_id\" = f.\"patent_id\" WHERE NOT b.\"bkwdCitations\" IS NULL AND NOT f.\"fwrdCitations_5\" IS NULL) AS citation_5 ON cpc.\"patent_id\" = citation_5.\"patent_id\" WHERE cpc.\"subsection_id\" IN ('C05', 'C06', 'C07', 'C08', 'C09', 'C10', 'C11', 'C12', 'C13') OR cpc.\"group_id\" IN ('A01G', 'A01H', 'A61K', 'A61P', 'A61Q', 'B01F', 'B01J', 'B81B', 'B82B', 'B82Y', 'G01N', 'G16H')) AS filterData ON app.\"patent_id\" = filterData.\"patent_id\" WHERE TRY_CAST(app.\"date\" AS DATE) < '2014-02-01' AND TRY_CAST(app.\"date\" AS DATE) >= '2014-01-01'" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "PATENTSVIEW", + "catalog": "patentsview", + "external_knowledge": "forward_backward_citation.md" + } + }, + { + "id": "sf_bq246", + "input": { + "query": "Retrieve U.S. patents with the number of forward citations within the first 3 years after the patent application date (i.e., patents citing the current patent within 3 years). Only include patents with both backward citations within 1 year before the application date and forward citations within 1 year after the application date. The query should focus on specific CPC categories, sort results by backward citations in descending order, and return the patent with the most backward citations, limiting to one result." + }, + "expected_output": { + "sql": "SELECT filterData.\"fwrdCitations_3\" FROM PATENTSVIEW.PATENTSVIEW.APPLICATION AS app JOIN (SELECT DISTINCT cpc.\"patent_id\", COALESCE(citation_3.\"bkwdCitations_3\", 0) AS \"bkwdCitations_3\", COALESCE(citation_3.\"fwrdCitations_3\", 0) AS \"fwrdCitations_3\" FROM PATENTSVIEW.PATENTSVIEW.CPC_CURRENT AS cpc LEFT JOIN (SELECT b.\"patent_id\", b.\"bkwdCitations_3\", f.\"fwrdCitations_3\" FROM (SELECT cited.\"patent_id\", COUNT(*) AS \"fwrdCitations_3\" FROM PATENTSVIEW.PATENTSVIEW.USPATENTCITATION AS cited JOIN PATENTSVIEW.PATENTSVIEW.APPLICATION AS apps ON cited.\"patent_id\" = apps.\"patent_id\" WHERE apps.\"country\" = 'US' AND cited.\"date\" >= apps.\"date\" AND TRY_CAST(cited.\"date\" AS DATE) <= DATE_ADD('YEAR', 1, TRY_CAST(apps.\"date\" AS DATE)) /* Citation within 1 year */ GROUP BY cited.\"patent_id\") AS f JOIN (SELECT cited.\"patent_id\", COUNT(*) AS \"bkwdCitations_3\" FROM PATENTSVIEW.PATENTSVIEW.USPATENTCITATION AS cited JOIN PATENTSVIEW.PATENTSVIEW.APPLICATION AS apps ON cited.\"patent_id\" = apps.\"patent_id\" WHERE apps.\"country\" = 'US' AND cited.\"date\" < apps.\"date\" AND TRY_CAST(cited.\"date\" AS DATE) >= DATE_ADD('YEAR', -1, TRY_CAST(apps.\"date\" AS DATE)) /* Citation within 1 year before */ GROUP BY cited.\"patent_id\") AS b ON b.\"patent_id\" = f.\"patent_id\" WHERE NOT b.\"bkwdCitations_3\" IS NULL AND NOT f.\"fwrdCitations_3\" IS NULL) AS citation_3 ON cpc.\"patent_id\" = citation_3.\"patent_id\") AS filterData ON app.\"patent_id\" = filterData.\"patent_id\" ORDER BY filterData.\"bkwdCitations_3\" DESC NULLS FIRST LIMIT 1" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "PATENTSVIEW", + "catalog": "patentsview", + "external_knowledge": null + } + }, + { + "id": "sf_bq052", + "input": { + "query": "Retrieve the following information for U.S. patents: The patent ID, title, and application date. The number of backward citations within 1 month before the application date (i.e., patents that cited the current patent before its application). The number of forward citations within 1 month after the application date (i.e., patents that cited the current patent after its application). The abstract text of the patent. Only include patents that belong to specific CPC categories, such as subsection 'C05' or group 'A01G'. The query should filter patents to include only those that have at least one backward citation or one forward citation in the 1-month period specified. Sort the results by application date and return all matching records." + }, + "expected_output": { + "sql": "SELECT app.\"patent_id\" AS \"patent_id\", patent.\"title\", app.\"date\" AS \"application_date\", filterData.\"bkwdCitations_1\", filterData.\"fwrdCitations_1\", summary.\"text\" AS \"summary_text\" FROM PATENTSVIEW.PATENTSVIEW.BRF_SUM_TEXT AS summary JOIN PATENTSVIEW.PATENTSVIEW.PATENT AS patent ON summary.\"patent_id\" = patent.\"id\" JOIN PATENTSVIEW.PATENTSVIEW.APPLICATION AS app ON app.\"patent_id\" = summary.\"patent_id\" JOIN (SELECT DISTINCT cpc.\"patent_id\", COALESCE(citation_1.\"bkwdCitations_1\", 0) AS \"bkwdCitations_1\", COALESCE(citation_1.\"fwrdCitations_1\", 0) AS \"fwrdCitations_1\" FROM PATENTSVIEW.PATENTSVIEW.CPC_CURRENT AS cpc JOIN (SELECT b.\"patent_id\", b.\"bkwdCitations_1\", f.\"fwrdCitations_1\" FROM (SELECT cited.\"patent_id\", COUNT(*) AS \"fwrdCitations_1\" FROM PATENTSVIEW.PATENTSVIEW.USPATENTCITATION AS cited JOIN PATENTSVIEW.PATENTSVIEW.APPLICATION AS apps ON cited.\"patent_id\" = apps.\"patent_id\" WHERE apps.\"country\" = 'US' AND cited.\"date\" >= apps.\"date\" AND TRY_CAST(cited.\"date\" AS DATE) <= DATE_ADD('MONTH', 1, TRY_CAST(apps.\"date\" AS DATE)) /* Citation within 1 month */ GROUP BY cited.\"patent_id\") AS f JOIN (SELECT cited.\"patent_id\", COUNT(*) AS \"bkwdCitations_1\" FROM PATENTSVIEW.PATENTSVIEW.USPATENTCITATION AS cited JOIN PATENTSVIEW.PATENTSVIEW.APPLICATION AS apps ON cited.\"patent_id\" = apps.\"patent_id\" WHERE apps.\"country\" = 'US' AND cited.\"date\" < apps.\"date\" AND TRY_CAST(cited.\"date\" AS DATE) >= DATE_ADD('MONTH', -1, TRY_CAST(apps.\"date\" AS DATE)) /* Citation within 1 month before */ GROUP BY cited.\"patent_id\") AS b ON b.\"patent_id\" = f.\"patent_id\" WHERE NOT b.\"bkwdCitations_1\" IS NULL AND NOT f.\"fwrdCitations_1\" IS NULL AND (b.\"bkwdCitations_1\" > 0 OR f.\"fwrdCitations_1\" > 0)) AS citation_1 ON cpc.\"patent_id\" = citation_1.\"patent_id\" WHERE cpc.\"subsection_id\" = 'C05' OR cpc.\"group_id\" = 'A01G') AS filterData ON app.\"patent_id\" = filterData.\"patent_id\" ORDER BY app.\"date\"" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "PATENTSVIEW", + "catalog": "patentsview", + "external_knowledge": null + } + }, + { + "id": "sf_bq182", + "input": { + "query": "Which primary programming languages, determined by the highest number of bytes in each repository, had at least 5 PullRequestEvents on January 18, 2023 across all their repositories?" + }, + "expected_output": { + "sql": "WITH event_data AS (SELECT \"type\", EXTRACT(YEAR FROM TO_TIMESTAMP(CAST(\"created_at\" AS DOUBLE) / 1000000)) AS \"year\", EXTRACT(QUARTER FROM TO_TIMESTAMP(CAST(\"created_at\" AS DOUBLE) / 1000000)) AS \"quarter\", REGEXP_REPLACE(CAST(JSON_EXTRACT(CAST(\"repo\" AS VARIANT), '$.url') AS VARCHAR), 'https:\\/\\/github\\.com\\/|https:\\/\\/api\\.github\\.com\\/repos\\/', '') AS \"name\" FROM GITHUB_REPOS_DATE.DAY._20230118), repo_languages AS (SELECT \"repo_name\" AS \"name\", \"lang\" FROM (SELECT \"repo_name\", FIRST_VALUE(\"language\") OVER (PARTITION BY \"repo_name\" ORDER BY \"bytes\" DESC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS \"lang\" FROM (SELECT \"repo_name\", JSON_EXTRACT(\"language\".value, '$.name') AS \"language\", JSON_EXTRACT(\"language\".value, '$.bytes') AS \"bytes\" FROM GITHUB_REPOS_DATE.GITHUB_REPOS.LANGUAGES CROSS JOIN UNNEST(INPUT => \"language\") AS \"language\"(SEQ, KEY, PATH, INDEX, VALUE, THIS))) WHERE NOT \"lang\" IS NULL GROUP BY \"repo_name\", \"lang\"), joined_data AS (SELECT a.\"type\" AS \"type\", b.\"lang\" AS \"language\", a.\"year\" AS \"year\", a.\"quarter\" AS \"quarter\" FROM event_data AS a JOIN repo_languages AS b ON a.\"name\" = b.\"name\"), count_data AS (SELECT \"language\", \"year\", \"quarter\", \"type\", COUNT(*) AS \"count\" FROM joined_data GROUP BY \"type\", \"language\", \"year\", \"quarter\" ORDER BY \"year\", \"quarter\", \"count\" DESC NULLS FIRST) SELECT REPLACE(\"language\", '\"', '') AS \"language_name\", \"count\" FROM count_data WHERE \"count\" >= 5 AND \"type\" = 'PullRequestEvent'" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "GITHUB_REPOS_DATE", + "catalog": "github_repos_date", + "external_knowledge": null + } + }, + { + "id": "sf_bq224", + "input": { + "query": "Which repository with an approved license in `licenses.md` had the highest combined total of forks, issues, and watches in April 2022?" + }, + "expected_output": { + "sql": "WITH allowed_repos AS (SELECT \"repo_name\", \"license\" FROM GITHUB_REPOS_DATE.GITHUB_REPOS.LICENSES WHERE \"license\" IN ('gpl-3.0', 'artistic-2.0', 'isc', 'cc0-1.0', 'epl-1.0', 'gpl-2.0', 'mpl-2.0', 'lgpl-2.1', 'bsd-2-clause', 'apache-2.0', 'mit', 'lgpl-3.0')), watch_counts AS (SELECT CAST(JSON_EXTRACT(JSON_PARSE(\"repo\"), '$.name') AS VARCHAR) AS \"repo\", COUNT(DISTINCT CAST(JSON_EXTRACT(JSON_PARSE(\"actor\"), '$.login') AS VARCHAR)) AS \"watches\" FROM GITHUB_REPOS_DATE.MONTH._202204 WHERE \"type\" = 'WatchEvent' GROUP BY JSON_EXTRACT(JSON_PARSE(\"repo\"), '$.name')), issue_counts AS (SELECT CAST(JSON_EXTRACT(JSON_PARSE(\"repo\"), '$.name') AS VARCHAR) AS \"repo\", COUNT(*) AS \"issue_events\" FROM GITHUB_REPOS_DATE.MONTH._202204 WHERE \"type\" = 'IssuesEvent' GROUP BY JSON_EXTRACT(JSON_PARSE(\"repo\"), '$.name')), fork_counts AS (SELECT CAST(JSON_EXTRACT(JSON_PARSE(\"repo\"), '$.name') AS VARCHAR) AS \"repo\", COUNT(*) AS \"forks\" FROM GITHUB_REPOS_DATE.MONTH._202204 WHERE \"type\" = 'ForkEvent' GROUP BY JSON_EXTRACT(JSON_PARSE(\"repo\"), '$.name')) SELECT ar.\"repo_name\" FROM allowed_repos AS ar INNER JOIN fork_counts AS fc ON ar.\"repo_name\" = fc.\"repo\" INNER JOIN issue_counts AS ic ON ar.\"repo_name\" = ic.\"repo\" INNER JOIN watch_counts AS wc ON ar.\"repo_name\" = wc.\"repo\" ORDER BY (fc.\"forks\" + ic.\"issue_events\" + wc.\"watches\") DESC NULLS FIRST LIMIT 1" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "GITHUB_REPOS_DATE", + "catalog": "github_repos_date", + "external_knowledge": null + } + }, + { + "id": "sf_bq233", + "input": { + "query": "Can you analyze the joined data from github repos files and github_repos contents, focusing only on files ending with '.py' or '.r', then extract Python modules from 'import' or 'from ... import' lines and R libraries from 'library(...)' lines, count their occurrences, and finally list the results sorted by language and by the number of occurrences in descending order?" + }, + "expected_output": { + "sql": "WITH extracted_modules AS (SELECT el.\"file_id\" AS \"file_id\", el.\"repo_name\", el.\"path\" AS \"path_\", REPLACE(line.value, '\"', '') AS \"line_\", CASE WHEN ENDS_WITH(el.\"path\", '.py') THEN 'python' WHEN ENDS_WITH(el.\"path\", '.r') THEN 'r' ELSE NULL END AS \"language\", CASE WHEN ENDS_WITH(el.\"path\", '.py') THEN CONCAT(ARRAY[REGEXP_EXTRACT(line.value, '\\bimport\\s+(\\w+)')], ARRAY[REGEXP_EXTRACT(line.value, '\\bfrom\\s+(\\w+)')]) WHEN ENDS_WITH(el.\"path\", '.r') THEN ARRAY[REGEXP_EXTRACT(line.value, 'library\\s*\\(\\s*([^\\s)]+)\\s*\\)')] ELSE ARRAY[] END AS \"modules\" FROM (SELECT ct.\"id\" AS \"file_id\", fl.\"repo_name\" AS \"repo_name\", fl.\"path\", SPLIT(REPLACE(ct.\"content\", '\n', ' \n'), '\n') AS \"lines\" FROM GITHUB_REPOS_DATE.GITHUB_REPOS.SAMPLE_FILES AS fl JOIN GITHUB_REPOS_DATE.GITHUB_REPOS.SAMPLE_CONTENTS AS ct ON fl.\"id\" = ct.\"id\") AS el CROSS JOIN UNNEST(input => el.\"lines\") AS line(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE (ENDS_WITH(\"path_\", '.py') AND (\"line_\" LIKE 'import %' OR \"line_\" LIKE 'from %')) OR (ENDS_WITH(\"path_\", '.r') AND \"line_\" LIKE 'library%(')), module_counts AS (SELECT em.\"language\", CAST(f.value AS VARCHAR) AS \"module\", COUNT(*) AS \"occurrence_count\" FROM extracted_modules AS em CROSS JOIN UNNEST(input => em.\"modules\") AS f(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE NOT em.\"modules\" IS NULL AND NOT f.value IS NULL GROUP BY em.\"language\", f.value), python AS (SELECT \"language\", \"module\", \"occurrence_count\" FROM module_counts WHERE \"language\" = 'python'), rlanguage AS (SELECT \"language\", \"module\", \"occurrence_count\" FROM module_counts AS mc_inner WHERE \"language\" = 'r') SELECT * FROM python UNION ALL SELECT * FROM rlanguage ORDER BY \"language\", \"occurrence_count\" DESC NULLS FIRST" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "GITHUB_REPOS", + "catalog": "github_repos", + "external_knowledge": null + } + }, + { + "id": "sf_bq248", + "input": { + "query": "Among all repositories that do not use any programming language whose name (case-insensitively) includes the substring \"python,\" what is the proportion of files whose paths include \"readme.md\" and whose contents contain the phrase \"Copyright (c)\"?" + }, + "expected_output": { + "sql": "WITH requests AS (SELECT D.\"id\", D.\"content\", E.\"repo_name\", E.\"path\" FROM (SELECT \"id\", \"content\" FROM GITHUB_REPOS.GITHUB_REPOS.SAMPLE_CONTENTS GROUP BY \"id\", \"content\") AS D INNER JOIN (SELECT C.\"id\", C.\"repo_name\", C.\"path\" FROM (SELECT \"id\", \"repo_name\", \"path\" FROM GITHUB_REPOS.GITHUB_REPOS.SAMPLE_FILES WHERE LOWER(\"path\") LIKE '%readme.md' GROUP BY \"path\", \"id\", \"repo_name\") AS C INNER JOIN (SELECT \"repo_name\", CAST(JSON_EXTRACT(language_struct.value, '$.name') AS VARCHAR) AS \"language_name\" FROM GITHUB_REPOS.GITHUB_REPOS.LANGUAGES CROSS JOIN UNNEST(input => \"language\") AS language_struct(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE NOT LOWER(CAST(JSON_EXTRACT(language_struct.value, '$.name') AS VARCHAR)) LIKE '%python%' GROUP BY \"language_name\", \"repo_name\") AS F ON C.\"repo_name\" = F.\"repo_name\") AS E ON D.\"id\" = E.\"id\") SELECT CAST((SELECT COUNT(*) FROM requests WHERE \"content\" LIKE '%Copyright (c)%') AS DOUBLE) / COUNT(*) AS \"proportion\" FROM requests" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "GITHUB_REPOS", + "catalog": "github_repos", + "external_knowledge": null + } + }, + { + "id": "sf_bq193", + "input": { + "query": "Retrieve all non-empty, non-commented lines from `README.md` files in GitHub repositories, excluding lines that are comments (either starting with `#` for Markdown or `//` for code comments). For each line, calculate how often each unique line appears across all repositories and return a comma-separated list of the programming languages used in each repository containing that line, sorted alphabetically, with the results ordered by the frequency of occurrence in descending order." + }, + "expected_output": { + "sql": "WITH content_extracted AS (SELECT \"D\".\"id\" AS \"id\", \"repo_name\", \"path\", SPLIT(\"content\", '\n') AS \"lines\", \"language_name\" FROM (SELECT \"id\", \"content\" FROM \"GITHUB_REPOS\".\"GITHUB_REPOS\".\"SAMPLE_CONTENTS\") AS \"D\" INNER JOIN (SELECT \"id\", \"C\".\"repo_name\" AS \"repo_name\", \"path\", \"language_name\" FROM (SELECT \"id\", \"repo_name\", \"path\" FROM \"GITHUB_REPOS\".\"GITHUB_REPOS\".\"SAMPLE_FILES\" WHERE LOWER(\"path\") LIKE '%readme.md') AS \"C\" INNER JOIN (SELECT \"repo_name\", JSON_EXTRACT(\"language_struct\".value, '$.name') AS \"language_name\" FROM (SELECT \"repo_name\", \"language\" FROM \"GITHUB_REPOS\".\"GITHUB_REPOS\".\"LANGUAGES\") CROSS JOIN CROSS JOIN UNNEST(INPUT => \"language\") AS \"language_struct\"(SEQ, KEY, PATH, INDEX, VALUE, THIS)) AS \"F\" ON \"C\".\"repo_name\" = \"F\".\"repo_name\") AS \"E\" ON \"E\".\"id\" = \"D\".\"id\"), non_empty_lines AS (SELECT \"line\".value AS \"line_\", \"language_name\" FROM content_extracted CROSS JOIN UNNEST(INPUT => \"lines\") AS \"line\"(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE TRIM(\"line\".value) <> '' AND NOT STARTS_WITH(TRIM(\"line\".value), '#') AND NOT STARTS_WITH(TRIM(\"line\".value), '//')), aggregated_languages AS (SELECT \"line_\", COUNT(*) AS \"frequency\", ARRAY_AGG(\"language_name\") FILTER(WHERE \"language_name\" IS NOT NULL) AS \"languages\" FROM non_empty_lines GROUP BY \"line_\") SELECT REGEXP_REPLACE(\"line_\", '^\"|\"$', '') AS \"line\", \"frequency\", ARRAY_JOIN(ARRAY_SORT(\"languages\"), ', ') AS \"languages_sorted\" FROM aggregated_languages ORDER BY \"frequency\" DESC NULLS FIRST" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "GITHUB_REPOS", + "catalog": "github_repos", + "external_knowledge": null + } + }, + { + "id": "sf_bq295", + "input": { + "query": "Using the 2017 GitHub Archive data for watch events, which three repositories that include at least one Python file (with a .py extension) smaller than 15,000 bytes and containing the substring \"def \" in its content have the highest total number of watch events for that year?" + }, + "expected_output": { + "sql": "WITH watched_repos AS (SELECT CAST(JSON_EXTRACT(JSON_PARSE(\"repo\"), '$.name') AS VARCHAR) AS \"repo\" FROM GITHUB_REPOS_DATE.MONTH._201701 WHERE \"type\" = 'WatchEvent' UNION ALL SELECT CAST(JSON_EXTRACT(JSON_PARSE(\"repo\"), '$.name') AS VARCHAR) AS \"repo\" FROM GITHUB_REPOS_DATE.MONTH._201702 WHERE \"type\" = 'WatchEvent' UNION ALL SELECT CAST(JSON_EXTRACT(JSON_PARSE(\"repo\"), '$.name') AS VARCHAR) AS \"repo\" FROM GITHUB_REPOS_DATE.MONTH._201703 WHERE \"type\" = 'WatchEvent' UNION ALL SELECT CAST(JSON_EXTRACT(JSON_PARSE(\"repo\"), '$.name') AS VARCHAR) AS \"repo\" FROM GITHUB_REPOS_DATE.MONTH._201704 WHERE \"type\" = 'WatchEvent' UNION ALL SELECT CAST(JSON_EXTRACT(JSON_PARSE(\"repo\"), '$.name') AS VARCHAR) AS \"repo\" FROM GITHUB_REPOS_DATE.MONTH._201705 WHERE \"type\" = 'WatchEvent' UNION ALL SELECT CAST(JSON_EXTRACT(JSON_PARSE(\"repo\"), '$.name') AS VARCHAR) AS \"repo\" FROM GITHUB_REPOS_DATE.MONTH._201706 WHERE \"type\" = 'WatchEvent' UNION ALL SELECT CAST(JSON_EXTRACT(JSON_PARSE(\"repo\"), '$.name') AS VARCHAR) AS \"repo\" FROM GITHUB_REPOS_DATE.MONTH._201707 WHERE \"type\" = 'WatchEvent' UNION ALL SELECT CAST(JSON_EXTRACT(JSON_PARSE(\"repo\"), '$.name') AS VARCHAR) AS \"repo\" FROM GITHUB_REPOS_DATE.MONTH._201708 WHERE \"type\" = 'WatchEvent' UNION ALL SELECT CAST(JSON_EXTRACT(JSON_PARSE(\"repo\"), '$.name') AS VARCHAR) AS \"repo\" FROM GITHUB_REPOS_DATE.MONTH._201709 WHERE \"type\" = 'WatchEvent' UNION ALL SELECT CAST(JSON_EXTRACT(JSON_PARSE(\"repo\"), '$.name') AS VARCHAR) AS \"repo\" FROM GITHUB_REPOS_DATE.MONTH._201710 WHERE \"type\" = 'WatchEvent' UNION ALL SELECT CAST(JSON_EXTRACT(JSON_PARSE(\"repo\"), '$.name') AS VARCHAR) AS \"repo\" FROM GITHUB_REPOS_DATE.MONTH._201711 WHERE \"type\" = 'WatchEvent' UNION ALL SELECT CAST(JSON_EXTRACT(JSON_PARSE(\"repo\"), '$.name') AS VARCHAR) AS \"repo\" FROM GITHUB_REPOS_DATE.MONTH._201712 WHERE \"type\" = 'WatchEvent'), repo_watch_counts AS (SELECT \"repo\", COUNT(*) AS \"watch_count\" FROM watched_repos GROUP BY \"repo\") SELECT REPLACE(r.\"repo\", '\"', '') AS \"repo\", r.\"watch_count\" FROM GITHUB_REPOS_DATE.GITHUB_REPOS.SAMPLE_FILES AS f JOIN GITHUB_REPOS_DATE.GITHUB_REPOS.SAMPLE_CONTENTS AS c ON f.\"id\" = c.\"id\" JOIN repo_watch_counts AS r ON f.\"repo_name\" = r.\"repo\" WHERE f.\"path\" LIKE '%.py' AND c.\"size\" < 15000 AND STRPOS(c.\"content\", 'def ') > 0 GROUP BY r.\"repo\", r.\"watch_count\" ORDER BY r.\"watch_count\" DESC NULLS FIRST LIMIT 3" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "GITHUB_REPOS_DATE", + "catalog": "github_repos_date", + "external_knowledge": null + } + }, + { + "id": "sf_bq255", + "input": { + "query": "How many commit messages are there in repositories that use the 'Shell' programming language and 'apache-2.0' license, where the length of the commit message is more than 5 characters but less than 10,000 characters, and the messages do not start with the word 'merge', 'update' or 'test'?" + }, + "expected_output": { + "sql": "SELECT COUNT(commits_table.\"message\") AS \"num_messages\" FROM (SELECT L.\"repo_name\", CAST(JSON_EXTRACT(language_struct.value, '$.name') AS VARCHAR) AS \"language_name\" FROM GITHUB_REPOS.GITHUB_REPOS.LANGUAGES AS L CROSS JOIN UNNEST(input => L.\"language\") AS language_struct(SEQ, KEY, PATH, INDEX, VALUE, THIS)) AS lang_table JOIN GITHUB_REPOS.GITHUB_REPOS.LICENSES AS license_table ON license_table.\"repo_name\" = lang_table.\"repo_name\" JOIN (SELECT * FROM GITHUB_REPOS.GITHUB_REPOS.SAMPLE_COMMITS) AS commits_table ON commits_table.\"repo_name\" = lang_table.\"repo_name\" WHERE license_table.\"license\" LIKE 'apache-2.0' AND lang_table.\"language_name\" LIKE 'Shell' AND LENGTH(commits_table.\"message\") > 5 AND LENGTH(commits_table.\"message\") < 10000 AND NOT LOWER(commits_table.\"message\") LIKE 'update%' AND NOT LOWER(commits_table.\"message\") LIKE 'test%' AND NOT LOWER(commits_table.\"message\") LIKE 'merge%'" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "GITHUB_REPOS", + "catalog": "github_repos", + "external_knowledge": null + } + }, + { + "id": "sf_bq377", + "input": { + "query": "Extract and count the frequency of all package names listed in the require section of JSON-formatted content" + }, + "expected_output": { + "sql": "WITH json_files AS (SELECT c.\"id\", JSON_EXTRACT(JSON_PARSE(c.\"content\"), '$.require') AS \"dependencies\" FROM GITHUB_REPOS.GITHUB_REPOS.SAMPLE_CONTENTS AS c), package_names AS (SELECT f.key AS \"package_name\" FROM json_files CROSS JOIN UNNEST(input => \"dependencies\") AS f(SEQ, KEY, PATH, INDEX, VALUE, THIS)) SELECT \"package_name\", COUNT(*) AS \"count\" FROM package_names WHERE NOT \"package_name\" IS NULL GROUP BY \"package_name\" ORDER BY \"count\" DESC NULLS FIRST" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "GITHUB_REPOS", + "catalog": "github_repos", + "external_knowledge": null + } + }, + { + "id": "sf_bq359", + "input": { + "query": "List the repository names and commit counts for the top two GitHub repositories with JavaScript as the primary language and the highest number of commits." + }, + "expected_output": { + "sql": "WITH repositories AS (SELECT t2.\"repo_name\", t2.\"language\" FROM (SELECT t1.\"repo_name\", t1.\"language\", RANK() OVER (PARTITION BY t1.\"repo_name\" ORDER BY t1.\"language_bytes\" DESC NULLS FIRST) AS \"rank\" FROM (SELECT l.\"repo_name\", CAST(JSON_EXTRACT(lang.value, '$.name') AS VARCHAR) AS \"language\", CAST(JSON_EXTRACT(lang.value, '$.bytes') AS DECIMAL(38, 0)) AS \"language_bytes\" FROM GITHUB_REPOS.GITHUB_REPOS.LANGUAGES AS l CROSS JOIN UNNEST(input => l.\"language\") AS lang(SEQ, KEY, PATH, INDEX, VALUE, THIS)) AS t1) AS t2 WHERE t2.\"rank\" = 1), python_repo AS (SELECT \"repo_name\", \"language\" FROM repositories WHERE \"language\" = 'JavaScript') SELECT sc.\"repo_name\", COUNT(sc.\"commit\") AS \"num_commits\" FROM GITHUB_REPOS.GITHUB_REPOS.SAMPLE_COMMITS AS sc INNER JOIN python_repo ON python_repo.\"repo_name\" = sc.\"repo_name\" GROUP BY sc.\"repo_name\" ORDER BY \"num_commits\" DESC NULLS FIRST LIMIT 2" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "GITHUB_REPOS", + "catalog": "github_repos", + "external_knowledge": null + } + }, + { + "id": "sf_bq252", + "input": { + "query": "Could you please find the name of the repository that contains the most copied non-binary Swift file in the dataset, ensuring each file is uniquely identified by its ID?" + }, + "expected_output": { + "sql": "WITH selected_repos AS (SELECT f.\"id\", f.\"repo_name\" AS \"repo_name\", f.\"path\" AS \"path\" FROM GITHUB_REPOS.GITHUB_REPOS.SAMPLE_FILES AS f), deduped_files AS (SELECT f.\"id\", MIN(f.\"repo_name\") AS \"repo_name\", MIN(f.\"path\") AS \"path\" FROM selected_repos AS f GROUP BY f.\"id\") SELECT f.\"repo_name\" FROM deduped_files AS f JOIN GITHUB_REPOS.GITHUB_REPOS.SAMPLE_CONTENTS AS c ON f.\"id\" = c.\"id\" WHERE NOT c.\"binary\" AND f.\"path\" LIKE '%.swift' ORDER BY c.\"copies\" DESC NULLS FIRST LIMIT 1" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "GITHUB_REPOS", + "catalog": "github_repos", + "external_knowledge": null + } + }, + { + "id": "sf_bq236", + "input": { + "query": "What are the top 5 zip codes of the areas in the United States that have experienced the most hail storm events in the past 10 years? Don't use data from hail reports table." + }, + "expected_output": { + "sql": "SELECT CONCAT(CAST(\"city\" AS VARCHAR), CAST(', ' AS VARCHAR), CAST(\"state_name\" AS VARCHAR)) AS \"city\", \"zip_code\", COUNT(\"event_id\") AS \"count_storms\" FROM (SELECT * FROM NOAA_DATA_PLUS.NOAA_HISTORIC_SEVERE_STORMS.STORMS_2014 UNION ALL SELECT * FROM NOAA_DATA_PLUS.NOAA_HISTORIC_SEVERE_STORMS.STORMS_2015 UNION ALL SELECT * FROM NOAA_DATA_PLUS.NOAA_HISTORIC_SEVERE_STORMS.STORMS_2016 UNION ALL SELECT * FROM NOAA_DATA_PLUS.NOAA_HISTORIC_SEVERE_STORMS.STORMS_2017 UNION ALL SELECT * FROM NOAA_DATA_PLUS.NOAA_HISTORIC_SEVERE_STORMS.STORMS_2018 UNION ALL SELECT * FROM NOAA_DATA_PLUS.NOAA_HISTORIC_SEVERE_STORMS.STORMS_2019 UNION ALL SELECT * FROM NOAA_DATA_PLUS.NOAA_HISTORIC_SEVERE_STORMS.STORMS_2020 UNION ALL SELECT * FROM NOAA_DATA_PLUS.NOAA_HISTORIC_SEVERE_STORMS.STORMS_2021 UNION ALL SELECT * FROM NOAA_DATA_PLUS.NOAA_HISTORIC_SEVERE_STORMS.STORMS_2022 UNION ALL SELECT * FROM NOAA_DATA_PLUS.NOAA_HISTORIC_SEVERE_STORMS.STORMS_2023 UNION ALL SELECT * FROM NOAA_DATA_PLUS.NOAA_HISTORIC_SEVERE_STORMS.STORMS_2024) AS storms JOIN NOAA_DATA_PLUS.GEO_US_BOUNDARIES.ZIP_CODES ON ST_WITHIN(ST_GEOGFROMWKB(storms.\"event_point\"), ST_GEOGFROMWKB(\"zip_code_geom\")) WHERE LOWER(storms.\"event_type\") = 'hail' GROUP BY \"zip_code\", \"city\", \"state_name\" ORDER BY \"count_storms\" DESC NULLS FIRST LIMIT 5" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "NOAA_DATA_PLUS", + "catalog": "noaa_data_plus", + "external_knowledge": "functions_st_within.md" + } + }, + { + "id": "sf_bq358", + "input": { + "query": "Can you tell me which bike trip in New York City on July 15, 2015, started and ended in ZIP Code areas with the highest average temperature for that day, as recorded by the Central Park weather station (WBAN '94728')? If there's more than one trip that meets these criteria, I'd like to know about the one that starts in the smallest ZIP Code and ends in the largest ZIP Code. Please return the starting and ending ZIP Codes of this trip." + }, + "expected_output": { + "sql": "SELECT \"ZIPSTART\".\"zip_code\" AS zip_code_start, \"ZIPEND\".\"zip_code\" AS zip_code_end FROM \"NEW_YORK_CITIBIKE_1\".\"NEW_YORK_CITIBIKE\".\"CITIBIKE_TRIPS\" AS \"TRI\" INNER JOIN \"NEW_YORK_CITIBIKE_1\".\"GEO_US_BOUNDARIES\".\"ZIP_CODES\" AS \"ZIPSTART\" ON ST_WITHIN(ST_POINT(\"TRI\".\"start_station_longitude\", \"TRI\".\"start_station_latitude\"), ST_GEOGFROMWKB(\"ZIPSTART\".\"zip_code_geom\")) INNER JOIN \"NEW_YORK_CITIBIKE_1\".\"GEO_US_BOUNDARIES\".\"ZIP_CODES\" AS \"ZIPEND\" ON ST_WITHIN(ST_POINT(\"TRI\".\"end_station_longitude\", \"TRI\".\"end_station_latitude\"), ST_GEOGFROMWKB(\"ZIPEND\".\"zip_code_geom\")) INNER JOIN \"NEW_YORK_CITIBIKE_1\".\"NOAA_GSOD\".\"GSOD2015\" AS \"WEA\" ON CAST(DATE_PARSE(CONCAT(CAST(CAST(DATE_FORMAT(\"WEA\".\"year\") AS VARCHAR) AS VARCHAR), CAST(CAST(LPAD(DATE_FORMAT(\"WEA\".\"mo\"), 2, '0') AS VARCHAR) AS VARCHAR), CAST(CAST(LPAD(DATE_FORMAT(\"WEA\".\"da\"), 2, '0') AS VARCHAR) AS VARCHAR)), '%Y%m%d') AS DATE) = DATE_TRUNC('DAY', TO_TIMESTAMP_NTZ(CAST(CAST(\"TRI\".\"starttime\" AS DOUBLE) AS DOUBLE) / 1000000)) WHERE \"WEA\".\"wban\" = '94728' AND DATE_TRUNC('DAY', TO_TIMESTAMP_NTZ(CAST(CAST(\"TRI\".\"starttime\" AS DOUBLE) AS DOUBLE) / 1000000)) = CAST('2015-07-15' AS DATE) ORDER BY \"WEA\".\"temp\" DESC NULLS FIRST, \"ZIPSTART\".\"zip_code\" ASC, \"ZIPEND\".\"zip_code\" DESC NULLS FIRST LIMIT 1" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "NEW_YORK_CITIBIKE_1", + "catalog": "new_york_citibike_1", + "external_knowledge": "functions_st_within.md" + } + }, + { + "id": "sf_bq050", + "input": { + "query": "I want to analyze bike trips in New York City for 2014 by linking trip data with weather information to understand how weather conditions (temperature, wind speed, and precipitation) affect bike trips between neighborhoods. For each combination of starting and ending neighborhoods, I need the following: 1. Total number of bike trips between the neighborhoods. 2. Average trip duration in minutes (rounded to 1 decimal). 3. Average temperature at the start of the trip (rounded to 1 decimal). 4. Average wind speed at the start (in meters per second, rounded to 1 decimal). 5. Average precipitation at the start (in centimeters, rounded to 1 decimal). 6. The month with the most trips (e.g., `4` for April). The data should be grouped by the starting and ending neighborhoods, with:`zip_codes` in `geo_us_boundaries` used to map the bike trip locations based on latitude and longitude. `zip_codes` in `cyclistic` used to obtain the borough and neighborhood names. Using weather data from the Central Park station for the trip date, covering all trips in 2014." + }, + "expected_output": { + "sql": "WITH data AS (SELECT \"ZIPSTARTNAME\".\"borough\" AS \"borough_start\", \"ZIPSTARTNAME\".\"neighborhood\" AS \"neighborhood_start\", \"ZIPENDNAME\".\"borough\" AS \"borough_end\", \"ZIPENDNAME\".\"neighborhood\" AS \"neighborhood_end\", CAST(CAST(\"TRI\".\"tripduration\" AS DOUBLE) / 60 AS DECIMAL(38, 0)) AS \"trip_minutes\", \"WEA\".\"temp\" AS \"temperature\", CAST(\"WEA\".\"wdsp\" AS DECIMAL(38, 0)) AS \"wind_speed\", \"WEA\".\"prcp\" AS \"precipitation\", EXTRACT(MONTH FROM CAST(CAST(\"TRI\".\"starttime\" AS TIMESTAMP) AS DATE)) AS \"start_month\" FROM \"NEW_YORK_CITIBIKE_1\".\"NEW_YORK_CITIBIKE\".\"CITIBIKE_TRIPS\" AS \"TRI\" INNER JOIN \"NEW_YORK_CITIBIKE_1\".\"GEO_US_BOUNDARIES\".\"ZIP_CODES\" AS \"ZIPSTART\" ON ST_WITHIN(ST_POINT(\"TRI\".\"start_station_longitude\", \"TRI\".\"start_station_latitude\"), ST_GEOGFROMWKB(\"ZIPSTART\".\"zip_code_geom\")) INNER JOIN \"NEW_YORK_CITIBIKE_1\".\"GEO_US_BOUNDARIES\".\"ZIP_CODES\" AS \"ZIPEND\" ON ST_WITHIN(ST_POINT(\"TRI\".\"end_station_longitude\", \"TRI\".\"end_station_latitude\"), ST_GEOGFROMWKB(\"ZIPEND\".\"zip_code_geom\")) INNER JOIN \"NEW_YORK_CITIBIKE_1\".\"NOAA_GSOD\".\"GSOD2014\" AS \"WEA\" ON CAST(DATE_PARSE(CONCAT(CAST(CAST(\"WEA\".\"year\" AS VARCHAR) AS VARCHAR), CAST(CAST(LPAD(\"WEA\".\"mo\", 2, '0') AS VARCHAR) AS VARCHAR), CAST(CAST(LPAD(\"WEA\".\"da\", 2, '0') AS VARCHAR) AS VARCHAR)), '%Y%m%d') AS DATE) = CAST(CAST(\"TRI\".\"starttime\" AS TIMESTAMP) AS DATE) INNER JOIN \"NEW_YORK_CITIBIKE_1\".\"CYCLISTIC\".\"ZIP_CODES\" AS \"ZIPSTARTNAME\" ON \"ZIPSTART\".\"zip_code\" = CAST(\"ZIPSTARTNAME\".\"zip\" AS VARCHAR) INNER JOIN \"NEW_YORK_CITIBIKE_1\".\"CYCLISTIC\".\"ZIP_CODES\" AS \"ZIPENDNAME\" ON \"ZIPEND\".\"zip_code\" = CAST(\"ZIPENDNAME\".\"zip\" AS VARCHAR) WHERE \"WEA\".\"wban\" = (SELECT \"wban\" FROM \"NEW_YORK_CITIBIKE_1\".\"NOAA_GSOD\".\"STATIONS\" WHERE \"state\" = 'NY' AND LOWER(\"name\") LIKE LOWER('%New York Central Park%') LIMIT 1) AND EXTRACT(YEAR FROM CAST(CAST(\"TRI\".\"starttime\" AS TIMESTAMP) AS DATE)) = 2014), agg_data AS (SELECT \"borough_start\", \"neighborhood_start\", \"borough_end\", \"neighborhood_end\", COUNT(*) AS \"num_trips\", ROUND(AVG(\"trip_minutes\"), 1) AS \"avg_trip_minutes\", ROUND(AVG(\"temperature\"), 1) AS \"avg_temperature\", ROUND(AVG(\"wind_speed\"), 1) AS \"avg_wind_speed\", ROUND(AVG(\"precipitation\"), 1) AS \"avg_precipitation\" FROM data GROUP BY \"borough_start\", \"neighborhood_start\", \"borough_end\", \"neighborhood_end\"), most_common_months AS (SELECT \"borough_start\", \"neighborhood_start\", \"borough_end\", \"neighborhood_end\", \"start_month\", ROW_NUMBER() OVER (PARTITION BY \"borough_start\", \"neighborhood_start\", \"borough_end\", \"neighborhood_end\" ORDER BY COUNT(*) DESC NULLS FIRST) AS \"row_num\" FROM data GROUP BY \"borough_start\", \"neighborhood_start\", \"borough_end\", \"neighborhood_end\", \"start_month\") SELECT a.*, m.\"start_month\" AS \"most_common_month\" FROM agg_data AS a JOIN most_common_months AS m ON a.\"borough_start\" = m.\"borough_start\" AND a.\"neighborhood_start\" = m.\"neighborhood_start\" AND a.\"borough_end\" = m.\"borough_end\" AND a.\"neighborhood_end\" = m.\"neighborhood_end\" AND m.\"row_num\" = 1 ORDER BY a.\"neighborhood_start\", a.\"neighborhood_end\"" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "NEW_YORK_CITIBIKE_1", + "catalog": "new_york_citibike_1", + "external_knowledge": "functions_st_within.md" + } + }, + { + "id": "sf_bq291", + "input": { + "query": "Can you provide a daily weather summary for July 2019 within a 5 km radius of latitude 26.75 and longitude 51.5? I need the maximum, minimum, and average temperatures; total precipitation; average cloud cover between 10 AM and 5 PM; total snowfall (when average temperature is below 32°F); and total rainfall (when average temperature is 32°F or above) for each forecast date. The data should correspond to forecasts created in July 2019 for the following day." + }, + "expected_output": { + "sql": "WITH daily_forecasts AS (SELECT \"TRI\".\"creation_time\", CAST(DATE_ADD('HOUR', 1, TO_TIMESTAMP_NTZ(CAST(CAST(JSON_EXTRACT(\"forecast\".value, '$.time') AS DOUBLE) AS DOUBLE) / 1000000)) AS DATE) AS \"local_forecast_date\", MAX(CASE WHEN NOT JSON_EXTRACT(\"forecast\".value, '$.temperature_2m_above_ground') IS NULL THEN JSON_EXTRACT(\"forecast\".value, '$.temperature_2m_above_ground') ELSE NULL END) AS \"max_temp\", MIN(CASE WHEN NOT JSON_EXTRACT(\"forecast\".value, '$.temperature_2m_above_ground') IS NULL THEN JSON_EXTRACT(\"forecast\".value, '$.temperature_2m_above_ground') ELSE NULL END) AS \"min_temp\", AVG(CASE WHEN NOT JSON_EXTRACT(\"forecast\".value, '$.temperature_2m_above_ground') IS NULL THEN JSON_EXTRACT(\"forecast\".value, '$.temperature_2m_above_ground') ELSE NULL END) AS \"avg_temp\", SUM(CASE WHEN NOT JSON_EXTRACT(\"forecast\".value, '$.total_precipitation_surface') IS NULL THEN JSON_EXTRACT(\"forecast\".value, '$.total_precipitation_surface') ELSE 0 END) AS \"total_precipitation\", AVG(CASE WHEN CAST(DATE_ADD('HOUR', 1, TO_TIMESTAMP_NTZ(CAST(CAST(JSON_EXTRACT(\"forecast\".value, '$.time') AS DOUBLE) AS DOUBLE) / 1000000)) AS TIME) BETWEEN '10:00:00' AND '17:00:00' AND NOT JSON_EXTRACT(\"forecast\".value, '$.total_cloud_cover_entire_atmosphere') IS NULL THEN JSON_EXTRACT(\"forecast\".value, '$.total_cloud_cover_entire_atmosphere') ELSE NULL END) AS \"avg_cloud_cover\", CASE WHEN AVG(JSON_EXTRACT(\"forecast\".value, '$.temperature_2m_above_ground')) < 32 THEN SUM(CASE WHEN NOT JSON_EXTRACT(\"forecast\".value, '$.total_precipitation_surface') IS NULL THEN JSON_EXTRACT(\"forecast\".value, '$.total_precipitation_surface') ELSE 0 END) ELSE 0 END AS \"total_snow\", CASE WHEN AVG(JSON_EXTRACT(\"forecast\".value, '$.temperature_2m_above_ground')) >= 32 THEN SUM(CASE WHEN NOT JSON_EXTRACT(\"forecast\".value, '$.total_precipitation_surface') IS NULL THEN JSON_EXTRACT(\"forecast\".value, '$.total_precipitation_surface') ELSE 0 END) ELSE 0 END AS \"total_rain\" FROM \"NOAA_GLOBAL_FORECAST_SYSTEM\".\"NOAA_GLOBAL_FORECAST_SYSTEM\".\"NOAA_GFS0P25\" AS \"TRI\" CROSS JOIN CROSS JOIN UNNEST(input => \"TRI\".\"forecast\") AS \"forecast\"(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE TO_TIMESTAMP_NTZ(CAST(CAST(\"TRI\".\"creation_time\" AS DOUBLE) AS DOUBLE) / 1000000) BETWEEN '2019-07-01' AND '2021-07-31' AND ST_DWITHIN(ST_GEOGFROMWKB(\"TRI\".\"geography\"), ST_POINT(26.75, 51.5), 5000) AND CAST(TO_TIMESTAMP_NTZ(CAST(CAST(JSON_EXTRACT(\"forecast\".value, '$.time') AS DOUBLE) AS DOUBLE) / 1000000) AS DATE) = DATE_ADD('DAY', 1, CAST(TO_TIMESTAMP_NTZ(CAST(CAST(\"TRI\".\"creation_time\" AS DOUBLE) AS DOUBLE) / 1000000) AS DATE)) GROUP BY \"TRI\".\"creation_time\", \"local_forecast_date\") SELECT TO_TIMESTAMP_NTZ(CAST(CAST(\"creation_time\" AS DOUBLE) AS DOUBLE) / 1000000), \"local_forecast_date\" AS \"forecast_date\", \"max_temp\", \"min_temp\", \"avg_temp\", \"total_precipitation\", \"avg_cloud_cover\", \"total_snow\", \"total_rain\" FROM daily_forecasts ORDER BY \"creation_time\", \"forecast_date\"" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "NOAA_GLOBAL_FORECAST_SYSTEM", + "catalog": "noaa_global_forecast_system", + "external_knowledge": "functions_st_within.md" + } + }, + { + "id": "sf_bq017", + "input": { + "query": "What are the five longest types of highways within the multipolygon boundary of Denmark (as defined by Wikidata ID 'Q35') by total length, analyzed through planet features?" + }, + "expected_output": { + "sql": "WITH bounding_area AS (SELECT \"geometry\" AS geometry FROM GEO_OPENSTREETMAP.GEO_OPENSTREETMAP.PLANET_FEATURES CROSS JOIN UNNEST(INPUT => planet_features.\"all_tags\") AS \"tag\"(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE \"feature_type\" = 'multipolygons' AND JSON_EXTRACT(\"tag\".value, '$.key') = 'wikidata' AND JSON_EXTRACT(\"tag\".value, '$.value') = 'Q35'), highway_info AS (SELECT SUM(ST_LENGTH(ST_GEOGRAPHYFROMWKB(planet_features.\"geometry\"))) AS highway_length, JSON_EXTRACT(\"tag\".value, '$.value') AS highway_type FROM GEO_OPENSTREETMAP.GEO_OPENSTREETMAP.PLANET_FEATURES AS planet_features, bounding_area CROSS JOIN CROSS JOIN UNNEST(INPUT => planet_features.\"all_tags\") AS \"tag\"(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE JSON_EXTRACT(\"tag\".value, '$.key') = 'highway' AND \"feature_type\" = 'lines' AND ST_DWITHIN(ST_GEOGFROMWKB(planet_features.\"geometry\"), ST_GEOGFROMWKB(bounding_area.geometry), 0.0) GROUP BY highway_type) SELECT REPLACE(highway_type, '\"', '') AS highway_type FROM highway_info ORDER BY highway_length DESC NULLS FIRST LIMIT 5" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "GEO_OPENSTREETMAP", + "catalog": "geo_openstreetmap", + "external_knowledge": "functions_st_dwithin.md" + } + }, + { + "id": "sf_bq349", + "input": { + "query": "Which OpenStreetMap ID from the planet features table corresponds to an administrative boundary, represented as multipolygons, whose total number of 'amenity'-tagged Points of Interest (POIs), as derived from the planet nodes table, is closest to the median count among all such boundaries?" + }, + "expected_output": { + "sql": "WITH bounding_area AS (SELECT \"osm_id\", \"geometry\" AS geometry, ST_AREA(ST_GEOGRAPHYFROMWKB(\"geometry\")) AS area FROM GEO_OPENSTREETMAP.GEO_OPENSTREETMAP.PLANET_FEATURES CROSS JOIN UNNEST(INPUT => PLANET_FEATURES.\"all_tags\") AS \"tag\"(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE \"feature_type\" = 'multipolygons' AND JSON_EXTRACT(\"tag\".value, '$.key') = 'boundary' AND JSON_EXTRACT(\"tag\".value, '$.value') = 'administrative'), poi AS (SELECT nodes.\"id\" AS poi_id, nodes.\"geometry\" AS poi_geometry, JSON_EXTRACT(tags.value, '$.value') AS poitype FROM GEO_OPENSTREETMAP.GEO_OPENSTREETMAP.PLANET_NODES AS nodes CROSS JOIN UNNEST(INPUT => nodes.\"all_tags\") AS tags(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE JSON_EXTRACT(tags.value, '$.key') = 'amenity'), poi_counts AS (SELECT ba.\"osm_id\", COUNT(poi.poi_id) AS total_pois FROM bounding_area AS ba JOIN poi ON ST_DWITHIN(ST_GEOGRAPHYFROMWKB(ba.geometry), ST_GEOGRAPHYFROMWKB(poi.poi_geometry), 0.0) GROUP BY ba.\"osm_id\"), median_value AS (SELECT APPROX_PERCENTILE(total_pois, 0.5) AS median_pois FROM poi_counts), closest_to_median AS (SELECT \"osm_id\", total_pois, ABS(total_pois - (SELECT median_pois FROM median_value)) AS diff_from_median FROM poi_counts) SELECT \"osm_id\" FROM closest_to_median ORDER BY diff_from_median LIMIT 1" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "GEO_OPENSTREETMAP", + "catalog": "geo_openstreetmap", + "external_knowledge": "functions_st_dwithin.md" + } + }, + { + "id": "sf_bq429", + "input": { + "query": "Which are the top five states with the greatest average difference in median income between 2015 and 2018 at the ZIP code level, and what is the corresponding average number of vulnerable employees across wholesale trade, natural resources and construction, arts and entertainment, information, and retail trade industries in 2017 according to the ACS Five-Year Estimates and ZIP code boundaries data?" + }, + "expected_output": { + "sql": "WITH median_income_diff_by_zipcode AS (WITH acs_2018 AS (SELECT \"geo_id\", \"median_income\" AS \"median_income_2018\" FROM CENSUS_BUREAU_ACS_2.CENSUS_BUREAU_ACS.\"ZIP_CODES_2018_5YR\"), acs_2015 AS (SELECT \"geo_id\", \"median_income\" AS \"median_income_2015\" FROM CENSUS_BUREAU_ACS_2.CENSUS_BUREAU_ACS.\"ZIP_CODES_2015_5YR\"), acs_diff AS (SELECT a18.\"geo_id\", (a18.\"median_income_2018\" - a15.\"median_income_2015\") AS \"median_income_diff\" FROM acs_2018 AS a18 JOIN acs_2015 AS a15 ON a18.\"geo_id\" = a15.\"geo_id\") SELECT \"geo_id\", AVG(\"median_income_diff\") AS \"avg_median_income_diff\" FROM acs_diff WHERE NOT \"median_income_diff\" IS NULL GROUP BY \"geo_id\"), base_census AS (SELECT geo.\"state_name\", AVG(i.\"avg_median_income_diff\") AS \"avg_median_income_diff\", AVG(\"employed_wholesale_trade\" * 0.38423645320197042 + \"occupation_natural_resources_construction_maintenance\" * 0.48071410777129553 + \"employed_arts_entertainment_recreation_accommodation_food\" * 0.89455676291236841 + \"employed_information\" * 0.31315240083507306 + \"employed_retail_trade\" * 0.51) AS \"avg_vulnerable\" FROM CENSUS_BUREAU_ACS_2.CENSUS_BUREAU_ACS.\"ZIP_CODES_2017_5YR\" AS census JOIN median_income_diff_by_zipcode AS i ON CAST(census.\"geo_id\" AS VARCHAR) = i.\"geo_id\" JOIN CENSUS_BUREAU_ACS_2.GEO_US_BOUNDARIES.\"ZIP_CODES\" AS geo ON census.\"geo_id\" = geo.\"zip_code\" GROUP BY geo.\"state_name\") SELECT \"state_name\", \"avg_median_income_diff\", \"avg_vulnerable\" FROM base_census ORDER BY \"avg_median_income_diff\" DESC NULLS FIRST LIMIT 5" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "CENSUS_BUREAU_ACS_2", + "catalog": "census_bureau_acs_2", + "external_knowledge": "avg_vulnerable_weights.md" + } + }, + { + "id": "sf_bq254", + "input": { + "query": "Among all multipolygons located within the same geographic area as the multipolygon associated with Wikidata item Q191, but lacking a 'wikidata' tag themselves, which two rank highest by the number of points that lie within their boundaries, and what are their names?" + }, + "expected_output": { + "sql": "WITH bounding_area AS (SELECT \"geometry\" AS geometry FROM GEO_OPENSTREETMAP.GEO_OPENSTREETMAP.PLANET_FEATURES CROSS JOIN UNNEST(INPUT => \"all_tags\") AS tag(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE \"feature_type\" = 'multipolygons' AND JSON_EXTRACT(tag.value, '$.key') = 'wikidata' AND JSON_EXTRACT(tag.value, '$.value') = 'Q191'), bounding_area_features AS (SELECT planet_features.\"osm_id\", planet_features.\"feature_type\", planet_features.\"geometry\", planet_features.\"all_tags\" FROM GEO_OPENSTREETMAP.GEO_OPENSTREETMAP.PLANET_FEATURES AS planet_features, bounding_area WHERE ST_DWITHIN(ST_GEOGFROMWKB(planet_features.\"geometry\"), ST_GEOGFROMWKB(bounding_area.geometry), 0.0)), osm_id_with_wikidata AS (SELECT DISTINCT baf.\"osm_id\" FROM bounding_area_features AS baf CROSS JOIN UNNEST(INPUT => baf.\"all_tags\") AS tag(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE JSON_EXTRACT(tag.value, '$.key') = 'wikidata'), polygons_wo_wikidata AS (SELECT baf.\"osm_id\", JSON_EXTRACT(tag.value, '$.value') AS name, baf.\"geometry\" AS geometry FROM bounding_area_features AS baf LEFT JOIN osm_id_with_wikidata AS wd ON baf.\"osm_id\" = wd.\"osm_id\" CROSS JOIN UNNEST(INPUT => \"all_tags\") AS tag(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE wd.\"osm_id\" IS NULL AND NOT baf.\"osm_id\" IS NULL AND baf.\"feature_type\" = 'multipolygons' AND JSON_EXTRACT(tag.value, '$.key') = 'name') SELECT TRIM(pww.name) AS name FROM bounding_area_features AS baf JOIN polygons_wo_wikidata AS pww ON ST_DWITHIN(ST_GEOGFROMWKB(baf.\"geometry\"), ST_GEOGFROMWKB(pww.geometry), 0.0) LEFT JOIN osm_id_with_wikidata AS wd ON baf.\"osm_id\" = wd.\"osm_id\" WHERE NOT wd.\"osm_id\" IS NULL AND baf.\"feature_type\" = 'points' GROUP BY pww.name ORDER BY COUNT(baf.\"osm_id\") DESC NULLS FIRST LIMIT 2" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "GEO_OPENSTREETMAP", + "catalog": "geo_openstreetmap", + "external_knowledge": "functions_st_dwithin.md" + } + }, + { + "id": "sf_bq289", + "input": { + "query": "Can you find the shortest distance between any two amenities (either a library, place of worship, or community center) located within Philadelphia, analyzed through pennsylvania table and planet features points?" + }, + "expected_output": { + "sql": "WITH philadelphia AS (SELECT * FROM GEO_OPENSTREETMAP_CENSUS_PLACES.GEO_US_CENSUS_PLACES.PLACES_PENNSYLVANIA WHERE \"place_name\" = 'Philadelphia'), amenities AS (SELECT features.*, JSON_EXTRACT(tags.value, '$.value') AS amenity FROM GEO_OPENSTREETMAP_CENSUS_PLACES.GEO_OPENSTREETMAP.PLANET_FEATURES_POINTS AS features CROSS JOIN philadelphia CROSS JOIN UNNEST(input => features.\"all_tags\") AS tags(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE ST_CONTAINS(ST_GEOGFROMWKB(philadelphia.\"place_geom\"), ST_GEOGFROMWKB(features.\"geometry\")) AND JSON_EXTRACT(tags.value, '$.key') = 'amenity' AND JSON_EXTRACT(tags.value, '$.value') IN ('library', 'place_of_worship', 'community_centre')), joiin AS (SELECT a1.*, a2.\"osm_id\" AS nearest_osm_id, ST_DISTANCE(ST_GEOGFROMWKB(a1.\"geometry\"), ST_GEOGFROMWKB(a2.\"geometry\")) AS distance, ROW_NUMBER() OVER (PARTITION BY a1.\"osm_id\" ORDER BY ST_DISTANCE(ST_GEOGFROMWKB(a1.\"geometry\"), ST_GEOGFROMWKB(a2.\"geometry\"))) AS row_num FROM amenities AS a1 CROSS JOIN amenities AS a2 WHERE a1.\"osm_id\" < a2.\"osm_id\" ORDER BY a1.\"osm_id\", distance) SELECT distance FROM joiin WHERE row_num = 1 ORDER BY distance ASC LIMIT 1" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "GEO_OPENSTREETMAP_CENSUS_PLACES", + "catalog": "geo_openstreetmap_census_places", + "external_knowledge": "functions_st_contains.md" + } + }, + { + "id": "sf_bq250", + "input": { + "query": "Based on the most recent 1km population grid data in Singapore before January 2023, using ST_CONVEXHULL to aggregate all population grid centroids into a bounding region and ST_INTERSECTS to identify hospitals from OpenStreetMap’s planet layer (layer_code in (2110, 2120)) that fall within this region, then calculating the distance from each grid cell to its nearest hospital, what is the total population of the grid cell that is farthest from any hospital?" + }, + "expected_output": { + "sql": "WITH country_name AS (SELECT 'Singapore' AS value), last_updated AS (SELECT MAX(\"last_updated\") AS value FROM GEO_OPENSTREETMAP_WORLDPOP.WORLDPOP.POPULATION_GRID_1KM AS pop INNER JOIN country_name ON (pop.\"country_name\" = country_name.value) WHERE \"last_updated\" < '2023-01-01'), aggregated_population AS (SELECT \"geo_id\", SUM(\"population\") AS sum_population, ST_POINT(\"longitude_centroid\", \"latitude_centroid\") AS centr /* 计算每个 geo_id 的中心点 */ FROM GEO_OPENSTREETMAP_WORLDPOP.WORLDPOP.POPULATION_GRID_1KM AS pop INNER JOIN country_name ON (pop.\"country_name\" = country_name.value) INNER JOIN last_updated ON (pop.\"last_updated\" = last_updated.value) GROUP BY \"geo_id\", \"longitude_centroid\", \"latitude_centroid\"), population AS (SELECT SUM(sum_population) AS sum_population, ST_ENVELOPE(ST_UNION_AGG(centr)) AS boundingbox /* 使用 ST_ENVELOPE 来代替 ST_CONVEXHULL */ FROM aggregated_population), hospitals AS (SELECT layer.\"geometry\" FROM GEO_OPENSTREETMAP_WORLDPOP.GEO_OPENSTREETMAP.PLANET_LAYERS AS layer INNER JOIN population ON ST_INTERSECTS(population.boundingbox, ST_GEOGFROMWKB(layer.\"geometry\")) WHERE layer.\"layer_code\" IN (2110, 2120)), distances AS (SELECT pop.\"geo_id\", pop.\"population\", MIN(ST_DISTANCE(ST_GEOGFROMWKB(pop.\"geog\"), ST_GEOGFROMWKB(hospitals.\"geometry\"))) AS distance FROM GEO_OPENSTREETMAP_WORLDPOP.WORLDPOP.POPULATION_GRID_1KM AS pop INNER JOIN country_name ON pop.\"country_name\" = country_name.value INNER JOIN last_updated ON pop.\"last_updated\" = last_updated.value CROSS JOIN hospitals WHERE pop.\"population\" > 0 GROUP BY \"geo_id\", \"population\") SELECT SUM(pd.\"population\") AS population FROM distances AS pd CROSS JOIN population AS p GROUP BY distance ORDER BY distance DESC NULLS FIRST LIMIT 1" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "GEO_OPENSTREETMAP_WORLDPOP", + "catalog": "geo_openstreetmap_worldpop", + "external_knowledge": "OpenStreetMap_data_in_layered_GIS_format.md" + } + }, + { + "id": "sf_bq083", + "input": { + "query": "Can you calculate the daily change in the market value of USDC tokens (address `0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48`) for 2023, based on Ethereum transactions? The change should be computed from minting (input pattern `0x40c10f19%`) and burning (input pattern `0x42966c68%`) operations. For each transaction, minting should be positive and burning negative. Extract the relevant amount from the 'input' field as a hexadecimal, convert it to millions, express it in USD format. Group the results by date and order them in descending order." + }, + "expected_output": { + "sql": "SELECT CAST(CAST(TO_TIMESTAMP_NTZ(CAST(\"block_timestamp\" AS DOUBLE) / 1000000) AS TIMESTAMP) AS DATE) AS \"Date\" /* 将时间戳转换为日期格式，除以1000000 */, DATE_FORMAT(SUM(CASE WHEN \"input\" LIKE '0x40c10f19%' THEN 1 ELSE -1 END * CAST(CONCAT(CAST('0x' AS VARCHAR), CAST(TRIM(LEADING '0' FROM SUBSTR(\"input\", CASE WHEN \"input\" LIKE '0x40c10f19%' THEN 75 ELSE 11 END, 64)) AS VARCHAR)) AS DOUBLE) / 1000000), '$999,999,999,999') AS \"Δ Total Market Value\" FROM \"CRYPTO\".\"CRYPTO_ETHEREUM\".\"TRANSACTIONS\" WHERE CAST(CAST(TO_TIMESTAMP_NTZ(CAST(\"block_timestamp\" AS DOUBLE) / 1000000) AS TIMESTAMP) AS DATE) BETWEEN '2023-01-01' AND '2023-12-31' AND \"to_address\" = '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48' /* USDC Token */ AND (\"input\" LIKE '0x42966c68%' /* Burn */ OR \"input\" LIKE '0x40c10f19%' /* Mint */) GROUP BY CAST(CAST(TO_TIMESTAMP_NTZ(CAST(\"block_timestamp\" AS DOUBLE) / 1000000) AS TIMESTAMP) AS DATE) ORDER BY \"Date\" DESC NULLS FIRST" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "CRYPTO", + "catalog": "crypto", + "external_knowledge": "Total_Market_Value_Change.md" + } + }, + { + "id": "sf_bq341", + "input": { + "query": "Which Ethereum address has the top 3 smallest positive balance from transactions involving the token at address \"0xa92a861fc11b99b24296af880011b47f9cafb5ab\"?" + }, + "expected_output": { + "sql": "WITH transaction_addresses AS (SELECT \"from_address\", \"to_address\", CAST(\"value\" AS DECIMAL(38, 0)) / 1000000 AS \"value\" FROM \"CRYPTO\".\"CRYPTO_ETHEREUM\".\"TOKEN_TRANSFERS\" WHERE \"token_address\" = '0xa92a861fc11b99b24296af880011b47f9cafb5ab'), out_addresses AS (SELECT \"from_address\", SUM(-1 * \"value\") AS \"total_value\" FROM transaction_addresses GROUP BY \"from_address\"), in_addresses AS (SELECT \"to_address\", SUM(\"value\") AS \"total_value\" FROM transaction_addresses GROUP BY \"to_address\"), all_addresses AS (SELECT \"from_address\" AS \"address\", \"total_value\" FROM out_addresses UNION ALL SELECT \"to_address\" AS \"address\", \"total_value\" FROM in_addresses) SELECT \"address\" FROM all_addresses GROUP BY \"address\" HAVING SUM(\"total_value\") > 0 ORDER BY SUM(\"total_value\") ASC LIMIT 3" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "CRYPTO", + "catalog": "crypto", + "external_knowledge": null + } + }, + { + "id": "sf_bq444", + "input": { + "query": "Can you pull the blockchain timestamp, block number, and transaction hash for the first five mint and burn events from Ethereum logs for the address '0x8ad599c3a0ff1de082011efddc58f1908eb6e6d8'? Please include mint events identified by the topic '0x7a53080ba414158be7ec69b987b5fb7d07dee101fe85488f0853ae16239d0bde' and burn events by '0x0c396cd989a39f4459b5fa1aed6a9a8dcdbc45908acfd67e028cd568da98982c', and order them by block timestamp from the oldest to the newest." + }, + "expected_output": { + "sql": "WITH parsed_burn_logs AS (SELECT logs.\"block_timestamp\" AS block_timestamp, logs.\"block_number\" AS block_number, logs.\"transaction_hash\" AS transaction_hash, logs.\"log_index\" AS log_index, JSON_PARSE(logs.\"data\") AS data, logs.\"topics\" FROM CRYPTO.CRYPTO_ETHEREUM.LOGS AS logs WHERE logs.\"address\" = '0x8ad599c3a0ff1de082011efddc58f1908eb6e6d8' AND logs.\"topics\"[1] = '0x0c396cd989a39f4459b5fa1aed6a9a8dcdbc45908acfd67e028cd568da98982c'), parsed_mint_logs AS (SELECT logs.\"block_timestamp\" AS block_timestamp, logs.\"block_number\" AS block_number, logs.\"transaction_hash\" AS transaction_hash, logs.\"log_index\" AS log_index, JSON_PARSE(logs.\"data\") AS data, logs.\"topics\" FROM CRYPTO.CRYPTO_ETHEREUM.LOGS AS logs WHERE logs.\"address\" = '0x8ad599c3a0ff1de082011efddc58f1908eb6e6d8' AND logs.\"topics\"[1] = '0x7a53080ba414158be7ec69b987b5fb7d07dee101fe85488f0853ae16239d0bde') SELECT block_timestamp, block_number, transaction_hash FROM parsed_mint_logs UNION ALL SELECT block_timestamp, block_number, transaction_hash FROM parsed_burn_logs ORDER BY block_timestamp LIMIT 5" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "CRYPTO", + "catalog": "crypto", + "external_knowledge": "ethereum_logs_and_events_overview.md" + } + }, + { + "id": "sf_bq334", + "input": { + "query": "Calculate the annual differences in Bitcoin output value averages between two methods: Merged input/output records: Combine the inputs and outputs tables, filter to only output records, and calculate yearly averages. Transactions table: Directly use the output_value field from the transactions table for yearly averages. Show the difference (merged outputs average minus transactions average) only for years with data in both methods." + }, + "expected_output": { + "sql": "WITH all_transactions AS (SELECT TO_TIMESTAMP_NTZ(CAST(\"block_timestamp\" AS DOUBLE) / 1000000) AS \"timestamp\" /* 将时间戳转换为日期时间格式 */, \"value\", 'input' AS \"type\" FROM \"CRYPTO\".\"CRYPTO_BITCOIN\".\"INPUTS\" UNION ALL SELECT TO_TIMESTAMP_NTZ(CAST(\"block_timestamp\" AS DOUBLE) / 1000000) AS \"timestamp\" /* 将时间戳转换为日期时间格式 */, \"value\", 'output' AS \"type\" FROM \"CRYPTO\".\"CRYPTO_BITCOIN\".\"OUTPUTS\"), filtered_transactions AS (SELECT EXTRACT(YEAR FROM \"timestamp\") AS \"year\", \"value\" FROM all_transactions WHERE \"type\" = 'output'), average_output_values AS (SELECT \"year\", AVG(\"value\") AS \"avg_value\" FROM filtered_transactions GROUP BY \"year\"), average_transaction_values AS (SELECT EXTRACT(YEAR FROM TO_TIMESTAMP_NTZ(CAST(\"block_timestamp\" AS DOUBLE) / 1000000)) AS \"year\" /* 同样转换时间戳 */, AVG(\"output_value\") AS \"avg_transaction_value\" FROM \"CRYPTO\".\"CRYPTO_BITCOIN\".\"TRANSACTIONS\" GROUP BY \"year\" ORDER BY \"year\"), common_years AS (SELECT ao.\"year\", ao.\"avg_value\" AS \"avg_output_value\", atv.\"avg_transaction_value\" FROM average_output_values AS ao JOIN average_transaction_values AS atv ON ao.\"year\" = atv.\"year\") SELECT \"year\", \"avg_transaction_value\" - \"avg_output_value\" AS \"difference\" FROM common_years ORDER BY \"year\"" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "CRYPTO", + "catalog": "crypto", + "external_knowledge": null + } + }, + { + "id": "sf_bq057", + "input": { + "query": "Which month (e.g., 3 for March) in 2021 witnessed the highest percentage of Bitcoin transaction volume occurring in CoinJoin transactions (defined as transactions with >2 outputs, output value ≤ input value, and having multiple equal-value outputs)? Also provide the percentage of all Bitcoin transactions that were CoinJoins, the percentage of UTXOs involved in CoinJoin transactions (average of input and output percentages), and the percentage of total Bitcoin volume that occurred in CoinJoin transactions for that month. Round all percentages to 1 decimal place." + }, + "expected_output": { + "sql": "WITH totals AS (/* Aggregate monthly totals for Bitcoin txs, input/output UTXOs, */ /* and input/output values (UTXO stands for Unspent Transaction Output) */ SELECT \"txs_tot\".\"block_timestamp_month\" AS tx_month, COUNT(\"txs_tot\".\"hash\") AS tx_count, SUM(\"txs_tot\".\"input_count\") AS tx_inputs, SUM(\"txs_tot\".\"output_count\") AS tx_outputs, CAST(SUM(\"txs_tot\".\"input_value\") AS DOUBLE) / 100000000 AS tx_input_val, CAST(SUM(\"txs_tot\".\"output_value\") AS DOUBLE) / 100000000 AS tx_output_val FROM CRYPTO.CRYPTO_BITCOIN.TRANSACTIONS AS \"txs_tot\" WHERE \"txs_tot\".\"block_timestamp_month\" BETWEEN CAST('2021-01-01' AS DATE) AND CAST('2021-12-31' AS DATE) GROUP BY \"txs_tot\".\"block_timestamp_month\" ORDER BY \"txs_tot\".\"block_timestamp_month\" DESC NULLS FIRST), coinjoinOuts AS (/* Builds a table where each row represents an output of a */ /* potential CoinJoin tx, defined as a tx that had more */ /* than two outputs and had a total output value less than its */ /* input value, per Adam Fiscor's description in this article: */ SELECT \"txs\".\"hash\", \"txs\".\"block_number\", \"txs\".\"block_timestamp_month\", \"txs\".\"input_count\", \"txs\".\"output_count\", \"txs\".\"input_value\", \"txs\".\"output_value\", JSON_EXTRACT(\"o\".value, '$.value') AS \"outputs_val\" FROM CRYPTO.CRYPTO_BITCOIN.TRANSACTIONS AS \"txs\" CROSS JOIN UNNEST(INPUT => \"txs\".\"outputs\") AS \"o\"(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE \"txs\".\"output_count\" > 2 AND \"txs\".\"output_value\" <= \"txs\".\"input_value\" AND \"txs\".\"block_timestamp_month\" BETWEEN CAST('2021-01-01' AS DATE) AND CAST('2021-12-31' AS DATE) ORDER BY \"txs\".\"block_number\", \"txs\".\"hash\" DESC NULLS FIRST), coinjoinTxs AS (/* Builds a table of just the distinct CoinJoin tx hashes */ /* which had more than one equal-value output. */ SELECT \"coinjoinouts\".\"hash\" AS \"cjhash\", \"coinjoinouts\".\"outputs_val\" AS outputVal, COUNT(*) AS cjOuts FROM coinjoinOuts AS \"coinjoinouts\" GROUP BY \"coinjoinouts\".\"hash\", \"coinjoinouts\".\"outputs_val\" HAVING COUNT(*) > 1), coinjoinsD AS (/* Filter out all potential CoinJoin txs that did not have */ /* more than one equal-value output. Do not list the */ /* outputs themselves, only the distinct tx hashes and */ /* their input/output counts and values. */ SELECT DISTINCT \"coinjoinouts\".\"hash\", \"coinjoinouts\".\"block_number\", \"coinjoinouts\".\"block_timestamp_month\", \"coinjoinouts\".\"input_count\", \"coinjoinouts\".\"output_count\", \"coinjoinouts\".\"input_value\", \"coinjoinouts\".\"output_value\" FROM coinjoinOuts AS \"coinjoinouts\" INNER JOIN coinjoinTxs AS \"coinjointxs\" ON \"coinjoinouts\".\"hash\" = \"coinjointxs\".\"cjhash\"), coinjoins AS (/* Aggregate monthly totals for CoinJoin txs, input/output UTXOs, */ /* and input/output values */ SELECT \"cjs\".\"block_timestamp_month\" AS cjs_month, COUNT(\"cjs\".\"hash\") AS cjs_count, SUM(\"cjs\".\"input_count\") AS cjs_inputs, SUM(\"cjs\".\"output_count\") AS cjs_outputs, CAST(SUM(\"cjs\".\"input_value\") AS DOUBLE) / 100000000 AS cjs_input_val, CAST(SUM(\"cjs\".\"output_value\") AS DOUBLE) / 100000000 AS cjs_output_val FROM coinjoinsD AS \"cjs\" GROUP BY \"cjs\".\"block_timestamp_month\" ORDER BY \"cjs\".\"block_timestamp_month\" DESC NULLS FIRST) SELECT EXTRACT(MONTH FROM tx_month) AS month, ROUND(CAST(coinjoins.cjs_count AS DOUBLE) / totals.tx_count * 100, 1) AS tx_percent /* Calculate resulting CoinJoin percentages: */ /* tx_percent = percent of monthly Bitcoin txs that were CoinJoins */, ROUND(CAST((CAST(coinjoins.cjs_inputs AS DOUBLE) / totals.tx_inputs + CAST(coinjoins.cjs_outputs AS DOUBLE) / totals.tx_outputs) AS DOUBLE) / 2 * 100, 1) AS utxos_percent /* utxos_percent = percent of monthly Bitcoin utxos that were CoinJoins */, ROUND(CAST(coinjoins.cjs_input_val AS DOUBLE) / totals.tx_input_val * 100, 1) AS value_percent /* value_percent = percent of monthly Bitcoin volume that took place */ /* in CoinJoined transactions */ FROM totals INNER JOIN coinjoins ON totals.tx_month = coinjoins.cjs_month ORDER BY value_percent DESC NULLS FIRST LIMIT 1" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "CRYPTO", + "catalog": "crypto", + "external_knowledge": null + } + }, + { + "id": "sf_bq068", + "input": { + "query": "Using double-entry bookkeeping principles by treating transaction inputs as debits (negative values) and outputs as credits (positive values) for all Bitcoin Cash transactions between 2014-03-01 and 2014-04-01, how can we calculate the maximum and minimum final balances grouped by address type from these transactions?" + }, + "expected_output": { + "sql": "WITH double_entry_book AS (/* debits */ SELECT ARRAY_JOIN(JSON_EXTRACT(\"inputs\".value, '$.addresses'), ',') AS \"address\" /* Use the correct JSON path notation */, JSON_EXTRACT(\"inputs\".value, '$.type') AS \"type\", -JSON_EXTRACT(\"inputs\".value, '$.value') AS \"value\" FROM CRYPTO.CRYPTO_BITCOIN_CASH.TRANSACTIONS CROSS JOIN UNNEST(INPUT => \"inputs\") AS \"inputs\"(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE TO_TIMESTAMP(CAST(\"block_timestamp\" AS DOUBLE) / 1000000) >= '2014-03-01' AND TO_TIMESTAMP(CAST(\"block_timestamp\" AS DOUBLE) / 1000000) < '2014-04-01' UNION ALL /* credits */ SELECT ARRAY_JOIN(JSON_EXTRACT(\"outputs\".value, '$.addresses'), ',') AS \"address\" /* Use the correct JSON path notation */, JSON_EXTRACT(\"outputs\".value, '$.type') AS \"type\", JSON_EXTRACT(\"outputs\".value, '$.value') AS \"value\" FROM CRYPTO.CRYPTO_BITCOIN_CASH.TRANSACTIONS CROSS JOIN UNNEST(INPUT => \"outputs\") AS \"outputs\"(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE TO_TIMESTAMP(CAST(\"block_timestamp\" AS DOUBLE) / 1000000) >= '2014-03-01' AND TO_TIMESTAMP(CAST(\"block_timestamp\" AS DOUBLE) / 1000000) < '2014-04-01'), address_balances AS (SELECT \"address\", \"type\", SUM(\"value\") AS \"balance\" FROM double_entry_book GROUP BY \"address\", \"type\"), max_min_balances AS (SELECT \"type\", MAX(\"balance\") AS max_balance, MIN(\"balance\") AS min_balance FROM address_balances GROUP BY \"type\") SELECT REPLACE(\"type\", '\"', '') AS \"type\" /* Replace double quotes with nothing */, max_balance, min_balance FROM max_min_balances ORDER BY \"type\"" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "CRYPTO", + "catalog": "crypto", + "external_knowledge": null + } + }, + { + "id": "sf_bq093", + "input": { + "query": "What were the maximum and minimum net balance changes for Ethereum Classic addresses on October 14, 2016? Calculate these by summing all transactions where addresses received funds (debits), sent funds (credits), and paid or received gas fees. Only include successful status transactions and exclude internal calls of types. For gas fees, consider both the fees paid by transaction senders and received by miners, calculated as multiplied by the gas price for both miners and senders" + }, + "expected_output": { + "sql": "WITH double_entry_book AS (/* Debits */ SELECT \"to_address\" AS \"address\", \"value\" AS \"value\" FROM CRYPTO.CRYPTO_ETHEREUM_CLASSIC.TRACES WHERE NOT \"to_address\" IS NULL AND \"status\" = 1 AND (NOT \"call_type\" IN ('delegatecall', 'callcode', 'staticcall') OR \"call_type\" IS NULL) AND CAST(CAST(TO_TIMESTAMP(CAST(\"block_timestamp\" AS DOUBLE) / 1000000) AS TIMESTAMP) AS DATE) = '2016-10-14' UNION ALL /* Credits */ SELECT \"from_address\" AS \"address\", -\"value\" AS \"value\" FROM CRYPTO.CRYPTO_ETHEREUM_CLASSIC.TRACES WHERE NOT \"from_address\" IS NULL AND \"status\" = 1 AND (NOT \"call_type\" IN ('delegatecall', 'callcode', 'staticcall') OR \"call_type\" IS NULL) AND CAST(CAST(TO_TIMESTAMP(CAST(\"block_timestamp\" AS DOUBLE) / 1000000) AS TIMESTAMP) AS DATE) = '2016-10-14' UNION ALL /* Transaction Fees Debits */ SELECT \"miner\" AS \"address\", SUM(CAST(\"receipt_gas_used\" AS DECIMAL(38, 0)) * CAST(\"gas_price\" AS DECIMAL(38, 0))) AS \"value\" FROM CRYPTO.CRYPTO_ETHEREUM_CLASSIC.TRANSACTIONS AS \"transactions\" JOIN CRYPTO.CRYPTO_ETHEREUM_CLASSIC.BLOCKS AS \"blocks\" ON \"blocks\".\"number\" = \"transactions\".\"block_number\" WHERE CAST(CAST(TO_TIMESTAMP(CAST(\"block_timestamp\" AS DOUBLE) / 1000000) AS TIMESTAMP) AS DATE) = '2016-10-14' GROUP BY \"blocks\".\"miner\" UNION ALL /* Transaction Fees Credits */ SELECT \"from_address\" AS \"address\", -(CAST(\"receipt_gas_used\" AS DECIMAL(38, 0)) * CAST(\"gas_price\" AS DECIMAL(38, 0))) AS \"value\" FROM CRYPTO.CRYPTO_ETHEREUM_CLASSIC.TRANSACTIONS WHERE CAST(CAST(TO_TIMESTAMP(CAST(\"block_timestamp\" AS DOUBLE) / 1000000) AS TIMESTAMP) AS DATE) = '2016-10-14'), net_changes AS (SELECT \"address\", SUM(\"value\") AS \"net_change\" FROM double_entry_book GROUP BY \"address\") SELECT MAX(\"net_change\") AS \"max_net_change\", MIN(\"net_change\") AS \"min_net_change\" FROM net_changes" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "CRYPTO", + "catalog": "crypto", + "external_knowledge": null + } + }, + { + "id": "sf_bq037", + "input": { + "query": "About the refined human genetic variations collected in phase 3 on 2015-02-20, I want to know the minimum and maximum start positions as well as the proportions of these two respectively for reference bases 'AT' and 'TA'." + }, + "expected_output": { + "sql": "WITH A AS (SELECT \"reference_bases\", \"start_position\" FROM \"HUMAN_GENOME_VARIANTS\".\"HUMAN_GENOME_VARIANTS\".\"_1000_GENOMES_PHASE_3_OPTIMIZED_SCHEMA_VARIANTS_20150220\" WHERE \"reference_bases\" IN ('AT', 'TA')), B AS (SELECT \"reference_bases\", MIN(\"start_position\") AS \"min_start_position\", MAX(\"start_position\") AS \"max_start_position\", COUNT(1) AS \"total_count\" FROM A GROUP BY \"reference_bases\"), min_counts AS (SELECT A.\"reference_bases\" /* Explicitly referencing the column from table A */, A.\"start_position\" AS \"min_start_position\", COUNT(1) AS \"min_count\" FROM A INNER JOIN B ON A.\"reference_bases\" = B.\"reference_bases\" WHERE A.\"start_position\" = B.\"min_start_position\" GROUP BY A.\"reference_bases\", A.\"start_position\"), max_counts AS (SELECT A.\"reference_bases\" /* Explicitly referencing the column from table A */, A.\"start_position\" AS \"max_start_position\", COUNT(1) AS \"max_count\" FROM A INNER JOIN B ON A.\"reference_bases\" = B.\"reference_bases\" WHERE A.\"start_position\" = B.\"max_start_position\" GROUP BY A.\"reference_bases\", A.\"start_position\") SELECT B.\"reference_bases\" /* Explicitly referencing the column from table B */, B.\"min_start_position\", CAST(min_counts.\"min_count\" AS DOUBLE) / B.\"total_count\" AS \"min_position_ratio\", B.\"max_start_position\", CAST(max_counts.\"max_count\" AS DOUBLE) / B.\"total_count\" AS \"max_position_ratio\" FROM B LEFT JOIN min_counts ON B.\"reference_bases\" = min_counts.\"reference_bases\" AND B.\"min_start_position\" = min_counts.\"min_start_position\" LEFT JOIN max_counts ON B.\"reference_bases\" = max_counts.\"reference_bases\" AND B.\"max_start_position\" = max_counts.\"max_start_position\" ORDER BY B.\"reference_bases\"" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "HUMAN_GENOME_VARIANTS", + "catalog": "human_genome_variants", + "external_knowledge": null + } + }, + { + "id": "sf_bq012", + "input": { + "query": "Calculate the average balance (in quadrillions, 10^15) of the top 10 Ethereum addresses by net balance, including incoming and outgoing transfers from traces (only successful transactions and excluding call types like delegatecall, callcode, and staticcall), miner rewards (sum of gas fees per block), and sender gas fee deductions. Exclude null addresses and round the result to two decimal places." + }, + "expected_output": { + "sql": "WITH double_entry_book AS (/* Debits */ SELECT \"to_address\" AS \"address\", \"value\" AS \"value\" FROM \"ETHEREUM_BLOCKCHAIN\".\"ETHEREUM_BLOCKCHAIN\".\"TRACES\" WHERE NOT \"to_address\" IS NULL AND \"status\" = 1 AND (NOT \"call_type\" IN ('delegatecall', 'callcode', 'staticcall') OR \"call_type\" IS NULL) UNION ALL /* Credits */ SELECT \"from_address\" AS \"address\", -\"value\" AS \"value\" FROM \"ETHEREUM_BLOCKCHAIN\".\"ETHEREUM_BLOCKCHAIN\".\"TRACES\" WHERE NOT \"from_address\" IS NULL AND \"status\" = 1 AND (NOT \"call_type\" IN ('delegatecall', 'callcode', 'staticcall') OR \"call_type\" IS NULL) UNION ALL /* Transaction fees debits */ SELECT \"miner\" AS \"address\", SUM(CAST(\"receipt_gas_used\" AS DECIMAL(38, 0)) * CAST(\"gas_price\" AS DECIMAL(38, 0))) AS \"value\" FROM \"ETHEREUM_BLOCKCHAIN\".\"ETHEREUM_BLOCKCHAIN\".\"TRANSACTIONS\" AS \"transactions\" JOIN \"ETHEREUM_BLOCKCHAIN\".\"ETHEREUM_BLOCKCHAIN\".\"BLOCKS\" AS \"blocks\" ON \"blocks\".\"number\" = \"transactions\".\"block_number\" GROUP BY \"blocks\".\"miner\" UNION ALL /* Transaction fees credits */ SELECT \"from_address\" AS \"address\", -(CAST(\"receipt_gas_used\" AS DECIMAL(38, 0)) * CAST(\"gas_price\" AS DECIMAL(38, 0))) AS \"value\" FROM \"ETHEREUM_BLOCKCHAIN\".\"ETHEREUM_BLOCKCHAIN\".\"TRANSACTIONS\"), top_10_balances AS (SELECT \"address\", SUM(\"value\") AS \"balance\" FROM double_entry_book GROUP BY \"address\" ORDER BY \"balance\" DESC NULLS FIRST LIMIT 10) SELECT ROUND(CAST(AVG(\"balance\") AS DOUBLE) / 1e15, 2) AS \"average_balance_trillion\" FROM top_10_balances" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "ETHEREUM_BLOCKCHAIN", + "catalog": "ethereum_blockchain", + "external_knowledge": null + } + }, + { + "id": "sf_bq187", + "input": { + "query": "Calculate the total circulating supply of 'BNB' tokens (in units divided by 10^18) by summing balances of all non-zero addresses, where each address’s balance equals its total received BNB minus sent BNB. Exclude transactions involving the zero address (0x000...) for both senders and receivers." + }, + "expected_output": { + "sql": "WITH tokenInfo AS (SELECT \"address\" FROM \"ETHEREUM_BLOCKCHAIN\".\"ETHEREUM_BLOCKCHAIN\".\"TOKENS\" WHERE \"name\" = 'BNB'), receivedTx AS (SELECT \"tx\".\"to_address\" AS \"addr\", \"tokens\".\"name\" AS \"name\", SUM(CAST(\"tx\".\"value\" AS DOUBLE) / POWER(10, 18)) AS \"amount_received\" FROM \"ETHEREUM_BLOCKCHAIN\".\"ETHEREUM_BLOCKCHAIN\".\"TOKEN_TRANSFERS\" AS \"tx\" JOIN tokenInfo ON \"tx\".\"token_address\" = tokenInfo.\"address\" JOIN \"ETHEREUM_BLOCKCHAIN\".\"ETHEREUM_BLOCKCHAIN\".\"TOKENS\" AS \"tokens\" ON \"tx\".\"token_address\" = \"tokens\".\"address\" WHERE \"tx\".\"to_address\" <> '0x0000000000000000000000000000000000000000' GROUP BY \"tx\".\"to_address\", \"tokens\".\"name\"), sentTx AS (SELECT \"tx\".\"from_address\" AS \"addr\", \"tokens\".\"name\" AS \"name\", SUM(CAST(\"tx\".\"value\" AS DOUBLE) / POWER(10, 18)) AS \"amount_sent\" FROM \"ETHEREUM_BLOCKCHAIN\".\"ETHEREUM_BLOCKCHAIN\".\"TOKEN_TRANSFERS\" AS \"tx\" JOIN tokenInfo ON \"tx\".\"token_address\" = tokenInfo.\"address\" JOIN \"ETHEREUM_BLOCKCHAIN\".\"ETHEREUM_BLOCKCHAIN\".\"TOKENS\" AS \"tokens\" ON \"tx\".\"token_address\" = \"tokens\".\"address\" WHERE \"tx\".\"from_address\" <> '0x0000000000000000000000000000000000000000' GROUP BY \"tx\".\"from_address\", \"tokens\".\"name\"), walletBalances AS (SELECT r.\"addr\", COALESCE(SUM(r.\"amount_received\"), 0) - COALESCE(SUM(s.\"amount_sent\"), 0) AS \"balance\" FROM receivedTx AS r LEFT JOIN sentTx AS s ON r.\"addr\" = s.\"addr\" GROUP BY r.\"addr\") SELECT SUM(\"balance\") AS \"circulating_supply\" FROM walletBalances" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "ETHEREUM_BLOCKCHAIN", + "catalog": "ethereum_blockchain", + "external_knowledge": null + } + }, + { + "id": "sf_bq294", + "input": { + "query": "Could you provide the details of the top 5 longest bike share trips that started between July 1, 2017, and December 31, 2017, including the trip ID, duration in seconds, start date, start station name, route (derived from start station name to end station name), bike number, subscriber type, member's birth year, the member's current age (calculated using the current year), an age classification based on whether the member is younger than 40, between 40 and 60, or older than 60, the member's gender, and the name of the region of the start station? Please exclude any trips where the start station name, member's birth year, or member's gender is not specified." + }, + "expected_output": { + "sql": "SELECT \"trip_id\", \"duration_sec\", CAST(CAST(TO_TIMESTAMP_LTZ(CAST(\"start_date\" AS DOUBLE) / 1000000) AS TIMESTAMP) AS DATE) AS \"star_date\", \"start_station_name\", CONCAT(CAST(\"start_station_name\" AS VARCHAR), CAST(' - ' AS VARCHAR), CAST(\"end_station_name\" AS VARCHAR)) AS \"route\", \"bike_number\", \"subscriber_type\", \"member_birth_year\", (EXTRACT(YEAR FROM CURRENT_DATE) - \"member_birth_year\") AS \"age\", CASE WHEN (EXTRACT(YEAR FROM CURRENT_DATE) - \"member_birth_year\") < 40 THEN 'Young (<40 Y.O)' WHEN (EXTRACT(YEAR FROM CURRENT_DATE) - \"member_birth_year\") BETWEEN 40 AND 60 THEN 'Adult (40-60 Y.O)' ELSE 'Senior Adult (>60 Y.O)' END AS \"age_class\", \"member_gender\", c.\"name\" AS \"region_name\" FROM \"SAN_FRANCISCO_PLUS\".\"SAN_FRANCISCO_BIKESHARE\".\"BIKESHARE_TRIPS\" AS a LEFT JOIN \"SAN_FRANCISCO_PLUS\".\"SAN_FRANCISCO_BIKESHARE\".\"BIKESHARE_STATION_INFO\" AS b ON a.\"start_station_id\" = b.\"station_id\" LEFT JOIN \"SAN_FRANCISCO_PLUS\".\"SAN_FRANCISCO_BIKESHARE\".\"BIKESHARE_REGIONS\" AS c ON b.\"region_id\" = c.\"region_id\" WHERE TO_TIMESTAMP_LTZ(CAST(\"start_date\" AS DOUBLE) / 1000000) BETWEEN '2017-07-01' AND '2017-12-31' AND NOT b.\"station_id\" IS NULL AND NOT \"member_birth_year\" IS NULL AND NOT \"member_gender\" IS NULL ORDER BY \"duration_sec\" DESC NULLS FIRST LIMIT 5" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "SAN_FRANCISCO_PLUS", + "catalog": "san_francisco_plus", + "external_knowledge": "trip_info.md" + } + }, + { + "id": "sf_bq260", + "input": { + "query": "From January 1, 2019, to April 30, 2022, how many users are at the youngest age and how many users are at the oldest age for each gender in the e-commerce platform, counting both youngest and oldest users separately for each gender?" + }, + "expected_output": { + "sql": "WITH filtered_users AS (SELECT \"first_name\", \"last_name\", \"gender\", \"age\", CAST(TO_TIMESTAMP(CAST(\"created_at\" AS DOUBLE) / 1000000.0) AS DATE) AS \"created_at\" FROM \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"USERS\" WHERE CAST(TO_TIMESTAMP(CAST(\"created_at\" AS DOUBLE) / 1000000.0) AS DATE) BETWEEN '2019-01-01' AND '2022-04-30'), youngest_ages AS (SELECT \"gender\", MIN(\"age\") AS \"age\" FROM filtered_users GROUP BY \"gender\"), oldest_ages AS (SELECT \"gender\", MAX(\"age\") AS \"age\" FROM filtered_users GROUP BY \"gender\"), youngest_oldest AS (SELECT u.\"first_name\", u.\"last_name\", u.\"gender\", u.\"age\", 'youngest' AS \"tag\" FROM filtered_users AS u JOIN youngest_ages AS y ON u.\"gender\" = y.\"gender\" AND u.\"age\" = y.\"age\" UNION ALL SELECT u.\"first_name\", u.\"last_name\", u.\"gender\", u.\"age\", 'oldest' AS \"tag\" FROM filtered_users AS u JOIN oldest_ages AS o ON u.\"gender\" = o.\"gender\" AND u.\"age\" = o.\"age\") SELECT \"tag\", \"gender\", COUNT(*) AS \"num\" FROM youngest_oldest GROUP BY \"tag\", \"gender\" ORDER BY \"tag\", \"gender\"" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "THELOOK_ECOMMERCE", + "catalog": "thelook_ecommerce", + "external_knowledge": null + } + }, + { + "id": "sf_bq263", + "input": { + "query": "Please create a month-by-month report for the year 2023 that focuses on the 'Sleep & Lounge' category, showing for each month the total sales, total cost, number of complete orders, total profit, and the profit-to-cost ratio, ensuring that the order is marked as 'Complete,' the creation date is between January 1, 2023, and December 31, 2023, and the cost data is accurately associated with the corresponding product through the order items. " + }, + "expected_output": { + "sql": "WITH d AS (SELECT a.\"order_id\", DATE_FORMAT(TO_TIMESTAMP(a.\"created_at\" / 1000000.0), 'YYYY-MM') AS \"month\" /* 格式化为年月 */, DATE_FORMAT(TO_TIMESTAMP(a.\"created_at\" / 1000000.0), 'YYYY') AS \"year\" /* 格式化为年份 */, b.\"product_id\", b.\"sale_price\", c.\"category\", c.\"cost\" FROM \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"ORDERS\" AS a JOIN \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"ORDER_ITEMS\" AS b ON a.\"order_id\" = b.\"order_id\" JOIN \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"PRODUCTS\" AS c ON b.\"product_id\" = c.\"id\" WHERE a.\"status\" = 'Complete' AND TO_TIMESTAMP(CAST(a.\"created_at\" AS DOUBLE) / 1000000.0) BETWEEN CAST('2023-01-01' AS TIMESTAMP) AND CAST('2023-12-31' AS TIMESTAMP) AND c.\"category\" = 'Sleep & Lounge'), e AS (SELECT \"month\", \"year\", \"sale_price\", \"category\", \"cost\", SUM(\"sale_price\") OVER (PARTITION BY \"month\", \"category\") AS \"TPV\", SUM(\"cost\") OVER (PARTITION BY \"month\", \"category\") AS \"total_cost\", COUNT(DISTINCT \"order_id\") OVER (PARTITION BY \"month\", \"category\") AS \"TPO\", SUM(\"sale_price\" - \"cost\") OVER (PARTITION BY \"month\", \"category\") AS \"total_profit\", SUM(CAST((\"sale_price\" - \"cost\") AS DOUBLE) / \"cost\") OVER (PARTITION BY \"month\", \"category\") AS \"Profit_to_cost_ratio\" FROM d) SELECT DISTINCT \"month\", \"category\", \"TPV\", \"total_cost\", \"TPO\", \"total_profit\", \"Profit_to_cost_ratio\" FROM e ORDER BY \"month\"" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "THELOOK_ECOMMERCE", + "catalog": "thelook_ecommerce", + "external_knowledge": null + } + }, + { + "id": "sf_bq264", + "input": { + "query": "Identify the difference in the number of the oldest and youngest users registered between January 1, 2019, and April 30, 2022, from our e-commerce platform data." + }, + "expected_output": { + "sql": "WITH youngest AS (SELECT \"gender\", \"id\", \"first_name\", \"last_name\", \"age\", 'youngest' AS \"tag\" FROM \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"USERS\" WHERE \"age\" = (SELECT MIN(\"age\") FROM \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"USERS\") AND TO_TIMESTAMP(CAST(\"created_at\" AS DOUBLE) / 1000000.0) BETWEEN CAST('2019-01-01' AS TIMESTAMP) AND CAST('2022-04-30' AS TIMESTAMP) GROUP BY \"gender\", \"id\", \"first_name\", \"last_name\", \"age\" ORDER BY \"gender\"), oldest AS (SELECT \"gender\", \"id\", \"first_name\", \"last_name\", \"age\", 'oldest' AS \"tag\" FROM \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"USERS\" WHERE \"age\" = (SELECT MAX(\"age\") FROM \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"USERS\") AND TO_TIMESTAMP(CAST(\"created_at\" AS DOUBLE) / 1000000.0) BETWEEN CAST('2019-01-01' AS TIMESTAMP) AND CAST('2022-04-30' AS TIMESTAMP) GROUP BY \"gender\", \"id\", \"first_name\", \"last_name\", \"age\" ORDER BY \"gender\"), TEMP_record AS (SELECT * FROM youngest UNION ALL SELECT * FROM oldest) SELECT SUM(CASE WHEN \"age\" = (SELECT MAX(\"age\") FROM \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"USERS\") THEN 1 END) - SUM(CASE WHEN \"age\" = (SELECT MIN(\"age\") FROM \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"USERS\") THEN 1 END) AS \"diff\" FROM TEMP_record" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "THELOOK_ECOMMERCE", + "catalog": "thelook_ecommerce", + "external_knowledge": null + } + }, + { + "id": "sf_bq265", + "input": { + "query": "Can you list the email addresses of the top 10 users who registered in 2019 and made purchases in 2019, ranking them by their highest average order value, where average order value is calculated by multiplying the number of items in each order by the sale price, summing this total across all orders for each user, and then dividing by the total number of orders?" + }, + "expected_output": { + "sql": "WITH main AS (SELECT \"id\" AS \"user_id\", \"email\", \"gender\", \"country\", \"traffic_source\" FROM \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"USERS\" WHERE TO_TIMESTAMP(CAST(\"created_at\" AS DOUBLE) / 1000000.0) BETWEEN CAST('2019-01-01' AS TIMESTAMP) AND CAST('2019-12-31' AS TIMESTAMP)), daate AS (SELECT \"user_id\", \"order_id\", CAST(TO_TIMESTAMP(CAST(\"created_at\" AS DOUBLE) / 1000000.0) AS DATE) AS \"order_date\", \"num_of_item\" FROM \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"ORDERS\" WHERE TO_TIMESTAMP(CAST(\"created_at\" AS DOUBLE) / 1000000.0) BETWEEN CAST('2019-01-01' AS TIMESTAMP) AND CAST('2019-12-31' AS TIMESTAMP)), orders AS (SELECT \"user_id\", \"order_id\", \"product_id\", \"sale_price\", \"status\" FROM \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"ORDER_ITEMS\" WHERE TO_TIMESTAMP(CAST(\"created_at\" AS DOUBLE) / 1000000.0) BETWEEN CAST('2019-01-01' AS TIMESTAMP) AND CAST('2019-12-31' AS TIMESTAMP)), nest AS (SELECT o.\"user_id\", o.\"order_id\", o.\"product_id\", d.\"order_date\", d.\"num_of_item\", ROUND(o.\"sale_price\", 2) AS \"sale_price\", ROUND(d.\"num_of_item\" * o.\"sale_price\", 2) AS \"total_sale\" FROM orders AS o INNER JOIN daate AS d ON o.\"order_id\" = d.\"order_id\" ORDER BY o.\"user_id\"), type AS (SELECT \"user_id\", MIN(nest.\"order_date\") AS \"cohort_date\", MAX(nest.\"order_date\") AS \"latest_shopping_date\", DATE_DIFF('MONTH', MIN(nest.\"order_date\"), MAX(nest.\"order_date\")) AS \"lifespan_months\", ROUND(SUM(\"total_sale\"), 2) AS \"ltv\", COUNT(\"order_id\") AS \"no_of_order\" FROM nest GROUP BY \"user_id\"), kite AS (SELECT m.\"user_id\", m.\"email\", m.\"gender\", m.\"country\", m.\"traffic_source\", EXTRACT(YEAR FROM n.\"cohort_date\") AS \"cohort_year\", n.\"latest_shopping_date\", n.\"lifespan_months\", n.\"ltv\", n.\"no_of_order\", ROUND(CAST(n.\"ltv\" AS DOUBLE) / n.\"no_of_order\", 2) AS \"avg_order_value\" FROM main AS m INNER JOIN type AS n ON m.\"user_id\" = n.\"user_id\") SELECT \"email\" FROM kite ORDER BY \"avg_order_value\" DESC NULLS FIRST LIMIT 10" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "THELOOK_ECOMMERCE", + "catalog": "thelook_ecommerce", + "external_knowledge": null + } + }, + { + "id": "sf_bq271", + "input": { + "query": "Please generate a report that, for each month in 2021, provides the number of orders, the number of unique purchasers, and the profit (calculated as the sum of product retail prices minus the sum of product costs), where the orders were placed during 2021 by users who registered in 2021 for inventory items created in 2021, and group the results by the users' country, product department, and product category." + }, + "expected_output": { + "sql": "WITH orders_x_order_items AS (SELECT orders.*, order_items.\"inventory_item_id\", order_items.\"sale_price\" FROM \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"ORDERS\" AS orders LEFT JOIN \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"ORDER_ITEMS\" AS order_items ON orders.\"order_id\" = order_items.\"order_id\" WHERE TO_TIMESTAMP_NTZ(CAST(orders.\"created_at\" AS DOUBLE) / 1000000) BETWEEN CAST('2021-01-01' AS TIMESTAMP) AND CAST('2021-12-31' AS TIMESTAMP)), orders_x_inventory AS (SELECT orders_x_order_items.*, inventory_items.\"product_category\", inventory_items.\"product_department\", inventory_items.\"product_retail_price\", inventory_items.\"product_distribution_center_id\", inventory_items.\"cost\", distribution_centers.\"name\" FROM orders_x_order_items LEFT JOIN \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"INVENTORY_ITEMS\" AS inventory_items ON orders_x_order_items.\"inventory_item_id\" = inventory_items.\"id\" LEFT JOIN \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"DISTRIBUTION_CENTERS\" AS distribution_centers ON inventory_items.\"product_distribution_center_id\" = distribution_centers.\"id\" WHERE TO_TIMESTAMP_NTZ(CAST(inventory_items.\"created_at\" AS DOUBLE) / 1000000) BETWEEN CAST('2021-01-01' AS TIMESTAMP) AND CAST('2021-12-31' AS TIMESTAMP)), orders_x_users AS (SELECT orders_x_inventory.*, users.\"country\" AS \"users_country\" FROM orders_x_inventory LEFT JOIN \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"USERS\" AS users ON orders_x_inventory.\"user_id\" = users.\"id\" WHERE TO_TIMESTAMP_NTZ(CAST(users.\"created_at\" AS DOUBLE) / 1000000) BETWEEN CAST('2021-01-01' AS TIMESTAMP) AND CAST('2021-12-31' AS TIMESTAMP)) SELECT DATE_TRUNC('MONTH', CAST(CAST(TO_TIMESTAMP_NTZ(CAST(orders_x_users.\"created_at\" AS DOUBLE) / 1000000) AS TIMESTAMP) AS DATE)) AS \"reporting_month\", orders_x_users.\"users_country\", orders_x_users.\"product_department\", orders_x_users.\"product_category\", COUNT(DISTINCT orders_x_users.\"order_id\") AS \"n_order\", COUNT(DISTINCT orders_x_users.\"user_id\") AS \"n_purchasers\", SUM(orders_x_users.\"product_retail_price\") - SUM(orders_x_users.\"cost\") AS \"profit\" FROM orders_x_users GROUP BY 1, 2, 3, 4 ORDER BY \"reporting_month\"" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "THELOOK_ECOMMERCE", + "catalog": "thelook_ecommerce", + "external_knowledge": null + } + }, + { + "id": "sf_bq273", + "input": { + "query": "Can you list the top 5 months from August 2022 to November 2023 where the profit from Facebook-sourced completed orders showed the largest month-over-month increase? Calculate profit as sales minus costs, group by delivery month, and include only orders created between August 2022 and November 2023. Compare each month's profit to its previous month to find the largest increases." + }, + "expected_output": { + "sql": "WITH orders AS (SELECT \"order_id\", \"user_id\", \"created_at\", DATE_TRUNC('MONTH', TO_TIMESTAMP_NTZ(CAST(\"delivered_at\" AS DOUBLE) / 1000000)) AS \"delivery_month\" /* Converting to timestamp */, \"status\" FROM \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"ORDERS\"), order_items AS (SELECT \"order_id\", \"product_id\", \"sale_price\" FROM \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"ORDER_ITEMS\"), products AS (SELECT \"id\", \"cost\" FROM \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"PRODUCTS\"), users AS (SELECT \"id\", \"traffic_source\" FROM \"THELOOK_ECOMMERCE\".\"THELOOK_ECOMMERCE\".\"USERS\"), filter_join AS (SELECT orders.\"order_id\", orders.\"user_id\", order_items.\"product_id\", orders.\"delivery_month\", orders.\"status\", order_items.\"sale_price\", products.\"cost\", users.\"traffic_source\" FROM orders JOIN order_items ON orders.\"order_id\" = order_items.\"order_id\" JOIN products ON order_items.\"product_id\" = products.\"id\" JOIN users ON orders.\"user_id\" = users.\"id\" WHERE orders.\"status\" = 'Complete' AND users.\"traffic_source\" = 'Facebook' AND TO_TIMESTAMP_NTZ(CAST(orders.\"created_at\" AS DOUBLE) / 1000000) BETWEEN CAST('2022-07-01' AS TIMESTAMP) AND CAST('2023-11-30' AS TIMESTAMP) /* Include July for calculation */), monthly_sales AS (SELECT \"delivery_month\", \"traffic_source\", SUM(\"sale_price\") AS \"total_revenue\", SUM(\"sale_price\") - SUM(\"cost\") AS \"total_profit\", COUNT(DISTINCT \"product_id\") AS \"product_quantity\", COUNT(DISTINCT \"order_id\") AS \"orders_quantity\", COUNT(DISTINCT \"user_id\") AS \"users_quantity\" FROM filter_join GROUP BY \"delivery_month\", \"traffic_source\") /* Filter to show only 8th month and onwards, but calculate using July */ SELECT current_month.\"delivery_month\", COALESCE(current_month.\"total_profit\" - previous_month.\"total_profit\", 0 /* If there is no previous month (i.e. for 8月), return 0 */) AS \"profit_vs_prior_month\" FROM monthly_sales AS current_month LEFT JOIN monthly_sales AS previous_month ON current_month.\"traffic_source\" = previous_month.\"traffic_source\" AND current_month.\"delivery_month\" = DATE_ADD('MONTH', -1, previous_month.\"delivery_month\") /* Correctly join to previous month */ WHERE current_month.\"delivery_month\" >= '2022-08-01' /* Only show August and later data, but use July for calculation */ ORDER BY \"profit_vs_prior_month\" DESC NULLS FIRST LIMIT 5" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "THELOOK_ECOMMERCE", + "catalog": "thelook_ecommerce", + "external_knowledge": null + } + }, + { + "id": "sf_bq028", + "input": { + "query": "Considering only the latest release versions of NPM package, which packages are the top 8 most popular based on the Github star number, as well as their versions?" + }, + "expected_output": { + "sql": "WITH HighestReleases AS (SELECT HR.\"Name\", HR.\"Version\" FROM (SELECT \"Name\", \"Version\", ROW_NUMBER() OVER (PARTITION BY \"Name\" ORDER BY CAST(JSON_EXTRACT(JSON_PARSE(\"VersionInfo\"), '$.Ordinal') AS DOUBLE) DESC NULLS FIRST) AS RowNumber FROM DEPS_DEV_V1.DEPS_DEV_V1.PACKAGEVERSIONS WHERE \"System\" = 'NPM' AND TO_BOOLEAN(JSON_EXTRACT(JSON_PARSE(\"VersionInfo\"), '$.IsRelease')) = TRUE) AS HR WHERE HR.RowNumber = 1), PVP AS (SELECT PVP.\"Name\", PVP.\"Version\", PVP.\"ProjectType\", PVP.\"ProjectName\" FROM DEPS_DEV_V1.DEPS_DEV_V1.PACKAGEVERSIONTOPROJECT AS PVP JOIN HighestReleases AS HR ON PVP.\"Name\" = HR.\"Name\" AND PVP.\"Version\" = HR.\"Version\" WHERE PVP.\"System\" = 'NPM' AND PVP.\"ProjectType\" = 'GITHUB') SELECT PVP.\"Name\", PVP.\"Version\" FROM PVP JOIN DEPS_DEV_V1.DEPS_DEV_V1.PROJECTS AS P ON PVP.\"ProjectType\" = P.\"Type\" AND PVP.\"ProjectName\" = P.\"Name\" ORDER BY P.\"StarsCount\" DESC NULLS FIRST LIMIT 8" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "DEPS_DEV_V1", + "catalog": "deps_dev_v1", + "external_knowledge": null + } + }, + { + "id": "sf_bq104", + "input": { + "query": "Based on the most recent refresh date, identify the top-ranked rising search term for the week that is exactly one year prior to the latest available week in the dataset." + }, + "expected_output": { + "sql": "WITH LatestWeek AS (SELECT DATE_ADD('WEEK', -52, MAX(\"week\")) AS \"last_year_week\" FROM GOOGLE_TRENDS.GOOGLE_TRENDS.TOP_RISING_TERMS), LatestRefreshDate AS (SELECT MAX(\"refresh_date\") AS \"latest_refresh_date\" FROM GOOGLE_TRENDS.GOOGLE_TRENDS.TOP_RISING_TERMS), RankedTerms AS (SELECT \"term\", \"week\", CASE WHEN \"score\" IS NULL THEN NULL ELSE \"dma_name\" END AS \"dma_name\", \"rank\", \"score\", ROW_NUMBER() OVER (PARTITION BY \"term\", \"week\" ORDER BY \"score\" DESC NULLS FIRST) AS rn FROM GOOGLE_TRENDS.GOOGLE_TRENDS.TOP_RISING_TERMS WHERE \"week\" = (SELECT \"last_year_week\" FROM LatestWeek) AND \"refresh_date\" = (SELECT \"latest_refresh_date\" FROM LatestRefreshDate)) SELECT \"term\" FROM RankedTerms WHERE rn = 1 ORDER BY \"rank\" LIMIT 1" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "GOOGLE_TRENDS", + "catalog": "google_trends", + "external_knowledge": null + } + }, + { + "id": "sf_bq121", + "input": { + "query": "How do the average reputation and number of badges vary among Stack Overflow users based on the number of complete years they have been members, considering only those who joined on or before October 1, 2021?" + }, + "expected_output": { + "sql": "WITH sub AS (SELECT \"users\".\"id\", CAST(TO_TIMESTAMP(CAST(MAX(\"users\".\"creation_date\") AS DOUBLE) / 1000000.0) AS DATE) AS \"user_creation_date\" /* 使用 MAX 聚合 creation_date 并转换为 DATE */, MAX(\"users\".\"reputation\") AS \"reputation\", SUM(CASE WHEN badges.\"user_id\" IS NULL THEN 0 ELSE 1 END) AS \"num_badges\" FROM \"STACKOVERFLOW\".\"STACKOVERFLOW\".\"USERS\" AS \"users\" LEFT JOIN \"STACKOVERFLOW\".\"STACKOVERFLOW\".\"BADGES\" AS badges ON \"users\".\"id\" = badges.\"user_id\" WHERE CAST(TO_TIMESTAMP(CAST(\"users\".\"creation_date\" AS DOUBLE) / 1000000.0) AS DATE) <= CAST('2021-10-01' AS DATE) GROUP BY \"users\".\"id\") SELECT DATE_DIFF('YEAR', \"user_creation_date\", CAST('2021-10-01' AS DATE)) AS \"user_tenure\", COUNT(1) AS \"Num_Users\", AVG(\"reputation\") AS \"Avg_Reputation\", AVG(\"num_badges\") AS \"Avg_Num_Badges\" FROM sub GROUP BY \"user_tenure\" ORDER BY \"user_tenure\"" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "STACKOVERFLOW", + "catalog": "stackoverflow", + "external_knowledge": null + } + }, + { + "id": "sf_bq345", + "input": { + "query": "How large are the DICOM image files with SEG or RTSTRUCT modalities and the SOP Class UID \"1.2.840.10008.5.1.4.1.1.66.4\", when grouped by collection, study, and series IDs, if they have no references to other series, images, or sources? Can you also provide a viewer URL formatted as \"https://viewer.imaging.datacommons.cancer.gov/viewer/\" followed by the study ID, and list these sizes in kilobytes, sorted from largest to smallest?" + }, + "expected_output": { + "sql": "WITH seg_rtstruct AS (SELECT \"collection_id\", \"StudyInstanceUID\", \"SeriesInstanceUID\", CONCAT(CAST('https://viewer.imaging.datacommons.cancer.gov/viewer/' AS VARCHAR), CAST(\"StudyInstanceUID\" AS VARCHAR)) AS \"viewer_url\", \"instance_size\" FROM \"IDC\".\"IDC_V17\".\"DICOM_ALL\" WHERE \"Modality\" IN ('SEG', 'RTSTRUCT') AND \"SOPClassUID\" = '1.2.840.10008.5.1.4.1.1.66.4' AND CARDINALITY(\"ReferencedSeriesSequence\") = 0 AND CARDINALITY(\"ReferencedImageSequence\") = 0 AND CARDINALITY(\"SourceImageSequence\") = 0) SELECT seg_rtstruct.\"collection_id\", seg_rtstruct.\"SeriesInstanceUID\", seg_rtstruct.\"StudyInstanceUID\", seg_rtstruct.\"viewer_url\", CAST(SUM(seg_rtstruct.\"instance_size\") AS DOUBLE) / 1024 AS \"collection_size_KB\" FROM seg_rtstruct GROUP BY seg_rtstruct.\"collection_id\", seg_rtstruct.\"SeriesInstanceUID\", seg_rtstruct.\"StudyInstanceUID\", seg_rtstruct.\"viewer_url\" ORDER BY \"collection_size_KB\" DESC NULLS FIRST" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "IDC", + "catalog": "idc", + "external_knowledge": null + } + }, + { + "id": "sf_bq346", + "input": { + "query": "In publicly accessible DICOM data where the Modality is 'SEG' and the SOPClassUID is '1.2.840.10008.5.1.4.1.1.66.4', and each segmentation references its original SOPInstanceUID, which five segmentation categories (by 'SegmentedPropertyCategory.CodeMeaning') occur most frequently?" + }, + "expected_output": { + "sql": "WITH sampled_sops AS (SELECT \"collection_id\", \"SeriesDescription\", \"SeriesInstanceUID\", \"SOPInstanceUID\" AS \"seg_SOPInstanceUID\", COALESCE(\"ReferencedSeriesSequence\"[1].\"ReferencedInstanceSequence\"[1].\"ReferencedSOPInstanceUID\", \"ReferencedImageSequence\"[1].\"ReferencedSOPInstanceUID\", \"SourceImageSequence\"[1].\"ReferencedSOPInstanceUID\") AS \"referenced_sop\" FROM \"IDC\".\"IDC_V17\".\"DICOM_ALL\" WHERE \"Modality\" = 'SEG' AND \"SOPClassUID\" = '1.2.840.10008.5.1.4.1.1.66.4' AND \"access\" = 'Public'), segmentations_data AS (SELECT dicom_all.\"collection_id\", dicom_all.\"PatientID\", dicom_all.\"SOPInstanceUID\", REPLACE(CAST(JSON_EXTRACT(segmentations.\"SegmentedPropertyCategory\", '$.CodeMeaning') AS VARCHAR), '\"', '') AS \"segmentation_category\", REPLACE(CAST(JSON_EXTRACT(segmentations.\"SegmentedPropertyType\", '$.CodeMeaning') AS VARCHAR), '\"', '') AS \"segmentation_type\" FROM sampled_sops JOIN \"IDC\".\"IDC_V17\".\"DICOM_ALL\" AS dicom_all ON sampled_sops.\"referenced_sop\" = dicom_all.\"SOPInstanceUID\" JOIN \"IDC\".\"IDC_V17\".\"SEGMENTATIONS\" AS segmentations ON segmentations.\"SOPInstanceUID\" = sampled_sops.\"seg_SOPInstanceUID\") SELECT \"segmentation_category\", COUNT(*) AS \"count_\" FROM segmentations_data GROUP BY \"segmentation_category\" ORDER BY \"count_\" DESC NULLS FIRST LIMIT 5" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "IDC", + "catalog": "idc", + "external_knowledge": null + } + }, + { + "id": "sf_bq347", + "input": { + "query": "From the union of the specified MR series with SeriesInstanceUID 1.3.6.1.4.1.14519.5.2.1.3671.4754.105976129314091491952445656147 and all associated segmentation instances, which modality has the greatest number of SOP instances in total, and how many are there?" + }, + "expected_output": { + "sql": "WITH union_mr_seg AS (SELECT \"dicom_all_mr\".\"SOPInstanceUID\", '' AS \"segPropertyTypeCodeMeaning\", '' AS \"segPropertyCategoryCodeMeaning\" FROM \"IDC\".\"IDC_V17\".\"DICOM_ALL\" AS \"dicom_all_mr\" WHERE \"dicom_all_mr\".\"SeriesInstanceUID\" IN ('1.3.6.1.4.1.14519.5.2.1.3671.4754.105976129314091491952445656147') UNION ALL SELECT \"dicom_all_seg\".\"SOPInstanceUID\", JSON_EXTRACT(\"segmentations\".\"SegmentedPropertyType\", '$.CodeMeaning') AS \"segPropertyTypeCodeMeaning\", JSON_EXTRACT(\"segmentations\".\"SegmentedPropertyCategory\", '$.CodeMeaning') AS \"segPropertyCategoryCodeMeaning\" FROM \"IDC\".\"IDC_V17\".\"DICOM_ALL\" AS \"dicom_all_seg\" JOIN \"IDC\".\"IDC_V17\".\"SEGMENTATIONS\" AS \"segmentations\" ON \"dicom_all_seg\".\"SOPInstanceUID\" = \"segmentations\".\"SOPInstanceUID\") SELECT \"dc_all\".\"Modality\", COUNT(*) AS \"count_\" FROM \"IDC\".\"IDC_V17\".\"DICOM_ALL\" AS \"dc_all\" INNER JOIN union_mr_seg ON \"dc_all\".\"SOPInstanceUID\" = union_mr_seg.\"SOPInstanceUID\" GROUP BY \"dc_all\".\"Modality\" ORDER BY \"count_\" DESC NULLS FIRST LIMIT 1" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "IDC", + "catalog": "idc", + "external_knowledge": null + } + }, + { + "id": "sf_bq390", + "input": { + "query": "In the \"qin_prostate_repeatability\" collection, please provide the distinct StudyInstanceUIDs for studies that include T2-weighted axial MR imaging and also contain anatomical structure segmentations labeled as \"Peripheral zone.\"" + }, + "expected_output": { + "sql": "WITH \"mr_studies\" /* Studies that have MR volumes */ AS (SELECT \"dicom_all_mr\".\"StudyInstanceUID\" FROM \"IDC\".\"IDC_V17\".\"DICOM_ALL\" AS \"dicom_all_mr\" WHERE \"Modality\" = 'MR' AND \"collection_id\" = 'qin_prostate_repeatability' AND CONTAINS(\"SeriesDescription\", 'T2 Weighted Axial')), \"seg_studies\" AS (SELECT \"dicom_all_seg\".\"StudyInstanceUID\" FROM \"IDC\".\"IDC_V17\".\"DICOM_ALL\" AS \"dicom_all_seg\" JOIN \"IDC\".\"IDC_V17\".\"SEGMENTATIONS\" AS \"segmentations\" ON \"dicom_all_seg\".\"SOPInstanceUID\" = \"segmentations\".\"SOPInstanceUID\" WHERE \"collection_id\" = 'qin_prostate_repeatability' AND CONTAINS(JSON_EXTRACT(\"segmentations\".\"SegmentedPropertyType\", '$.CodeMeaning'), 'Peripheral zone') AND JSON_EXTRACT(\"segmentations\".\"SegmentedPropertyCategory\", '$.CodeMeaning') = 'Anatomical Structure') SELECT DISTINCT \"mr_studies\".\"StudyInstanceUID\" FROM \"mr_studies\" JOIN \"seg_studies\" ON \"mr_studies\".\"StudyInstanceUID\" = \"seg_studies\".\"StudyInstanceUID\"" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "IDC", + "catalog": "idc", + "external_knowledge": null + } + }, + { + "id": "sf_bq421", + "input": { + "query": "Can you list all unique pairs of embedding medium and staining substance code meanings, along with the number of occurrences for each pair, based on distinct embedding medium and staining substance codes from the 'SM' modality in the DICOM dataset's un-nested specimen preparation sequences, ensuring that the codes are from the SCT coding scheme?" + }, + "expected_output": { + "sql": "WITH SpecimenPreparationSequence_unnested AS (SELECT d.\"SOPInstanceUID\", CAST(JSON_EXTRACT(concept_name_code_sequence.value, '$.CodeMeaning') AS VARCHAR) AS \"cnc_cm\", CAST(JSON_EXTRACT(concept_name_code_sequence.value, '$.CodingSchemeDesignator') AS VARCHAR) AS \"cnc_csd\", CAST(JSON_EXTRACT(concept_name_code_sequence.value, '$.CodeValue') AS VARCHAR) AS \"cnc_val\", CAST(JSON_EXTRACT(concept_code_sequence.value, '$.CodeMeaning') AS VARCHAR) AS \"ccs_cm\", CAST(JSON_EXTRACT(concept_code_sequence.value, '$.CodingSchemeDesignator') AS VARCHAR) AS \"ccs_csd\", CAST(JSON_EXTRACT(concept_code_sequence.value, '$.CodeValue') AS VARCHAR) AS \"ccs_val\" FROM \"IDC\".\"IDC_V17\".\"DICOM_ALL\" AS d CROSS JOIN UNNEST(input => d.\"SpecimenDescriptionSequence\") AS spec_desc(SEQ, KEY, PATH, INDEX, VALUE, THIS) CROSS JOIN UNNEST(input => JSON_EXTRACT(spec_desc.value, '$.SpecimenPreparationSequence')) AS prep_seq(SEQ, KEY, PATH, INDEX, VALUE, THIS) CROSS JOIN UNNEST(input => JSON_EXTRACT(prep_seq.value, '$.SpecimenPreparationStepContentItemSequence')) AS prep_step(SEQ, KEY, PATH, INDEX, VALUE, THIS) CROSS JOIN UNNEST(input => JSON_EXTRACT(prep_step.value, '$.ConceptNameCodeSequence')) AS concept_name_code_sequence(SEQ, KEY, PATH, INDEX, VALUE, THIS) CROSS JOIN UNNEST(input => JSON_EXTRACT(prep_step.value, '$.ConceptCodeSequence')) AS concept_code_sequence(SEQ, KEY, PATH, INDEX, VALUE, THIS)), slide_embedding AS (SELECT \"SOPInstanceUID\", ARRAY_AGG(DISTINCT (CONCAT(CAST(\"ccs_cm\" AS VARCHAR), CAST(':' AS VARCHAR), CAST(\"ccs_csd\" AS VARCHAR), CAST(':' AS VARCHAR), CAST(\"ccs_val\" AS VARCHAR)))) FILTER(WHERE (CONCAT(CAST(\"ccs_cm\" AS VARCHAR), CAST(':' AS VARCHAR), CAST(\"ccs_csd\" AS VARCHAR), CAST(':' AS VARCHAR), CAST(\"ccs_val\" AS VARCHAR))) IS NOT NULL) AS \"embeddingMedium_code_str\" FROM SpecimenPreparationSequence_unnested WHERE \"cnc_csd\" = 'SCT' AND \"cnc_val\" = '430863003' /* CodeMeaning is 'Embedding medium' */ GROUP BY \"SOPInstanceUID\"), slide_staining AS (SELECT \"SOPInstanceUID\", ARRAY_AGG(DISTINCT (CONCAT(CAST(\"ccs_cm\" AS VARCHAR), CAST(':' AS VARCHAR), CAST(\"ccs_csd\" AS VARCHAR), CAST(':' AS VARCHAR), CAST(\"ccs_val\" AS VARCHAR)))) FILTER(WHERE (CONCAT(CAST(\"ccs_cm\" AS VARCHAR), CAST(':' AS VARCHAR), CAST(\"ccs_csd\" AS VARCHAR), CAST(':' AS VARCHAR), CAST(\"ccs_val\" AS VARCHAR))) IS NOT NULL) AS \"staining_usingSubstance_code_str\" FROM SpecimenPreparationSequence_unnested WHERE \"cnc_csd\" = 'SCT' AND \"cnc_val\" = '424361007' /* CodeMeaning is 'Using substance' */ GROUP BY \"SOPInstanceUID\"), embedding_data AS (SELECT d.\"SOPInstanceUID\", d.\"instance_size\", e.\"embeddingMedium_code_str\", s.\"staining_usingSubstance_code_str\" FROM \"IDC\".\"IDC_V17\".\"DICOM_ALL\" AS d LEFT JOIN slide_embedding AS e ON d.\"SOPInstanceUID\" = e.\"SOPInstanceUID\" LEFT JOIN slide_staining AS s ON d.\"SOPInstanceUID\" = s.\"SOPInstanceUID\" WHERE d.\"Modality\" = 'SM') SELECT SPLIT_PART(CAST(embeddingMedium_CodeMeaning_flat.VALUE AS VARCHAR), ':', 1) AS \"embeddingMedium_CodeMeaning\", SPLIT_PART(CAST(staining_usingSubstance_CodeMeaning_flat.VALUE AS VARCHAR), ':', 1) AS \"staining_usingSubstance_CodeMeaning\", COUNT(*) AS \"count_\" FROM embedding_data CROSS JOIN UNNEST(input => embedding_data.\"embeddingMedium_code_str\") AS embeddingMedium_CodeMeaning_flat(SEQ, KEY, PATH, INDEX, VALUE, THIS) CROSS JOIN UNNEST(input => embedding_data.\"staining_usingSubstance_code_str\") AS staining_usingSubstance_CodeMeaning_flat(SEQ, KEY, PATH, INDEX, VALUE, THIS) GROUP BY SPLIT_PART(CAST(embeddingMedium_CodeMeaning_flat.VALUE AS VARCHAR), ':', 1), SPLIT_PART(CAST(staining_usingSubstance_CodeMeaning_flat.VALUE AS VARCHAR), ':', 1)" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "IDC", + "catalog": "idc", + "external_knowledge": null + } + }, + { + "id": "sf_bq422", + "input": { + "query": "Using the 'nlst' collection's CT images, calculate and compare two separate metrics: 1) The average series size in MiB for the top 3 patients with the highest slice interval difference tolerance (defined as the difference between the maximum and minimum unique slice intervals across all their series), and 2) The average series size in MiB for the top 3 patients with the highest exposure difference (defined as the difference between the maximum and minimum unique exposure values across all their series). For each patient, calculate the series size by summing the instance sizes of all images in that series and converting to MiB. Return the results as two separate groups labeled \"Top 3 by Slice Interval\" and \"Top 3 by Max Exposure\" with their respective average series sizes." + }, + "expected_output": { + "sql": "WITH nonLocalizerRawData AS (SELECT \"SeriesInstanceUID\", \"StudyInstanceUID\", \"PatientID\", TRY_CAST(CAST(\"Exposure\" AS VARCHAR) AS DOUBLE) AS \"Exposure\" /* 直接从 bid 获取 Exposure */, TRY_CAST(CAST(axes.VALUE AS VARCHAR) AS DOUBLE) AS \"zImagePosition\", LEAD(TRY_CAST(CAST(axes.VALUE AS VARCHAR) AS DOUBLE)) OVER (PARTITION BY \"SeriesInstanceUID\" ORDER BY TRY_CAST(CAST(axes.VALUE AS VARCHAR) AS DOUBLE)) - TRY_CAST(CAST(axes.VALUE AS VARCHAR) AS DOUBLE) AS \"slice_interval\", \"instance_size\" AS \"instanceSize\" FROM \"IDC\".\"IDC_V17\".\"DICOM_ALL\" AS \"bid\" CROSS JOIN UNNEST(input => \"bid\".\"ImagePositionPatient\") AS axes(SEQ, KEY, PATH, INDEX, VALUE, THIS) /* 使用 LATERAL FLATTEN 展开数组 */ WHERE \"collection_id\" = 'nlst' AND \"Modality\" = 'CT'), geometryChecks AS (SELECT \"SeriesInstanceUID\", \"StudyInstanceUID\", \"PatientID\", ARRAY_AGG(DISTINCT \"slice_interval\") FILTER(WHERE \"slice_interval\" IS NOT NULL) AS \"sliceIntervalDifferences\", ARRAY_AGG(DISTINCT \"Exposure\") FILTER(WHERE \"Exposure\" IS NOT NULL) AS \"distinctExposures\", CAST(CAST(SUM(\"instanceSize\") AS DOUBLE) / 1024 AS DOUBLE) / 1024 AS \"seriesSizeInMB\" FROM nonLocalizerRawData GROUP BY \"SeriesInstanceUID\", \"StudyInstanceUID\", \"PatientID\"), patientMetrics AS (SELECT \"PatientID\", MAX(TRY_CAST(CAST(sid.VALUE AS VARCHAR) AS DOUBLE)) AS \"maxSliceIntervalDifference\", MIN(TRY_CAST(CAST(sid.VALUE AS VARCHAR) AS DOUBLE)) AS \"minSliceIntervalDifference\", MAX(TRY_CAST(CAST(sid.VALUE AS VARCHAR) AS DOUBLE)) - MIN(TRY_CAST(CAST(sid.VALUE AS VARCHAR) AS DOUBLE)) AS \"sliceIntervalDifferenceTolerance\", MAX(TRY_CAST(CAST(de.VALUE AS VARCHAR) AS DOUBLE)) AS \"maxExposure\", MIN(TRY_CAST(CAST(de.VALUE AS VARCHAR) AS DOUBLE)) AS \"minExposure\", MAX(TRY_CAST(CAST(de.VALUE AS VARCHAR) AS DOUBLE)) - MIN(TRY_CAST(CAST(de.VALUE AS VARCHAR) AS DOUBLE)) AS \"maxExposureDifference\", \"seriesSizeInMB\" FROM geometryChecks CROSS JOIN UNNEST(input => \"sliceIntervalDifferences\") AS sid(SEQ, KEY, PATH, INDEX, VALUE, THIS) CROSS JOIN UNNEST(input => \"distinctExposures\") AS de(SEQ, KEY, PATH, INDEX, VALUE, THIS) /* 展开 distinctExposures */ WHERE NOT sid.VALUE IS NULL AND NOT de.VALUE IS NULL GROUP BY \"PatientID\", \"seriesSizeInMB\"), top3BySliceInterval AS (SELECT \"PatientID\", \"seriesSizeInMB\" FROM patientMetrics ORDER BY \"sliceIntervalDifferenceTolerance\" DESC NULLS FIRST LIMIT 3), top3ByMaxExposure AS (SELECT \"PatientID\", \"seriesSizeInMB\" FROM patientMetrics ORDER BY \"maxExposureDifference\" DESC NULLS FIRST LIMIT 3) SELECT 'Top 3 by Slice Interval' AS \"MetricGroup\", AVG(\"seriesSizeInMB\") AS \"AverageSeriesSizeInMB\" FROM top3BySliceInterval UNION ALL SELECT 'Top 3 by Max Exposure' AS \"MetricGroup\", AVG(\"seriesSizeInMB\") AS \"AverageSeriesSizeInMB\" FROM top3ByMaxExposure" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "IDC", + "catalog": "idc", + "external_knowledge": null + } + }, + { + "id": "sf_bq219", + "input": { + "query": "In the Iowa Liquor Sales dataset, starting from January 1, 2022 through the last fully completed month, which two liquor categories, each contributing an average of at least 1% to the monthly sales volume over at least 24 months of available data, have the lowest Pearson correlation coefficient when comparing their monthly percentages of total liquor sales across those months, and what are their names?" + }, + "expected_output": { + "sql": "WITH MonthlyTotals AS (SELECT DATE_FORMAT(\"date\", 'YYYY-MM') AS \"month\", SUM(\"volume_sold_gallons\") AS \"total_monthly_volume\" FROM IOWA_LIQUOR_SALES.IOWA_LIQUOR_SALES.\"SALES\" WHERE \"date\" >= '2022-01-01' AND DATE_FORMAT(\"date\", 'YYYY-MM') < DATE_FORMAT(CURRENT_DATE, '%Y-%m') GROUP BY DATE_FORMAT(\"date\", 'YYYY-MM')), MonthCategory AS (SELECT DATE_FORMAT(\"date\", 'YYYY-MM') AS \"month\", \"category\", \"category_name\", SUM(\"volume_sold_gallons\") AS \"category_monthly_volume\", CASE WHEN \"total_monthly_volume\" <> 0 THEN (CAST(SUM(\"volume_sold_gallons\") AS DOUBLE) / \"total_monthly_volume\") * 100 ELSE NULL END AS \"category_pct_of_month_volume\" FROM IOWA_LIQUOR_SALES.IOWA_LIQUOR_SALES.\"SALES\" AS Sales LEFT JOIN MonthlyTotals ON DATE_FORMAT(Sales.\"date\", 'YYYY-MM') = MonthlyTotals.\"month\" WHERE Sales.\"date\" >= '2022-01-01' AND DATE_FORMAT(Sales.\"date\", 'YYYY-MM') < DATE_FORMAT(CURRENT_DATE, '%Y-%m') GROUP BY DATE_FORMAT(Sales.\"date\", 'YYYY-MM'), \"category\", \"category_name\", \"total_monthly_volume\"), middle_info AS (SELECT Category1.\"category\" AS \"category1\", Category1.\"category_name\" AS \"category_name1\", Category2.\"category\" AS \"category2\", Category2.\"category_name\" AS \"category_name2\", COUNT(DISTINCT Category1.\"month\") AS \"num_months\", CORR(Category1.\"category_pct_of_month_volume\", Category2.\"category_pct_of_month_volume\") AS \"category_corr_across_months\", AVG(Category1.\"category_pct_of_month_volume\") AS \"category1_avg_pct_of_month_volume\", AVG(Category2.\"category_pct_of_month_volume\") AS \"category2_avg_pct_of_month_volume\" FROM MonthCategory AS Category1 INNER JOIN MonthCategory AS Category2 ON Category1.\"month\" = Category2.\"month\" GROUP BY Category1.\"category\", Category1.\"category_name\", Category2.\"category\", Category2.\"category_name\" HAVING \"num_months\" >= 24 AND \"category1_avg_pct_of_month_volume\" >= 1 AND \"category2_avg_pct_of_month_volume\" >= 1) SELECT \"category_name1\", \"category_name2\" FROM middle_info ORDER BY \"category_corr_across_months\" LIMIT 1" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "IOWA_LIQUOR_SALES", + "catalog": "iowa_liquor_sales", + "external_knowledge": null + } + }, + { + "id": "sf_bq043", + "input": { + "query": "What are the RNA expression levels of the genes MDM2, TP53, CDKN1A, and CCNE1, along with associated clinical information, in bladder cancer patients with CDKN2A mutations in the 'TCGA-BLCA' project? Use clinical data from the Genomic Data Commons Release 39, data about somatic mutations derived from the hg19 human genome reference in Feb 2017." + }, + "expected_output": { + "sql": "SELECT genex.\"case_barcode\" AS \"case_barcode\", genex.\"sample_barcode\" AS \"sample_barcode\", genex.\"aliquot_barcode\" AS \"aliquot_barcode\", genex.\"HGNC_gene_symbol\" AS \"HGNC_gene_symbol\", clinical_info.\"Variant_Type\" AS \"Variant_Type\", genex.\"gene_id\" AS \"gene_id\", genex.\"normalized_count\" AS \"normalized_count\", genex.\"project_short_name\" AS \"project_short_name\", clinical_info.\"demo__gender\" AS \"gender\", clinical_info.\"demo__vital_status\" AS \"vital_status\", clinical_info.\"demo__days_to_death\" AS \"days_to_death\" FROM (SELECT case_list.\"Variant_Type\" AS \"Variant_Type\", case_list.\"case_barcode\" AS \"case_barcode\", clinical.\"demo__gender\", clinical.\"demo__vital_status\", clinical.\"demo__days_to_death\" FROM (SELECT mutation.\"case_barcode\", mutation.\"Variant_Type\" FROM \"TCGA\".\"TCGA_VERSIONED\".\"SOMATIC_MUTATION_HG19_DCC_2017_02\" AS mutation WHERE mutation.\"Hugo_Symbol\" = 'CDKN2A' AND mutation.\"project_short_name\" = 'TCGA-BLCA' GROUP BY mutation.\"case_barcode\", mutation.\"Variant_Type\" ORDER BY mutation.\"case_barcode\") AS case_list /* end case_list */ INNER JOIN \"TCGA\".\"TCGA_VERSIONED\".\"CLINICAL_GDC_R39\" AS clinical ON case_list.\"case_barcode\" = clinical.\"submitter_id\" /* end clinical annotation */) AS clinical_info INNER JOIN \"TCGA\".\"TCGA_VERSIONED\".\"RNASEQ_HG19_GDC_2017_02\" AS genex ON genex.\"case_barcode\" = clinical_info.\"case_barcode\" WHERE genex.\"HGNC_gene_symbol\" IN ('MDM2', 'TP53', 'CDKN1A', 'CCNE1') ORDER BY \"case_barcode\", \"HGNC_gene_symbol\"" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "TCGA", + "catalog": "tcga", + "external_knowledge": null + } + }, + { + "id": "sf_bq176", + "input": { + "query": "Identify the case barcodes from the TCGA-LAML study with the highest weighted average copy number in cytoband 15q11 on chromosome 15, using segment data and cytoband overlaps from TCGA's genomic and Mitelman databases." + }, + "expected_output": { + "sql": "WITH copy AS (SELECT \"case_barcode\", \"chromosome\", \"start_pos\", \"end_pos\", MAX(\"copy_number\") AS \"copy_number\" FROM \"TCGA_MITELMAN\".\"TCGA_VERSIONED\".\"COPY_NUMBER_SEGMENT_ALLELIC_HG38_GDC_R23\" WHERE \"project_short_name\" = 'TCGA-LAML' GROUP BY \"case_barcode\", \"chromosome\", \"start_pos\", \"end_pos\"), total_cases AS (SELECT COUNT(DISTINCT \"case_barcode\") AS \"total\" FROM copy), cytob AS (SELECT \"chromosome\", \"cytoband_name\", \"hg38_start\", \"hg38_stop\" FROM \"TCGA_MITELMAN\".\"PROD\".\"CYTOBANDS_HG38\"), joined AS (SELECT cytob.\"chromosome\", cytob.\"cytoband_name\", cytob.\"hg38_start\", cytob.\"hg38_stop\", copy.\"case_barcode\", CAST((ABS(cytob.\"hg38_stop\" - cytob.\"hg38_start\") + ABS(copy.\"end_pos\" - copy.\"start_pos\") - ABS(cytob.\"hg38_stop\" - copy.\"end_pos\") - ABS(cytob.\"hg38_start\" - copy.\"start_pos\")) AS DOUBLE) / 2.0 AS \"overlap\", copy.\"copy_number\" FROM copy LEFT JOIN cytob ON cytob.\"chromosome\" = copy.\"chromosome\" WHERE (cytob.\"hg38_start\" >= copy.\"start_pos\" AND copy.\"end_pos\" >= cytob.\"hg38_start\") OR (copy.\"start_pos\" >= cytob.\"hg38_start\" AND copy.\"start_pos\" <= cytob.\"hg38_stop\")), INFO AS (SELECT \"chromosome\", \"cytoband_name\", \"hg38_start\", \"hg38_stop\", \"case_barcode\", ROUND(CAST(SUM(\"overlap\" * \"copy_number\") AS DOUBLE) / SUM(\"overlap\")) AS \"copy_number\" FROM joined GROUP BY \"chromosome\", \"cytoband_name\", \"hg38_start\", \"hg38_stop\", \"case_barcode\") SELECT \"case_barcode\" FROM INFO WHERE \"chromosome\" = 'chr15' AND \"cytoband_name\" = '15q11' ORDER BY \"copy_number\" DESC NULLS FIRST LIMIT 1" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "TCGA_MITELMAN", + "catalog": "tcga_mitelman", + "external_knowledge": null + } + }, + { + "id": "sf_bq150", + "input": { + "query": "Assess whether different genetic variants affect the log10-transformed TP53 expression levels in TCGA-BRCA samples using sequencing and mutation data. Provide the total number of samples, the number of mutation types, the mean square between groups, the mean square within groups, and the F-statistic." + }, + "expected_output": { + "sql": "WITH cohortExpr AS (SELECT \"sample_barcode\", LOG(10, \"normalized_count\") AS \"expr\" FROM \"TCGA_HG19_DATA_V0\".\"TCGA_HG19_DATA_V0\".\"RNASEQ_GENE_EXPRESSION_UNC_RSEM\" WHERE \"project_short_name\" = 'TCGA-BRCA' AND \"HGNC_gene_symbol\" = 'TP53' AND NOT \"normalized_count\" IS NULL AND \"normalized_count\" > 0), cohortVar AS (SELECT \"Variant_Type\", \"sample_barcode_tumor\" AS \"sample_barcode\" FROM \"TCGA_HG19_DATA_V0\".\"TCGA_HG19_DATA_V0\".\"SOMATIC_MUTATION_MC3\" WHERE \"SYMBOL\" = 'TP53'), cohort AS (SELECT e.\"sample_barcode\" AS \"sample_barcode\", v.\"Variant_Type\" AS \"group_name\", e.\"expr\" FROM cohortExpr AS e JOIN cohortVar AS v ON e.\"sample_barcode\" = v.\"sample_barcode\"), grandMeanTable AS (SELECT AVG(\"expr\") AS \"grand_mean\" FROM cohort), groupMeansTable AS (SELECT AVG(\"expr\") AS \"group_mean\", \"group_name\", COUNT(\"sample_barcode\") AS \"n\" FROM cohort GROUP BY \"group_name\"), ssBetween AS (SELECT g.\"group_name\", g.\"group_mean\", gm.\"grand_mean\", g.\"n\", g.\"n\" * POWER(g.\"group_mean\" - gm.\"grand_mean\", 2) AS \"n_diff_sq\" FROM groupMeansTable AS g CROSS JOIN grandMeanTable AS gm), ssWithin AS (SELECT c.\"group_name\" AS \"group_name\", c.\"expr\", b.\"group_mean\", b.\"n\" AS \"n\", POWER(c.\"expr\" - b.\"group_mean\", 2) AS \"s2\" FROM cohort AS c JOIN ssBetween AS b ON c.\"group_name\" = b.\"group_name\"), numerator AS (SELECT CAST(SUM(\"n_diff_sq\") AS DOUBLE) / (COUNT(\"group_name\") - 1) AS \"mean_sq_between\" FROM ssBetween), denominator AS (SELECT COUNT(DISTINCT \"group_name\") AS \"k\", COUNT(\"group_name\") AS \"n\", CAST(SUM(\"s2\") AS DOUBLE) / (COUNT(\"group_name\") - COUNT(DISTINCT \"group_name\")) AS \"mean_sq_within\" FROM ssWithin) SELECT \"n\", \"k\", \"mean_sq_between\", \"mean_sq_within\", CAST(\"mean_sq_between\" AS DOUBLE) / \"mean_sq_within\" AS \"F\" FROM numerator, denominator" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "TCGA_HG19_DATA_V0", + "catalog": "tcga_hg19_data_v0", + "external_knowledge": "TCGA_F_Score.md" + } + }, + { + "id": "sf_bq155", + "input": { + "query": "In the TCGA-BRCA cohort of patients who are 80 years old or younger at diagnosis and have a pathological stage of Stage I, Stage II, or Stage IIA, calculate the t-statistic derived from the Pearson correlation between the log10-transformed average RNA-Seq expression levels (using HTSeq__Counts + 1) of the gene SNORA31 and the average microRNA-Seq expression levels of all unique microRNAs, only considering pairs with more than 25 samples and where the absolute Pearson correlation coefficient is between 0.3 and 1.0" + }, + "expected_output": { + "sql": "WITH cohort AS (SELECT \"case_barcode\" FROM \"TCGA_HG38_DATA_V0\".\"TCGA_BIOCLIN_V0\".\"CLINICAL\" WHERE \"project_short_name\" = 'TCGA-BRCA' AND \"age_at_diagnosis\" <= 80 AND \"pathologic_stage\" IN ('Stage I', 'Stage II', 'Stage IIA')), table1 AS (SELECT \"symbol\", \"data\" AS \"rnkdata\", \"ParticipantBarcode\" FROM (SELECT \"gene_name\" AS \"symbol\", AVG(LOG(10, \"HTSeq__Counts\" + 1)) AS \"data\", \"case_barcode\" AS \"ParticipantBarcode\" FROM \"TCGA_HG38_DATA_V0\".\"TCGA_HG38_DATA_V0\".\"RNASEQ_GENE_EXPRESSION\" WHERE \"case_barcode\" IN (SELECT \"case_barcode\" FROM cohort) AND \"gene_name\" = 'SNORA31' AND NOT \"HTSeq__Counts\" IS NULL GROUP BY \"ParticipantBarcode\", \"symbol\")), table2 AS (SELECT \"symbol\", \"data\" AS \"rnkdata\", \"ParticipantBarcode\" FROM (SELECT \"mirna_id\" AS \"symbol\", AVG(\"reads_per_million_miRNA_mapped\") AS \"data\", \"case_barcode\" AS \"ParticipantBarcode\" FROM \"TCGA_HG38_DATA_V0\".\"TCGA_HG38_DATA_V0\".\"MIRNASEQ_EXPRESSION\" WHERE \"case_barcode\" IN (SELECT \"case_barcode\" FROM cohort) AND NOT \"mirna_id\" IS NULL AND NOT \"reads_per_million_miRNA_mapped\" IS NULL GROUP BY \"ParticipantBarcode\", \"symbol\")), summ_table AS (SELECT n1.\"symbol\" AS \"symbol1\", n2.\"symbol\" AS \"symbol2\", COUNT(n1.\"ParticipantBarcode\") AS \"n\", CORR(n1.\"rnkdata\", n2.\"rnkdata\") AS \"correlation\" FROM table1 AS n1 INNER JOIN table2 AS n2 ON n1.\"ParticipantBarcode\" = n2.\"ParticipantBarcode\" GROUP BY \"symbol1\", \"symbol2\") SELECT \"symbol1\", \"symbol2\", ABS(\"correlation\") * SQRT(CAST((\"n\" - 2) AS DOUBLE) / (1 - \"correlation\" * \"correlation\")) AS \"t\" FROM summ_table WHERE \"n\" > 25 AND ABS(\"correlation\") >= 0.3 AND ABS(\"correlation\") < 1.0" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "TCGA_HG38_DATA_V0", + "catalog": "tcga_hg38_data_v0", + "external_knowledge": null + } + }, + { + "id": "sf_bq153", + "input": { + "query": "Calculate, for each histology type specified in the 'icd_o_3_histology' field (excluding those enclosed in square brackets), the average of the per-patient average log10(normalized_count + 1) expression levels of the IGF2 gene among LGG patients with valid IGF2 expression data. Match gene expression and clinical data using the ParticipantBarcode field." + }, + "expected_output": { + "sql": "WITH table1 AS (SELECT \"Symbol\" AS \"symbol\", AVG(LOG(10, \"normalized_count\" + 1)) AS \"data\", \"ParticipantBarcode\" FROM PANCANCER_ATLAS_1.PANCANCER_ATLAS_FILTERED.EBPP_ADJUSTPANCAN_ILLUMINAHISEQ_RNASEQV2_GENEXP_FILTERED WHERE \"Study\" = 'LGG' AND \"Symbol\" = 'IGF2' AND NOT \"normalized_count\" IS NULL GROUP BY \"ParticipantBarcode\", \"symbol\"), table2 AS (SELECT \"symbol\", \"avgdata\" AS \"data\", \"ParticipantBarcode\" FROM (SELECT 'icd_o_3_histology' AS \"symbol\", \"icd_o_3_histology\" AS \"avgdata\", \"bcr_patient_barcode\" AS \"ParticipantBarcode\" FROM PANCANCER_ATLAS_1.PANCANCER_ATLAS_FILTERED.CLINICAL_PANCAN_PATIENT_WITH_FOLLOWUP_FILTERED WHERE \"acronym\" = 'LGG' AND NOT \"icd_o_3_histology\" IS NULL AND NOT REGEXP_LIKE(\"icd_o_3_histology\", '^(\\[.*\\]$)'))), table_data AS (SELECT n1.\"data\" AS \"data1\", n2.\"data\" AS \"data2\", n1.\"ParticipantBarcode\" FROM table1 AS n1 INNER JOIN table2 AS n2 ON n1.\"ParticipantBarcode\" = n2.\"ParticipantBarcode\") SELECT \"data2\" AS \"Histology_Type\", AVG(\"data1\") AS \"Average_Log_Expression\" FROM table_data GROUP BY \"data2\"" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "PANCANCER_ATLAS_1", + "catalog": "pancancer_atlas_1", + "external_knowledge": null + } + }, + { + "id": "sf_bq158", + "input": { + "query": "Which top five histological types of breast cancer (BRCA) in the PanCancer Atlas exhibit the highest percentage of CDH1 gene mutations?" + }, + "expected_output": { + "sql": "WITH table1 AS (SELECT \"histological_type\" AS \"data1\", \"bcr_patient_barcode\" AS \"ParticipantBarcode\" FROM \"PANCANCER_ATLAS_1\".\"PANCANCER_ATLAS_FILTERED\".\"CLINICAL_PANCAN_PATIENT_WITH_FOLLOWUP_FILTERED\" WHERE \"acronym\" = 'BRCA' AND NOT \"histological_type\" IS NULL), table2 AS (SELECT \"Hugo_Symbol\" AS \"symbol\", \"ParticipantBarcode\" FROM \"PANCANCER_ATLAS_1\".\"PANCANCER_ATLAS_FILTERED\".\"MC3_MAF_V5_ONE_PER_TUMOR_SAMPLE\" WHERE \"Study\" = 'BRCA' AND \"Hugo_Symbol\" = 'CDH1' AND \"FILTER\" = 'PASS' GROUP BY \"ParticipantBarcode\", \"symbol\"), summ_table AS (SELECT n1.\"data1\", CASE WHEN n2.\"ParticipantBarcode\" IS NULL THEN 'NO' ELSE 'YES' END AS \"data2\", COUNT(*) AS \"Nij\" FROM table1 AS n1 LEFT JOIN table2 AS n2 ON n1.\"ParticipantBarcode\" = n2.\"ParticipantBarcode\" GROUP BY n1.\"data1\", \"data2\"), percentages AS (SELECT \"data1\", SUM(CASE WHEN \"data2\" = 'YES' THEN \"Nij\" ELSE 0 END) AS \"mutation_count\", SUM(\"Nij\") AS \"total\", CAST(SUM(CASE WHEN \"data2\" = 'YES' THEN \"Nij\" ELSE 0 END) AS DOUBLE) / SUM(\"Nij\") AS \"mutation_percentage\" FROM summ_table GROUP BY \"data1\") SELECT \"data1\" AS \"Histological_Type\" FROM percentages ORDER BY \"mutation_percentage\" DESC NULLS FIRST LIMIT 5" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "PANCANCER_ATLAS_1", + "catalog": "pancancer_atlas_1", + "external_knowledge": null + } + }, + { + "id": "sf_bq159", + "input": { + "query": "Calculate the chi-square value to assess the association between histological types and the presence of CDH1 gene mutations in BRCA patients using data from the PanCancer Atlas. Focus on patients with known histological types and consider only reliable mutation entries. Exclude any histological types or mutation statuses with marginal totals less than or equal to 10. Match clinical and mutation data using ParticipantBarcode" + }, + "expected_output": { + "sql": "WITH table1 AS (SELECT \"symbol\", \"avgdata\" AS \"data\", \"ParticipantBarcode\" FROM (SELECT 'histological_type' AS \"symbol\", \"histological_type\" AS \"avgdata\", \"bcr_patient_barcode\" AS \"ParticipantBarcode\" FROM \"PANCANCER_ATLAS_1\".\"PANCANCER_ATLAS_FILTERED\".\"CLINICAL_PANCAN_PATIENT_WITH_FOLLOWUP_FILTERED\" WHERE \"acronym\" = 'BRCA' AND NOT \"histological_type\" IS NULL)), table2 AS (SELECT \"symbol\", \"ParticipantBarcode\" FROM (SELECT \"Hugo_Symbol\" AS \"symbol\", \"ParticipantBarcode\" AS \"ParticipantBarcode\" FROM \"PANCANCER_ATLAS_1\".\"PANCANCER_ATLAS_FILTERED\".\"MC3_MAF_V5_ONE_PER_TUMOR_SAMPLE\" WHERE \"Study\" = 'BRCA' AND \"Hugo_Symbol\" = 'CDH1' AND \"FILTER\" = 'PASS' GROUP BY \"ParticipantBarcode\", \"symbol\")), summ_table AS (SELECT n1.\"data\" AS \"data1\", CASE WHEN n2.\"ParticipantBarcode\" IS NULL THEN 'NO' ELSE 'YES' END AS \"data2\", COUNT(*) AS \"Nij\" FROM table1 AS n1 LEFT JOIN table2 AS n2 ON n1.\"ParticipantBarcode\" = n2.\"ParticipantBarcode\" GROUP BY n1.\"data\", \"data2\"), expected_table AS (SELECT \"data1\", \"data2\" FROM (SELECT \"data1\", SUM(\"Nij\") AS \"Ni\" FROM summ_table GROUP BY \"data1\") AS Ni_table CROSS JOIN (SELECT \"data2\", SUM(\"Nij\") AS \"Nj\" FROM summ_table GROUP BY \"data2\") AS Nj_table WHERE Ni_table.\"Ni\" > 10 AND Nj_table.\"Nj\" > 10), contingency_table AS (SELECT T1.\"data1\", T1.\"data2\", COALESCE(T2.\"Nij\", 0) AS \"Nij\", CAST((SUM(T2.\"Nij\") OVER (PARTITION BY T1.\"data1\")) * (SUM(T2.\"Nij\") OVER (PARTITION BY T1.\"data2\")) AS DOUBLE) / SUM(T2.\"Nij\") OVER () AS \"E_nij\" FROM expected_table AS T1 LEFT JOIN summ_table AS T2 ON T1.\"data1\" = T2.\"data1\" AND T1.\"data2\" = T2.\"data2\") SELECT SUM(CAST((\"Nij\" - \"E_nij\") * (\"Nij\" - \"E_nij\") AS DOUBLE) / \"E_nij\") AS \"Chi2\" FROM contingency_table" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "PANCANCER_ATLAS_1", + "catalog": "pancancer_atlas_1", + "external_knowledge": null + } + }, + { + "id": "sf_bq166", + "input": { + "query": "Using segment-level copy number data from the copy_number_segment_allelic_hg38_gdc_r23 dataset restricted to 'TCGA-KIRC' samples, merge these segments with the cytogenetic band definitions in 'CytoBands_hg38' to identify each sample’s maximum copy number per cytoband. Classify these maximum copy numbers into amplifications (>3), gains (=3), homozygous deletions (=0), heterozygous deletions (=1), or normal (=2), then calculate the frequency of each subtype out of the total number of distinct cases, and finally present these frequencies as percentages sorted by chromosome and cytoband." + }, + "expected_output": { + "sql": "WITH copy AS (SELECT \"case_barcode\", \"chromosome\", \"start_pos\", \"end_pos\", MAX(\"copy_number\") AS \"copy_number\" FROM \"TCGA_MITELMAN\".\"TCGA_VERSIONED\".\"COPY_NUMBER_SEGMENT_ALLELIC_HG38_GDC_R23\" WHERE \"project_short_name\" = 'TCGA-KIRC' GROUP BY \"case_barcode\", \"chromosome\", \"start_pos\", \"end_pos\"), total_cases AS (SELECT COUNT(DISTINCT \"case_barcode\") AS \"total\" FROM copy), cytob AS (SELECT \"chromosome\", \"cytoband_name\", \"hg38_start\", \"hg38_stop\" FROM \"TCGA_MITELMAN\".\"PROD\".\"CYTOBANDS_HG38\"), joined AS (SELECT cytob.\"chromosome\", cytob.\"cytoband_name\", cytob.\"hg38_start\", cytob.\"hg38_stop\", copy.\"case_barcode\", copy.\"copy_number\" FROM copy LEFT JOIN cytob ON cytob.\"chromosome\" = copy.\"chromosome\" WHERE (cytob.\"hg38_start\" >= copy.\"start_pos\" AND copy.\"end_pos\" >= cytob.\"hg38_start\") OR (copy.\"start_pos\" >= cytob.\"hg38_start\" AND copy.\"start_pos\" <= cytob.\"hg38_stop\")), cbands AS (SELECT \"chromosome\", \"cytoband_name\", \"hg38_start\", \"hg38_stop\", \"case_barcode\", MAX(\"copy_number\") AS \"copy_number\" FROM joined GROUP BY \"chromosome\", \"cytoband_name\", \"hg38_start\", \"hg38_stop\", \"case_barcode\"), aberrations AS (SELECT \"chromosome\", \"cytoband_name\", SUM(CASE WHEN \"copy_number\" > 3 THEN 1 ELSE 0 END) AS \"total_amp\" /* Amplifications: more than two copies for diploid > 4 */, SUM(CASE WHEN \"copy_number\" = 3 THEN 1 ELSE 0 END) AS \"total_gain\" /* Gains: at most two extra copies */, SUM(CASE WHEN \"copy_number\" = 0 THEN 1 ELSE 0 END) AS \"total_homodel\" /* Homozygous deletions, or complete deletions */, SUM(CASE WHEN \"copy_number\" = 1 THEN 1 ELSE 0 END) AS \"total_heterodel\" /* Heterozygous deletions, 1 copy lost */, SUM(CASE WHEN \"copy_number\" = 2 THEN 1 ELSE 0 END) AS \"total_normal\" /* Normal for Diploid = 2 */ FROM cbands GROUP BY \"chromosome\", \"cytoband_name\") SELECT aberrations.\"chromosome\", aberrations.\"cytoband_name\", total_cases.\"total\", CAST(100 * aberrations.\"total_amp\" AS DOUBLE) / total_cases.\"total\" AS \"freq_amp\", CAST(100 * aberrations.\"total_gain\" AS DOUBLE) / total_cases.\"total\" AS \"freq_gain\", CAST(100 * aberrations.\"total_homodel\" AS DOUBLE) / total_cases.\"total\" AS \"freq_homodel\", CAST(100 * aberrations.\"total_heterodel\" AS DOUBLE) / total_cases.\"total\" AS \"freq_heterodel\", CAST(100 * aberrations.\"total_normal\" AS DOUBLE) / total_cases.\"total\" AS \"freq_normal\" FROM aberrations, total_cases ORDER BY aberrations.\"chromosome\", aberrations.\"cytoband_name\"" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "TCGA_MITELMAN", + "catalog": "tcga_mitelman", + "external_knowledge": "Comprehensive_Guide_to_Copy_Number_Variations_in_Cancer_Genomics.md" + } + }, + { + "id": "sf_bq412", + "input": { + "query": "Please retrieve the page URLs, first shown time, last shown time, removal reason, violation category, and the lower and upper bounds of times shown for the five most recently removed ads in the Croatia region (region code 'HR'), where the times shown availability date is null, the times shown lower bound exceeds 10,000, the times shown upper bound is below 25,000, and the ads used at least one non-unused audience selection approach among demographics, geographic location, contextual signals, customer lists, or topics of interest, ordering the resulting ads by their last shown time in descending order." + }, + "expected_output": { + "sql": "SELECT \"creative_page_url\", TO_TIMESTAMP(JSON_EXTRACT(\"region_stat\".value, '$.first_shown')) AS \"first_shown\", TO_TIMESTAMP(JSON_EXTRACT(\"region_stat\".value, '$.last_shown')) AS \"last_shown\", REPLACE(REPLACE(\"disapproval\"[1].\"removal_reason\", '\"\"', '\"'), '\"', '') AS \"removal_reason\", REPLACE(REPLACE(\"disapproval\"[1].\"violation_category\", '\"\"', '\"'), '\"', '') AS \"violation_category\", JSON_EXTRACT(\"region_stat\".value, '$.times_shown_lower_bound') AS \"times_shown_lower\", JSON_EXTRACT(\"region_stat\".value, '$.times_shown_upper_bound') AS \"times_shown_upper\" FROM \"GOOGLE_ADS\".\"GOOGLE_ADS_TRANSPARENCY_CENTER\".\"REMOVED_CREATIVE_STATS\" CROSS JOIN UNNEST(input => \"region_stats\") AS \"region_stat\"(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE JSON_EXTRACT(\"region_stat\".value, '$.region_code') = 'HR' AND JSON_EXTRACT(\"region_stat\".value, '$.times_shown_availability_date') IS NULL AND JSON_EXTRACT(\"region_stat\".value, '$.times_shown_lower_bound') > 10000 AND JSON_EXTRACT(\"region_stat\".value, '$.times_shown_upper_bound') < 25000 AND (JSON_EXTRACT(\"audience_selection_approach_info\", '$.demographic_info') <> 'CRITERIA_UNUSED' OR JSON_EXTRACT(\"audience_selection_approach_info\", '$.geo_location') <> 'CRITERIA_UNUSED' OR JSON_EXTRACT(\"audience_selection_approach_info\", '$.contextual_signals') <> 'CRITERIA_UNUSED' OR JSON_EXTRACT(\"audience_selection_approach_info\", '$.customer_lists') <> 'CRITERIA_UNUSED' OR JSON_EXTRACT(\"audience_selection_approach_info\", '$.topics_of_interest') <> 'CRITERIA_UNUSED') ORDER BY \"last_shown\" DESC NULLS FIRST LIMIT 5" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "GOOGLE_ADS", + "catalog": "google_ads", + "external_knowledge": null + } + }, + { + "id": "sf_bq070", + "input": { + "query": "Could you provide a clean, structured dataset from dicom_all table that only includes SM images marked as VOLUME from the TCGA-LUAD and TCGA-LUSC collections, excluding any slides with compression type “other,” where the specimen preparation step explicitly has “Embedding medium” set to “Tissue freezing medium,” and ensuring that the tissue type is only “normal” or “tumor” and the cancer subtype is reported accordingly?" + }, + "expected_output": { + "sql": "WITH sm_images AS (SELECT \"SeriesInstanceUID\" AS \"digital_slide_id\", \"StudyInstanceUID\" AS \"case_id\", \"ContainerIdentifier\" AS \"physical_slide_id\", \"PatientID\" AS \"patient_id\", \"TotalPixelMatrixColumns\" AS \"width\", \"TotalPixelMatrixRows\" AS \"height\", \"collection_id\", \"crdc_instance_uuid\", \"gcs_url\", CAST(\"SharedFunctionalGroupsSequence\"[1].\"PixelMeasuresSequence\"[1].\"PixelSpacing\"[1] AS DOUBLE) AS \"pixel_spacing\", CASE \"TransferSyntaxUID\" WHEN '1.2.840.10008.1.2.4.50' THEN 'jpeg' WHEN '1.2.840.10008.1.2.4.91' THEN 'jpeg2000' ELSE 'other' END AS \"compression\" FROM IDC.IDC_V17.DICOM_ALL WHERE \"Modality\" = 'SM' AND \"ImageType\"[3] = 'VOLUME'), tissue_types AS (SELECT DISTINCT * FROM (SELECT \"SeriesInstanceUID\" AS \"digital_slide_id\", CASE CAST(JSON_EXTRACT(\"steps_unnested2\".value, '$.CodeValue') AS VARCHAR) WHEN '17621005' THEN 'normal' /* meaning: 'Normal' (i.e., non-neoplastic) */ WHEN '86049000' THEN 'tumor' /* meaning: 'Neoplasm, Primary' */ ELSE 'other' /* meaning: 'Neoplasm, Metastatic' */ END AS \"tissue_type\" FROM IDC.IDC_V17.DICOM_ALL CROSS JOIN CROSS JOIN UNNEST(input => \"SpecimenDescriptionSequence\"[1].\"PrimaryAnatomicStructureSequence\") AS \"steps_unnested1\"(SEQ, KEY, PATH, INDEX, VALUE, THIS) CROSS JOIN CROSS JOIN UNNEST(input => JSON_EXTRACT(\"steps_unnested1\".value, '$.PrimaryAnatomicStructureModifierSequence')) AS \"steps_unnested2\"(SEQ, KEY, PATH, INDEX, VALUE, THIS))), specimen_preparation_sequence_items AS (SELECT DISTINCT * FROM (SELECT \"SeriesInstanceUID\" AS \"digital_slide_id\", CAST(JSON_EXTRACT(\"steps_unnested2\".value, '$.ConceptNameCodeSequence[0].CodeMeaning') AS VARCHAR) AS \"item_name\", CAST(JSON_EXTRACT(\"steps_unnested2\".value, '$.ConceptCodeSequence[0].CodeMeaning') AS VARCHAR) AS \"item_value\" FROM IDC.IDC_V17.DICOM_ALL CROSS JOIN CROSS JOIN UNNEST(input => \"SpecimenDescriptionSequence\"[1].\"SpecimenPreparationSequence\") AS \"steps_unnested1\"(SEQ, KEY, PATH, INDEX, VALUE, THIS) CROSS JOIN CROSS JOIN UNNEST(input => JSON_EXTRACT(\"steps_unnested1\".value, '$.SpecimenPreparationStepContentItemSequence')) AS \"steps_unnested2\"(SEQ, KEY, PATH, INDEX, VALUE, THIS))) SELECT a.*, b.\"tissue_type\", REPLACE(REPLACE(a.\"collection_id\", 'tcga_luad', 'luad'), 'tcga_lusc', 'lscc') AS \"cancer_subtype\" FROM sm_images AS a JOIN tissue_types AS b ON b.\"digital_slide_id\" = a.\"digital_slide_id\" JOIN specimen_preparation_sequence_items AS c ON c.\"digital_slide_id\" = a.\"digital_slide_id\" WHERE (a.\"collection_id\" = 'tcga_luad' OR a.\"collection_id\" = 'tcga_lusc') AND a.\"compression\" <> 'other' AND (b.\"tissue_type\" = 'normal' OR b.\"tissue_type\" = 'tumor') AND (c.\"item_name\" = 'Embedding medium' AND c.\"item_value\" = 'Tissue freezing medium') ORDER BY a.\"crdc_instance_uuid\"" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "IDC", + "catalog": "idc", + "external_knowledge": "dicom_dataset_selection.md" + } + }, + { + "id": "sf_bq320", + "input": { + "query": "In the dicom_pivot table, how many unique StudyInstanceUID values exactly match the SegmentedPropertyTypeCodeSequence of \"15825003\" (case-insensitive) and also have a collection_id of either \"Community\" or \"nsclc_radiomics\"?" + }, + "expected_output": { + "sql": "SELECT COUNT(*) AS \"total_count\" FROM IDC.IDC_V17.DICOM_PIVOT AS \"dicom_pivot\" WHERE \"StudyInstanceUID\" IN (SELECT \"StudyInstanceUID\" FROM IDC.IDC_V17.DICOM_PIVOT AS \"dicom_pivot\" WHERE \"StudyInstanceUID\" IN (SELECT \"StudyInstanceUID\" FROM IDC.IDC_V17.DICOM_PIVOT AS \"dicom_pivot\" WHERE LOWER(\"dicom_pivot\".\"SegmentedPropertyTypeCodeSequence\") LIKE LOWER('15825003') GROUP BY \"StudyInstanceUID\" INTERSECT SELECT \"StudyInstanceUID\" FROM IDC.IDC_V17.DICOM_PIVOT AS \"dicom_pivot\" WHERE \"dicom_pivot\".\"collection_id\" IN ('Community', 'nsclc_radiomics') GROUP BY \"StudyInstanceUID\") GROUP BY \"StudyInstanceUID\")" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "IDC", + "catalog": "idc", + "external_knowledge": null + } + }, + { + "id": "sf_bq321", + "input": { + "query": "How many unique StudyInstanceUIDs are there from the DWI, T2 Weighted Axial, Apparent Diffusion Coefficient series, and T2 Weighted Axial Segmentations in the 'qin_prostate_repeatability' collection?" + }, + "expected_output": { + "sql": "WITH relevant_series AS (SELECT DISTINCT \"StudyInstanceUID\" FROM IDC.IDC_V17.DICOM_ALL WHERE \"collection_id\" = 'qin_prostate_repeatability' AND \"SeriesDescription\" IN ('DWI', 'T2 Weighted Axial', 'Apparent Diffusion Coefficient', 'T2 Weighted Axial Segmentations', 'Apparent Diffusion Coefficient Segmentations')), t2_seg_lesion_series AS (SELECT DISTINCT \"StudyInstanceUID\" FROM IDC.IDC_V17.DICOM_ALL CROSS JOIN CROSS JOIN UNNEST(input => \"SegmentSequence\") AS segSeq(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE \"collection_id\" = 'qin_prostate_repeatability' AND \"SeriesDescription\" = 'T2 Weighted Axial Segmentations') SELECT COUNT(DISTINCT \"StudyInstanceUID\") AS \"total_count\" FROM (SELECT \"StudyInstanceUID\" FROM relevant_series UNION ALL SELECT \"StudyInstanceUID\" FROM t2_seg_lesion_series)" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "IDC", + "catalog": "idc", + "external_knowledge": null + } + }, + { + "id": "sf_bq455", + "input": { + "query": "Identify the top five CT scan series by size (in MiB), including their SeriesInstanceUID, series number, patient ID, and series size. These series must be from the CT modality and not part of the 'nlst' collection. Exclude any series where the ImageType is classified as 'LOCALIZER' or where the TransferSyntaxUID is either '1.2.840.10008.1.2.4.70' or '1.2.840.10008.1.2.4.51' (i.e., JPEG compressed). The selected series must have consistent slice intervals, exposure levels, image orientation (with only one unique ImageOrientationPatient value), pixel spacing, image positions (both z-axis and xy positions), and pixel dimensions (rows and columns). Ensure that the number of images matches the number of unique z-axis positions, indicating no duplicate slices. Additionally, the z-axis component of the cross product of the x and y direction cosines from ImageOrientationPatient must have an absolute value between 0.99 and 1.01, ensuring alignment with the expected imaging plane. Finally, order the results by series size in descending order and limit the output to the top five series satisfying these conditions." + }, + "expected_output": { + "sql": "WITH localizerAndJpegCompressedSeries /* Create a common table expression (CTE) named localizerAndJpegCompressedSeries */ AS (SELECT \"SeriesInstanceUID\" FROM IDC.IDC_V17.\"DICOM_ALL\" AS bid WHERE \"ImageType\" = 'LOCALIZER' OR \"TransferSyntaxUID\" IN ('1.2.840.10008.1.2.4.70', '1.2.840.10008.1.2.4.51')), imageOrientation /* Create a common table expression (CTE) for x_vector calculation (first three elements) */ AS (SELECT \"SeriesInstanceUID\", ARRAY_AGG(CAST(part.value AS DOUBLE)) FILTER(WHERE CAST(part.value AS DOUBLE) IS NOT NULL) AS \"x_vector\" FROM IDC.IDC_V17.\"DICOM_ALL\" AS bid CROSS JOIN UNNEST(input => bid.\"ImageOrientationPatient\") AS part(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE part.index BETWEEN 0 AND 2 GROUP BY \"SeriesInstanceUID\"), imageOrientationY /* Create a common table expression (CTE) for y_vector calculation (next three elements) */ AS (SELECT \"SeriesInstanceUID\", ARRAY_AGG(CAST(part.value AS DOUBLE)) FILTER(WHERE CAST(part.value AS DOUBLE) IS NOT NULL) AS \"y_vector\" FROM IDC.IDC_V17.\"DICOM_ALL\" AS bid CROSS JOIN UNNEST(input => bid.\"ImageOrientationPatient\") AS part(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE part.index BETWEEN 3 AND 5 GROUP BY \"SeriesInstanceUID\"), nonLocalizerRawData /* Create a common table expression (CTE) named nonLocalizerRawData */ AS (SELECT bid.\"SeriesInstanceUID\" /* Added table alias bid */, bid.\"StudyInstanceUID\", bid.\"PatientID\", bid.\"SOPInstanceUID\", bid.\"SliceThickness\", bid.\"ImageType\", bid.\"TransferSyntaxUID\", bid.\"SeriesNumber\", bid.\"aws_bucket\", bid.\"crdc_series_uuid\", CAST(bid.\"Exposure\" AS DOUBLE) AS \"Exposure\" /* Use CAST directly */, CAST(ipp.value AS DOUBLE) AS \"zImagePosition\" /* Use CAST directly */, CONCAT(CAST(ipp2.value AS VARCHAR), CAST('/' AS VARCHAR), CAST(ipp3.value AS VARCHAR)) AS \"xyImagePosition\", LEAD(CAST(ipp.value AS DOUBLE)) OVER (PARTITION BY bid.\"SeriesInstanceUID\" ORDER BY CAST(ipp.value AS DOUBLE)) - CAST(ipp.value AS DOUBLE) AS \"slice_interval\", ARRAY_JOIN(bid.\"ImageOrientationPatient\", '/') AS \"iop\", bid.\"PixelSpacing\", bid.\"Rows\" AS \"pixelRows\", bid.\"Columns\" AS \"pixelColumns\", bid.\"instance_size\" AS \"instanceSize\" FROM IDC.IDC_V17.\"DICOM_ALL\" AS bid LEFT JOIN CROSS JOIN UNNEST(input => bid.\"ImagePositionPatient\") AS ipp(SEQ, KEY, PATH, INDEX, VALUE, THIS) LEFT JOIN CROSS JOIN UNNEST(input => bid.\"ImagePositionPatient\") AS ipp2(SEQ, KEY, PATH, INDEX, VALUE, THIS) LEFT JOIN CROSS JOIN UNNEST(input => bid.\"ImagePositionPatient\") AS ipp3(SEQ, KEY, PATH, INDEX, VALUE, THIS) WHERE bid.\"collection_id\" <> 'nlst' AND bid.\"Modality\" = 'CT' AND ipp.index = 2 AND ipp2.index = 0 AND ipp3.index = 1 AND bid.\"SeriesInstanceUID\" <> ALL (SELECT \"SeriesInstanceUID\" FROM localizerAndJpegCompressedSeries)), crossProduct /* Cross product calculation */ AS (SELECT nld.\"SOPInstanceUID\" /* Added table alias nld */, nld.\"SeriesInstanceUID\" /* Added table alias nld */, ROW((\"x_vector\"[2] * \"y_vector\"[3] - \"x_vector\"[3] * \"y_vector\"[2]), (\"x_vector\"[3] * \"y_vector\"[1] - \"x_vector\"[1] * \"y_vector\"[3]), (\"x_vector\"[1] * \"y_vector\"[2] - \"x_vector\"[2] * \"y_vector\"[1])) AS \"xyCrossProduct\" FROM nonLocalizerRawData AS nld /* Added alias for nonLocalizerRawData */ JOIN imageOrientation AS io ON nld.\"SeriesInstanceUID\" = io.\"SeriesInstanceUID\" JOIN imageOrientationY AS ioy ON nld.\"SeriesInstanceUID\" = ioy.\"SeriesInstanceUID\"), crossProductElements /* Cross product elements extraction and row numbering */ AS (SELECT cp.\"SOPInstanceUID\", cp.\"SeriesInstanceUID\", elem.value, ROW_NUMBER() OVER (PARTITION BY cp.\"SOPInstanceUID\", cp.\"SeriesInstanceUID\" ORDER BY elem.value) AS rn FROM crossProduct AS cp/* Use LATERAL FLATTEN to explode the cross product object into individual 'x', 'y', and 'z' */ CROSS JOIN UNNEST(input => ARRAY[cp.\"xyCrossProduct\"['x'], cp.\"xyCrossProduct\"['y'], cp.\"xyCrossProduct\"['z']]) AS elem(SEQ, KEY, PATH, INDEX, VALUE, THIS) /* Simplified 'elem.value' reference here */), dotProduct /* Dot product calculation */ AS (SELECT cpe.\"SOPInstanceUID\", cpe.\"SeriesInstanceUID\", SUM(CASE WHEN cpe.rn = 1 THEN cpe.value * 0 /* x * 0 */ WHEN cpe.rn = 2 THEN cpe.value * 0 /* y * 0 */ WHEN cpe.rn = 3 THEN cpe.value * 1 /* z * 1 */ END) AS \"xyDotProduct\" FROM crossProductElements AS cpe GROUP BY cpe.\"SOPInstanceUID\", cpe.\"SeriesInstanceUID\"), geometryChecks /* Geometry checks for series consistency */ AS (SELECT gc.\"SeriesInstanceUID\" /* Added table alias gc */, gc.\"SeriesNumber\", gc.\"aws_bucket\", gc.\"crdc_series_uuid\", gc.\"StudyInstanceUID\", gc.\"PatientID\", ARRAY_AGG(DISTINCT gc.\"slice_interval\") FILTER(WHERE gc.\"slice_interval\" IS NOT NULL) AS \"sliceIntervalDifferences\", ARRAY_AGG(DISTINCT gc.\"Exposure\") FILTER(WHERE gc.\"Exposure\" IS NOT NULL) AS \"distinctExposures\", COUNT(DISTINCT gc.\"iop\") AS \"iopCount\", COUNT(DISTINCT gc.\"PixelSpacing\") AS \"pixelSpacingCount\", COUNT(DISTINCT gc.\"zImagePosition\") AS \"positionCount\", COUNT(DISTINCT gc.\"xyImagePosition\") AS \"xyPositionCount\", COUNT(DISTINCT gc.\"SOPInstanceUID\") AS \"sopInstanceCount\", COUNT(DISTINCT gc.\"SliceThickness\") AS \"sliceThicknessCount\", COUNT(DISTINCT gc.\"Exposure\") AS \"exposureCount\", COUNT(DISTINCT gc.\"pixelRows\") AS \"pixelRowCount\", COUNT(DISTINCT gc.\"pixelColumns\") AS \"pixelColumnCount\", dp.\"xyDotProduct\" /* Added xyDotProduct from dotProduct */, CAST(CAST(SUM(gc.\"instanceSize\") AS DOUBLE) / 1024 AS DOUBLE) / 1024 AS \"seriesSizeInMiB\" FROM nonLocalizerRawData AS gc /* Added table alias gc */ JOIN dotProduct AS dp ON gc.\"SeriesInstanceUID\" = dp.\"SeriesInstanceUID\" AND gc.\"SOPInstanceUID\" = dp.\"SOPInstanceUID\" GROUP BY gc.\"SeriesInstanceUID\", gc.\"SeriesNumber\", gc.\"aws_bucket\", gc.\"crdc_series_uuid\", gc.\"StudyInstanceUID\", gc.\"PatientID\", dp.\"xyDotProduct\" /* Include xyDotProduct in GROUP BY */ HAVING COUNT(DISTINCT gc.\"iop\") = 1 AND COUNT(DISTINCT gc.\"PixelSpacing\") = 1 AND COUNT(DISTINCT gc.\"SOPInstanceUID\") = COUNT(DISTINCT gc.\"zImagePosition\") AND COUNT(DISTINCT gc.\"xyImagePosition\") = 1 AND COUNT(DISTINCT gc.\"pixelRows\") = 1 AND COUNT(DISTINCT gc.\"pixelColumns\") = 1 AND ABS(dp.\"xyDotProduct\") BETWEEN 0.99 AND 1.01) SELECT geometryChecks.\"SeriesInstanceUID\" /* Added table alias */, geometryChecks.\"SeriesNumber\" /* Added table alias */, geometryChecks.\"PatientID\" /* Added table alias */, geometryChecks.\"seriesSizeInMiB\" FROM geometryChecks ORDER BY geometryChecks.\"seriesSizeInMiB\" DESC NULLS FIRST LIMIT 5" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "IDC", + "catalog": "idc", + "external_knowledge": null + } + }, + { + "id": "sf_bq167", + "input": { + "query": "Identify the pair of Kaggle users involved in ForumMessageVotes such that one user has given the other the greatest distinct number of upvotes, then also display how many upvotes that recipient returned. Present the usernames of both users, the total distinct upvotes one received from the other, and the upvotes they gave back, sorting by the highest received count and then by the highest given count, and show only the top result." + }, + "expected_output": { + "sql": "WITH UserPairUpvotes AS (SELECT ToUsers.\"UserName\" AS \"ToUserName\", FromUsers.\"UserName\" AS \"FromUserName\", COUNT(DISTINCT \"ForumMessageVotes\".\"Id\") AS \"UpvoteCount\" FROM META_KAGGLE.META_KAGGLE.FORUMMESSAGEVOTES AS \"ForumMessageVotes\" INNER JOIN META_KAGGLE.META_KAGGLE.USERS AS FromUsers ON FromUsers.\"Id\" = \"ForumMessageVotes\".\"FromUserId\" INNER JOIN META_KAGGLE.META_KAGGLE.USERS AS ToUsers ON ToUsers.\"Id\" = \"ForumMessageVotes\".\"ToUserId\" GROUP BY ToUsers.\"UserName\", FromUsers.\"UserName\"), TopPairs AS (SELECT \"ToUserName\", \"FromUserName\", \"UpvoteCount\", ROW_NUMBER() OVER (ORDER BY \"UpvoteCount\" DESC NULLS FIRST) AS \"Rank\" FROM UserPairUpvotes), ReciprocalUpvotes AS (SELECT t.\"ToUserName\", t.\"FromUserName\", t.\"UpvoteCount\" AS \"UpvotesReceived\", COALESCE(u.\"UpvoteCount\", 0) AS \"UpvotesGiven\" FROM TopPairs AS t LEFT JOIN UserPairUpvotes AS u ON t.\"ToUserName\" = u.\"FromUserName\" AND t.\"FromUserName\" = u.\"ToUserName\" WHERE t.\"Rank\" = 1) SELECT \"ToUserName\" AS \"UpvotedUserName\", \"FromUserName\" AS \"UpvotingUserName\", \"UpvotesReceived\" AS \"UpvotesReceivedByUpvotedUser\", \"UpvotesGiven\" AS \"UpvotesGivenByUpvotedUser\" FROM ReciprocalUpvotes ORDER BY \"UpvotesReceived\" DESC NULLS FIRST, \"UpvotesGiven\" DESC NULLS FIRST" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "META_KAGGLE", + "catalog": "meta_kaggle", + "external_knowledge": null + } + }, + { + "id": "sf_bq072", + "input": { + "query": "Please provide, for each age from 12 through 18 (inclusive), the total number of deaths and the number of deaths among individuals identified as Black (based on race descriptions containing the word ‘black’), specifically for deaths associated with ICD-10 codes whose descriptions include the word ‘vehicle’ and for deaths associated with ICD-10 codes whose descriptions include the word ‘firearm.’ Use the EntityAxisConditions table to determine which ICD-10 codes were involved in each death, rather than joining ICD-10 code information directly on the death records." + }, + "expected_output": { + "sql": "WITH BlackRace AS (SELECT CAST(\"Code\" AS INTEGER) AS CODE FROM DEATH.DEATH.RACE WHERE LOWER(\"Description\") LIKE '%black%') SELECT v.\"Age\", v.\"Total\" AS \"Vehicle_Total\", v.\"Black\" AS \"Vehicle_Black\", g.\"Total\" AS \"Gun_Total\", g.\"Black\" AS \"Gun_Black\" FROM (SELECT \"Age\", COUNT(*) AS \"Total\", COUNT_IF(\"Race\" IN (SELECT CODE FROM BlackRace)) AS \"Black\" FROM DEATH.DEATH.DEATHRECORDS AS d JOIN (SELECT DISTINCT e.\"DeathRecordId\" AS \"id\" FROM DEATH.DEATH.ENTITYAXISCONDITIONS AS e JOIN (SELECT * FROM DEATH.DEATH.ICD10CODE WHERE LOWER(\"Description\") LIKE '%vehicle%') AS c ON e.\"Icd10Code\" = c.\"Code\") AS f ON d.\"Id\" = f.\"id\" WHERE \"Age\" BETWEEN 12 AND 18 GROUP BY \"Age\") AS v /* Vehicle */ JOIN (SELECT \"Age\", COUNT(*) AS \"Total\", COUNT_IF(\"Race\" IN (SELECT CODE FROM BlackRace)) AS \"Black\" FROM DEATH.DEATH.DEATHRECORDS AS d JOIN (SELECT DISTINCT e.\"DeathRecordId\" AS \"id\" FROM DEATH.DEATH.ENTITYAXISCONDITIONS AS e JOIN (SELECT \"Code\", \"Description\" FROM DEATH.DEATH.ICD10CODE WHERE \"Description\" LIKE '%firearm%') AS c ON e.\"Icd10Code\" = c.\"Code\") AS f ON d.\"Id\" = f.\"id\" WHERE \"Age\" BETWEEN 12 AND 18 GROUP BY \"Age\") AS g ON g.\"Age\" = v.\"Age\"" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "DEATH", + "catalog": "death", + "external_knowledge": null + } + }, + { + "id": "local003", + "input": { + "query": "According to the RFM definition document, calculate the average sales per order for each customer within distinct RFM segments, considering only 'delivered' orders. Use the customer unique identifier. Clearly define how to calculate Recency based on the latest purchase timestamp and specify the criteria for classifying RFM segments. The average sales should be computed as the total spend divided by the total number of orders. Please analyze and report the differences in average sales across the RFM segments" + }, + "expected_output": { + "sql": "WITH RecencyScore AS (SELECT customer_unique_id, MAX(order_purchase_timestamp) AS last_purchase, NTILE(5) OVER (ORDER BY MAX(order_purchase_timestamp) DESC NULLS FIRST) AS recency FROM orders JOIN customers USING (customer_id) WHERE order_status = 'delivered' GROUP BY customer_unique_id), FrequencyScore AS (SELECT customer_unique_id, COUNT(order_id) AS total_orders, NTILE(5) OVER (ORDER BY COUNT(order_id) DESC NULLS FIRST) AS frequency FROM orders JOIN customers USING (customer_id) WHERE order_status = 'delivered' GROUP BY customer_unique_id), MonetaryScore AS (SELECT customer_unique_id, SUM(price) AS total_spent, NTILE(5) OVER (ORDER BY SUM(price) DESC NULLS FIRST) AS monetary FROM orders JOIN order_items USING (order_id) JOIN customers USING (customer_id) WHERE order_status = 'delivered' GROUP BY customer_unique_id), RFM /* 2. Assign each customer to a group */ AS (SELECT last_purchase, total_orders, total_spent, CASE WHEN recency = 1 AND frequency + monetary IN (1, 2, 3, 4) THEN \"Champions\" WHEN recency IN (4, 5) AND frequency + monetary IN (1, 2) THEN \"Can't Lose Them\" WHEN recency IN (4, 5) AND frequency + monetary IN (3, 4, 5, 6) THEN \"Hibernating\" WHEN recency IN (4, 5) AND frequency + monetary IN (7, 8, 9, 10) THEN \"Lost\" WHEN recency IN (2, 3) AND frequency + monetary IN (1, 2, 3, 4) THEN \"Loyal Customers\" WHEN recency = 3 AND frequency + monetary IN (5, 6) THEN \"Needs Attention\" WHEN recency = 1 AND frequency + monetary IN (7, 8) THEN \"Recent Users\" WHEN recency = 1 AND frequency + monetary IN (5, 6) OR recency = 2 AND frequency + monetary IN (5, 6, 7, 8) THEN \"Potentital Loyalists\" WHEN recency = 1 AND frequency + monetary IN (9, 10) THEN \"Price Sensitive\" WHEN recency = 2 AND frequency + monetary IN (9, 10) THEN \"Promising\" WHEN recency = 3 AND frequency + monetary IN (7, 8, 9, 10) THEN \"About to Sleep\" END AS RFM_Bucket FROM RecencyScore JOIN FrequencyScore USING (customer_unique_id) JOIN MonetaryScore USING (customer_unique_id)) SELECT RFM_Bucket, AVG(CAST(total_spent AS DOUBLE) / total_orders) AS avg_sales_per_customer FROM RFM GROUP BY RFM_Bucket" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "E_commerce", + "catalog": "e_commerce", + "external_knowledge": "RFM.md" + } + }, + { + "id": "local004", + "input": { + "query": "Could you tell me the number of orders, average payment per order and customer lifespan in weeks of the 3 custumers with the highest average payment per order, where the lifespan is calculated by subtracting the earliest purchase date from the latest purchase date in days, dividing by seven, and if the result is less than seven days, setting it to 1.0?" + }, + "expected_output": { + "sql": "WITH CustomerData AS (SELECT customer_unique_id, COUNT(DISTINCT orders.order_id) AS order_count, SUM(payment_value) AS total_payment, JULIANDAY(MIN(order_purchase_timestamp)) AS first_order_day, JULIANDAY(MAX(order_purchase_timestamp)) AS last_order_day FROM customers JOIN orders USING (customer_id) JOIN order_payments USING (order_id) GROUP BY customer_unique_id) SELECT customer_unique_id, order_count AS PF, ROUND(CAST(total_payment AS DOUBLE) / order_count, 2) AS AOV, CASE WHEN (last_order_day - first_order_day) < 7 THEN 1 ELSE CAST((last_order_day - first_order_day) AS DOUBLE) / 7 END AS ACL FROM CustomerData ORDER BY AOV DESC NULLS FIRST LIMIT 3" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "E_commerce", + "catalog": "e_commerce", + "external_knowledge": null + } + }, + { + "id": "local008", + "input": { + "query": "I would like to know the given names of baseball players who have achieved the highest value of games played, runs, hits, and home runs, with their corresponding score values." + }, + "expected_output": { + "sql": "WITH player_stats AS (SELECT b.player_id, p.name_given AS player_name, SUM(b.g) AS games_played, SUM(b.r) AS runs, SUM(b.h) AS hits, SUM(b.hr) AS home_runs FROM player AS p JOIN batting AS b ON p.player_id = b.player_id GROUP BY b.player_id, p.name_given) SELECT 'Games Played' AS Category, player_name AS Player_Name, games_played AS Batting_Table_Topper FROM player_stats WHERE games_played = (SELECT MAX(games_played) FROM player_stats) UNION ALL SELECT 'Runs' AS Category, player_name AS Player_Name, runs AS Batting_Table_Topper FROM player_stats WHERE runs = (SELECT MAX(runs) FROM player_stats) UNION ALL SELECT 'Hits' AS Category, player_name AS Player_Name, hits AS Batting_Table_Topper FROM player_stats WHERE hits = (SELECT MAX(hits) FROM player_stats) UNION ALL SELECT 'Home Runs' AS Category, player_name AS Player_Name, home_runs AS Batting_Table_Topper FROM player_stats WHERE home_runs = (SELECT MAX(home_runs) FROM player_stats)" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "Baseball", + "catalog": "baseball", + "external_knowledge": null + } + }, + { + "id": "local017", + "input": { + "query": "In which year were the two most common causes of traffic accidents different from those in other years?" + }, + "expected_output": { + "sql": "WITH AnnualTotals AS (SELECT STRFTIME('%Y', collision_date) AS Year, COUNT(case_id) AS AnnualTotal FROM collisions GROUP BY Year), CategoryTotals AS (SELECT STRFTIME('%Y', collision_date) AS Year, pcf_violation_category AS Category, COUNT(case_id) AS Subtotal FROM collisions GROUP BY Year, Category), CategoryPercentages AS (SELECT ct.Year, ct.Category, ROUND(CAST((ct.Subtotal * 100.0) AS DOUBLE) / at.AnnualTotal, 1) AS PercentageOfAnnualRoadIncidents FROM CategoryTotals AS ct JOIN AnnualTotals AS at ON ct.Year = at.Year), RankedCategories AS (SELECT Year, Category, PercentageOfAnnualRoadIncidents, ROW_NUMBER() OVER (PARTITION BY Year ORDER BY PercentageOfAnnualRoadIncidents DESC NULLS FIRST) AS Rank FROM CategoryPercentages), TopTwoCategories AS (SELECT Year, LISTAGG(Category, ', ') AS TopCategories FROM RankedCategories WHERE Rank <= 2 GROUP BY Year), UniqueYear AS (SELECT Year FROM TopTwoCategories GROUP BY TopCategories HAVING COUNT(Year) = 1), results AS (SELECT rc.Year, rc.Category, rc.PercentageOfAnnualRoadIncidents FROM UniqueYear AS u JOIN RankedCategories AS rc ON u.Year = rc.Year WHERE rc.Rank <= 2) SELECT DISTINCT Year FROM results" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "California_Traffic_Collision", + "catalog": "california_traffic_collision", + "external_knowledge": null + } + }, + { + "id": "local019", + "input": { + "query": "For the NXT title that had the shortest match (excluding titles with \"title change\"), what were the names of the two wrestlers involved?" + }, + "expected_output": { + "sql": "WITH MatchDetails AS (SELECT b.name AS titles, m.duration AS match_duration, CONCAT(CAST(w1.name AS VARCHAR), CAST(' vs ' AS VARCHAR), CAST(w2.name AS VARCHAR)) AS matches, m.win_type AS win_type, l.name AS location, e.name AS event, ROW_NUMBER() OVER (PARTITION BY b.name ORDER BY m.duration ASC) AS rank FROM Belts AS b INNER JOIN Matches AS m ON m.title_id = b.id INNER JOIN Wrestlers AS w1 ON w1.id = m.winner_id INNER JOIN Wrestlers AS w2 ON w2.id = m.loser_id INNER JOIN Cards AS c ON c.id = m.card_id INNER JOIN Locations AS l ON l.id = c.location_id INNER JOIN Events AS e ON e.id = c.event_id INNER JOIN Promotions AS p ON p.id = c.promotion_id WHERE p.name = 'NXT' AND m.duration <> '' AND b.name <> '' AND b.name <> ALL (SELECT name FROM Belts WHERE name LIKE '%title change%')), Rank1 AS (SELECT titles, match_duration, matches, win_type, location, event FROM MatchDetails WHERE rank = 1) SELECT SUBSTR(matches, 1, STRPOS(matches, ' vs ') - 1) AS wrestler1, SUBSTR(matches, STRPOS(matches, ' vs ') + 4) AS wrestler2 FROM Rank1 ORDER BY match_duration LIMIT 1" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "WWE", + "catalog": "wwe", + "external_knowledge": null + } + }, + { + "id": "local022", + "input": { + "query": "Retrieve the names of players who scored no less than 100 runs in a match while playing for the team that lost that match." + }, + "expected_output": { + "sql": "/* Step 1: Calculate players' total runs in each match */ WITH player_runs AS (SELECT bbb.striker AS player_id, bbb.match_id, SUM(bsc.runs_scored) AS total_runs FROM ball_by_ball AS bbb JOIN batsman_scored AS bsc ON bbb.match_id = bsc.match_id AND bbb.over_id = bsc.over_id AND bbb.ball_id = bsc.ball_id AND bbb.innings_no = bsc.innings_no GROUP BY bbb.striker, bbb.match_id HAVING SUM(bsc.runs_scored) >= 100), losing_teams /* Step 2: Identify losing teams for each match */ AS (SELECT match_id, CASE WHEN match_winner = team_1 THEN team_2 ELSE team_1 END AS loser FROM match), players_in_losing_teams /* Step 3: Combine the above results to get players who scored 100 or more runs in losing teams */ AS (SELECT pr.player_id, pr.match_id FROM player_runs AS pr JOIN losing_teams AS lt ON pr.match_id = lt.match_id JOIN player_match AS pm ON pr.player_id = pm.player_id AND pr.match_id = pm.match_id AND lt.loser = pm.team_id) /* Step 4: Select distinct player names from the player table */ SELECT DISTINCT p.player_name FROM player AS p JOIN players_in_losing_teams AS plt ON p.player_id = plt.player_id ORDER BY p.player_name" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "IPL", + "catalog": "ipl", + "external_knowledge": null + } + }, + { + "id": "local023", + "input": { + "query": "Please help me find the names of top 5 players with the highest average runs per match in season 5, along with their batting averages." + }, + "expected_output": { + "sql": "WITH runs_scored AS (SELECT bb.striker AS player_id, bb.match_id, bs.runs_scored AS runs FROM ball_by_ball AS bb JOIN batsman_scored AS bs ON bb.match_id = bs.match_id AND bb.over_id = bs.over_id AND bb.ball_id = bs.ball_id AND bb.innings_no = bs.innings_no WHERE bb.match_id IN (SELECT match_id FROM match WHERE season_id = 5)), total_runs AS (SELECT player_id, match_id, SUM(runs) AS total_runs FROM runs_scored GROUP BY player_id, match_id), batting_averages AS (SELECT player_id, SUM(total_runs) AS runs, COUNT(match_id) AS num_matches, ROUND(SUM(total_runs) / CAST(COUNT(match_id) AS DOUBLE), 3) AS batting_avg FROM total_runs GROUP BY player_id ORDER BY batting_avg DESC NULLS FIRST LIMIT 5) SELECT p.player_name, b.batting_avg FROM player AS p JOIN batting_averages AS b ON p.player_id = b.player_id ORDER BY b.batting_avg DESC NULLS FIRST" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "IPL", + "catalog": "ipl", + "external_knowledge": null + } + }, + { + "id": "local029", + "input": { + "query": "Please identify the top three customers, based on their customer_unique_id, who have the highest number of delivered orders, and provide the average payment value, city, and state for each of these customers." + }, + "expected_output": { + "sql": "WITH customer_orders AS (SELECT c.customer_unique_id, COUNT(o.order_id) AS Total_Orders_By_Customers, AVG(p.payment_value) AS Average_Payment_By_Customer, c.customer_city, c.customer_state FROM olist_customers AS c JOIN olist_orders AS o ON c.customer_id = o.customer_id JOIN olist_order_payments AS p ON o.order_id = p.order_id WHERE o.order_status = 'delivered' GROUP BY c.customer_unique_id, c.customer_city, c.customer_state) SELECT Average_Payment_By_Customer, customer_city, customer_state FROM customer_orders ORDER BY Total_Orders_By_Customers DESC NULLS FIRST LIMIT 3" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "Brazilian_E_Commerce", + "catalog": "brazilian_e_commerce", + "external_knowledge": null + } + }, + { + "id": "local038", + "input": { + "query": "Could you help me determine which actor starred most frequently in English-language children's category films that were rated either G or PG, had a running time of 120 minutes or less, and were released between 2000 and 2010? Please provide the actor's full name." + }, + "expected_output": { + "sql": "SELECT CONCAT(CAST(actor.first_name AS VARCHAR), CAST(' ' AS VARCHAR), CAST(actor.last_name AS VARCHAR)) AS full_name FROM actor INNER JOIN film_actor ON actor.actor_id = film_actor.actor_id INNER JOIN film ON film_actor.film_id = film.film_id INNER JOIN film_category ON film.film_id = film_category.film_id INNER JOIN category ON film_category.category_id = category.category_id /* Join with the language table */ INNER JOIN language ON film.language_id = language.language_id WHERE category.name = 'Children' AND film.release_year BETWEEN 2000 AND 2010 AND film.rating IN ('G', 'PG') AND language.name = 'English' AND film.length <= 120 GROUP BY actor.actor_id, actor.first_name, actor.last_name ORDER BY COUNT(film.film_id) DESC NULLS FIRST LIMIT 1" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "Pagila", + "catalog": "pagila", + "external_knowledge": null + } + }, + { + "id": "local039", + "input": { + "query": "Please help me find the film category with the highest total rental hours in cities where the city's name either starts with \"A\" or contains a hyphen. " + }, + "expected_output": { + "sql": "SELECT category.name FROM category INNER JOIN film_category USING (category_id) INNER JOIN film USING (film_id) INNER JOIN inventory USING (film_id) INNER JOIN rental USING (inventory_id) INNER JOIN customer USING (customer_id) INNER JOIN address USING (address_id) INNER JOIN city USING (city_id) WHERE LOWER(city.city) LIKE 'a%' OR city.city LIKE '%-%' GROUP BY category.name ORDER BY SUM(CAST((JULIANDAY(rental.return_date) - JULIANDAY(rental.rental_date)) * 24 AS INTEGER)) DESC NULLS FIRST LIMIT 1" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "Pagila", + "catalog": "pagila", + "external_knowledge": null + } + }, + { + "id": "local058", + "input": { + "query": "Can you provide a list of hardware product segments along with their unique product counts for 2020 in the output, ordered by the highest percentage increase in unique fact sales products from 2020 to 2021?" + }, + "expected_output": { + "sql": "WITH UniqueProducts2020 AS (SELECT dp.segment, COUNT(DISTINCT fsm.product_code) AS unique_products_2020 FROM hardware_fact_sales_monthly AS fsm JOIN hardware_dim_product AS dp ON fsm.product_code = dp.product_code WHERE fsm.fiscal_year = 2020 GROUP BY dp.segment), UniqueProducts2021 AS (SELECT dp.segment, COUNT(DISTINCT fsm.product_code) AS unique_products_2021 FROM hardware_fact_sales_monthly AS fsm JOIN hardware_dim_product AS dp ON fsm.product_code = dp.product_code WHERE fsm.fiscal_year = 2021 GROUP BY dp.segment) SELECT spc.segment, spc.unique_products_2020 AS product_count_2020 FROM UniqueProducts2020 AS spc JOIN UniqueProducts2021 AS fup ON spc.segment = fup.segment ORDER BY CAST(((fup.unique_products_2021 - spc.unique_products_2020) * 100.0) AS DOUBLE) / (spc.unique_products_2020) DESC NULLS FIRST" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "education_business", + "catalog": "education_business", + "external_knowledge": null + } + }, + { + "id": "local066", + "input": { + "query": "Based on our customer pizza order information, summarize the total quantity of each ingredient used in the pizzas we delivered. Output the name and quantity for each ingredient." + }, + "expected_output": { + "sql": "WITH cte_cleaned_customer_orders AS (SELECT *, ROW_NUMBER() OVER () AS original_row_number FROM pizza_clean_customer_orders), split_regular_toppings AS (SELECT pizza_id, TRIM(SUBSTR(toppings, 1, STRPOS(CONCAT(CAST(toppings AS VARCHAR), CAST(',' AS VARCHAR)), ',') - 1)) AS topping_id, SUBSTR(CONCAT(CAST(toppings AS VARCHAR), CAST(',' AS VARCHAR)), STRPOS(CONCAT(CAST(toppings AS VARCHAR), CAST(',' AS VARCHAR)), ',') + 1) AS remaining_toppings FROM pizza_recipes UNION ALL SELECT pizza_id, TRIM(SUBSTR(remaining_toppings, 1, STRPOS(remaining_toppings, ',') - 1)) AS topping_id, SUBSTR(remaining_toppings, STRPOS(remaining_toppings, ',') + 1) AS remaining_toppings FROM split_regular_toppings WHERE remaining_toppings <> ''), cte_base_toppings AS (SELECT t1.order_id, t1.customer_id, t1.pizza_id, t1.order_time, t1.original_row_number, t2.topping_id FROM cte_cleaned_customer_orders AS t1 LEFT JOIN split_regular_toppings AS t2 ON t1.pizza_id = t2.pizza_id), split_exclusions AS (SELECT order_id, customer_id, pizza_id, order_time, original_row_number, TRIM(SUBSTR(exclusions, 1, STRPOS(CONCAT(CAST(exclusions AS VARCHAR), CAST(',' AS VARCHAR)), ',') - 1)) AS topping_id, SUBSTR(CONCAT(CAST(exclusions AS VARCHAR), CAST(',' AS VARCHAR)), STRPOS(CONCAT(CAST(exclusions AS VARCHAR), CAST(',' AS VARCHAR)), ',') + 1) AS remaining_exclusions FROM cte_cleaned_customer_orders WHERE NOT exclusions IS NULL UNION ALL SELECT order_id, customer_id, pizza_id, order_time, original_row_number, TRIM(SUBSTR(remaining_exclusions, 1, STRPOS(remaining_exclusions, ',') - 1)) AS topping_id, SUBSTR(remaining_exclusions, STRPOS(remaining_exclusions, ',') + 1) AS remaining_exclusions FROM split_exclusions WHERE remaining_exclusions <> ''), split_extras AS (SELECT order_id, customer_id, pizza_id, order_time, original_row_number, TRIM(SUBSTR(extras, 1, STRPOS(CONCAT(CAST(extras AS VARCHAR), CAST(',' AS VARCHAR)), ',') - 1)) AS topping_id, SUBSTR(CONCAT(CAST(extras AS VARCHAR), CAST(',' AS VARCHAR)), STRPOS(CONCAT(CAST(extras AS VARCHAR), CAST(',' AS VARCHAR)), ',') + 1) AS remaining_extras FROM cte_cleaned_customer_orders WHERE NOT extras IS NULL UNION ALL SELECT order_id, customer_id, pizza_id, order_time, original_row_number, TRIM(SUBSTR(remaining_extras, 1, STRPOS(remaining_extras, ',') - 1)) AS topping_id, SUBSTR(remaining_extras, STRPOS(remaining_extras, ',') + 1) AS remaining_extras FROM split_extras WHERE remaining_extras <> ''), cte_combined_orders AS (SELECT order_id, customer_id, pizza_id, order_time, original_row_number, topping_id FROM cte_base_toppings WHERE topping_id <> ALL (SELECT topping_id FROM split_exclusions WHERE split_exclusions.order_id = cte_base_toppings.order_id) UNION ALL SELECT order_id, customer_id, pizza_id, order_time, original_row_number, topping_id FROM split_extras) SELECT t2.topping_name, COUNT(*) AS topping_count FROM cte_combined_orders AS t1 JOIN pizza_toppings AS t2 ON t1.topping_id = t2.topping_id GROUP BY t2.topping_name ORDER BY topping_count DESC NULLS FIRST" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "modern_data", + "catalog": "modern_data", + "external_knowledge": null + } + }, + { + "id": "local065", + "input": { + "query": "Calculate the total income from Meat Lovers pizzas priced at $12 and Vegetarian pizzas at $10. Include any extra toppings charged at $1 each. Ensure that canceled orders are filtered out. How much money has Pizza Runner earned in total?" + }, + "expected_output": { + "sql": "WITH get_extras_count AS (WITH RECURSIVE split_extras(order_id, each_extra, remaining_extras) AS (SELECT order_id, TRIM(SUBSTR(extras, 1, STRPOS(CONCAT(CAST(extras AS VARCHAR), CAST(',' AS VARCHAR)), ',') - 1)) AS each_extra, SUBSTR(CONCAT(CAST(extras AS VARCHAR), CAST(',' AS VARCHAR)), STRPOS(CONCAT(CAST(extras AS VARCHAR), CAST(',' AS VARCHAR)), ',') + 1) AS remaining_extras FROM pizza_clean_customer_orders UNION ALL SELECT order_id, TRIM(SUBSTR(remaining_extras, 1, STRPOS(remaining_extras, ',') - 1)) AS each_extra, SUBSTR(remaining_extras, STRPOS(remaining_extras, ',') + 1) FROM split_extras WHERE remaining_extras <> '') SELECT order_id, COUNT(each_extra) AS total_extras FROM split_extras GROUP BY order_id), calculate_totals AS (SELECT t1.order_id, t1.pizza_id, SUM(CASE WHEN pizza_id = 1 THEN 12 WHEN pizza_id = 2 THEN 10 END) AS total_price, t3.total_extras FROM pizza_clean_customer_orders AS t1 JOIN pizza_clean_runner_orders AS t2 ON t2.order_id = t1.order_id LEFT JOIN get_extras_count AS t3 ON t3.order_id = t1.order_id WHERE t2.cancellation IS NULL GROUP BY t1.order_id, t1.pizza_id, t3.total_extras) SELECT SUM(total_price) + SUM(total_extras) AS total_income FROM calculate_totals" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "modern_data", + "catalog": "modern_data", + "external_knowledge": null + } + }, + { + "id": "local075", + "input": { + "query": "Can you provide a breakdown of how many times each product was viewed, how many times they were added to the shopping cart, and how many times they were left in the cart without being purchased? Also, give me the count of actual purchases for each product. Ensure that products with a page id in (1, 2, 12, 13) are filtered out." + }, + "expected_output": { + "sql": "WITH product_viewed AS (SELECT t1.page_id, SUM(CASE WHEN event_type = 1 THEN 1 ELSE 0 END) AS n_page_views, SUM(CASE WHEN event_type = 2 THEN 1 ELSE 0 END) AS n_added_to_cart FROM shopping_cart_page_hierarchy AS t1 JOIN shopping_cart_events AS t2 ON t1.page_id = t2.page_id WHERE NOT t1.product_id IS NULL GROUP BY t1.page_id), product_purchased AS (SELECT t2.page_id, SUM(CASE WHEN event_type = 2 THEN 1 ELSE 0 END) AS purchased_from_cart FROM shopping_cart_page_hierarchy AS t1 JOIN shopping_cart_events AS t2 ON t1.page_id = t2.page_id WHERE NOT t1.product_id IS NULL AND EXISTS(SELECT visit_id FROM shopping_cart_events WHERE event_type = 3 AND t2.visit_id = visit_id) AND NOT t1.page_id IN (1, 2, 12, 13) GROUP BY t2.page_id), product_abandoned AS (SELECT t2.page_id, SUM(CASE WHEN event_type = 2 THEN 1 ELSE 0 END) AS abandoned_in_cart FROM shopping_cart_page_hierarchy AS t1 JOIN shopping_cart_events AS t2 ON t1.page_id = t2.page_id WHERE NOT t1.product_id IS NULL AND NOT EXISTS(SELECT visit_id FROM shopping_cart_events WHERE event_type = 3 AND t2.visit_id = visit_id) AND NOT t1.page_id IN (1, 2, 12, 13) GROUP BY t2.page_id) SELECT t1.page_id, t1.page_name, t2.n_page_views AS \"number of product being viewed\", t2.n_added_to_cart AS \"number added to the cart\", t4.abandoned_in_cart AS \"without being purchased in cart\", t3.purchased_from_cart AS \"count of actual purchases\" FROM shopping_cart_page_hierarchy AS t1 JOIN product_viewed AS t2 ON t2.page_id = t1.page_id JOIN product_purchased AS t3 ON t3.page_id = t1.page_id JOIN product_abandoned AS t4 ON t4.page_id = t1.page_id" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "bank_sales_trading", + "catalog": "bank_sales_trading", + "external_knowledge": null + } + }, + { + "id": "local078", + "input": { + "query": "Identify the top 10 and bottom 10 interest categories based on their highest composition values across all months. For each category, display the time(MM-YYYY), interest name, and the composition value" + }, + "expected_output": { + "sql": "WITH get_interest_rank AS (SELECT t1.month_year, t2.interest_name, t1.composition, RANK() OVER (PARTITION BY t2.interest_name ORDER BY t1.composition DESC NULLS FIRST) AS interest_rank FROM interest_metrics AS t1 JOIN interest_map AS t2 ON t1.interest_id = t2.id WHERE NOT t1.month_year IS NULL), get_top_10 AS (SELECT month_year, interest_name, composition FROM get_interest_rank WHERE interest_rank = 1 ORDER BY composition DESC NULLS FIRST LIMIT 10), get_bottom_10 AS (SELECT month_year, interest_name, composition FROM get_interest_rank WHERE interest_rank = 1 ORDER BY composition ASC LIMIT 10) SELECT * FROM get_top_10 UNION SELECT * FROM get_bottom_10 ORDER BY composition DESC NULLS FIRST" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "bank_sales_trading", + "catalog": "bank_sales_trading", + "external_knowledge": null + } + }, + { + "id": "local099", + "input": { + "query": "I need you to look into the actor collaborations and tell me how many actors have made more films with Yash Chopra than with any other director. This will help us understand his influence on the industry better." + }, + "expected_output": { + "sql": "WITH YASH_CHOPRAS_PID AS (SELECT TRIM(P.PID) AS PID FROM Person AS P WHERE TRIM(P.Name) = 'Yash Chopra'), NUM_OF_MOV_BY_ACTOR_DIRECTOR AS (SELECT TRIM(MC.PID) AS ACTOR_PID, TRIM(MD.PID) AS DIRECTOR_PID, COUNT(DISTINCT TRIM(MD.MID)) AS NUM_OF_MOV FROM M_Cast AS MC JOIN M_Director AS MD ON TRIM(MC.MID) = TRIM(MD.MID) GROUP BY ACTOR_PID, DIRECTOR_PID), NUM_OF_MOVIES_BY_YC AS (SELECT NM.ACTOR_PID, NM.DIRECTOR_PID, NM.NUM_OF_MOV AS NUM_OF_MOV_BY_YC FROM NUM_OF_MOV_BY_ACTOR_DIRECTOR AS NM JOIN YASH_CHOPRAS_PID AS YCP ON NM.DIRECTOR_PID = YCP.PID), MAX_MOV_BY_OTHER_DIRECTORS AS (SELECT ACTOR_PID, MAX(NUM_OF_MOV) AS MAX_NUM_OF_MOV FROM NUM_OF_MOV_BY_ACTOR_DIRECTOR AS NM JOIN YASH_CHOPRAS_PID AS YCP ON NM.DIRECTOR_PID <> YCP.PID GROUP BY ACTOR_PID), ACTORS_MOV_COMPARISION AS (SELECT NMY.ACTOR_PID, CASE WHEN NMY.NUM_OF_MOV_BY_YC > COALESCE(NMO.MAX_NUM_OF_MOV, 0) THEN 'Y' ELSE 'N' END AS MORE_MOV_BY_YC FROM NUM_OF_MOVIES_BY_YC AS NMY LEFT OUTER JOIN MAX_MOV_BY_OTHER_DIRECTORS AS NMO ON NMY.ACTOR_PID = NMO.ACTOR_PID) SELECT COUNT(DISTINCT TRIM(P.PID)) AS \"Number of actor\" FROM Person AS P WHERE TRIM(P.PID) IN (SELECT DISTINCT ACTOR_PID FROM ACTORS_MOV_COMPARISION WHERE MORE_MOV_BY_YC = 'Y')" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "Db-IMDB", + "catalog": "db-imdb", + "external_knowledge": null + } + }, + { + "id": "local131", + "input": { + "query": "Could you list each musical style with the number of times it appears as a 1st, 2nd, or 3rd preference in a single row per style?" + }, + "expected_output": { + "sql": "SELECT Musical_Styles.StyleName, COUNT(RankedPreferences.FirstStyle) AS FirstPreference, COUNT(RankedPreferences.SecondStyle) AS SecondPreference, COUNT(RankedPreferences.ThirdStyle) AS ThirdPreference FROM Musical_Styles, (SELECT (CASE WHEN Musical_Preferences.PreferenceSeq = 1 THEN Musical_Preferences.StyleID ELSE NULL END) AS FirstStyle, (CASE WHEN Musical_Preferences.PreferenceSeq = 2 THEN Musical_Preferences.StyleID ELSE NULL END) AS SecondStyle, (CASE WHEN Musical_Preferences.PreferenceSeq = 3 THEN Musical_Preferences.StyleID ELSE NULL END) AS ThirdStyle FROM Musical_Preferences) AS RankedPreferences WHERE Musical_Styles.StyleID = RankedPreferences.FirstStyle OR Musical_Styles.StyleID = RankedPreferences.SecondStyle OR Musical_Styles.StyleID = RankedPreferences.ThirdStyle GROUP BY StyleID, StyleName HAVING COUNT(FirstStyle) > 0 OR COUNT(SecondStyle) > 0 OR COUNT(ThirdStyle) > 0 ORDER BY FirstPreference DESC NULLS FIRST, SecondPreference DESC NULLS FIRST, ThirdPreference DESC NULLS FIRST, StyleID" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "EntertainmentAgency", + "catalog": "entertainmentagency", + "external_knowledge": null + } + }, + { + "id": "local163", + "input": { + "query": "Which university faculty members' salaries are closest to the average salary for their respective ranks? Please provide the ranks, first names, last names, and salaries.university" + }, + "expected_output": { + "sql": "WITH AvgSalaries AS (SELECT facrank AS FacRank, AVG(facsalary) AS AvSalary FROM university_faculty GROUP BY facrank), SalaryDifferences AS (SELECT university_faculty.facrank AS FacRank, university_faculty.facfirstname AS FacFirstName, university_faculty.faclastname AS FacLastName, university_faculty.facsalary AS Salary, ABS(university_faculty.facsalary - AvgSalaries.AvSalary) AS Diff FROM university_faculty JOIN AvgSalaries ON university_faculty.facrank = AvgSalaries.FacRank), MinDifferences AS (SELECT FacRank, MIN(Diff) AS MinDiff FROM SalaryDifferences GROUP BY FacRank) SELECT s.FacRank, s.FacFirstName, s.FacLastName, s.Salary FROM SalaryDifferences AS s JOIN MinDifferences AS m ON s.FacRank = m.FacRank AND s.Diff = m.MinDiff" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "education_business", + "catalog": "education_business", + "external_knowledge": null + } + }, + { + "id": "local197", + "input": { + "query": "Among our top 10 paying customers, can you identify the largest change in payment amounts from one month to the immediately following month? Specifically, please determine for which customer and during which month this maximum month-over-month difference occurred, and provide the difference rounded to two decimal places." + }, + "expected_output": { + "sql": "WITH result_table AS (SELECT STRFTIME('%m', pm.payment_date) AS pay_mon, customer_id, COUNT(pm.amount) AS pay_countpermon, SUM(pm.amount) AS pay_amount FROM payment AS pm GROUP BY pay_mon, customer_id), top10_customer AS (SELECT customer_id, SUM(tb.pay_amount) AS total_payments FROM result_table AS tb GROUP BY customer_id ORDER BY SUM(tb.pay_amount) DESC NULLS FIRST LIMIT 10), difference_per_mon AS (SELECT pay_mon AS month_number, pay_mon AS month, tb.pay_countpermon, tb.pay_amount, ABS(tb.pay_amount - LAG(tb.pay_amount) OVER (PARTITION BY tb.customer_id)) AS diff FROM result_table AS tb JOIN top10_customer AS top ON top.customer_id = tb.customer_id) SELECT month, ROUND(max_diff, 2) AS max_diff FROM (SELECT month, diff, month_number, MAX(diff) OVER (PARTITION BY month) AS max_diff FROM difference_per_mon) AS max_per_mon WHERE diff = max_diff ORDER BY max_diff DESC NULLS FIRST LIMIT 1" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "sqlite-sakila", + "catalog": "sqlite-sakila", + "external_knowledge": null + } + }, + { + "id": "local199", + "input": { + "query": "Can you identify the year and month with the highest rental orders created by the store's staff for each store? Please list the store ID, the year, the month, and the total rentals for those dates." + }, + "expected_output": { + "sql": "WITH result_table AS (SELECT STRFTIME('%Y', RE.RENTAL_DATE) AS YEAR, STRFTIME('%m', RE.RENTAL_DATE) AS RENTAL_MONTH, ST.STORE_ID, COUNT(RE.RENTAL_ID) AS count FROM RENTAL AS RE JOIN STAFF AS ST ON RE.STAFF_ID = ST.STAFF_ID GROUP BY YEAR, RENTAL_MONTH, ST.STORE_ID), monthly_sales AS (SELECT YEAR, RENTAL_MONTH, STORE_ID, SUM(count) AS total_rentals FROM result_table GROUP BY YEAR, RENTAL_MONTH, STORE_ID), store_max_sales AS (SELECT STORE_ID, YEAR, RENTAL_MONTH, total_rentals, MAX(total_rentals) OVER (PARTITION BY STORE_ID) AS max_rentals FROM monthly_sales) SELECT STORE_ID, YEAR, RENTAL_MONTH, total_rentals FROM store_max_sales WHERE total_rentals = max_rentals ORDER BY STORE_ID" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "sqlite-sakila", + "catalog": "sqlite-sakila", + "external_knowledge": null + } + }, + { + "id": "local210", + "input": { + "query": "Can you identify the hubs that saw more than a 20% increase in finished orders from February to March?" + }, + "expected_output": { + "sql": "WITH february_orders AS (SELECT h.hub_name AS hub_name, COUNT(*) AS orders_february FROM orders AS o LEFT JOIN stores AS s ON o.store_id = s.store_id LEFT JOIN hubs AS h ON s.hub_id = h.hub_id WHERE o.order_created_month = 2 AND o.order_status = 'FINISHED' GROUP BY h.hub_name), march_orders AS (SELECT h.hub_name AS hub_name, COUNT(*) AS orders_march FROM orders AS o LEFT JOIN stores AS s ON o.store_id = s.store_id LEFT JOIN hubs AS h ON s.hub_id = h.hub_id WHERE o.order_created_month = 3 AND o.order_status = 'FINISHED' GROUP BY h.hub_name) SELECT fo.hub_name FROM february_orders AS fo LEFT JOIN march_orders AS mo ON fo.hub_name = mo.hub_name WHERE fo.orders_february > 0 AND mo.orders_march > 0 AND (CAST((mo.orders_march - fo.orders_february) AS REAL) / CAST(fo.orders_february AS REAL)) > 0.2 /* Filter for hubs with more than a 20% increase */" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "delivery_center", + "catalog": "delivery_center", + "external_knowledge": null + } + }, + { + "id": "local219", + "input": { + "query": "In each league, considering all seasons, which single team has the fewest total match wins based on comparing home and away goals, including teams with zero wins, ensuring that if multiple teams tie for the fewest wins, only one team is returned for each league?" + }, + "expected_output": { + "sql": "WITH match_view AS (SELECT M.id, L.name AS league, M.season, M.match_api_id, T.team_long_name AS home_team, TM.team_long_name AS away_team, M.home_team_goal, M.away_team_goal, P1.player_name AS home_gk, P2.player_name AS home_center_back_1, P3.player_name AS home_center_back_2, P4.player_name AS home_right_back, P5.player_name AS home_left_back, P6.player_name AS home_midfield_1, P7.player_name AS home_midfield_2, P8.player_name AS home_midfield_3, P9.player_name AS home_midfield_4, P10.player_name AS home_second_forward, P11.player_name AS home_center_forward, P12.player_name AS away_gk, P13.player_name AS away_center_back_1, P14.player_name AS away_center_back_2, P15.player_name AS away_right_back, P16.player_name AS away_left_back, P17.player_name AS away_midfield_1, P18.player_name AS away_midfield_2, P19.player_name AS away_midfield_3, P20.player_name AS away_midfield_4, P21.player_name AS away_second_forward, P22.player_name AS away_center_forward, M.goal, M.card FROM match AS M LEFT JOIN league AS L ON M.league_id = L.id LEFT JOIN team AS T ON M.home_team_api_id = T.team_api_id LEFT JOIN team AS TM ON M.away_team_api_id = TM.team_api_id LEFT JOIN player AS P1 ON M.home_player_1 = P1.player_api_id LEFT JOIN player AS P2 ON M.home_player_2 = P2.player_api_id LEFT JOIN player AS P3 ON M.home_player_3 = P3.player_api_id LEFT JOIN player AS P4 ON M.home_player_4 = P4.player_api_id LEFT JOIN player AS P5 ON M.home_player_5 = P5.player_api_id LEFT JOIN player AS P6 ON M.home_player_6 = P6.player_api_id LEFT JOIN player AS P7 ON M.home_player_7 = P7.player_api_id LEFT JOIN player AS P8 ON M.home_player_8 = P8.player_api_id LEFT JOIN player AS P9 ON M.home_player_9 = P9.player_api_id LEFT JOIN player AS P10 ON M.home_player_10 = P10.player_api_id LEFT JOIN player AS P11 ON M.home_player_11 = P11.player_api_id LEFT JOIN player AS P12 ON M.away_player_1 = P12.player_api_id LEFT JOIN player AS P13 ON M.away_player_2 = P13.player_api_id LEFT JOIN player AS P14 ON M.away_player_3 = P14.player_api_id LEFT JOIN player AS P15 ON M.away_player_4 = P15.player_api_id LEFT JOIN player AS P16 ON M.away_player_5 = P16.player_api_id LEFT JOIN player AS P17 ON M.away_player_6 = P17.player_api_id LEFT JOIN player AS P18 ON M.away_player_7 = P18.player_api_id LEFT JOIN player AS P19 ON M.away_player_8 = P19.player_api_id LEFT JOIN player AS P20 ON M.away_player_9 = P20.player_api_id LEFT JOIN player AS P21 ON M.away_player_10 = P21.player_api_id LEFT JOIN player AS P22 ON M.away_player_11 = P22.player_api_id), match_score AS (/* Displaying teams and their goals as home_team */ SELECT id, home_team AS team, CASE WHEN home_team_goal > away_team_goal THEN 1 ELSE 0 END AS Winning_match FROM match_view UNION ALL /* Displaying teams and their goals as away_team */ SELECT id, away_team AS team, CASE WHEN away_team_goal > home_team_goal THEN 1 ELSE 0 END AS Winning_match FROM match_view), winning_matches AS (/* Displaying total match wins for each team */ SELECT MV.league, M.team, COUNT(CASE WHEN M.Winning_match = 1 THEN 1 END) AS wins, ROW_NUMBER() OVER (PARTITION BY MV.league ORDER BY COUNT(CASE WHEN M.Winning_match = 1 THEN 1 END) ASC) AS rn FROM match_score AS M JOIN match_view AS MV ON M.id = MV.id GROUP BY MV.league, team ORDER BY league, wins ASC) SELECT league, team FROM winning_matches WHERE rn = 1 /* Getting the team with the least number of wins in each league */ ORDER BY league" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "EU_soccer", + "catalog": "eu_soccer", + "external_knowledge": null + } + }, + { + "id": "local301", + "input": { + "query": "For weekly-sales data, I need an analysis of our sales performance around mid-June for the years 2018, 2019, and 2020. Specifically, calculate the percentage change in sales between the four weeks leading up to June 15 and the four weeks following June 15 for each year." + }, + "expected_output": { + "sql": "SELECT before_effect, after_effect, after_effect - before_effect AS change_amount, ROUND(((CAST(after_effect * 1.0 AS DOUBLE) / before_effect) - 1) * 100, 2) AS percent_change, '2018' AS year FROM (SELECT SUM(CASE WHEN delta_weeks BETWEEN 1 AND 4 THEN sales END) AS after_effect, SUM(CASE WHEN delta_weeks BETWEEN -3 AND 0 THEN sales END) AS before_effect FROM (SELECT week_date, ROUND(CAST((JULIANDAY(week_date) - JULIANDAY('2018-06-15')) AS DOUBLE) / 7.0) + 1 AS delta_weeks, sales FROM cleaned_weekly_sales) AS add_delta_weeks) AS add_before_after UNION ALL SELECT before_effect, after_effect, after_effect - before_effect AS change_amount, ROUND(((CAST(after_effect * 1.0 AS DOUBLE) / before_effect) - 1) * 100, 2) AS percent_change, '2019' AS year FROM (SELECT SUM(CASE WHEN delta_weeks BETWEEN 1 AND 4 THEN sales END) AS after_effect, SUM(CASE WHEN delta_weeks BETWEEN -3 AND 0 THEN sales END) AS before_effect FROM (SELECT week_date, ROUND(CAST((JULIANDAY(week_date) - JULIANDAY('2019-06-15')) AS DOUBLE) / 7.0) + 1 AS delta_weeks, sales FROM cleaned_weekly_sales) AS add_delta_weeks) AS add_before_after UNION ALL SELECT before_effect, after_effect, after_effect - before_effect AS change_amount, ROUND(((CAST(after_effect * 1.0 AS DOUBLE) / before_effect) - 1) * 100, 2) AS percent_change, '2020' AS year FROM (SELECT SUM(CASE WHEN delta_weeks BETWEEN 1 AND 4 THEN sales END) AS after_effect, SUM(CASE WHEN delta_weeks BETWEEN -3 AND 0 THEN sales END) AS before_effect FROM (SELECT week_date, ROUND(CAST((JULIANDAY(week_date) - JULIANDAY('2020-06-15')) AS DOUBLE) / 7.0) + 1 AS delta_weeks, sales FROM cleaned_weekly_sales) AS add_delta_weeks) AS add_before_after ORDER BY year" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "bank_sales_trading", + "catalog": "bank_sales_trading", + "external_knowledge": null + } + }, + { + "id": "local309", + "input": { + "query": "For each year, which driver and which constructor scored the most points? I want the full name of each driver." + }, + "expected_output": { + "sql": "WITH year_points AS (SELECT races.year, CONCAT(CAST(drivers.forename AS VARCHAR), CAST(' ' AS VARCHAR), CAST(drivers.surname AS VARCHAR)) AS driver, constructors.name AS constructor, SUM(results.points) AS points FROM results LEFT JOIN races ON results.race_id = races.race_id /* Ensure these columns exist in your schema */ LEFT JOIN drivers ON results.driver_id = drivers.driver_id /* Ensure these columns exist in your schema */ LEFT JOIN constructors ON results.constructor_id = constructors.constructor_id /* Ensure these columns exist in your schema */ GROUP BY races.year, driver UNION SELECT races.year, NULL AS driver, constructors.name AS constructor, SUM(results.points) AS points FROM results LEFT JOIN races ON results.race_id = races.race_id /* Ensure these columns exist in your schema */ LEFT JOIN drivers ON results.driver_id = drivers.driver_id /* Ensure these columns exist in your schema */ LEFT JOIN constructors ON results.constructor_id = constructors.constructor_id /* Ensure these columns exist in your schema */ GROUP BY races.year, constructor), max_points AS (SELECT year, MAX(CASE WHEN NOT driver IS NULL THEN points ELSE NULL END) AS max_driver_points, MAX(CASE WHEN NOT constructor IS NULL THEN points ELSE NULL END) AS max_constructor_points FROM year_points GROUP BY year) SELECT max_points.year, drivers_year_points.driver, constructors_year_points.constructor FROM max_points LEFT JOIN year_points AS drivers_year_points ON max_points.year = drivers_year_points.year AND max_points.max_driver_points = drivers_year_points.points AND NOT drivers_year_points.driver IS NULL LEFT JOIN year_points AS constructors_year_points ON max_points.year = constructors_year_points.year AND max_points.max_constructor_points = constructors_year_points.points AND NOT constructors_year_points.constructor IS NULL ORDER BY max_points.year" + }, + "metadata": { + "difficulty": "complex", + "question_type": "join", + "source": "spider2_lite", + "db": "f1", + "catalog": "f1", + "external_knowledge": null + } + } +] \ No newline at end of file diff --git a/backend/app/sync_om_metadata.py b/backend/app/sync_om_metadata.py new file mode 100644 index 0000000..ee7da5a --- /dev/null +++ b/backend/app/sync_om_metadata.py @@ -0,0 +1,1021 @@ +#!/usr/bin/env python3 +""" +Sync OpenMetadata metadata into the app's `tables` rows. + +Direction: OpenMetadata -> App DB. OM is the source of truth for which +tables exist, including their canonical name/catalog/schema casing. + +Uses OpenMetadata's REST API directly (via `requests`) instead of the +`openmetadata-ingestion` SDK, because that SDK pins sqlalchemy<2 while +sqlmodel (used by this app's ORM) requires sqlalchemy>=2 -- the two cannot +coexist in one environment. This script only needs read-only GET calls, so +the SDK isn't necessary here. + +For every table found in OpenMetadata under the `local_trino` service +(the sole service used -- it already federates every Snowflake database +as its own Trino catalog via generate_trino_catalogs.py, so a separate +native Snowflake_Prod service would just duplicate the same data): + - If a matching row already exists in the app DB `tables` table it is + updated. Matching is done first by exact (service, catalog, + schema_name, name), then -- if that misses -- CASE-INSENSITIVELY + against every existing app row, since a row that differs from OM + only by case is the same table, not a new one. `openmetadata_json` + is refreshed (and `embedding`, if --recompute-embeddings is + passed), and the row's service/catalog/schema_name/name are + realigned to OM's current casing. + - If no matching row exists (not even case-insensitively), a NEW row + is inserted with: + id = freshly generated UUID + service/catalog/schema_name/name = from OpenMetadata + status = "sandbox" if catalog == "minio" (the genuine local + demo data), else "production" (everything else is a + Trino-federated Snowflake catalog, i.e. spider2-snow + data) + owner_id = "system" + oasis_source_id = the OpenMetadata table's own entity id (om_table["id"]) + openmetadata_json = synced payload + embedding = always computed for new rows (there's no prior + embedding to preserve), regardless of whether + --recompute-embeddings was passed + +If more than one existing app row matches an OM table case-insensitively +(a duplicate), the row whose oasis_source_id exactly matches the OM +table's own entity id is preferred as canonical; otherwise the lowest-id +row is picked deterministically. The other row(s) are left in place and +reported as duplicates -- pass --merge-case-duplicates to delete them. + +Rows whose oasis_source_id starts with a protected prefix (currently just +"airlines." -- these are seeded directly into Postgres by infra_init.py's +_ensure_airlines_registered(), bypassing OM entirely) are never considered +for ghost/duplicate reporting, even though they'll never have an OM match. + +Rows that exist in the app DB but have NO OpenMetadata match under their +own exact (service, catalog, schema, name) OR case-insensitively against +every table OM actually has (across all TARGET_SERVICES) are "ghost" rows. +This is also how any old rows tagged with a now-retired service (e.g. a +former Snowflake_Prod) get surfaced: since it's no longer in +TARGET_SERVICES, those rows will never match anything. Ghost rows are +reported in the logs; this script does not delete them. + +Commits happen in batches (BATCH_SIZE rows) rather than one commit for the +whole run. If a batch fails (e.g. a constraint violation on one row), that +batch is rolled back and retried row-by-row so only the actual bad row(s) +are skipped -- everything already committed in prior batches stays in the +DB, and everything else in the failed batch still gets saved. The same +batching applies to duplicate-row merging. + +By default this does NOT recompute embeddings for EXISTING rows (only +openmetadata_json is touched for them). Pass --recompute-embeddings to also +regenerate the embedding column for existing rows from the freshly-synced +description + columns, using the same embedding pipeline as the seed +script. New rows always get an embedding computed, since they don't have +one yet. + +CATALOG COVERAGE: this script can only sync what OpenMetadata already +knows about, which in turn can only be what Trino currently has catalogs +loaded for. If you've recently run scripts/generate_trino_catalogs.py to +add more Snowflake databases, make sure you (1) restarted Trino so it +picks up the new .properties files, and (2) re-triggered (or waited for) +the OpenMetadata ingestion pipeline, BEFORE running this script -- otherwise +this script will happily run to completion while only seeing a fraction of +your intended catalogs. This script now logs the distinct catalogs it sees +from OM on every run (see "OM catalog coverage" below) specifically so that +gap is visible immediately instead of requiring separate debugging. + +Requires in .env: + OPENMETADATA_URL, OM_JWT_TOKEN + Whatever core.db.engine / app.config.settings already need + (DB connection, EMBEDDER_URL, EMBEDDER_MODEL, EMBEDDER_KEY) + +Run with: + uv run scripts/sync_om_metadata.py [--recompute-embeddings] [--merge-case-duplicates] +""" + +import argparse +import json +import logging +import os +import re +import uuid + +import requests +import sqlglot +from core.db.engine import engine +from core.embeddings import EXPECTED_EMBEDDING_DIM, get_embedding as core_get_embedding +from core.models.models import EnrichmentVersion, Table as AppTable +from dotenv import load_dotenv +from sqlmodel import Session, col, select + +from app.config import settings + +logging.basicConfig( + level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s" +) +logger = logging.getLogger("OM_Metadata_Sync") + +load_dotenv(".env") + +OM_URL = os.environ.get("OPENMETADATA_URL", "http://localhost:8585/api").rstrip("/") +OM_JWT_TOKEN = os.environ.get("OPENMETADATA_TOKEN") +QUERY_LIMIT_PER_TABLE = int(os.environ.get("OM_SAMPLE_QUERIES_PER_TABLE", "5")) +PAGE_SIZE = 100 + +# Pull everything under this OM service -- no filtering. local_trino is the +# sole source now: it already federates every Snowflake database as its own +# Trino catalog (see generate_trino_catalogs.py), so there's no need for a +# separate native Snowflake_Prod service -- that would just duplicate data +# already visible here. +TARGET_SERVICES = ["local_trino"] + +# The one catalog under local_trino that is genuinely local demo data (per +# infra_init.py's own DDL). Every other catalog under local_trino is a +# Trino-federated mirror of a Snowflake database (see +# generate_trino_catalogs.py) -- i.e. spider2-snow data. Used to set +# status: anything NOT in this set is treated as "from spider" -> production. +LOCAL_ONLY_CATALOGS = {"minio"} + +# Rows whose oasis_source_id starts with one of these prefixes were seeded +# directly into Postgres by infra_init.py (bypassing OM entirely -- see +# _ensure_airlines_registered()) and must never be treated as ghosts/dupes, +# even though they'll never have an OM match. +PROTECTED_OASIS_SOURCE_ID_PREFIXES = ("airlines.",) + +SYSTEM_OWNER_ID = "system" + +# Commit in batches so a single problem row only costs that batch, not the +# entire run (a single failed INSERT rolls back everything since the last +# commit -- keeping this small limits the blast radius). +BATCH_SIZE = 200 + +# ── Spider2-Snow golden questions: fetched live from GitHub ──────────────── +# +# Previously golden questions were read from a local spider2_questions.json +# file. That file had gone stale (it actually contained Spider2-Lite +# questions, not Spider2-Snow), and being a local file it's also fragile -- +# if it moves or gets renamed, load_golden_questions() silently does nothing. +# Fetching directly from the same GitHub source the evaluation service uses +# removes that whole class of problem: there's no local file to drift out of +# sync or go missing. See the evaluation service's Spider2SnowDownloader for +# the canonical version of this logic (this is a lightweight standalone copy +# since this script lives in a different project/repo). +SPIDER2_SNOW_JSONL_URL = ( + "https://raw.githubusercontent.com/xlang-ai/Spider2/main/" + "spider2-snow/spider2-snow.jsonl" +) +SPIDER2_SNOW_GOLD_SQL_BASE_URL = ( + "https://raw.githubusercontent.com/xlang-ai/Spider2/main/" + "spider2-snow/evaluation_suite/gold/sql/" +) +GITHUB_TIMEOUT = 15 # seconds per request + + +def normalize_key( + service: str | None, catalog: str | None, schema: str | None, name: str | None +) -> tuple: + """Case-insensitive matching key. OM is the source of truth for casing, + but its reported casing for a table can drift between syncs, so exact + string matching alone isn't reliable for deciding "is this the same + table" -- only for deciding what to store/display.""" + return ( + (service or "").lower(), + (catalog or "").lower(), + (schema or "").lower(), + (name or "").lower(), + ) + + +def om_get(path: str, params: dict | None = None) -> dict: + if not OM_JWT_TOKEN: + raise ValueError("OM_JWT_TOKEN not found in env. Set it in your .env file.") + resp = requests.get( + f"{OM_URL}/v1{path}", + headers={"Authorization": f"Bearer {OM_JWT_TOKEN}"}, + params=params or {}, + timeout=30, + ) + resp.raise_for_status() + return resp.json() + + +def fetch_om_tables_by_service(service_name: str) -> dict: + """ + Fetch OM tables for a given service, keyed by (catalog, schema, name). + + Defensive: OM's `service=` filter on /v1/tables is not reliably strict -- + it can return tables belonging to a different actual service. So we also + request the `service` field on each table and verify it actually matches + `service_name` before indexing it. Mismatches are dropped and logged + rather than silently mis-tagged with the wrong service in our DB. + """ + logger.info(f"Fetching OpenMetadata tables for service '{service_name}'...") + index = {} + after = None + fields = "columns,description,tags,owners,databaseSchema,database,service" + + while True: + params = {"service": service_name, "fields": fields, "limit": PAGE_SIZE} + if after: + params["after"] = after + + page = om_get("/tables", params=params) + for t in page.get("data", []): + actual_service = (t.get("service") or {}).get("name") + if actual_service != service_name: + logger.warning( + f"OM returned a table under service filter '{service_name}' but its actual " + f"service is '{actual_service}' (fqn={t.get('fullyQualifiedName')}) -- skipping. " + f"Add '{actual_service}' to TARGET_SERVICES if it should be synced." + ) + continue + + catalog = (t.get("database") or {}).get("name") + schema = (t.get("databaseSchema") or {}).get("name") + name = t.get("name") + index[(catalog, schema, name)] = t + + after = page.get("paging", {}).get("after") + if not after: + break + + logger.info( + f"Found {len(index)} tables in OpenMetadata for service '{service_name}' (after verification)." + ) + return index + + +def fetch_sample_queries(table_fqn: str, limit: int) -> list[str]: + try: + resp = om_get( + "/queries", + params={"entityFQN": table_fqn, "limit": limit, "fields": "query"}, + ) + return [item["query"] for item in resp.get("data", []) if item.get("query")] + except Exception as e: + logger.warning(f"Could not fetch sample queries for '{table_fqn}': {e}") + return [] + + +def build_om_payload(om_table: dict) -> dict: + columns = [ + { + "name": col.get("name"), + "dataType": col.get("dataTypeDisplay") or col.get("dataType"), + "description": col.get("description"), + "tags": col.get("tags"), + } + for col in om_table.get("columns", []) + ] + + table_fqn = om_table.get("fullyQualifiedName") + sample_queries = ( + fetch_sample_queries(table_fqn, QUERY_LIMIT_PER_TABLE) if table_fqn else [] + ) + + return { + "fqn": table_fqn, + "description": om_table.get("description"), + "owners": om_table.get("owners"), + "tags": om_table.get("tags"), + "columns": columns, + "sample_queries": sample_queries, + } + + +def get_embedding_text(description: str, columns: list[dict]) -> str: + col_names = ", ".join(c["name"] for c in columns if c.get("name")) + return f"Description: {description or ''}. Columns: {col_names}" + + +def get_embedding(text: str) -> list[float]: + emb = core_get_embedding( + text=text, + embedder_url=settings.EMBEDDER_URL, + embedder_model=settings.EMBEDDER_MODEL, + embedder_key=settings.EMBEDDER_KEY, + ) + if emb is None: + logger.warning("Error getting embedding for text, falling back to zero-vector") + return [0.0] * EXPECTED_EMBEDDING_DIM + return emb + + +def status_for_catalog(catalog: str | None) -> str: + # Anything other than the genuine local demo catalog (minio) is a + # Trino-federated Snowflake catalog, i.e. spider2-snow data -> production. + return "sandbox" if (catalog or "").lower() in LOCAL_ONLY_CATALOGS else "production" + + +def flush_delete_batch(session: Session, batch: list[tuple]) -> tuple[int, int]: + """ + Commit a batch of (obj, key_str) pairs that were session.delete()'d. + Same pattern as flush_batch: if the bulk commit fails, roll back and + retry each deletion individually so one bad row doesn't block the rest. + + Returns (succeeded, failed) counts. + """ + if not batch: + return 0, 0 + + try: + session.commit() + return len(batch), 0 + except Exception as e: + session.rollback() + logger.error( + f"Batch delete failed ({len(batch)} rows) -- retrying individually. Batch error: {e}" + ) + succeeded = 0 + failed = 0 + for obj, key_str in batch: + try: + session.delete(obj) + session.commit() + succeeded += 1 + except Exception as row_e: + session.rollback() + logger.error(f"Failed to delete row {key_str}: {row_e}") + failed += 1 + return succeeded, failed + + +def flush_batch(session: Session, batch: list[tuple]) -> tuple[int, int]: + """ + Commit a batch of (obj, key_str) pairs. If the bulk commit fails, roll + back and retry each row individually so we only lose the actual bad + row(s) instead of the whole batch. + + Returns (succeeded, failed) counts. + """ + if not batch: + return 0, 0 + + try: + session.commit() + return len(batch), 0 + except Exception as e: + session.rollback() + logger.error( + f"Batch commit failed ({len(batch)} rows) -- retrying rows individually " + f"to isolate the bad one(s). Batch error: {e}" + ) + succeeded = 0 + failed = 0 + for obj, key_str in batch: + try: + session.add(obj) + session.commit() + succeeded += 1 + except Exception as row_e: + session.rollback() + logger.error(f"Skipping row {key_str} -- failed to commit: {row_e}") + failed += 1 + return succeeded, failed + + +def extract_tables_from_sql(sql: str) -> list[str]: + # Regex to find table names in FROM or JOIN clauses + matches = re.findall( + r'\b(?:from|join|update|into)\s+([a-zA-Z0-9_"\.]+)', sql, re.IGNORECASE + ) + tables = [] + for match in matches: + parts = [p.replace('"', "").lower().strip() for p in match.split(".")] + if parts: + tables.append(parts[-1]) + return tables + + +def sync_spider_schemas(session: Session, force: bool = False): + spider_tables = session.exec( + select(AppTable).where(AppTable.owner_id == "spider2") + ).all() + logger.info( + f"Syncing schemas (EnrichmentVersion) for {len(spider_tables)} Spider2-Snow tables..." + ) + synced_schemas = 0 + failed_schemas = 0 + + for table in spider_tables: + try: + data = table.openmetadata_json or {} + description = data.get("description") or "" + om_columns = data.get("columns") or [] + + if not om_columns and not description: + continue + + def parse_columns(cols): + parsed = [] + for c in cols: + col_def = { + "name": c.get("name"), + "description": c.get("description") or "", + "dataType": c.get("dataType"), + "is_geo": False, + "is_time": False, + } + if "children" in c: + col_def["children"] = parse_columns(c["children"]) + parsed.append(col_def) + return parsed + + columns = parse_columns(om_columns) + + existing_enrichment = session.exec( + select(EnrichmentVersion) + .where(EnrichmentVersion.table_id == table.id) + .order_by(col(EnrichmentVersion.version).desc()) + ).first() + + if existing_enrichment and existing_enrichment.data and not force: + existing_data = existing_enrichment.data + if ( + existing_data.get("table_description") == description + and existing_data.get("columns") == columns + ): + continue + + next_version = ( + (existing_enrichment.version + 1) if existing_enrichment else 1 + ) + new_enrichment = EnrichmentVersion( + table_id=table.id, + version=next_version, + data={"table_description": description, "columns": columns}, + ) + session.add(new_enrichment) + synced_schemas += 1 + except Exception as e: + logger.error( + f"Failed to sync schema for table {table.name} ({table.id}): {e}" + ) + failed_schemas += 1 + + session.commit() + logger.info(f"Schema sync completed: {synced_schemas} ok, {failed_schemas} failed.") + + +def _fetch_spider2_snow_gold_sql(instance_id: str) -> str | None: + """Download the gold SQL file for a single Spider2-Snow instance.""" + url = f"{SPIDER2_SNOW_GOLD_SQL_BASE_URL}{instance_id}.sql" + try: + resp = requests.get(url, timeout=GITHUB_TIMEOUT) + resp.raise_for_status() + return resp.text.strip() + except Exception as exc: + logger.debug(f"No gold SQL available for {instance_id}: {exc}") + return None + + +def _translate_snowflake_to_trino(snowflake_sql: str, instance_id: str) -> str: + """Translate gold SQL from Snowflake to Trino dialect via sqlglot.""" + try: + results = sqlglot.transpile(snowflake_sql, read="snowflake", write="trino") + if results: + trino_sql = results[0] + # Clean up sqlglot's comma + CROSS JOIN UNNEST formatting bug. + trino_sql = trino_sql.replace(", CROSS JOIN UNNEST", " CROSS JOIN UNNEST") + trino_sql = trino_sql.replace(", CROSS JOIN UNNEST", " CROSS JOIN UNNEST") + trino_sql = trino_sql.replace(", cross join unnest", " cross join unnest") + trino_sql = trino_sql.replace(", cross join unnest", " cross join unnest") + return trino_sql + except Exception as exc: + logger.warning(f"SQL translation failed for {instance_id}: {exc}") + # Fall back to the original SQL -- Trino may still accept it as-is. + return snowflake_sql + + +def fetch_spider2_snow_questions() -> list[dict]: + """ + Fetch Spider2-Snow benchmark questions straight from the xlang-ai/Spider2 + GitHub repo, with gold SQL translated from Snowflake to Trino dialect. + + Returns a list of dicts shaped like the old spider2_questions.json + entries did, so load_golden_questions() below needs no other changes: + {"input": {"query": ...}, "expected_output": {"sql": ...}, + "metadata": {"db": ..., "difficulty": ..., "question_type": ...}} + """ + logger.info(f"Fetching Spider2-Snow questions from {SPIDER2_SNOW_JSONL_URL} ...") + resp = requests.get(SPIDER2_SNOW_JSONL_URL, timeout=GITHUB_TIMEOUT) + resp.raise_for_status() + + raw_questions = [] + for line in resp.text.splitlines(): + line = line.strip() + if not line: + continue + try: + raw_questions.append(json.loads(line)) + except json.JSONDecodeError: + continue + + # Only Snowflake-derived questions (sf_ prefix) belong to Spider2-Snow. + sf_questions = [ + q for q in raw_questions if q.get("instance_id", "").startswith("sf_") + ] + logger.info( + f"spider2-snow.jsonl: {len(raw_questions)} total rows, " + f"{len(sf_questions)} sf_ (Snowflake) questions." + ) + + # Diagnostic: how many distinct Snowflake databases (db_id) does the + # full sf_ question set reference? Compared against the "OM catalog + # coverage" log emitted in main() below, this tells you at a glance + # what fraction of the benchmark's databases you've actually federated. + distinct_db_ids = sorted( + {q.get("db_id", "") for q in sf_questions if q.get("db_id")} + ) + logger.info( + f"spider2-snow.jsonl sf_ questions reference {len(distinct_db_ids)} distinct db_id(s): " + f"{distinct_db_ids}" + ) + + questions: list[dict] = [] + skipped_no_gold = 0 + for q in sf_questions: + instance_id = q["instance_id"] + db = q.get("db_id", "") + + gold_sql = _fetch_spider2_snow_gold_sql(instance_id) + if not gold_sql: + skipped_no_gold += 1 + continue + + trino_sql = _translate_snowflake_to_trino(gold_sql, instance_id) + + questions.append( + { + "id": instance_id, + "input": {"query": q["instruction"]}, + "expected_output": {"sql": trino_sql}, + "metadata": { + "db": db, + "difficulty": "complex", + "question_type": "join", + }, + } + ) + + logger.info( + f"Fetched {len(questions)} Spider2-Snow questions with gold SQL from GitHub " + f"({skipped_no_gold} skipped -- no released gold SQL on GitHub for that instance; " + f"this is expected, the maintainers only release gold SQL for a subset of instances)." + ) + return questions + + +def load_golden_questions(session: Session): + try: + questions = fetch_spider2_snow_questions() + except Exception as e: + logger.error(f"Failed to fetch Spider2-Snow questions from GitHub: {e}") + return + + if not questions: + logger.warning( + "No Spider2-Snow questions were fetched from GitHub -- nothing to load." + ) + return + + tables = session.exec(select(AppTable).where(AppTable.owner_id == "spider2")).all() + tables_by_catalog = {} + for t in tables: + tables_by_catalog.setdefault(t.catalog.lower(), []).append(t) + + logger.info( + f"App DB currently has {len(tables_by_catalog)} distinct spider2 catalog(s) " + f"available to match golden questions against: {sorted(tables_by_catalog.keys())}" + ) + + from core.models.models import DifficultyLevel, GoldenQuestion, QuestionType + + def get_difficulty(diff_str): + diff_str = str(diff_str).lower().strip() + if diff_str == "medium": + return DifficultyLevel.medium + elif diff_str == "complex": + return DifficultyLevel.complex + return DifficultyLevel.simple + + def get_question_type(q_type_str): + q_type_str = str(q_type_str).lower().strip() + if q_type_str == "join": + return QuestionType.join + elif q_type_str == "geo": + return QuestionType.geo + elif q_type_str == "aggregate": + return QuestionType.aggregate + elif q_type_str == "time_series": + return QuestionType.time_series + return QuestionType.simple + + inserted = 0 + skipped = 0 + failed = 0 + failed_dbs: dict[str, int] = {} + + for item in questions: + question_text = item["input"]["query"] + expected_sql = item["expected_output"]["sql"] + # Same db -> catalog normalization as the evaluation service's + # Spider2SnowDownloader (db.lower(), spaces -> underscores), so + # this lines up with how catalogs are actually named in Trino. + db = item["metadata"]["db"].lower().strip().replace(" ", "_") + difficulty = get_difficulty(item["metadata"].get("difficulty", "simple")) + q_type = get_question_type(item["metadata"].get("question_type", "simple")) + + catalog_tables = tables_by_catalog.get(db, []) + if not catalog_tables: + failed += 1 + failed_dbs[db] = failed_dbs.get(db, 0) + 1 + continue + + ref_tables = extract_tables_from_sql(expected_sql) + target_table = None + + for ref_t in ref_tables: + for t in catalog_tables: + if t.name.lower() == ref_t: + target_table = t + break + if target_table: + break + + if not target_table: + catalog_tables_sorted = sorted(catalog_tables, key=lambda t: t.id) + target_table = catalog_tables_sorted[0] + + existing_q = session.exec( + select(GoldenQuestion) + .where(GoldenQuestion.table_id == target_table.id) + .where(GoldenQuestion.question == question_text) + ).first() + + if existing_q: + skipped += 1 + else: + new_q = GoldenQuestion( + id=str(uuid.uuid4()), + table_id=target_table.id, + question=question_text, + expected_sql=expected_sql, + difficulty=difficulty, + question_type=q_type, + ) + session.add(new_q) + inserted += 1 + + session.commit() + logger.info( + f"Golden questions load completed: {inserted} inserted, {skipped} skipped, {failed} failed (no matching catalog)." + ) + + if failed_dbs: + logger.warning( + f"{len(failed_dbs)} distinct db_id(s) had NO matching app-DB catalog at all " + f"(these catalogs were never synced -- see 'OM catalog coverage' earlier in the " + f"log, or check generate_trino_catalogs.py / Trino restart status): " + f"{sorted(failed_dbs.keys())}" + ) + + tables_with_zero = [] + for t in tables: + q_count = session.exec( + select(GoldenQuestion).where(GoldenQuestion.table_id == t.id) + ).first() + if not q_count: + tables_with_zero.append(f"{t.catalog}.{t.schema_name}.{t.name}") + + if tables_with_zero: + logger.warning( + f"Found {len(tables_with_zero)} Spider2-Snow tables with 0 golden questions: {tables_with_zero}" + ) + + +def main( + recompute_embeddings: bool, + merge_case_duplicates: bool, + sync_schema: bool, + load_questions: bool, +): + with Session(engine) as session: + app_tables = session.exec(select(AppTable)).all() + logger.info(f"Loaded {len(app_tables)} existing rows from app 'tables' table.") + + # Exact-case index (fast path) plus a case-insensitive index (fallback + # path -- and the thing that stops this script from re-inserting the + # same table under a different case every time OM's casing drifts). + app_index = { + (t.service, t.catalog, t.schema_name, t.name): t for t in app_tables + } + app_index_by_norm: dict[tuple, list[AppTable]] = {} + for t in app_tables: + app_index_by_norm.setdefault( + normalize_key(t.service, t.catalog, t.schema_name, t.name), [] + ).append(t) + + matched_row_ids: set[str] = set() + + om_by_service = { + svc: fetch_om_tables_by_service(svc) for svc in TARGET_SERVICES + } + + # ── Catalog coverage diagnostic ───────────────────────────────────── + # Log exactly which catalogs OpenMetadata is reporting *before* we do + # any matching. This is the fastest way to notice "I generated 129 + # Trino catalogs but OM (and this script) only sees 11 of them" -- + # usually because Trino wasn't restarted after new .properties files + # were written, or the OM ingestion pipeline hasn't run since. + om_catalogs_seen = sorted( + { + (catalog or "").lower() + for om_index in om_by_service.values() + for (catalog, _schema, _name) in om_index.keys() + } + ) + logger.info( + f"OM catalog coverage: OpenMetadata currently exposes {len(om_catalogs_seen)} " + f"distinct catalog(s) under {TARGET_SERVICES}: {om_catalogs_seen}" + ) + + updated, inserted, failed = 0, 0, 0 + batch: list[tuple] = [] + + for service, om_index in om_by_service.items(): + for (catalog, schema, name), om_table in om_index.items(): + key = (service, catalog, schema, name) + norm_key = normalize_key(service, catalog, schema, name) + key_str = f"{service}.{catalog}.{schema}.{name}" + payload = build_om_payload(om_table) + + existing = app_index.get(key) + + if existing is None: + # No exact-case match. Fall back to a case-insensitive + # lookup -- OM's casing for this table may simply have + # drifted since the row was first synced, and that's + # still the SAME table, not a new one. + candidates = [ + c + for c in app_index_by_norm.get(norm_key, []) + if c.id not in matched_row_ids + ] + if candidates: + om_id = om_table.get("id") + exact_source_matches = [ + c for c in candidates if c.oasis_source_id == om_id + ] + if exact_source_matches: + existing = exact_source_matches[0] + else: + candidates_sorted = sorted(candidates, key=lambda c: c.id) + existing = candidates_sorted[0] + if len(candidates) > 1: + others = ", ".join( + f"{c.service}.{c.catalog}.{c.schema_name}.{c.name}" + for c in candidates + if c.id != existing.id + ) + logger.warning( + f"{len(candidates)} existing app rows match {key_str} " + f"case-insensitively -- updating " + f"{existing.service}.{existing.catalog}.{existing.schema_name}.{existing.name} " + f"as canonical and leaving pre-existing duplicate(s) [{others}] in place " + f"(pass --merge-case-duplicates to remove them)." + ) + + is_spider = ( + (catalog or "").lower() not in {"minio", "airlines", "admin_db"} + and (schema or "").lower() != "information_schema" + and not (om_table.get("id") or "").startswith( + PROTECTED_OASIS_SOURCE_ID_PREFIXES + ) + ) + + # Only sync tables whose owner would be "spider2". + # Skip system tables (minio, airlines, admin_db catalogs, + # information_schema schema) entirely -- no update, no insert. + if not is_spider: + logger.debug(f"Skipping system table {key_str}") + continue + + if existing is not None: + if (existing.oasis_source_id or "").startswith( + PROTECTED_OASIS_SOURCE_ID_PREFIXES + ): + logger.info(f"Skipping protected table {key_str}") + continue + + matched_row_ids.add(existing.id) + # Realign casing to OM's current, canonical values. This + # is what prevents the next run from treating this same + # row as a miss and inserting yet another duplicate. + if ( + existing.service, + existing.catalog, + existing.schema_name, + existing.name, + ) != key: + logger.info( + f"Realigning casing: " + f"{existing.service}.{existing.catalog}.{existing.schema_name}.{existing.name} " + f"-> {key_str}" + ) + existing.service = service + existing.catalog = catalog + existing.schema_name = schema + existing.name = name + existing.openmetadata_json = payload + existing.owner_id = "spider2" + if recompute_embeddings: + embed_text = get_embedding_text( + payload["description"], payload["columns"] + ) + embedding = get_embedding(embed_text) + if embedding is None or all(v == 0.0 for v in embedding): + logger.warning( + f"Embedding fell back to zero-vector for {key_str}" + ) + existing.embedding = embedding + session.add(existing) + batch.append((existing, key_str)) + logger.info(f"Queued update for {key_str}") + else: + embed_text = get_embedding_text( + payload["description"], payload["columns"] + ) + embedding = get_embedding(embed_text) + if embedding is None or all(v == 0.0 for v in embedding): + logger.warning( + f"Embedding fell back to zero-vector for {key_str}" + ) + new_row = AppTable( + id=str(uuid.uuid4()), + name=name, + schema_name=schema, + catalog=catalog, + service=service, + status=status_for_catalog(catalog), + owner_id="spider2", + oasis_source_id=om_table.get("id"), + openmetadata_json=payload, + embedding=embedding, + ) + session.add(new_row) + matched_row_ids.add(new_row.id) + batch.append((new_row, key_str)) + logger.info( + f"Queued insert for {key_str} " + f"(status={new_row.status}, owner_id={new_row.owner_id}, " + f"oasis_source_id={new_row.oasis_source_id})" + ) + + if len(batch) >= BATCH_SIZE: + succeeded, batch_failed = flush_batch(session, batch) + failed += batch_failed + logger.info( + f"Committed batch: {succeeded} ok, {batch_failed} failed" + ) + batch = [] + + # Flush any remaining rows in the final partial batch. + succeeded, batch_failed = flush_batch(session, batch) + failed += batch_failed + if succeeded or batch_failed: + logger.info(f"Committed final batch: {succeeded} ok, {batch_failed} failed") + + # Rows in the app DB that no OM table matched this run (not even + # case-insensitively) -- candidates for ghost cleanup. Duplicates + # (a second+ row that matched an OM table case-insensitively but + # wasn't picked as canonical) are excluded here -- they're handled + # separately below, since "no OM match at all" and "OM has this + # table but we already have two rows for it" are different problems. + unmatched = [ + t + for t in app_tables + if t.id not in matched_row_ids + and not (t.oasis_source_id or "").startswith( + PROTECTED_OASIS_SOURCE_ID_PREFIXES + ) + ] + + normalized_om_keys = set() + for om_index in om_by_service.values(): + for (catalog, schema, name), om_table in om_index.items(): + actual_service = (om_table.get("service") or {}).get("name") or "" + normalized_om_keys.add( + normalize_key(actual_service, catalog, schema, name) + ) + + ghost_rows = [ + t + for t in unmatched + if normalize_key(t.service, t.catalog, t.schema_name, t.name) + not in normalized_om_keys + ] + # Rows that DID match an OM table's normalized key but were excluded + # from matched_row_ids because another row was already picked as + # canonical for that same table this run -- i.e. genuine pre-existing + # duplicates, not ghosts. + duplicate_rows = [t for t in unmatched if t not in ghost_rows] + + # matched_row_ids includes both updated existing rows and newly + # inserted rows; split them back out by checking which ids existed + # before this run started. + pre_run_ids = {t.id for t in app_tables} + updated = len([rid for rid in matched_row_ids if rid in pre_run_ids]) + inserted = len([rid for rid in matched_row_ids if rid not in pre_run_ids]) + + if not ghost_rows: + logger.info("No ghost rows found.") + else: + logger.warning( + f"Found {len(ghost_rows)} ghost row(s) with no OM match under any casing/service:" + ) + for t in ghost_rows[:20]: + logger.warning(f" {t.service}.{t.catalog}.{t.schema_name}.{t.name}") + if len(ghost_rows) > 20: + logger.warning(f" ...and {len(ghost_rows) - 20} more") + + merged, merge_failed = 0, 0 + + if not duplicate_rows: + logger.info("No pre-existing case-duplicate rows found.") + elif not merge_case_duplicates: + logger.warning( + f"Found {len(duplicate_rows)} pre-existing duplicate row(s) that match an OM table " + f"case-insensitively but weren't kept as the canonical row this run. NOT deleting -- " + f"pass --merge-case-duplicates to remove them. Sample:" + ) + for t in duplicate_rows[:20]: + logger.warning( + f" would delete: {t.service}.{t.catalog}.{t.schema_name}.{t.name}" + ) + if len(duplicate_rows) > 20: + logger.warning(f" ...and {len(duplicate_rows) - 20} more") + else: + merge_batch: list[tuple] = [] + for t in duplicate_rows: + key_str = f"{t.service}.{t.catalog}.{t.schema_name}.{t.name}" + session.delete(t) + merge_batch.append((t, key_str)) + logger.info(f"Merging (deleting) duplicate row: {key_str}") + if len(merge_batch) >= BATCH_SIZE: + succeeded, batch_failed = flush_delete_batch(session, merge_batch) + merged += succeeded + merge_failed += batch_failed + merge_batch = [] + succeeded, batch_failed = flush_delete_batch(session, merge_batch) + merged += succeeded + merge_failed += batch_failed + + logger.info( + f"\nDone. Updated {updated} existing rows, inserted {inserted} new rows, " + f"{failed} rows failed to commit (see errors above). " + f"Ghost rows (no OM match, reported only): {len(ghost_rows)}. " + f"Case duplicates: {len(duplicate_rows)} found, {merged} merged/deleted, " + f"{merge_failed} merge failures (pass --merge-case-duplicates to enable; without it, " + f"they are reported only)." + ) + + # Task 2: Schema Sync for Spider2-Snow tables + sync_spider_schemas(session, force=sync_schema) + + # Task 3: Golden Questions loading + if load_questions: + load_golden_questions(session) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--recompute-embeddings", + action="store_true", + help="Also regenerate the embedding column for EXISTING rows from the freshly-synced " + "description/columns. New rows always get an embedding computed on insert.", + ) + parser.add_argument( + "--merge-case-duplicates", + action="store_true", + help="Delete pre-existing duplicate rows that match an OM table case-insensitively but " + "were not kept as the canonical row (e.g. leftovers from before case-insensitive " + "matching was added). Without this flag, duplicates are only reported.", + ) + parser.add_argument( + "--sync-schema", + action="store_true", + help="Force re-syncing of schemas (EnrichmentVersion) even if openmetadata_json columns are unchanged.", + ) + parser.add_argument( + "--load-golden-questions", + action="store_true", + help="Fetch Spider2-Snow golden questions + gold SQL live from GitHub and link them to " + "Spider2-Snow tables in the DB.", + ) + args = parser.parse_args() + main( + recompute_embeddings=args.recompute_embeddings, + merge_case_duplicates=args.merge_case_duplicates, + sync_schema=args.sync_schema, + load_questions=args.load_golden_questions, + ) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 226e152..28bfddb 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -26,6 +26,7 @@ dependencies = [ "core", "mcp>=1.2.0", "python-core-utils[keycloak] @ git+ssh://git@github.com/matzpen-agency/python-core-utils.git@1.2.0#egg=python-core-utils", + "sqlglot>=30.12.0", ] [dependency-groups] diff --git a/backend/uv.lock b/backend/uv.lock index 9c4330b..a84987f 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -1963,6 +1963,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e2/22/dbf013a12ec759e54a34a119e9e217435b3f71b2dd5c61a7ade0a25dae87/sqlalchemy-2.0.51-py3-none-any.whl", hash = "sha256:bb024d8b621d0be75f4f44ecc7c950450026e76d66dc8f791bb5331d7fed59d5", size = 1944334, upload-time = "2026-06-15T16:09:22.418Z" }, ] +[[package]] +name = "sqlglot" +version = "30.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/ed/a6c45aec29353b6392ea34548c40af3ac6ffd6bc5572cf23b2ce250876fc/sqlglot-30.12.0.tar.gz", hash = "sha256:6b8369704662d4f654bc934cea4dd31c916c2a571b389210cb9e951a275e5fd9", size = 5905110, upload-time = "2026-06-26T14:09:40.408Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/9e/82a390ecc85f066ff80affa01d195f744e3de60ad4d695b8de31c9a66da3/sqlglot-30.12.0-py3-none-any.whl", hash = "sha256:86cccc610073c645c03e72b55b60ae0518aa3253a7fc3bd56551370d003c6554", size = 707583, upload-time = "2026-06-26T14:09:38.525Z" }, +] + [[package]] name = "sqlmodel" version = "0.0.22" @@ -2024,6 +2033,7 @@ dependencies = [ { name = "python-jose", extra = ["cryptography"] }, { name = "python-multipart" }, { name = "requests" }, + { name = "sqlglot" }, { name = "sqlmodel" }, { name = "trino" }, { name = "uvicorn", extra = ["standard"] }, @@ -2063,6 +2073,7 @@ requires-dist = [ { name = "python-jose", extras = ["cryptography"], specifier = "==3.3.0" }, { name = "python-multipart", specifier = "==0.0.20" }, { name = "requests", specifier = "==2.32.3" }, + { name = "sqlglot", specifier = ">=30.12.0" }, { name = "sqlmodel", specifier = "==0.0.22" }, { name = "trino", specifier = "==0.328.0" }, { name = "uvicorn", extras = ["standard"], specifier = "==0.32.1" }, diff --git a/docker-compose.yml b/docker-compose.yml index f6d1f46..e9336fa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -471,6 +471,9 @@ services: timeout: 5s retries: 5 start_period: 300s + networks: + - default + - shared-net agent: build: @@ -542,4 +545,8 @@ volumes: secrets: deploy_key: - file: ./deploy_key \ No newline at end of file + file: ./deploy_key + +networks: + shared-net: + external: true \ No newline at end of file diff --git a/frontend/src/components/monitoring/RunHistoryTable.tsx b/frontend/src/components/monitoring/RunHistoryTable.tsx index 5930420..ba985c9 100644 --- a/frontend/src/components/monitoring/RunHistoryTable.tsx +++ b/frontend/src/components/monitoring/RunHistoryTable.tsx @@ -312,9 +312,15 @@ interface RunHistoryTableProps { tableId?: string; limit?: number; compact?: boolean; + excludeTableIds?: Set; } -export function RunHistoryTable({ tableId, limit = 50, compact = false }: RunHistoryTableProps) { +export function RunHistoryTable({ + tableId, + limit = 50, + compact = false, + excludeTableIds, +}: RunHistoryTableProps) { const [selectedRunId, setSelectedRunId] = useState(null); const [page, setPage] = useState(0); const pageSize = compact ? 5 : 10; @@ -334,8 +340,12 @@ export function RunHistoryTable({ tableId, limit = 50, compact = false }: RunHis }, }); - const paged = runs.slice(page * pageSize, (page + 1) * pageSize); - const totalPages = Math.ceil(runs.length / pageSize); + const visibleRuns = excludeTableIds?.size + ? runs.filter((r) => !r.table_id || !excludeTableIds.has(r.table_id)) + : runs; + + const paged = visibleRuns.slice(page * pageSize, (page + 1) * pageSize); + const totalPages = Math.ceil(visibleRuns.length / pageSize); if (isLoading) return ( @@ -451,8 +461,8 @@ export function RunHistoryTable({ tableId, limit = 50, compact = false }: RunHis {totalPages > 1 && (
- Showing {page * pageSize + 1}–{Math.min((page + 1) * pageSize, runs.length)} of{' '} - {runs.length} runs + Showing {page * pageSize + 1}–{Math.min((page + 1) * pageSize, visibleRuns.length)} of{' '} + {visibleRuns.length} runs
{/* Table */} @@ -161,7 +210,7 @@ export function TableList() {
- {data.map((table) => ( + {displayedData.map((table) => (
- {run.id.slice(0, 8)}… + + {run.id.slice(0, 8)}… + {run.triggered_by} diff --git a/frontend/src/components/tables/TableList.css b/frontend/src/components/tables/TableList.css index b2411e8..2cf78df 100644 --- a/frontend/src/components/tables/TableList.css +++ b/frontend/src/components/tables/TableList.css @@ -37,15 +37,33 @@ .table-name-cell { font-weight: 600; color: var(--accent-hover); + max-width: 180px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + display: inline-block; + vertical-align: middle; } .table-schema-code { font-size: 12px; color: var(--text-muted); + max-width: 140px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + display: inline-block; + vertical-align: middle; } .table-owner-cell { color: var(--text-secondary); + max-width: 120px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + display: inline-block; + vertical-align: middle; } .table-updated-cell { diff --git a/frontend/src/components/tables/TableList.tsx b/frontend/src/components/tables/TableList.tsx index 9c356cd..2fc90aa 100644 --- a/frontend/src/components/tables/TableList.tsx +++ b/frontend/src/components/tables/TableList.tsx @@ -164,21 +164,33 @@ export function TableList() { {data.map((table) => (
- {table.name} + + {table.name} + - {table.service} + + {table.service} + - {table.catalog} + + {table.catalog} + - {table.schema_name} + + {table.schema_name} + {table.owner_id} + + {table.owner_id} + + {dayjs(table.updated_at).format('MMM D, YYYY HH:mm')}
@@ -206,6 +255,25 @@ export function TableList() { ))}
+ {data.length > visibleCount && ( +
+ +
+ )} )} diff --git a/frontend/src/pages/EvaluationsPage.css b/frontend/src/pages/EvaluationsPage.css index 8bfc789..202e9ec 100644 --- a/frontend/src/pages/EvaluationsPage.css +++ b/frontend/src/pages/EvaluationsPage.css @@ -1,5 +1,43 @@ /* EvaluationsPage styles */ +/* Spider2 visibility toggle inside the filters row */ +.table-filters__spider2-toggle { + display: inline-flex; + align-items: center; + gap: 5px; + padding: 0 12px; + height: 32px; + border: 1px solid var(--border); + border-radius: var(--radius-sm); + background: transparent; + color: var(--text-muted); + font-size: 12.5px; + font-weight: 500; + cursor: pointer; + white-space: nowrap; + flex-shrink: 0; + transition: + color 0.15s, + border-color 0.15s, + background 0.15s; +} + +.table-filters__spider2-toggle:hover:not(:disabled) { + color: var(--text-primary); + border-color: var(--accent); +} + +.table-filters__spider2-toggle:disabled { + opacity: 0.65; + cursor: default; +} + +.table-filters__spider2-toggle--active { + color: var(--accent); + border-color: var(--accent); + background: rgba(var(--accent-rgb, 99, 102, 241), 0.08); +} + .run-trigger-panel__actions { display: flex; gap: 8px; @@ -11,13 +49,158 @@ padding: 24px; } +.run-trigger-panel__count { + font-size: 11.5px; + color: var(--text-muted); + margin: 2px 2px 10px; +} + .run-trigger-panel__grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(220px, 1fr)); gap: 8px; margin-bottom: 16px; + max-height: 480px; + overflow-y: auto; + padding: 2px; +} + +/* ── Filters ── */ + +.table-filters { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 14px; + padding: 12px 0 16px; + margin: 4px 0 4px; + border-bottom: 1px solid var(--border); +} + +.table-filters__search { + display: flex; + align-items: center; + gap: 8px; + flex: 1 1 240px; + min-width: 200px; + padding: 6px 10px; + border-radius: 7px; + background: rgba(255, 255, 255, 0.03); + border: 1px solid var(--border); + color: var(--text-muted); + transition: border-color 0.18s ease; +} + +.table-filters__search:focus-within { + border-color: var(--accent); + color: var(--text-primary); +} + +.table-filters__search input { + flex: 1; + border: none; + outline: none; + background: transparent; + font-size: 13px; + color: var(--text-primary); +} + +.table-filters__search input::placeholder { + color: var(--text-muted); +} + +.table-filters__clear-icon { + display: flex; + align-items: center; + border: none; + background: transparent; + color: var(--text-muted); + cursor: pointer; + padding: 2px; +} + +.table-filters__clear-icon:hover { + color: var(--text-primary); +} + +.table-filters__group { + display: flex; + align-items: center; + gap: 8px; + flex: 0 1 220px; + min-width: 180px; +} + +.table-filters__group-label { + font-size: 10.5px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--text-muted); + white-space: nowrap; +} + +.table-filters__select { + flex: 1; + min-width: 0; +} + +.table-filters__select .ant-select-selector { + background: rgba(255, 255, 255, 0.03) !important; + border: 1px solid var(--border) !important; + border-radius: 7px !important; + box-shadow: none !important; +} + +.table-filters__select .ant-select-selection-placeholder, +.table-filters__select .ant-select-selection-item { + font-size: 12.5px; + color: var(--text-primary); +} + +.table-filters__select .ant-select-selection-item { + background: var(--accent-dim) !important; + border: 1px solid var(--accent) !important; + color: var(--accent-hover) !important; + border-radius: 5px !important; +} + +.table-filters__select:hover .ant-select-selector { + border-color: var(--accent) !important; +} + +.table-filters__select.ant-select-focused .ant-select-selector { + border-color: var(--accent) !important; + box-shadow: 0 0 0 2px var(--accent-dim) !important; +} + +.table-filters__select.ant-select-disabled .ant-select-selector { + opacity: 0.5; } +.table-filters__select-dropdown { + background: var(--bg-base) !important; + border: 1px solid var(--border) !important; + border-radius: 8px !important; +} + +.table-filters__select-dropdown .ant-select-item { + color: var(--text-primary); + font-size: 12.5px; + border-radius: 5px; +} + +.table-filters__select-dropdown .ant-select-item-option-selected { + background: var(--accent-dim) !important; + color: var(--accent-hover) !important; +} + +.table-filters__select-dropdown .ant-select-item-option-active { + background: rgba(255, 255, 255, 0.05) !important; +} + +/* ── Table cards ── */ + .table-card { padding: 10px 12px; border-radius: 8px; @@ -27,11 +210,26 @@ transition: all 0.18s ease; } +.table-card:hover { + border-color: var(--accent); +} + .table-card--selected { background: var(--accent-dim); border-color: var(--accent); } +.table-card--incomplete { + opacity: 0.6; + cursor: not-allowed; + border-color: rgba(239, 68, 68, 0.3); + background: rgba(239, 68, 68, 0.04); +} + +.table-card--incomplete:hover { + border-color: rgba(239, 68, 68, 0.5); +} + .table-card__header { display: flex; align-items: center; @@ -43,23 +241,48 @@ width: 7px; height: 7px; border-radius: 50%; + flex-shrink: 0; } .table-card__name { font-size: 13px; font-weight: 600; color: var(--text-primary); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; } .table-card__name--selected { color: var(--accent-hover); } +.table-card__incomplete-badge { + margin-left: auto; + font-size: 10px; + font-weight: 700; + color: #ef4444; + background: rgba(239, 68, 68, 0.12); + padding: 1px 6px; + border-radius: 4px; + white-space: nowrap; + flex-shrink: 0; +} + .table-card__info { font-size: 11.5px; color: var(--text-muted); } +.table-card__missing { + margin-top: 4px; + font-size: 10.5px; + color: #ef4444; + opacity: 0.9; +} + +/* ── Banners ── */ + .launch-success-banner { padding: 10px 14px; border-radius: 8px; @@ -92,40 +315,6 @@ cursor: pointer; } -/* ── Incomplete table cards ── */ - -.table-card--incomplete { - opacity: 0.6; - cursor: not-allowed; - border-color: rgba(239, 68, 68, 0.3); - background: rgba(239, 68, 68, 0.04); -} - -.table-card--incomplete:hover { - border-color: rgba(239, 68, 68, 0.5); -} - -.table-card__incomplete-badge { - margin-left: auto; - font-size: 10px; - font-weight: 700; - color: #ef4444; - background: rgba(239, 68, 68, 0.12); - padding: 1px 6px; - border-radius: 4px; - white-space: nowrap; - flex-shrink: 0; -} - -.table-card__missing { - margin-top: 4px; - font-size: 10.5px; - color: #ef4444; - opacity: 0.9; -} - -/* ── Dataset running notice ── */ - .dataset-running-notice { display: flex; align-items: flex-start; @@ -145,6 +334,15 @@ color: #818cf8; } +@media (prefers-reduced-motion: reduce) { + .dataset-running-notice, + .table-card, + .table-filters__chip { + animation: none; + transition: none; + } +} + @keyframes pulse-border { 0%, 100% { diff --git a/frontend/src/pages/EvaluationsPage.tsx b/frontend/src/pages/EvaluationsPage.tsx index c9c936e..b8ae934 100644 --- a/frontend/src/pages/EvaluationsPage.tsx +++ b/frontend/src/pages/EvaluationsPage.tsx @@ -1,7 +1,18 @@ -import { useState } from 'react'; +import { useEffect, useMemo, useState, useTransition } from 'react'; import { useQuery } from '@tanstack/react-query'; -import { App } from 'antd'; -import { CalendarClock, Check, Database, History, Loader2, PlayCircle } from 'lucide-react'; +import { App, Select } from 'antd'; +import { + CalendarClock, + Check, + Database, + Eye, + EyeOff, + History, + Loader2, + PlayCircle, + Search, + X, +} from 'lucide-react'; import { orchestrationApi } from '../api/orchestration'; import { EmptySlate, SectionHeader, Spinner } from '../components/common/EvalUI'; @@ -20,19 +31,184 @@ import './EvaluationsPage.css'; type Tab = 'history' | 'schedules' | 'run'; +const ALL_STATUSES = ['production', 'sandbox', 'verified', 'draft', 'degraded']; + +// ── Table filters ─────────────────────────────────────────────────────────── +function TableFilters({ + search, + setSearch, + statusOptions, + activeStatuses, + setActiveStatuses, + ownerOptions, + activeOwners, + setActiveOwners, + onClear, + showSpider2, + onToggleSpider2, + spider2Pending, +}: { + search: string; + setSearch: (v: string) => void; + statusOptions: string[]; + activeStatuses: string[]; + setActiveStatuses: (v: string[]) => void; + ownerOptions: string[]; + activeOwners: string[]; + setActiveOwners: (v: string[]) => void; + onClear: () => void; + showSpider2: boolean; + onToggleSpider2: () => void; + spider2Pending: boolean; +}) { + return ( +
+
+ + setSearch(e.target.value)} + /> + {search && ( + + )} +
+ +
+ Status + ({ label: owner, value: owner }))} + disabled={!ownerOptions.length} + maxTagCount={2} + /> +
+ + + + +
+ ); +} + // ── Run trigger panel ────────────────────────────────────────────────────────── -function RunTriggerPanel({ onLaunch }: { onLaunch?: () => void }) { +function RunTriggerPanel({ + onLaunch, + showSpider2, + onToggleSpider2, + spider2Pending, +}: { + onLaunch?: () => void; + showSpider2: boolean; + onToggleSpider2: () => void; + spider2Pending: boolean; +}) { const [selectedTableIds, setSelectedTableIds] = useState([]); const [triggeredBy] = useState('user'); const [launched, setLaunched] = useState(false); + const [search, setSearch] = useState(''); + const [activeStatuses, setActiveStatuses] = useState([]); + const [activeOwners, setActiveOwners] = useState([]); + const [visibleCount, setVisibleCount] = useState(50); + const [sentinel, setSentinel] = useState(null); const { message } = App.useApp(); + useEffect(() => { + setVisibleCount(50); + }, [search, activeStatuses, activeOwners, showSpider2]); + + useEffect(() => { + if (!sentinel) return; + const observer = new IntersectionObserver( + (entries) => { + if (entries[0].isIntersecting) { + setVisibleCount((prev) => prev + 50); + } + }, + { rootMargin: '100px' }, + ); + observer.observe(sentinel); + return () => observer.disconnect(); + }, [sentinel]); + const { data: tables = [], isLoading: tablesLoading } = useTables(); const { data: readiness = {} } = useEvalReadiness(); const triggerMut = useTriggerOrchestrationRun(); + const statusOptions = useMemo( + () => Array.from(new Set([...ALL_STATUSES, ...tables.map((t: Table) => t.status)])), + [tables], + ); + + const ownerOptions = useMemo( + () => + Array.from( + new Set(tables.map((t: Table) => (t as any).owner_id).filter(Boolean)), + ).sort() as string[], + [tables], + ); + + const filteredTables = useMemo(() => { + const q = search.trim().toLowerCase(); + return tables.filter((t: Table) => { + if (!showSpider2 && (t as any).owner_id === 'spider2') return false; + if (activeStatuses.length > 0 && !activeStatuses.includes(t.status)) return false; + const owner = (t as any).owner_id; + if (activeOwners.length > 0 && !activeOwners.includes(owner)) return false; + if (!q) return true; + const serviceCatalog = (t as any).service_catalog ?? (t as any).serviceCatalog ?? ''; + const haystack = `${t.name} ${t.schema_name} ${serviceCatalog}`.toLowerCase(); + return haystack.includes(q); + }); + }, [tables, search, activeStatuses, activeOwners, showSpider2]); + + const displayedTables = useMemo(() => { + return filteredTables.slice(0, visibleCount); + }, [filteredTables, visibleCount]); + const handleLaunch = () => { triggerMut.mutate( { tableIds: selectedTableIds, triggeredBy }, @@ -61,9 +237,15 @@ function RunTriggerPanel({ onLaunch }: { onLaunch?: () => void }) { ); }; + const clearFilters = () => { + setSearch(''); + setActiveStatuses([]); + setActiveOwners([]); + }; + const selectAll = () => setSelectedTableIds( - tables.filter((t: Table) => readiness[t.id]?.ready !== false).map((t: Table) => t.id), + filteredTables.filter((t: Table) => readiness[t.id]?.ready !== false).map((t: Table) => t.id), ); const clearAll = () => setSelectedTableIds([]); @@ -85,6 +267,23 @@ function RunTriggerPanel({ onLaunch }: { onLaunch?: () => void }) { } /> + {!tablesLoading && tables.length > 0 && ( + + )} + {tablesLoading ? (
@@ -95,55 +294,84 @@ function RunTriggerPanel({ onLaunch }: { onLaunch?: () => void }) { title="No tables found" sub="Add tables first via the Tables section" /> + ) : !filteredTables.length ? ( + } + title="No tables match your filters" + sub="Try adjusting the search or filters above" + /> ) : ( -
- {tables.map((t: Table) => { - const selected = selectedTableIds.includes(t.id); - const tableReadiness = readiness[t.id]; - const isIncomplete = tableReadiness !== undefined && !tableReadiness.ready; - const statusColor: Record = { - production: '#10b981', - sandbox: '#f59e0b', - verified: '#3b82f6', - draft: '#64748b', - degraded: '#ef4444', - }; - const sc = statusColor[t.status] ?? '#64748b'; - return ( -
toggleTable(t.id)} - className={`table-card${selected ? ' table-card--selected' : ''}${isIncomplete ? ' table-card--incomplete' : ''}`} - title={isIncomplete ? `Missing: ${tableReadiness.missing.join(', ')}` : undefined} - > -
-
- - {t.name} - - {isIncomplete && ( + <> +
+ Showing {filteredTables.length} of {tables.length} tables +
+
+ {displayedTables.map((t: Table) => { + const selected = selectedTableIds.includes(t.id); + const tableReadiness = readiness[t.id]; + const isIncomplete = tableReadiness !== undefined && !tableReadiness.ready; + const statusColor: Record = { + production: '#10b981', + sandbox: '#f59e0b', + verified: '#3b82f6', + draft: '#64748b', + degraded: '#ef4444', + }; + const sc = statusColor[t.status] ?? '#64748b'; + return ( +
toggleTable(t.id)} + className={`table-card${selected ? ' table-card--selected' : ''}${isIncomplete ? ' table-card--incomplete' : ''}`} + title={isIncomplete ? `Missing: ${tableReadiness.missing.join(', ')}` : undefined} + > +
+
- ⚠ Incomplete + {t.name} + {isIncomplete && ( + + ⚠ Incomplete + + )} +
+
+ {t.schema_name} · {t.status} +
+ {isIncomplete && ( +
+ Missing: {tableReadiness.missing.join(', ')} +
)}
-
- {t.schema_name} · {t.status} -
- {isIncomplete && ( -
- Missing: {tableReadiness.missing.join(', ')} -
- )} + ); + })} + {filteredTables.length > visibleCount && ( +
+
- ); - })} -
+ )} +
+ )} {launched && ( @@ -255,6 +483,21 @@ export function EvaluationsPage() { const [activeTab, setActiveTab] = useState('history'); const [runningDataset, setRunningDataset] = useState(null); const [runningRunId, setRunningRunId] = useState(null); + const [showSpider2, setShowSpider2] = useState(false); + const [spider2Pending, startSpider2Transition] = useTransition(); + + const handleToggleSpider2 = () => { + startSpider2Transition(() => { + setShowSpider2((v) => !v); + }); + }; + + // All tables — used to build spider2 table ID set for run history filtering + const { data: allTables = [] } = useTables(); + const spider2TableIds = useMemo( + () => new Set(allTables.filter((t: any) => t.owner_id === 'spider2').map((t: any) => t.id)), + [allTables], + ); // Poll the specific run's details if we have a running run useQuery({ @@ -331,7 +574,10 @@ export function EvaluationsPage() { title="Execution History" sub="All evaluation runs across tables — click any row to view full report" /> - +
)} @@ -339,7 +585,12 @@ export function EvaluationsPage() { {activeTab === 'run' && (
- + { test.beforeEach(async ({ page }) => { diff --git a/frontend/tests/real-agent.spec.ts b/frontend/tests/real-agent.spec.ts index 380efed..ea483cc 100644 --- a/frontend/tests/real-agent.spec.ts +++ b/frontend/tests/real-agent.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { expect,test } from '@playwright/test'; test.describe('Real Agent Execution', () => { test('should run query against real backend', async ({ page }) => { diff --git a/scripts/generate_trino_catalogs.py b/scripts/generate_trino_catalogs.py index ef7316c..0832d4e 100644 --- a/scripts/generate_trino_catalogs.py +++ b/scripts/generate_trino_catalogs.py @@ -8,6 +8,16 @@ Idempotent: skips databases whose catalog file already exists. +IMPORTANT: Trino (file-based catalog config) only loads .properties files +from infra/trino/etc/catalog/ at container STARTUP. Running this script +while Trino is already running will create new files that Trino will NOT +see until it is restarted. After running this script, always: + docker compose restart trino +(or `docker compose up -d --force-recreate trino`) before running +sync_om_metadata.py, or newly-added catalogs will silently be invisible +to OpenMetadata and the app DB, even though the .properties files exist +on disk. + Usage: uv run --with python-dotenv --with snowflake-connector-python scripts/generate_trino_catalogs.py @@ -21,6 +31,7 @@ from pathlib import Path from dotenv import load_dotenv import snowflake.connector +import json logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("generate_trino_catalogs") @@ -88,6 +99,31 @@ def get_snowflake_databases(account: str, user: str, password: str, role: str, w conn.close() +def get_question_referenced_db_ids() -> set[str]: + """ + Fetch the Spider2-Snow sf_ question set and return the distinct db_id + values it references (uppercased, matching Snowflake's SHOW DATABASES + casing). Used to skip generating Trino catalogs for databases that will + never have a single golden question -- no point ingesting/syncing them + at all. + """ + import requests as _requests + url = "https://raw.githubusercontent.com/xlang-ai/Spider2/main/spider2-snow/spider2-snow.jsonl" + resp = _requests.get(url, timeout=15) + resp.raise_for_status() + db_ids = set() + for line in resp.text.splitlines(): + line = line.strip() + if not line: + continue + try: + q = json.loads(line) + except json.JSONDecodeError: + continue + if q.get("instance_id", "").startswith("sf_") and q.get("db_id"): + db_ids.add(q["db_id"].upper()) + return db_ids + # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- @@ -110,15 +146,31 @@ def main() -> None: extra_deny = {db.strip().upper() for db in deny_env.split(",") if db.strip()} deny_list = SYSTEM_DENY | extra_deny - # Fetch ALL live Snowflake databases + # Fetch ALL live Snowflake databases visible to this role/account. If this + # number looks low (e.g. you expected ~129 spider2-snow databases but only + # see a handful), that's a Snowflake grants problem -- this script can + # only ever federate what SHOW DATABASES actually returns for sf_role. all_dbs = get_snowflake_databases(sf_account, sf_user, sf_password, sf_role, sf_warehouse) - logger.info(f"Snowflake reports {len(all_dbs)} total databases.") + logger.info(f"Snowflake reports {len(all_dbs)} total databases visible to role '{sf_role}':") + for db in sorted(all_dbs): + logger.info(f" {db}") - # Filter out system/denied databases — everything else gets a catalog file - target_dbs = [db for db in all_dbs if db.upper() not in deny_list] - logger.info(f"Will ensure {len(target_dbs)} catalog file(s) exist (excluding {len(deny_list)} denied databases).") + referenced_db_ids = get_question_referenced_db_ids() + logger.info(f"Golden questions reference {len(referenced_db_ids)} distinct database(s).") + + target_dbs = [ + db for db in all_dbs + if db.upper() not in deny_list and db.upper() in referenced_db_ids + ] + logger.info( + f"Will ensure {len(target_dbs)} catalog file(s) exist " + f"(restricted to golden-question databases; excluding {len(deny_list)} denied " + f"database(s), and {len([d for d in all_dbs if d.upper() not in deny_list]) - len(target_dbs)} " + f"database(s) with zero golden questions)." + ) created, skipped = 0, 0 + created_names, skipped_names = [], [] for db in sorted(target_dbs): catalog_name = sanitize_catalog_name(db) props_file = CATALOG_DIR / f"{catalog_name}.properties" @@ -126,18 +178,37 @@ def main() -> None: if props_file.exists(): logger.info(f" [SKIP] {props_file.name} (already exists)") skipped += 1 + skipped_names.append(catalog_name) continue content = TEMPLATE.format(database=db) props_file.write_text(content) logger.info(f" [CREATED] {props_file.name} → Snowflake DB '{db}'") created += 1 + created_names.append(catalog_name) + total_files = len(list(CATALOG_DIR.glob('*.properties'))) logger.info( f"\nDone. Created: {created} Skipped (already existed): {skipped} " - f"Total catalog files: {len(list(CATALOG_DIR.glob('*.properties')))}" + f"Total catalog files on disk: {total_files}" ) + if created: + logger.warning( + "\n" + "==================================================================\n" + f"{created} NEW catalog file(s) were just written to {CATALOG_DIR}/.\n" + "Trino only loads catalog .properties files at container STARTUP.\n" + "It will NOT see these new catalogs until you restart it:\n" + "\n" + " docker compose restart trino\n" + "\n" + "Do this BEFORE re-running the OpenMetadata ingestion pipeline or\n" + "scripts/sync_om_metadata.py, or the new catalogs will silently be\n" + "invisible to both and you'll see the same tables/catalogs as before.\n" + "==================================================================" + ) + if __name__ == "__main__": - main() + main() \ No newline at end of file From 4bad16e81abcc2cf1e50f6fda2d36b2d222a28f2 Mon Sep 17 00:00:00 2001 From: Yodan Bargida Date: Mon, 20 Jul 2026 11:31:54 +0300 Subject: [PATCH 09/11] add catalog to table and schema in trino --- backend/app/routers/evaluation.py | 7 +++++-- backend/app/routers/orchestration.py | 5 ++++- core/src/core/trino.py | 25 +++++++++++++++++++------ 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/backend/app/routers/evaluation.py b/backend/app/routers/evaluation.py index 748779b..b02b845 100644 --- a/backend/app/routers/evaluation.py +++ b/backend/app/routers/evaluation.py @@ -157,6 +157,7 @@ def _build_questions_payload(questions: list, table: Table) -> list: "expected_sql": q.expected_sql or "", "table_id": q.table_id, "schema_name": table.schema_name, + "catalog_name": table.catalog, "question_type": q.question_type.value if hasattr(q.question_type, "value") else str(q.question_type), @@ -270,7 +271,9 @@ def execute_single_table_eval(table_id: str, run_id: str, session: Session) -> f try: req = { "dataset_name": dataset_name, - "additional_tables": [table.name], + "additional_tables": [ + f"{table.catalog}.{table.schema_name}.{table.name}" + ], } resp = requests.post( f"{settings.EVALUATION_SERVICE_URL}/text-to-sql/evaluation/run-single-dataset", @@ -426,7 +429,7 @@ def _run_candidate_eval( try: req = { "dataset_name": dataset_name, - "additional_tables": [table.name], + "additional_tables": [f"{table.catalog}.{table.schema_name}.{table.name}"], } resp = requests.post( f"{settings.EVALUATION_SERVICE_URL}/text-to-sql/evaluation/run-single-dataset", diff --git a/backend/app/routers/orchestration.py b/backend/app/routers/orchestration.py index 0eca9c9..5012999 100644 --- a/backend/app/routers/orchestration.py +++ b/backend/app/routers/orchestration.py @@ -297,13 +297,16 @@ def _run_dataset_pipeline(dataset_name: str, run_id: str): prod_tables = session.exec( select(Table).where(Table.owner_id == "spider2") ).all() + table_names = [ + f"{t.catalog}.{t.schema_name}.{t.name}" for t in prod_tables + ] else: prod_tables = session.exec( select(Table) .where(Table.status == TableStatus.production) .where(Table.owner_id != "spider2") ).all() - table_names = [t.name for t in prod_tables] + table_names = [t.name for t in prod_tables] req = { "dataset_name": dataset_name, diff --git a/core/src/core/trino.py b/core/src/core/trino.py index 6e2e8f0..7e1c66f 100644 --- a/core/src/core/trino.py +++ b/core/src/core/trino.py @@ -23,8 +23,15 @@ class TrinoExecutionResult(BaseModel): error_message: str | None = None -def get_trino_connection(): - """Create a real Trino DBAPI connection from settings.""" +def get_trino_connection(catalog: str | None = None, schema: str | None = None): + """Create a real Trino DBAPI connection from settings. + + Args: + catalog: Override the default catalog (settings.TRINO_CATALOG). + Pass a table's specific catalog to run queries in the right context + (e.g. Spider2 Snowflake catalogs instead of the default 'minio'). + schema: Override the default schema (settings.TRINO_SCHEMA). + """ auth = None if settings.TRINO_CERT_PATH and settings.TRINO_KEY_PATH: auth = trino.auth.CertificateAuthentication( @@ -39,8 +46,8 @@ def get_trino_connection(): host=settings.TRINO_HOST, port=settings.TRINO_PORT, user=settings.TRINO_USER, - catalog=settings.TRINO_CATALOG, - schema=settings.TRINO_SCHEMA, + catalog=catalog or settings.TRINO_CATALOG, + schema=schema or settings.TRINO_SCHEMA, http_scheme=settings.TRINO_HTTP_SCHEME, auth=auth, request_timeout=settings.TRINO_REQUEST_TIMEOUT, @@ -48,7 +55,13 @@ def get_trino_connection(): ) -def execute_query_sync(sql: str, table_id: str = "", params: tuple | dict | list | None = None) -> TrinoExecutionResult: +def execute_query_sync( + sql: str, + table_id: str = "", + params: tuple | dict | list | None = None, + catalog: str | None = None, + schema: str | None = None, +) -> TrinoExecutionResult: """ Execute a SQL query against the real Trino cluster. """ @@ -67,7 +80,7 @@ def execute_query_sync(sql: str, table_id: str = "", params: tuple | dict | list conn = None cur = None try: - conn = get_trino_connection() + conn = get_trino_connection(catalog=catalog, schema=schema) cur = conn.cursor() if params is not None: cur.execute(sql, params) From e44b4e5c5b952a9f85bb13e2ee81fa068e49dc3b Mon Sep 17 00:00:00 2001 From: Yodan Bargida Date: Wed, 22 Jul 2026 16:28:57 +0300 Subject: [PATCH 10/11] fix ui bugs --- backend/app/routers/evaluation.py | 41 ++++++++- backend/app/routers/orchestration.py | 32 ++++++- backend/app/services/langfuse_client.py | 33 +++++-- backend/app/services/scheduler.py | 6 +- .../components/monitoring/RunHistoryTable.css | 6 ++ .../components/monitoring/RunHistoryTable.tsx | 86 ++++++++++++++---- .../src/components/tables/EvaluationTab.css | 7 ++ .../src/components/tables/EvaluationTab.tsx | 91 +++++++++++-------- frontend/src/pages/ControlCenterPage.tsx | 17 +++- frontend/src/pages/SandboxPage.tsx | 12 ++- text2sql_test | 0 11 files changed, 257 insertions(+), 74 deletions(-) create mode 100644 text2sql_test diff --git a/backend/app/routers/evaluation.py b/backend/app/routers/evaluation.py index b02b845..de97887 100644 --- a/backend/app/routers/evaluation.py +++ b/backend/app/routers/evaluation.py @@ -176,9 +176,15 @@ def _map_and_save_run_metrics( run: EvalRun, eval_resp: RunDatasetResponse, session: Session, run_id: str ): run.score = eval_resp.accuracy.contains_accuracy - run.pass_rate = 1.0 - eval_resp.failure_rate - run.fail_rate = eval_resp.failure_rate - run.total_questions = eval_resp.total_cases + # Guard: if no questions were evaluated, pass_rate is meaningless — use 0.0 + if eval_resp.total_cases == 0: + run.pass_rate = 0.0 + run.fail_rate = 1.0 + else: + run.pass_rate = 1.0 - eval_resp.failure_rate + run.fail_rate = eval_resp.failure_rate + if eval_resp.total_cases > 0 or not run.total_questions: + run.total_questions = eval_resp.total_cases run.duration_seconds = eval_resp.duration_seconds run.status = EvalStatus.completed run.completed_at = datetime.now() @@ -251,10 +257,16 @@ def execute_single_table_eval(table_id: str, run_id: str, session: Session) -> f if not questions: run.status = EvalStatus.failed run.score = 0.0 + run.total_questions = 0 session.add(run) session.commit() return 0.0 + if run.total_questions == 0: + run.total_questions = len(questions) + session.add(run) + session.commit() + table = session.get(Table, table_id) dataset_name = f"text2sql_sandbox_{table_id}" @@ -286,6 +298,8 @@ def execute_single_table_eval(table_id: str, run_id: str, session: Session) -> f logger.error(f"[Eval] Table {table_id} evaluation failed via API: {e}") run.status = EvalStatus.failed run.score = 0.0 + if questions: + run.total_questions = len(questions) session.add(run) session.commit() return 0.0 @@ -355,6 +369,7 @@ def _run_production_dataset_eval( run = EvalRun( table_id=None, + total_questions=len(all_production_questions), status=EvalStatus.running, triggered_by="promotion-baseline", promotion_run_id=promotion_run_id, @@ -363,7 +378,10 @@ def _run_production_dataset_eval( session.commit() session.refresh(run) - table_names = [t.name for t in prod_tables] + # Send schema.table format so the agent can validate tables without ambiguity + table_names = [ + f"{t.schema_name}.{t.name}" if t.schema_name else t.name for t in prod_tables + ] try: req = { "dataset_name": PRODUCTION_DATASET_NAME, @@ -380,6 +398,7 @@ def _run_production_dataset_eval( logger.error(f"[Promotion/Phase-A] Baseline eval failed: {e}") run.status = EvalStatus.failed run.score = 0.0 + run.total_questions = len(all_production_questions) session.add(run) session.commit() return 0.0 @@ -408,6 +427,7 @@ def _run_candidate_eval( ) -> float: run = EvalRun( table_id=table.id, + total_questions=len(questions), status=EvalStatus.running, triggered_by="promotion-candidate", promotion_run_id=promotion_run_id, @@ -442,6 +462,7 @@ def _run_candidate_eval( logger.error(f"[Promotion/Phase-B] Candidate eval failed: {e}") run.status = EvalStatus.failed run.score = 0.0 + run.total_questions = len(questions) session.add(run) session.commit() return 0.0 @@ -479,6 +500,7 @@ def _run_regression_eval( run = EvalRun( table_id=None, + total_questions=len(all_production_questions), status=EvalStatus.running, triggered_by="promotion-regression", promotion_run_id=promotion_run_id, @@ -487,6 +509,10 @@ def _run_regression_eval( session.commit() session.refresh(run) + # Send schema.table format so the agent can validate tables without ambiguity + table_names = [ + f"{t.schema_name}.{t.name}" if t.schema_name else t.name for t in prod_tables + ] try: req = { "dataset_name": PRODUCTION_DATASET_NAME, @@ -503,6 +529,7 @@ def _run_regression_eval( logger.error(f"[Promotion/Phase-B] Regression eval failed: {e}") run.status = EvalStatus.failed run.score = 0.0 + run.total_questions = len(all_production_questions) session.add(run) session.commit() return 0.0 @@ -840,7 +867,11 @@ def trigger_eval( detail=f"Cannot run evaluation. Missing: {'; '.join(missing)}.", ) - run = EvalRun(table_id=table_id, status=EvalStatus.running) + run = EvalRun( + table_id=table_id, + total_questions=len(questions), + status=EvalStatus.running, + ) session.add(run) session.commit() session.refresh(run) diff --git a/backend/app/routers/orchestration.py b/backend/app/routers/orchestration.py index 5012999..580a39a 100644 --- a/backend/app/routers/orchestration.py +++ b/backend/app/routers/orchestration.py @@ -255,7 +255,14 @@ def trigger_evaluation_run( runs = [] for table_id in table_ids: table = session.get(Table, table_id) - run = EvalRun(table_id=table_id, triggered_by=triggered_by) + questions = session.exec( + select(GoldenQuestion).where(GoldenQuestion.table_id == table_id) + ).all() + run = EvalRun( + table_id=table_id, + total_questions=len(questions), + triggered_by=triggered_by, + ) session.add(run) session.commit() session.refresh(run) @@ -380,8 +387,25 @@ def trigger_dataset_run( except Exception as e: logger.warning(f"[Eval] Production dataset sync failed: {e}") + total_q_count = 0 + if dataset_name == "text2sql_production": + total_q_count = len(all_production_questions) + elif dataset_name == "spider2": + spider2_tables = session.exec( + select(Table).where(Table.owner_id == "spider2") + ).all() + s2_ids = [t.id for t in spider2_tables] + if s2_ids: + total_q_count = len( + session.exec( + select(GoldenQuestion).where(GoldenQuestion.table_id.in_(s2_ids)) + ).all() + ) + # 2. Create the run record - run = EvalRun(table_id=None, triggered_by=dataset_name) + run = EvalRun( + table_id=None, total_questions=total_q_count, triggered_by=dataset_name + ) session.add(run) session.commit() session.refresh(run) @@ -480,7 +504,9 @@ def get_run_report(run_id: str, session: Session = Depends(get_session)): "status": run.status, "triggered_by": run.triggered_by, "promotion_run_id": run.promotion_run_id, - "is_publishable": run.score > 0.00, + "is_publishable": run.score >= 0.50 + if run.status == EvalStatus.completed + else False, "regression_detected": run.regression_detected, "regression_delta": run.regression_delta, "failure_breakdown": run.failure_breakdown or {}, diff --git a/backend/app/services/langfuse_client.py b/backend/app/services/langfuse_client.py index 80fc4a1..bb7987a 100644 --- a/backend/app/services/langfuse_client.py +++ b/backend/app/services/langfuse_client.py @@ -314,12 +314,26 @@ def sync_dataset(self, dataset_name: str, questions: list) -> object: for item in existing_items: qid = (item.get("metadata") or {}).get("question_id") if qid: + exp_out = item.get("expectedOutput") or {} + if isinstance(exp_out, dict): + exp_sql = exp_out.get("sql") or exp_out.get("response") or "" + elif isinstance(exp_out, str): + exp_sql = exp_out + else: + exp_sql = "" + + inp_in = item.get("input") or {} + if isinstance(inp_in, dict): + inp_q = inp_in.get("query") or inp_in.get("question") or "" + elif isinstance(inp_in, str): + inp_q = inp_in + else: + inp_q = "" + existing_by_qid[qid] = { "langfuse_id": item["id"], - "question_text": (item.get("input") or {}).get("query", ""), - "expected_sql": (item.get("expectedOutput") or {}).get( - "response", "" - ), + "question_text": inp_q.strip(), + "expected_sql": exp_sql.strip(), } # Build lookup for desired state: question_id → question dict @@ -337,9 +351,11 @@ def sync_dataset(self, dataset_name: str, questions: list) -> object: for qid in to_check: desired = desired_by_qid[qid] existing = existing_by_qid[qid] + desired_text = (desired.get("question_text") or "").strip() + desired_sql = (desired.get("expected_sql") or "").strip() if ( - desired["question_text"] != existing["question_text"] - or desired["expected_sql"] != existing["expected_sql"] + desired_text != existing["question_text"] + or desired_sql != existing["expected_sql"] ): to_update.add(qid) @@ -378,6 +394,7 @@ def sync_dataset(self, dataset_name: str, questions: list) -> object: for qid in items_to_create: q = desired_by_qid[qid] try: + sql_str = (q.get("expected_sql") or "").strip() self._tracer.client.create_dataset_item( dataset_name=dataset_name, id=q["question_id"], @@ -385,7 +402,7 @@ def sync_dataset(self, dataset_name: str, questions: list) -> object: "query": q["question_text"], "databases": [q.get("schema_name", q["table_id"])], }, - expected_output={"response": q["expected_sql"]}, + expected_output={"sql": sql_str, "response": sql_str}, metadata={ "split": q.get("split", ""), "difficulty": str(q.get("difficulty", "")).lower().strip(), @@ -402,6 +419,8 @@ def sync_dataset(self, dataset_name: str, questions: list) -> object: ) self.flush() + if items_to_delete or items_to_create: + time.sleep(0.5) self.logger.info( f"[LangfuseDatasetService] Sync complete for '{dataset_name}': " diff --git a/backend/app/services/scheduler.py b/backend/app/services/scheduler.py index 909e607..f31e046 100644 --- a/backend/app/services/scheduler.py +++ b/backend/app/services/scheduler.py @@ -59,7 +59,11 @@ def _run_scheduled_evaluation(schedule_id: str): logger.info(f"[Scheduler] Skipping {table_id} — no questions") continue - run = EvalRun(table_id=table_id, triggered_by="scheduler") + run = EvalRun( + table_id=table_id, + total_questions=len(questions), + triggered_by="scheduler", + ) session.add(run) session.commit() session.refresh(run) diff --git a/frontend/src/components/monitoring/RunHistoryTable.css b/frontend/src/components/monitoring/RunHistoryTable.css index a594645..b549eb0 100644 --- a/frontend/src/components/monitoring/RunHistoryTable.css +++ b/frontend/src/components/monitoring/RunHistoryTable.css @@ -103,6 +103,12 @@ color: #ef4444; } +.publishable-status--running { + background: rgba(245, 158, 11, 0.08); + border: 1px solid rgba(245, 158, 11, 0.25); + color: #f59e0b; +} + .regression-alert { padding: 10px 14px; border-radius: 8px; diff --git a/frontend/src/components/monitoring/RunHistoryTable.tsx b/frontend/src/components/monitoring/RunHistoryTable.tsx index ba985c9..077b615 100644 --- a/frontend/src/components/monitoring/RunHistoryTable.tsx +++ b/frontend/src/components/monitoring/RunHistoryTable.tsx @@ -57,13 +57,29 @@ function RunDetailDrawer({ runId, onClose }: { runId: string; onClose: () => voi {[ { label: 'Score', - value: `${Math.round(report.overall_score * 100)}%`, - color: report.overall_score >= 0.5 ? '#10b981' : '#ef4444', + value: + report.status === 'running' + ? 'Evaluating…' + : `${Math.round(report.overall_score * 100)}%`, + color: + report.status === 'running' + ? '#f59e0b' + : report.overall_score >= 0.5 + ? '#10b981' + : '#ef4444', }, { label: 'Pass Rate', - value: `${Math.round(report.pass_rate * 100)}%`, - color: report.pass_rate >= 0.5 ? '#10b981' : '#ef4444', + value: + report.status === 'running' + ? 'Calculating…' + : `${Math.round(report.pass_rate * 100)}%`, + color: + report.status === 'running' + ? '#f59e0b' + : report.pass_rate >= 0.5 + ? '#10b981' + : '#ef4444', }, { label: 'Questions', value: report.total_questions, color: 'var(--text-primary)' }, { @@ -83,12 +99,30 @@ function RunDetailDrawer({ runId, onClose }: { runId: string; onClose: () => voi {/* Publishable status */}
- {report.is_publishable ? : } - {report.is_publishable - ? 'Ready to publish (score ≥ 50%)' - : 'Not publishable — score below 50%'} + + {report.status === 'running' ? ( + + ) : report.is_publishable ? ( + + ) : ( + + )} + + {report.status === 'running' + ? 'Evaluation running — publishability pending evaluation completion' + : report.status === 'failed' + ? 'Not publishable — evaluation run failed' + : report.is_publishable + ? 'Ready to publish (score ≥ 50%)' + : 'Not publishable — score below 50%'}
{/* Regression */} @@ -420,12 +454,28 @@ export function RunHistoryTable({ )}
- = 0.5 ? '#10b981' : '#ef4444' }} - > - {Math.round(run.score * 100)}% - + {run.status === 'running' ? ( + + + Evaluating… + + ) : ( + = 0.5 ? '#10b981' : '#ef4444' }} + > + {Math.round(run.score * 100)}% + + )} {run.regression_detected && ( @@ -434,7 +484,11 @@ export function RunHistoryTable({
- + {run.status === 'running' ? ( + Calculating… + ) : ( + + )} {!compact && {run.total_questions}} diff --git a/frontend/src/components/tables/EvaluationTab.css b/frontend/src/components/tables/EvaluationTab.css index ae05b4e..38eda77 100644 --- a/frontend/src/components/tables/EvaluationTab.css +++ b/frontend/src/components/tables/EvaluationTab.css @@ -88,6 +88,13 @@ background: rgba(239, 68, 68, 0.08); } +.score-ring--running { + color: #f59e0b; + border-color: #f59e0b; + background: rgba(245, 158, 11, 0.08); + font-size: 11px; +} + /* ── Readiness banner ──────────────────────────────── */ .eval-readiness-banner { diff --git a/frontend/src/components/tables/EvaluationTab.tsx b/frontend/src/components/tables/EvaluationTab.tsx index 6a26c3a..b5e117b 100644 --- a/frontend/src/components/tables/EvaluationTab.tsx +++ b/frontend/src/components/tables/EvaluationTab.tsx @@ -1,3 +1,4 @@ +import { useMemo } from 'react'; import { useTranslation } from 'react-i18next'; import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; import { App } from 'antd'; @@ -6,6 +7,7 @@ import { AlertTriangle, BarChart2, CheckCircle, Play } from 'lucide-react'; import { enrichmentApi, evalApi, questionsApi } from '../../api/client'; import { ErrorState } from '../common/ErrorState'; +import { Spinner } from '../common/EvalUI'; import { SkeletonTable } from '../common/Skeleton'; import './EvaluationTab.css'; @@ -14,7 +16,14 @@ interface Props { tableId: string; } -function ScoreRing({ score }: { score: number }) { +function ScoreRing({ score, status }: { score: number; status?: string }) { + if (status === 'running') { + return ( +
+ +
+ ); + } const pct = Math.round(score * 100); const cls = pct >= 50 ? 'score-ring--high' : 'score-ring--low'; return
{pct}%
; @@ -73,6 +82,20 @@ export function EvaluationTab({ tableId }: Props) { }, }); + // Deduplicate runs so triggered run never appears twice + const allRuns = useMemo(() => { + const map = new Map(); + if (triggerMutation.data) { + map.set(triggerMutation.data.id, triggerMutation.data); + } + (runs ?? []).forEach((r: any) => { + map.set(r.id, r); + }); + return Array.from(map.values()) + .filter((run) => run.triggered_by !== 'promotion') + .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()); + }, [triggerMutation.data, runs]); + if (isLoading) return ; if (isError) return ; @@ -117,7 +140,7 @@ export function EvaluationTab({ tableId }: Props) { {triggerMutation.data && (
- +
Latest Run: {triggerMutation.data.table_name || tableId.slice(0, 8)} @@ -129,7 +152,7 @@ export function EvaluationTab({ tableId }: Props) {
)} - {(!runs || runs.length === 0) && !triggerMutation.data ? ( + {allRuns.length === 0 ? (
@@ -150,39 +173,35 @@ export function EvaluationTab({ tableId }: Props) { - {[...(triggerMutation.data ? [triggerMutation.data] : []), ...(runs ?? [])] - .filter((run) => run.triggered_by !== 'promotion') - .map((run) => ( - - - {run.id.slice(0, 8)}… - - - - - - - {run.status} - - - - {run.triggered_by} - - - {dayjs(run.created_at).format('MMM D, HH:mm')} - - - ))} + {allRuns.map((run) => ( + + + {run.id.slice(0, 8)}… + + + + + + + {run.status} + + + + {run.triggered_by} + + {dayjs(run.created_at).format('MMM D, HH:mm')} + + ))}
diff --git a/frontend/src/pages/ControlCenterPage.tsx b/frontend/src/pages/ControlCenterPage.tsx index 129fd56..4da1b82 100644 --- a/frontend/src/pages/ControlCenterPage.tsx +++ b/frontend/src/pages/ControlCenterPage.tsx @@ -137,16 +137,25 @@ function RecentRunRow({ height: 36, borderRadius: 8, flexShrink: 0, - background: run.score >= 0.5 ? 'rgba(16,185,129,0.15)' : 'rgba(239,68,68,0.15)', + background: + run.status === 'running' + ? 'rgba(245,158,11,0.15)' + : run.score >= 0.5 + ? 'rgba(16,185,129,0.15)' + : 'rgba(239,68,68,0.15)', display: 'flex', alignItems: 'center', justifyContent: 'center', - fontSize: 13, + fontSize: run.status === 'running' ? 10 : 13, fontWeight: 800, - color: run.score >= 0.5 ? '#10b981' : '#ef4444', + color: run.status === 'running' ? '#f59e0b' : run.score >= 0.5 ? '#10b981' : '#ef4444', }} > - {Math.round(run.score * 100)}% + {run.status === 'running' ? ( + + ) : ( + `${Math.round(run.score * 100)}%` + )}
diff --git a/frontend/src/pages/SandboxPage.tsx b/frontend/src/pages/SandboxPage.tsx index 8c71d84..7a6d790 100644 --- a/frontend/src/pages/SandboxPage.tsx +++ b/frontend/src/pages/SandboxPage.tsx @@ -4,10 +4,18 @@ import dayjs from 'dayjs'; import { FlaskConical } from 'lucide-react'; import { ErrorState } from '../components/common/ErrorState'; +import { Spinner } from '../components/common/EvalUI'; import { SkeletonTable } from '../components/common/Skeleton'; import { useAllEvalRuns } from '../hooks/useEvaluations'; -function ScoreRing({ score }: { score: number }) { +function ScoreRing({ score, status }: { score: number; status?: string }) { + if (status === 'running') { + return ( +
+ +
+ ); + } const pct = Math.round(score * 100); const cls = pct >= 50 ? 'score-ring--high' : 'score-ring--low'; return
{pct}%
; @@ -79,7 +87,7 @@ export function SandboxPage() { - + Date: Thu, 23 Jul 2026 11:20:47 +0300 Subject: [PATCH 11/11] make ui look better in evaluations run --- agent/src/agent/nodes/schema_explorer.py | 37 +++++++++++++ agent/src/agent/routers/chat.py | 16 +++++- backend/app/routers/evaluation.py | 27 +--------- backend/app/routers/orchestration.py | 52 ++++++++++++++----- .../components/monitoring/RunHistoryTable.css | 10 ++++ .../components/monitoring/RunHistoryTable.tsx | 34 ++++++++++-- .../src/components/tables/EvaluationTab.tsx | 5 ++ 7 files changed, 138 insertions(+), 43 deletions(-) diff --git a/agent/src/agent/nodes/schema_explorer.py b/agent/src/agent/nodes/schema_explorer.py index ca1dab3..5245dcf 100644 --- a/agent/src/agent/nodes/schema_explorer.py +++ b/agent/src/agent/nodes/schema_explorer.py @@ -33,6 +33,8 @@ ) from core.cache import get_cache_service from core.embeddings import get_embedding +from core.trino import execute_query_sync + # Initialize LLM llm = get_llm("schema_explorer") @@ -202,6 +204,7 @@ def hybrid_search_tables( table.id in allowed or table.name in allowed or f"{table.schema_name}.{table.name}" in allowed + or f"{table.catalog}.{table.schema_name}.{table.name}" in allowed ) ) else: @@ -212,6 +215,7 @@ def hybrid_search_tables( table.id in allowed or table.name in allowed or f"{table.schema_name}.{table.name}" in allowed + or f"{table.catalog}.{table.schema_name}.{table.name}" in allowed ) ) @@ -282,6 +286,13 @@ def hybrid_search_tables( : settings.HYBRID_SEARCH_MAX_TABLES ] + # If scoping_mode == "strict" or allowed_tables was specified, + # ensure explicitly allowed tables are never dropped due to low keyword/vector scores + if allowed_tables_set: + for table in allowed_tables_set: + if table.id not in combined_ids: + combined_ids.append(table.id) + result_tables = [] for tid in combined_ids: t = session.get(Table, tid) @@ -312,6 +323,32 @@ async def get_table_profile(table_id: str) -> str: ).first() if not profile: + # Fallback to querying Trino directly if no static profile exists in DB + try: + trino_res = execute_query_sync( + f"DESCRIBE {table.catalog}.{table.schema_name}.{table.name}" + ) + if trino_res.success and trino_res.rows: + cols = [ + { + "name": row[0], + "type": row[1], + "sample_values": [], + "null_count": 0, + } + for row in trino_res.rows + ] + res = { + "table_id": table_id, + "table_name": f"{table.catalog}.{table.schema_name}.{table.name}", + "row_count": 0, + "columns": cols, + "description": "", + } + return json.dumps(res) + except Exception as e: + print(f"Trino DESCRIBE fallback failed for {table.name}: {e}") + return json.dumps( { "error": f"No completed profile found for Table ID {table_id}. Make sure to trigger profiling first." diff --git a/agent/src/agent/routers/chat.py b/agent/src/agent/routers/chat.py index 5ae4390..c810765 100644 --- a/agent/src/agent/routers/chat.py +++ b/agent/src/agent/routers/chat.py @@ -83,8 +83,20 @@ async def chat_endpoint( from sqlalchemy import or_ for allowed in request.allowed_tables: parts = allowed.split(".") - if len(parts) == 2: - cond = or_(Table.id == allowed, Table.name == allowed, (Table.schema_name == parts[0]) & (Table.name == parts[1])) + if len(parts) == 3: + cond = or_( + Table.id == allowed, + Table.name == allowed, + (Table.catalog == parts[0]) + & (Table.schema_name == parts[1]) + & (Table.name == parts[2]), + ) + elif len(parts) == 2: + cond = or_( + Table.id == allowed, + Table.name == allowed, + (Table.schema_name == parts[0]) & (Table.name == parts[1]), + ) else: cond = or_(Table.id == allowed, Table.name == allowed) diff --git a/backend/app/routers/evaluation.py b/backend/app/routers/evaluation.py index de97887..9b374e9 100644 --- a/backend/app/routers/evaluation.py +++ b/backend/app/routers/evaluation.py @@ -38,6 +38,7 @@ from sqlmodel import Session, desc, select from app.config import settings +from app.routers.orchestration import get_run_report as get_orchestration_report from app.services.langfuse_client import langfuse_client logger = logging.getLogger(__name__) @@ -976,31 +977,7 @@ def get_results(run_id: str, session: Session = Depends(get_session)): @router.get("/eval/{run_id}/report") def get_run_report(run_id: str, session: Session = Depends(get_session)): - run = session.get(EvalRun, run_id) - if not run: - raise HTTPException(status_code=404, detail="Eval run not found") - - results = session.exec(select(EvalResult).where(EvalResult.run_id == run_id)).all() - - total = len(results) - passes = sum(1 for r in results if r.status == "pass") - - return { - "run_id": run_id, - "table_id": run.table_id, - "contains_execution_accuracy": run.score, - "pass_rate": round(passes / total, 3) if total else 0, - "total_questions": total, - "is_publishable": run.score >= 0.50, - "regression_detected": run.regression_detected, - "regression_delta": run.regression_delta, - "status": run.status, - "created_at": run.created_at.isoformat(), - "per_question": [ - {"question_id": r.question_id, "score": r.score, "status": r.status} - for r in results - ], - } + return get_orchestration_report(run_id, session) @router.get("/eval/{run_id}/regression-diff") diff --git a/backend/app/routers/orchestration.py b/backend/app/routers/orchestration.py index 580a39a..edc22f8 100644 --- a/backend/app/routers/orchestration.py +++ b/backend/app/routers/orchestration.py @@ -493,13 +493,50 @@ def get_run_report(run_id: str, session: Session = Depends(get_session)): ).all() question_map = {q.id: q.question for q in questions} + if not results and run.table_id: + table_questions = session.exec( + select(GoldenQuestion).where(GoldenQuestion.table_id == run.table_id) + ).all() + is_running = run.status == EvalStatus.running + per_question = [ + { + "question_id": q.id, + "question": q.question, + "score": None if is_running else 0.0, + "status": "pending" if is_running else "fail", + "failure_type": None + if is_running + else ( + "evaluation_failed" if run.status == EvalStatus.failed else "failed" + ), + } + for q in table_questions + ] + else: + per_question = [ + { + "question_id": r.question_id, + "question": question_map.get(r.question_id, r.question_id), + "score": r.score, + "status": r.status, + "failure_type": r.error_type, + } + for r in results + ] + + failure_breakdown = run.failure_breakdown or ( + {"evaluation_failed": run.total_questions or len(per_question)} + if run.status == EvalStatus.failed + else {} + ) + return { "run_id": run_id, "table_id": run.table_id, "overall_score": run.score, "pass_rate": run.pass_rate, "fail_rate": run.fail_rate, - "total_questions": run.total_questions, + "total_questions": run.total_questions or len(per_question), "duration_seconds": run.duration_seconds, "status": run.status, "triggered_by": run.triggered_by, @@ -509,20 +546,11 @@ def get_run_report(run_id: str, session: Session = Depends(get_session)): else False, "regression_detected": run.regression_detected, "regression_delta": run.regression_delta, - "failure_breakdown": run.failure_breakdown or {}, + "failure_breakdown": failure_breakdown, "dimension_averages": run.dimension_averages or {}, "started_at": run.started_at.isoformat(), "completed_at": run.completed_at.isoformat() if run.completed_at else None, - "per_question": [ - { - "question_id": r.question_id, - "question": question_map.get(r.question_id), - "score": r.score, - "status": r.status, - "failure_type": r.error_type, - } - for r in results - ], + "per_question": per_question, } diff --git a/frontend/src/components/monitoring/RunHistoryTable.css b/frontend/src/components/monitoring/RunHistoryTable.css index b549eb0..9fefaaf 100644 --- a/frontend/src/components/monitoring/RunHistoryTable.css +++ b/frontend/src/components/monitoring/RunHistoryTable.css @@ -211,6 +211,16 @@ flex-shrink: 0; } +@keyframes pulse { + 0%, + 100% { + opacity: 0.4; + } + 50% { + opacity: 1; + } +} + .question-item__id { flex: 1; font-size: 11.5px; diff --git a/frontend/src/components/monitoring/RunHistoryTable.tsx b/frontend/src/components/monitoring/RunHistoryTable.tsx index 077b615..5b91acf 100644 --- a/frontend/src/components/monitoring/RunHistoryTable.tsx +++ b/frontend/src/components/monitoring/RunHistoryTable.tsx @@ -15,6 +15,7 @@ function RunDetailDrawer({ runId, onClose }: { runId: string; onClose: () => voi queryKey: ['run-report', runId], queryFn: () => orchestrationApi.getRunReport(runId), enabled: !!runId, + refetchInterval: (query) => (query.state.data?.status === 'running' ? 4000 : false), }); const isRegressionRun = report?.triggered_by === 'promotion-regression'; @@ -192,18 +193,43 @@ function RunDetailDrawer({ runId, onClose }: { runId: string; onClose: () => voi
{(q.question ?? q.question_id).slice(0, 16)}…
= 0.5 ? '#10b981' : '#ef4444' }} + style={{ + color: + q.status === 'pending' + ? '#f59e0b' + : q.score >= 0.5 + ? '#10b981' + : '#ef4444', + }} > - {q.score >= 0.5 ? 100 : 0}% + {q.status === 'pending' ? '—' : q.score >= 0.5 ? '100%' : '0%'}
- {q.failure_type && ( + {q.status === 'pending' && ( + + evaluating… + + )} + {q.status !== 'pending' && q.failure_type && ( {q.failure_type} )}
diff --git a/frontend/src/components/tables/EvaluationTab.tsx b/frontend/src/components/tables/EvaluationTab.tsx index b5e117b..8e022d9 100644 --- a/frontend/src/components/tables/EvaluationTab.tsx +++ b/frontend/src/components/tables/EvaluationTab.tsx @@ -42,6 +42,11 @@ export function EvaluationTab({ tableId }: Props) { } = useQuery({ queryKey: ['eval-runs', tableId], queryFn: () => evalApi.listRuns(tableId), + refetchInterval: (query) => { + const data = query.state.data; + const hasRunning = Array.isArray(data) && data.some((r) => r.status === 'running'); + return hasRunning ? 3000 : false; + }, }); const { data: enrichment } = useQuery({