spider2-dbt — harbor_view dbt/DuckDB parity with ade-bench

src/razorback/benchmarks/spider2_dbt/harbor_view.py

@@ -1,8 +1,14 @@
 from __future__ import annotations
 
+import shlex
+import shutil
 from pathlib import Path
 from typing import Literal
 
+from razorback.benchmarks.spider2_dbt.preflight import (
+    preflight_script_text,
+    resolve_spider2_db_name,
+)
 from razorback.harbor_tasks.leakage import DEFAULT_SOLUTION_DENY_GLOBS
 from razorback.harbor_tasks.materialize import materialize_harbor_task_view
 
@@ -20,6 +26,26 @@
     "**/golden/**",
 )
 
+# The dbt project root inside the running container. The r5 verifier
+# (spider2-dbt-duckdb-match-verifier) and run-wiring entity read this same
+# path; do not let later layers drift from it (Task 0 contract).
+_APP_ROOT = "/app"
+
+# Source-side directory (inside the materialized view) holding the dbt
+# project. spider2-dbt nests the dbt project under `dbt_project/` — the one
+# structural divergence from ade-bench's `project/`.
+_DBT_PROJECT_DIRNAME = "dbt_project"
+
+_BUILD_CONTEXT_MARKER = (
+    "# Razorback: land spider2-dbt project + source DuckDB at /app before agent runtime."
+)
+_DBT_DEPS_LAYER_MARKER = (
+    "# Razorback: install declared dbt packages before agent runtime."
+)
+_SPIDER2_WORKSPACE_PREFLIGHT_MARKER = (
+    "# Razorback: validate spider2-dbt source DuckDB before agent runtime."
+)
+
 
 def materialize_spider2_harbor_task_view(
     *,
@@ -29,7 +55,7 @@ def materialize_spider2_harbor_task_view(
     docker_image: str | None = None,
     view_mode: Literal["copy", "link"] = "copy",
 ) -> Path:
-    return materialize_harbor_task_view(
+    view = materialize_harbor_task_view(
         source_task_dir=source_task_dir,
         view_root=view_root,
         benchmark_kind="spider2-dbt",
@@ -43,3 +69,199 @@ def materialize_spider2_harbor_task_view(
         exclude_globs=SPIDER2_DBT_DENY_GLOBS,
         view_mode=view_mode,
     )
+    # RIDER (Codex finding 2): stage dbt_project/ (incl. the source .duckdb)
+    # into the build context and COPY it to /app BEFORE the preflight RUN, so
+    # the preflight `--workspace /app` can never fail on a missing project.
+    _ensure_spider2_build_context_layer(view)
+    _ensure_dbt_deps_image_layer(view)
+    _ensure_workspace_preflight_image_layer(view, task_slug=task_slug)
+    return view
+
+
+def _has_dbt_project(view_dir: Path) -> bool:
+    """spider2-dbt nests the dbt project under `dbt_project/` (or under
+    `environment/dbt_project/`)."""
+    return _dbt_project_dir(view_dir) is not None
+
+
+def _dbt_project_dir(view_dir: Path) -> Path | None:
+    """The dbt project root inside the view (`dbt_project/` or
+    `environment/dbt_project/`), if present.
+
+    This is the on-disk stand-in for the container's `/app` dbt root: the
+    source `.duckdb` and any `profiles.yml` live here, so it is the workspace
+    `resolve_spider2_db_name` reads to pin `/app/<db_name>.duckdb`.
+    """
+    direct = view_dir / _DBT_PROJECT_DIRNAME
+    if direct.is_dir():
+        return direct
+    nested = view_dir / "environment" / _DBT_PROJECT_DIRNAME
+    if nested.is_dir():
+        return nested
+    return None
+
+
+def _has_dbt_packages_manifest(view_dir: Path) -> bool:
+    return (
+        (view_dir / _DBT_PROJECT_DIRNAME / "packages.yml").is_file()
+        or (
+            view_dir / "environment" / _DBT_PROJECT_DIRNAME / "packages.yml"
+        ).is_file()
+    )
+
+
+def _ensure_spider2_build_context_layer(view_dir: Path) -> None:
+    """Land the dbt project + source DuckDB at /app inside the image build.
+
+    The Docker build context is the view's `environment/` directory, so the
+    dbt project (which the materializer reflects to `<view>/dbt_project/`)
+    must be staged *inside* `environment/` for a COPY to reach it. This makes
+    the entity own the minimal COPY/context the preflight RUN depends on,
+    rather than assuming run-wiring already placed the project at /app.
+    """
+    if not _has_dbt_project(view_dir):
+        return
+
+    dockerfile = view_dir / "environment" / "Dockerfile"
+    if not dockerfile.is_file():
+        return
+
+    text = dockerfile.read_text()
+    if _BUILD_CONTEXT_MARKER in text:
+        return
+
+    environment_dir = view_dir / "environment"
+    source_project = view_dir / _DBT_PROJECT_DIRNAME
+    staged_project = environment_dir / _DBT_PROJECT_DIRNAME
+    if source_project.is_dir() and not staged_project.exists():
+        # Stage into the build context so the COPY source resolves. Copy
+        # (not move) so the view's own dbt_project/ remains intact for
+        # downstream consumers/tests.
+        shutil.copytree(source_project, staged_project)
+
+    block = "\n".join(
+        [
+            _BUILD_CONTEXT_MARKER,
+            f"COPY {_DBT_PROJECT_DIRNAME}/ {_APP_ROOT}/",
+        ]
+    )
+    # In `view_mode="link"` the reflected Dockerfile is a symlink back into the
+    # shared source tree; writing through it would follow the link and corrupt
+    # the version-controlled source. Replace the symlink with a real,
+    # view-owned file so the layer injection stays inside the view.
+    if dockerfile.is_symlink():
+        dockerfile.unlink()
+    dockerfile.write_text(_insert_before_final_cmd(text, block))
+
+
+def _ensure_dbt_deps_image_layer(view_dir: Path) -> None:
+    """Install declared dbt packages during image build for dbt spider2 tasks."""
+    if not _has_dbt_packages_manifest(view_dir):
+        return
+
+    dockerfile = view_dir / "environment" / "Dockerfile"
+    if not dockerfile.is_file():
+        return
+
+    text = dockerfile.read_text()
+    if _DBT_DEPS_LAYER_MARKER in text:
+        return
+
+    block = "\n".join(
+        [
+            _DBT_DEPS_LAYER_MARKER,
+            "RUN if [ -f /app/packages.yml ]; then cd /app && dbt deps; fi",
+        ]
+    )
+    # In `view_mode="link"` the reflected Dockerfile is a symlink back into the
+    # shared source tree; writing through it would follow the link and corrupt
+    # the version-controlled source. Replace the symlink with a real,
+    # view-owned file so the layer injection stays inside the view.
+    if dockerfile.is_symlink():
+        dockerfile.unlink()
+    dockerfile.write_text(_insert_before_final_cmd(text, block))
+
+
+def _ensure_workspace_preflight_image_layer(
+    view_dir: Path, *, task_slug: str
+) -> None:
+    """Validate the source DuckDB at build time, before the agent runs.
+
+    Gated on `_has_dbt_project`: spider2-dbt has no task families, so the
+    preflight is injected whenever the task is a dbt project. By the time this
+    RUN executes, the build-context layer has already COPY'd dbt_project/ (and
+    its .duckdb) to /app, so `--workspace /app` cannot fail on a missing
+    project.
+    """
+    if not _has_dbt_project(view_dir):
+        return
+
+    environment_dir = view_dir / "environment"
+    dockerfile = environment_dir / "Dockerfile"
+    if not dockerfile.is_file():
+        return
+
+    script_path = environment_dir / "razorback_spider2_preflight.py"
+    # In `view_mode="link"` a source task that ships a file with this exact name
+    # is reflected as a symlink back into the shared source tree; writing through
+    # it would follow the link and corrupt the version-controlled source. Replace
+    # the symlink with a real, view-owned file so the write stays inside the view
+    # (mirrors the Dockerfile/task.toml unlink-then-write guards).
+    if script_path.is_symlink():
+        script_path.unlink()
+    script_path.write_text(preflight_script_text())
+
+    text = dockerfile.read_text()
+    if _SPIDER2_WORKSPACE_PREFLIGHT_MARKER in text:
+        return
+
+    # Pin the agent-facing DuckDB via the SHARED resolver so the build-time
+    # preflight validates the SAME `/app/<db_name>.duckdb` the agent (and the
+    # r5 verifier) operate against — never a glob-first under multi/stale-DB
+    # drift. Resolution fails CLOSED (raises) when >1 *.duckdb exists and none
+    # is pinned; that aborts the materialize, the correct fail-closed point.
+    project_dir = _dbt_project_dir(view_dir)
+    db_name = resolve_spider2_db_name(
+        project_dir if project_dir is not None else view_dir,
+        task_slug=task_slug,
+    )
+    command = " ".join(
+        [
+            "python",
+            "/tmp/razorback_spider2_preflight.py",
+            "--task-id",
+            shlex.quote(task_slug),
+            "--workspace",
+            _APP_ROOT,
+            "--db-name",
+            shlex.quote(db_name),
+        ]
+    )
+    block = "\n".join(
+        [
+            _SPIDER2_WORKSPACE_PREFLIGHT_MARKER,
+            "COPY razorback_spider2_preflight.py /tmp/razorback_spider2_preflight.py",
+            f"RUN {command}",
+        ]
+    )
+    # In `view_mode="link"` the reflected Dockerfile is a symlink back into the
+    # shared source tree; writing through it would follow the link and corrupt
+    # the version-controlled source. Replace the symlink with a real,
+    # view-owned file so the layer injection stays inside the view.
+    if dockerfile.is_symlink():
+        dockerfile.unlink()
+    dockerfile.write_text(_insert_before_final_cmd(text, block))
+
+
+def _insert_before_final_cmd(text: str, block: str) -> str:
+    lines = text.rstrip().splitlines()
+    insert_at = None
+    for idx, line in enumerate(lines):
+        if line.lstrip().startswith("CMD "):
+            insert_at = idx
+    block_lines = ["", *block.splitlines()]
+    if insert_at is None:
+        lines.extend(block_lines)
+    else:
+        lines[insert_at:insert_at] = block_lines
+    return "\n".join(lines) + "\n"