diff --git a/docs/hackbot/tracing.md b/docs/hackbot/tracing.md index a7ebe630a1..92ba80fbe1 100644 --- a/docs/hackbot/tracing.md +++ b/docs/hackbot/tracing.md @@ -13,6 +13,15 @@ It is **opt-in**: the runtime only traces when it has W&B credentials, and never fails a run if Weave can't start. `WEAVE_PROJECT` picks the destination project (a bare `project` or `entity/project`; defaults to `hackbot-test`). +## Linking a run to its traces + +The runtime tags every span with the hackbot run id (surfaced by Weave as +`attributes.hackbot.run_id`, see [tracing.py](../../libs/hackbot-runtime/hackbot_runtime/tracing.py)). +The Hackbot UI links each run page to the Weave **Agents** view filtered on that +attribute, so all of a run's conversations show up together. The UI's +`WEAVE_PROJECT` (`entity/project`, default `moz-bugbug/hackbot-dev`) must point at +the project the agents trace into. + **Locally**, add the key to your root `.env`; the agent's `compose.yml` should pass `WANDB_API_KEY` through to the agent container. diff --git a/libs/hackbot-runtime/hackbot_runtime/runtime.py b/libs/hackbot-runtime/hackbot_runtime/runtime.py index 7b926e5fd7..30af26eea0 100644 --- a/libs/hackbot-runtime/hackbot_runtime/runtime.py +++ b/libs/hackbot-runtime/hackbot_runtime/runtime.py @@ -193,7 +193,7 @@ def run(entrypoint: AgentMain, config: ConfigArg = None) -> NoReturn: try: _configure_auth() - with trace_agent(entrypoint): + with trace_agent(entrypoint, ctx.run_id): outcome: object = entrypoint(ctx) except Exception as exc: log.exception("Agent raised an exception") @@ -210,7 +210,7 @@ def run_async(entrypoint: AsyncAgentMain, config: ConfigArg = None) -> NoReturn: try: _configure_auth() - with trace_agent(entrypoint): + with trace_agent(entrypoint, ctx.run_id): outcome: object = asyncio.run(entrypoint(ctx)) except Exception as exc: log.exception("Agent raised an exception") diff --git a/libs/hackbot-runtime/hackbot_runtime/tracing.py b/libs/hackbot-runtime/hackbot_runtime/tracing.py index 83ba0f71bd..4691148146 100644 --- a/libs/hackbot-runtime/hackbot_runtime/tracing.py +++ b/libs/hackbot-runtime/hackbot_runtime/tracing.py @@ -19,15 +19,27 @@ import os from collections.abc import Callable, Iterator from pathlib import Path +from typing import TYPE_CHECKING + +from opentelemetry import trace as otel_trace +from opentelemetry.sdk.trace import SpanProcessor, TracerProvider from hackbot_runtime import wandb_wif +if TYPE_CHECKING: + from opentelemetry.context import Context + from opentelemetry.sdk.trace import Span + log = logging.getLogger("hackbot_runtime") # Weave project traces land in when the deploy doesn't set WEAVE_PROJECT. Accepts # either "project" or "entity/project". DEFAULT_WEAVE_PROJECT = "hackbot-test" +# Weave's tagging namespace: the server surfaces this as `attributes.hackbot.run_id` +# on every call, which the Hackbot UI uses to link a run to its traces. +RUN_ID_SPAN_ATTRIBUTE = "wandb.attributes.hackbot.run_id" + def resolve_agent_name(entrypoint: Callable) -> str: """The running agent's name, derived from its ``main()`` source file. @@ -64,9 +76,23 @@ def _init_weave() -> bool: return False +class _RunIdSpanProcessor(SpanProcessor): + def __init__(self, run_id: str) -> None: + self._run_id = run_id + + def on_start(self, span: "Span", parent_context: "Context | None" = None) -> None: + span.set_attribute(RUN_ID_SPAN_ATTRIBUTE, self._run_id) + + +def _tag_spans_with_run_id(run_id: str) -> None: + provider = otel_trace.get_tracer_provider() + if isinstance(provider, TracerProvider): + provider.add_span_processor(_RunIdSpanProcessor(run_id)) + + @contextlib.contextmanager -def trace_agent(entrypoint: Callable) -> Iterator[None]: - """Trace the agent run and label its Weave spans with the agent's name. +def trace_agent(entrypoint: Callable, run_id: str) -> Iterator[None]: + """Trace the agent run, labelled with the agent's name and tagged with the run id. A no-op when tracing isn't configured (no W&B credentials). """ @@ -76,6 +102,7 @@ def trace_agent(entrypoint: Callable) -> Iterator[None]: from weave.conversation import agent_name_override + _tag_spans_with_run_id(run_id) agent = resolve_agent_name(entrypoint) log.info("Enabled Weave tracing for agent %s", agent) with agent_name_override(agent): diff --git a/libs/hackbot-runtime/tests/test_tracing.py b/libs/hackbot-runtime/tests/test_tracing.py index 4a1bee7135..f777c881a7 100644 --- a/libs/hackbot-runtime/tests/test_tracing.py +++ b/libs/hackbot-runtime/tests/test_tracing.py @@ -2,6 +2,7 @@ import pytest from hackbot_runtime import tracing, wandb_wif +from opentelemetry.sdk.trace import TracerProvider from weave.conversation.agent_context import resolve_agent_name as weave_agent_name @@ -69,7 +70,7 @@ def test_init_weave_enabled_by_wif_token_file(monkeypatch): def test_trace_agent_is_noop_without_credentials(): entry = _entrypoint_at("/app/hackbot_agents/build_repair/__main__.py") - with tracing.trace_agent(entry): + with tracing.trace_agent(entry, "run-1"): assert weave_agent_name("claude_agent_sdk") == "claude_agent_sdk" @@ -77,6 +78,28 @@ def test_trace_agent_labels_spans_when_enabled(monkeypatch): monkeypatch.setattr(tracing, "_init_weave", lambda: True) entry = _entrypoint_at("/app/hackbot_agents/build_repair/__main__.py") - with tracing.trace_agent(entry): + with tracing.trace_agent(entry, "run-1"): assert weave_agent_name("claude_agent_sdk") == "build-repair" assert weave_agent_name("claude_agent_sdk") == "claude_agent_sdk" + + +def test_trace_agent_tags_spans_with_run_id(monkeypatch): + monkeypatch.setattr(tracing, "_init_weave", lambda: True) + provider = TracerProvider() + monkeypatch.setattr(tracing.otel_trace, "get_tracer_provider", lambda: provider) + entry = _entrypoint_at("/app/hackbot_agents/build_repair/__main__.py") + + with tracing.trace_agent(entry, "run-1"): + with provider.get_tracer("test").start_as_current_span("turn") as span: + pass + + assert span.attributes[tracing.RUN_ID_SPAN_ATTRIBUTE] == "run-1" + + +def test_trace_agent_skips_tagging_without_sdk_provider(monkeypatch): + monkeypatch.setattr(tracing, "_init_weave", lambda: True) + monkeypatch.setattr(tracing.otel_trace, "get_tracer_provider", lambda: object()) + entry = _entrypoint_at("/app/hackbot_agents/build_repair/__main__.py") + + with tracing.trace_agent(entry, "run-1"): + pass diff --git a/services/hackbot-ui/.env.example b/services/hackbot-ui/.env.example index 560468e4e7..906208ae32 100644 --- a/services/hackbot-ui/.env.example +++ b/services/hackbot-ui/.env.example @@ -5,6 +5,9 @@ HACKBOT_API_URL=http://localhost:8080 # EXTERNAL_API_KEY. Stays server-side — it is never shipped to the browser. HACKBOT_API_KEY= +# Weave project (entity/project) the agents trace into; the run page links there. +WEAVE_PROJECT=moz-bugbug/hackbot-dev + # Public base URL of THIS web app, used by better-auth for callbacks. BETTER_AUTH_URL=http://localhost:3000 diff --git a/services/hackbot-ui/README.md b/services/hackbot-ui/README.md index c14c2d4828..a22d87f5f7 100644 --- a/services/hackbot-ui/README.md +++ b/services/hackbot-ui/README.md @@ -132,6 +132,7 @@ must match its `X-API-Key`. | ---------------------- | -------------------------------------------------- | | `HACKBOT_API_URL` | Base URL of hackbot-api (no trailing slash) | | `HACKBOT_API_KEY` | Value for the `X-API-Key` header (server-side) | +| `WEAVE_PROJECT` | Weave `entity/project` for run trace links | | `BETTER_AUTH_URL` | Public base URL of this app | | `BETTER_AUTH_SECRET` | Session signing secret (`openssl rand -base64 32`) | | `GOOGLE_CLIENT_ID` | Google OAuth client ID | diff --git a/services/hackbot-ui/app/runs/[runId]/page.tsx b/services/hackbot-ui/app/runs/[runId]/page.tsx index 244da0dbd5..c85a56569f 100644 --- a/services/hackbot-ui/app/runs/[runId]/page.tsx +++ b/services/hackbot-ui/app/runs/[runId]/page.tsx @@ -1,4 +1,5 @@ import { RunDetail } from "@/components/RunDetail"; +import { weaveProject, weaveRunTracesUrl } from "@/lib/weave"; export default async function RunPage({ params, @@ -7,5 +8,11 @@ export default async function RunPage({ }) { const { runId } = await params; - return ; + return ( + + ); } diff --git a/services/hackbot-ui/components/RunDetail.tsx b/services/hackbot-ui/components/RunDetail.tsx index 64dc77779a..7f9935f6a8 100644 --- a/services/hackbot-ui/components/RunDetail.tsx +++ b/services/hackbot-ui/components/RunDetail.tsx @@ -53,7 +53,13 @@ function extractLog(run: RunDoc): string | null { return null; } -export function RunDetail({ runId }: { runId: string }) { +export function RunDetail({ + runId, + tracesUrl, +}: { + runId: string; + tracesUrl: string; +}) { const router = useRouter(); const [run, setRun] = useState(null); const [error, setError] = useState(null); @@ -232,6 +238,12 @@ export function RunDetail({ runId }: { runId: string }) {
{run.execution_name}
)} +
Traces
+
+ + Weave + +