diff --git a/.github/workflows/publish-container-images.yml b/.github/workflows/publish-container-images.yml index ac6d205..29a7fb3 100644 --- a/.github/workflows/publish-container-images.yml +++ b/.github/workflows/publish-container-images.yml @@ -66,6 +66,7 @@ jobs: ghcr.io/githubsecuritylab/seclab-shell-network-analysis ghcr.io/githubsecuritylab/seclab-shell-source-access ghcr.io/githubsecuritylab/seclab-shell-sast + ghcr.io/githubsecuritylab/seclab-shell-reproduction ) for image in "${images[@]}"; do diff --git a/scripts/audit_v2/run_audit_v2.sh b/scripts/audit_v2/run_audit_v2.sh new file mode 100755 index 0000000..feda5c1 --- /dev/null +++ b/scripts/audit_v2/run_audit_v2.sh @@ -0,0 +1,168 @@ +#!/bin/bash +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +# Run the audit v2 pipeline against a repository. +# +# The five stages are separate taskflows rather than one file on purpose. Each +# one is expensive, and each ends at a durable checkpoint in the finding +# ledger, so a stage can be rerun on its own without redoing the ones before +# it. That is also why stage state lives in the ledger rather than in taskflow +# outputs: multi-model tasks do not feed a shared result channel, so the ledger +# is the only place the stages can meet. +# +# Usage: ./scripts/audit_v2/run_audit_v2.sh [options] +# +# Options: +# -m Override the model config each taskflow declares. +# Use seclab_taskflows.configs.model_config_audit_v2_lowercost +# for cheaper exploratory runs. +# -s Run a single stage: survey|hunt|contest|reproduce|report. +# Repeatable. Default: all five, in order. +# --from Run from this stage to the end, after fixing a stage +# that failed part way through. +# --no-reproduce Skip reproduction. Findings then top out at `confirmed` +# and the report says so. +# -h, --help Show this message. + +set -euo pipefail + +ALL_STAGES=(survey hunt contest reproduce report) +STAGES=() +FROM_STAGE="" +SKIP_REPRODUCE=false +MODEL_CONFIG_FLAG=() + +usage() { + sed -n '6,27p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' +} + +while [[ $# -gt 0 && "$1" == -* ]]; do + case "$1" in + -m) + MODEL_CONFIG_FLAG=(-m "$2") + shift 2 + ;; + -s) + STAGES+=("$2") + shift 2 + ;; + --from) + FROM_STAGE="$2" + shift 2 + ;; + --no-reproduce) + SKIP_REPRODUCE=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +REPO="${1:-}" +if [ -z "$REPO" ]; then + usage >&2 + exit 1 +fi + +# Catch a mistyped target here rather than several minutes into a stage. The +# repo is threaded through to every tool as a single owner/repo string, so it +# has to be exactly that: one slash, and neither half empty or containing +# whitespace. +if [[ ! "$REPO" =~ ^[^/[:space:]]+/[^/[:space:]]+$ ]]; then + echo "Expected a target of the form , got: ${REPO}" >&2 + echo "For example: ${BASH_SOURCE[0]} octocat/hello-world" >&2 + exit 1 +fi + +if [ -n "$FROM_STAGE" ] && [ ${#STAGES[@]} -gt 0 ]; then + echo "Use either --from or -s, not both." >&2 + exit 1 +fi + +if [ -n "$FROM_STAGE" ]; then + seen=false + for stage in "${ALL_STAGES[@]}"; do + [ "$stage" = "$FROM_STAGE" ] && seen=true + [ "$seen" = true ] && STAGES+=("$stage") + done + if [ "$seen" != true ]; then + echo "Unknown stage: ${FROM_STAGE}" >&2 + exit 1 + fi +fi + +if [ ${#STAGES[@]} -eq 0 ]; then + STAGES=("${ALL_STAGES[@]}") +fi + +for stage in "${STAGES[@]}"; do + valid=false + for known in "${ALL_STAGES[@]}"; do + [ "$stage" = "$known" ] && valid=true + done + if [ "$valid" != true ]; then + echo "Unknown stage: ${stage}" >&2 + exit 1 + fi +done + +if [ "$SKIP_REPRODUCE" = true ]; then + filtered=() + for stage in "${STAGES[@]}"; do + [ "$stage" = "reproduce" ] || filtered+=("$stage") + done + STAGES=(${filtered[@]+"${filtered[@]}"}) +fi + +if [ ${#STAGES[@]} -eq 0 ]; then + echo "No stages left to run." >&2 + exit 1 +fi + +# Reproduction is the one stage that executes attacker-controlled input, so a +# missing image is worth catching now rather than halfway through a finding. +for stage in "${STAGES[@]}"; do + if [ "$stage" = "reproduce" ] && + ! docker image inspect ghcr.io/githubsecuritylab/seclab-shell-reproduction:latest >/dev/null 2>&1; then + echo "The reproduction image is missing. Build it with:" >&2 + echo " ./scripts/build_container_images.sh reproduction" >&2 + exit 1 + fi +done + +# Source access containers run with CONTAINER_PERSIST, and their name is a hash +# of image, workspace and network rather than of the workspace's contents. A +# container left over from a run whose workspace has since been recreated keeps +# a bind mount on the old directory, so /workspace comes up empty and every +# stage reads nothing. Drop them and let this run make its own. +stale=$(docker ps -aq --filter "name=^seclab-persist-" 2>/dev/null || true) +if [ -n "$stale" ]; then + echo "Removing stale persistent containers" + # shellcheck disable=SC2086 + docker rm -f $stale >/dev/null +fi + +echo "audit v2: ${REPO}" +echo "stages: ${STAGES[*]}" +echo + +for stage in "${STAGES[@]}"; do + echo "=== ${stage} ===" + python -m seclab_taskflow_agent \ + ${MODEL_CONFIG_FLAG[@]+"${MODEL_CONFIG_FLAG[@]}"} \ + -t "seclab_taskflows.taskflows.audit_v2.${stage}" \ + -g repo="${REPO}" + echo +done + +echo "The findings are in the ledger. Re-read the report at any time with:" +echo " python -m seclab_taskflow_agent -t seclab_taskflows.taskflows.audit_v2.report -g repo=${REPO}" diff --git a/scripts/build_container_images.sh b/scripts/build_container_images.sh index 031434c..07f4062 100755 --- a/scripts/build_container_images.sh +++ b/scripts/build_container_images.sh @@ -6,7 +6,7 @@ # Must be run from the root of the seclab-taskflows repository. # Images must be rebuilt whenever a Dockerfile changes. # -# Usage: ./scripts/build_container_images.sh [base|malware|network|source-access|sast|all] +# Usage: ./scripts/build_container_images.sh [base|malware|network|source-access|sast|reproduction|all] # default: all set -euo pipefail @@ -41,6 +41,11 @@ build_sast() { docker build -t "${IMAGE_PREFIX}/seclab-shell-sast:latest" "${CONTAINERS_DIR}/sast/" } +build_reproduction() { + echo "Building ${IMAGE_PREFIX}/seclab-shell-reproduction..." + docker build -t "${IMAGE_PREFIX}/seclab-shell-reproduction:latest" "${CONTAINERS_DIR}/reproduction/" +} + target="${1:-all}" case "$target" in @@ -62,16 +67,21 @@ case "$target" in build_base build_sast ;; + reproduction) + build_base + build_reproduction + ;; all) build_base build_malware build_network build_source_access build_sast + build_reproduction ;; *) echo "Unknown target: $target" >&2 - echo "Usage: $0 [base|malware|network|source-access|sast|all]" >&2 + echo "Usage: $0 [base|malware|network|source-access|sast|reproduction|all]" >&2 exit 1 ;; esac diff --git a/src/seclab_taskflows/configs/model_config_audit_v2.yaml b/src/seclab_taskflows/configs/model_config_audit_v2.yaml new file mode 100644 index 0000000..8c33441 --- /dev/null +++ b/src/seclab_taskflows/configs/model_config_audit_v2.yaml @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +# Model assignment for the audit v2 pipeline. +# +# The stage names below are chosen so that models can be swapped per stage +# without editing any taskflow. Two properties are deliberate: +# +# 1. The three hunt slots come from three different model families. Models in +# the same family tend to miss the same things, so mixing families buys +# real coverage rather than three correlated opinions. +# +# 2. The adjudicator belongs to a different family than either advocate, so it +# is never grading an argument written by a sibling model. +# +# Swap any entry for a different model if your entitlements differ; nothing in +# the taskflows depends on a specific provider. +# +# Two things to check when swapping. +# +# Backend. Each family is driven through its own SDK: Claude over the native +# Anthropic Messages API (`anthropic_sdk`), everything else over the OpenAI +# surface (`openai_agents`). All of it still goes to CAPI; the backend only +# decides which wire protocol is spoken. `backend` is set explicitly on every +# slot rather than left to the default, so a slot's provider and its SDK +# cannot drift apart unnoticed. +# +# Endpoint. CAPI serves most models from exactly one endpoint, and asking for +# the wrong one fails the whole stage with "model X is not accessible via the +# /chat/completions endpoint". Probed at the time of writing: gemini-3.6-flash +# is chat completions only; gpt-5.6-sol is responses only; gpt-5.4 and +# gpt-5-mini serve both; the Claude models answer on the native +# `/v1/messages` surface, which is `api_type: messages`. `responses` is the +# newer API and is preferred wherever a model offers it. Check a new model on +# both endpoints before relying on it. +# +# Reasoning effort is set on every slot that supports it, including the +# Anthropic ones, where the backend turns it into adaptive thinking. + +seclab-taskflow-agent: + version: "1.0" + filetype: model_config +models: + # Cheap, high-volume bookkeeping: fetching, clearing, summarising ledger + # state and applying attribution. Not a reasoning slot, but it drives tool + # calls in a loop, so it is not the smallest model either. + general_tasks: gpt-5.4 + # Attack-surface mapping and component inventory. Everything downstream hunts + # only what this stage found, so it is not a place to economise. + survey: gpt-5.6-sol + # Three independent hunters, one per family, each the strongest code-analysis + # model that family currently serves. + # + # xAI is deliberately absent everywhere in this file. CAPI rejects grok-4.5 + # for security analysis at the platform level, not the model level: any + # request whose content is vulnerability analysis comes back + # `403 permission-denied ... Failed check: SAFETY_CHECK_TYPE_CYBER`, down to + # a five-line snippet. It is unusable for every role in this pipeline. + hunt_gpt: gpt-5.6-sol + hunt_claude: claude-opus-5 + hunt_gemini: gemini-3.6-flash + # Adversarial contest. Advocates are strong; the judge is from the one family + # that writes neither argument, so it is never grading a sibling's case. + prosecution: gpt-5.6-sol + defense: claude-opus-5 + adjudication: gemini-3.6-flash + # Dynamic reachability validation is long-horizon tool use inside a + # container: the stage has to actually drive the target and observe the flow + # reach the sink, not describe it. claude-opus-5 soft-refuses this work under + # content filtering (it returns a single text turn and calls no tools), so + # this slot runs claude-opus-4.8, which does the same job without tripping it. + reproduction: claude-opus-4.8 + # Final write-up. + reporting: gpt-5.6-sol +model_settings: + general_tasks: + backend: openai_agents + api_type: responses + survey: + backend: openai_agents + api_type: responses + reasoning: + effort: medium + hunt_gpt: + backend: openai_agents + api_type: responses + reasoning: + effort: high + hunt_claude: + backend: anthropic_sdk + api_type: messages + reasoning: + effort: high + hunt_gemini: + backend: openai_agents + api_type: chat_completions + prosecution: + backend: openai_agents + api_type: responses + reasoning: + effort: high + defense: + backend: anthropic_sdk + api_type: messages + reasoning: + effort: high + adjudication: + backend: openai_agents + api_type: chat_completions + reproduction: + backend: anthropic_sdk + api_type: messages + reasoning: + effort: high + reporting: + backend: openai_agents + api_type: responses + reasoning: + effort: medium diff --git a/src/seclab_taskflows/configs/model_config_audit_v2_lowercost.yaml b/src/seclab_taskflows/configs/model_config_audit_v2_lowercost.yaml new file mode 100644 index 0000000..59f091b --- /dev/null +++ b/src/seclab_taskflows/configs/model_config_audit_v2_lowercost.yaml @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +# Lower-cost model assignment for the audit v2 pipeline. +# +# Same stage names as model_config_audit_v2, so it is a drop-in replacement via +# `-c seclab_taskflows.configs.model_config_audit_v2_lowercost`. Cross-family +# diversity is preserved where it matters most (hunt breadth and an +# independent adjudicator), but every slot uses a cheaper model. + +seclab-taskflow-agent: + version: "1.0" + filetype: model_config +models: + general_tasks: gpt-5.4-nano + survey: gpt-5-mini + hunt_gpt: gpt-5.4 + hunt_claude: claude-haiku-4.5 + hunt_gemini: gemini-3.6-flash + prosecution: gpt-5.4 + defense: claude-haiku-4.5 + adjudication: gemini-3.6-flash + reproduction: claude-sonnet-4.6 + reporting: gpt-5-mini +model_settings: + general_tasks: + backend: openai_agents + api_type: responses + survey: + backend: openai_agents + api_type: responses + hunt_gpt: + backend: openai_agents + api_type: responses + reasoning: + effort: high + hunt_claude: + backend: anthropic_sdk + api_type: messages + reasoning: + effort: high + hunt_gemini: + backend: openai_agents + api_type: chat_completions + prosecution: + backend: openai_agents + api_type: responses + reasoning: + effort: high + defense: + backend: anthropic_sdk + api_type: messages + reasoning: + effort: high + adjudication: + backend: openai_agents + api_type: chat_completions + reproduction: + backend: anthropic_sdk + api_type: messages + reasoning: + effort: high + reporting: + backend: openai_agents + api_type: responses diff --git a/src/seclab_taskflows/containers/base/Dockerfile b/src/seclab_taskflows/containers/base/Dockerfile index 9a99e3e..0cbc0d3 100644 --- a/src/seclab_taskflows/containers/base/Dockerfile +++ b/src/seclab_taskflows/containers/base/Dockerfile @@ -6,4 +6,24 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ bash coreutils python3 python3-pip curl wget git ca-certificates \ file binutils xxd \ && rm -rf /var/lib/apt/lists/* + +# Debian bookworm ships Go 1.19, which refuses to build any module declaring a +# newer toolchain, so take Go from upstream. dpkg's architecture names match +# Go's release naming for amd64 and arm64. +ARG GO_VERSION=1.23.4 +RUN arch="$(dpkg --print-architecture)" \ + && case "$arch" in \ + amd64) sha256="6924efde5de86fe277676e929dc9917d466efa02fb934197bc2eba35d5680971" ;; \ + arm64) sha256="16e5017863a7f6071363782b1b8042eb12c6ca4f4cd71528b2123f0a1275b13e" ;; \ + *) echo "unsupported architecture: $arch" >&2; exit 1 ;; \ + esac \ + && curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${arch}.tar.gz" -o /tmp/go.tar.gz \ + && echo "${sha256} /tmp/go.tar.gz" | sha256sum -c - \ + && tar -C /usr/local -xzf /tmp/go.tar.gz \ + && rm /tmp/go.tar.gz +ENV PATH="/usr/local/go/bin:/root/go/bin:${PATH}" +# A login shell re-reads /etc/profile and would drop the PATH set above, so +# make the toolchain reachable that way too. +RUN printf 'export PATH="/usr/local/go/bin:/root/go/bin:$PATH"\n' > /etc/profile.d/golang.sh + WORKDIR /workspace diff --git a/src/seclab_taskflows/containers/reproduction/Dockerfile b/src/seclab_taskflows/containers/reproduction/Dockerfile new file mode 100644 index 0000000..a1f0b84 --- /dev/null +++ b/src/seclab_taskflows/containers/reproduction/Dockerfile @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +# Reproduction container for the audit v2 pipeline. +# +# This is the only image in which candidate exploits are actually executed. It +# carries the runtimes and debugging tools needed to stand a target up and +# trigger a finding, but it is meant to run with no network by default; the +# operator opts into egress explicitly when a target needs to fetch its own +# dependencies. + +FROM ghcr.io/githubsecuritylab/seclab-shell-base:latest +# `ltrace` is deliberately absent: it has no arm64 package on bookworm, which +# would make this image build on CI but not on Apple Silicon. strace covers it. +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential gdb valgrind strace \ + python3-venv python3-dev \ + nodejs npm \ + default-jre-headless \ + jq netcat-openbsd socat procps lsof psmisc \ + ripgrep tree less \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace diff --git a/src/seclab_taskflows/mcp_servers/audit_v2/finding_ledger.py b/src/seclab_taskflows/mcp_servers/audit_v2/finding_ledger.py new file mode 100644 index 0000000..8d769a7 --- /dev/null +++ b/src/seclab_taskflows/mcp_servers/audit_v2/finding_ledger.py @@ -0,0 +1,687 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +"""MCP server exposing the audit v2 finding ledger. + +The ledger is the source of truth for the v2 audit pipeline. Each stage reads +the findings it is responsible for and writes its evidence back, rather than +threading large payloads between tasks. That keeps stages independently +resumable and makes the promotion rules auditable. + +Promotion is enforced here, not in prompts: a finding only becomes +``confirmed`` through adjudication, and only becomes ``reproduced`` when a +reproduction attempt observed its flow reaching the sink from a ``confirmed`` +state. +""" + +import json +import logging + +from fastmcp import FastMCP +from pydantic import Field +from seclab_taskflow_agent.path_utils import log_file_name, mcp_data_dir +from sqlalchemy.orm import Session + +from .finding_ledger_models import ( + CONTEST_POSITIONS, + CONTEST_ROLES, + FINDING_STATES, + OUTCOME_REPRODUCED, + POSITION_EXPLOITABLE, + POSITION_NOT_EXPLOITABLE, + REPRODUCTION_OUTCOMES, + ROLE_ADJUDICATION, + ROLE_DEFENSE, + ROLE_PROSECUTION, + SEVERITIES, + STATE_CANDIDATE, + STATE_CONFIRMED, + STATE_DUPLICATE, + STATE_REJECTED, + STATE_REPRODUCED, + Base, + ContestVerdict, + Finding, + ReproductionAttempt, +) +from ..utils import normalizes_repo, process_repo +from .schema_init import open_state_engine + +logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(levelname)s - %(message)s", + filename=log_file_name("mcp_finding_ledger.log"), + filemode="a", +) + +MEMORY = mcp_data_dir("seclab-taskflows", "finding_ledger", "FINDING_LEDGER_DIR") + + +def _merge_labels(*label_groups) -> str: + """Union comma-separated model labels, preserving first-seen order.""" + seen = [] + for group in label_groups: + for raw in (group or "").split(","): + label = raw.strip() + if label and label not in seen: + seen.append(label) + return ", ".join(seen) + + +def finding_to_dict(f): + try: + locations = json.loads(f.locations or "[]") + except (json.JSONDecodeError, ValueError): + locations = [] + return { + "finding_id": f.id, + "repo": f.repo.lower(), + "component": f.component, + "title": f.title, + "vuln_class": f.vuln_class, + "language": f.language, + "source": f.source, + "sink": f.sink, + "flow": f.flow, + "locations": locations, + "hypothesis": f.hypothesis, + "proposed_by": f.proposed_by, + "state": f.state, + "severity": f.severity, + "disposition_reason": f.disposition_reason, + "duplicate_of": f.duplicate_of, + } + + +def verdict_to_dict(v): + return { + "verdict_id": v.id, + "finding_id": v.finding_id, + "role": v.role, + "model": v.model, + "position": v.position, + "rationale": v.rationale, + } + + +def attempt_to_dict(a): + return { + "attempt_id": a.id, + "finding_id": a.finding_id, + "model": a.model, + "harness": a.harness, + "outcome": a.outcome, + "observed": a.observed, + } + + +class InvalidLedgerValueError(ValueError): + """Raised when a caller supplies a value outside an allowed set.""" + + +def _require(value: str, allowed, name: str) -> str: + """Validate an enum-like argument, raising a message the model can act on.""" + normalized = (value or "").strip().lower() + if normalized not in allowed: + msg = f"invalid {name} {value!r}; expected one of: {', '.join(allowed)}" + raise InvalidLedgerValueError(msg) + return normalized + + +def _repo_mismatch(finding, repo, action: str): + """Message refusing a cross-repo write, or None when the finding matches. + + Finding ids are global within the SQLite file, so every write that names a + repo has to check the finding it targets actually belongs to that repo; + otherwise one repo's stage could attach evidence to another's finding. + """ + if finding.repo != repo: + return ( + f"Finding {finding.id} belongs to {finding.repo!r}, not {repo!r}; " + f"refusing to {action} across repositories" + ) + return None + + +class FindingLedgerBackend: + """Durable store for the audit v2 finding lifecycle. + + The ledger is the only channel between pipeline stages, so it always + materialises a real database file. Other MCP servers in this package fall + back to an in-memory database when their state directory is missing, but + that would be silent data loss here: an audit would appear to run, promote + findings, and then have nothing to show at the end. + """ + + def __init__(self, state_dir: str): + self.state_dir = state_dir + self.engine = open_state_engine( + state_dir, + "finding_ledger.db", + Base, + [ + Finding.__table__, + ContestVerdict.__table__, + ReproductionAttempt.__table__, + ], + ) + + def dispose(self): + """Release the SQLite file handle held by the engine. + + A long-lived server never needs this, but a test that opens a ledger in + a temporary directory has to let go of the file before the directory can + be removed, which on Windows fails while any handle is still open. + """ + self.engine.dispose() + + # -- writes ------------------------------------------------------------ + + @normalizes_repo + def store_finding( + self, + repo, + component, + title, + vuln_class, + language, + source, + sink, + flow, + locations, + hypothesis, + proposed_by, + ): + with Session(self.engine) as session: + finding = Finding( + repo=repo, + component=component, + title=title, + vuln_class=vuln_class, + language=language or "", + source=source or "", + sink=sink or "", + flow=flow or "", + locations=json.dumps(list(locations or [])), + hypothesis=hypothesis or "", + proposed_by=proposed_by or "", + state=STATE_CANDIDATE, + ) + session.add(finding) + session.commit() + return finding.id + + @normalizes_repo + def attribute_finding(self, repo, finding_id, proposed_by): + """Union a model label onto a finding's ``proposed_by``. + + Hunters cannot name the model they are running as, so the labels they + self-report are unreliable and, worse, they collide: two hunters that + both guess "unknown" union down to a single label, and a path that two + families found independently then reads as one. The runner does know + which model produced each branch, so attribution is applied afterwards + from the branch records rather than trusted from inside the branch. + """ + with Session(self.engine) as session: + finding = session.get(Finding, finding_id) + if finding is None: + return f"No finding with id {finding_id}" + mismatch = _repo_mismatch(finding, repo, "attribute") + if mismatch: + return mismatch + label = (proposed_by or "").strip() + if not label: + return "proposed_by must be a non-empty model label" + finding.proposed_by = _merge_labels(finding.proposed_by, label) + labels = finding.proposed_by + session.commit() + return f"Finding {finding_id} was proposed by: {labels}" + + @normalizes_repo + def attribute_findings(self, repo, attributions): + """Apply a whole run's worth of attribution in one call. + + ``attributions`` is one entry per hunt branch, as + ``{"proposed_by":