From 44f756464935ebcf5336aadc36719810eb1480e9 Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Wed, 29 Jul 2026 14:11:54 -0400 Subject: [PATCH 01/12] Add audit v2: contested, reproduction-gated vulnerability discovery Five-stage pipeline (survey, hunt, contest, reproduce, report) whose finding lifecycle is enforced by a new finding_ledger MCP server rather than by prompt text. Adds the audit_v2 taskflows, personalities, prompts, toolboxes, the container images and a corpus render test. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: dadec5f9-3bf8-449c-84d6-db45be19bb6a --- .../workflows/publish-container-images.yml | 1 + scripts/audit_v2/run_audit_v2.sh | 146 +++++ scripts/build_container_images.sh | 14 +- .../configs/model_config_audit_v2.yaml | 57 ++ .../model_config_audit_v2_lowercost.yaml | 39 ++ .../containers/reproduction/Dockerfile | 23 + .../mcp_servers/finding_ledger.py | 553 ++++++++++++++++++ .../mcp_servers/finding_ledger_models.py | 141 +++++ .../personalities/exploit_defender.yaml | 49 ++ .../personalities/exploit_prosecutor.yaml | 41 ++ .../personalities/finding_adjudicator.yaml | 59 ++ .../personalities/reproduction_engineer.yaml | 54 ++ .../personalities/vulnerability_hunter.yaml | 52 ++ .../prompts/audit_v2/contest_rules.yaml | 55 ++ .../prompts/audit_v2/evidence_rules.yaml | 32 + .../prompts/audit_v2/finding_contract.yaml | 47 ++ .../prompts/audit_v2/reproduction_rules.yaml | 65 ++ .../prompts/audit_v2/severity_rubric.yaml | 38 ++ .../taskflows/audit_v2/README.md | 171 ++++++ .../taskflows/audit_v2/contest.yaml | 202 +++++++ .../taskflows/audit_v2/hunt.yaml | 209 +++++++ .../taskflows/audit_v2/report.yaml | 162 +++++ .../taskflows/audit_v2/reproduce.yaml | 119 ++++ .../taskflows/audit_v2/survey.yaml | 190 ++++++ .../container_shell_reproduction.yaml | 51 ++ .../toolboxes/finding_ledger.yaml | 46 ++ tests/test_finding_ledger.py | 367 ++++++++++++ tests/test_taskflow_corpus.py | 197 +++++++ 28 files changed, 3178 insertions(+), 2 deletions(-) create mode 100755 scripts/audit_v2/run_audit_v2.sh create mode 100644 src/seclab_taskflows/configs/model_config_audit_v2.yaml create mode 100644 src/seclab_taskflows/configs/model_config_audit_v2_lowercost.yaml create mode 100644 src/seclab_taskflows/containers/reproduction/Dockerfile create mode 100644 src/seclab_taskflows/mcp_servers/finding_ledger.py create mode 100644 src/seclab_taskflows/mcp_servers/finding_ledger_models.py create mode 100644 src/seclab_taskflows/personalities/exploit_defender.yaml create mode 100644 src/seclab_taskflows/personalities/exploit_prosecutor.yaml create mode 100644 src/seclab_taskflows/personalities/finding_adjudicator.yaml create mode 100644 src/seclab_taskflows/personalities/reproduction_engineer.yaml create mode 100644 src/seclab_taskflows/personalities/vulnerability_hunter.yaml create mode 100644 src/seclab_taskflows/prompts/audit_v2/contest_rules.yaml create mode 100644 src/seclab_taskflows/prompts/audit_v2/evidence_rules.yaml create mode 100644 src/seclab_taskflows/prompts/audit_v2/finding_contract.yaml create mode 100644 src/seclab_taskflows/prompts/audit_v2/reproduction_rules.yaml create mode 100644 src/seclab_taskflows/prompts/audit_v2/severity_rubric.yaml create mode 100644 src/seclab_taskflows/taskflows/audit_v2/README.md create mode 100644 src/seclab_taskflows/taskflows/audit_v2/contest.yaml create mode 100644 src/seclab_taskflows/taskflows/audit_v2/hunt.yaml create mode 100644 src/seclab_taskflows/taskflows/audit_v2/report.yaml create mode 100644 src/seclab_taskflows/taskflows/audit_v2/reproduce.yaml create mode 100644 src/seclab_taskflows/taskflows/audit_v2/survey.yaml create mode 100644 src/seclab_taskflows/toolboxes/container_shell_reproduction.yaml create mode 100644 src/seclab_taskflows/toolboxes/finding_ledger.yaml create mode 100644 tests/test_finding_ledger.py create mode 100644 tests/test_taskflow_corpus.py diff --git a/.github/workflows/publish-container-images.yml b/.github/workflows/publish-container-images.yml index ac6d205..29a7fb3 100644 --- a/.github/workflows/publish-container-images.yml +++ b/.github/workflows/publish-container-images.yml @@ -66,6 +66,7 @@ jobs: ghcr.io/githubsecuritylab/seclab-shell-network-analysis ghcr.io/githubsecuritylab/seclab-shell-source-access ghcr.io/githubsecuritylab/seclab-shell-sast + ghcr.io/githubsecuritylab/seclab-shell-reproduction ) for image in "${images[@]}"; do diff --git a/scripts/audit_v2/run_audit_v2.sh b/scripts/audit_v2/run_audit_v2.sh new file mode 100755 index 0000000..aa6cc57 --- /dev/null +++ b/scripts/audit_v2/run_audit_v2.sh @@ -0,0 +1,146 @@ +#!/bin/bash +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +# Run the audit v2 pipeline against a repository. +# +# The five stages are separate taskflows rather than one file on purpose. Each +# one is expensive, and each ends at a durable checkpoint in the finding +# ledger, so a stage can be rerun on its own without redoing the ones before +# it. That is also why stage state lives in the ledger rather than in taskflow +# outputs: multi-model tasks do not feed a shared result channel, so the ledger +# is the only place the stages can meet. +# +# Usage: ./scripts/audit_v2/run_audit_v2.sh [options] +# +# Options: +# -m Override the model config each taskflow declares. +# Use seclab_taskflows.configs.model_config_audit_v2_lowercost +# for cheaper exploratory runs. +# -s Run a single stage: survey|hunt|contest|reproduce|report. +# Repeatable. Default: all five, in order. +# --from Run from this stage to the end, after fixing a stage +# that failed part way through. +# --no-reproduce Skip reproduction. Findings then top out at `confirmed` +# and the report says so. +# -h, --help Show this message. + +set -euo pipefail + +ALL_STAGES=(survey hunt contest reproduce report) +STAGES=() +FROM_STAGE="" +SKIP_REPRODUCE=false +MODEL_CONFIG_FLAG=() + +usage() { + sed -n '6,27p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' +} + +while [[ $# -gt 0 && "$1" == -* ]]; do + case "$1" in + -m) + MODEL_CONFIG_FLAG=(-m "$2") + shift 2 + ;; + -s) + STAGES+=("$2") + shift 2 + ;; + --from) + FROM_STAGE="$2" + shift 2 + ;; + --no-reproduce) + SKIP_REPRODUCE=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +REPO="${1:-}" +if [ -z "$REPO" ]; then + usage >&2 + exit 1 +fi + +if [ -n "$FROM_STAGE" ] && [ ${#STAGES[@]} -gt 0 ]; then + echo "Use either --from or -s, not both." >&2 + exit 1 +fi + +if [ -n "$FROM_STAGE" ]; then + seen=false + for stage in "${ALL_STAGES[@]}"; do + [ "$stage" = "$FROM_STAGE" ] && seen=true + [ "$seen" = true ] && STAGES+=("$stage") + done + if [ "$seen" != true ]; then + echo "Unknown stage: ${FROM_STAGE}" >&2 + exit 1 + fi +fi + +if [ ${#STAGES[@]} -eq 0 ]; then + STAGES=("${ALL_STAGES[@]}") +fi + +for stage in "${STAGES[@]}"; do + valid=false + for known in "${ALL_STAGES[@]}"; do + [ "$stage" = "$known" ] && valid=true + done + if [ "$valid" != true ]; then + echo "Unknown stage: ${stage}" >&2 + exit 1 + fi +done + +if [ "$SKIP_REPRODUCE" = true ]; then + filtered=() + for stage in "${STAGES[@]}"; do + [ "$stage" = "reproduce" ] || filtered+=("$stage") + done + STAGES=(${filtered[@]+"${filtered[@]}"}) +fi + +if [ ${#STAGES[@]} -eq 0 ]; then + echo "No stages left to run." >&2 + exit 1 +fi + +# Reproduction is the one stage that executes attacker-controlled input, so a +# missing image is worth catching now rather than halfway through a finding. +for stage in "${STAGES[@]}"; do + if [ "$stage" = "reproduce" ] && + ! docker image inspect ghcr.io/githubsecuritylab/seclab-shell-reproduction:latest >/dev/null 2>&1; then + echo "The reproduction image is missing. Build it with:" >&2 + echo " ./scripts/build_container_images.sh reproduction" >&2 + exit 1 + fi +done + +echo "audit v2: ${REPO}" +echo "stages: ${STAGES[*]}" +echo + +for stage in "${STAGES[@]}"; do + echo "=== ${stage} ===" + python -m seclab_taskflow_agent \ + ${MODEL_CONFIG_FLAG[@]+"${MODEL_CONFIG_FLAG[@]}"} \ + -t "seclab_taskflows.taskflows.audit_v2.${stage}" \ + -g repo="${REPO}" + echo +done + +echo "The findings are in the ledger. Re-read the report at any time with:" +echo " python -m seclab_taskflow_agent -t seclab_taskflows.taskflows.audit_v2.report -g repo=${REPO}" diff --git a/scripts/build_container_images.sh b/scripts/build_container_images.sh index 031434c..07f4062 100755 --- a/scripts/build_container_images.sh +++ b/scripts/build_container_images.sh @@ -6,7 +6,7 @@ # Must be run from the root of the seclab-taskflows repository. # Images must be rebuilt whenever a Dockerfile changes. # -# Usage: ./scripts/build_container_images.sh [base|malware|network|source-access|sast|all] +# Usage: ./scripts/build_container_images.sh [base|malware|network|source-access|sast|reproduction|all] # default: all set -euo pipefail @@ -41,6 +41,11 @@ build_sast() { docker build -t "${IMAGE_PREFIX}/seclab-shell-sast:latest" "${CONTAINERS_DIR}/sast/" } +build_reproduction() { + echo "Building ${IMAGE_PREFIX}/seclab-shell-reproduction..." + docker build -t "${IMAGE_PREFIX}/seclab-shell-reproduction:latest" "${CONTAINERS_DIR}/reproduction/" +} + target="${1:-all}" case "$target" in @@ -62,16 +67,21 @@ case "$target" in build_base build_sast ;; + reproduction) + build_base + build_reproduction + ;; all) build_base build_malware build_network build_source_access build_sast + build_reproduction ;; *) echo "Unknown target: $target" >&2 - echo "Usage: $0 [base|malware|network|source-access|sast|all]" >&2 + echo "Usage: $0 [base|malware|network|source-access|sast|reproduction|all]" >&2 exit 1 ;; esac diff --git a/src/seclab_taskflows/configs/model_config_audit_v2.yaml b/src/seclab_taskflows/configs/model_config_audit_v2.yaml new file mode 100644 index 0000000..d8d27a3 --- /dev/null +++ b/src/seclab_taskflows/configs/model_config_audit_v2.yaml @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +# Model assignment for the audit v2 pipeline. +# +# The stage names below are chosen so that models can be swapped per stage +# without editing any taskflow. Two properties are deliberate: +# +# 1. The three hunt slots come from three different model families. Models in +# the same family tend to miss the same things, so mixing families buys +# real coverage rather than three correlated opinions. +# +# 2. The adjudicator belongs to a different family than either advocate, so it +# is never grading an argument written by a sibling model. +# +# Swap any entry for a different model if your entitlements differ; nothing in +# the taskflows depends on a specific provider. + +seclab-taskflow-agent: + version: "1.0" + filetype: model_config +models: + # Cheap, high-volume bookkeeping: fetching, clearing, summarising ledger state. + general_tasks: gpt-5-mini + # Attack-surface mapping and component inventory. + survey: gpt-5.4 + # Three independent hunters, one per model family. + hunt_gpt: gpt-5.6-sol + hunt_claude: claude-sonnet-5 + hunt_gemini: gemini-3.6-flash + # Adversarial contest. Advocates are strong; the judge is from a third family. + prosecution: gpt-5.6-sol + defense: claude-sonnet-5 + adjudication: grok-4.5 + # Dynamic reproduction is long-horizon tool use inside a container. + reproduction: claude-sonnet-5 + # Final write-up. + reporting: gpt-5.5 +model_settings: + general_tasks: + api_type: responses + survey: + api_type: responses + reasoning: + effort: medium + hunt_gpt: + api_type: responses + reasoning: + effort: high + prosecution: + api_type: responses + reasoning: + effort: high + reporting: + api_type: responses + reasoning: + effort: medium diff --git a/src/seclab_taskflows/configs/model_config_audit_v2_lowercost.yaml b/src/seclab_taskflows/configs/model_config_audit_v2_lowercost.yaml new file mode 100644 index 0000000..15c41ff --- /dev/null +++ b/src/seclab_taskflows/configs/model_config_audit_v2_lowercost.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +# Lower-cost model assignment for the audit v2 pipeline. +# +# Same stage names as model_config_audit_v2, so it is a drop-in replacement via +# `-c seclab_taskflows.configs.model_config_audit_v2_lowercost`. Cross-family +# diversity is preserved where it matters most (hunt breadth and an +# independent adjudicator), but every slot uses a cheaper model. + +seclab-taskflow-agent: + version: "1.0" + filetype: model_config +models: + general_tasks: gpt-5.4-nano + survey: gpt-5-mini + hunt_gpt: gpt-5.4 + hunt_claude: claude-haiku-4.5 + hunt_gemini: gemini-3.6-flash + prosecution: gpt-5.4 + defense: claude-haiku-4.5 + adjudication: gemini-3.6-flash + reproduction: claude-sonnet-4.6 + reporting: gpt-5-mini +model_settings: + general_tasks: + api_type: responses + survey: + api_type: responses + hunt_gpt: + api_type: responses + reasoning: + effort: medium + prosecution: + api_type: responses + reasoning: + effort: medium + reporting: + api_type: responses diff --git a/src/seclab_taskflows/containers/reproduction/Dockerfile b/src/seclab_taskflows/containers/reproduction/Dockerfile new file mode 100644 index 0000000..1f5dc60 --- /dev/null +++ b/src/seclab_taskflows/containers/reproduction/Dockerfile @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +# Reproduction container for the audit v2 pipeline. +# +# This is the only image in which candidate exploits are actually executed. It +# carries the runtimes and debugging tools needed to stand a target up and +# trigger a finding, but it is meant to run with no network by default; the +# operator opts into egress explicitly when a target needs to fetch its own +# dependencies. + +FROM ghcr.io/githubsecuritylab/seclab-shell-base:latest +# `ltrace` is deliberately absent: it has no arm64 package on bookworm, which +# would make this image build on CI but not on Apple Silicon. strace covers it. +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential gdb valgrind strace \ + python3-venv python3-dev \ + nodejs npm \ + default-jre-headless \ + jq netcat-openbsd socat procps lsof psmisc \ + ripgrep tree less \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /workspace diff --git a/src/seclab_taskflows/mcp_servers/finding_ledger.py b/src/seclab_taskflows/mcp_servers/finding_ledger.py new file mode 100644 index 0000000..782dcdc --- /dev/null +++ b/src/seclab_taskflows/mcp_servers/finding_ledger.py @@ -0,0 +1,553 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +"""MCP server exposing the audit v2 finding ledger. + +The ledger is the source of truth for the v2 audit pipeline. Each stage reads +the findings it is responsible for and writes its evidence back, rather than +threading large payloads between tasks. That keeps stages independently +resumable and makes the promotion rules auditable. + +Promotion is enforced here, not in prompts: a finding only becomes +``confirmed`` through adjudication, and only becomes ``reproduced`` when a +reproduction attempt actually triggered it from a ``confirmed`` state. +""" + +import json +import logging + +from fastmcp import FastMCP +from pydantic import Field +from seclab_taskflow_agent.path_utils import log_file_name, mcp_data_dir +from sqlalchemy import create_engine +from sqlalchemy.orm import Session + +from pathlib import Path + +from .finding_ledger_models import ( + CONTEST_POSITIONS, + CONTEST_ROLES, + FINDING_STATES, + OUTCOME_REPRODUCED, + POSITION_EXPLOITABLE, + POSITION_NOT_EXPLOITABLE, + REPRODUCTION_OUTCOMES, + ROLE_ADJUDICATION, + ROLE_DEFENSE, + ROLE_PROSECUTION, + SEVERITIES, + STATE_CANDIDATE, + STATE_CONFIRMED, + STATE_DUPLICATE, + STATE_REJECTED, + STATE_REPRODUCED, + Base, + ContestVerdict, + Finding, + ReproductionAttempt, +) +from .utils import process_repo + +logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(levelname)s - %(message)s", + filename=log_file_name("mcp_finding_ledger.log"), + filemode="a", +) + +MEMORY = mcp_data_dir("seclab-taskflows", "finding_ledger", "FINDING_LEDGER_DIR") + + +def _merge_labels(*label_groups) -> str: + """Union comma-separated model labels, preserving first-seen order.""" + seen = [] + for group in label_groups: + for raw in (group or "").split(","): + label = raw.strip() + if label and label not in seen: + seen.append(label) + return ", ".join(seen) + + +def finding_to_dict(f): + try: + locations = json.loads(f.locations or "[]") + except (json.JSONDecodeError, ValueError): + locations = [] + return { + "finding_id": f.id, + "repo": f.repo.lower(), + "component": f.component, + "title": f.title, + "vuln_class": f.vuln_class, + "language": f.language, + "source": f.source, + "sink": f.sink, + "flow": f.flow, + "locations": locations, + "hypothesis": f.hypothesis, + "proposed_by": f.proposed_by, + "state": f.state, + "severity": f.severity, + "disposition_reason": f.disposition_reason, + "duplicate_of": f.duplicate_of, + } + + +def verdict_to_dict(v): + return { + "verdict_id": v.id, + "finding_id": v.finding_id, + "role": v.role, + "model": v.model, + "position": v.position, + "rationale": v.rationale, + } + + +def attempt_to_dict(a): + return { + "attempt_id": a.id, + "finding_id": a.finding_id, + "model": a.model, + "harness": a.harness, + "outcome": a.outcome, + "observed": a.observed, + } + + +class InvalidLedgerValueError(ValueError): + """Raised when a caller supplies a value outside an allowed set.""" + + +def _require(value: str, allowed, name: str) -> str: + """Validate an enum-like argument, raising a message the model can act on.""" + normalized = (value or "").strip().lower() + if normalized not in allowed: + msg = f"invalid {name} {value!r}; expected one of: {', '.join(allowed)}" + raise InvalidLedgerValueError(msg) + return normalized + + +class FindingLedgerBackend: + def __init__(self, state_dir: str): + self.state_dir = state_dir + db_dir = ( + f"sqlite:///{self.state_dir}/finding_ledger.db" + if Path(self.state_dir).exists() + else "sqlite://" + ) + self.engine = create_engine(db_dir, echo=False) + Base.metadata.create_all( + self.engine, + tables=[ + Finding.__table__, + ContestVerdict.__table__, + ReproductionAttempt.__table__, + ], + ) + + # -- writes ------------------------------------------------------------ + + def store_finding( + self, + repo, + component, + title, + vuln_class, + language, + source, + sink, + flow, + locations, + hypothesis, + proposed_by, + ): + with Session(self.engine) as session: + finding = Finding( + repo=repo, + component=component, + title=title, + vuln_class=vuln_class, + language=language or "", + source=source or "", + sink=sink or "", + flow=flow or "", + locations=json.dumps(list(locations or [])), + hypothesis=hypothesis or "", + proposed_by=proposed_by or "", + state=STATE_CANDIDATE, + ) + session.add(finding) + session.commit() + return finding.id + + def store_contest_verdict(self, repo, finding_id, role, model, position, rationale): + role = _require(role, CONTEST_ROLES, "role") + position = _require(position, CONTEST_POSITIONS, "position") + with Session(self.engine) as session: + finding = session.get(Finding, finding_id) + if finding is None: + return f"No finding with id {finding_id}" + session.add( + ContestVerdict( + finding_id=finding_id, + repo=repo, + role=role, + model=model or "", + position=position, + rationale=rationale or "", + ) + ) + session.commit() + return f"Recorded {role} verdict ({position}) for finding {finding_id}" + + def adjudicate_finding(self, repo, finding_id, position, severity, rationale): + """Resolve a contested finding. This is the only path to ``confirmed``. + + Adjudication requires that both advocates have actually filed. A + contest with only one side is not a contest, and letting a finding + reach ``confirmed`` on an unopposed argument would quietly undo the + thing this stage exists to do. + """ + position = _require(position, CONTEST_POSITIONS, "position") + severity = _require(severity, SEVERITIES, "severity") + with Session(self.engine) as session: + finding = session.get(Finding, finding_id) + if finding is None: + return f"No finding with id {finding_id}" + if finding.state == STATE_REPRODUCED: + return f"Finding {finding_id} is already reproduced; adjudication ignored" + filed = { + role + for (role,) in session.query(ContestVerdict.role).filter( + ContestVerdict.finding_id == finding_id + ) + } + missing = [r for r in (ROLE_PROSECUTION, ROLE_DEFENSE) if r not in filed] + if missing: + return ( + f"Finding {finding_id} cannot be adjudicated yet; no " + f"{' or '.join(missing)} verdict has been filed. Call " + f"`store_contest_verdict` for each side first." + ) + if position == POSITION_EXPLOITABLE: + finding.state = STATE_CONFIRMED + elif position == POSITION_NOT_EXPLOITABLE: + finding.state = STATE_REJECTED + else: + finding.state = STATE_CANDIDATE + finding.severity = severity + finding.disposition_reason = rationale or "" + session.add( + ContestVerdict( + finding_id=finding_id, + repo=repo, + role=ROLE_ADJUDICATION, + model="", + position=position, + rationale=rationale or "", + ) + ) + session.commit() + return f"Finding {finding_id} adjudicated {position}; state is now {finding.state}" + + def merge_duplicate_finding(self, repo, duplicate_id, canonical_id): + """Fold one candidate into another, carrying its provenance across. + + Independent hunters converging on the same path is the useful signal + here, so the canonical finding accumulates every model label that + proposed it rather than discarding the duplicates' provenance. + """ + if duplicate_id == canonical_id: + return f"Finding {duplicate_id} cannot be a duplicate of itself" + with Session(self.engine) as session: + duplicate = session.get(Finding, duplicate_id) + if duplicate is None: + return f"No finding with id {duplicate_id}" + canonical = session.get(Finding, canonical_id) + if canonical is None: + return f"No finding with id {canonical_id}" + for label, finding in (("duplicate", duplicate), ("canonical", canonical)): + if finding.repo != repo: + return ( + f"Finding {finding.id} belongs to {finding.repo!r}, not {repo!r}; " + f"refusing to merge across repositories ({label})" + ) + if canonical.state == STATE_DUPLICATE: + return ( + f"Finding {canonical_id} is itself a duplicate of " + f"{canonical.duplicate_of}; merge into that one instead" + ) + if duplicate.state != STATE_CANDIDATE: + return ( + f"Finding {duplicate_id} is {duplicate.state!r}; only candidates " + f"can be merged as duplicates" + ) + labels = _merge_labels(canonical.proposed_by, duplicate.proposed_by) + canonical.proposed_by = labels + duplicate.state = STATE_DUPLICATE + duplicate.duplicate_of = canonical_id + session.commit() + return ( + f"Finding {duplicate_id} merged into {canonical_id}; " + f"{canonical_id} was proposed by: {labels}" + ) + + def store_reproduction_attempt(self, repo, finding_id, model, harness, outcome, observed): + """Record a dynamic trigger attempt; only a real trigger promotes state.""" + outcome = _require(outcome, REPRODUCTION_OUTCOMES, "outcome") + with Session(self.engine) as session: + finding = session.get(Finding, finding_id) + if finding is None: + return f"No finding with id {finding_id}" + session.add( + ReproductionAttempt( + finding_id=finding_id, + repo=repo, + model=model or "", + harness=harness or "", + outcome=outcome, + observed=observed or "", + ) + ) + promoted = False + if outcome == OUTCOME_REPRODUCED and finding.state == STATE_CONFIRMED: + finding.state = STATE_REPRODUCED + promoted = True + session.commit() + if promoted: + return f"Finding {finding_id} reproduced; state is now {STATE_REPRODUCED}" + if outcome == OUTCOME_REPRODUCED: + return ( + f"Recorded reproduction for finding {finding_id}, but its state is " + f"{finding.state!r} (must be {STATE_CONFIRMED!r} to be promoted)" + ) + return f"Recorded {outcome} reproduction attempt for finding {finding_id}" + + def clear_findings_for_repo(self, repo): + with Session(self.engine) as session: + ids = [f.id for f in session.query(Finding).filter_by(repo=repo).all()] + if ids: + session.query(ContestVerdict).filter(ContestVerdict.finding_id.in_(ids)).delete( + synchronize_session=False + ) + session.query(ReproductionAttempt).filter( + ReproductionAttempt.finding_id.in_(ids) + ).delete(synchronize_session=False) + session.query(Finding).filter_by(repo=repo).delete() + session.commit() + return f"Cleared {len(ids)} findings for {repo}" + + # -- reads ------------------------------------------------------------- + + def get_findings(self, repo, state=None): + with Session(self.engine) as session: + query = session.query(Finding).filter_by(repo=repo) + if state: + query = query.filter_by(state=state) + return [finding_to_dict(f) for f in query.all()] + + def get_finding(self, finding_id): + with Session(self.engine) as session: + finding = session.get(Finding, finding_id) + if finding is None: + return None + data = finding_to_dict(finding) + data["verdicts"] = [ + verdict_to_dict(v) + for v in session.query(ContestVerdict).filter_by(finding_id=finding_id).all() + ] + data["reproduction_attempts"] = [ + attempt_to_dict(a) + for a in session.query(ReproductionAttempt).filter_by(finding_id=finding_id).all() + ] + return data + + def find_similar_findings(self, repo, component, vuln_class): + with Session(self.engine) as session: + query = session.query(Finding).filter_by(repo=repo) + if component: + query = query.filter_by(component=component) + if vuln_class: + query = query.filter_by(vuln_class=vuln_class) + return [finding_to_dict(f) for f in query.all()] + + def get_ledger_summary(self, repo): + with Session(self.engine) as session: + findings = session.query(Finding).filter_by(repo=repo).all() + counts = dict.fromkeys(FINDING_STATES, 0) + for f in findings: + counts[f.state] = counts.get(f.state, 0) + 1 + return {"repo": repo.lower(), "total": len(findings), "by_state": counts} + + +backend = FindingLedgerBackend(MEMORY) + +mcp = FastMCP("FindingLedger") + + +@mcp.tool() +def store_finding( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), + component: str = Field(description="Directory or module the finding belongs to"), + title: str = Field(description="Short one-line description of the finding"), + vuln_class: str = Field(description="Vulnerability class, e.g. CWE-22 or 'path traversal'"), + language: str = Field(description="Primary language of the affected code", default=""), + source: str = Field(description="Where the untrusted input originates", default=""), + sink: str = Field(description="The dangerous operation reached by the input", default=""), + flow: str = Field(description="How the source reaches the sink", default=""), + locations: list[str] = Field( + description="Evidence locations as 'path:line' strings", default_factory=list + ), + hypothesis: str = Field(description="Why this may be exploitable", default=""), + proposed_by: str = Field(description="Label of the model proposing the finding", default=""), +): + """Store a new candidate finding and return its id.""" + repo = process_repo(owner, repo) + finding_id = backend.store_finding( + repo, + component, + title, + vuln_class, + language, + source, + sink, + flow, + locations, + hypothesis, + proposed_by, + ) + return json.dumps({"finding_id": finding_id, "state": STATE_CANDIDATE}) + + +@mcp.tool() +def get_findings( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), + state: str = Field( + description=f"Optional state filter, one of: {', '.join(FINDING_STATES)}", default="" + ), +): + """Get all findings for a repository, optionally filtered by lifecycle state.""" + repo = process_repo(owner, repo) + return json.dumps(backend.get_findings(repo, state or None)) + + +@mcp.tool() +def get_finding( + finding_id: int = Field(description="The ID of the finding"), +): + """Get one finding with all its contest verdicts and reproduction attempts.""" + result = backend.get_finding(finding_id) + if result is None: + return f"No finding with id {finding_id}" + return json.dumps(result) + + +@mcp.tool() +def find_similar_findings( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), + component: str = Field(description="Component to match", default=""), + vuln_class: str = Field(description="Vulnerability class to match", default=""), +): + """Find existing findings in the same component and class, to avoid duplicates.""" + repo = process_repo(owner, repo) + return json.dumps(backend.find_similar_findings(repo, component, vuln_class)) + + +@mcp.tool() +def store_contest_verdict( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), + finding_id: int = Field(description="The ID of the finding being contested"), + role: str = Field(description=f"One of: {', '.join(CONTEST_ROLES)}"), + position: str = Field(description=f"One of: {', '.join(CONTEST_POSITIONS)}"), + rationale: str = Field(description="Evidence-backed argument for this position", default=""), + model: str = Field(description="Label of the model taking this position", default=""), +): + """Record a prosecution or defense position on a finding. Does not change state.""" + repo = process_repo(owner, repo) + try: + return backend.store_contest_verdict(repo, finding_id, role, model, position, rationale) + except InvalidLedgerValueError as exc: + return f"Error: {exc}" + + +@mcp.tool() +def adjudicate_finding( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), + finding_id: int = Field(description="The ID of the finding to adjudicate"), + position: str = Field(description=f"One of: {', '.join(CONTEST_POSITIONS)}"), + severity: str = Field(description=f"One of: {', '.join(SEVERITIES)}"), + rationale: str = Field(description="Why the prosecution or defense prevailed", default=""), +): + """Resolve a contested finding. This is the only way a finding becomes confirmed.""" + repo = process_repo(owner, repo) + try: + return backend.adjudicate_finding(repo, finding_id, position, severity, rationale) + except InvalidLedgerValueError as exc: + return f"Error: {exc}" + + +@mcp.tool() +def merge_duplicate_finding( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), + duplicate_id: int = Field(description="The ID of the finding to fold away"), + canonical_id: int = Field(description="The ID of the finding to keep"), +): + """Mark one candidate as a duplicate of another, merging its model provenance. + + Only candidates can be merged. The canonical finding keeps a combined + `proposed_by` list, so convergence between independent hunters is preserved. + """ + repo = process_repo(owner, repo) + return backend.merge_duplicate_finding(repo, duplicate_id, canonical_id) + + +@mcp.tool() +def store_reproduction_attempt( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), + finding_id: int = Field(description="The ID of the finding being reproduced"), + outcome: str = Field(description=f"One of: {', '.join(REPRODUCTION_OUTCOMES)}"), + harness: str = Field(description="The exact commands or PoC used", default=""), + observed: str = Field(description="What actually happened when the PoC ran", default=""), + model: str = Field(description="Label of the model that ran the attempt", default=""), +): + """Record a dynamic reproduction attempt run inside the sandboxed container.""" + repo = process_repo(owner, repo) + try: + return backend.store_reproduction_attempt( + repo, finding_id, model, harness, outcome, observed + ) + except InvalidLedgerValueError as exc: + return f"Error: {exc}" + + +@mcp.tool() +def get_ledger_summary( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), +): + """Get counts of findings by lifecycle state for a repository.""" + repo = process_repo(owner, repo) + return json.dumps(backend.get_ledger_summary(repo)) + + +@mcp.tool() +def clear_findings_for_repo( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), +): + """Delete all findings and their evidence for a repository.""" + repo = process_repo(owner, repo) + return backend.clear_findings_for_repo(repo) + + +if __name__ == "__main__": + mcp.run(show_banner=False) diff --git a/src/seclab_taskflows/mcp_servers/finding_ledger_models.py b/src/seclab_taskflows/mcp_servers/finding_ledger_models.py new file mode 100644 index 0000000..113791c --- /dev/null +++ b/src/seclab_taskflows/mcp_servers/finding_ledger_models.py @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +"""SQLAlchemy models for the audit v2 finding ledger. + +The ledger tracks a finding through an explicit lifecycle: + +``candidate`` -> (adversarial contest) -> ``confirmed`` | ``rejected`` +``confirmed`` -> (dynamic reproduction) -> ``reproduced`` +``candidate`` -> (deduplication) -> ``duplicate`` + +State is never set directly by a model. It is derived by the backend from +adjudication and reproduction records, so a finding cannot reach a stronger +state than its recorded evidence supports. +""" + +from datetime import datetime, timezone + +from sqlalchemy import Column, DateTime, ForeignKey, Integer, Text +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +class Base(DeclarativeBase): + pass + + +# Lifecycle states, ordered weakest to strongest. +STATE_CANDIDATE = "candidate" +STATE_DUPLICATE = "duplicate" +STATE_REJECTED = "rejected" +STATE_CONFIRMED = "confirmed" +STATE_REPRODUCED = "reproduced" + +FINDING_STATES = ( + STATE_CANDIDATE, + STATE_DUPLICATE, + STATE_REJECTED, + STATE_CONFIRMED, + STATE_REPRODUCED, +) + +# Contest roles and the positions a role may take. +ROLE_PROSECUTION = "prosecution" +ROLE_DEFENSE = "defense" +ROLE_ADJUDICATION = "adjudication" +CONTEST_ROLES = (ROLE_PROSECUTION, ROLE_DEFENSE, ROLE_ADJUDICATION) + +POSITION_EXPLOITABLE = "exploitable" +POSITION_NOT_EXPLOITABLE = "not_exploitable" +POSITION_UNCERTAIN = "uncertain" +CONTEST_POSITIONS = (POSITION_EXPLOITABLE, POSITION_NOT_EXPLOITABLE, POSITION_UNCERTAIN) + +# Reproduction outcomes. +OUTCOME_REPRODUCED = "reproduced" +OUTCOME_NOT_REPRODUCED = "not_reproduced" +OUTCOME_INCONCLUSIVE = "inconclusive" +REPRODUCTION_OUTCOMES = (OUTCOME_REPRODUCED, OUTCOME_NOT_REPRODUCED, OUTCOME_INCONCLUSIVE) + +SEVERITIES = ("critical", "high", "medium", "low", "none") + + +class Finding(Base): + """A single candidate or confirmed vulnerability in a repository.""" + + __tablename__ = "finding" + + id: Mapped[int] = mapped_column(primary_key=True) + repo: Mapped[str] + component: Mapped[str] + title: Mapped[str] + vuln_class: Mapped[str] + language: Mapped[str] = mapped_column(default="") + # Taint-style triple describing the claimed issue. + source: Mapped[str] = mapped_column(Text, default="") + sink: Mapped[str] = mapped_column(Text, default="") + flow: Mapped[str] = mapped_column(Text, default="") + # JSON-encoded list of "path:line" strings. + locations: Mapped[str] = mapped_column(Text, default="[]") + hypothesis: Mapped[str] = mapped_column(Text, default="") + # Comma-separated model labels. More than one means independent hunters + # converged on the same path, which is a meaningful prior. + proposed_by: Mapped[str] = mapped_column(default="") + state: Mapped[str] = mapped_column(default=STATE_CANDIDATE) + severity: Mapped[str] = mapped_column(default="") + disposition_reason: Mapped[str] = mapped_column(Text, default="") + # Set when this finding was folded into another as a duplicate. + duplicate_of: Mapped[int | None] = mapped_column(Integer, default=None, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime, default=_utcnow) + + def __repr__(self): + return ( + f"" + ) + + +class ContestVerdict(Base): + """One role's position on a finding during adversarial validation.""" + + __tablename__ = "contest_verdict" + + id: Mapped[int] = mapped_column(primary_key=True) + finding_id = Column(Integer, ForeignKey("finding.id", ondelete="CASCADE")) + repo: Mapped[str] + role: Mapped[str] + model: Mapped[str] = mapped_column(default="") + position: Mapped[str] + rationale: Mapped[str] = mapped_column(Text, default="") + created_at: Mapped[datetime] = mapped_column(DateTime, default=_utcnow) + + def __repr__(self): + return ( + f"" + ) + + +class ReproductionAttempt(Base): + """A dynamic attempt to trigger a finding inside a sandboxed container.""" + + __tablename__ = "reproduction_attempt" + + id: Mapped[int] = mapped_column(primary_key=True) + finding_id = Column(Integer, ForeignKey("finding.id", ondelete="CASCADE")) + repo: Mapped[str] + model: Mapped[str] = mapped_column(default="") + harness: Mapped[str] = mapped_column(Text, default="") + outcome: Mapped[str] + observed: Mapped[str] = mapped_column(Text, default="") + created_at: Mapped[datetime] = mapped_column(DateTime, default=_utcnow) + + def __repr__(self): + return ( + f"" + ) diff --git a/src/seclab_taskflows/personalities/exploit_defender.yaml b/src/seclab_taskflows/personalities/exploit_defender.yaml new file mode 100644 index 0000000..6b1e7e8 --- /dev/null +++ b/src/seclab_taskflows/personalities/exploit_defender.yaml @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: personality + version: "1.0" + +personality: | + You are the defense in an adversarial review of a candidate vulnerability. + Your job is to find the specific, citable reason the finding is wrong, and + most of the time there is one. Automated code audit produces far more + plausible-looking findings than real ones, and you are the reason the report + does not fill up with them. + + You break findings in order of how often each actually works: + + 1. **The source is not attacker-controlled.** It comes from configuration, + from a trusted caller, from the operator's own command line, or from a + value the attacker cannot influence. Show where it really originates. + 2. **The path is not reachable.** The entry point is dead code, a test + helper, an example, gated behind a feature flag that is off, or behind a + privilege the attacker would have to already hold. + 3. **Something stops it.** There is a check, an escape, a parameterised + query, a type conversion, an allow-list, a framework guarantee. Read it + and show that it actually holds for this payload, on every branch. + 4. **The flow does not connect.** The claimed hops do not really call each + other; the value is copied, replaced, or dropped somewhere in the middle. + 5. **The impact is not what is claimed.** The sink is reached, but with data + or in a context that does not produce the claimed consequence. + + You are rigorous, not reflexive. A defense that says "this is probably + handled by the framework" without reading the framework is worthless, and the + adjudicator will see straight through it. If you have to reach for a + hypothetical mitigation you did not find in the code, you have already lost + the argument, and the honest move is to concede. + + Conceding is a legitimate outcome. If the finding survives everything you can + throw at it, record `exploitable` and say which of your attacks failed and + why. That is a strong signal, and it is worth more than a defense you cannot + support. + +task: | + Argue, from cited code, that the candidate finding you are given is not + exploitable, and record your position in the finding ledger. + +toolboxes: + - seclab_taskflow_agent.toolboxes.memcache + - seclab_taskflows.toolboxes.container_shell_source_access + - seclab_taskflows.toolboxes.finding_ledger diff --git a/src/seclab_taskflows/personalities/exploit_prosecutor.yaml b/src/seclab_taskflows/personalities/exploit_prosecutor.yaml new file mode 100644 index 0000000..d4afcb1 --- /dev/null +++ b/src/seclab_taskflows/personalities/exploit_prosecutor.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: personality + version: "1.0" + +personality: | + You are the prosecution in an adversarial review of a candidate vulnerability. + Your job is to build the strongest honest case that the finding is real and + exploitable, and to build it out of code rather than assertion. + + A prosecution that convinces the adjudicator does three things: + + 1. It shows the source is genuinely attacker-controlled, by naming the entry + point and the code path that carries the value inward. + 2. It shows the flow survives, by reading every check between source and sink + and explaining why each one fails to stop the payload. Encoding that is + reversed later, validation that runs on the wrong value, a check that is + skipped on some branch, a normalisation that happens before the dangerous + transformation rather than after. + 3. It states the concrete impact and the preconditions honestly, including + what the attacker must already have. + + You know the ways a defense will try to kill a finding, and you address them + before they are raised: "that input is validated", "that path is unreachable", + "only an admin can call that", "the framework escapes it". If one of those is + actually true, you say so. You are trying to find real bugs, not to win. + + Conceding is a legitimate and valuable outcome. If reading the code shows the + finding is wrong, record `not_exploitable` and explain what kills it. That + saves the audit more than a hollow win costs it. + +task: | + Argue, from cited code, that the candidate finding you are given is + exploitable, and record your position in the finding ledger. + +toolboxes: + - seclab_taskflow_agent.toolboxes.memcache + - seclab_taskflows.toolboxes.container_shell_source_access + - seclab_taskflows.toolboxes.finding_ledger diff --git a/src/seclab_taskflows/personalities/finding_adjudicator.yaml b/src/seclab_taskflows/personalities/finding_adjudicator.yaml new file mode 100644 index 0000000..9d0704d --- /dev/null +++ b/src/seclab_taskflows/personalities/finding_adjudicator.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: personality + version: "1.0" + +personality: | + You are the adjudicator in an adversarial review of a candidate vulnerability. + Two other models have argued the finding, one for and one against. You decide. + + You are not a vote counter and you are not a summariser. Both arguments are + evidence, not authority. When they disagree about what the code does, you go + and read the code yourself and settle it. The single most common way this + role fails is by siding with whichever argument was written more confidently, + so treat confident prose with suspicion and check its citations. + + How you decide: + + - Identify the exact point of disagreement. Usually it is one specific + question: is this value attacker-controlled, does this check hold, is this + entry point reachable. Resolve that question against the source. + - Verify citations that carry weight. If an argument rests on a line, open + the line. An argument whose citations do not say what it claims they say + loses, regardless of how well it reads. + - Weigh what was not addressed. If the defense never engaged with the + prosecution's strongest hop, that silence counts against it, and the same + is true in reverse. + - Decide on the specific path, not on the general area or on how likely this + class of bug is in this kind of project. + + Your three outcomes: + + - `exploitable` — the prosecution's path holds against the defense's attacks + and against your own reading. The finding becomes confirmed and goes on to + dynamic reproduction. + - `not_exploitable` — something concrete and citable stops it. The finding is + rejected. Name what stops it, so a reader can check you. + - `uncertain` — the code genuinely does not settle it, usually because it + depends on runtime configuration, on a dependency's behaviour, or on data + you cannot see. The finding stays a candidate rather than being discarded. + + Use `uncertain` when it is true, and do not use it to avoid making a call. + A decision you can justify is worth more than a hedge, in both directions: + rejecting a finding that should have been rejected is as valuable as + confirming one that should be confirmed. + + Whatever you decide, your rationale is what a human reviewer will read first. + Write it so that someone who has not seen either argument can follow the + reasoning and check it against the code. + +task: | + Decide whether a contested finding is exploitable, assign its severity, and + record the adjudication in the finding ledger. + +toolboxes: + - seclab_taskflow_agent.toolboxes.memcache + - seclab_taskflows.toolboxes.container_shell_source_access + - seclab_taskflows.toolboxes.finding_ledger diff --git a/src/seclab_taskflows/personalities/reproduction_engineer.yaml b/src/seclab_taskflows/personalities/reproduction_engineer.yaml new file mode 100644 index 0000000..0667f53 --- /dev/null +++ b/src/seclab_taskflows/personalities/reproduction_engineer.yaml @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: personality + version: "1.0" + +personality: | + You are an exploit engineer. You are given a finding that survived + adversarial review, and your job is to find out whether it is actually true + by making it happen inside a sandboxed container. + + You are the last honest step in the audit. Everything before you was + reasoning about code; you are the only stage that observes behaviour. That + makes you the only stage that can tell the difference between a vulnerability + and a very well-argued misunderstanding, and it means your reports have to be + literal. You report what the container printed, not what you expected it to + print. + + How you work: + + 1. Understand the target well enough to run it. Read the README, the build + files, the test suite, the example configuration. The tests are usually + the fastest route to a working invocation of the vulnerable code path. + 2. Stand the target up. Build it, install it, start the service, whatever it + takes. Note every step, because the reproduction is only useful if + somebody else can repeat it. + 3. Establish the control first. Drive the path with benign input and record + what normal looks like. Without that, the malicious result proves nothing. + 4. Then attack it. Start from the entry point the finding names, with a + minimal, non-destructive payload: read a marker file you created, echo a + marker string, trigger the crash under gdb or valgrind. + 5. Iterate on failure. A first attempt that does not work usually means your + payload or your invocation is wrong, not that the finding is wrong. Change + one thing at a time and keep going while you are still learning something. + 6. Know when to stop. If the target cannot be built or started with what you + have, that is an `inconclusive` result and it is a legitimate one. Say + exactly where you got stuck. + + You are working in an isolated container with no network by default. If a + step genuinely requires fetching dependencies, record that in your notes + rather than working around it. + + You never fabricate output. If you did not see it in the container, it did + not happen. + +task: | + Attempt to reproduce a confirmed finding inside the reproduction container, + and record exactly what you observed in the finding ledger. + +toolboxes: + - seclab_taskflow_agent.toolboxes.memcache + - seclab_taskflows.toolboxes.container_shell_reproduction + - seclab_taskflows.toolboxes.finding_ledger diff --git a/src/seclab_taskflows/personalities/vulnerability_hunter.yaml b/src/seclab_taskflows/personalities/vulnerability_hunter.yaml new file mode 100644 index 0000000..1eed9b0 --- /dev/null +++ b/src/seclab_taskflows/personalities/vulnerability_hunter.yaml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: personality + version: "1.0" + +personality: | + You are a vulnerability researcher auditing source code. You work across + languages and across kinds of software: web applications, libraries, parsers, + command line tools, daemons, build tooling, and native code. You do not + assume the target is a web application, and you do not go looking only for + the vulnerability classes that happen to be common in web applications. + + What you are good at is following data. You start from something an attacker + controls and you trace it, hop by hop, through the real call graph, until it + either reaches something dangerous or is genuinely neutralised. You read the + bodies of the functions in between, including the ones that sound safe. + + You know what untrusted input looks like in code that is not a web server: + + - anything parsed from a file, archive, image, packet, or serialised blob + whose contents came from somewhere else + - arguments and environment that a lower-privileged process controls + - data read back out of a database, cache, or queue that some other user put + there + - filenames, paths, and metadata inside archives and repositories + - responses from a network peer, including ones the target chose to contact + - for a library, whatever the library's documented API tells its callers is + safe to pass in + + You know what dangerous means beyond the obvious sinks: memory corruption in + native code, deserialisation, template and expression evaluation, path + resolution, process execution, dynamic import and reflection, SQL and command + string construction, cryptographic misuse, and authorisation checks that can + be skipped. + + You are deliberately broad at this stage. Your candidates will be argued + against by a model whose job is to destroy them, so you do not need to + pre-emptively discard anything you can defend with code. But you also do not + pad: a candidate you cannot trace is worse than no candidate, because it + costs the contest stage real time. + +task: | + Hunt for concrete, attacker-reachable vulnerabilities in the component you + are given, and file each one in the finding ledger as a candidate. + +toolboxes: + - seclab_taskflow_agent.toolboxes.memcache + - seclab_taskflows.toolboxes.container_shell_source_access + - seclab_taskflows.toolboxes.repo_context + - seclab_taskflows.toolboxes.finding_ledger diff --git a/src/seclab_taskflows/prompts/audit_v2/contest_rules.yaml b/src/seclab_taskflows/prompts/audit_v2/contest_rules.yaml new file mode 100644 index 0000000..251c5ae --- /dev/null +++ b/src/seclab_taskflows/prompts/audit_v2/contest_rules.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + version: "1.0" + filetype: prompt + +prompt: | + ## How the contest works + + Every candidate finding is argued by two independent models before anything + is decided about it: + + - the **prosecution** argues that the finding is exploitable, + - the **defense** argues that it is not, + - an **adjudicator** from a third model family reads both arguments and the + code, and decides. + + The point of this is not debate for its own sake. Most false positives in + automated code audit survive because nobody was ever tasked with killing + them. Your job in your assigned role is to make the strongest honest case you + can, so that whichever side survives has actually been tested. + + ## Rules that bind both sides + + 1. Argue from code, not from plausibility. Every load-bearing sentence needs + a `path/to/file.ext:line` behind it. + + 2. Do not argue your side into a position the code does not support. If the + evidence goes against you, say so explicitly and record it. A prosecution + that concedes a real sanitiser, or a defense that concedes a real gap, is + doing its job. Winning an argument about a finding that turns out to be + wrong costs the audit more than losing one. + + 3. Attack the specific path, not the general area. "This project validates + input elsewhere" is not a defense of this flow. "The value is dangerous in + general" is not a prosecution of this flow. + + 4. Address the other side's argument when you have it. A rebuttal that + ignores the strongest opposing point is not a rebuttal. + + ## Positions + + Record exactly one of: + + - `exploitable` — the path is reachable by an attacker and reaches the + sink in a state that causes the claimed impact. + - `not_exploitable` — something concrete stops it. Name it and cite it: an + effective check, an unreachable entry point, a type + that cannot carry the payload, a caller that is always + trusted. + - `uncertain` — you could not resolve it from the code available. Say + precisely what you would need to see. Use this + honestly; it routes the finding to dynamic + reproduction rather than throwing it away. diff --git a/src/seclab_taskflows/prompts/audit_v2/evidence_rules.yaml b/src/seclab_taskflows/prompts/audit_v2/evidence_rules.yaml new file mode 100644 index 0000000..25585a4 --- /dev/null +++ b/src/seclab_taskflows/prompts/audit_v2/evidence_rules.yaml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + version: "1.0" + filetype: prompt + +prompt: | + ## Evidence rules + + These rules apply to every claim you make in this audit. + + 1. Every claim about the code must point at code you actually read, cited as + `path/to/file.ext:line`. If you did not open the file, you do not know + what is in it. + + 2. Never infer behaviour from a name. `sanitize_path()`, `is_safe()` and + `validate_input()` are just identifiers until you have read their bodies. + Follow the call through to the implementation before relying on it. + + 3. Do not speculate. If a step in your reasoning depends on something you + could not confirm, say which step and what you were unable to confirm, + rather than filling the gap with a plausible assumption. + + 4. Absence of evidence is not evidence. "I could not find a check" is only + meaningful once you have searched for one; say where you looked. + + 5. Prefer being wrong out loud to being vague. A precise claim that can be + checked and refuted is worth more here than a hedged one that cannot. + + 6. Do not ask the operator for permission to continue. Work autonomously + until the task is done, then report what you found. diff --git a/src/seclab_taskflows/prompts/audit_v2/finding_contract.yaml b/src/seclab_taskflows/prompts/audit_v2/finding_contract.yaml new file mode 100644 index 0000000..a2385f7 --- /dev/null +++ b/src/seclab_taskflows/prompts/audit_v2/finding_contract.yaml @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + version: "1.0" + filetype: prompt + +prompt: | + ## What a finding is + + A finding is a concrete, attacker-reachable path from untrusted input to a + dangerous operation. It is not a code smell, a missing best practice, or a + hardening opportunity. If nothing an attacker controls reaches the dangerous + operation, it is not a finding. + + Every finding you file must name all three parts of the path: + + - **source** — where the untrusted data enters, with a file and line. Say + who controls it and why they are untrusted. + - **sink** — the dangerous operation reached, with a file and line. + - **flow** — how the data actually gets from source to sink: the calls it + passes through, and every check it does or does not survive on the way. + + A finding whose flow is "the value is eventually used unsafely" is not a + finding, it is a guess. Trace the real path. + + `locations` must list the file and line of each hop that matters, as + `path/to/file.ext:line` strings, starting at the source and ending at the + sink. + + `hypothesis` states what an attacker gains: read arbitrary files, execute + commands as the service user, bypass authentication, and so on. Be specific + about the impact, and about what the attacker must already have in order to + reach the source. + + ## Filing findings + + Before filing, call `find_similar_findings` for the same component and + vulnerability class. If your path is the same as an existing finding, do not + file a duplicate. + + Call `store_finding` once per distinct path. Set `proposed_by` to the label + you were given for this hunt, so the ledger records which model proposed it. + + Findings enter the ledger as `candidate`. That is all they are at this point: + you are proposing a hypothesis for adversarial review, not declaring a + vulnerability. Do not describe your own candidates as confirmed. diff --git a/src/seclab_taskflows/prompts/audit_v2/reproduction_rules.yaml b/src/seclab_taskflows/prompts/audit_v2/reproduction_rules.yaml new file mode 100644 index 0000000..085f186 --- /dev/null +++ b/src/seclab_taskflows/prompts/audit_v2/reproduction_rules.yaml @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + version: "1.0" + filetype: prompt + +prompt: | + ## What counts as a reproduction + + A reproduction is something you watched happen. You built or started the + target inside the reproduction container, you sent the input, and you read + the result in the container's output. + + These are reproductions: + + - a response that contains data the attacker should not be able to read + - a file that exists on disk after the request that should not exist + - a process that ran and left evidence you can show + - a crash, an assertion failure, a stack trace, a sanitizer report + - a non-zero exit or hang that is directly attributable to the input + + These are **not** reproductions, no matter how convincing they look: + + - a proof of concept you wrote but did not run + - a description of what the code "would" do + - a successful request that you did not verify actually reached the sink + - output you reconstructed rather than copied from the container + + ## Recording the outcome + + Record exactly one outcome: + + - `reproduced` — you triggered it and observed the impact. Put the exact + commands in `harness` and the actual observed output in + `observed`. Only this outcome promotes a confirmed + finding. + - `not_reproduced` — you got the target running, drove the path, and it did + not do what the finding claims. Say what happened + instead; that is a real result and it is valuable. + - `inconclusive` — you could not get far enough to find out. Say exactly + where you stopped: the build failed, a dependency was + unavailable, the service would not start, the entry + point needs credentials you do not have. + + `inconclusive` is not a failure and it is not a polite `reproduced`. A + finding that stays confirmed-but-unreproduced is an honest result. A finding + marked reproduced on evidence you did not see is a lie in the report. + + ## Working method + + 1. Read the finding first: its source, sink, flow, and the adjudicator's + reasoning. You are testing that specific path. + 2. Get the target running before you try to exploit it. Note in `harness` + what you had to do; the next person needs it. + 3. Establish a control. Show the benign input behaving normally, then show + the malicious input behaving differently. A result with no control is not + evidence of anything. + 4. Keep the payload minimal and non-destructive. Read a marker file, echo a + marker string, trigger the crash. Do not do damage to prove you could. + 5. If you cannot reach the sink through the intended entry point, do not + reach it by calling the vulnerable function directly and calling that a + reproduction. That proves the sink is dangerous, which was never in + dispute. Record it as `inconclusive` and say the entry point was not + reachable. diff --git a/src/seclab_taskflows/prompts/audit_v2/severity_rubric.yaml b/src/seclab_taskflows/prompts/audit_v2/severity_rubric.yaml new file mode 100644 index 0000000..36cb120 --- /dev/null +++ b/src/seclab_taskflows/prompts/audit_v2/severity_rubric.yaml @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + version: "1.0" + filetype: prompt + +prompt: | + ## Severity + + Severity describes what an attacker gets and what it costs them to get it. + Rate the demonstrated impact of this specific path, not the worst case the + vulnerability class can reach in general. + + - `critical` — unauthenticated remote code execution, unauthenticated full + data access, or complete authentication bypass, reachable by + default with no unusual configuration. + - `high` — code execution, arbitrary file read or write, or privilege + escalation that needs low-privileged access, a non-default + but realistic configuration, or a plausible user interaction. + - `medium` — meaningful but bounded impact: information disclosure of + non-secret data, denial of service, a partial bypass, or a + path that needs privileges most users do not have. + - `low` — real but minor: limited disclosure, a bypass that requires + conditions an attacker cannot reliably arrange, or impact an + attacker with the required access already had anyway. + - `none` — the finding was rejected. There is no impact. + + Two things pull severity down, and you should apply them: + + - **Preconditions.** If the attacker must already be an administrator, the + finding is worth less. State the preconditions and rate accordingly. + - **Reachability.** If the path is only reachable from an entry point that is + disabled by default, say so and rate accordingly. + + One thing does not pull severity down: the fact that the code is a library + rather than an application. Judge library findings by what happens to the + applications that use the library as documented. diff --git a/src/seclab_taskflows/taskflows/audit_v2/README.md b/src/seclab_taskflows/taskflows/audit_v2/README.md new file mode 100644 index 0000000..5033487 --- /dev/null +++ b/src/seclab_taskflows/taskflows/audit_v2/README.md @@ -0,0 +1,171 @@ +# Audit v2 + +A vulnerability discovery pipeline that treats a finding as a claim to be +prosecuted, not a report to be filed. + +## Why this exists + +The v1 audit taskflows ask a model to read code and say what looks wrong. That +works, and it also produces a lot of confident prose about vulnerabilities that +do not exist. The two things it is missing are adversarial review and proof. + +Audit v2 adds both, and makes them structural rather than advisory: + +1. **Every candidate is contested.** A prosecutor argues the finding is real, a + defender argues it is not, and a third model from a different family + adjudicates. Findings that survive are marked `confirmed`; the rest are + `rejected` with a reason. +2. **Confirmed findings must be reproduced.** The reproduction stage stands the + target up in a container and actually triggers the bug. Only an observed + trigger promotes a finding to `reproduced`. + +It is also not web-specific. The survey stage asks where untrusted data crosses +a trust boundary, which is a question you can ask of a parser, a daemon, a +library or a build plugin just as well as of a web app. + +## The lifecycle is enforced in code + +The important design decision is that a finding's state is derived by the +`finding_ledger` MCP server, never set by a model. + +``` + store_finding + | + v + [candidate] ------ merge_duplicate_finding ---> [duplicate] + | + adjudicate_finding + / \ + v v + [rejected] [confirmed] + | + store_reproduction_attempt + (outcome: triggered) + | + v + [reproduced] +``` + +No tool accepts a state as an argument. A model can record evidence, and the +backend decides what that evidence entitles the finding to: + +- `adjudicate_finding` is the only path to `confirmed`, and it is only + reachable after both sides of the contest have filed their arguments. +- `store_reproduction_attempt` only promotes a finding that is already + `confirmed`, and only when it observed a real trigger. +- A `reproduced` finding is immune to later adjudication. Once something has + been demonstrated, no amount of subsequent argument un-demonstrates it. +- `merge_duplicate_finding` only folds `candidate` findings, refuses to merge a + finding into itself or into another duplicate, and refuses to merge across + repositories. + +This matters because prompts are advice and code is not. A model that decides +mid-run that its finding is obviously real cannot promote it by saying so. + +## Convergence is signal, not noise + +Three hunters from three model families run over each component. When two of +them independently land on the same path, `merge_duplicate_finding` unions +their `proposed_by` labels rather than discarding the duplicate's provenance. +A finding proposed by three families is a materially different object from one +proposed by a single model, and the adjudicator gets to see that. + +## Model assignment + +Defined in `configs/model_config_audit_v2.yaml`. + +| Role | Model | Why | +| --- | --- | --- | +| `general_tasks` | gpt-5-mini | Cheap bookkeeping: fetching and summarising ledger state | +| `survey` | gpt-5.4 | Long-context mapping work | +| `hunt_gpt` | gpt-5.6-sol | | +| `hunt_claude` | claude-sonnet-5 | Different family, different blind spots | +| `hunt_gemini` | gemini-3.6-flash | Third family, cheap enough to run wide | +| `prosecution` | gpt-5.6-sol | | +| `defense` | claude-sonnet-5 | | +| `adjudication` | grok-4.5 | Deliberately not a sibling of either advocate | +| `reproduction` | claude-sonnet-5 | Long agentic tool-use loops in a container | +| `reporting` | gpt-5.5 | | + +The adjudicator's family is the point. A model grading an argument written by a +sibling shares its priors, including the wrong ones. Only the gpt slots set +`reasoning.effort`, because that setting is provider-specific; swap any entry +for a model your account is entitled to. Use +`model_config_audit_v2_lowercost.yaml` for exploratory runs. + +## Running it + +Build the reproduction image once: + +```bash +./scripts/build_container_images.sh reproduction +``` + +Then run the pipeline: + +```bash +./scripts/audit_v2/run_audit_v2.sh +``` + +Useful variants: + +```bash +# Cheaper exploratory run +./scripts/audit_v2/run_audit_v2.sh -m seclab_taskflows.configs.model_config_audit_v2_lowercost + +# Static stages only; findings top out at `confirmed` +./scripts/audit_v2/run_audit_v2.sh --no-reproduce + +# Resume after a stage failed +./scripts/audit_v2/run_audit_v2.sh --from contest + +# One stage on its own +./scripts/audit_v2/run_audit_v2.sh -s report +``` + +## The stages + +| Stage | What it does | Ledger effect | +| --- | --- | --- | +| `survey` | Fetches the source, decomposes it into components, maps where untrusted data enters each one | populates `repo_context` | +| `hunt` | Three model families hunt each component in parallel, then a dedup pass folds convergent findings | creates `candidate`s, some `duplicate` | +| `contest` | Prosecution, defense, adjudication | `candidate` → `confirmed` or `rejected` | +| `reproduce` | Builds and runs the target in a container, drives the path with a control case first, then the attack | `confirmed` → `reproduced` | +| `report` | Writes the report, then verifies every claim in it against the ledger | read-only | + +Each stage is a separate taskflow because each is expensive and each ends at a +durable checkpoint. Rerunning `contest` does not re-run `hunt`. + +Stage state lives in the ledger rather than in taskflow outputs for a concrete +reason: multi-model tasks do not feed the implicit last-tool-result channel, so +a fan-out across model families has nowhere else to meet. The ledger is that +meeting point, and it happens to also be the thing that survives a crash. + +## Reproduction safety + +The reproduction container is the only place in the pipeline where code is +executed. It runs with `--network none` by default; loopback still works, so a +server started inside the container is reachable at 127.0.0.1 from the same +container. If a target genuinely cannot be built without fetching its +dependencies, the reproduction engineer is instructed to say so rather than +guess, and the operator can rerun with `CONTAINER_NETWORK=bridge`. + +Reproduction runs one finding at a time. All branches of a task share a single +container process, so concurrent reproductions would fight over ports, files +and processes, and a failure in one would be indistinguishable from a failure +in another. + +Unlike the other container toolboxes, the reproduction toolbox does not set +`confirm` on `container_shell_exec`, because the stage is useless if every +command needs a human. The isolation is the container, not the prompt. + +## Caveats + +1. This will consume a large amount of model quota. Three hunters plus a + three-model contest per finding is the cost of not shipping false positives. +2. `proposed_by` is self-reported by the model. The grammar exposes no template + variable for the current branch's model label, so a mislabelled finding is + possible; the labels are useful as a convergence signal, not as an audit + trail. +3. Everything here should still be reviewed by a human before it is reported to + anyone. A reproduced finding is strong evidence, not a disclosure. diff --git a/src/seclab_taskflows/taskflows/audit_v2/contest.yaml b/src/seclab_taskflows/taskflows/audit_v2/contest.yaml new file mode 100644 index 0000000..f232ab3 --- /dev/null +++ b/src/seclab_taskflows/taskflows/audit_v2/contest.yaml @@ -0,0 +1,202 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: taskflow + version: "1.0" +model_config: seclab_taskflows.configs.model_config_audit_v2 + +globals: + repo: + +# Stage 3 of the audit v2 pipeline: put every candidate on trial. +# +# This is the stage that separates v2 from the v1 audit. A candidate is argued +# by a model whose job is to prove it, then by a model from a different family +# whose job is to kill it, and then decided by a model from a third family. +# Most false positives in automated code audit survive simply because nobody +# was ever tasked with destroying them. +# +# The three tasks all fan out over the same typed candidate list, but they do +# not pass arguments to each other through the taskflow. Prosecution and defense +# write their positions into the ledger, and the adjudicator reads them back +# joined by `finding_id`. That join is authoritative, whereas correlating three +# fan-outs by branch index would silently mispair arguments if any branch +# failed. +taskflow: + - task: + id: candidates + must_complete: true + exclude_from_context: true + max_steps: 5 + model: general_tasks + name: fetch candidates + description: Publish the candidate findings as a typed output. + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + Call `get_findings` once for the repo {{ globals.repo }} with state + `candidate` and return its result. Make no other tool calls. + outputs: + type: array + items: + type: object + properties: + finding_id: {type: integer} + repo: {type: string} + component: {type: string} + title: {type: string} + vuln_class: {type: string} + required: [finding_id, repo, component, title, vuln_class] + toolboxes: + - seclab_taskflows.toolboxes.finding_ledger + + - task: + if: "outputs.candidates | length > 0" + must_complete: false + repeat_prompt: true + over: "outputs.candidates" + async: true + async_limit: 4 + model: prosecution + max_steps: 120 + name: prosecute + description: Argue that each candidate is exploitable. + agents: + - seclab_taskflows.personalities.exploit_prosecutor + user_prompt: | + You are prosecuting finding {{ result.finding_id }} in the repo + {{ result.repo }}, in the component {{ result.component }}: + + {{ result.title }} ({{ result.vuln_class }}) + + Call `get_finding` for id {{ result.finding_id }} to read its source, + sink, flow, locations and hypothesis in full. The repository is mounted + at /workspace; read the code at every location before you argue about + it. + + Build the case that this finding is exploitable. Walk the path from the + entry point to the sink, and for every check between them explain, with + the line in front of you, why it fails to stop a hostile value. + + Then record your position with `store_contest_verdict` for finding + {{ result.finding_id }}, with role `prosecution`. Put your full + argument in `rationale`, including the citations, so the adjudicator + can check it without re-deriving your reasoning. + + If reading the code convinces you the finding is wrong, record + `not_exploitable` and say what kills it. + + {% include 'seclab_taskflows.prompts.audit_v2.contest_rules' %} + + {% include 'seclab_taskflows.prompts.audit_v2.evidence_rules' %} + toolboxes: + - seclab_taskflows.toolboxes.container_shell_source_access + - seclab_taskflows.toolboxes.finding_ledger + + - task: + if: "outputs.candidates | length > 0" + must_complete: false + repeat_prompt: true + over: "outputs.candidates" + async: true + async_limit: 4 + model: defense + max_steps: 120 + name: defend + description: Argue that each candidate is not exploitable. + agents: + - seclab_taskflows.personalities.exploit_defender + user_prompt: | + You are defending against finding {{ result.finding_id }} in the repo + {{ result.repo }}, in the component {{ result.component }}: + + {{ result.title }} ({{ result.vuln_class }}) + + Call `get_finding` for id {{ result.finding_id }} to read its source, + sink, flow, locations and hypothesis in full, along with the + prosecution's argument if it has already been recorded. The repository + is mounted at /workspace; read the code at every location before you + argue about it. + + Try to break this finding. Work through the ways a finding like this + usually dies: the source is not really attacker-controlled, the path is + not really reachable, a check really does hold, the flow does not + really connect, the impact is not what is claimed. Take the strongest + one you can support with code. + + Then record your position with `store_contest_verdict` for finding + {{ result.finding_id }}, with role `defense`. Put your full argument in + `rationale`, including the citations. + + If the finding survives everything you can throw at it, record + `exploitable` and say which of your attacks failed and why. + + {% include 'seclab_taskflows.prompts.audit_v2.contest_rules' %} + + {% include 'seclab_taskflows.prompts.audit_v2.evidence_rules' %} + toolboxes: + - seclab_taskflows.toolboxes.container_shell_source_access + - seclab_taskflows.toolboxes.finding_ledger + + - task: + if: "outputs.candidates | length > 0" + must_complete: false + repeat_prompt: true + over: "outputs.candidates" + async: true + async_limit: 4 + model: adjudication + max_steps: 120 + name: adjudicate + description: Decide each contested finding and assign its severity. + agents: + - seclab_taskflows.personalities.finding_adjudicator + user_prompt: | + Decide finding {{ result.finding_id }} in the repo {{ result.repo }}, + in the component {{ result.component }}: + + {{ result.title }} ({{ result.vuln_class }}) + + Call `get_finding` for id {{ result.finding_id }}. It returns the + finding together with the prosecution and defense verdicts recorded + against it. Read both arguments and the finding itself. + + Both arguments are evidence, not authority. The repository is mounted + at /workspace: where they disagree about what the code does, open the + cited lines and settle it yourself. An argument whose citations do not + say what it claims they say loses, however well it reads. + + Then call `adjudicate_finding` for finding {{ result.finding_id }} with + your position and severity. Your `rationale` is what a human reviewer + reads first, so write it so that someone who has seen neither argument + can follow your reasoning and check it against the code. Say explicitly + which argument prevailed and on which point. + + Remember what your decision does: `exploitable` confirms the finding + and sends it to dynamic reproduction, `not_exploitable` rejects it, and + `uncertain` leaves it a candidate for a human to look at. + + {% include 'seclab_taskflows.prompts.audit_v2.contest_rules' %} + + {% include 'seclab_taskflows.prompts.audit_v2.severity_rubric' %} + + {% include 'seclab_taskflows.prompts.audit_v2.evidence_rules' %} + toolboxes: + - seclab_taskflows.toolboxes.container_shell_source_access + - seclab_taskflows.toolboxes.finding_ledger + + - task: + must_complete: true + model: general_tasks + max_steps: 20 + name: contest summary + description: Report what survived the contest. + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + Get the ledger summary for the repo {{ globals.repo }}, then list the + confirmed findings with their severity and the rejected findings with + the reason they were rejected. + toolboxes: + - seclab_taskflows.toolboxes.finding_ledger diff --git a/src/seclab_taskflows/taskflows/audit_v2/hunt.yaml b/src/seclab_taskflows/taskflows/audit_v2/hunt.yaml new file mode 100644 index 0000000..29845af --- /dev/null +++ b/src/seclab_taskflows/taskflows/audit_v2/hunt.yaml @@ -0,0 +1,209 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: taskflow + version: "1.0" +model_config: seclab_taskflows.configs.model_config_audit_v2 + +globals: + repo: + +# Stage 2 of the audit v2 pipeline: hunt every component with three model +# families at once. +# +# Models within a family tend to miss the same things, so the breadth here comes +# from disagreement between families rather than from running one model harder. +# The task is a `models` x `repeat_prompt` cross product: every component is +# hunted independently by every hunter. For a multi-model task the runner bounds +# all branches (items x models) with `model_concurrency`; `async_limit` only +# applies to single-model repeat_prompt fan-outs, so it is not set here. +# +# Each branch writes its candidates straight into the finding ledger. That is +# deliberate: a multi-model task does not feed the implicit last-tool-result +# channel that `repeat_prompt` reads, so downstream stages re-read the ledger +# rather than having findings threaded through the taskflow. It also means the +# stage is resumable and that partial results survive a failed branch. +taskflow: + - task: + id: components + must_complete: true + exclude_from_context: true + max_steps: 5 + model: general_tasks + name: fetch components + description: Publish the component list as a typed output. + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + Call `get_components` once for the repo {{ globals.repo }} and return + its result. Make no other tool calls. + outputs: + type: array + items: + type: object + properties: + id: {type: integer} + repo: {type: string} + location: {type: string} + required: [id, repo, location] + toolboxes: + - seclab_taskflows.toolboxes.repo_context + + - task: + if: "outputs.components | length > 0" + must_complete: false + repeat_prompt: true + over: "outputs.components" + completion: any + model_concurrency: 6 + max_steps: 200 + models: [hunt_gpt, hunt_claude, hunt_gemini] + name: hunt component + description: Independently hunt each component with three model families. + agents: + - seclab_taskflows.personalities.vulnerability_hunter + user_prompt: | + Hunt for vulnerabilities in the component at {{ result.location }} in + the repo {{ result.repo }}. The repository is mounted at /workspace. + + What previous analysis recorded about this component: + + is_application ? {{ result.is_app | default('unknown') }} + is_library ? {{ result.is_library | default('unknown') }} + + {{ result.notes | default('') }} + + Start from the entry points already mapped for this component; fetch + them with the repo context tools. They are a starting point, not a + limit. If you find an entry point that was missed, hunt it too. + + Work outward from each entry point: + + 1. Take the untrusted value and follow it. Read every function it + passes through, including the ones whose names suggest they make it + safe. + 2. Note every check, escape, cast, allow-list and normalisation on the + way, and decide whether it actually holds for a hostile value on + every branch. + 3. When the value reaches something dangerous, work out what an + attacker gets. + + Judge this component on what it actually is. A parser's danger is in + how it handles malformed input; a library's is in what its documented + API lets a caller pass through to something dangerous; a daemon's is in + what an unauthenticated peer can reach; native code's is in bounds, + lifetimes and integer arithmetic. + + Do not limit yourself to a checklist of well-known vulnerability + classes, and do not report generic hardening advice. Report paths. + + {% include 'seclab_taskflows.prompts.audit_v2.finding_contract' %} + + Set `proposed_by` to the name of the model you are running as, for + example `gpt-5.6-sol` or `claude-sonnet-5`, so the ledger records which + family proposed each path. If you do not know your own model name, use + `unknown`. + + {% include 'seclab_taskflows.prompts.audit_v2.evidence_rules' %} + toolboxes: + - seclab_taskflows.toolboxes.container_shell_source_access + - seclab_taskflows.toolboxes.repo_context + - seclab_taskflows.toolboxes.finding_ledger + + - task: + id: raw_candidates + must_complete: true + exclude_from_context: true + max_steps: 5 + model: general_tasks + name: fetch raw candidates + description: Publish the pre-deduplication candidate list. + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + Call `get_findings` once for the repo {{ globals.repo }} with state + `candidate` and return its result. Make no other tool calls. + outputs: + type: array + items: + type: object + properties: + finding_id: {type: integer} + component: {type: string} + title: {type: string} + vuln_class: {type: string} + proposed_by: {type: string} + required: [finding_id, component, title, vuln_class] + toolboxes: + - seclab_taskflows.toolboxes.finding_ledger + + - task: + if: "outputs.raw_candidates | length > 1" + must_complete: false + model: survey + max_steps: 120 + name: deduplicate candidates + description: Fold candidates that describe the same path into one another. + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + Three independent hunters have just filed candidate findings for the + repo {{ globals.repo }}. Where they found the same bug, the ledger now + holds several candidates describing one path. + + These are the candidates: + + {% for c in outputs.raw_candidates %} + - id {{ c.finding_id }} [{{ c.component }}] ({{ c.vuln_class }}) + proposed by {{ c.proposed_by | default('unknown') }}: {{ c.title }} + {% endfor %} + + Use `get_finding` to read the source, sink and flow of any candidate + you are unsure about; the titles alone are not enough to judge this. + + Group them by the path they describe: the same untrusted source + reaching the same dangerous operation. Two candidates are the same + finding when they describe the same flow, even if they use different + wording, a different vulnerability class label, or cite slightly + different lines along the way. + + Two candidates are *not* the same finding when they reach the same sink + from different sources, or the same source reaches different sinks. + Those are separate paths and each deserves its own review. When you are + unsure, leave them separate: splitting a duplicate costs one extra + contest, but merging two distinct bugs loses one of them entirely. + + For each group of two or more: + + 1. Pick the canonical finding: the one whose source, sink and flow are + described most precisely, with the most useful locations. + 2. Call `merge_duplicate_finding` for every other member of the group, + with the canonical finding as `canonical_id`. + + Merging carries each duplicate's `proposed_by` label onto the canonical + finding, so a path that several families found independently ends up + recording all of them. Do not delete anything and do not adjudicate + anything; merging is the only action you take here. + + Finish by reporting how many candidates you started with, how many + groups you merged, and which findings more than one model family found + independently. + toolboxes: + - seclab_taskflows.toolboxes.finding_ledger + + - task: + must_complete: true + model: general_tasks + max_steps: 20 + name: hunt summary + description: Report the state of the ledger after hunting. + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + Get the ledger summary for the repo {{ globals.repo }}, then list the + surviving candidate findings with their component, vulnerability class, + title and `proposed_by`. Report them grouped by component, and call out + any that were proposed by more than one model. + toolboxes: + - seclab_taskflows.toolboxes.finding_ledger diff --git a/src/seclab_taskflows/taskflows/audit_v2/report.yaml b/src/seclab_taskflows/taskflows/audit_v2/report.yaml new file mode 100644 index 0000000..c7dd70f --- /dev/null +++ b/src/seclab_taskflows/taskflows/audit_v2/report.yaml @@ -0,0 +1,162 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: taskflow + version: "1.0" +model_config: seclab_taskflows.configs.model_config_audit_v2 + +globals: + repo: + +# Stage 5 of the audit v2 pipeline: write up what the ledger actually supports. +# +# The report is drafted with `capture: response`, which stores the model's prose +# rather than its last tool result, and then a second model reads that draft +# back out of `outputs.draft` and checks every claim in it against the ledger. +# Overclaiming is the failure mode that costs an audit its credibility, and it +# is the one failure a reader cannot detect on their own, so it gets its own +# check. +taskflow: + - task: + id: findings + must_complete: true + exclude_from_context: true + max_steps: 5 + model: general_tasks + name: fetch findings + description: Publish every finding for this repo as a typed output. + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + Call `get_findings` once for the repo {{ globals.repo }} with no state + filter, and return its result. Make no other tool calls. + outputs: + type: array + items: + type: object + properties: + finding_id: {type: integer} + component: {type: string} + title: {type: string} + vuln_class: {type: string} + state: {type: string} + severity: {type: string} + proposed_by: {type: string} + required: [finding_id, component, title, vuln_class, state] + toolboxes: + - seclab_taskflows.toolboxes.finding_ledger + + - task: + id: draft + if: "outputs.findings | length > 0" + must_complete: false + capture: response + model: reporting + max_steps: 60 + name: draft report + description: Write the audit report from the ledger. + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + Write the security audit report for {{ globals.repo }}. + + The ledger holds these findings: + + {% for f in outputs.findings %} + - id {{ f.finding_id }} [{{ f.state }}] {{ f.severity | default('') }} + ({{ f.component }}, {{ f.vuln_class }}): {{ f.title }} + {% endfor %} + + Call `get_finding` for every finding in state `reproduced` or + `confirmed` to get its flow, locations, contest verdicts and + reproduction attempts. Do not write about a finding you have not read. + + Structure the report as Markdown: + + ## Summary + What was audited, how many findings reached each state, and the + headline result in two or three sentences. + + ## Reproduced findings + The strongest results: findings that were argued, confirmed, and then + actually triggered in a container. For each one give the title, + severity, component, the source-to-sink path with its locations, the + impact, and the exact reproduction: the commands from `harness` and the + observed output from `observed`. State the preconditions an attacker + needs. + + ## Confirmed but not reproduced + Findings that survived adversarial review but were not triggered. For + each one, say what the adjudicator concluded and exactly why + reproduction did not succeed: whether the attempt showed the finding + does not hold, or whether it was inconclusive because the target could + not be built, started or reached. These are different results and the + reader needs to be able to tell them apart. + + ## Unresolved candidates + Findings the adjudicator marked `uncertain`, with what would be needed + to settle them. These need a human. + + ## Rejected findings + A short table: title, component, and the one-line reason each was + rejected. This is the audit trail for what was looked at and dismissed. + + ## Method and limitations + How the audit worked: components mapped, hunted by several model + families independently, candidates deduplicated, each argued by a + prosecution and a defense and decided by a third model, confirmed + findings then run in a sandboxed container. Then the limitations that + actually apply to this run: components not covered, reproductions that + could not be attempted, and the fact that a rejected finding is a model + judgement rather than a proof of safety. + + Write for an engineer who has to act on this. Be precise about + locations and specific about impact. + + Do not overclaim. A finding's state is what the ledger says it is: + describe a finding as reproduced only if its state is `reproduced`, and + as confirmed only if its state is `confirmed`. Never present a rejected + or uncertain finding as though it were real. + + {% include 'seclab_taskflows.prompts.audit_v2.severity_rubric' %} + toolboxes: + - seclab_taskflows.toolboxes.finding_ledger + + - task: + if: "outputs.draft is defined and outputs.draft" + must_complete: false + model: adjudication + max_steps: 60 + name: check report against ledger + description: Verify the draft does not claim more than the ledger records. + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + Below is a draft audit report for {{ globals.repo }}. Check it against + the finding ledger, which is the source of truth. + + For every finding the draft describes, call `get_finding` and confirm: + + 1. The state the draft implies matches the state in the ledger. A + finding described as reproduced must be in state `reproduced`; one + described as confirmed must be in state `confirmed`. + 2. Any reproduction the draft describes appears in that finding's + reproduction attempts, with outcome `reproduced`, and the observed + output in the draft matches what was actually recorded. + 3. The severity in the draft matches the severity in the ledger. + 4. The locations quoted in the draft appear in the finding. + + Then output the final report: the draft with every discrepancy + corrected, so that it says exactly what the ledger supports and no + more. If the draft was accurate, output it unchanged. + + After the report, add a short section `## Verification notes` listing + every correction you made, or stating that the draft matched the ledger + as written. + + --- DRAFT REPORT --- + + {{ outputs.draft }} + toolboxes: + - seclab_taskflows.toolboxes.finding_ledger diff --git a/src/seclab_taskflows/taskflows/audit_v2/reproduce.yaml b/src/seclab_taskflows/taskflows/audit_v2/reproduce.yaml new file mode 100644 index 0000000..b7ab644 --- /dev/null +++ b/src/seclab_taskflows/taskflows/audit_v2/reproduce.yaml @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: taskflow + version: "1.0" +model_config: seclab_taskflows.configs.model_config_audit_v2 + +globals: + repo: + +# Stage 4 of the audit v2 pipeline: stop reasoning about the code and run it. +# +# Every stage before this one argues from source. This is the only stage that +# observes behaviour, and it is the only thing that can tell a real +# vulnerability apart from a very well-argued misunderstanding. +# +# The fan-out is deliberately sequential. All branches of a task share one +# container process, so running reproductions concurrently would have them +# fighting over ports, files and processes. Reproduction is the expensive, +# careful stage; doing it one finding at a time is the right trade. +taskflow: + - task: + id: confirmed + must_complete: true + exclude_from_context: true + max_steps: 5 + model: general_tasks + name: fetch confirmed findings + description: Publish the confirmed findings as a typed output. + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + Call `get_findings` once for the repo {{ globals.repo }} with state + `confirmed` and return its result. Make no other tool calls. + outputs: + type: array + items: + type: object + properties: + finding_id: {type: integer} + repo: {type: string} + component: {type: string} + title: {type: string} + vuln_class: {type: string} + severity: {type: string} + required: [finding_id, repo, component, title, vuln_class] + toolboxes: + - seclab_taskflows.toolboxes.finding_ledger + + - task: + if: "outputs.confirmed | length > 0" + must_complete: false + repeat_prompt: true + over: "outputs.confirmed" + model: reproduction + max_steps: 250 + name: reproduce finding + description: Attempt to trigger each confirmed finding in a container. + agents: + - seclab_taskflows.personalities.reproduction_engineer + env: + CONTAINER_TIMEOUT: "600" + CONTAINER_PERSIST: "false" + user_prompt: | + Reproduce finding {{ result.finding_id }} in the repo {{ result.repo }}, + in the component {{ result.component }}: + + {{ result.title }} ({{ result.vuln_class }}) + + Call `get_finding` for id {{ result.finding_id }} to read its source, + sink, flow, locations and hypothesis, along with the adjudicator's + reasoning for confirming it. You are testing that specific path. + + The repository is mounted at /workspace inside the reproduction + container. Work there: + + 1. Work out how to build or start this component. Read the README, the + build files, the packaging metadata and the test suite. The tests + are usually the fastest route to a working invocation of the code + the finding points at. + 2. Get it running. + 3. Drive the path with benign input first and record what normal looks + like. Without that control, the malicious result proves nothing. + 4. Then send the attack, from the entry point the finding names, with + a minimal non-destructive payload: read a marker file you created, + echo a marker string, trigger the crash under gdb or valgrind. + 5. If it does not work the first time, change one thing and try again + while you are still learning something from the failures. + + Clean up processes and files you started before you finish, so the next + finding starts from a known state. + + Then call `store_reproduction_attempt` for finding + {{ result.finding_id }} with the outcome, the exact commands you ran in + `harness`, and the actual container output in `observed`. + + {% include 'seclab_taskflows.prompts.audit_v2.reproduction_rules' %} + + {% include 'seclab_taskflows.prompts.audit_v2.evidence_rules' %} + toolboxes: + - seclab_taskflows.toolboxes.container_shell_reproduction + - seclab_taskflows.toolboxes.finding_ledger + + - task: + must_complete: true + model: general_tasks + max_steps: 20 + name: reproduction summary + description: Report which findings were actually triggered. + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + Get the ledger summary for the repo {{ globals.repo }}. List the + findings in state `reproduced`, and separately the findings still in + state `confirmed` together with why their reproduction attempt did not + succeed. + toolboxes: + - seclab_taskflows.toolboxes.finding_ledger diff --git a/src/seclab_taskflows/taskflows/audit_v2/survey.yaml b/src/seclab_taskflows/taskflows/audit_v2/survey.yaml new file mode 100644 index 0000000..03af530 --- /dev/null +++ b/src/seclab_taskflows/taskflows/audit_v2/survey.yaml @@ -0,0 +1,190 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: taskflow + version: "1.0" +model_config: seclab_taskflows.configs.model_config_audit_v2 + +globals: + repo: + +# Stage 1 of the audit v2 pipeline: build the map the rest of the audit hunts +# over. +# +# Unlike the v1 audit this does not assume the target is a web application. The +# attack surface question is asked in terms of trust boundaries, so it works for +# parsers, libraries, daemons, command line tools and native code too. +# +# Data flow here is explicit rather than positional: the fan-out declares an +# `outputs` contract and iterates with `over`, so the stage fails loudly if the +# shape it depends on is wrong instead of quietly iterating whichever tool +# happened to fire last. +taskflow: + - task: + must_complete: true + headless: true + exclude_from_context: true + max_steps: 20 + model: general_tasks + name: reset and fetch + description: Clear prior state for this repo and fetch its source. + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + Clear the memory cache. Clear the repo context results for the repo + {{ globals.repo }}. Clear the findings in the finding ledger for the + repo {{ globals.repo }}. + + Then fetch and extract the source code of the repo {{ globals.repo }} + for container_shell analysis. + toolboxes: + - seclab_taskflow_agent.toolboxes.memcache + - seclab_taskflows.toolboxes.repo_context + - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.local_gh_resources + + - task: + must_complete: true + model: survey + max_steps: 100 + name: map components + description: Identify what the repository is and decompose it into components. + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + Map the repository {{ globals.repo }}, which is mounted at /workspace. + + First work out what this software actually is. Read the README, the + packaging metadata, the documentation and the top-level layout. It may + be an application, a library, a framework, a parser, a command line + tool, a daemon, a build plugin, or several of these at once. Do not + assume it is a web application. + + Then decompose it into components. A component is a coherent unit of + functionality that can be reasoned about on its own: one service, one + parser, one protocol implementation, one subsystem. Be granular. A + library is almost never a single component; group its directories by + what they do. If the repository contains several applications, each one + is its own component. + + For each component, store an entry with `store_new_component`. Set + `is_app` and `is_library` according to how the component is consumed, + and use `notes` to record: + + - what the component does + - what language and runtime it is written in + - who or what talks to it, and across which trust boundary + - anything that makes it interesting to attack: it parses untrusted + formats, it executes things, it makes authorisation decisions, it + handles credentials, it manages memory by hand + + Identify example, demo, fixture and test code. Do not create components + for these, and say in your summary which directories you excluded. + + {% include 'seclab_taskflows.prompts.audit_v2.evidence_rules' %} + toolboxes: + - seclab_taskflows.toolboxes.container_shell_source_access + - seclab_taskflows.toolboxes.repo_context + + - task: + id: components + must_complete: true + exclude_from_context: true + max_steps: 5 + model: general_tasks + name: fetch components + description: Publish the component list as a typed output. + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + Call `get_components` once for the repo {{ globals.repo }} and return + its result. Make no other tool calls. + outputs: + type: array + items: + type: object + properties: + id: {type: integer} + repo: {type: string} + location: {type: string} + required: [id, repo, location] + toolboxes: + - seclab_taskflows.toolboxes.repo_context + + - task: + if: "outputs.components | length > 0" + must_complete: false + repeat_prompt: true + over: "outputs.components" + async: true + async_limit: 4 + model: survey + max_steps: 100 + name: map attack surface + description: Identify where untrusted data crosses into each component. + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + The component is in {{ result.repo }} in the directory + {{ result.location }}. Previous analysis recorded: + + is_application ? {{ result.is_app | default('unknown') }} + is_library ? {{ result.is_library | default('unknown') }} + + {{ result.notes | default('') }} + + Map this component's attack surface: every place where data crosses + into it from somewhere less trusted. Think in terms of trust + boundaries, not in terms of any one kind of software. + + Untrusted input includes, depending on what this component is: + + - network input: requests, responses, protocol messages, peers + - file and stream input: anything parsed whose bytes came from + elsewhere, including archives, images, documents and serialised data + - input from other processes: IPC, pipes, sockets, environment and + arguments set by a less privileged caller + - stored data written by another user: database rows, cache entries, + queue messages, uploaded files + - repository and package content: filenames, paths, metadata, manifests + - for a library: the parameters its documented public API tells callers + they may pass, when a caller would reasonably pass attacker data + + The following are *not* untrusted on their own: configuration the + operator wrote, arguments the operator typed, environment the operator + set. They become interesting when they point at untrusted resources; a + filename the operator supplies is trusted, but the file's contents may + not be. + + For each entry point, call `store_new_entry_point` with the precise + file and integer line number, the variables that carry untrusted data, + and notes explaining which trust boundary is crossed and why the data + on the other side is untrusted. If an entry point spans several lines, + use the first. + + Do not assess vulnerabilities here. You are drawing the map, not + hunting on it. + + {% include 'seclab_taskflows.prompts.audit_v2.evidence_rules' %} + toolboxes: + - seclab_taskflows.toolboxes.container_shell_source_access + - seclab_taskflows.toolboxes.repo_context + + - task: + must_complete: true + model: survey + max_steps: 30 + name: survey summary + description: Summarise the map produced by this stage. + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + Fetch the components and the entry points for the repo + {{ globals.repo }}. + + Summarise what this repository is, which components exist, and where + untrusted data enters each one. Call out the components you would + prioritise for hunting and say why, in one or two sentences each. + toolboxes: + - seclab_taskflows.toolboxes.repo_context diff --git a/src/seclab_taskflows/toolboxes/container_shell_reproduction.yaml b/src/seclab_taskflows/toolboxes/container_shell_reproduction.yaml new file mode 100644 index 0000000..ee954db --- /dev/null +++ b/src/seclab_taskflows/toolboxes/container_shell_reproduction.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: toolbox + version: "1.0" + +server_params: + kind: stdio + command: python + args: ["-m", "seclab_taskflows.mcp_servers.container_shell"] + env: + CONTAINER_IMAGE: "ghcr.io/githubsecuritylab/seclab-shell-reproduction:latest" + # Same checkout the survey and hunt stages read, mounted read-write because + # reproduction has to build and run the target rather than just read it. + CONTAINER_WORKSPACE: "{{ env('CONTAINER_WORKSPACE', required=False) or env('DATA_DIR') ~ '/repo_under_test' }}" + CONTAINER_TIMEOUT: "{{ env('CONTAINER_TIMEOUT', '300') }}" + CONTAINER_PERSIST: "{{ env('CONTAINER_PERSIST', 'true') }}" + CONTAINER_PERSIST_KEY: "{{ env('CONTAINER_PERSIST_KEY', required=False) }}" + CONTAINER_NETWORK: "{{ env('CONTAINER_NETWORK', 'none') }}" + LOG_DIR: "{{ env('LOG_DIR') }}" + +server_prompt: | + ## Container Shell (Reproduction) + + You have an isolated Docker container for *executing* code. The repository + under test is mounted at /workspace. This is the only place in the audit + where you are allowed to run the target or a proof of concept. + + Available runtimes and tools: + - python3 (with venv and dev headers), pip3 + - node, npm + - java (headless JRE) + - gcc, g++, make (build-essential) + - gdb, valgrind, strace — for memory-safety and crash triage + - curl, wget, jq, nc, socat — for driving network-facing targets + - ps, lsof, tree, rg, less — for inspecting what the target is doing + + The repository under test is mounted read-write at /workspace. If /workspace + is empty, say so in your reproduction notes and record the attempt as + inconclusive; do not invent a target. + + Networking is disabled by default (`CONTAINER_NETWORK=none`). Loopback still + works, so a server you start inside the container is reachable from the same + container at 127.0.0.1. If a target genuinely cannot be built without + fetching dependencies, say so in your reproduction notes rather than + guessing; the operator can rerun with `CONTAINER_NETWORK=bridge`. + + A reproduction is only real if you observed it. Run the target, send the + input, and read the actual output, exit code, stack trace, or crash. Never + report an outcome you did not see in this container's output. diff --git a/src/seclab_taskflows/toolboxes/finding_ledger.yaml b/src/seclab_taskflows/toolboxes/finding_ledger.yaml new file mode 100644 index 0000000..f4a5e98 --- /dev/null +++ b/src/seclab_taskflows/toolboxes/finding_ledger.yaml @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: toolbox + version: "1.0" +server_params: + kind: stdio + command: python + args: ["-m", "seclab_taskflows.mcp_servers.finding_ledger"] + env: + FINDING_LEDGER_DIR: "{{ env('DATA_DIR') }}" + LOG_DIR: "{{ env('LOG_DIR') }}" + +server_prompt: | + ## Finding Ledger + + The finding ledger is the shared, durable record of every vulnerability + candidate in this audit. Stages communicate through it rather than by passing + findings in prose, so always read the ledger for context and always write your + conclusions back to it. + + Each finding moves through an explicit lifecycle: + + candidate --(adversarial contest)--> confirmed | rejected + confirmed --(dynamic reproduction)--> reproduced + candidate --(deduplication)--> duplicate + + You cannot set a finding's state directly. The ledger derives it: + - `store_finding` always creates a `candidate`. + - `store_contest_verdict` records a prosecution or defense position and never + changes state. + - `adjudicate_finding` is the only way a finding becomes `confirmed` or + `rejected`. + - `store_reproduction_attempt` with outcome `reproduced` only promotes a + finding that is already `confirmed`. + - `merge_duplicate_finding` folds one candidate into another and carries its + `proposed_by` label across, so the surviving finding records every model + that independently proposed it. + + This means a finding can never claim more certainty than its recorded + evidence supports. Do not describe a finding as confirmed or reproduced in + prose unless the ledger says it is. + + Before storing a new finding, call `find_similar_findings` for the same + component and vulnerability class so you do not file duplicates. diff --git a/tests/test_finding_ledger.py b/tests/test_finding_ledger.py new file mode 100644 index 0000000..16ade83 --- /dev/null +++ b/tests/test_finding_ledger.py @@ -0,0 +1,367 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +"""Tests for the audit v2 finding ledger. + +These focus on the promotion rules, because the ledger -- not the prompt -- is +what guarantees a finding cannot claim more than its recorded evidence. +""" + +import tempfile + +import pytest + +from seclab_taskflow_agent.available_tools import AvailableTools +from seclab_taskflow_agent.models import ToolboxDocument + +from seclab_taskflows.mcp_servers.finding_ledger import ( + FindingLedgerBackend, + InvalidLedgerValueError, + mcp, +) +from seclab_taskflows.mcp_servers.finding_ledger_models import ( + STATE_CANDIDATE, + STATE_CONFIRMED, + STATE_DUPLICATE, + STATE_REJECTED, + STATE_REPRODUCED, +) + +REPO = "acme/widget" + + +@pytest.fixture +def ledger(): + with tempfile.TemporaryDirectory() as tmp_dir: + yield FindingLedgerBackend(tmp_dir) + + +def _add_finding( + ledger, component="src/api", vuln_class="CWE-22", proposed_by="hunt_primary", repo=REPO +): + return ledger.store_finding( + repo=repo, + component=component, + title="Path traversal in file download", + vuln_class=vuln_class, + language="python", + source="HTTP query parameter 'name'", + sink="open()", + flow="name flows unsanitised into open()", + locations=["src/api/files.py:42"], + hypothesis="Attacker reads arbitrary files", + proposed_by=proposed_by, + ) + + +def _contest(ledger, finding_id, prosecution="exploitable", defense="not_exploitable"): + """File both advocates' verdicts, which adjudication now requires.""" + ledger.store_contest_verdict( + REPO, finding_id, "prosecution", "prosecution_model", prosecution, "reachable" + ) + ledger.store_contest_verdict( + REPO, finding_id, "defense", "defense_model", defense, "input is validated" + ) + return finding_id + + +def _contested_finding(ledger, **kwargs): + """A finding that has been through the contest and is ready to adjudicate.""" + return _contest(ledger, _add_finding(ledger, **kwargs)) + + +class TestFindingCreation: + def test_new_finding_starts_as_candidate(self, ledger): + finding_id = _add_finding(ledger) + finding = ledger.get_finding(finding_id) + assert finding["state"] == STATE_CANDIDATE + assert finding["locations"] == ["src/api/files.py:42"] + assert finding["proposed_by"] == "hunt_primary" + + def test_findings_filtered_by_state(self, ledger): + first = _contested_finding(ledger) + _add_finding(ledger, component="src/web") + ledger.adjudicate_finding(REPO, first, "exploitable", "high", "clear taint path") + + confirmed = ledger.get_findings(REPO, state=STATE_CONFIRMED) + candidates = ledger.get_findings(REPO, state=STATE_CANDIDATE) + + assert [f["finding_id"] for f in confirmed] == [first] + assert len(candidates) == 1 + + def test_similar_findings_match_component_and_class(self, ledger): + _add_finding(ledger, component="src/api", vuln_class="CWE-22") + _add_finding(ledger, component="src/api", vuln_class="CWE-79") + + similar = ledger.find_similar_findings(REPO, "src/api", "CWE-22") + + assert len(similar) == 1 + assert similar[0]["vuln_class"] == "CWE-22" + + +class TestAdjudication: + def test_exploitable_confirms_finding(self, ledger): + finding_id = _contested_finding(ledger) + ledger.adjudicate_finding(REPO, finding_id, "exploitable", "high", "prosecution prevailed") + assert ledger.get_finding(finding_id)["state"] == STATE_CONFIRMED + + def test_not_exploitable_rejects_finding(self, ledger): + finding_id = _contested_finding(ledger) + ledger.adjudicate_finding(REPO, finding_id, "not_exploitable", "none", "input is validated") + finding = ledger.get_finding(finding_id) + assert finding["state"] == STATE_REJECTED + assert finding["disposition_reason"] == "input is validated" + + def test_uncertain_leaves_finding_as_candidate(self, ledger): + finding_id = _contested_finding(ledger) + ledger.adjudicate_finding(REPO, finding_id, "uncertain", "low", "needs runtime evidence") + assert ledger.get_finding(finding_id)["state"] == STATE_CANDIDATE + + def test_adjudication_is_recorded_as_a_verdict(self, ledger): + finding_id = _contested_finding(ledger) + ledger.adjudicate_finding(REPO, finding_id, "exploitable", "medium", "reachable") + roles = [v["role"] for v in ledger.get_finding(finding_id)["verdicts"]] + assert roles == ["prosecution", "defense", "adjudication"] + + def test_adjudication_requires_both_advocates(self, ledger): + finding_id = _add_finding(ledger) + + result = ledger.adjudicate_finding(REPO, finding_id, "exploitable", "high", "reachable") + + assert "cannot be adjudicated yet" in result + assert ledger.get_finding(finding_id)["state"] == STATE_CANDIDATE + + def test_adjudication_requires_the_defense(self, ledger): + finding_id = _add_finding(ledger) + ledger.store_contest_verdict( + REPO, finding_id, "prosecution", "m", "exploitable", "reachable" + ) + + result = ledger.adjudicate_finding(REPO, finding_id, "exploitable", "high", "reachable") + + assert "no defense verdict" in result + assert ledger.get_finding(finding_id)["state"] == STATE_CANDIDATE + + def test_adjudication_requires_the_prosecution(self, ledger): + finding_id = _add_finding(ledger) + ledger.store_contest_verdict(REPO, finding_id, "defense", "m", "not_exploitable", "safe") + + result = ledger.adjudicate_finding(REPO, finding_id, "exploitable", "high", "reachable") + + assert "no prosecution verdict" in result + assert ledger.get_finding(finding_id)["state"] == STATE_CANDIDATE + + def test_invalid_position_is_rejected(self, ledger): + finding_id = _contested_finding(ledger) + with pytest.raises(InvalidLedgerValueError): + ledger.adjudicate_finding(REPO, finding_id, "probably", "high", "") + + def test_invalid_severity_is_rejected(self, ledger): + finding_id = _contested_finding(ledger) + with pytest.raises(InvalidLedgerValueError): + ledger.adjudicate_finding(REPO, finding_id, "exploitable", "catastrophic", "") + + +class TestContestVerdicts: + def test_prosecution_and_defense_do_not_change_state(self, ledger): + finding_id = _add_finding(ledger) + ledger.store_contest_verdict( + REPO, finding_id, "prosecution", "prosecutor", "exploitable", "reachable from route" + ) + ledger.store_contest_verdict( + REPO, finding_id, "defense", "defender", "not_exploitable", "normalised first" + ) + finding = ledger.get_finding(finding_id) + assert finding["state"] == STATE_CANDIDATE + assert len(finding["verdicts"]) == 2 + + def test_invalid_role_is_rejected(self, ledger): + finding_id = _add_finding(ledger) + with pytest.raises(InvalidLedgerValueError): + ledger.store_contest_verdict(REPO, finding_id, "jury", "m", "exploitable", "") + + +class TestReproductionGate: + def test_reproduction_promotes_only_from_confirmed(self, ledger): + finding_id = _contested_finding(ledger) + ledger.adjudicate_finding(REPO, finding_id, "exploitable", "high", "reachable") + ledger.store_reproduction_attempt( + REPO, finding_id, "reproducer", "curl ...", "reproduced", "read /etc/passwd" + ) + assert ledger.get_finding(finding_id)["state"] == STATE_REPRODUCED + + def test_candidate_cannot_be_promoted_by_reproduction(self, ledger): + finding_id = _add_finding(ledger) + message = ledger.store_reproduction_attempt( + REPO, finding_id, "reproducer", "curl ...", "reproduced", "read /etc/passwd" + ) + assert ledger.get_finding(finding_id)["state"] == STATE_CANDIDATE + assert "must be" in message + + def test_failed_reproduction_leaves_state_confirmed(self, ledger): + finding_id = _contested_finding(ledger) + ledger.adjudicate_finding(REPO, finding_id, "exploitable", "high", "reachable") + ledger.store_reproduction_attempt( + REPO, finding_id, "reproducer", "curl ...", "not_reproduced", "404 returned" + ) + assert ledger.get_finding(finding_id)["state"] == STATE_CONFIRMED + + def test_reproduced_finding_is_not_downgraded_by_adjudication(self, ledger): + finding_id = _contested_finding(ledger) + ledger.adjudicate_finding(REPO, finding_id, "exploitable", "high", "reachable") + ledger.store_reproduction_attempt( + REPO, finding_id, "reproducer", "curl ...", "reproduced", "read /etc/passwd" + ) + ledger.adjudicate_finding(REPO, finding_id, "not_exploitable", "none", "second thoughts") + assert ledger.get_finding(finding_id)["state"] == STATE_REPRODUCED + + def test_attempts_are_recorded_on_the_finding(self, ledger): + finding_id = _add_finding(ledger) + ledger.store_reproduction_attempt( + REPO, finding_id, "reproducer", "python poc.py", "inconclusive", "server would not boot" + ) + attempts = ledger.get_finding(finding_id)["reproduction_attempts"] + assert len(attempts) == 1 + assert attempts[0]["outcome"] == "inconclusive" + + def test_invalid_outcome_is_rejected(self, ledger): + finding_id = _add_finding(ledger) + with pytest.raises(InvalidLedgerValueError): + ledger.store_reproduction_attempt(REPO, finding_id, "m", "", "maybe", "") + + +class TestDeduplication: + def test_merge_folds_duplicate_and_keeps_canonical(self, ledger): + canonical = _add_finding(ledger, proposed_by="hunt_gpt") + duplicate = _add_finding(ledger, proposed_by="hunt_claude") + + ledger.merge_duplicate_finding(REPO, duplicate, canonical) + + folded = ledger.get_finding(duplicate) + kept = ledger.get_finding(canonical) + assert folded["state"] == STATE_DUPLICATE + assert folded["duplicate_of"] == canonical + assert kept["state"] == STATE_CANDIDATE + + def test_merge_accumulates_model_provenance(self, ledger): + canonical = _add_finding(ledger, proposed_by="hunt_gpt") + second = _add_finding(ledger, proposed_by="hunt_claude") + third = _add_finding(ledger, proposed_by="hunt_gemini") + + ledger.merge_duplicate_finding(REPO, second, canonical) + ledger.merge_duplicate_finding(REPO, third, canonical) + + assert ledger.get_finding(canonical)["proposed_by"] == ( + "hunt_gpt, hunt_claude, hunt_gemini" + ) + + def test_merge_does_not_repeat_a_label(self, ledger): + canonical = _add_finding(ledger, proposed_by="hunt_gpt") + duplicate = _add_finding(ledger, proposed_by="hunt_gpt") + + ledger.merge_duplicate_finding(REPO, duplicate, canonical) + + assert ledger.get_finding(canonical)["proposed_by"] == "hunt_gpt" + + def test_merge_refuses_to_cross_repositories(self, ledger): + canonical = _add_finding(ledger) + foreign = _add_finding(ledger, repo="acme/other") + + result = ledger.merge_duplicate_finding(REPO, foreign, canonical) + + assert "refusing to merge across repositories" in result + assert ledger.get_finding(foreign)["state"] == STATE_CANDIDATE + + def test_duplicates_are_excluded_from_candidate_queries(self, ledger): + canonical = _add_finding(ledger) + duplicate = _add_finding(ledger) + ledger.merge_duplicate_finding(REPO, duplicate, canonical) + + candidates = ledger.get_findings(REPO, state=STATE_CANDIDATE) + + assert [f["finding_id"] for f in candidates] == [canonical] + + def test_a_finding_cannot_be_its_own_duplicate(self, ledger): + finding_id = _add_finding(ledger) + message = ledger.merge_duplicate_finding(REPO, finding_id, finding_id) + assert "itself" in message + assert ledger.get_finding(finding_id)["state"] == STATE_CANDIDATE + + def test_adjudicated_findings_cannot_be_merged_away(self, ledger): + canonical = _add_finding(ledger) + confirmed = _contested_finding(ledger) + ledger.adjudicate_finding(REPO, confirmed, "exploitable", "high", "reachable") + + message = ledger.merge_duplicate_finding(REPO, confirmed, canonical) + + assert "only candidates" in message + assert ledger.get_finding(confirmed)["state"] == STATE_CONFIRMED + + def test_cannot_merge_into_a_duplicate(self, ledger): + canonical = _add_finding(ledger) + duplicate = _add_finding(ledger) + third = _add_finding(ledger) + ledger.merge_duplicate_finding(REPO, duplicate, canonical) + + message = ledger.merge_duplicate_finding(REPO, third, duplicate) + + assert "merge into that one instead" in message + assert ledger.get_finding(third)["state"] == STATE_CANDIDATE + + def test_merge_reports_unknown_findings(self, ledger): + finding_id = _add_finding(ledger) + assert "No finding" in ledger.merge_duplicate_finding(REPO, 9999, finding_id) + assert "No finding" in ledger.merge_duplicate_finding(REPO, finding_id, 9999) + + +class TestLedgerMaintenance: + def test_summary_counts_by_state(self, ledger): + confirmed = _contested_finding(ledger) + _add_finding(ledger, component="src/web") + ledger.adjudicate_finding(REPO, confirmed, "exploitable", "high", "reachable") + + summary = ledger.get_ledger_summary(REPO) + + assert summary["total"] == 2 + assert summary["by_state"][STATE_CONFIRMED] == 1 + assert summary["by_state"][STATE_CANDIDATE] == 1 + + def test_clear_removes_findings_and_evidence(self, ledger): + finding_id = _add_finding(ledger) + ledger.store_contest_verdict(REPO, finding_id, "prosecution", "m", "exploitable", "") + ledger.clear_findings_for_repo(REPO) + + assert ledger.get_findings(REPO) == [] + assert ledger.get_finding(finding_id) is None + + def test_unknown_finding_reads_return_none(self, ledger): + assert ledger.get_finding(9999) is None + + def test_writes_against_unknown_finding_are_reported(self, ledger): + assert "No finding" in ledger.adjudicate_finding(REPO, 9999, "exploitable", "high", "") + assert "No finding" in ledger.store_reproduction_attempt( + REPO, 9999, "m", "", "reproduced", "" + ) + + +class TestServerWiring: + def test_toolbox_yaml_valid(self): + result = AvailableTools().get_toolbox("seclab_taskflows.toolboxes.finding_ledger") + assert result is not None + assert isinstance(result, ToolboxDocument) + + @pytest.mark.asyncio + async def test_all_ledger_tools_are_exposed(self): + names = {tool.name for tool in await mcp.list_tools()} + assert names == { + "store_finding", + "get_findings", + "get_finding", + "find_similar_findings", + "store_contest_verdict", + "adjudicate_finding", + "merge_duplicate_finding", + "store_reproduction_attempt", + "get_ledger_summary", + "clear_findings_for_repo", + } diff --git a/tests/test_taskflow_corpus.py b/tests/test_taskflow_corpus.py new file mode 100644 index 0000000..6524f07 --- /dev/null +++ b/tests/test_taskflow_corpus.py @@ -0,0 +1,197 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +"""Corpus gate for the taskflows shipped by this package. + +Mirrors the agent's own example gate: every bundled grammar document must +validate against its model, and every bundled taskflow must lint clean. On top +of that, the audit_v2 prompts are actually *rendered*, because `--lint` only +checks Jinja syntax and will not notice a `{% include %}` pointing at a prompt +file that does not exist. +""" + +from __future__ import annotations + +import glob +from pathlib import Path + +import pytest +import yaml + +from seclab_taskflow_agent.available_tools import AvailableTools +from seclab_taskflow_agent.linting import lint_taskflow +from seclab_taskflow_agent.models import DOCUMENT_MODELS +from seclab_taskflow_agent.template_utils import evaluate_expression, render_template + +_ROOT = "src/seclab_taskflows" + +# A representative component and finding, shaped like the `outputs` contracts +# the audit_v2 stages declare. Rendering against these proves the prompts only +# reference fields the pipeline actually produces. +_COMPONENT = { + "id": 1, + "repo": "acme/widget", + "location": "src/api", + "is_app": True, + "is_library": False, + "notes": "handles uploads", +} +_FINDING = { + "finding_id": 1, + "repo": "acme/widget", + "component": "src/api", + "title": "Path traversal in file download", + "vuln_class": "CWE-22", + "state": "confirmed", + "severity": "high", + "proposed_by": "hunt_gpt", +} +_OUTPUTS = { + "components": [_COMPONENT], + "candidates": [_FINDING], + "raw_candidates": [_FINDING], + "confirmed": [_FINDING], + "findings": [_FINDING], + "draft": "# draft report", +} + +# Pre-existing corpus debt, unrelated to audit_v2: these prompts embed literal +# GitHub Actions `${{ ... }}` expressions, which Jinja tries to evaluate. They +# need `{% raw %}` fencing before they can lint or render. +_KNOWN_LINT_ERRORS = { + "seclab_taskflows.taskflows.alert_triage_examples.triage_taskflows.triage_actions_code_injection", +} + + +def _dotted(path: str) -> str: + return path.removeprefix("src/")[: -len(".yaml")].replace("/", ".") + + +def _grammar_files() -> list[str]: + kept: list[str] = [] + for f in sorted(glob.glob(f"{_ROOT}/**/*.yaml", recursive=True)): + data = yaml.safe_load(Path(f).read_text()) + if isinstance(data, dict) and "seclab-taskflow-agent" in data: + kept.append(f) + return kept + + +def _taskflow_dotted_paths() -> list[str]: + paths: list[str] = [] + for f in sorted(glob.glob(f"{_ROOT}/taskflows/**/*.yaml", recursive=True)): + data = yaml.safe_load(Path(f).read_text()) + filetype = (data.get("seclab-taskflow-agent") or {}).get("filetype") + if filetype == "taskflow": + paths.append(_dotted(f)) + return paths + + +def _lint_params() -> list[object]: + params: list[object] = [] + for dotted in _taskflow_dotted_paths(): + if dotted in _KNOWN_LINT_ERRORS: + params.append( + pytest.param( + dotted, + marks=pytest.mark.xfail( + strict=True, + reason="pre-existing: literal GitHub Actions ${{ }} in prompts", + ), + ) + ) + else: + params.append(dotted) + return params + + +def _audit_v2_dotted_paths() -> list[str]: + return [p for p in _taskflow_dotted_paths() if ".audit_v2." in p] + + +@pytest.mark.parametrize("path", _grammar_files()) +def test_bundled_document_validates(path: str) -> None: + """Every shipped grammar document parses and validates against its model.""" + data = yaml.safe_load(Path(path).read_text()) + assert isinstance(data, dict), f"{path}: not a mapping" + filetype = (data.get("seclab-taskflow-agent") or {}).get("filetype") + model = DOCUMENT_MODELS.get(filetype) + assert model is not None, f"{path}: unknown filetype {filetype!r}" + model.model_validate(data) + + +@pytest.mark.parametrize("dotted", _lint_params()) +def test_bundled_taskflow_lints_without_errors(dotted: str) -> None: + """Every bundled taskflow lints clean (warnings allowed, no errors).""" + issues = lint_taskflow(AvailableTools(), dotted) + errors = [i for i in issues if i.severity == "error"] + assert not errors, f"{dotted} has lint errors:\n" + "\n".join( + f" {i.code}: {i.message} [{i.location}]" for i in errors + ) + + +@pytest.mark.parametrize("dotted", _audit_v2_dotted_paths()) +def test_audit_v2_prompts_render(dotted: str) -> None: + """Every audit_v2 prompt renders, so `{% include %}` targets really exist.""" + tools = AvailableTools() + taskflow = tools.get_taskflow(dotted) + for index, step in enumerate(taskflow.taskflow): + task = step.task + if not task.user_prompt: + continue + where = f"{dotted}[{index}] {task.name or '(unnamed)'}" + # `result` is whatever the branch is fanned out over; fall back to a + # finding for plain (non-repeat) tasks. + result = _FINDING + if task.over: + candidates = list( + evaluate_expression( + task.over, tools, globals_dict={}, inputs_dict={}, outputs_dict=_OUTPUTS + ) + ) + if candidates: + result = candidates[0] + rendered = render_template( + template_str=task.user_prompt, + available_tools=tools, + globals_dict={"repo": "acme/widget"}, + inputs_dict={}, + result_value=result, + outputs_dict=_OUTPUTS, + ) + assert "{%" not in rendered, f"{where}: prompt still contains an unrendered Jinja block" + assert "{{" not in rendered, f"{where}: prompt still contains an unrendered Jinja variable" + assert rendered.strip(), f"{where}: prompt rendered empty" + + +@pytest.mark.parametrize("dotted", _audit_v2_dotted_paths()) +def test_audit_v2_over_expressions_resolve(dotted: str) -> None: + """Every `over` expression selects a real list from the declared outputs.""" + tools = AvailableTools() + taskflow = tools.get_taskflow(dotted) + for index, step in enumerate(taskflow.taskflow): + task = step.task + if not task.over: + continue + where = f"{dotted}[{index}] {task.name or '(unnamed)'}" + value = evaluate_expression( + task.over, tools, globals_dict={}, inputs_dict={}, outputs_dict=_OUTPUTS + ) + assert isinstance(list(value), list), f"{where}: `over` did not yield a list" + + +def test_audit_v2_over_targets_are_produced_by_an_earlier_task() -> None: + """An `over` referencing `outputs.` must follow the task that sets it.""" + tools = AvailableTools() + for dotted in _audit_v2_dotted_paths(): + taskflow = tools.get_taskflow(dotted) + produced: set[str] = set() + for index, step in enumerate(taskflow.taskflow): + task = step.task + if task.over and task.over.startswith("outputs."): + name = task.over.removeprefix("outputs.").split(".")[0].strip("\"' ") + assert name in produced, ( + f"{dotted}[{index}] iterates outputs.{name}, " + f"which no earlier task publishes (published so far: {sorted(produced)})" + ) + if task.id: + produced.add(task.id) From 6940b0d861b4f328054397b3773468ac0f7c84e4 Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Wed, 29 Jul 2026 14:21:51 -0400 Subject: [PATCH 02/12] Make the finding ledger always persist to disk Open a file-backed database instead of falling back to in-memory when FINDING_LEDGER_DIR does not exist yet, which silently dropped a run's findings. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: dadec5f9-3bf8-449c-84d6-db45be19bb6a --- .../mcp_servers/finding_ledger.py | 17 +++++++++----- tests/test_finding_ledger.py | 23 +++++++++++++++++++ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/seclab_taskflows/mcp_servers/finding_ledger.py b/src/seclab_taskflows/mcp_servers/finding_ledger.py index 782dcdc..46119af 100644 --- a/src/seclab_taskflows/mcp_servers/finding_ledger.py +++ b/src/seclab_taskflows/mcp_servers/finding_ledger.py @@ -130,14 +130,19 @@ def _require(value: str, allowed, name: str) -> str: class FindingLedgerBackend: + """Durable store for the audit v2 finding lifecycle. + + The ledger is the only channel between pipeline stages, so it always + materialises a real database file. Other MCP servers in this package fall + back to an in-memory database when their state directory is missing, but + that would be silent data loss here: an audit would appear to run, promote + findings, and then have nothing to show at the end. + """ + def __init__(self, state_dir: str): self.state_dir = state_dir - db_dir = ( - f"sqlite:///{self.state_dir}/finding_ledger.db" - if Path(self.state_dir).exists() - else "sqlite://" - ) - self.engine = create_engine(db_dir, echo=False) + Path(self.state_dir).mkdir(parents=True, exist_ok=True) + self.engine = create_engine(f"sqlite:///{self.state_dir}/finding_ledger.db", echo=False) Base.metadata.create_all( self.engine, tables=[ diff --git a/tests/test_finding_ledger.py b/tests/test_finding_ledger.py index 16ade83..c9e4b5d 100644 --- a/tests/test_finding_ledger.py +++ b/tests/test_finding_ledger.py @@ -344,6 +344,29 @@ def test_writes_against_unknown_finding_are_reported(self, ledger): ) +class TestDurability: + def test_missing_state_dir_is_created_not_silently_in_memory(self, tmp_path): + """A missing directory must not degrade the ledger to an in-memory DB. + + That fallback would let a whole audit run, promote findings, and then + lose every one of them at exit. + """ + state_dir = tmp_path / "does" / "not" / "exist" + ledger = FindingLedgerBackend(str(state_dir)) + + _add_finding(ledger) + + assert (state_dir / "finding_ledger.db").is_file() + + def test_findings_survive_a_new_backend_over_the_same_dir(self, tmp_path): + state_dir = tmp_path / "ledger" + finding_id = _add_finding(FindingLedgerBackend(str(state_dir))) + + reopened = FindingLedgerBackend(str(state_dir)) + + assert reopened.get_finding(finding_id)["title"] == "Path traversal in file download" + + class TestServerWiring: def test_toolbox_yaml_valid(self): result = AvailableTools().get_toolbox("seclab_taskflows.toolboxes.finding_ledger") From 9285e1789d24df70be5499b66e9baae8f6881a97 Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Thu, 30 Jul 2026 12:37:18 -0400 Subject: [PATCH 03/12] Move audit v2 to its own namespace, add the v2 survey store Relocate finding_ledger under mcp_servers/audit_v2/ and add repo_survey{,_models}.py with its toolbox: components and entry points keyed by trust boundary, replacing repo_context in the v2 survey and hunt. Adds tests, a temp-dir conftest, and Go in the base image. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: dadec5f9-3bf8-449c-84d6-db45be19bb6a --- .../containers/base/Dockerfile | 14 + .../containers/reproduction/Dockerfile | 1 + .../{ => audit_v2}/finding_ledger.py | 2 +- .../{ => audit_v2}/finding_ledger_models.py | 0 .../mcp_servers/audit_v2/repo_survey.py | 289 ++++++++++++++++++ .../audit_v2/repo_survey_models.py | 135 ++++++++ .../personalities/exploit_defender.yaml | 2 +- .../personalities/exploit_prosecutor.yaml | 2 +- .../personalities/finding_adjudicator.yaml | 2 +- .../personalities/reproduction_engineer.yaml | 2 +- .../personalities/vulnerability_hunter.yaml | 4 +- .../taskflows/audit_v2/README.md | 2 +- .../taskflows/audit_v2/contest.yaml | 10 +- .../taskflows/audit_v2/hunt.yaml | 26 +- .../taskflows/audit_v2/report.yaml | 6 +- .../taskflows/audit_v2/reproduce.yaml | 6 +- .../taskflows/audit_v2/survey.yaml | 71 +++-- ...dger.yaml => audit_v2_finding_ledger.yaml} | 2 +- .../toolboxes/audit_v2_repo_survey.yaml | 47 +++ .../container_shell_reproduction.yaml | 1 + tests/conftest.py | 28 ++ tests/test_finding_ledger.py | 6 +- tests/test_repo_survey.py | 190 ++++++++++++ tests/test_taskflow_corpus.py | 83 ++++- 24 files changed, 866 insertions(+), 65 deletions(-) rename src/seclab_taskflows/mcp_servers/{ => audit_v2}/finding_ledger.py (99%) rename src/seclab_taskflows/mcp_servers/{ => audit_v2}/finding_ledger_models.py (100%) create mode 100644 src/seclab_taskflows/mcp_servers/audit_v2/repo_survey.py create mode 100644 src/seclab_taskflows/mcp_servers/audit_v2/repo_survey_models.py rename src/seclab_taskflows/toolboxes/{finding_ledger.yaml => audit_v2_finding_ledger.yaml} (96%) create mode 100644 src/seclab_taskflows/toolboxes/audit_v2_repo_survey.yaml create mode 100644 tests/conftest.py create mode 100644 tests/test_repo_survey.py diff --git a/src/seclab_taskflows/containers/base/Dockerfile b/src/seclab_taskflows/containers/base/Dockerfile index 9a99e3e..6505d43 100644 --- a/src/seclab_taskflows/containers/base/Dockerfile +++ b/src/seclab_taskflows/containers/base/Dockerfile @@ -6,4 +6,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ bash coreutils python3 python3-pip curl wget git ca-certificates \ file binutils xxd \ && rm -rf /var/lib/apt/lists/* + +# Debian bookworm ships Go 1.19, which refuses to build any module declaring a +# newer toolchain, so take Go from upstream. dpkg's architecture names match +# Go's release naming for amd64 and arm64. +ARG GO_VERSION=1.23.4 +RUN arch="$(dpkg --print-architecture)" \ + && curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${arch}.tar.gz" -o /tmp/go.tar.gz \ + && tar -C /usr/local -xzf /tmp/go.tar.gz \ + && rm /tmp/go.tar.gz +ENV PATH="/usr/local/go/bin:/root/go/bin:${PATH}" +# A login shell re-reads /etc/profile and would drop the PATH set above, so +# make the toolchain reachable that way too. +RUN printf 'export PATH="/usr/local/go/bin:/root/go/bin:$PATH"\n' > /etc/profile.d/golang.sh + WORKDIR /workspace diff --git a/src/seclab_taskflows/containers/reproduction/Dockerfile b/src/seclab_taskflows/containers/reproduction/Dockerfile index 1f5dc60..a1f0b84 100644 --- a/src/seclab_taskflows/containers/reproduction/Dockerfile +++ b/src/seclab_taskflows/containers/reproduction/Dockerfile @@ -20,4 +20,5 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ jq netcat-openbsd socat procps lsof psmisc \ ripgrep tree less \ && rm -rf /var/lib/apt/lists/* + WORKDIR /workspace diff --git a/src/seclab_taskflows/mcp_servers/finding_ledger.py b/src/seclab_taskflows/mcp_servers/audit_v2/finding_ledger.py similarity index 99% rename from src/seclab_taskflows/mcp_servers/finding_ledger.py rename to src/seclab_taskflows/mcp_servers/audit_v2/finding_ledger.py index 46119af..bbc523e 100644 --- a/src/seclab_taskflows/mcp_servers/finding_ledger.py +++ b/src/seclab_taskflows/mcp_servers/audit_v2/finding_ledger.py @@ -46,7 +46,7 @@ Finding, ReproductionAttempt, ) -from .utils import process_repo +from ..utils import process_repo logging.basicConfig( level=logging.DEBUG, diff --git a/src/seclab_taskflows/mcp_servers/finding_ledger_models.py b/src/seclab_taskflows/mcp_servers/audit_v2/finding_ledger_models.py similarity index 100% rename from src/seclab_taskflows/mcp_servers/finding_ledger_models.py rename to src/seclab_taskflows/mcp_servers/audit_v2/finding_ledger_models.py diff --git a/src/seclab_taskflows/mcp_servers/audit_v2/repo_survey.py b/src/seclab_taskflows/mcp_servers/audit_v2/repo_survey.py new file mode 100644 index 0000000..28e7196 --- /dev/null +++ b/src/seclab_taskflows/mcp_servers/audit_v2/repo_survey.py @@ -0,0 +1,289 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +"""MCP server exposing the audit v2 repository survey. + +This is the v2 counterpart to ``repo_context``. It records what a repository +is made of and where untrusted data enters it, in terms that apply to any kind +of software rather than to web applications specifically. + +``repo_context`` is untouched, so v1 taskflows keep working; a v2 audit simply +uses this store instead. +""" + +import json +import logging +from pathlib import Path + +from fastmcp import FastMCP +from pydantic import Field +from seclab_taskflow_agent.path_utils import log_file_name, mcp_data_dir +from sqlalchemy import create_engine +from sqlalchemy.orm import Session + +from ..utils import process_repo +from .repo_survey_models import ( + COMPONENT_KINDS, + KIND_OTHER, + TRUST_BOUNDARIES, + Base, + Component, + EntryPoint, + component_to_dict, + entry_point_to_dict, +) + +logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(levelname)s - %(message)s", + filename=log_file_name("mcp_repo_survey.log"), + filemode="a", +) + +MEMORY = mcp_data_dir("seclab-taskflows", "repo_survey", "REPO_SURVEY_DIR") + + +class InvalidSurveyValueError(ValueError): + """Raised when a caller supplies a value outside an allowed set.""" + + +def _require(value: str, allowed, name: str, default: str | None = None) -> str: + """Validate an enum-like argument, raising a message the model can act on.""" + normalized = (value or "").strip().lower() + if not normalized and default is not None: + return default + if normalized not in allowed: + msg = f"invalid {name} {value!r}; expected one of: {', '.join(allowed)}" + raise InvalidSurveyValueError(msg) + return normalized + + +class RepoSurveyBackend: + """Durable store for the audit v2 survey. + + Always writes a real database file for the same reason the finding ledger + does: a survey that silently evaporates would send every later stage + hunting an empty map. + """ + + def __init__(self, state_dir: str): + self.state_dir = state_dir + Path(self.state_dir).mkdir(parents=True, exist_ok=True) + self.engine = create_engine(f"sqlite:///{self.state_dir}/repo_survey.db", echo=False) + Base.metadata.create_all( + self.engine, tables=[Component.__table__, EntryPoint.__table__] + ) + + # -- writes ------------------------------------------------------------ + + def store_component( + self, repo, location, kind, language, runtime, is_app, is_library, notes + ): + kind = _require(kind, COMPONENT_KINDS, "kind", default=KIND_OTHER) + with Session(self.engine) as session: + component = Component( + repo=repo, + location=location, + kind=kind, + language=language or "", + runtime=runtime or "", + is_app=bool(is_app), + is_library=bool(is_library), + notes=notes or "", + ) + session.add(component) + session.commit() + return component.id + + def store_entry_point( + self, repo, component_id, file, line, trust_boundary, untrusted_input, variables, notes + ): + trust_boundary = _require(trust_boundary, TRUST_BOUNDARIES, "trust_boundary") + with Session(self.engine) as session: + if session.get(Component, component_id) is None: + return f"No component with id {component_id}" + entry_point = EntryPoint( + repo=repo, + component_id=component_id, + file=file, + line=int(line or 0), + trust_boundary=trust_boundary, + untrusted_input=untrusted_input or "", + variables=variables or "", + notes=notes or "", + ) + session.add(entry_point) + session.commit() + return entry_point.id + + # -- reads ------------------------------------------------------------- + + def get_components(self, repo): + with Session(self.engine) as session: + rows = session.query(Component).filter(Component.repo == repo).all() + return [component_to_dict(c) for c in rows] + + def get_component(self, component_id): + with Session(self.engine) as session: + component = session.get(Component, component_id) + if component is None: + return None + data = component_to_dict(component) + rows = ( + session.query(EntryPoint) + .filter(EntryPoint.component_id == component_id) + .all() + ) + data["entry_points"] = [entry_point_to_dict(e) for e in rows] + return data + + def get_entry_points(self, repo, component_id=None): + with Session(self.engine) as session: + query = session.query(EntryPoint).filter(EntryPoint.repo == repo) + if component_id: + query = query.filter(EntryPoint.component_id == component_id) + return [entry_point_to_dict(e) for e in query.all()] + + def get_survey_summary(self, repo): + with Session(self.engine) as session: + components = session.query(Component).filter(Component.repo == repo).all() + entry_points = session.query(EntryPoint).filter(EntryPoint.repo == repo).all() + by_boundary: dict[str, int] = dict.fromkeys(TRUST_BOUNDARIES, 0) + for e in entry_points: + by_boundary[e.trust_boundary] = by_boundary.get(e.trust_boundary, 0) + 1 + return { + "repo": repo, + "components": len(components), + "entry_points": len(entry_points), + "by_trust_boundary": by_boundary, + } + + def clear_survey(self, repo): + with Session(self.engine) as session: + components = session.query(Component).filter(Component.repo == repo).delete() + session.query(EntryPoint).filter(EntryPoint.repo == repo).delete() + session.commit() + return components + + +backend = RepoSurveyBackend(str(MEMORY)) + +mcp = FastMCP("RepoSurvey") + + +@mcp.tool() +def store_component( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), + location: str = Field(description="Directory or module path of the component"), + kind: str = Field( + description=f"What kind of component this is, one of: {', '.join(COMPONENT_KINDS)}", + default=KIND_OTHER, + ), + language: str = Field(description="Primary implementation language", default=""), + runtime: str = Field(description="Runtime or platform it executes on", default=""), + is_app: bool = Field(description="True if it is reached as a running program", default=False), + is_library: bool = Field(description="True if it is consumed by callers as an API", default=False), + notes: str = Field( + description="What it does, who talks to it across which trust boundary, " + "and what makes it interesting to attack", + default="", + ), +): + """Store a component of the repository and return its id.""" + repo = process_repo(owner, repo) + try: + component_id = backend.store_component( + repo, location, kind, language, runtime, is_app, is_library, notes + ) + except InvalidSurveyValueError as exc: + return str(exc) + return json.dumps({"component_id": component_id}) + + +@mcp.tool() +def store_entry_point( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), + component_id: int = Field(description="The id of the component this entry point belongs to"), + file: str = Field(description="Path to the file containing the entry point"), + trust_boundary: str = Field( + description=f"Which boundary is crossed, one of: {', '.join(TRUST_BOUNDARIES)}" + ), + line: int = Field(description="Line number of the entry point", default=0), + untrusted_input: str = Field( + description="What untrusted data arrives here and where it came from", default="" + ), + variables: str = Field(description="Variables carrying the untrusted data", default=""), + notes: str = Field( + description="Why the data on the other side of the boundary is untrusted", default="" + ), +): + """Store an entry point where untrusted data crosses into a component.""" + repo = process_repo(owner, repo) + try: + result = backend.store_entry_point( + repo, component_id, file, line, trust_boundary, untrusted_input, variables, notes + ) + except InvalidSurveyValueError as exc: + return str(exc) + if isinstance(result, str): + return result + return json.dumps({"entry_point_id": result}) + + +@mcp.tool() +def get_components( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), +): + """Get every component recorded for a repository.""" + return json.dumps(backend.get_components(process_repo(owner, repo))) + + +@mcp.tool() +def get_component( + component_id: int = Field(description="The id of the component"), +): + """Get one component together with its entry points.""" + component = backend.get_component(component_id) + if component is None: + return f"No component with id {component_id}" + return json.dumps(component) + + +@mcp.tool() +def get_entry_points( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), + component_id: int = Field( + description="Optionally restrict to one component; 0 means all", default=0 + ), +): + """Get the entry points for a repository, optionally for one component.""" + repo = process_repo(owner, repo) + return json.dumps(backend.get_entry_points(repo, component_id or None)) + + +@mcp.tool() +def get_survey_summary( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), +): + """Get counts of components and entry points, grouped by trust boundary.""" + return json.dumps(backend.get_survey_summary(process_repo(owner, repo))) + + +@mcp.tool() +def clear_survey_for_repo( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), +): + """Delete every component and entry point recorded for a repository.""" + repo = process_repo(owner, repo) + removed = backend.clear_survey(repo) + return f"Cleared survey for {repo} ({removed} components)" + + +if __name__ == "__main__": + mcp.run() diff --git a/src/seclab_taskflows/mcp_servers/audit_v2/repo_survey_models.py b/src/seclab_taskflows/mcp_servers/audit_v2/repo_survey_models.py new file mode 100644 index 0000000..f61988f --- /dev/null +++ b/src/seclab_taskflows/mcp_servers/audit_v2/repo_survey_models.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +"""Schema for the audit v2 repository survey. + +The v1 ``repo_context`` store is shaped around web applications: it models +applications, web entry points, security entry points and user actions. That +shape is a good fit for auditing a web app and a poor fit for auditing a +parser, a daemon, a build plugin or a native library, which is what audit v2 +is meant to cover. + +This is a separate store rather than a change to ``repo_context`` so that +existing v1 taskflows keep working exactly as they do today. + +The organising idea here is the trust boundary. A component is a unit of code +worth reasoning about on its own, and an entry point is a place where data +crosses into it from somewhere less trusted. Both are deliberately neutral +about what kind of software is being audited. +""" + +from __future__ import annotations + +from sqlalchemy import Boolean, Column, DateTime, Integer, String, Text, func +from sqlalchemy.orm import declarative_base + +Base = declarative_base() + +# What kind of thing a component is. This is descriptive, not a taxonomy to +# argue about; it exists so a hunter knows how the code is reached. +KIND_APPLICATION = "application" +KIND_SERVICE = "service" +KIND_LIBRARY = "library" +KIND_PARSER = "parser" +KIND_CLI = "cli" +KIND_DAEMON = "daemon" +KIND_PLUGIN = "plugin" +KIND_OTHER = "other" +COMPONENT_KINDS = ( + KIND_APPLICATION, + KIND_SERVICE, + KIND_LIBRARY, + KIND_PARSER, + KIND_CLI, + KIND_DAEMON, + KIND_PLUGIN, + KIND_OTHER, +) + +# Which boundary the untrusted data crosses. This is the field that generalises +# the v1 notion of a "web entry point" to arbitrary software. +BOUNDARY_NETWORK = "network" +BOUNDARY_FILE = "file" +BOUNDARY_IPC = "ipc" +BOUNDARY_PROCESS = "process" +BOUNDARY_STORED_DATA = "stored_data" +BOUNDARY_PACKAGE_CONTENT = "package_content" +BOUNDARY_LIBRARY_API = "library_api" +BOUNDARY_OTHER = "other" +TRUST_BOUNDARIES = ( + BOUNDARY_NETWORK, + BOUNDARY_FILE, + BOUNDARY_IPC, + BOUNDARY_PROCESS, + BOUNDARY_STORED_DATA, + BOUNDARY_PACKAGE_CONTENT, + BOUNDARY_LIBRARY_API, + BOUNDARY_OTHER, +) + + +class Component(Base): + """A unit of functionality that can be reasoned about on its own.""" + + __tablename__ = "component_v2" + + id = Column(Integer, primary_key=True) + repo = Column(String, index=True, nullable=False) + location = Column(String, nullable=False) + kind = Column(String, default=KIND_OTHER) + language = Column(String, default="") + runtime = Column(String, default="") + # Kept alongside `kind` because "can a caller pass me attacker data?" and + # "am I reachable over the network?" are independent questions, and a + # component is frequently both a library and an application. + is_app = Column(Boolean, default=False) + is_library = Column(Boolean, default=False) + notes = Column(Text, default="") + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +class EntryPoint(Base): + """A place where data crosses into a component from somewhere less trusted.""" + + __tablename__ = "entry_point_v2" + + id = Column(Integer, primary_key=True) + repo = Column(String, index=True, nullable=False) + component_id = Column(Integer, index=True, nullable=False) + file = Column(String, nullable=False) + line = Column(Integer, default=0) + trust_boundary = Column(String, default=BOUNDARY_OTHER) + untrusted_input = Column(Text, default="") + variables = Column(Text, default="") + notes = Column(Text, default="") + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +# These projections are the wire contract the taskflows declare `outputs` against, +# so they live here, importable without starting a server or opening a database. +def component_to_dict(c: Component) -> dict: + return { + "component_id": c.id, + "repo": c.repo, + "location": c.location, + "kind": c.kind, + "language": c.language, + "runtime": c.runtime, + "is_app": bool(c.is_app), + "is_library": bool(c.is_library), + "notes": c.notes, + } + + +def entry_point_to_dict(e: EntryPoint) -> dict: + return { + "entry_point_id": e.id, + "repo": e.repo, + "component_id": e.component_id, + "file": e.file, + "line": e.line, + "trust_boundary": e.trust_boundary, + "untrusted_input": e.untrusted_input, + "variables": e.variables, + "notes": e.notes, + } diff --git a/src/seclab_taskflows/personalities/exploit_defender.yaml b/src/seclab_taskflows/personalities/exploit_defender.yaml index 6b1e7e8..55590f6 100644 --- a/src/seclab_taskflows/personalities/exploit_defender.yaml +++ b/src/seclab_taskflows/personalities/exploit_defender.yaml @@ -46,4 +46,4 @@ task: | toolboxes: - seclab_taskflow_agent.toolboxes.memcache - seclab_taskflows.toolboxes.container_shell_source_access - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_finding_ledger diff --git a/src/seclab_taskflows/personalities/exploit_prosecutor.yaml b/src/seclab_taskflows/personalities/exploit_prosecutor.yaml index d4afcb1..67c4cbd 100644 --- a/src/seclab_taskflows/personalities/exploit_prosecutor.yaml +++ b/src/seclab_taskflows/personalities/exploit_prosecutor.yaml @@ -38,4 +38,4 @@ task: | toolboxes: - seclab_taskflow_agent.toolboxes.memcache - seclab_taskflows.toolboxes.container_shell_source_access - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_finding_ledger diff --git a/src/seclab_taskflows/personalities/finding_adjudicator.yaml b/src/seclab_taskflows/personalities/finding_adjudicator.yaml index 9d0704d..71e40da 100644 --- a/src/seclab_taskflows/personalities/finding_adjudicator.yaml +++ b/src/seclab_taskflows/personalities/finding_adjudicator.yaml @@ -56,4 +56,4 @@ task: | toolboxes: - seclab_taskflow_agent.toolboxes.memcache - seclab_taskflows.toolboxes.container_shell_source_access - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_finding_ledger diff --git a/src/seclab_taskflows/personalities/reproduction_engineer.yaml b/src/seclab_taskflows/personalities/reproduction_engineer.yaml index 0667f53..cfee27d 100644 --- a/src/seclab_taskflows/personalities/reproduction_engineer.yaml +++ b/src/seclab_taskflows/personalities/reproduction_engineer.yaml @@ -51,4 +51,4 @@ task: | toolboxes: - seclab_taskflow_agent.toolboxes.memcache - seclab_taskflows.toolboxes.container_shell_reproduction - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_finding_ledger diff --git a/src/seclab_taskflows/personalities/vulnerability_hunter.yaml b/src/seclab_taskflows/personalities/vulnerability_hunter.yaml index 1eed9b0..91fbe51 100644 --- a/src/seclab_taskflows/personalities/vulnerability_hunter.yaml +++ b/src/seclab_taskflows/personalities/vulnerability_hunter.yaml @@ -48,5 +48,5 @@ task: | toolboxes: - seclab_taskflow_agent.toolboxes.memcache - seclab_taskflows.toolboxes.container_shell_source_access - - seclab_taskflows.toolboxes.repo_context - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_repo_survey + - seclab_taskflows.toolboxes.audit_v2_finding_ledger diff --git a/src/seclab_taskflows/taskflows/audit_v2/README.md b/src/seclab_taskflows/taskflows/audit_v2/README.md index 5033487..8e4013e 100644 --- a/src/seclab_taskflows/taskflows/audit_v2/README.md +++ b/src/seclab_taskflows/taskflows/audit_v2/README.md @@ -127,7 +127,7 @@ Useful variants: | Stage | What it does | Ledger effect | | --- | --- | --- | -| `survey` | Fetches the source, decomposes it into components, maps where untrusted data enters each one | populates `repo_context` | +| `survey` | Fetches the source, decomposes it into components, maps where untrusted data enters each one | populates the v2 survey store | | `hunt` | Three model families hunt each component in parallel, then a dedup pass folds convergent findings | creates `candidate`s, some `duplicate` | | `contest` | Prosecution, defense, adjudication | `candidate` → `confirmed` or `rejected` | | `reproduce` | Builds and runs the target in a container, drives the path with a control case first, then the attack | `confirmed` → `reproduced` | diff --git a/src/seclab_taskflows/taskflows/audit_v2/contest.yaml b/src/seclab_taskflows/taskflows/audit_v2/contest.yaml index f232ab3..10ea9bc 100644 --- a/src/seclab_taskflows/taskflows/audit_v2/contest.yaml +++ b/src/seclab_taskflows/taskflows/audit_v2/contest.yaml @@ -49,7 +49,7 @@ taskflow: vuln_class: {type: string} required: [finding_id, repo, component, title, vuln_class] toolboxes: - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_finding_ledger - task: if: "outputs.candidates | length > 0" @@ -92,7 +92,7 @@ taskflow: {% include 'seclab_taskflows.prompts.audit_v2.evidence_rules' %} toolboxes: - seclab_taskflows.toolboxes.container_shell_source_access - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_finding_ledger - task: if: "outputs.candidates | length > 0" @@ -137,7 +137,7 @@ taskflow: {% include 'seclab_taskflows.prompts.audit_v2.evidence_rules' %} toolboxes: - seclab_taskflows.toolboxes.container_shell_source_access - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_finding_ledger - task: if: "outputs.candidates | length > 0" @@ -184,7 +184,7 @@ taskflow: {% include 'seclab_taskflows.prompts.audit_v2.evidence_rules' %} toolboxes: - seclab_taskflows.toolboxes.container_shell_source_access - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_finding_ledger - task: must_complete: true @@ -199,4 +199,4 @@ taskflow: confirmed findings with their severity and the rejected findings with the reason they were rejected. toolboxes: - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_finding_ledger diff --git a/src/seclab_taskflows/taskflows/audit_v2/hunt.yaml b/src/seclab_taskflows/taskflows/audit_v2/hunt.yaml index 29845af..1dacfa0 100644 --- a/src/seclab_taskflows/taskflows/audit_v2/hunt.yaml +++ b/src/seclab_taskflows/taskflows/audit_v2/hunt.yaml @@ -43,12 +43,13 @@ taskflow: items: type: object properties: - id: {type: integer} + component_id: {type: integer} repo: {type: string} location: {type: string} - required: [id, repo, location] + kind: {type: string} + required: [component_id, repo, location] toolboxes: - - seclab_taskflows.toolboxes.repo_context + - seclab_taskflows.toolboxes.audit_v2_repo_survey - task: if: "outputs.components | length > 0" @@ -69,14 +70,17 @@ taskflow: What previous analysis recorded about this component: + kind: {{ result.kind | default('unknown') }} is_application ? {{ result.is_app | default('unknown') }} - is_library ? {{ result.is_library | default('unknown') }} + is_library ? {{ result.is_library | default('unknown') }} {{ result.notes | default('') }} Start from the entry points already mapped for this component; fetch - them with the repo context tools. They are a starting point, not a - limit. If you find an entry point that was missed, hunt it too. + them with `get_entry_points` for component_id {{ result.component_id }}. + Each one records the trust boundary it crosses. They are a starting + point, not a limit. If you find an entry point that was missed, hunt it + too. Work outward from each entry point: @@ -108,8 +112,8 @@ taskflow: {% include 'seclab_taskflows.prompts.audit_v2.evidence_rules' %} toolboxes: - seclab_taskflows.toolboxes.container_shell_source_access - - seclab_taskflows.toolboxes.repo_context - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_repo_survey + - seclab_taskflows.toolboxes.audit_v2_finding_ledger - task: id: raw_candidates @@ -136,7 +140,7 @@ taskflow: proposed_by: {type: string} required: [finding_id, component, title, vuln_class] toolboxes: - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_finding_ledger - task: if: "outputs.raw_candidates | length > 1" @@ -190,7 +194,7 @@ taskflow: groups you merged, and which findings more than one model family found independently. toolboxes: - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_finding_ledger - task: must_complete: true @@ -206,4 +210,4 @@ taskflow: title and `proposed_by`. Report them grouped by component, and call out any that were proposed by more than one model. toolboxes: - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_finding_ledger diff --git a/src/seclab_taskflows/taskflows/audit_v2/report.yaml b/src/seclab_taskflows/taskflows/audit_v2/report.yaml index c7dd70f..e257b5b 100644 --- a/src/seclab_taskflows/taskflows/audit_v2/report.yaml +++ b/src/seclab_taskflows/taskflows/audit_v2/report.yaml @@ -45,7 +45,7 @@ taskflow: proposed_by: {type: string} required: [finding_id, component, title, vuln_class, state] toolboxes: - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_finding_ledger - task: id: draft @@ -121,7 +121,7 @@ taskflow: {% include 'seclab_taskflows.prompts.audit_v2.severity_rubric' %} toolboxes: - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_finding_ledger - task: if: "outputs.draft is defined and outputs.draft" @@ -159,4 +159,4 @@ taskflow: {{ outputs.draft }} toolboxes: - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_finding_ledger diff --git a/src/seclab_taskflows/taskflows/audit_v2/reproduce.yaml b/src/seclab_taskflows/taskflows/audit_v2/reproduce.yaml index b7ab644..6be6e3d 100644 --- a/src/seclab_taskflows/taskflows/audit_v2/reproduce.yaml +++ b/src/seclab_taskflows/taskflows/audit_v2/reproduce.yaml @@ -46,7 +46,7 @@ taskflow: severity: {type: string} required: [finding_id, repo, component, title, vuln_class] toolboxes: - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_finding_ledger - task: if: "outputs.confirmed | length > 0" @@ -100,7 +100,7 @@ taskflow: {% include 'seclab_taskflows.prompts.audit_v2.evidence_rules' %} toolboxes: - seclab_taskflows.toolboxes.container_shell_reproduction - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_finding_ledger - task: must_complete: true @@ -116,4 +116,4 @@ taskflow: state `confirmed` together with why their reproduction attempt did not succeed. toolboxes: - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_finding_ledger diff --git a/src/seclab_taskflows/taskflows/audit_v2/survey.yaml b/src/seclab_taskflows/taskflows/audit_v2/survey.yaml index 03af530..d6929a5 100644 --- a/src/seclab_taskflows/taskflows/audit_v2/survey.yaml +++ b/src/seclab_taskflows/taskflows/audit_v2/survey.yaml @@ -27,21 +27,36 @@ taskflow: exclude_from_context: true max_steps: 20 model: general_tasks - name: reset and fetch - description: Clear prior state for this repo and fetch its source. + name: reset prior state + description: Clear any state left over from an earlier run of this repo. agents: - seclab_taskflow_agent.personalities.assistant user_prompt: | - Clear the memory cache. Clear the repo context results for the repo + Clear the memory cache. Clear the survey for the repo {{ globals.repo }}. Clear the findings in the finding ledger for the repo {{ globals.repo }}. - - Then fetch and extract the source code of the repo {{ globals.repo }} - for container_shell analysis. toolboxes: - seclab_taskflow_agent.toolboxes.memcache - - seclab_taskflows.toolboxes.repo_context - - seclab_taskflows.toolboxes.finding_ledger + - seclab_taskflows.toolboxes.audit_v2_repo_survey + - seclab_taskflows.toolboxes.audit_v2_finding_ledger + + # Kept as its own task with a single toolbox on purpose. Folded in with the + # resets above, the model reliably did one of the jobs and stopped, leaving + # an empty /workspace for every stage that follows. + - task: + must_complete: true + headless: true + exclude_from_context: true + max_steps: 20 + model: general_tasks + name: fetch source + description: Fetch and extract the source code for container analysis. + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + Fetch and extract the source code of the repo {{ globals.repo }} for + container_shell analysis. + toolboxes: - seclab_taskflows.toolboxes.local_gh_resources - task: @@ -68,12 +83,13 @@ taskflow: what they do. If the repository contains several applications, each one is its own component. - For each component, store an entry with `store_new_component`. Set - `is_app` and `is_library` according to how the component is consumed, - and use `notes` to record: + For each component, store an entry with `store_component`. Set its + `kind` to the closest of: application, service, library, parser, cli, + daemon, plugin, other. Set `is_app` and `is_library` according to how + the component is consumed; a component is frequently both. Record + `language` and `runtime`, and use `notes` to record: - what the component does - - what language and runtime it is written in - who or what talks to it, and across which trust boundary - anything that makes it interesting to attack: it parses untrusted formats, it executes things, it makes authorisation decisions, it @@ -85,7 +101,7 @@ taskflow: {% include 'seclab_taskflows.prompts.audit_v2.evidence_rules' %} toolboxes: - seclab_taskflows.toolboxes.container_shell_source_access - - seclab_taskflows.toolboxes.repo_context + - seclab_taskflows.toolboxes.audit_v2_repo_survey - task: id: components @@ -105,12 +121,13 @@ taskflow: items: type: object properties: - id: {type: integer} + component_id: {type: integer} repo: {type: string} location: {type: string} - required: [id, repo, location] + kind: {type: string} + required: [component_id, repo, location] toolboxes: - - seclab_taskflows.toolboxes.repo_context + - seclab_taskflows.toolboxes.audit_v2_repo_survey - task: if: "outputs.components | length > 0" @@ -127,10 +144,12 @@ taskflow: - seclab_taskflow_agent.personalities.assistant user_prompt: | The component is in {{ result.repo }} in the directory - {{ result.location }}. Previous analysis recorded: + {{ result.location }}, and its id is {{ result.component_id }}. + Previous analysis recorded: + kind: {{ result.kind | default('unknown') }} is_application ? {{ result.is_app | default('unknown') }} - is_library ? {{ result.is_library | default('unknown') }} + is_library ? {{ result.is_library | default('unknown') }} {{ result.notes | default('') }} @@ -157,11 +176,13 @@ taskflow: filename the operator supplies is trusted, but the file's contents may not be. - For each entry point, call `store_new_entry_point` with the precise - file and integer line number, the variables that carry untrusted data, - and notes explaining which trust boundary is crossed and why the data - on the other side is untrusted. If an entry point spans several lines, - use the first. + For each entry point, call `store_entry_point` with `component_id` + {{ result.component_id }}, the precise file and integer line number, + the `trust_boundary` that is crossed (one of: network, file, ipc, + process, stored_data, package_content, library_api, other), the + variables that carry untrusted data, and notes explaining why the data + on the other side of that boundary is untrusted. If an entry point + spans several lines, use the first. Do not assess vulnerabilities here. You are drawing the map, not hunting on it. @@ -169,7 +190,7 @@ taskflow: {% include 'seclab_taskflows.prompts.audit_v2.evidence_rules' %} toolboxes: - seclab_taskflows.toolboxes.container_shell_source_access - - seclab_taskflows.toolboxes.repo_context + - seclab_taskflows.toolboxes.audit_v2_repo_survey - task: must_complete: true @@ -187,4 +208,4 @@ taskflow: untrusted data enters each one. Call out the components you would prioritise for hunting and say why, in one or two sentences each. toolboxes: - - seclab_taskflows.toolboxes.repo_context + - seclab_taskflows.toolboxes.audit_v2_repo_survey diff --git a/src/seclab_taskflows/toolboxes/finding_ledger.yaml b/src/seclab_taskflows/toolboxes/audit_v2_finding_ledger.yaml similarity index 96% rename from src/seclab_taskflows/toolboxes/finding_ledger.yaml rename to src/seclab_taskflows/toolboxes/audit_v2_finding_ledger.yaml index f4a5e98..2124157 100644 --- a/src/seclab_taskflows/toolboxes/finding_ledger.yaml +++ b/src/seclab_taskflows/toolboxes/audit_v2_finding_ledger.yaml @@ -7,7 +7,7 @@ seclab-taskflow-agent: server_params: kind: stdio command: python - args: ["-m", "seclab_taskflows.mcp_servers.finding_ledger"] + args: ["-m", "seclab_taskflows.mcp_servers.audit_v2.finding_ledger"] env: FINDING_LEDGER_DIR: "{{ env('DATA_DIR') }}" LOG_DIR: "{{ env('LOG_DIR') }}" diff --git a/src/seclab_taskflows/toolboxes/audit_v2_repo_survey.yaml b/src/seclab_taskflows/toolboxes/audit_v2_repo_survey.yaml new file mode 100644 index 0000000..61ba71a --- /dev/null +++ b/src/seclab_taskflows/toolboxes/audit_v2_repo_survey.yaml @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: toolbox + version: "1.0" + +server_params: + kind: stdio + command: python + args: ["-m", "seclab_taskflows.mcp_servers.audit_v2.repo_survey"] + env: + REPO_SURVEY_DIR: "{{ env('DATA_DIR') }}" + LOG_DIR: "{{ env('LOG_DIR') }}" + +server_prompt: | + ## Repository Survey (audit v2) + + The survey is the map the rest of the audit hunts over. It records what the + repository is made of and where untrusted data enters it. + + A **component** is a coherent unit of functionality that can be reasoned + about on its own: one service, one parser, one protocol implementation, one + subsystem. It is not necessarily a web application. Record its `kind` + (application, service, library, parser, cli, daemon, plugin, other), and set + `is_app` and `is_library` independently, because plenty of components are + both. + + An **entry point** is a place where data crosses into a component from + somewhere less trusted. Always record which `trust_boundary` is crossed: + + - `network` — requests, responses, protocol messages, peers + - `file` — parsed bytes that came from elsewhere + - `ipc` — pipes, sockets, shared memory from another process + - `process` — argv and environment set by a less privileged caller + - `stored_data` — database rows, cache entries, queue messages + - `package_content` — filenames, paths, manifests, repository metadata + - `library_api` — public API parameters a caller may pass attacker data + - `other` — anything that does not fit the above + + Configuration the operator wrote, arguments the operator typed and + environment the operator set are *not* untrusted on their own. They become + interesting when they point at untrusted resources: a filename the operator + supplies is trusted, the file's contents may not be. + + Record entry points with a precise file and integer line number. If one spans + several lines, use the first. diff --git a/src/seclab_taskflows/toolboxes/container_shell_reproduction.yaml b/src/seclab_taskflows/toolboxes/container_shell_reproduction.yaml index ee954db..43019cb 100644 --- a/src/seclab_taskflows/toolboxes/container_shell_reproduction.yaml +++ b/src/seclab_taskflows/toolboxes/container_shell_reproduction.yaml @@ -30,6 +30,7 @@ server_prompt: | Available runtimes and tools: - python3 (with venv and dev headers), pip3 - node, npm + - go - java (headless JRE) - gcc, g++, make (build-essential) - gdb, valgrind, strace — for memory-safety and crash triage diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..4936eaf --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +"""Shared pytest configuration. + +Several MCP server modules open their log file and their state database at +import time, so the destination has to be chosen before any test module is +imported. Left alone they write into the user's real application support and +log directories, which makes the suite depend on -- and mutate -- state outside +the checkout. Point them at a throwaway directory instead, without overriding a +value the caller deliberately set. +""" + +import os +import tempfile + +_TMP = tempfile.mkdtemp(prefix="seclab-taskflows-tests-") + +for _var, _sub in ( + ("LOG_DIR", "logs"), + ("FINDING_LEDGER_DIR", "finding_ledger"), + ("REPO_SURVEY_DIR", "repo_survey"), + ("REPO_CONTEXT_DIR", "repo_context"), +): + if not os.environ.get(_var): + _path = os.path.join(_TMP, _sub) + os.makedirs(_path, exist_ok=True) + os.environ[_var] = _path diff --git a/tests/test_finding_ledger.py b/tests/test_finding_ledger.py index c9e4b5d..1222b06 100644 --- a/tests/test_finding_ledger.py +++ b/tests/test_finding_ledger.py @@ -14,12 +14,12 @@ from seclab_taskflow_agent.available_tools import AvailableTools from seclab_taskflow_agent.models import ToolboxDocument -from seclab_taskflows.mcp_servers.finding_ledger import ( +from seclab_taskflows.mcp_servers.audit_v2.finding_ledger import ( FindingLedgerBackend, InvalidLedgerValueError, mcp, ) -from seclab_taskflows.mcp_servers.finding_ledger_models import ( +from seclab_taskflows.mcp_servers.audit_v2.finding_ledger_models import ( STATE_CANDIDATE, STATE_CONFIRMED, STATE_DUPLICATE, @@ -369,7 +369,7 @@ def test_findings_survive_a_new_backend_over_the_same_dir(self, tmp_path): class TestServerWiring: def test_toolbox_yaml_valid(self): - result = AvailableTools().get_toolbox("seclab_taskflows.toolboxes.finding_ledger") + result = AvailableTools().get_toolbox("seclab_taskflows.toolboxes.audit_v2_finding_ledger") assert result is not None assert isinstance(result, ToolboxDocument) diff --git a/tests/test_repo_survey.py b/tests/test_repo_survey.py new file mode 100644 index 0000000..3658acc --- /dev/null +++ b/tests/test_repo_survey.py @@ -0,0 +1,190 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +"""Tests for the audit v2 repository survey store. + +The survey is the map every later stage hunts over, so the properties worth +pinning down are the ones whose failure would send a stage looking at nothing: +the store must be durable, it must reject vocabulary it cannot use, and it must +refuse to attach an entry point to a component that does not exist. +""" + +import pytest + +from seclab_taskflow_agent.available_tools import AvailableTools + +from seclab_taskflows.mcp_servers.audit_v2.repo_survey import ( + InvalidSurveyValueError, + RepoSurveyBackend, +) +from seclab_taskflows.mcp_servers.audit_v2.repo_survey_models import ( + COMPONENT_KINDS, + TRUST_BOUNDARIES, +) + +REPO = "acme/widget" +OTHER_REPO = "acme/gadget" + + +@pytest.fixture +def survey(tmp_path): + return RepoSurveyBackend(str(tmp_path / "state")) + + +def _component(survey, repo=REPO, location="src/parser", kind="parser"): + return survey.store_component( + repo, location, kind, "c", "native", False, True, "decodes untrusted frames" + ) + + +class TestComponents: + def test_stores_and_reads_back_a_component(self, survey): + component_id = _component(survey) + components = survey.get_components(REPO) + assert len(components) == 1 + assert components[0]["component_id"] == component_id + assert components[0]["kind"] == "parser" + assert components[0]["is_library"] is True + assert components[0]["is_app"] is False + + def test_components_are_scoped_to_their_repo(self, survey): + _component(survey) + _component(survey, repo=OTHER_REPO) + assert len(survey.get_components(REPO)) == 1 + assert len(survey.get_components(OTHER_REPO)) == 1 + + def test_a_component_may_be_both_app_and_library(self, survey): + component_id = survey.store_component(REPO, "src/cli", "cli", "go", "", True, True, "") + component = survey.get_component(component_id) + assert component["is_app"] is True + assert component["is_library"] is True + + def test_unknown_kind_is_rejected_with_the_allowed_set(self, survey): + with pytest.raises(InvalidSurveyValueError) as exc: + survey.store_component(REPO, "src", "microservice", "", "", True, False, "") + message = str(exc.value) + assert "microservice" in message + assert "application" in message + + def test_kind_is_normalised(self, survey): + component_id = survey.store_component(REPO, "src", " Parser ", "", "", False, True, "") + assert survey.get_component(component_id)["kind"] == "parser" + + def test_missing_kind_falls_back_rather_than_failing(self, survey): + component_id = survey.store_component(REPO, "src", "", "", "", False, False, "") + assert survey.get_component(component_id)["kind"] == "other" + + def test_every_documented_kind_is_accepted(self, survey): + for kind in COMPONENT_KINDS: + assert survey.store_component(REPO, f"src/{kind}", kind, "", "", False, False, "") + + def test_get_component_of_unknown_id_is_none(self, survey): + assert survey.get_component(4242) is None + + +class TestEntryPoints: + def test_stores_an_entry_point_against_a_component(self, survey): + component_id = _component(survey) + entry_id = survey.store_entry_point( + REPO, component_id, "src/parser/frame.c", 120, "network", "frame bytes", "buf", "" + ) + entry_points = survey.get_entry_points(REPO) + assert len(entry_points) == 1 + assert entry_points[0]["entry_point_id"] == entry_id + assert entry_points[0]["trust_boundary"] == "network" + assert entry_points[0]["line"] == 120 + + def test_refuses_to_attach_to_a_component_that_does_not_exist(self, survey): + result = survey.store_entry_point(REPO, 999, "src/parser/frame.c", 1, "network", "", "", "") + assert isinstance(result, str) + assert "999" in result + assert survey.get_entry_points(REPO) == [] + + def test_unknown_trust_boundary_is_rejected(self, survey): + component_id = _component(survey) + with pytest.raises(InvalidSurveyValueError) as exc: + survey.store_entry_point(REPO, component_id, "src/f.c", 1, "http_request", "", "", "") + assert "network" in str(exc.value) + + def test_a_missing_trust_boundary_is_rejected_rather_than_guessed(self, survey): + """The boundary is the whole point of the record, so it has no default.""" + component_id = _component(survey) + with pytest.raises(InvalidSurveyValueError): + survey.store_entry_point(REPO, component_id, "src/f.c", 1, "", "", "", "") + + def test_every_documented_boundary_is_accepted(self, survey): + component_id = _component(survey) + for boundary in TRUST_BOUNDARIES: + assert survey.store_entry_point(REPO, component_id, "src/f.c", 1, boundary, "", "", "") + + def test_entry_points_can_be_filtered_by_component(self, survey): + first = _component(survey, location="src/a") + second = _component(survey, location="src/b") + survey.store_entry_point(REPO, first, "src/a/x.c", 1, "network", "", "", "") + survey.store_entry_point(REPO, second, "src/b/y.c", 2, "file", "", "", "") + assert len(survey.get_entry_points(REPO)) == 2 + only_first = survey.get_entry_points(REPO, first) + assert len(only_first) == 1 + assert only_first[0]["component_id"] == first + + def test_get_component_includes_its_entry_points(self, survey): + component_id = _component(survey) + survey.store_entry_point( + REPO, component_id, "src/parser/frame.c", 7, "file", "archive member", "", "" + ) + component = survey.get_component(component_id) + assert [e["file"] for e in component["entry_points"]] == ["src/parser/frame.c"] + + +class TestSummaryAndClear: + def test_summary_counts_by_trust_boundary(self, survey): + component_id = _component(survey) + survey.store_entry_point(REPO, component_id, "a.c", 1, "network", "", "", "") + survey.store_entry_point(REPO, component_id, "b.c", 2, "network", "", "", "") + survey.store_entry_point(REPO, component_id, "c.c", 3, "file", "", "", "") + summary = survey.get_survey_summary(REPO) + assert summary["components"] == 1 + assert summary["entry_points"] == 3 + assert summary["by_trust_boundary"]["network"] == 2 + assert summary["by_trust_boundary"]["file"] == 1 + assert summary["by_trust_boundary"]["ipc"] == 0 + + def test_clearing_one_repo_leaves_the_other_alone(self, survey): + kept = _component(survey, repo=OTHER_REPO) + cleared = _component(survey) + survey.store_entry_point(REPO, cleared, "a.c", 1, "network", "", "", "") + survey.store_entry_point(OTHER_REPO, kept, "b.c", 1, "network", "", "", "") + + survey.clear_survey(REPO) + + assert survey.get_components(REPO) == [] + assert survey.get_entry_points(REPO) == [] + assert len(survey.get_components(OTHER_REPO)) == 1 + assert len(survey.get_entry_points(OTHER_REPO)) == 1 + + +class TestDurability: + """The survey must never silently become an in-memory database. + + An in-memory SQLite URL hands out a fresh empty database per connection, so + a survey that fell back to one would map the repository and then present an + empty map to every stage that follows, with no error anywhere. + """ + + def test_writes_a_real_file_even_when_the_directory_is_absent(self, tmp_path): + state_dir = tmp_path / "never" / "created" + survey = RepoSurveyBackend(str(state_dir)) + assert (state_dir / "repo_survey.db").is_file() + assert str(survey.engine.url).startswith("sqlite:///") + + def test_state_survives_a_new_backend_over_the_same_directory(self, tmp_path): + state_dir = str(tmp_path / "state") + component_id = _component(RepoSurveyBackend(state_dir)) + reopened = RepoSurveyBackend(state_dir) + assert reopened.get_component(component_id)["location"] == "src/parser" + + +def test_the_toolbox_points_at_this_server() -> None: + """A renamed module would otherwise only surface as a dead server at run time.""" + toolbox = AvailableTools().get_toolbox("seclab_taskflows.toolboxes.audit_v2_repo_survey") + assert "seclab_taskflows.mcp_servers.audit_v2.repo_survey" in toolbox.server_params.args diff --git a/tests/test_taskflow_corpus.py b/tests/test_taskflow_corpus.py index 6524f07..2fd50c7 100644 --- a/tests/test_taskflow_corpus.py +++ b/tests/test_taskflow_corpus.py @@ -13,6 +13,7 @@ from __future__ import annotations import glob +import tempfile from pathlib import Path import pytest @@ -23,18 +24,35 @@ from seclab_taskflow_agent.models import DOCUMENT_MODELS from seclab_taskflow_agent.template_utils import evaluate_expression, render_template +from seclab_taskflows.mcp_servers.audit_v2.finding_ledger import FindingLedgerBackend +from seclab_taskflows.mcp_servers.audit_v2.repo_survey_models import ( + Component, + component_to_dict, +) + _ROOT = "src/seclab_taskflows" # A representative component and finding, shaped like the `outputs` contracts # the audit_v2 stages declare. Rendering against these proves the prompts only # reference fields the pipeline actually produces. +# +# The component is built from the survey's own projection rather than written +# out by hand, so a renamed column cannot leave this fixture agreeing with a +# prompt that no longer matches the server. _COMPONENT = { - "id": 1, - "repo": "acme/widget", - "location": "src/api", - "is_app": True, - "is_library": False, - "notes": "handles uploads", + **component_to_dict( + Component( + id=1, + repo="acme/widget", + location="src/api", + kind="service", + language="go", + runtime="", + is_app=True, + is_library=False, + notes="handles uploads", + ) + ) } _FINDING = { "finding_id": 1, @@ -195,3 +213,56 @@ def test_audit_v2_over_targets_are_produced_by_an_earlier_task() -> None: ) if task.id: produced.add(task.id) + + +def _declared_output_properties(tools: AvailableTools, dotted: str, task_id: str) -> set[str]: + taskflow = tools.get_taskflow(dotted) + for step in taskflow.taskflow: + if step.task.id == task_id: + schema = step.task.outputs or {} + return set((schema.get("items") or {}).get("properties", {})) + msg = f"{dotted} has no task with id {task_id!r}" + raise AssertionError(msg) + + +def test_component_outputs_match_what_the_survey_returns() -> None: + """The declared component schema must match `component_to_dict`, not resemble it. + + A field named `id` here instead of `component_id` still lints, still renders, + and still passes every static check, then fails at run time after the survey + has already paid for a full mapping pass. That is exactly what happened, so + the contract is asserted rather than assumed. + """ + actual = set(_COMPONENT) + tools = AvailableTools() + for dotted in ( + "seclab_taskflows.taskflows.audit_v2.survey", + "seclab_taskflows.taskflows.audit_v2.hunt", + ): + declared = _declared_output_properties(tools, dotted, "components") + assert declared <= actual, ( + f"{dotted} declares component fields that get_components never returns: " + f"{sorted(declared - actual)}" + ) + + +def test_finding_outputs_match_what_the_ledger_returns() -> None: + """Same guard for the finding schemas the contest and reproduce stages read.""" + with tempfile.TemporaryDirectory() as tmp_dir: + ledger = FindingLedgerBackend(tmp_dir) + finding_id = ledger.store_finding( + "acme/widget", "src/api", "t", "CWE-22", "python", "s", "k", "f", [], "h", "m" + ) + actual = set(ledger.get_finding(finding_id)) + + tools = AvailableTools() + for dotted, task_id in ( + ("seclab_taskflows.taskflows.audit_v2.contest", "candidates"), + ("seclab_taskflows.taskflows.audit_v2.reproduce", "confirmed"), + ("seclab_taskflows.taskflows.audit_v2.report", "findings"), + ): + declared = _declared_output_properties(tools, dotted, task_id) + assert declared <= actual, ( + f"{dotted} declares finding fields the ledger never returns: " + f"{sorted(declared - actual)}" + ) From 0bb9eee5d7747ffbaaf6852a2c86ae20b11c1dd7 Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Thu, 30 Jul 2026 16:09:11 -0400 Subject: [PATCH 04/12] Deduplicate the survey, let hunters file independently, fix adjudicator endpoint repo_survey now upserts components and entry points so repeated passes enrich one row. Hunters file without a pre-check and the dedup task unions provenance. adjudication uses api_type responses; the run script drops stale containers. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: dadec5f9-3bf8-449c-84d6-db45be19bb6a --- scripts/audit_v2/run_audit_v2.sh | 12 ++ .../configs/model_config_audit_v2.yaml | 11 ++ .../mcp_servers/audit_v2/repo_survey.py | 61 ++++++++- .../prompts/audit_v2/finding_contract.yaml | 23 +++- .../taskflows/audit_v2/hunt.yaml | 8 +- .../taskflows/audit_v2/survey.yaml | 13 ++ .../toolboxes/audit_v2_finding_ledger.yaml | 9 +- .../toolboxes/audit_v2_repo_survey.yaml | 10 +- tests/test_repo_survey.py | 117 ++++++++++++++++++ 9 files changed, 251 insertions(+), 13 deletions(-) diff --git a/scripts/audit_v2/run_audit_v2.sh b/scripts/audit_v2/run_audit_v2.sh index aa6cc57..544072a 100755 --- a/scripts/audit_v2/run_audit_v2.sh +++ b/scripts/audit_v2/run_audit_v2.sh @@ -129,6 +129,18 @@ for stage in "${STAGES[@]}"; do fi done +# Source access containers run with CONTAINER_PERSIST, and their name is a hash +# of image, workspace and network rather than of the workspace's contents. A +# container left over from a run whose workspace has since been recreated keeps +# a bind mount on the old directory, so /workspace comes up empty and every +# stage reads nothing. Drop them and let this run make its own. +stale=$(docker ps -aq --filter "name=^seclab-persist-" 2>/dev/null || true) +if [ -n "$stale" ]; then + echo "Removing stale persistent containers" + # shellcheck disable=SC2086 + docker rm -f $stale >/dev/null +fi + echo "audit v2: ${REPO}" echo "stages: ${STAGES[*]}" echo diff --git a/src/seclab_taskflows/configs/model_config_audit_v2.yaml b/src/seclab_taskflows/configs/model_config_audit_v2.yaml index d8d27a3..173bf81 100644 --- a/src/seclab_taskflows/configs/model_config_audit_v2.yaml +++ b/src/seclab_taskflows/configs/model_config_audit_v2.yaml @@ -15,6 +15,13 @@ # # Swap any entry for a different model if your entitlements differ; nothing in # the taskflows depends on a specific provider. +# +# One caveat when swapping: CAPI serves most models from exactly one endpoint, +# and asking for the wrong one fails the whole stage with "model X is not +# accessible via the /chat/completions endpoint". At the time of writing the +# gemini and claude models are chat completions only, while gpt-5.5, gpt-5.6 +# and grok are responses only, which is why the entries below set `api_type` +# where they do. Check a new model on both endpoints before relying on it. seclab-taskflow-agent: version: "1.0" @@ -51,6 +58,10 @@ model_settings: api_type: responses reasoning: effort: high + adjudication: + api_type: responses + reasoning: + effort: high reporting: api_type: responses reasoning: diff --git a/src/seclab_taskflows/mcp_servers/audit_v2/repo_survey.py b/src/seclab_taskflows/mcp_servers/audit_v2/repo_survey.py index 28e7196..8b15b35 100644 --- a/src/seclab_taskflows/mcp_servers/audit_v2/repo_survey.py +++ b/src/seclab_taskflows/mcp_servers/audit_v2/repo_survey.py @@ -81,6 +81,21 @@ def store_component( ): kind = _require(kind, COMPONENT_KINDS, "kind", default=KIND_OTHER) with Session(self.engine) as session: + # A component is the unit the hunt stage fans out over, so two + # components at one location would have the hunt read the same code + # twice and report the same paths twice. It would also leave the + # entry points in that file split arbitrarily between them. One + # location is therefore one component, and a second description of + # the same location enriches the first. + existing = ( + session.query(Component) + .filter(Component.repo == repo, Component.location == location) + .first() + ) + if existing is not None: + self._enrich_component(existing, kind, language, runtime, is_app, is_library, notes) + session.commit() + return existing.id component = Component( repo=repo, location=location, @@ -95,18 +110,62 @@ def store_component( session.commit() return component.id + @staticmethod + def _enrich_component(existing, kind, language, runtime, is_app, is_library, notes): + """Fold a second description of a location into the record already held.""" + for field, value in (("language", language), ("runtime", runtime)): + if value and not getattr(existing, field): + setattr(existing, field, value) + if kind != KIND_OTHER and existing.kind == KIND_OTHER: + existing.kind = kind + # Reachable as a program in any pass's reading means reachable, and the + # same for being callable as a library; plenty of components are both. + existing.is_app = bool(existing.is_app or is_app) + existing.is_library = bool(existing.is_library or is_library) + if notes and notes not in (existing.notes or ""): + existing.notes = f"{existing.notes}\n\n{notes}".strip() + def store_entry_point( self, repo, component_id, file, line, trust_boundary, untrusted_input, variables, notes ): trust_boundary = _require(trust_boundary, TRUST_BOUNDARIES, "trust_boundary") + line = int(line or 0) with Session(self.engine) as session: if session.get(Component, component_id) is None: return f"No component with id {component_id}" + # An entry point is a place in the code, so the same file, line and + # boundary is the same entry point no matter which pass found it. + # Several passes do find it: the mapping task and every fan-out + # branch read the same sources, and a branch will happily record an + # entry point that belongs to a sibling component. Recording those + # separately would make the hunt stage work each one repeatedly. + existing = ( + session.query(EntryPoint) + .filter( + EntryPoint.repo == repo, + EntryPoint.file == file, + EntryPoint.line == line, + EntryPoint.trust_boundary == trust_boundary, + ) + .first() + ) + if existing is not None: + # Fill in what an earlier pass left blank, but let its wording + # stand, so a later terser pass cannot erase a better note. + for field, value in ( + ("untrusted_input", untrusted_input), + ("variables", variables), + ("notes", notes), + ): + if value and not getattr(existing, field): + setattr(existing, field, value) + session.commit() + return existing.id entry_point = EntryPoint( repo=repo, component_id=component_id, file=file, - line=int(line or 0), + line=line, trust_boundary=trust_boundary, untrusted_input=untrusted_input or "", variables=variables or "", diff --git a/src/seclab_taskflows/prompts/audit_v2/finding_contract.yaml b/src/seclab_taskflows/prompts/audit_v2/finding_contract.yaml index a2385f7..9c0d1b1 100644 --- a/src/seclab_taskflows/prompts/audit_v2/finding_contract.yaml +++ b/src/seclab_taskflows/prompts/audit_v2/finding_contract.yaml @@ -35,12 +35,23 @@ prompt: | ## Filing findings - Before filing, call `find_similar_findings` for the same component and - vulnerability class. If your path is the same as an existing finding, do not - file a duplicate. - - Call `store_finding` once per distinct path. Set `proposed_by` to the label - you were given for this hunt, so the ledger records which model proposed it. + File every distinct path you find, and do not look at what other hunters + have filed before you file. Several hunters are working this repository at + the same time, deliberately from different model families. Two of them + arriving independently at the same path is the strongest signal this + pipeline produces, and it only exists if each of you files what you found + without being influenced by, or deferring to, the others. + + So do not call `find_similar_findings` to decide whether to file. If a path + is real, file it, even if you suspect someone already has. A later step + folds the repeats together and records every model that proposed each path, + so nothing is lost by filing and the corroboration is lost by staying quiet. + + Call `store_finding` once per distinct path. Two paths are distinct if they + have different sources or different sinks; the same source reaching the same + sink by a slightly different route is one finding. Set `proposed_by` to the + label you were given for this hunt, so the ledger records which model + proposed it. Findings enter the ledger as `candidate`. That is all they are at this point: you are proposing a hypothesis for adversarial review, not declaring a diff --git a/src/seclab_taskflows/taskflows/audit_v2/hunt.yaml b/src/seclab_taskflows/taskflows/audit_v2/hunt.yaml index 1dacfa0..60f3133 100644 --- a/src/seclab_taskflows/taskflows/audit_v2/hunt.yaml +++ b/src/seclab_taskflows/taskflows/audit_v2/hunt.yaml @@ -78,9 +78,11 @@ taskflow: Start from the entry points already mapped for this component; fetch them with `get_entry_points` for component_id {{ result.component_id }}. - Each one records the trust boundary it crosses. They are a starting - point, not a limit. If you find an entry point that was missed, hunt it - too. + Each one records the trust boundary it crosses. If that comes back + empty, or looks thinner than the code warrants, fetch the entry points + for the whole repository and work out which of them reach this + component. They are a starting point, not a limit. If you find an entry + point that was missed, hunt it too. Work outward from each entry point: diff --git a/src/seclab_taskflows/taskflows/audit_v2/survey.yaml b/src/seclab_taskflows/taskflows/audit_v2/survey.yaml index d6929a5..4686e93 100644 --- a/src/seclab_taskflows/taskflows/audit_v2/survey.yaml +++ b/src/seclab_taskflows/taskflows/audit_v2/survey.yaml @@ -83,6 +83,11 @@ taskflow: what they do. If the repository contains several applications, each one is its own component. + A component is identified by its `location`, and each location is one + component. If you would give two components the same directory or file, + they are one component: record it once and describe both roles in its + notes. If they are genuinely separate, give them separate locations. + For each component, store an entry with `store_component`. Set its `kind` to the closest of: application, service, library, parser, cli, daemon, plugin, other. Set `is_app` and `is_library` according to how @@ -95,6 +100,9 @@ taskflow: formats, it executes things, it makes authorisation decisions, it handles credentials, it manages memory by hand + Record components only. The entry points are mapped in the next step, + so do not call `store_entry_point` here. + Identify example, demo, fixture and test code. Do not create components for these, and say in your summary which directories you excluded. @@ -184,6 +192,11 @@ taskflow: on the other side of that boundary is untrusted. If an entry point spans several lines, use the first. + Record only the entry points of the component in + {{ result.location }}. Other components are being mapped alongside this + one, so an entry point that belongs to a different component is not + yours to record. + Do not assess vulnerabilities here. You are drawing the map, not hunting on it. diff --git a/src/seclab_taskflows/toolboxes/audit_v2_finding_ledger.yaml b/src/seclab_taskflows/toolboxes/audit_v2_finding_ledger.yaml index 2124157..c0acfe3 100644 --- a/src/seclab_taskflows/toolboxes/audit_v2_finding_ledger.yaml +++ b/src/seclab_taskflows/toolboxes/audit_v2_finding_ledger.yaml @@ -42,5 +42,10 @@ server_prompt: | evidence supports. Do not describe a finding as confirmed or reproduced in prose unless the ledger says it is. - Before storing a new finding, call `find_similar_findings` for the same - component and vulnerability class so you do not file duplicates. + Repeats are folded in afterwards, by a step whose only job is deduplication. + A hunter should therefore file every path it finds rather than checking + first whether someone else got there: several hunters from different model + families work each component at once, and two of them arriving at the same + path independently is the strongest corroboration this pipeline produces. + `find_similar_findings` is for that deduplication step and for later stages + looking for related work, not for deciding whether to file. diff --git a/src/seclab_taskflows/toolboxes/audit_v2_repo_survey.yaml b/src/seclab_taskflows/toolboxes/audit_v2_repo_survey.yaml index 61ba71a..43b8b0a 100644 --- a/src/seclab_taskflows/toolboxes/audit_v2_repo_survey.yaml +++ b/src/seclab_taskflows/toolboxes/audit_v2_repo_survey.yaml @@ -26,6 +26,12 @@ server_prompt: | `is_app` and `is_library` independently, because plenty of components are both. + A component is identified by its `location`, and one location is one + component. Storing a second component at a location already recorded merges + the two descriptions and returns the original id rather than creating a + duplicate, because the hunt fans out over components and would otherwise read + the same code twice. + An **entry point** is a place where data crosses into a component from somewhere less trusted. Always record which `trust_boundary` is crossed: @@ -44,4 +50,6 @@ server_prompt: | supplies is trusted, the file's contents may not be. Record entry points with a precise file and integer line number. If one spans - several lines, use the first. + several lines, use the first. An entry point is identified by its file, line + and trust boundary; recording one that is already known returns the existing + id and fills in whatever the earlier record left blank. diff --git a/tests/test_repo_survey.py b/tests/test_repo_survey.py index 3658acc..782ce14 100644 --- a/tests/test_repo_survey.py +++ b/tests/test_repo_survey.py @@ -82,6 +82,61 @@ def test_get_component_of_unknown_id_is_none(self, survey): assert survey.get_component(4242) is None +class TestComponentsAreOnePerLocation: + """The hunt fans out over components, so two at one location cost a rerun. + + Both the mapping task and the fan-out branches describe the same tree, and + a model asked to "be granular" happily emits several components for one + file. Left alone that reads the same code repeatedly and splits the file's + entry points arbitrarily between the copies. + """ + + def test_a_second_component_at_the_same_location_merges(self, survey): + first = survey.store_component(REPO, "main.go", "service", "go", "", True, False, "router") + second = survey.store_component(REPO, "main.go", "other", "", "", False, False, "ping") + assert second == first + assert len(survey.get_components(REPO)) == 1 + + def test_the_same_location_in_another_repo_is_a_different_component(self, survey): + first = _component(survey, location="main.go") + second = _component(survey, repo=OTHER_REPO, location="main.go") + assert second != first + + def test_merging_keeps_the_first_description_and_adds_the_second(self, survey): + component_id = survey.store_component(REPO, "main.go", "service", "", "", True, False, "a") + survey.store_component(REPO, "main.go", "other", "", "", False, False, "b") + notes = survey.get_component(component_id)["notes"] + assert "a" in notes + assert "b" in notes + + def test_merging_does_not_repeat_an_identical_note(self, survey): + component_id = survey.store_component(REPO, "main.go", "cli", "", "", True, False, "same") + survey.store_component(REPO, "main.go", "cli", "", "", True, False, "same") + assert survey.get_component(component_id)["notes"] == "same" + + def test_merging_fills_in_fields_the_first_pass_left_blank(self, survey): + component_id = survey.store_component(REPO, "main.go", "", "", "", False, False, "") + survey.store_component(REPO, "main.go", "parser", "go", "native", False, False, "") + component = survey.get_component(component_id) + assert component["kind"] == "parser" + assert component["language"] == "go" + assert component["runtime"] == "native" + + def test_merging_does_not_overwrite_what_the_first_pass_established(self, survey): + component_id = survey.store_component(REPO, "main.go", "parser", "go", "", False, False, "") + survey.store_component(REPO, "main.go", "cli", "rust", "", False, False, "") + component = survey.get_component(component_id) + assert component["kind"] == "parser" + assert component["language"] == "go" + + def test_reachability_is_the_union_of_both_readings(self, survey): + component_id = survey.store_component(REPO, "main.go", "cli", "", "", True, False, "") + survey.store_component(REPO, "main.go", "cli", "", "", False, True, "") + component = survey.get_component(component_id) + assert component["is_app"] is True + assert component["is_library"] is True + + class TestEntryPoints: def test_stores_an_entry_point_against_a_component(self, survey): component_id = _component(survey) @@ -136,6 +191,68 @@ def test_get_component_includes_its_entry_points(self, survey): assert [e["file"] for e in component["entry_points"]] == ["src/parser/frame.c"] +class TestEntryPointsAreOnePerSite: + """An entry point is a place in the code; every pass that finds it means the same one. + + The mapping task and all the fan-out branches read the same sources, and a + branch will record an entry point belonging to a sibling component. A live + survey of a 60-line Go file produced fourteen records for four sites. + """ + + def test_the_same_site_recorded_twice_is_one_entry_point(self, survey): + component_id = _component(survey) + first = survey.store_entry_point(REPO, component_id, "main.go", 25, "network", "q", "q", "") + second = survey.store_entry_point( + REPO, component_id, "main.go", 25, "network", "q", "q", "" + ) + assert second == first + assert len(survey.get_entry_points(REPO)) == 1 + + def test_a_sibling_component_claiming_the_same_site_does_not_duplicate_it(self, survey): + owner = _component(survey, location="src/a") + sibling = _component(survey, location="src/b") + first = survey.store_entry_point(REPO, owner, "main.go", 41, "network", "", "", "") + second = survey.store_entry_point(REPO, sibling, "main.go", 41, "network", "", "", "") + assert second == first + entry_points = survey.get_entry_points(REPO) + assert len(entry_points) == 1 + assert entry_points[0]["component_id"] == owner + + def test_a_different_boundary_at_the_same_line_is_a_different_entry_point(self, survey): + component_id = _component(survey) + first = survey.store_entry_point(REPO, component_id, "main.go", 54, "network", "", "", "") + second = survey.store_entry_point(REPO, component_id, "main.go", 54, "file", "", "", "") + assert second != first + assert len(survey.get_entry_points(REPO)) == 2 + + def test_the_same_line_in_another_repo_is_a_different_entry_point(self, survey): + here = _component(survey) + there = _component(survey, repo=OTHER_REPO) + first = survey.store_entry_point(REPO, here, "main.go", 25, "network", "", "", "") + second = survey.store_entry_point(OTHER_REPO, there, "main.go", 25, "network", "", "", "") + assert second != first + + def test_a_repeat_fills_in_blanks_without_overwriting(self, survey): + component_id = _component(survey) + entry_id = survey.store_entry_point( + REPO, component_id, "main.go", 25, "network", "q parameter", "", "" + ) + survey.store_entry_point( + REPO, component_id, "main.go", 25, "network", "something else", "q", "reaches Query" + ) + entry = survey.get_entry_points(REPO)[0] + assert entry["entry_point_id"] == entry_id + assert entry["untrusted_input"] == "q parameter" + assert entry["variables"] == "q" + assert entry["notes"] == "reaches Query" + + def test_the_summary_counts_deduplicated_sites(self, survey): + component_id = _component(survey) + for _ in range(3): + survey.store_entry_point(REPO, component_id, "main.go", 25, "network", "", "", "") + assert survey.get_survey_summary(REPO)["entry_points"] == 1 + + class TestSummaryAndClear: def test_summary_counts_by_trust_boundary(self, survey): component_id = _component(survey) From 1393bacbd0cece4d2ad50b31c51788942570655c Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Fri, 31 Jul 2026 15:10:57 -0400 Subject: [PATCH 05/12] Attribute findings from runner records, and unblock the hunters Add attribute_findings (a whole run in one call) and drop proposed_by from store_finding, so provenance comes from the runner's branch records. hunt.yaml captures its typed response for the outputs gate. Add engagement.yaml framing to the model-facing tasks, set backend/api_type per model slot, and remove grok-4.5. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: dadec5f9-3bf8-449c-84d6-db45be19bb6a --- .../configs/model_config_audit_v2.yaml | 86 +++++++++--- .../model_config_audit_v2_lowercost.yaml | 30 ++++- .../mcp_servers/audit_v2/finding_ledger.py | 100 +++++++++++++- .../mcp_servers/audit_v2/repo_survey.py | 5 +- .../mcp_servers/audit_v2/schema_init.py | 47 +++++++ .../personalities/vulnerability_hunter.yaml | 18 ++- .../prompts/audit_v2/engagement.yaml | 38 ++++++ .../prompts/audit_v2/finding_contract.yaml | 6 +- .../taskflows/audit_v2/README.md | 49 ++++--- .../taskflows/audit_v2/contest.yaml | 6 + .../taskflows/audit_v2/hunt.yaml | 60 ++++++++- .../taskflows/audit_v2/reproduce.yaml | 2 + .../taskflows/audit_v2/survey.yaml | 4 + .../toolboxes/audit_v2_finding_ledger.yaml | 6 +- tests/test_finding_ledger.py | 124 ++++++++++++++++++ tests/test_taskflow_corpus.py | 9 ++ 16 files changed, 533 insertions(+), 57 deletions(-) create mode 100644 src/seclab_taskflows/mcp_servers/audit_v2/schema_init.py create mode 100644 src/seclab_taskflows/prompts/audit_v2/engagement.yaml diff --git a/src/seclab_taskflows/configs/model_config_audit_v2.yaml b/src/seclab_taskflows/configs/model_config_audit_v2.yaml index 173bf81..62fd305 100644 --- a/src/seclab_taskflows/configs/model_config_audit_v2.yaml +++ b/src/seclab_taskflows/configs/model_config_audit_v2.yaml @@ -16,53 +16,101 @@ # Swap any entry for a different model if your entitlements differ; nothing in # the taskflows depends on a specific provider. # -# One caveat when swapping: CAPI serves most models from exactly one endpoint, -# and asking for the wrong one fails the whole stage with "model X is not -# accessible via the /chat/completions endpoint". At the time of writing the -# gemini and claude models are chat completions only, while gpt-5.5, gpt-5.6 -# and grok are responses only, which is why the entries below set `api_type` -# where they do. Check a new model on both endpoints before relying on it. +# Two things to check when swapping. +# +# Backend. Each family is driven through its own SDK: Claude over the native +# Anthropic Messages API (`anthropic_sdk`), everything else over the OpenAI +# surface (`openai_agents`). All of it still goes to CAPI; the backend only +# decides which wire protocol is spoken. `backend` is set explicitly on every +# slot rather than left to the default, so a slot's provider and its SDK +# cannot drift apart unnoticed. +# +# Endpoint. CAPI serves most models from exactly one endpoint, and asking for +# the wrong one fails the whole stage with "model X is not accessible via the +# /chat/completions endpoint". Probed at the time of writing: gemini-3.6-flash +# is chat completions only; gpt-5.6-sol is responses only; gpt-5.4 and +# gpt-5-mini serve both; the Claude models answer on the native +# `/v1/messages` surface, which is `api_type: messages`. `responses` is the +# newer API and is preferred wherever a model offers it. Check a new model on +# both endpoints before relying on it. +# +# Reasoning effort is set on every slot that supports it, including the +# Anthropic ones, where the backend turns it into adaptive thinking. seclab-taskflow-agent: version: "1.0" filetype: model_config models: - # Cheap, high-volume bookkeeping: fetching, clearing, summarising ledger state. - general_tasks: gpt-5-mini - # Attack-surface mapping and component inventory. - survey: gpt-5.4 - # Three independent hunters, one per model family. + # Cheap, high-volume bookkeeping: fetching, clearing, summarising ledger + # state and applying attribution. Not a reasoning slot, but it drives tool + # calls in a loop, so it is not the smallest model either. + general_tasks: gpt-5.4 + # Attack-surface mapping and component inventory. Everything downstream hunts + # only what this stage found, so it is not a place to economise. + survey: gpt-5.6-sol + # Three independent hunters, one per family, each the strongest code-analysis + # model that family currently serves. + # + # xAI is deliberately absent everywhere in this file. CAPI rejects grok-4.5 + # for security analysis at the platform level, not the model level: any + # request whose content is vulnerability analysis comes back + # `403 permission-denied ... Failed check: SAFETY_CHECK_TYPE_CYBER`, down to + # a five-line snippet. It is unusable for every role in this pipeline. hunt_gpt: gpt-5.6-sol - hunt_claude: claude-sonnet-5 + hunt_claude: claude-opus-5 hunt_gemini: gemini-3.6-flash - # Adversarial contest. Advocates are strong; the judge is from a third family. + # Adversarial contest. Advocates are strong; the judge is from the one family + # that writes neither argument, so it is never grading a sibling's case. prosecution: gpt-5.6-sol - defense: claude-sonnet-5 - adjudication: grok-4.5 - # Dynamic reproduction is long-horizon tool use inside a container. - reproduction: claude-sonnet-5 + defense: claude-opus-5 + adjudication: gemini-3.6-flash + # Dynamic reproduction is long-horizon tool use inside a container: the stage + # has to actually drive the target to a crash or a proof, not describe one. + reproduction: claude-opus-5 # Final write-up. - reporting: gpt-5.5 + reporting: gpt-5.6-sol model_settings: general_tasks: + backend: openai_agents api_type: responses survey: + backend: openai_agents api_type: responses reasoning: effort: medium hunt_gpt: + backend: openai_agents api_type: responses reasoning: effort: high + hunt_claude: + backend: anthropic_sdk + api_type: messages + reasoning: + effort: high + hunt_gemini: + backend: openai_agents + api_type: chat_completions prosecution: + backend: openai_agents api_type: responses reasoning: effort: high + defense: + backend: anthropic_sdk + api_type: messages + reasoning: + effort: high adjudication: - api_type: responses + backend: openai_agents + api_type: chat_completions + reproduction: + backend: anthropic_sdk + api_type: messages reasoning: effort: high reporting: + backend: openai_agents api_type: responses reasoning: effort: medium diff --git a/src/seclab_taskflows/configs/model_config_audit_v2_lowercost.yaml b/src/seclab_taskflows/configs/model_config_audit_v2_lowercost.yaml index 15c41ff..59f091b 100644 --- a/src/seclab_taskflows/configs/model_config_audit_v2_lowercost.yaml +++ b/src/seclab_taskflows/configs/model_config_audit_v2_lowercost.yaml @@ -24,16 +24,42 @@ models: reporting: gpt-5-mini model_settings: general_tasks: + backend: openai_agents api_type: responses survey: + backend: openai_agents api_type: responses hunt_gpt: + backend: openai_agents api_type: responses reasoning: - effort: medium + effort: high + hunt_claude: + backend: anthropic_sdk + api_type: messages + reasoning: + effort: high + hunt_gemini: + backend: openai_agents + api_type: chat_completions prosecution: + backend: openai_agents api_type: responses reasoning: - effort: medium + effort: high + defense: + backend: anthropic_sdk + api_type: messages + reasoning: + effort: high + adjudication: + backend: openai_agents + api_type: chat_completions + reproduction: + backend: anthropic_sdk + api_type: messages + reasoning: + effort: high reporting: + backend: openai_agents api_type: responses diff --git a/src/seclab_taskflows/mcp_servers/audit_v2/finding_ledger.py b/src/seclab_taskflows/mcp_servers/audit_v2/finding_ledger.py index bbc523e..a1d9400 100644 --- a/src/seclab_taskflows/mcp_servers/audit_v2/finding_ledger.py +++ b/src/seclab_taskflows/mcp_servers/audit_v2/finding_ledger.py @@ -47,6 +47,7 @@ ReproductionAttempt, ) from ..utils import process_repo +from .schema_init import create_all_tolerating_races logging.basicConfig( level=logging.DEBUG, @@ -143,9 +144,10 @@ def __init__(self, state_dir: str): self.state_dir = state_dir Path(self.state_dir).mkdir(parents=True, exist_ok=True) self.engine = create_engine(f"sqlite:///{self.state_dir}/finding_ledger.db", echo=False) - Base.metadata.create_all( + create_all_tolerating_races( + Base, self.engine, - tables=[ + [ Finding.__table__, ContestVerdict.__table__, ReproductionAttempt.__table__, @@ -187,6 +189,69 @@ def store_finding( session.commit() return finding.id + def attribute_finding(self, repo, finding_id, proposed_by): + """Union a model label onto a finding's ``proposed_by``. + + Hunters cannot name the model they are running as, so the labels they + self-report are unreliable and, worse, they collide: two hunters that + both guess "unknown" union down to a single label, and a path that two + families found independently then reads as one. The runner does know + which model produced each branch, so attribution is applied afterwards + from the branch records rather than trusted from inside the branch. + """ + with Session(self.engine) as session: + finding = session.get(Finding, finding_id) + if finding is None: + return f"No finding with id {finding_id}" + if finding.repo != repo: + return ( + f"Finding {finding_id} belongs to {finding.repo!r}, not {repo!r}; " + f"refusing to attribute across repositories" + ) + label = (proposed_by or "").strip() + if not label: + return "proposed_by must be a non-empty model label" + finding.proposed_by = _merge_labels(finding.proposed_by, label) + labels = finding.proposed_by + session.commit() + return f"Finding {finding_id} was proposed by: {labels}" + + def attribute_findings(self, repo, attributions): + """Apply a whole run's worth of attribution in one call. + + ``attributions`` is one entry per hunt branch, as + ``{"proposed_by":