Add audit taskflows v2: contested, reproduction-gated vulnerability discovery

.github/workflows/publish-container-images.yml

@@ -66,6 +66,7 @@ jobs:
           ghcr.io/githubsecuritylab/seclab-shell-network-analysis
           ghcr.io/githubsecuritylab/seclab-shell-source-access
           ghcr.io/githubsecuritylab/seclab-shell-sast
+          ghcr.io/githubsecuritylab/seclab-shell-reproduction
         )
 
         for image in "${images[@]}"; do

scripts/audit_v2/run_audit_v2.sh

@@ -0,0 +1,168 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: GitHub, Inc.
+# SPDX-License-Identifier: MIT
+
+# Run the audit v2 pipeline against a repository.
+#
+# The five stages are separate taskflows rather than one file on purpose. Each
+# one is expensive, and each ends at a durable checkpoint in the finding
+# ledger, so a stage can be rerun on its own without redoing the ones before
+# it. That is also why stage state lives in the ledger rather than in taskflow
+# outputs: multi-model tasks do not feed a shared result channel, so the ledger
+# is the only place the stages can meet.
+#
+# Usage: ./scripts/audit_v2/run_audit_v2.sh [options] <owner/repo>
+#
+# Options:
+#   -m <model_config>   Override the model config each taskflow declares.
+#                       Use seclab_taskflows.configs.model_config_audit_v2_lowercost
+#                       for cheaper exploratory runs.
+#   -s <stage>          Run a single stage: survey|hunt|contest|reproduce|report.
+#                       Repeatable. Default: all five, in order.
+#   --from <stage>      Run from this stage to the end, after fixing a stage
+#                       that failed part way through.
+#   --no-reproduce      Skip reproduction. Findings then top out at `confirmed`
+#                       and the report says so.
+#   -h, --help          Show this message.
+
+set -euo pipefail
+
+ALL_STAGES=(survey hunt contest reproduce report)
+STAGES=()
+FROM_STAGE=""
+SKIP_REPRODUCE=false
+MODEL_CONFIG_FLAG=()
+
+usage() {
+    sed -n '6,27p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
+}
+
+while [[ $# -gt 0 && "$1" == -* ]]; do
+    case "$1" in
+        -m)
+            MODEL_CONFIG_FLAG=(-m "$2")
+            shift 2
+            ;;
+        -s)
+            STAGES+=("$2")
+            shift 2
+            ;;
+        --from)
+            FROM_STAGE="$2"
+            shift 2
+            ;;
+        --no-reproduce)
+            SKIP_REPRODUCE=true
+            shift
+            ;;
+        -h|--help)
+            usage
+            exit 0
+            ;;
+        *)
+            echo "Unknown option: $1" >&2
+            usage >&2
+            exit 1
+            ;;
+    esac
+done
+
+REPO="${1:-}"
+if [ -z "$REPO" ]; then
+    usage >&2
+    exit 1
+fi
+
+# Catch a mistyped target here rather than several minutes into a stage. The
+# repo is threaded through to every tool as a single owner/repo string, so it
+# has to be exactly that: one slash, and neither half empty or containing
+# whitespace.
+if [[ ! "$REPO" =~ ^[^/[:space:]]+/[^/[:space:]]+$ ]]; then
+    echo "Expected a target of the form <owner/repo>, got: ${REPO}" >&2
+    echo "For example: ${BASH_SOURCE[0]} octocat/hello-world" >&2
+    exit 1
+fi
+
+if [ -n "$FROM_STAGE" ] && [ ${#STAGES[@]} -gt 0 ]; then
+    echo "Use either --from or -s, not both." >&2
+    exit 1
+fi
+
+if [ -n "$FROM_STAGE" ]; then
+    seen=false
+    for stage in "${ALL_STAGES[@]}"; do
+        [ "$stage" = "$FROM_STAGE" ] && seen=true
+        [ "$seen" = true ] && STAGES+=("$stage")
+    done
+    if [ "$seen" != true ]; then
+        echo "Unknown stage: ${FROM_STAGE}" >&2
+        exit 1
+    fi
+fi
+
+if [ ${#STAGES[@]} -eq 0 ]; then
+    STAGES=("${ALL_STAGES[@]}")
+fi
+
+for stage in "${STAGES[@]}"; do
+    valid=false
+    for known in "${ALL_STAGES[@]}"; do
+        [ "$stage" = "$known" ] && valid=true
+    done
+    if [ "$valid" != true ]; then
+        echo "Unknown stage: ${stage}" >&2
+        exit 1
+    fi
+done
+
+if [ "$SKIP_REPRODUCE" = true ]; then
+    filtered=()
+    for stage in "${STAGES[@]}"; do
+        [ "$stage" = "reproduce" ] || filtered+=("$stage")
+    done
+    STAGES=(${filtered[@]+"${filtered[@]}"})
+fi
+
+if [ ${#STAGES[@]} -eq 0 ]; then
+    echo "No stages left to run." >&2
+    exit 1
+fi
+
+# Reproduction is the one stage that executes attacker-controlled input, so a
+# missing image is worth catching now rather than halfway through a finding.
+for stage in "${STAGES[@]}"; do
+    if [ "$stage" = "reproduce" ] &&
+        ! docker image inspect ghcr.io/githubsecuritylab/seclab-shell-reproduction:latest >/dev/null 2>&1; then
+        echo "The reproduction image is missing. Build it with:" >&2
+        echo "  ./scripts/build_container_images.sh reproduction" >&2
+        exit 1
+    fi
+done
+
+# Source access containers run with CONTAINER_PERSIST, and their name is a hash
+# of image, workspace and network rather than of the workspace's contents. A
+# container left over from a run whose workspace has since been recreated keeps
+# a bind mount on the old directory, so /workspace comes up empty and every
+# stage reads nothing. Drop them and let this run make its own.
+stale=$(docker ps -aq --filter "name=^seclab-persist-" 2>/dev/null || true)
+if [ -n "$stale" ]; then
+    echo "Removing stale persistent containers"
+    # shellcheck disable=SC2086
+    docker rm -f $stale >/dev/null
+fi
+
+echo "audit v2: ${REPO}"
+echo "stages:   ${STAGES[*]}"
+echo
+
+for stage in "${STAGES[@]}"; do
+    echo "=== ${stage} ==="
+    python -m seclab_taskflow_agent \
+        ${MODEL_CONFIG_FLAG[@]+"${MODEL_CONFIG_FLAG[@]}"} \
+        -t "seclab_taskflows.taskflows.audit_v2.${stage}" \
+        -g repo="${REPO}"
+    echo
+done
+
+echo "The findings are in the ledger. Re-read the report at any time with:"
+echo "  python -m seclab_taskflow_agent -t seclab_taskflows.taskflows.audit_v2.report -g repo=${REPO}"

scripts/build_container_images.sh

@@ -6,7 +6,7 @@
 # Must be run from the root of the seclab-taskflows repository.
 # Images must be rebuilt whenever a Dockerfile changes.
 #
-# Usage: ./scripts/build_container_images.sh [base|malware|network|source-access|sast|all]
+# Usage: ./scripts/build_container_images.sh [base|malware|network|source-access|sast|reproduction|all]
 #   default: all
 
 set -euo pipefail
@@ -41,6 +41,11 @@ build_sast() {
     docker build -t "${IMAGE_PREFIX}/seclab-shell-sast:latest" "${CONTAINERS_DIR}/sast/"
 }
 
+build_reproduction() {
+    echo "Building ${IMAGE_PREFIX}/seclab-shell-reproduction..."
+    docker build -t "${IMAGE_PREFIX}/seclab-shell-reproduction:latest" "${CONTAINERS_DIR}/reproduction/"
+}
+
 target="${1:-all}"
 
 case "$target" in
@@ -62,16 +67,21 @@ case "$target" in
         build_base
         build_sast
         ;;
+    reproduction)
+        build_base
+        build_reproduction
+        ;;
     all)
         build_base
         build_malware
         build_network
         build_source_access
         build_sast
+        build_reproduction
         ;;
     *)
         echo "Unknown target: $target" >&2
-        echo "Usage: $0 [base|malware|network|source-access|sast|all]" >&2
+        echo "Usage: $0 [base|malware|network|source-access|sast|reproduction|all]" >&2
         exit 1
         ;;
 esac

src/seclab_taskflows/configs/model_config_audit_v2.yaml

@@ -0,0 +1,119 @@
+# SPDX-FileCopyrightText: GitHub, Inc.
+# SPDX-License-Identifier: MIT
+
+# Model assignment for the audit v2 pipeline.
+#
+# The stage names below are chosen so that models can be swapped per stage
+# without editing any taskflow. Two properties are deliberate:
+#
+# 1. The three hunt slots come from three different model families. Models in
+#    the same family tend to miss the same things, so mixing families buys
+#    real coverage rather than three correlated opinions.
+#
+# 2. The adjudicator belongs to a different family than either advocate, so it
+#    is never grading an argument written by a sibling model.
+#
+# Swap any entry for a different model if your entitlements differ; nothing in
+# the taskflows depends on a specific provider.
+#
+# Two things to check when swapping.
+#
+# Backend. Each family is driven through its own SDK: Claude over the native
+# Anthropic Messages API (`anthropic_sdk`), everything else over the OpenAI
+# surface (`openai_agents`). All of it still goes to CAPI; the backend only
+# decides which wire protocol is spoken. `backend` is set explicitly on every
+# slot rather than left to the default, so a slot's provider and its SDK
+# cannot drift apart unnoticed.
+#
+# Endpoint. CAPI serves most models from exactly one endpoint, and asking for
+# the wrong one fails the whole stage with "model X is not accessible via the
+# /chat/completions endpoint". Probed at the time of writing: gemini-3.6-flash
+# is chat completions only; gpt-5.6-sol is responses only; gpt-5.4 and
+# gpt-5-mini serve both; the Claude models answer on the native
+# `/v1/messages` surface, which is `api_type: messages`. `responses` is the
+# newer API and is preferred wherever a model offers it. Check a new model on
+# both endpoints before relying on it.
+#
+# Reasoning effort is set on every slot that supports it, including the
+# Anthropic ones, where the backend turns it into adaptive thinking.
+
+seclab-taskflow-agent:
+  version: "1.0"
+  filetype: model_config
+models:
+  # Cheap, high-volume bookkeeping: fetching, clearing, summarising ledger
+  # state and applying attribution. Not a reasoning slot, but it drives tool
+  # calls in a loop, so it is not the smallest model either.
+  general_tasks: gpt-5.4
+  # Attack-surface mapping and component inventory. Everything downstream hunts
+  # only what this stage found, so it is not a place to economise.
+  survey: gpt-5.6-sol
+  # Three independent hunters, one per family, each the strongest code-analysis
+  # model that family currently serves.
+  #
+  # xAI is deliberately absent everywhere in this file. CAPI rejects grok-4.5
+  # for security analysis at the platform level, not the model level: any
+  # request whose content is vulnerability analysis comes back
+  # `403 permission-denied ... Failed check: SAFETY_CHECK_TYPE_CYBER`, down to
+  # a five-line snippet. It is unusable for every role in this pipeline.
+  hunt_gpt: gpt-5.6-sol
+  hunt_claude: claude-opus-5
+  hunt_gemini: gemini-3.6-flash
+  # Adversarial contest. Advocates are strong; the judge is from the one family
+  # that writes neither argument, so it is never grading a sibling's case.
+  prosecution: gpt-5.6-sol
+  defense: claude-opus-5
+  adjudication: gemini-3.6-flash
+  # Dynamic reachability validation is long-horizon tool use inside a
+  # container: the stage has to actually drive the target and observe the flow
+  # reach the sink, not describe it. claude-opus-5 soft-refuses this work under
+  # content filtering (it returns a single text turn and calls no tools), so
+  # this slot runs claude-opus-4.8, which does the same job without tripping it.
+  reproduction: claude-opus-4.8
+  # Final write-up.
+  reporting: gpt-5.6-sol
+model_settings:
+  general_tasks:
+    backend: openai_agents
+    api_type: responses
+  survey:
+    backend: openai_agents
+    api_type: responses
+    reasoning:
+      effort: medium
+  hunt_gpt:
+    backend: openai_agents
+    api_type: responses
+    reasoning:
+      effort: high
+  hunt_claude:
+    backend: anthropic_sdk
+    api_type: messages
+    reasoning:
+      effort: high
+  hunt_gemini:
+    backend: openai_agents
+    api_type: chat_completions
+  prosecution:
+    backend: openai_agents
+    api_type: responses
+    reasoning:
+      effort: high
+  defense:
+    backend: anthropic_sdk
+    api_type: messages
+    reasoning:
+      effort: high
+  adjudication:
+    backend: openai_agents
+    api_type: chat_completions
+  reproduction:
+    backend: anthropic_sdk
+    api_type: messages
+    reasoning:
+      effort: high
+  reporting:
+    backend: openai_agents
+    api_type: responses
+    reasoning:
+      effort: medium