From 26077d84f9b9fcab5a956cd65b56e608a286d204 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 9 Jul 2026 15:35:59 +0200 Subject: [PATCH] Remove Claude Code automations --- .github/workflows/agent-automation.yml | 166 --------------- .../workflows/braintrust-security-audit.yml | 155 -------------- .../workflows/issue-revalidation-audit.yml | 196 ------------------ .github/workflows/library-gap-audit.yml | 167 --------------- 4 files changed, 684 deletions(-) delete mode 100644 .github/workflows/agent-automation.yml delete mode 100644 .github/workflows/braintrust-security-audit.yml delete mode 100644 .github/workflows/issue-revalidation-audit.yml delete mode 100644 .github/workflows/library-gap-audit.yml diff --git a/.github/workflows/agent-automation.yml b/.github/workflows/agent-automation.yml deleted file mode 100644 index c109eba..0000000 --- a/.github/workflows/agent-automation.yml +++ /dev/null @@ -1,166 +0,0 @@ -name: Agent Automation - -on: - schedule: - - cron: "0 6 * * 3" - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: provider-gap-audit-scheduled-${{ github.ref }} - cancel-in-progress: false - -jobs: - audit-sdk-coverage: - name: Audit SDK Coverage (${{ matrix.repo }}) - runs-on: ubuntu-latest - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - include: - - owner: braintrustdata - repo: braintrust-sdk-javascript - max_issues: 3 - - owner: braintrustdata - repo: braintrust-sdk-python - max_issues: 3 - - owner: braintrustdata - repo: braintrust-sdk-go - max_issues: 3 - - owner: braintrustdata - repo: braintrust-sdk-ruby - max_issues: 3 - - owner: braintrustdata - repo: braintrust-sdk-rust - max_issues: 3 - - owner: braintrustdata - repo: braintrust-sdk-java - max_issues: 3 - - owner: braintrustdata - repo: braintrust-sdk-dotnet - max_issues: 3 - steps: - - name: Generate GitHub App token - id: app-token - uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 - with: - app-id: ${{ secrets.BRAINTRUST_BOT_APP_ID }} - private-key: ${{ secrets.BRAINTRUST_BOT_PRIVATE_KEY }} - owner: ${{ matrix.owner }} - repositories: | - ${{ matrix.repo }} - permission-contents: read - permission-issues: write - - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - repository: ${{ matrix.owner }}/${{ matrix.repo }} - token: ${{ steps.app-token.outputs.token }} - persist-credentials: false - - - name: Run Claude provider gap audit - uses: anthropics/claude-code-action@df37d2f0760a4b5683a6e617c9325bc1a36443f6 # v1.0.75 - with: - anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - github_token: ${{ steps.app-token.outputs.token }} - show_full_output: "true" - display_report: "true" - plugin_marketplaces: | - https://github.com/braintrustdata/braintrust-claude-plugin.git - plugins: | - trace-claude-code@braintrust-claude-plugin - settings: | - { - "env": { - "TARGET_REPO_OWNER": "${{ matrix.owner }}", - "TARGET_REPO_NAME": "${{ matrix.repo }}", - "TRACE_TO_BRAINTRUST": "true", - "BRAINTRUST_CC_PROJECT": "${{ vars.BRAINTRUST_CC_PROJECT }}", - "BRAINTRUST_API_KEY": "${{ secrets.BRAINTRUST_API_KEY }}" - } - } - prompt: | - # Goal - - Find important instrumentation gaps in this repository's generative-AI and embeddings integrations. - - A gap means the upstream SDK or framework clearly supports something that this repository does not yet instrument, or instruments with materially less detail. - - # Scope - - - Infer the relevant surfaces from the checked-out repository itself. - - Focus only on repo-owned generative or embeddings execution instrumentation surfaces with meaningful upstream API, docs, and release surfaces. - - In-scope examples include generations/completions/responses APIs, embeddings APIs, streaming generation flows, tool-calling during model execution, and agentic run/tracing surfaces. - - Explicitly exclude CRUD-style parity work (resource create/list/retrieve/update/delete APIs, admin/configuration endpoints, and similar non-generative management APIs). - - Ignore generic runtime or infrastructure integrations that do not have an upstream AI API surface to compare against. - - Do not open parity issues for deprecated or no-op surfaces unless you find real deprecation drift or docs drift worth reporting. - - # Process - - 1. Inspect local code, tests, docs, examples, and e2e scenarios to understand what is already instrumented. - 2. For each relevant surface, independently discover the current official upstream docs and recent official releases or changelogs. - 3. Compare current upstream generative and embeddings execution capabilities to current Braintrust instrumentation in this repo. - 4. Always check the latest Braintrust docs at https://www.braintrust.dev/docs before deciding how to describe a gap. - 5. Search existing GitHub issues for duplicates before creating anything. - - Only search in `TARGET_REPO_OWNER/TARGET_REPO_NAME`. - - For every GitHub MCP tool call (`mcp__github__search_issues`, `mcp__github__list_issues`, `mcp__github__get_issue`, `mcp__github__get_issue_comments`, `mcp__github__create_issue`), explicitly pass owner=`TARGET_REPO_OWNER` and repo=`TARGET_REPO_NAME`. `TARGET_REPO_OWNER` and `TARGET_REPO_NAME` are environment variables. - - Never query or create issues in the repository running this workflow. - 6. Only act on high-confidence, concrete gaps tied to missing generative or embeddings execution APIs, unsupported model execution call patterns, or missing execution instrumentation detail. - - # Examples - - ## Good - - - The upstream SDK now has a stable `responses.stream()` helper, but this repo does not instrument it at all, or instruments it without final result metadata that other similar APIs already capture here. - - This repo instruments a provider's basic text generation API, but not its newer tool-calling or agent tracing API even though that API is now official and documented. - - The upstream SDK has an official embeddings API surface (for example batch embeddings options), but this repo does not instrument those embeddings calls with the same depth as comparable model execution APIs. - - ## Bad - - - A vague suspicion that "something in streaming may be missing" without a concrete upstream API and a concrete repo gap. - - Opening a separate issue for every release note bullet when they all describe the same missing instrumentation area. - - Filing an issue for CRUD-only endpoint coverage (for example files/threads/assistants management APIs) when no generative or embeddings execution instrumentation gap is involved. - - # If You Find Actionable Non-Duplicate Gaps - - - Create at most ${{ matrix.max_issues }} issues in this run. - - Create one issue per distinct gap. - - Keep each issue concise, concrete, and source-backed. - - Prefix every created issue title with `[bot] `. - - If the repository supports issue types, choose whichever type fits best (`Bug`, `Feature`, or `Task`). If issue types are unavailable, create the issue without type. - - Include a hidden marker comment near the top of the issue body in this exact form: - - ```html - - ``` - - Each issue should clearly include: - - - what instrumentation is missing - - whether Braintrust docs suggest the capability is `supported`, `unclear`, or `not_found` - - exact upstream sources - - exact Braintrust docs source or sources - - exact local repo files you inspected - - # Duplicate Handling - - - Do not create an issue if an open issue already covers the same gap. - - Treat a matching hidden marker comment or a clearly equivalent open issue as a duplicate. - - If duplicate checking is inconclusive, do not create the issue. - - # Constraints - - - Discover source URLs yourself. Do not rely on a preset list. - - Prefer official docs and official release sources. - - Do not create comments. - - Do not update, close, or label existing issues. - - Do not create pull requests. - - If there are no high-confidence non-duplicate gaps, do nothing. - claude_args: | - --model claude-sonnet-4-6 - --max-turns 60 - --allowedTools "Read,Glob,Grep,LS,WebSearch,WebFetch,mcp__github__get_issue,mcp__github__get_issue_comments,mcp__github__search_issues,mcp__github__list_issues,mcp__github__create_issue" - --disallowedTools "Bash,Edit,MultiEdit,Write,Replace,NotebookEditCell,mcp__github__create_issue_comment,mcp__github__update_issue,mcp__github__create_pr,mcp__github__create_or_update_file,mcp__github__delete_file,mcp__github_file_ops__commit_files,mcp__github_file_ops__delete_files" diff --git a/.github/workflows/braintrust-security-audit.yml b/.github/workflows/braintrust-security-audit.yml deleted file mode 100644 index 79b1c97..0000000 --- a/.github/workflows/braintrust-security-audit.yml +++ /dev/null @@ -1,155 +0,0 @@ -name: Braintrust Security Audit - -on: - schedule: - - cron: "0 7 * * 2" - workflow_dispatch: - -permissions: {} - -concurrency: - group: braintrust-security-audit-scheduled-${{ github.ref }} - cancel-in-progress: false - -jobs: - audit-braintrust-security: - name: Audit braintrust security - runs-on: ubuntu-latest - timeout-minutes: 90 - steps: - - name: Generate source repository token - id: source-token - uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 - with: - app-id: ${{ secrets.BRAINTRUST_BOT_APP_ID }} - private-key: ${{ secrets.BRAINTRUST_BOT_PRIVATE_KEY }} - owner: braintrustdata - repositories: | - braintrust - permission-contents: read - - - name: Generate issue repository token - id: issue-token - uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 - with: - app-id: ${{ secrets.BRAINTRUST_BOT_APP_ID }} - private-key: ${{ secrets.BRAINTRUST_BOT_PRIVATE_KEY }} - owner: braintrustdata - repositories: | - braintrust-security-scan-issues - permission-issues: write - - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - repository: braintrustdata/braintrust - token: ${{ steps.source-token.outputs.token }} - persist-credentials: false - - - name: Run Claude security audit - uses: anthropics/claude-code-action@df37d2f0760a4b5683a6e617c9325bc1a36443f6 # v1.0.75 - with: - anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - github_token: ${{ steps.issue-token.outputs.token }} - show_full_output: "true" - display_report: "true" - settings: | - { - "env": { - "SOURCE_REPO_OWNER": "braintrustdata", - "SOURCE_REPO_NAME": "braintrust", - "ISSUE_REPO_OWNER": "braintrustdata", - "ISSUE_REPO_NAME": "braintrust-security-scan-issues" - } - } - prompt: | - # Goal - - Find high-confidence, actionable security vulnerabilities in the checked-out `SOURCE_REPO_OWNER/SOURCE_REPO_NAME` repository and create concise issue reports only in `ISSUE_REPO_OWNER/ISSUE_REPO_NAME`. - - # Scope - - - Inspect only the checked-out source repository for source-code, configuration, dependency, infrastructure-as-code, authentication, authorization, data exposure, injection, tenant isolation, cryptography, secret handling, webhook, API, and deployment security issues. - - Assume `SOURCE_REPO_OWNER/SOURCE_REPO_NAME` is private. Other repositories in the GitHub organization may not be private. - - Focus on vulnerabilities with a concrete affected code path, configuration, or dependency and a plausible exploit or abuse path. - - Prioritize reporting vulnerabilities with the highest realistic likelihood of discovery by an attacker and the highest impact if exploited. - - Pay particular attention to user-provided AI provider secrets and API keys. Trace concrete code paths where they are accepted, propagated, logged, stored, cached, or sent to providers, and prioritize findings where they are exposed in plaintext outside the minimum required execution path, persisted or cached in plaintext, or spread to components that do not need them. - - Treat tests, docs, examples, generated files, and scripts as supporting evidence, but do not report findings that only affect non-production examples unless they create a realistic production risk. - - Exclude speculative hardening ideas, broad best-practice suggestions, style issues, missing tests without a vulnerability, and purely theoretical issues without repo evidence. - - Do not report secret-scanning alerts or raw leaked secrets. If you encounter a secret-looking value in source, describe only the file/path and risk class without including the secret value. - - # Process - - 1. Inspect local code, configs, tests, docs, examples, CI, deployment, and infrastructure files to understand security-relevant behavior. - 2. Follow security-sensitive flows across trust boundaries before deciding a finding is real. - 3. When external behavior matters, prefer official docs, official release notes, or authoritative project sources. - 4. Search existing GitHub issues for duplicates before creating anything. - - Only search in `ISSUE_REPO_OWNER/ISSUE_REPO_NAME`. - - For every GitHub MCP tool call (`mcp__github__search_issues`, `mcp__github__list_issues`, `mcp__github__get_issue`, `mcp__github__get_issue_comments`, `mcp__github__create_issue`), explicitly pass owner=`ISSUE_REPO_OWNER` and repo=`ISSUE_REPO_NAME`. `ISSUE_REPO_OWNER` and `ISSUE_REPO_NAME` are environment variables. - - Never query, create, update, or comment on issues in `SOURCE_REPO_OWNER/SOURCE_REPO_NAME`. - - Never query, create, update, or comment on issues in the repository running this workflow. - 5. Only create issues for high-confidence, non-duplicate vulnerabilities that a maintainer could act on from the report. - - # Examples - - ## Good - - - An authorization check is missing on a server-side route and the inspected code shows a tenant or project boundary can be crossed. - - A user-controlled value reaches a shell, SQL, template, path, SSRF, or deserialization sink without adequate validation or escaping. - - A production deployment or infrastructure configuration grants materially broader access than the code path needs, with an exact affected file and impact. - - A dependency or framework behavior creates a vulnerability in this repository's concrete usage, with official or authoritative source evidence. - - ## Bad - - - "Consider adding rate limiting" without showing an abuse path tied to a concrete endpoint or security control. - - "This might be vulnerable" based only on a suspicious function name. - - A finding that depends on ignoring an existing validation, authorization, or environment guard. - - A raw secret value pasted into the issue body. - - Any issue created in `SOURCE_REPO_OWNER/SOURCE_REPO_NAME`. - - # If You Find Actionable Non-Duplicate Vulnerabilities - - - Create at most 3 issues in this run. - - Create one issue per distinct vulnerability. - - Keep each issue concise, concrete, and source-backed. - - Prefix every created issue title with `[bot] `. - - If the repository supports issue types, choose whichever type fits best (`Bug`, `Feature`, or `Task`). If issue types are unavailable, create the issue without type. - - Include a hidden marker comment near the top of the issue body in this exact form: - - ```html - - ``` - - Each issue should clearly include: - - - affected source repository: `SOURCE_REPO_OWNER/SOURCE_REPO_NAME` - - severity: `critical`, `high`, `medium`, or `low` - - affected source files and code paths inspected - - vulnerability summary - - concrete impact - - evidence and reasoning - - reproduction, exploit sketch, or abuse path when possible without harmful detail - - suggested fix direction - - exact external sources inspected, if any - - # Duplicate Handling - - - Do not create an issue if an open issue in `ISSUE_REPO_OWNER/ISSUE_REPO_NAME` already covers the same vulnerability. - - Treat a matching hidden marker comment or a clearly equivalent open issue as a duplicate. - - If duplicate checking is inconclusive, do not create the issue. - - # Constraints - - - Create issues only in `ISSUE_REPO_OWNER/ISSUE_REPO_NAME`. - - Do not create comments. - - Do not update, close, label, assign, or milestone existing issues. - - Do not create pull requests. - - Do not modify files. - - Do not run shell commands. - - Do not include raw secrets, private keys, session tokens, API keys, passwords, or exploit payloads that would materially increase risk. - - Prefer local repo evidence plus authoritative external sources over inference. - - If there are no high-confidence non-duplicate vulnerabilities, do nothing. - claude_args: | - --model claude-opus-4-6 - --max-turns 125 - --allowedTools "Read,Glob,Grep,LS,WebSearch,WebFetch,mcp__github__get_issue,mcp__github__get_issue_comments,mcp__github__search_issues,mcp__github__list_issues,mcp__github__create_issue" - --disallowedTools "Bash,Edit,MultiEdit,Write,Replace,NotebookEditCell,mcp__github__create_issue_comment,mcp__github__update_issue,mcp__github__create_pr,mcp__github__create_or_update_file,mcp__github__delete_file,mcp__github_file_ops__commit_files,mcp__github_file_ops__delete_files" diff --git a/.github/workflows/issue-revalidation-audit.yml b/.github/workflows/issue-revalidation-audit.yml deleted file mode 100644 index 32b8de1..0000000 --- a/.github/workflows/issue-revalidation-audit.yml +++ /dev/null @@ -1,196 +0,0 @@ -name: Issue Revalidation Audit - -on: - schedule: - - cron: "0 6 1 * *" - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: issue-revalidation-audit-scheduled-${{ github.ref }} - cancel-in-progress: false - -jobs: - revalidate-audit-issues: - name: Revalidate Audit Issues (${{ matrix.repo }}) - runs-on: ubuntu-latest - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - include: - - owner: braintrustdata - repo: braintrust-sdk-javascript - - owner: braintrustdata - repo: braintrust-sdk-python - - owner: braintrustdata - repo: braintrust-sdk-go - - owner: braintrustdata - repo: braintrust-sdk-ruby - - owner: braintrustdata - repo: braintrust-sdk-rust - - owner: braintrustdata - repo: braintrust-sdk-java - - owner: braintrustdata - repo: braintrust-sdk-dotnet - steps: - - name: Generate GitHub App token - id: app-token - uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 - with: - app-id: ${{ secrets.BRAINTRUST_BOT_APP_ID }} - private-key: ${{ secrets.BRAINTRUST_BOT_PRIVATE_KEY }} - owner: ${{ matrix.owner }} - repositories: | - ${{ matrix.repo }} - permission-contents: read - permission-issues: write - - - name: Check for open audit issues - id: audit-issues - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - run: | - count="$(gh api --paginate --slurp "repos/${{ matrix.owner }}/${{ matrix.repo }}/issues?state=open&per_page=100" \ - | jq 'flatten | map(select(.pull_request | not) | select((.body // "") | contains(" - ``` - - - For `resolved`, `obsolete`, and `duplicate`, comment only when you are also closing the issue. - - For `outdated_but_actionable`, comment without closing and explain exactly what changed and why the issue remains open. - - If an issue already has a revalidation comment with the same classification and there is no materially new evidence, do nothing. - - If an issue already has a revalidation comment and the classification has changed to `resolved`, `obsolete`, or `duplicate`, leave one final closing comment and then close the issue. - - If an issue already has a revalidation comment and the issue remains `outdated_but_actionable`, only leave another comment if the new comment adds materially new evidence, scope changes, or guidance. - - For `duplicate`, reference the surviving open issue number and close only when the overlap is clear. - - If duplicate detection is uncertain, do nothing. - - If resolution is uncertain, do nothing. - - If obsolescence is uncertain, do nothing. - - If you are unsure whether an issue is `obsolete` or `outdated_but_actionable`, prefer `outdated_but_actionable` or no action over closing. - - # Good Reasons To Close - - - `resolved`: the local repository now clearly implements or tests the exact capability the issue described. - - `obsolete`: the upstream surface was removed, deprecated into a non-actionable path, or the issue's requested gap no longer makes sense based on current official docs, and there is no materially similar actionable gap to preserve. - - `duplicate`: another open issue with the same underlying gap exists and is clearly the better canonical issue. - - # Good Reasons To Comment Without Closing - - - `outdated_but_actionable`: the issue still points at a real gap, but API names, docs links, release context, or scope details have changed enough that a reader would benefit from an update comment. - - `outdated_but_actionable`: the original issue overstates or slightly misstates the current gap, but the remaining actionable gap is still substantial enough to keep the issue open. - - # Bad Reasons To Close - - - The issue is old or inactive. - - The issue looks low priority. - - The evidence is mixed or incomplete. - - The gap narrowed but still materially exists. - - The wording is stale but the underlying issue is still real. - - There is only a vaguely similar open issue. - - # Constraints - - - For every GitHub MCP tool call (`mcp__github__list_issues`, `mcp__github__search_issues`, `mcp__github__get_issue`, `mcp__github__get_issue_comments`, `mcp__github__create_issue_comment`, `mcp__github__update_issue`), explicitly pass owner=`TARGET_REPO_OWNER` and repo=`TARGET_REPO_NAME`. - - Never query or modify the repository running this workflow unless it is also the target repository from the environment variables. - - Never create new issues. - - Never update issue titles or bodies. - - Never reopen issues. - - Never change labels, assignees, or milestones. - - When calling `mcp__github__update_issue`, only use it to close an issue. Do not pass title, body, labels, assignees, or milestone fields. - - Never touch human-created issues. - - Never touch issues without one of the audit markers. - - Never create repetitive or materially redundant comments. - - Prefer local repo evidence plus official docs/releases over inference. - - If there are no high-confidence actions, do nothing. - claude_args: | - --model claude-sonnet-4-6 - --max-turns 60 - --allowedTools "Read,Glob,Grep,LS,WebSearch,WebFetch,mcp__github__list_issues,mcp__github__search_issues,mcp__github__get_issue,mcp__github__get_issue_comments,mcp__github__create_issue_comment,mcp__github__update_issue" - --disallowedTools "Bash,Edit,MultiEdit,Write,Replace,NotebookEditCell,mcp__github__create_issue,mcp__github__create_pr,mcp__github__create_or_update_file,mcp__github__delete_file,mcp__github_file_ops__commit_files,mcp__github_file_ops__delete_files" diff --git a/.github/workflows/library-gap-audit.yml b/.github/workflows/library-gap-audit.yml deleted file mode 100644 index 961782a..0000000 --- a/.github/workflows/library-gap-audit.yml +++ /dev/null @@ -1,167 +0,0 @@ -name: Library Gap Audit - -on: - schedule: - - cron: "0 6 * * 1" - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: library-gap-audit-scheduled-${{ github.ref }} - cancel-in-progress: false - -jobs: - audit-library-coverage: - name: Audit Library Coverage (${{ matrix.repo }}) - runs-on: ubuntu-latest - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - include: - - owner: braintrustdata - repo: braintrust-sdk-javascript - max_issues: 3 - - owner: braintrustdata - repo: braintrust-sdk-python - max_issues: 3 - - owner: braintrustdata - repo: braintrust-sdk-go - max_issues: 3 - - owner: braintrustdata - repo: braintrust-sdk-ruby - max_issues: 3 - - owner: braintrustdata - repo: braintrust-sdk-rust - max_issues: 3 - - owner: braintrustdata - repo: braintrust-sdk-java - max_issues: 3 - - owner: braintrustdata - repo: braintrust-sdk-dotnet - max_issues: 3 - steps: - - name: Generate GitHub App token - id: app-token - uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 - with: - app-id: ${{ secrets.BRAINTRUST_BOT_APP_ID }} - private-key: ${{ secrets.BRAINTRUST_BOT_PRIVATE_KEY }} - owner: ${{ matrix.owner }} - repositories: | - ${{ matrix.repo }} - permission-contents: read - permission-issues: write - - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - repository: ${{ matrix.owner }}/${{ matrix.repo }} - token: ${{ steps.app-token.outputs.token }} - persist-credentials: false - - - name: Run Claude library gap audit - uses: anthropics/claude-code-action@df37d2f0760a4b5683a6e617c9325bc1a36443f6 # v1.0.75 - with: - anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - github_token: ${{ steps.app-token.outputs.token }} - show_full_output: "true" - display_report: "true" - plugin_marketplaces: | - https://github.com/braintrustdata/braintrust-claude-plugin.git - plugins: | - trace-claude-code@braintrust-claude-plugin - settings: | - { - "env": { - "TARGET_REPO_OWNER": "${{ matrix.owner }}", - "TARGET_REPO_NAME": "${{ matrix.repo }}", - "TRACE_TO_BRAINTRUST": "true", - "BRAINTRUST_CC_PROJECT": "${{ vars.BRAINTRUST_CC_PROJECT }}", - "BRAINTRUST_API_KEY": "${{ secrets.BRAINTRUST_API_KEY }}" - } - } - prompt: | - # Goal - - Find important libraries that this repository should instrument for generative-AI, embeddings, or agent execution, but does not yet instrument. - - A gap means an external library with a meaningful execution API surface and official documentation/release signals that is not currently instrumented in this repository. - - # Scope - - - Infer currently instrumented libraries from this repository's code, tests, docs, examples, and e2e scenarios. - - Focus only on execution-oriented generative-AI, embeddings, and agent-run libraries with meaningful upstream API, docs, and release surfaces. - - In-scope examples include model generation/completions/responses surfaces, embeddings execution surfaces, streaming generation flows, tool-calling during model execution, and agent-run/tracing surfaces. - - Explicitly exclude CRUD/admin/configuration/non-execution libraries and generic runtime infrastructure tooling. - - Exclude speculative, weak-signal, or low-confidence library candidates. - - It is not in scope to point out missing instrumentation for libraries that a braintrust SDK already has instrumentation for. Only point out libraries that are completely uninstrumented yet. - - Do not open issues for deprecated/no-op surfaces unless there is concrete deprecation or docs drift worth reporting. - - # Process - - 1. Inspect local code, tests, docs, examples, and e2e scenarios to determine what libraries are already instrumented. - 2. Independently discover candidate libraries with official upstream docs and recent official release/changelog signals. - 3. For each candidate, compare current upstream execution capabilities to current Braintrust instrumentation in this repo. - 4. Always check the latest Braintrust docs at https://www.braintrust.dev/docs before deciding how to describe a gap. - 5. Search existing GitHub issues for duplicates before creating anything. - - Only search in `TARGET_REPO_OWNER/TARGET_REPO_NAME`. - - For every GitHub MCP tool call (`mcp__github__search_issues`, `mcp__github__list_issues`, `mcp__github__get_issue`, `mcp__github__get_issue_comments`, `mcp__github__create_issue`), explicitly pass owner=`TARGET_REPO_OWNER` and repo=`TARGET_REPO_NAME`. `TARGET_REPO_OWNER` and `TARGET_REPO_NAME` are environment variables. - - Never query or create issues in the repository running this workflow. - 6. Only act on high-confidence, concrete, source-backed library instrumentation gaps. - - # Examples - - ## Good - - - A widely used official SDK for model execution in this language ecosystem has stable docs/releases and this repo does not instrument it at all. - - This repo instruments provider SDK A and B, but not provider SDK C with similar official generative/embeddings execution surfaces and active releases. - - A library with clear agent-run/tracing execution APIs is missing entirely from instrumentation coverage in this repo. - - ## Bad - - - Vague claims that "we may be missing some library support" without exact library, source URLs, and concrete repo evidence. - - Filing issues for non-execution libraries or configuration/management-only APIs. - - Opening multiple issues for the same library gap when one issue covers the missing instrumentation scope. - - # If You Find Actionable Non-Duplicate Gaps - - - Create at most ${{ matrix.max_issues }} issues in this run. - - Create one issue per distinct library gap. - - Keep each issue concise, concrete, and source-backed. - - Prefix every created issue title with `[bot] `. - - If the repository supports issue types, choose whichever type fits best (`Bug`, `Feature`, or `Task`). If issue types are unavailable, create the issue without type. - - Include a hidden marker comment near the top of the issue body in this exact form: - - ```html - - ``` - - Each issue should clearly include: - - - which library and what instrumentation is missing - - whether Braintrust docs suggest the capability is `supported`, `unclear`, or `not_found` - - exact upstream sources - - exact Braintrust docs source or sources - - exact local repo files you inspected - - # Duplicate Handling - - - Do not create an issue if an open issue already covers the same gap. - - Treat a matching hidden marker comment or a clearly equivalent open issue as a duplicate. - - If duplicate checking is inconclusive, do not create the issue. - - # Constraints - - - Discover source URLs yourself. Do not rely on a preset list. - - Prefer official docs and official release sources. - - Do not create comments. - - Do not update, close, or label existing issues. - - Do not create pull requests. - - If there are no high-confidence non-duplicate gaps, do nothing. - claude_args: | - --model claude-sonnet-4-6 - --max-turns 100 - --allowedTools "Read,Glob,Grep,LS,WebSearch,WebFetch,mcp__github__get_issue,mcp__github__get_issue_comments,mcp__github__search_issues,mcp__github__list_issues,mcp__github__create_issue" - --disallowedTools "Bash,Edit,MultiEdit,Write,Replace,NotebookEditCell,mcp__github__create_issue_comment,mcp__github__update_issue,mcp__github__create_pr,mcp__github__create_or_update_file,mcp__github__delete_file,mcp__github_file_ops__commit_files,mcp__github_file_ops__delete_files"