From 694b0e669e544d956733cb7badd21f6075f43c49 Mon Sep 17 00:00:00 2001 From: "releaser-ai-plugin[bot]" <273148615+releaser-ai-plugin[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2026 12:06:26 +0000 Subject: [PATCH] chore: sync skills (agent-skills-v0.922.0, context-mill@v1.51.0) --- .claude-plugin/marketplace.json | 2 +- .claude-plugin/plugin.json | 2 +- .codex-plugin/plugin.json | 2 +- .cursor-plugin/plugin.json | 2 +- gemini-extension.json | 2 +- .../references/dedupe-and-memory.md | 2 +- .../references/report-contract.md | 14 +++- .../references/scout-patterns.md | 4 +- skills/creating-online-evaluations/SKILL.md | 57 ++++++++------- .../references/evaluation-payload.md | 25 +++++-- .../creating-replay-vision-scanners/SKILL.md | 2 +- skills/exploring-llm-evaluations/SKILL.md | 27 ++++--- .../SKILL.md | 3 +- skills/inbox-exploration/SKILL.md | 73 ++++++++++++++----- .../references/flutter.md | 2 +- .../references/web.md | 2 +- .../references/flutter.md | 2 +- .../references/web.md | 2 +- .../references/posthog-js.md | 4 +- .../references/anthropic.md | 6 ++ .../references/azure-openai.md | 6 ++ .../references/cerebras.md | 6 ++ .../references/cohere.md | 6 ++ .../references/deepseek.md | 6 ++ .../references/fireworks-ai.md | 6 ++ .../references/google.md | 6 ++ .../references/groq.md | 6 ++ .../references/helicone.md | 6 ++ .../references/hugging-face.md | 6 ++ .../references/mistral.md | 6 ++ .../references/ollama.md | 6 ++ .../references/openai.md | 6 ++ .../references/openrouter.md | 6 ++ .../references/perplexity.md | 6 ++ .../references/together-ai.md | 6 ++ .../references/xai.md | 6 ++ .../references/best-practices.md | 2 + .../references/architecture.md | 9 +++ .../instrument-metrics/references/basics.md | 2 +- .../references/start-here.md | 12 ++- skills/managing-experiment-lifecycle/SKILL.md | 11 +++ skills/querying-posthog-data/SKILL.md | 2 +- .../references/example-error-tracking.md | 4 +- .../references/example-logs.md | 2 +- .../references/example-session-replay.md | 8 +- .../references/example-sessions.md | 2 +- skills/resolving-ingestion-warnings/SKILL.md | 12 +-- skills/setting-up-data-catalog/SKILL.md | 27 +++++-- .../signals-scout-ai-observability/SKILL.md | 4 +- .../signals-scout-anomaly-detection/SKILL.md | 2 +- .../references/report-contract.md | 12 ++- .../references/watchlist-and-memory.md | 4 +- skills/signals-scout-apm/SKILL.md | 4 +- skills/signals-scout-conversations/SKILL.md | 2 +- .../SKILL.md | 2 +- .../signals-scout-customer-analytics/SKILL.md | 4 +- skills/signals-scout-data-pipelines/SKILL.md | 4 +- skills/signals-scout-data-warehouse/SKILL.md | 4 +- skills/signals-scout-error-tracking/SKILL.md | 4 +- skills/signals-scout-experiments/SKILL.md | 4 +- skills/signals-scout-feature-flags/SKILL.md | 2 +- .../references/conventions.md | 6 +- skills/signals-scout-health-checks/SKILL.md | 4 +- .../signals-scout-inbox-validation/SKILL.md | 4 +- skills/signals-scout-insight-alerts/SKILL.md | 4 +- skills/signals-scout-logs/SKILL.md | 4 +- skills/signals-scout-mcp-tool-calls/SKILL.md | 4 +- .../signals-scout-observability-gaps/SKILL.md | 2 +- .../signals-scout-product-analytics/SKILL.md | 4 +- skills/signals-scout-replay-vision/SKILL.md | 4 +- .../signals-scout-revenue-analytics/SKILL.md | 2 +- skills/signals-scout-session-replay/SKILL.md | 4 +- skills/signals-scout-skills-store/SKILL.md | 4 +- skills/signals-scout-surveys/SKILL.md | 4 +- skills/signals-scout-tasks/SKILL.md | 2 +- skills/signals-scout-web-analytics/SKILL.md | 4 +- skills/signals-scout-web-vitals/SKILL.md | 8 +- .../references/usage-type-routing.md | 57 ++++++++------- 78 files changed, 406 insertions(+), 191 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 2848573..7f0d66d 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -12,7 +12,7 @@ "displayName": "PostHog", "source": "./", "description": "Access PostHog analytics, feature flags, experiments, error tracking, and insights directly from your AI coding tool. Optionally capture Claude Code sessions to PostHog LLM Analytics.", - "version": "1.1.62", + "version": "1.1.63", "author": { "name": "PostHog", "email": "hey@posthog.com", diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 5325958..9fcd8e8 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "posthog", "description": "Access PostHog analytics, feature flags, experiments, error tracking, and insights directly from your AI coding tool. Optionally capture Claude Code sessions to PostHog LLM Analytics.", - "version": "1.1.62", + "version": "1.1.63", "author": { "name": "PostHog", "email": "hey@posthog.com", diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 6638b86..31d5083 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "posthog", - "version": "1.0.60", + "version": "1.0.61", "description": "Access PostHog analytics, feature flags, experiments, error tracking, and insights directly from Codex", "author": { "name": "PostHog", diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json index a7c8457..40137c8 100644 --- a/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "posthog", "displayName": "PostHog", - "version": "1.1.56", + "version": "1.1.57", "description": "Access PostHog analytics, feature flags, experiments, error tracking, and insights directly from Cursor", "author": { "name": "PostHog", diff --git a/gemini-extension.json b/gemini-extension.json index cd26fe2..bd00d13 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -1,6 +1,6 @@ { "name": "posthog", - "version": "1.0.58", + "version": "1.0.59", "description": "Access PostHog analytics, feature flags, experiments, error tracking, and insights directly from Gemini CLI", "mcpServers": { "posthog": { diff --git a/skills/authoring-scouts/references/dedupe-and-memory.md b/skills/authoring-scouts/references/dedupe-and-memory.md index ff2ae9f..7cf0595 100644 --- a/skills/authoring-scouts/references/dedupe-and-memory.md +++ b/skills/authoring-scouts/references/dedupe-and-memory.md @@ -10,7 +10,7 @@ Every scout classifies each candidate finding against prior runs, the inbox, and Bake this classifier into the scout's Decide section: 1. **Net new** — no prior run mentions the topic, no inbox report and no scratchpad entry covers it. → Author a report via `emit_report` if it clears the report bar (see [`report-contract.md`](report-contract.md)). -2. **Material update on an existing live report** — a live report already covers the topic (one this scout authored last run, or a pipeline report), but there's new evidence (a different corroborating source, a fresh deploy correlation, contradicting data, a meaningful escalation in scope). → **`edit_report` it** — `append_note` with the fresh evidence, or rewrite `title`/`summary` on a report the scout authored. +2. **Material update on an existing live report** — a live report already covers the topic (one this scout authored last run, or a pipeline report), but there's new evidence (a different corroborating source, a fresh deploy correlation, contradicting data, a meaningful escalation in scope). → **`edit_report` it** — use `append_evidence` for the new observation, `append_note` for a reading of it, or rewrite `title`/`summary` on a report the scout authored. Don't mint a near-duplicate. **Live reports only:** `edit_report` never changes a report's status, so if the prior report is suppressed or resolved and the issue is genuinely back, author a **fresh** report (citing the prior `report_id` in the summary) rather than editing a closed one nobody will see. 3. **Same fact already covered** — an existing report already captures the same evidence shape, nothing has changed. → Skip. diff --git a/skills/authoring-scouts/references/report-contract.md b/skills/authoring-scouts/references/report-contract.md index 5b945f5..6d60b59 100644 --- a/skills/authoring-scouts/references/report-contract.md +++ b/skills/authoring-scouts/references/report-contract.md @@ -231,15 +231,21 @@ The fleet's reviewer map should compound over time. ## `edit_report` — update an existing report -Rewrite `title`/`summary`, append a note, set `suggested_reviewers`, and/or replace `charts` / `suggested_prompts` on a report that already exists. -Pass `run_id` (the current run) and `report_id`, plus at least one of `title`, `summary`, `append_note`, `suggested_reviewers`, `charts`, `suggested_prompts`. -An edit that supplies content (`title`, `summary`, `charts`, `suggested_prompts`, `append_note`, or a reviewer `reason`) passes the same safety judge as `emit_report`; an unsafe edit is rejected whole and the report keeps what it had. +Rewrite `title`/`summary`, append evidence or a note, set `suggested_reviewers`, and/or replace `charts` / `suggested_prompts` on a report that already exists. +Pass `run_id` (the current run) and `report_id`, plus at least one of `title`, `summary`, `append_note`, `append_evidence`, `suggested_reviewers`, `charts`, `suggested_prompts`. +An edit that supplies content (`title`, `summary`, `charts`, `suggested_prompts`, `append_note`, `append_evidence`, or a reviewer `reason`) passes the same safety judge as `emit_report`; an unsafe edit is rejected whole and the report keeps what it had. `edit_report` can target **any** of the team's inbox reports — not just ones a scout authored. That makes it the right tool when a later run learns something about a report the pipeline (or another scout) created. Rules of good behavior: -- **Prefer `append_note` over rewriting** `title`/`summary` on a report you didn't author. +- Use **`append_evidence`** for a new observation that a reader can check. + It takes the same `{description, source_id}` items as `emit_report`, and each one lands in the report's evidence rail as a bound signal, so the report's `signal_count` and `total_weight` grow with it. +- Use **`append_note`** for commentary — a reading of the report that adds nothing to check, such as the owning team already knowing, or a deploy having fixed it. + Send both in one call when an observation needs a reading alongside it. +- **A recovery is a note, not evidence.** `signal_count` and `total_weight` only grow, and both feed the inbox ranking, so evidence that an issue is over would rank the report as stronger. +- **At the cap, the note is the channel that still lands.** Emit plus every append share the report's **50** evidence rows, and the grouping pipeline can raise the count too, so a long-lived report can fill up. An append past the cap is rejected and the report keeps what it had. +- Prefer these additive fields over rewriting `title`/`summary` on a report you didn't author. A note is additive and audit-friendly (it carries your scout as the author); a rewrite silently overwrites a human- or pipeline-authored headline. - **Don't fight an in-flight pipeline.** A report the summary/research workflow is mid-run on can have its fields overwritten under you. If a report is actively being worked, append a note rather than rewriting. diff --git a/skills/authoring-scouts/references/scout-patterns.md b/skills/authoring-scouts/references/scout-patterns.md index 003827a..4307179 100644 --- a/skills/authoring-scouts/references/scout-patterns.md +++ b/skills/authoring-scouts/references/scout-patterns.md @@ -390,8 +390,8 @@ So the trigger for this pattern is any of: **a judgment with more than one axis* - **Bound what you write for non-candidates.** "Record which axis failed" is right for items that are close, and ruinous as a blanket rule on a busy queue — one `remember` call per rejected item can spend the run before the real candidates get read. Persist a **state transition** (an item that changed axis since last run) or a capped set of near-misses, and roll the rest into one aggregate backlog entry. - **Close the loop on what you filed — and know what closing it can and cannot do.** A "ready to pick up" report is wrong the moment someone picks it up, and it costs a person duplicating work already underway. - Re-check each `report:` entry every run and `edit_report` once the item is assigned, PR-linked, or closed — but note that `edit_report` mutates `title`, `summary`, `append_note`, `suggested_reviewers`, `charts`, and `suggested_prompts` **only**. - It cannot change status or actionability, so an appended note does not retire the report. + Re-check each `report:` entry every run and `edit_report` once the item is assigned, PR-linked, or closed — but note that `edit_report` mutates `title`, `summary`, `append_note`, `append_evidence`, `suggested_reviewers`, `charts`, and `suggested_prompts` **only**. + It cannot change status or actionability, so an appended note or evidence row does not retire the report. Rewrite the **title and summary** so the stale framing is gone from the surface a human scans, and leave the status change to a person. - **Routing the outcome is part of the design.** On the report channel a queue scout can hand work straight to a draft PR: `actionability: immediately_actionable` + `repository` + a `priority` makes the report **eligible** to autostart one. Eligible is not automatic — the team's autostart toggle, its priority threshold, the org's self-driving quota, and resolving a runner identity each gate it independently, so a correctly-filed report can sit still for reasons that have nothing to do with the scout. diff --git a/skills/creating-online-evaluations/SKILL.md b/skills/creating-online-evaluations/SKILL.md index 3ed7862..51fbd86 100644 --- a/skills/creating-online-evaluations/SKILL.md +++ b/skills/creating-online-evaluations/SKILL.md @@ -2,15 +2,15 @@ name: creating-online-evaluations description: > Author continuously-running online evaluations in PostHog AI observability, grounded in real failure - modes you've identified. Use when the user wants evaluations that automatically score new generations - or whole traces going forward — "create an eval to catch X", "continuously check that responses do Y", + modes you've identified. Use when the user wants evaluations that automatically score new generations or + whole traces going forward — "create an eval to catch X", "continuously check that responses do Y", "turn these failures into evals". Covers letting the explored data decide how many evals to create, - proposing that set in plain language and asking the user which ones they want, choosing the target and - eval type (hog / llm_judge / sentiment), configuring a provider, model, and usable provider key for an - llm_judge eval, scoping which generations trigger it via conditions, creating disabled, verifying scope, - and enabling. Falls back to proposing a sentiment eval when no failure mode is worth catching. - Finding and ranking the failure modes worth evaluating is its own job — use exploring-ai-failures first. - To debug or manage evaluations that already exist, use exploring-llm-evaluations. + proposing that set for the user to pick, choosing the target and eval type (hog / llm_judge / + sentiment), configuring a provider and model for an llm_judge eval (a provider key gates enabling, not + creation), scoping which generations trigger it via conditions, creating disabled, verifying scope, and + enabling. Proposes a sentiment eval when no failure mode is worth catching. Finding and ranking the + failure modes worth evaluating is its own job — use exploring-ai-failures first. To debug or manage + evaluations that already exist, use exploring-llm-evaluations. --- # Creating online evaluations @@ -129,11 +129,11 @@ set into production, which is noise and (for a judge) cost the user didn't agree ### 2.1 — Choose the eval type -| Use… | When the criterion is… | -| ----------- | ------------------------------------------------------------------------------------------------------------------------------------- | -| `hog` | Structural / rule-based (JSON parses, length, regex, tool-call shape). Cheap, deterministic, **no provider key needed.** | -| `llm_judge` | Subjective / fuzzy (tone, factuality, on-topic). Costs an LLM call per run; needs a provider, model, and usable provider key. | -| `sentiment` | You want sentiment labels on user messages, not a pass/fail (unless very specifically asked for, usually not relevant to this skill). | +| Use… | When the criterion is… | +| ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `hog` | Structural / rule-based (JSON parses, length, regex, tool-call shape). Cheap, deterministic, **no provider key needed.** | +| `llm_judge` | Subjective / fuzzy (tone, factuality, on-topic). Costs an LLM call per run; needs a provider and model. A usable provider key is only needed to enable it, not to create a draft. | +| `sentiment` | You want sentiment labels on user messages, not a pass/fail (unless very specifically asked for, usually not relevant to this skill). | Reach for `hog` first, escalate to `llm_judge` if there is no deterministic way to check for what we want to check. @@ -210,12 +210,14 @@ when exact structure matters. ### 2.3 — Configure the LLM judge -An `llm_judge` evaluation requires a valid `provider` and `model`. It also needs a usable provider key -when it runs. `provider_key_id` controls whether the evaluation pins one specific key: +An `llm_judge` evaluation requires only a valid `provider` and `model` to create as a draft. A usable +provider key is needed to enable it, not to save it, so no `ok` key is a reason to keep the eval disabled, +not a reason to stop. `provider_key_id` controls whether the evaluation pins one specific key: - Set `provider_key_id` to the UUID of an `ok`-state key for the same provider to pin it. -- Set `provider_key_id` to `null` to use the team's active provider key. The active key must be in the - `ok` state and use the same provider as `model_configuration.provider`. +- Set `provider_key_id` to `null` to use the team's active provider key. This is always a valid saved + value. To enable the eval, the active key must be in the `ok` state and use the same provider as + `model_configuration.provider`. Hog and sentiment evaluations skip this step. @@ -229,10 +231,12 @@ Confirm the provider and model with `llma-evaluation-judge-models`. Call it with no arguments to see the whole catalog at once. Providers PostHog funds no models for come back empty unless you pass `key_id` for one of the team's keys; the response's `providers` list flags which ones those are. Prefer pinning the chosen key so a later team-wide active-key change does not change how the evaluation runs. -Leave `provider_key_id` as `null` only after `llma-evaluation-config-get` confirms the active key is usable and its provider matches. +Leave `provider_key_id` as `null` to run on the team's active key; before enabling, confirm with +`llma-evaluation-config-get` that the active key is usable and its provider matches. -If there is no usable key, you may still create a disabled draft for the user to review. Do not spot-run or -enable it. Ask the user to add or validate a key in the UI before continuing. +No usable key does not block creation. Create the disabled draft with a valid `provider` and `model` for +the user to review. Do not spot-run or enable it, and ask the user to add or validate a key in the UI so it +can be enabled later. ### 2.4 — Create it disabled @@ -261,8 +265,9 @@ posthog:llma-evaluation-create ``` For `llm_judge`, swap `evaluation_config` to `{ "prompt": "…" }` and add -`"model_configuration": { "provider": "openai", "model": "gpt-5-mini", "provider_key_id": "" }`. -Use `null` only when the active team key is `ok` and uses the same provider. Full field reference: +`"model_configuration": { "provider": "openai", "model": "gpt-5-mini", "provider_key_id": null }`. +Pin `provider_key_id` to the UUID of an `ok`-state key from `llma-provider-key-list` to run on one specific +key; `null` runs on the team's active key. Either value saves a draft. Full field reference: [references/evaluation-payload.md](references/evaluation-payload.md). ### 2.5 — Verify the scope before enabling @@ -283,7 +288,8 @@ For generation targets, `count()` is the run volume. For trace targets, count di `$ai_trace_id` values because matching generations from the same trace schedule only one run. If volume is high, set `rollout_percentage` below 100 to sample. Spot-check the evaluator with -`llma-evaluation-test-hog` (hog) or `llma-evaluation-run` against one generation (llm_judge). +`llma-evaluation-test-hog` (hog) or `llma-evaluation-run` against one generation (llm_judge; skip the +spot-run for a judge with no usable key, per 2.3). Both tools currently use generation samples; for a trace target they can check shared source or prompt behavior, but they do not reproduce the complete settled trace. Review the first live trace results before increasing rollout. @@ -354,8 +360,9 @@ creating so the user can review and toggle it in the UI. criterion genuinely can't be coded. - **Always create disabled, verify scope, then enable.** An eval firing on the wrong events is worse than none — noise, and (for llm_judge) cost. -- **Configure llm_judge credentials before running.** A judge needs a valid provider and model plus a usable - provider key. `provider_key_id` may be `null` only when the matching active team key can be used. +- **A judge draft needs only a provider and model.** A usable provider key is needed to enable it, not to + save it, so no `ok` key means keep it disabled — not stop. `provider_key_id` may be `null` to run on the + team's active key. - **`bytecode` is server-written** for hog evals — never pass it; send only `evaluation_config.source`. - For cluster-scoped evals, identify the cluster with `exploring-llm-clusters`, then translate its event filter into `conditions`. diff --git a/skills/creating-online-evaluations/references/evaluation-payload.md b/skills/creating-online-evaluations/references/evaluation-payload.md index 1580785..3793a96 100644 --- a/skills/creating-online-evaluations/references/evaluation-payload.md +++ b/skills/creating-online-evaluations/references/evaluation-payload.md @@ -12,7 +12,7 @@ schemas below are rendered from the backend Pydantic models at build time, so th | `evaluation_type` | yes | `"hog"`, `"llm_judge"`, or `"sentiment"`. | | `evaluation_config` | yes | Shape depends on `evaluation_type` (below). | | `output_type` | yes | `"boolean"` for `hog`/`llm_judge`; `"sentiment"` for `sentiment`. | -| `output_config` | no | `{ "allows_na": bool }` for boolean; `{}` for sentiment. | +| `output_config` | no | `{ "allows_na": bool, "true_is_failure": bool }` for boolean; `{}` for sentiment. | | `model_configuration` | llm_judge only | Provider + model; key ID optional. Rejected on `hog`/`sentiment`. | | `target` | no | `"generation"` (default), `"trace"`, or `"session"`. Sentiment supports only `"generation"`. | | `target_config` | trace/session only | Settle config discriminated on `strategy` (below); defaults to a 30-minute fixed window for `trace`, and to a 1-hour inactivity window for `session`. | @@ -133,6 +133,11 @@ Valid `(evaluation_type, output_type)` pairs: `(hog, boolean)`, `(llm_judge, boo "default": false, "title": "Allows Na", "type": "boolean" + }, + "true_is_failure": { + "default": false, + "title": "True Is Failure", + "type": "boolean" } }, "title": "BooleanOutputConfig", @@ -141,6 +146,8 @@ Valid `(evaluation_type, output_type)` pairs: `(hog, boolean)`, `(llm_judge, boo ``` `allows_na: true` lets the evaluator return N/A (skip) in addition to pass/fail. +`true_is_failure: true` treats a raw `true` result as a failure and a raw `false` result as a pass. +Leave it `false` for the default mapping. ### sentiment output @@ -152,11 +159,12 @@ Empty object: `{}`. | ----------------- | -------- | ------------------------------------------------------------------------------ | | `provider` | yes | One of `openai`, `anthropic`, `gemini`, `openrouter`, `fireworks`, `azure_openai`, `together_ai`. | | `model` | yes | Model id, e.g. `gpt-5-mini`. Validate against `llma-evaluation-judge-models`. | -| `provider_key_id` | no | UUID of an `ok`-state key for the same provider. `null` uses the matching active team key. | +| `provider_key_id` | no | UUID of an `ok`-state key for the same provider, or `null` to use the active team key. | -`provider` and `model` are required. Pin `provider_key_id` to run on one specific key. Leave it `null` only -when `llma-evaluation-config-get` shows an `ok`-state active key for the same provider. A disabled draft may -be saved without a usable key, but it cannot be tested or enabled until a key can be resolved. +`provider` and `model` are required. Pin `provider_key_id` to run on one specific key, or set it to `null` +to run on the team's active key — both save a valid draft. To enable an unpinned eval, the active key must +be `ok` and use the same provider (`llma-evaluation-config-get` shows it). A disabled draft may be saved +without a usable key, but it cannot be tested or enabled until a key can be resolved. ## `conditions` @@ -250,7 +258,7 @@ and `item.output` when exact structure matters. "evaluation_config": { "prompt": "Return true if the assistant's reply stays on the user's topic, false if it changes subject. Return N/A if the user did not ask a question." }, "output_type": "boolean", "output_config": { "allows_na": true }, - "model_configuration": { "provider": "openai", "model": "gpt-5-mini", "provider_key_id": "" }, + "model_configuration": { "provider": "openai", "model": "gpt-5-mini", "provider_key_id": null }, "target": "generation", "target_config": {}, "conditions": [{ "id": "default", "rollout_percentage": 100, "properties": [] }], @@ -258,5 +266,6 @@ and `item.output` when exact structure matters. } ``` -Set `provider_key_id` to `null` only when the team's active key is in the `ok` state and its provider is -`openai`. +`provider_key_id: null` runs on the team's active key and always saves a valid draft. Pin it to an +`ok`-state key uuid from `llma-provider-key-list` to run on one specific key. To enable an unpinned eval, +the team's active key must be `ok` and its provider `openai`. diff --git a/skills/creating-replay-vision-scanners/SKILL.md b/skills/creating-replay-vision-scanners/SKILL.md index 25f35ed..0d9ac89 100644 --- a/skills/creating-replay-vision-scanners/SKILL.md +++ b/skills/creating-replay-vision-scanners/SKILL.md @@ -49,7 +49,7 @@ Pick a `scanner_type` and write its `scanner_config`. Every type needs a `prompt | `monitor` | Open-ended observation against a prompt (e.g. "flag rage clicks") | `{"prompt": "..."}`; optional `"allow_inconclusive": true` (off by default, so the model must answer yes or no) | | `classifier` | Assigns tags from a fixed label set | `{"prompt": "...", "tags": ["tag-a", "tag-b"]}` — `tags` needs ≥1 entry; optional `"multi_label": false` (defaults to true), `"allow_freeform_tags": true` (off by default) | | `scorer` | Numeric score on a rubric | `{"prompt": "...", "scale": {"min": 1, "max": 5, "label": "frustration"}}` — `min` < `max`; `label` optional | -| `summarizer` | Free-text summary, plus facet embeddings for search | `{"prompt": "..."}`; optional `"length": "short" \| "medium" \| "long"` (default `"medium"`). Embeddings are always on | +| `summarizer` | Free-text summary | `{"prompt": "..."}`; optional `"length": "short" \| "medium" \| "long"` (default `"medium"`). Embeddings are always on | `scanner_type` is **locked after creation** — to change it you delete and recreate, so confirm the type is right up front, and get the `scanner_config` shape right (a wrong shape is a create error, not a silent diff --git a/skills/exploring-llm-evaluations/SKILL.md b/skills/exploring-llm-evaluations/SKILL.md index eaef6f0..4d342a2 100644 --- a/skills/exploring-llm-evaluations/SKILL.md +++ b/skills/exploring-llm-evaluations/SKILL.md @@ -60,21 +60,23 @@ All `llma-evaluation-*` tools are defined in `products/ai_observability/mcp/tool Every run of an evaluation emits an `$ai_evaluation` event. Key properties: -| Property | Meaning | -| ---------------------------- | --------------------------------------------------------------- | -| `$ai_evaluation_id` | UUID of the evaluation config | -| `$ai_evaluation_name` | Human-readable name | -| `$ai_target_event_id` | UUID of the `$ai_generation` event being scored | -| `$ai_trace_id` | Parent trace ID (for jumping to the trace UI) | -| `$ai_evaluation_result_type` | Result kind: `boolean` or `sentiment` | -| `$ai_evaluation_result` | For boolean evaluations: `true` = pass, `false` = fail | -| `$ai_evaluation_reasoning` | Free-text explanation (set by the LLM judge or Hog code) | -| `$ai_evaluation_applicable` | `false` when the evaluator decided the generation is N/A | -| `$ai_sentiment_label` | For sentiment evaluations: `positive`, `neutral`, or `negative` | -| `$ai_sentiment_score` | Confidence score for the winning sentiment label | +| Property | Meaning | +| ---------------------------- | -------------------------------------------------------------------------------- | +| `$ai_evaluation_id` | UUID of the evaluation config | +| `$ai_evaluation_name` | Human-readable name | +| `$ai_target_event_id` | UUID of the `$ai_generation` event being scored | +| `$ai_trace_id` | Parent trace ID (for jumping to the trace UI) | +| `$ai_evaluation_result_type` | Result kind: `boolean` or `sentiment` | +| `$ai_evaluation_result` | Raw boolean result. Use the evaluation's output config to map it to pass or fail | +| `$ai_evaluation_reasoning` | Free-text explanation (set by the LLM judge or Hog code) | +| `$ai_evaluation_applicable` | `false` when the evaluator decided the generation is N/A | +| `$ai_sentiment_label` | For sentiment evaluations: `positive`, `neutral`, or `negative` | +| `$ai_sentiment_score` | Confidence score for the winning sentiment label | When `$ai_evaluation_applicable = false`, the run counts as N/A regardless of `$ai_evaluation_result`. For evaluations that don't support N/A, this property may be `null` — treat null as "applicable". +For boolean evaluations, `output_config.true_is_failure: false` maps `true` to pass and `false` to fail. +Set it to `true` for detector-style evaluations where `true` means the evaluator found a problem. ## Workflow: investigate why an evaluation is failing @@ -270,6 +272,7 @@ LLM judges require organisation AI data processing approval. Hog evaluators do n | Add an LLM-judge evaluator | `llma-evaluation-create` with `evaluation_type: "llm_judge"`, `evaluation_config.prompt`, and a `model_configuration` | | Tweak the source or prompt | `llma-evaluation-update` (edits `evaluation_config.source` for Hog, `evaluation_config.prompt` for LLM judge) | | Toggle N/A handling | `llma-evaluation-update` with `output_config.allows_na` | +| Set failure polarity | `llma-evaluation-update` with `output_config.true_is_failure` | | Disable temporarily | `llma-evaluation-update` with `enabled: false` | | Remove | `llma-evaluation-delete` (soft-delete via PATCH `{deleted: true}`) | diff --git a/skills/exploring-replay-vision-observations/SKILL.md b/skills/exploring-replay-vision-observations/SKILL.md index 742ab88..036d980 100644 --- a/skills/exploring-replay-vision-observations/SKILL.md +++ b/skills/exploring-replay-vision-observations/SKILL.md @@ -20,8 +20,7 @@ and doing something useful with it. For creating or sizing scanners, use [[creat - `classifier` → one or more `tags` from the scanner's label set, plus `tags_freeform` when the scanner allows freeform tags, and the `reasoning`. - `scorer` → a numeric `score` on the scanner's `scale`, and the `reasoning`. - - `summarizer` → a `title` and free-text `summary`, plus the facets that get embedded for search - (`intent`, `outcome`, `friction_points`, `keywords`). + - `summarizer` → a `title` and free-text `summary`. - **Only `succeeded` observations carry a finding.** Triage the rest by `status`/`error_reason` (see below). - **Observations are LLM judgments, not ground truth.** One observation is one model's read of one session — corroborate before you act on it. diff --git a/skills/inbox-exploration/SKILL.md b/skills/inbox-exploration/SKILL.md index 2fbdd04..f728594 100644 --- a/skills/inbox-exploration/SKILL.md +++ b/skills/inbox-exploration/SKILL.md @@ -60,6 +60,7 @@ _underlying detail_ — pair them when the user wants to dig in. | ------------------------------------- | ------------------------------------------------------------------------------------------------------------- | | `inbox-reports-list` | Paginated list of reports with filters (status, search, etc.) | | `inbox-reports-retrieve` | Full detail for a single report | +| `inbox-reports-claim` | Claim or release a report, and optionally attach its implementation PR | | `inbox-report-artefacts-list` | A report's full work log — `signal_finding` evidence, status judgments, commits, task runs, notes (read-only) | | `inbox-report-artefacts-retrieve` | Full detail for a single artefact (read-only) | | `inbox-reports-set-state` | Resolve (`resolved`), dismiss (`suppressed`), or snooze (`potential`) a single report | @@ -71,12 +72,12 @@ _underlying detail_ — pair them when the user wants to dig in. The `inbox-reports-*-list` / `-retrieve`, `inbox-report-artefacts-list` / `-retrieve`, and `inbox-source-configs-*-list` / `-retrieve` tools are read-only. The exposed writes are `inbox-reports-set-state` (resolve / dismiss / snooze a single report), -`inbox-reports-bulk-set-state` (the same transition for 1–100 reports in one call) — see +`inbox-reports-bulk-set-state` (the same transition for 1–100 reports in one call), +`inbox-reports-claim` (claim / release / attach a PR), see _Workflow: resolve, dismiss, or snooze a report_ — and `inbox-source-configs-partial-update`, which flips a source's `enabled` flag on or off (e.g. `{enabled: false}` to stop a source feeding the inbox); -`-create` / `-update` exist too for standing a source up or replacing it wholesale. Other writes -(pause processing, set `implementation_pr_url`) are not exposed via MCP today — the PR link is -populated on the product surface when a PR is opened against a report. +`-create` / `-update` exist too for standing a source up or replacing it wholesale. Pausing +processing is not exposed via MCP. ## Terminology @@ -198,6 +199,10 @@ inbox-reports-list If `count: 0` comes back, jump to the empty/unconfigured workflow above before saying "your inbox is empty" — the right reply depends on whether sources are configured. +A claimed report keeps its pipeline status, so this queue also returns work someone else already +owns. Add `"unclaimed": true` when the user is asking what is free to pick up, and +`"assignee": "me"` to list what the caller already owns and may want to resume. + ### Step 2 — Summarize by source and actionability For each report, the response includes: @@ -211,6 +216,10 @@ For each report, the response includes: report (see "What 'suggested reviewer' means" above — it's based on GitHub commit authorship of the relevant code, mapped to PostHog users via linked GitHub identity) - `implementation_pr_url` — if a PR has been opened against this report +- `work_state` — `unclaimed`, `working`, `in_review`, or `done` +- `assignee` — who claimed the report (a user, an internal task, or an external agent), or `null` + when nobody has. Say so when you list a report someone else owns, so it doesn't read as free + work - `_posthogUrl` — clickable deep-link to the report; **always include this in your response** Group the results so the user can scan quickly. **Lead with reports where @@ -285,7 +294,7 @@ implement.** Reports from `signals_scout` (and any LLM-research source) are espe double-checking; their `summary` often reads as a confident root-cause with file and function names, but it can be stale or wrong. -### Step 1 — Retrieve and check it isn't already handled +### Step 1 — Retrieve, check existing work, and claim it ```json inbox-reports-retrieve @@ -297,9 +306,23 @@ Before doing any work, look at: - `already_addressed` — if `true`, the fix may already be in flight or merged; confirm with the user before duplicating it. - `implementation_pr_url` — if a PR is already linked, surface it instead of opening a second one. +- `work_state` and `assignee` — show whether someone else has already picked it up. A takeover is + allowed, but make it deliberate rather than overlooking active work. - `status` — only `ready` reports carry a finished judgment. A `candidate` / `pending_input` report hasn't been researched yet; don't implement off a half-formed summary. +Before starting research or implementation, claim the report: + +```json +posthog:inbox-reports-claim +{ "report_id": "" } +``` + +The claim is attributed to the current internal task or external MCP client. Claims do not expire, so +release the report whenever you walk away without landing a fix — including when you dismiss it instead. +Release clears ownership only; an attached pull request stays on the report, and release works from any +status, so a dismissed report can be released too. + ### Step 2 — Verify the diagnosis against the code (do not skip) Start by reading the report's work log — its evidence and the judgments behind it: @@ -317,7 +340,13 @@ way the report describes, and actually produces the described failure. As a deep the raw underlying signals via the `signals` skill (`metadata.report_id`) if you need the signal text behind the curated findings. If the diagnosis doesn't hold up, say so and stop — a wrong report is itself a useful finding (and a candidate for _dismiss_ below), not a license to write a -speculative fix. +speculative fix. Release the claim you took in Step 1 before you stop, so the report returns to the +unclaimed view instead of reading as active work nobody is doing: + +```json +posthog:inbox-reports-claim +{ "report_id": "", "release": true } +``` ### Step 3 — Scope the fix to the right layer @@ -329,15 +358,25 @@ speculative fix. area-specific skills), make the change minimal, and add a regression test that would have caught the reported failure. -### Step 4 — Open the PR and link it back +### Step 4 — Open the PR and attach it + +Open the PR following the repo's PR conventions, then attach it to the report: + +```json +posthog:inbox-reports-claim +{ + "report_id": "", + "pr_url": "https://github.com/example/repository/pull/123" +} +``` -Open the PR following the repo's PR conventions. There is no MCP tool to set -`implementation_pr_url` — that link is populated on the product surface when a PR is opened -against the report. So reference the report in the PR description (its `_posthogUrl`) and tell the -user which report the PR addresses, so the loop is traceable. +The same PR may be attached to multiple reports. Connected repositories receive immediate state +validation and webhook updates. PRs from unconnected repositories are accepted with unknown state, +so resolve those reports manually after the work lands. Also reference the report's `_posthogUrl` +in the PR description so the loop is traceable from either side. -**Don't resolve a report because you opened a PR.** When the fix ships as a PR, the merge is what -resolves the report — the tasks GitHub webhook does it automatically. Resolving by hand at PR-open +**Don't resolve a report because you opened a PR.** When the fix ships as a connected PR, the merge +resolves the report automatically. Resolving by hand at PR-open time asserts work that hasn't landed, and a reviewer looking at the inbox can't tell the difference. Manual resolve is for fixes a PR merge will never cover — a skill-body change, a config change, a `NO_REPO` report — see the workflow below. @@ -488,12 +527,12 @@ inbox-source-configs-partial-update - `suppressed` reports are excluded by default; pass `status: "suppressed"` explicitly if the user wants to see hidden items - The inbox writes exposed via MCP are `inbox-reports-set-state` (resolve / dismiss / snooze one - report), `inbox-reports-bulk-set-state` (the same for 1–100 reports), and + report), `inbox-reports-bulk-set-state` (the same for 1–100 reports), `inbox-reports-claim` + (claim / release / attach a PR), and `inbox-source-configs-partial-update` (toggle a source's `enabled` flag). To _act_ on a report - (implement a fix), verify the diagnosis against the code first, then open a PR — see + (implement a fix), claim it, verify the diagnosis against the code, then open and attach a PR — see _Workflow: act on an actionable report_. A PR-backed fix is resolved automatically when the PR - merges, so don't resolve it by hand at PR-open time; setting `implementation_pr_url` happens on - the product surface, not via MCP. Always also surface the `_posthogUrl` deep-link + merges, so don't resolve it by hand at PR-open time. Always also surface the `_posthogUrl` deep-link - **Never implement a report's fix straight from its `summary`.** Reports — especially `signals_scout` ones — are LLM diagnoses; confirm the cited files / functions / behavior in the actual code before writing a fix. A report that doesn't hold up is a dismissal candidate, not a diff --git a/skills/instrument-error-tracking/references/flutter.md b/skills/instrument-error-tracking/references/flutter.md index 388ea1c..28c550f 100644 --- a/skills/instrument-error-tracking/references/flutter.md +++ b/skills/instrument-error-tracking/references/flutter.md @@ -108,7 +108,7 @@ Copy page ...