From fd0a669e1309f7a39095694000fad45b87545452 Mon Sep 17 00:00:00 2001 From: "releaser-ai-plugin[bot]" <273148615+releaser-ai-plugin[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 12:07:31 +0000 Subject: [PATCH] chore: sync skills (agent-skills-v0.880.0, context-mill@v1.51.0) --- .claude-plugin/marketplace.json | 2 +- .claude-plugin/plugin.json | 2 +- .codex-plugin/plugin.json | 2 +- .cursor-plugin/plugin.json | 2 +- gemini-extension.json | 2 +- skills/creating-online-evaluations/SKILL.md | 57 ++++++++------- .../references/evaluation-payload.md | 16 ++-- skills/inbox-exploration/SKILL.md | 73 ++++++++++++++----- .../references/flutter.md | 2 +- .../references/web.md | 2 +- .../references/flutter.md | 2 +- .../references/web.md | 2 +- .../references/posthog-js.md | 4 +- .../references/anthropic.md | 6 ++ .../references/azure-openai.md | 6 ++ .../references/cerebras.md | 6 ++ .../references/cohere.md | 6 ++ .../references/deepseek.md | 6 ++ .../references/fireworks-ai.md | 6 ++ .../references/google.md | 6 ++ .../references/groq.md | 6 ++ .../references/helicone.md | 6 ++ .../references/hugging-face.md | 6 ++ .../references/mistral.md | 6 ++ .../references/ollama.md | 6 ++ .../references/openai.md | 6 ++ .../references/openrouter.md | 6 ++ .../references/perplexity.md | 6 ++ .../references/together-ai.md | 6 ++ .../references/xai.md | 6 ++ .../references/best-practices.md | 2 + .../references/architecture.md | 9 +++ .../instrument-metrics/references/basics.md | 2 +- .../references/start-here.md | 12 ++- skills/managing-experiment-lifecycle/SKILL.md | 11 +++ skills/querying-posthog-data/SKILL.md | 2 +- .../references/example-error-tracking.md | 4 +- .../references/example-logs.md | 2 +- .../references/example-session-replay.md | 8 +- .../references/example-sessions.md | 2 +- skills/resolving-ingestion-warnings/SKILL.md | 12 +-- skills/setting-up-data-catalog/SKILL.md | 27 +++++-- .../references/usage-type-routing.md | 57 ++++++++------- 43 files changed, 307 insertions(+), 113 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 2848573..7f0d66d 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -12,7 +12,7 @@ "displayName": "PostHog", "source": "./", "description": "Access PostHog analytics, feature flags, experiments, error tracking, and insights directly from your AI coding tool. Optionally capture Claude Code sessions to PostHog LLM Analytics.", - "version": "1.1.62", + "version": "1.1.63", "author": { "name": "PostHog", "email": "hey@posthog.com", diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 5325958..9fcd8e8 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "posthog", "description": "Access PostHog analytics, feature flags, experiments, error tracking, and insights directly from your AI coding tool. Optionally capture Claude Code sessions to PostHog LLM Analytics.", - "version": "1.1.62", + "version": "1.1.63", "author": { "name": "PostHog", "email": "hey@posthog.com", diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 6638b86..31d5083 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "posthog", - "version": "1.0.60", + "version": "1.0.61", "description": "Access PostHog analytics, feature flags, experiments, error tracking, and insights directly from Codex", "author": { "name": "PostHog", diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json index a7c8457..40137c8 100644 --- a/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "posthog", "displayName": "PostHog", - "version": "1.1.56", + "version": "1.1.57", "description": "Access PostHog analytics, feature flags, experiments, error tracking, and insights directly from Cursor", "author": { "name": "PostHog", diff --git a/gemini-extension.json b/gemini-extension.json index cd26fe2..bd00d13 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -1,6 +1,6 @@ { "name": "posthog", - "version": "1.0.58", + "version": "1.0.59", "description": "Access PostHog analytics, feature flags, experiments, error tracking, and insights directly from Gemini CLI", "mcpServers": { "posthog": { diff --git a/skills/creating-online-evaluations/SKILL.md b/skills/creating-online-evaluations/SKILL.md index 3ed7862..51fbd86 100644 --- a/skills/creating-online-evaluations/SKILL.md +++ b/skills/creating-online-evaluations/SKILL.md @@ -2,15 +2,15 @@ name: creating-online-evaluations description: > Author continuously-running online evaluations in PostHog AI observability, grounded in real failure - modes you've identified. Use when the user wants evaluations that automatically score new generations - or whole traces going forward — "create an eval to catch X", "continuously check that responses do Y", + modes you've identified. Use when the user wants evaluations that automatically score new generations or + whole traces going forward — "create an eval to catch X", "continuously check that responses do Y", "turn these failures into evals". Covers letting the explored data decide how many evals to create, - proposing that set in plain language and asking the user which ones they want, choosing the target and - eval type (hog / llm_judge / sentiment), configuring a provider, model, and usable provider key for an - llm_judge eval, scoping which generations trigger it via conditions, creating disabled, verifying scope, - and enabling. Falls back to proposing a sentiment eval when no failure mode is worth catching. - Finding and ranking the failure modes worth evaluating is its own job — use exploring-ai-failures first. - To debug or manage evaluations that already exist, use exploring-llm-evaluations. + proposing that set for the user to pick, choosing the target and eval type (hog / llm_judge / + sentiment), configuring a provider and model for an llm_judge eval (a provider key gates enabling, not + creation), scoping which generations trigger it via conditions, creating disabled, verifying scope, and + enabling. Proposes a sentiment eval when no failure mode is worth catching. Finding and ranking the + failure modes worth evaluating is its own job — use exploring-ai-failures first. To debug or manage + evaluations that already exist, use exploring-llm-evaluations. --- # Creating online evaluations @@ -129,11 +129,11 @@ set into production, which is noise and (for a judge) cost the user didn't agree ### 2.1 — Choose the eval type -| Use… | When the criterion is… | -| ----------- | ------------------------------------------------------------------------------------------------------------------------------------- | -| `hog` | Structural / rule-based (JSON parses, length, regex, tool-call shape). Cheap, deterministic, **no provider key needed.** | -| `llm_judge` | Subjective / fuzzy (tone, factuality, on-topic). Costs an LLM call per run; needs a provider, model, and usable provider key. | -| `sentiment` | You want sentiment labels on user messages, not a pass/fail (unless very specifically asked for, usually not relevant to this skill). | +| Use… | When the criterion is… | +| ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `hog` | Structural / rule-based (JSON parses, length, regex, tool-call shape). Cheap, deterministic, **no provider key needed.** | +| `llm_judge` | Subjective / fuzzy (tone, factuality, on-topic). Costs an LLM call per run; needs a provider and model. A usable provider key is only needed to enable it, not to create a draft. | +| `sentiment` | You want sentiment labels on user messages, not a pass/fail (unless very specifically asked for, usually not relevant to this skill). | Reach for `hog` first, escalate to `llm_judge` if there is no deterministic way to check for what we want to check. @@ -210,12 +210,14 @@ when exact structure matters. ### 2.3 — Configure the LLM judge -An `llm_judge` evaluation requires a valid `provider` and `model`. It also needs a usable provider key -when it runs. `provider_key_id` controls whether the evaluation pins one specific key: +An `llm_judge` evaluation requires only a valid `provider` and `model` to create as a draft. A usable +provider key is needed to enable it, not to save it, so no `ok` key is a reason to keep the eval disabled, +not a reason to stop. `provider_key_id` controls whether the evaluation pins one specific key: - Set `provider_key_id` to the UUID of an `ok`-state key for the same provider to pin it. -- Set `provider_key_id` to `null` to use the team's active provider key. The active key must be in the - `ok` state and use the same provider as `model_configuration.provider`. +- Set `provider_key_id` to `null` to use the team's active provider key. This is always a valid saved + value. To enable the eval, the active key must be in the `ok` state and use the same provider as + `model_configuration.provider`. Hog and sentiment evaluations skip this step. @@ -229,10 +231,12 @@ Confirm the provider and model with `llma-evaluation-judge-models`. Call it with no arguments to see the whole catalog at once. Providers PostHog funds no models for come back empty unless you pass `key_id` for one of the team's keys; the response's `providers` list flags which ones those are. Prefer pinning the chosen key so a later team-wide active-key change does not change how the evaluation runs. -Leave `provider_key_id` as `null` only after `llma-evaluation-config-get` confirms the active key is usable and its provider matches. +Leave `provider_key_id` as `null` to run on the team's active key; before enabling, confirm with +`llma-evaluation-config-get` that the active key is usable and its provider matches. -If there is no usable key, you may still create a disabled draft for the user to review. Do not spot-run or -enable it. Ask the user to add or validate a key in the UI before continuing. +No usable key does not block creation. Create the disabled draft with a valid `provider` and `model` for +the user to review. Do not spot-run or enable it, and ask the user to add or validate a key in the UI so it +can be enabled later. ### 2.4 — Create it disabled @@ -261,8 +265,9 @@ posthog:llma-evaluation-create ``` For `llm_judge`, swap `evaluation_config` to `{ "prompt": "…" }` and add -`"model_configuration": { "provider": "openai", "model": "gpt-5-mini", "provider_key_id": "" }`. -Use `null` only when the active team key is `ok` and uses the same provider. Full field reference: +`"model_configuration": { "provider": "openai", "model": "gpt-5-mini", "provider_key_id": null }`. +Pin `provider_key_id` to the UUID of an `ok`-state key from `llma-provider-key-list` to run on one specific +key; `null` runs on the team's active key. Either value saves a draft. Full field reference: [references/evaluation-payload.md](references/evaluation-payload.md). ### 2.5 — Verify the scope before enabling @@ -283,7 +288,8 @@ For generation targets, `count()` is the run volume. For trace targets, count di `$ai_trace_id` values because matching generations from the same trace schedule only one run. If volume is high, set `rollout_percentage` below 100 to sample. Spot-check the evaluator with -`llma-evaluation-test-hog` (hog) or `llma-evaluation-run` against one generation (llm_judge). +`llma-evaluation-test-hog` (hog) or `llma-evaluation-run` against one generation (llm_judge; skip the +spot-run for a judge with no usable key, per 2.3). Both tools currently use generation samples; for a trace target they can check shared source or prompt behavior, but they do not reproduce the complete settled trace. Review the first live trace results before increasing rollout. @@ -354,8 +360,9 @@ creating so the user can review and toggle it in the UI. criterion genuinely can't be coded. - **Always create disabled, verify scope, then enable.** An eval firing on the wrong events is worse than none — noise, and (for llm_judge) cost. -- **Configure llm_judge credentials before running.** A judge needs a valid provider and model plus a usable - provider key. `provider_key_id` may be `null` only when the matching active team key can be used. +- **A judge draft needs only a provider and model.** A usable provider key is needed to enable it, not to + save it, so no `ok` key means keep it disabled — not stop. `provider_key_id` may be `null` to run on the + team's active key. - **`bytecode` is server-written** for hog evals — never pass it; send only `evaluation_config.source`. - For cluster-scoped evals, identify the cluster with `exploring-llm-clusters`, then translate its event filter into `conditions`. diff --git a/skills/creating-online-evaluations/references/evaluation-payload.md b/skills/creating-online-evaluations/references/evaluation-payload.md index 1580785..1bee939 100644 --- a/skills/creating-online-evaluations/references/evaluation-payload.md +++ b/skills/creating-online-evaluations/references/evaluation-payload.md @@ -152,11 +152,12 @@ Empty object: `{}`. | ----------------- | -------- | ------------------------------------------------------------------------------ | | `provider` | yes | One of `openai`, `anthropic`, `gemini`, `openrouter`, `fireworks`, `azure_openai`, `together_ai`. | | `model` | yes | Model id, e.g. `gpt-5-mini`. Validate against `llma-evaluation-judge-models`. | -| `provider_key_id` | no | UUID of an `ok`-state key for the same provider. `null` uses the matching active team key. | +| `provider_key_id` | no | UUID of an `ok`-state key for the same provider, or `null` to use the active team key. | -`provider` and `model` are required. Pin `provider_key_id` to run on one specific key. Leave it `null` only -when `llma-evaluation-config-get` shows an `ok`-state active key for the same provider. A disabled draft may -be saved without a usable key, but it cannot be tested or enabled until a key can be resolved. +`provider` and `model` are required. Pin `provider_key_id` to run on one specific key, or set it to `null` +to run on the team's active key — both save a valid draft. To enable an unpinned eval, the active key must +be `ok` and use the same provider (`llma-evaluation-config-get` shows it). A disabled draft may be saved +without a usable key, but it cannot be tested or enabled until a key can be resolved. ## `conditions` @@ -250,7 +251,7 @@ and `item.output` when exact structure matters. "evaluation_config": { "prompt": "Return true if the assistant's reply stays on the user's topic, false if it changes subject. Return N/A if the user did not ask a question." }, "output_type": "boolean", "output_config": { "allows_na": true }, - "model_configuration": { "provider": "openai", "model": "gpt-5-mini", "provider_key_id": "" }, + "model_configuration": { "provider": "openai", "model": "gpt-5-mini", "provider_key_id": null }, "target": "generation", "target_config": {}, "conditions": [{ "id": "default", "rollout_percentage": 100, "properties": [] }], @@ -258,5 +259,6 @@ and `item.output` when exact structure matters. } ``` -Set `provider_key_id` to `null` only when the team's active key is in the `ok` state and its provider is -`openai`. +`provider_key_id: null` runs on the team's active key and always saves a valid draft. Pin it to an +`ok`-state key uuid from `llma-provider-key-list` to run on one specific key. To enable an unpinned eval, +the team's active key must be `ok` and its provider `openai`. diff --git a/skills/inbox-exploration/SKILL.md b/skills/inbox-exploration/SKILL.md index 2fbdd04..f728594 100644 --- a/skills/inbox-exploration/SKILL.md +++ b/skills/inbox-exploration/SKILL.md @@ -60,6 +60,7 @@ _underlying detail_ — pair them when the user wants to dig in. | ------------------------------------- | ------------------------------------------------------------------------------------------------------------- | | `inbox-reports-list` | Paginated list of reports with filters (status, search, etc.) | | `inbox-reports-retrieve` | Full detail for a single report | +| `inbox-reports-claim` | Claim or release a report, and optionally attach its implementation PR | | `inbox-report-artefacts-list` | A report's full work log — `signal_finding` evidence, status judgments, commits, task runs, notes (read-only) | | `inbox-report-artefacts-retrieve` | Full detail for a single artefact (read-only) | | `inbox-reports-set-state` | Resolve (`resolved`), dismiss (`suppressed`), or snooze (`potential`) a single report | @@ -71,12 +72,12 @@ _underlying detail_ — pair them when the user wants to dig in. The `inbox-reports-*-list` / `-retrieve`, `inbox-report-artefacts-list` / `-retrieve`, and `inbox-source-configs-*-list` / `-retrieve` tools are read-only. The exposed writes are `inbox-reports-set-state` (resolve / dismiss / snooze a single report), -`inbox-reports-bulk-set-state` (the same transition for 1–100 reports in one call) — see +`inbox-reports-bulk-set-state` (the same transition for 1–100 reports in one call), +`inbox-reports-claim` (claim / release / attach a PR), see _Workflow: resolve, dismiss, or snooze a report_ — and `inbox-source-configs-partial-update`, which flips a source's `enabled` flag on or off (e.g. `{enabled: false}` to stop a source feeding the inbox); -`-create` / `-update` exist too for standing a source up or replacing it wholesale. Other writes -(pause processing, set `implementation_pr_url`) are not exposed via MCP today — the PR link is -populated on the product surface when a PR is opened against a report. +`-create` / `-update` exist too for standing a source up or replacing it wholesale. Pausing +processing is not exposed via MCP. ## Terminology @@ -198,6 +199,10 @@ inbox-reports-list If `count: 0` comes back, jump to the empty/unconfigured workflow above before saying "your inbox is empty" — the right reply depends on whether sources are configured. +A claimed report keeps its pipeline status, so this queue also returns work someone else already +owns. Add `"unclaimed": true` when the user is asking what is free to pick up, and +`"assignee": "me"` to list what the caller already owns and may want to resume. + ### Step 2 — Summarize by source and actionability For each report, the response includes: @@ -211,6 +216,10 @@ For each report, the response includes: report (see "What 'suggested reviewer' means" above — it's based on GitHub commit authorship of the relevant code, mapped to PostHog users via linked GitHub identity) - `implementation_pr_url` — if a PR has been opened against this report +- `work_state` — `unclaimed`, `working`, `in_review`, or `done` +- `assignee` — who claimed the report (a user, an internal task, or an external agent), or `null` + when nobody has. Say so when you list a report someone else owns, so it doesn't read as free + work - `_posthogUrl` — clickable deep-link to the report; **always include this in your response** Group the results so the user can scan quickly. **Lead with reports where @@ -285,7 +294,7 @@ implement.** Reports from `signals_scout` (and any LLM-research source) are espe double-checking; their `summary` often reads as a confident root-cause with file and function names, but it can be stale or wrong. -### Step 1 — Retrieve and check it isn't already handled +### Step 1 — Retrieve, check existing work, and claim it ```json inbox-reports-retrieve @@ -297,9 +306,23 @@ Before doing any work, look at: - `already_addressed` — if `true`, the fix may already be in flight or merged; confirm with the user before duplicating it. - `implementation_pr_url` — if a PR is already linked, surface it instead of opening a second one. +- `work_state` and `assignee` — show whether someone else has already picked it up. A takeover is + allowed, but make it deliberate rather than overlooking active work. - `status` — only `ready` reports carry a finished judgment. A `candidate` / `pending_input` report hasn't been researched yet; don't implement off a half-formed summary. +Before starting research or implementation, claim the report: + +```json +posthog:inbox-reports-claim +{ "report_id": "" } +``` + +The claim is attributed to the current internal task or external MCP client. Claims do not expire, so +release the report whenever you walk away without landing a fix — including when you dismiss it instead. +Release clears ownership only; an attached pull request stays on the report, and release works from any +status, so a dismissed report can be released too. + ### Step 2 — Verify the diagnosis against the code (do not skip) Start by reading the report's work log — its evidence and the judgments behind it: @@ -317,7 +340,13 @@ way the report describes, and actually produces the described failure. As a deep the raw underlying signals via the `signals` skill (`metadata.report_id`) if you need the signal text behind the curated findings. If the diagnosis doesn't hold up, say so and stop — a wrong report is itself a useful finding (and a candidate for _dismiss_ below), not a license to write a -speculative fix. +speculative fix. Release the claim you took in Step 1 before you stop, so the report returns to the +unclaimed view instead of reading as active work nobody is doing: + +```json +posthog:inbox-reports-claim +{ "report_id": "", "release": true } +``` ### Step 3 — Scope the fix to the right layer @@ -329,15 +358,25 @@ speculative fix. area-specific skills), make the change minimal, and add a regression test that would have caught the reported failure. -### Step 4 — Open the PR and link it back +### Step 4 — Open the PR and attach it + +Open the PR following the repo's PR conventions, then attach it to the report: + +```json +posthog:inbox-reports-claim +{ + "report_id": "", + "pr_url": "https://github.com/example/repository/pull/123" +} +``` -Open the PR following the repo's PR conventions. There is no MCP tool to set -`implementation_pr_url` — that link is populated on the product surface when a PR is opened -against the report. So reference the report in the PR description (its `_posthogUrl`) and tell the -user which report the PR addresses, so the loop is traceable. +The same PR may be attached to multiple reports. Connected repositories receive immediate state +validation and webhook updates. PRs from unconnected repositories are accepted with unknown state, +so resolve those reports manually after the work lands. Also reference the report's `_posthogUrl` +in the PR description so the loop is traceable from either side. -**Don't resolve a report because you opened a PR.** When the fix ships as a PR, the merge is what -resolves the report — the tasks GitHub webhook does it automatically. Resolving by hand at PR-open +**Don't resolve a report because you opened a PR.** When the fix ships as a connected PR, the merge +resolves the report automatically. Resolving by hand at PR-open time asserts work that hasn't landed, and a reviewer looking at the inbox can't tell the difference. Manual resolve is for fixes a PR merge will never cover — a skill-body change, a config change, a `NO_REPO` report — see the workflow below. @@ -488,12 +527,12 @@ inbox-source-configs-partial-update - `suppressed` reports are excluded by default; pass `status: "suppressed"` explicitly if the user wants to see hidden items - The inbox writes exposed via MCP are `inbox-reports-set-state` (resolve / dismiss / snooze one - report), `inbox-reports-bulk-set-state` (the same for 1–100 reports), and + report), `inbox-reports-bulk-set-state` (the same for 1–100 reports), `inbox-reports-claim` + (claim / release / attach a PR), and `inbox-source-configs-partial-update` (toggle a source's `enabled` flag). To _act_ on a report - (implement a fix), verify the diagnosis against the code first, then open a PR — see + (implement a fix), claim it, verify the diagnosis against the code, then open and attach a PR — see _Workflow: act on an actionable report_. A PR-backed fix is resolved automatically when the PR - merges, so don't resolve it by hand at PR-open time; setting `implementation_pr_url` happens on - the product surface, not via MCP. Always also surface the `_posthogUrl` deep-link + merges, so don't resolve it by hand at PR-open time. Always also surface the `_posthogUrl` deep-link - **Never implement a report's fix straight from its `summary`.** Reports — especially `signals_scout` ones — are LLM diagnoses; confirm the cited files / functions / behavior in the actual code before writing a fix. A report that doesn't hold up is a dismissal candidate, not a diff --git a/skills/instrument-error-tracking/references/flutter.md b/skills/instrument-error-tracking/references/flutter.md index 388ea1c..28c550f 100644 --- a/skills/instrument-error-tracking/references/flutter.md +++ b/skills/instrument-error-tracking/references/flutter.md @@ -108,7 +108,7 @@ Copy page ...