From f9beae92274f54665efc53340c9b1eac493039f7 Mon Sep 17 00:00:00 2001 From: JacobPEvans <20714140+JacobPEvans-personal@users.noreply.github.com> Date: Sun, 21 Jun 2026 14:12:37 -0400 Subject: [PATCH 1/4] fix: correct 8 group-scoped resource API paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eight group-scoped resources used endpoint paths that return HTTP 404 on Cribl 4.17.1 (verified live against a real Leader). The originals were inconsistent — some under system/, one with no library prefix, two with a nested lib/sds/... segment, and several with abbreviated names — so the commands built from them could never reach the API. parsers system/parsers -> lib/parsers schemas schemas -> lib/schemas db-connections lib/db-connections -> lib/database-connections conditions lib/conditions -> conditions sds-rules lib/sds/rules -> lib/sds-rules sds-rulesets lib/sds/rulesets -> lib/sds-rulesets appscope lib/appscope -> lib/appscope-configs hmac-functions lib/hmac -> lib/hmac-functions registry.py drives every factory-generated CRUD command, so this also repairs the existing `cribl parsers list`, `schemas`, `sds-rules`, `appscope`, etc. commands — not just the new group export/import that surfaced the problem. --- cribl_cli/commands/registry.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cribl_cli/commands/registry.py b/cribl_cli/commands/registry.py index 0eb8de9..84e0204 100644 --- a/cribl_cli/commands/registry.py +++ b/cribl_cli/commands/registry.py @@ -20,22 +20,22 @@ class CommandRegistration: REGISTRY: list[CommandRegistration] = [ # Group-scoped (full CRUD) - CommandRegistration("parsers", EndpointConfig("group", "system/parsers")), - CommandRegistration("schemas", EndpointConfig("group", "schemas")), + CommandRegistration("parsers", EndpointConfig("group", "lib/parsers")), + CommandRegistration("schemas", EndpointConfig("group", "lib/schemas")), CommandRegistration("regex", EndpointConfig("group", "lib/regex")), CommandRegistration("grok", EndpointConfig("group", "lib/grok")), CommandRegistration("event-breakers", EndpointConfig("group", "lib/breakers")), CommandRegistration("global-vars", EndpointConfig("group", "lib/vars")), - CommandRegistration("db-connections", EndpointConfig("group", "lib/db-connections")), + CommandRegistration("db-connections", EndpointConfig("group", "lib/database-connections")), CommandRegistration("secrets", EndpointConfig("group", "system/secrets")), CommandRegistration("credentials", EndpointConfig("group", "system/credentials")), CommandRegistration("collectors", EndpointConfig("group", "collectors")), - CommandRegistration("conditions", EndpointConfig("group", "lib/conditions")), + CommandRegistration("conditions", EndpointConfig("group", "conditions")), CommandRegistration("parquet-schemas", EndpointConfig("group", "lib/parquet-schemas")), CommandRegistration("protobuf-libs", EndpointConfig("group", "lib/protobuf-libs")), - CommandRegistration("sds-rules", EndpointConfig("group", "lib/sds/rules")), - CommandRegistration("sds-rulesets", EndpointConfig("group", "lib/sds/rulesets")), - CommandRegistration("appscope", EndpointConfig("group", "lib/appscope")), + CommandRegistration("sds-rules", EndpointConfig("group", "lib/sds-rules")), + CommandRegistration("sds-rulesets", EndpointConfig("group", "lib/sds-rulesets")), + CommandRegistration("appscope", EndpointConfig("group", "lib/appscope-configs")), # Group-scoped (limited) CommandRegistration("certificates", EndpointConfig("group", "system/certificates"), ["list", "get", "create", "delete"]), CommandRegistration("samples", EndpointConfig("group", "system/samples"), ["list", "get", "create", "delete"]), @@ -43,7 +43,7 @@ class CommandRegistration: CommandRegistration("lookups", EndpointConfig("group", "system/lookups")), # packs: hand-written command in commands/packs.py (export, install, upgrade) CommandRegistration("executors", EndpointConfig("group", "executors"), ["list", "get"]), - CommandRegistration("hmac-functions", EndpointConfig("group", "lib/hmac"), ["list", "get"]), + CommandRegistration("hmac-functions", EndpointConfig("group", "lib/hmac-functions"), ["list", "get"]), CommandRegistration("functions", EndpointConfig("group", "system/functions"), ["list", "get"]), # Global-scoped (full CRUD) CommandRegistration("users", EndpointConfig("global", "system/users")), From 9d899e3db4be34d5a679fe9dda71cb96aa3ae361 Mon Sep 17 00:00:00 2001 From: JacobPEvans <20714140+JacobPEvans-personal@users.noreply.github.com> Date: Sun, 21 Jun 2026 14:22:35 -0400 Subject: [PATCH 2/4] feat: add whole-group export/import for worker groups and edge fleets Add a `groups` command that moves an entire worker group's or edge fleet's config between groups (a fleet is the same API object with `isFleet: true`, so both flow through one path): - `groups export ` pulls every group-scoped resource to stdout, or `--out-dir` writes one file per resource type (dir/files locked to 0700/0600). - `groups import ` upserts the config back. It is staged, never deployed: routes are skipped unless `--with-routes`, and `--deploy` requires `--yes`. - Secrets, credentials, and certificates are excluded unless `--include-sensitive`; everything skipped or failed is reported in `_meta`. - Import honors each resource type's registry operations: read-only types (executors, hmac-functions, functions) are skipped and create-only types without `update` (certificates, samples, scripts) are created, not PATCHed. The resource list is derived from `commands/registry.py` plus the hand-written sources/destinations/pipelines/packs/routes, so it never drifts from the rest of the CLI. Route import uses a new public `replace_route_table()` helper in `api/endpoints/routes.py` (wholesale swap, preserving edge/stream format). Depends on the registry path corrections in #2 (the export/import surfaced the 404s those fix). --- CLAUDE.md | 53 +++- cribl_cli/api/endpoints/routes.py | 16 ++ cribl_cli/cli.py | 3 +- cribl_cli/commands/groups.py | 121 +++++++++ cribl_cli/utils/group_transfer.py | 406 ++++++++++++++++++++++++++++++ tests/unit/test_group_transfer.py | 113 +++++++++ 6 files changed, 703 insertions(+), 9 deletions(-) create mode 100644 cribl_cli/commands/groups.py create mode 100644 cribl_cli/utils/group_transfer.py create mode 100644 tests/unit/test_group_transfer.py diff --git a/CLAUDE.md b/CLAUDE.md index 4dfcb0a..514433b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,10 +12,13 @@ Python CLI built with Click. Two kinds of commands: -1. **Hand-written (25):** `cribl_cli/commands/*.py` — complex operations (workers, routes, search, edge, health report, ingest dashboard, billing, finops, etc.) +1. **Hand-written (26):** `cribl_cli/commands/*.py` — complex operations + (workers, groups export/import, routes, search, edge, health report, + ingest dashboard, billing, finops, etc.) 2. **Factory-generated (52):** Declared in `commands/registry.py`, built by `commands/command_factory.py` — standard CRUD Key modules: + - `cli.py` — Click group (`CriblCLI`), registers all commands, skips auth for `config` subcommand - `api/client.py` — httpx client with `AuthTransport` and `DryRunTransport` - `api/endpoint_factory.py` — generic CRUD for four scope types (group/global/search/lake) @@ -34,30 +37,64 @@ Key modules: ## CLI Commands -Worker groups are managed via `workers`, not `groups`. +Worker groups are listed and managed via `workers` (there is no `groups list`). +The `groups` command is separate: it exports and imports an entire worker group's +or edge fleet's config. Both are the same API object — a fleet is a group with +`isFleet: true`. -**Hand-written commands:** alerts, billing, config, destinations, edge, finops, health (check, report, cpu), ingest (dashboard, query), jobs, kms, license-usage, logger, metrics, notebooks, overview, packs, pipelines, preview, profiler, routes, search, sources, system, version, workers +**Hand-written commands:** alerts, billing, config, destinations, edge, finops, +groups, health (check, report, cpu), ingest (dashboard, query), jobs, kms, +license-usage, logger, metrics, notebooks, overview, packs, pipelines, preview, +profiler, routes, search, sources, system, version, workers **Key subcommands:** + - `workers list` — list worker groups; `workers nodes` — list individual worker nodes (supports `-g` group filter) - `edge nodes` — list individual edge nodes (supports `-f` fleet filter) - -**Factory-generated commands (standard CRUD):** ai-settings, alert-monitors, appscope, auth-settings, banners, certificates, collectors, conditions, credentials, dashboard-categories, dashboards, dataset-providers, datasets, datatypes, db-connections, encryption-keys, event-breakers, executors, feature-flags, functions, git-settings, global-vars, grok, hmac-functions, lake-datasets, licenses, lookups, macros, messages, notification-targets, notifications, outposts, parquet-schemas, parsers, policies, protobuf-libs, regex, roles, samples, saved-searches, schemas, scripts, sds-rules, sds-rulesets, secrets, storage-locations, subscriptions, teams, trust-policies, usage-groups, users, workspaces +- `groups export ` — pull all config for one worker group or edge fleet + (JSON to stdout, or `--out-dir` for one file per resource type); `groups import + ` — push it back (see Safety Rules) + +**Factory-generated commands (standard CRUD):** ai-settings, alert-monitors, +appscope, auth-settings, banners, certificates, collectors, conditions, +credentials, dashboard-categories, dashboards, dataset-providers, datasets, +datatypes, db-connections, encryption-keys, event-breakers, executors, +feature-flags, functions, git-settings, global-vars, grok, hmac-functions, +lake-datasets, licenses, lookups, macros, messages, notification-targets, +notifications, outposts, parquet-schemas, parsers, policies, protobuf-libs, +regex, roles, samples, saved-searches, schemas, scripts, sds-rules, +sds-rulesets, secrets, storage-locations, subscriptions, teams, trust-policies, +usage-groups, users, workspaces ## Safety Rules - **Never replace the route table wholesale.** `routes create` fetches existing routes, inserts before the catch-all, then updates. - **Always confirm before deploying.** `version deploy` pushes config to live workers. +- **`groups import` is staged, never deployed.** It upserts config but never + auto-commits or deploys; routes are skipped unless `--with-routes`, and + `--deploy` requires `--yes`. Review with `version status`/`diff` first. +- **`groups export` excludes secrets by default.** Secrets, credentials, and + certificates are omitted unless `--include-sensitive`; what was skipped or + failed is always reported in the `_meta` block. - Cloud OAuth audience is always `https://api.cribl.cloud`, not the org-specific URL. ## Conventions - New CRUD resources go in `commands/registry.py` — only write a hand-written command file if the resource needs custom logic. -- Merge-on-update pattern: fetch existing object, strip server-only fields (`status`, `notifications`), deep-merge user updates (nested dicts are merged recursively, not replaced). +- Merge-on-update pattern: fetch existing object, strip server-only fields + (`status`, `notifications`), deep-merge user updates (nested dicts are merged + recursively, not replaced). - `--dry-run` logs request details to stderr and raises `DryRunAbort` (caught by error handler, exits 0). - Config tests mock the filesystem to avoid touching real `~/.criblrc`. -- `health report` aggregates nodes, capacity alerts, versions, unhealthy IO, and error logs into a single command. Supports `--json`, `--skip-errors`, and `-g` group filter. +- `health report` aggregates nodes, capacity alerts, versions, unhealthy IO, and + error logs into a single command. Supports `--json`, `--skip-errors`, and `-g` + group filter. - `ingest dashboard` shows daily ingest totals (events/bytes in/out) by source (Stream, Edge, Search). Supports `--json`, `--table`, and `--hours`. - `ingest query` runs a raw metric query — accepts a JSON payload as argument. - `dashboards/` contains Cribl Search dashboard definitions for Daily Ingest (overview, by source, by route). Deploy with `cribl dashboards create "$(cat dashboards/.json)"`. -- When running CLI commands to read data, use default JSON output (no `--table`). JSON is structured and easier to parse. Only use `--table` if the user explicitly asks for it. +- `groups export`/`groups import` move a whole group's config; the resource list + is derived from `commands/registry.py` group-scoped entries plus the + hand-written sources/destinations/pipelines/packs/routes, so it never drifts. +- When running CLI commands to read data, use default JSON output (no `--table`). + JSON is structured and easier to parse. Only use `--table` if the user + explicitly asks for it. diff --git a/cribl_cli/api/endpoints/routes.py b/cribl_cli/api/endpoints/routes.py index 59ce6d7..fac0d1c 100644 --- a/cribl_cli/api/endpoints/routes.py +++ b/cribl_cli/api/endpoints/routes.py @@ -87,6 +87,22 @@ def list_routes(client: httpx.Client, group: str) -> Any: return result +def replace_route_table( + client: httpx.Client, group: str, items: list[dict[str, Any]] +) -> Any: + """Replace the whole route table's items in one PATCH. + + Routes are a single document, not individually addressable, so importing one + group's routes into another means swapping the entire items array at once + rather than upserting routes one by one. Fetches the live table first so the + edge/stream wrapper format is preserved on the way back. Used by + ``groups import --with-routes``. + """ + table = _fetch_route_table(client, group) + table["items"] = list(items) + return _patch_route_table(client, group, table) + + def get_route(client: httpx.Client, group: str, route_id: str) -> Any: """Get a single route by ID from the route table.""" table = _fetch_route_table(client, group) diff --git a/cribl_cli/cli.py b/cribl_cli/cli.py index 05cfffa..423c5c0 100644 --- a/cribl_cli/cli.py +++ b/cribl_cli/cli.py @@ -66,6 +66,7 @@ def _register_commands() -> None: # Hand-written commands from cribl_cli.commands.config_cmd import config_group from cribl_cli.commands.workers import workers_group + from cribl_cli.commands.groups import groups_group from cribl_cli.commands.sources import sources_group from cribl_cli.commands.destinations import destinations_group from cribl_cli.commands.metrics import metrics_group @@ -91,7 +92,7 @@ def _register_commands() -> None: from cribl_cli.commands.finops import finops_group for group in [ - config_group, workers_group, sources_group, destinations_group, + config_group, workers_group, groups_group, sources_group, destinations_group, metrics_group, search_group, notebooks_group, pipelines_group, routes_group, jobs_group, version_group, system_group, edge_group, kms_group, preview_group, logger_group, profiler_group, health_group, diff --git a/cribl_cli/commands/groups.py b/cribl_cli/commands/groups.py new file mode 100644 index 0000000..3c66154 --- /dev/null +++ b/cribl_cli/commands/groups.py @@ -0,0 +1,121 @@ +"""Whole-group config export/import for worker groups and edge fleets.""" +from __future__ import annotations + +import json +import sys + +import click + +from cribl_cli.api.client import get_client +from cribl_cli.api.endpoints.version import commit_version +from cribl_cli.api.endpoints.workers import deploy_group +from cribl_cli.output.formatter import format_output +from cribl_cli.utils.errors import handle_error +from cribl_cli.utils.group_resolver import resolve_group +from cribl_cli.utils.group_transfer import ( + apply, + collect, + format_caveat, + read_input, + write_dir, +) + + +@click.group("groups", help="Export and import all config for a worker group or edge fleet.") +def groups_group(): + pass + + +@groups_group.command("export", help="Pull all config for one worker group or edge fleet.") +@click.argument("group") +@click.option("--out-dir", default=None, help="Write one file per resource type under //.") +@click.option("--include-sensitive", is_flag=True, help="Include secrets, credentials, and certificates.") +@click.option("--include-packs", is_flag=True, help="Include pack configurations (definitions only).") +@click.option("--include-lookups", is_flag=True, help="Include lookup configurations (definitions only).") +@click.option("--table", "use_table", is_flag=True, help="Table output (stdout mode only).") +def groups_export(group, out_dir, include_sensitive, include_packs, include_lookups, use_table): + """Export every group-scoped resource for GROUP (a worker group or edge fleet). + + Outputs one aggregated JSON object to stdout, or use --out-dir to write a + file per resource type. Sensitive resources are excluded unless + --include-sensitive is passed; a caveat of everything skipped or failed is + always printed to stderr. + """ + try: + client = get_client() + g = resolve_group(client, group) + result = collect( + client, g, include_sensitive=include_sensitive, + include_packs=include_packs, include_lookups=include_lookups + ) + + if out_dir: + base = write_dir(result, out_dir) + click.echo(format_output({"written": str(base), "resource_types": len(result["resources"])})) + else: + click.echo(format_output(result, table=use_table)) + + click.echo(format_caveat(result), err=True) + except Exception as e: + handle_error(e) + + +@groups_group.command("import", help="Push exported config into a worker group or edge fleet.") +@click.argument("group") +@click.option("--in", "in_path", default=None, help="Read payload from a JSON file or an --out-dir directory.") +@click.option("--with-routes", is_flag=True, help="Also replace the route table (wholesale; off by default).") +@click.option("--with-packs", is_flag=True, help="Also import packs (off by default).") +@click.option("--with-lookups", is_flag=True, help="Also import lookups (off by default).") +@click.option("--commit", "commit_message", default=None, help="Commit staged changes with this message.") +@click.option("--deploy", is_flag=True, help="Commit and deploy to live workers (requires --yes).") +@click.option("--yes", is_flag=True, help="Confirm deployment.") +def groups_import(group, in_path, with_routes, with_packs, with_lookups, commit_message, deploy, yes): + """Import config into GROUP (a worker group or edge fleet). + + Reads an export payload from --in (a file or an --out-dir directory) or from + stdin, then upserts each resource. The route table is left untouched unless + --with-routes is passed. Nothing is deployed: changes stay staged until you + review them and deploy explicitly, or pass --deploy --yes. + """ + try: + if in_path: + payload = read_input(in_path) + else: + raw = sys.stdin.read() + if not raw.strip(): + raise ValueError("No input. Pass --in FILE|DIR or pipe an export JSON on stdin.") + payload = json.loads(raw) + + client = get_client() + target = resolve_group(client, group) + + if deploy and not yes: + click.echo( + "WARNING: --deploy commits and pushes config to live workers. Pass --yes to confirm.", + err=True, + ) + sys.exit(1) + + report = apply( + client, target, payload, + with_routes=with_routes, with_packs=with_packs, with_lookups=with_lookups + ) + click.echo(format_output(report)) + + if commit_message or deploy: + message = commit_message or "Import group config" + commit_version(client, target, message) + click.echo(f"Committed: {message}", err=True) + + if deploy: + deploy_group(client, target) + click.echo(f"Deployed to {target}.", err=True) + else: + click.echo( + f"Config staged (not deployed). Review with " + f"`cribl version status -g {target}` / `cribl version diff -g {target}`, " + f"then `cribl version deploy -g {target} --yes`.", + err=True, + ) + except Exception as e: + handle_error(e) diff --git a/cribl_cli/utils/group_transfer.py b/cribl_cli/utils/group_transfer.py new file mode 100644 index 0000000..d811388 --- /dev/null +++ b/cribl_cli/utils/group_transfer.py @@ -0,0 +1,406 @@ +"""Export and import all config for a single worker group or edge fleet. + +A worker group and an edge fleet are the same API object — an entry in +``/api/v1/master/groups`` whose config lives under ``/api/v1/m/{group}/…`` +(a fleet is just one with ``isFleet: true``). This module pulls every +group-scoped resource for one such object (export) and pushes it back into +another (import). + +Resource → endpoint mappings are sourced from ``commands/registry.py`` so the +two never drift. Sensitive resources (secrets, credentials, certificates) are +excluded by default; stream-only resources are skipped for edge fleets. Routes +(a single wholesale-replaced table) and packs (need their binary archive) get +special handling on import. Everything skipped or failed is reported in the +``_meta`` block — the export is never silently incomplete. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import httpx + +from cribl_cli.api.endpoint_factory import EndpointConfig, Endpoints +from cribl_cli.api.endpoints.routes import list_routes, replace_route_table +from cribl_cli.commands.registry import ALL_OPS, REGISTRY +from cribl_cli.utils.validation import deep_merge + +# Group-scoped resources not declared in the registry (they have hand-written +# command modules), mapped to their API path under /api/v1/m/{group}/. +HANDWRITTEN_GENERIC: dict[str, str] = { + "sources": "system/inputs", + "destinations": "system/outputs", + "pipelines": "pipelines", + "packs": "packs", +} + +# Excluded from a default export; opt in with include_sensitive=True. +SENSITIVE: frozenset[str] = frozenset({"secrets", "credentials", "certificates"}) + +# Not applicable to edge fleets (Stream-only), skipped when isFleet is True. +STREAM_ONLY: frozenset[str] = frozenset( + {"collectors", "db-connections", "hmac-functions", "executors", "functions"} +) + +# Server-managed fields stripped before any write. +SERVER_FIELDS: tuple[str, ...] = ("status", "notifications") + +# Dependency order for import: library items first, then pipelines, then +# inputs/outputs/collectors, then notifications, then routes last. Names absent +# here fall back to alphabetical after the ordered ones. Mirrors the ordering +# used by the cribl-migration tool. +IMPORT_ORDER: tuple[str, ...] = ( + "packs", + "lookups", + "regex", + "event-breakers", + "parsers", + "global-vars", + "grok", + "schemas", + "parquet-schemas", + "db-connections", + "hmac-functions", + "appscope", + "conditions", + "protobuf-libs", + "sds-rules", + "sds-rulesets", + "samples", + "scripts", + "functions", + "executors", + "pipelines", + "sources", + "destinations", + "collectors", + "notifications", + "alert-monitors", + "routes", +) + +# Informational caveats: object config is exported, file payloads are not. +BINARY_CAVEAT: tuple[str, ...] = ( + "packs (install archive .crbl not exported/imported)", + "lookups (CSV/binary file contents not exported)", +) + + +def _registry_group_configs() -> dict[str, EndpointConfig]: + return { + reg.name: reg.endpoint + for reg in REGISTRY + if reg.endpoint.scope == "group" and "list" in reg.operations + } + + +# name -> EndpointConfig for every generic (non-routes) group resource. +RESOURCE_CONFIGS: dict[str, EndpointConfig] = { + **{name: EndpointConfig("group", path) for name, path in HANDWRITTEN_GENERIC.items()}, + **_registry_group_configs(), +} + +# name -> write operations the API actually supports, sourced from the registry +# so it never drifts. apply() uses this to skip read-only resources (no create +# and no update) and to avoid PATCHing create-only ones. Hand-written generics +# (sources/destinations/pipelines/packs) support full CRUD. +RESOURCE_OPERATIONS: dict[str, frozenset[str]] = { + **{name: frozenset(ALL_OPS) for name in HANDWRITTEN_GENERIC}, + **{reg.name: frozenset(reg.operations) for reg in REGISTRY if reg.endpoint.scope == "group"}, +} + + +def _order_key(name: str) -> tuple[int, str]: + try: + return (IMPORT_ORDER.index(name), "") + except ValueError: + return (len(IMPORT_ORDER), name) + + +def plan_resources( + is_fleet: bool, include_sensitive: bool, include_packs: bool = False, include_lookups: bool = False +) -> tuple[list[tuple[str, EndpointConfig]], dict[str, list[str]]]: + """Return (included generic resources in dependency order, skipped-by-reason).""" + included: list[tuple[str, EndpointConfig]] = [] + skipped: dict[str, list[str]] = {"sensitive": [], "stream_only": [], "omitted_by_default": []} + + for name in sorted(RESOURCE_CONFIGS, key=_order_key): + if name in SENSITIVE and not include_sensitive: + skipped["sensitive"].append(name) + elif name in STREAM_ONLY and is_fleet: + skipped["stream_only"].append(name) + elif name in {"packs", "lookups"} and not (include_packs if name == "packs" else include_lookups): + skipped["omitted_by_default"].append(name) + else: + included.append((name, RESOURCE_CONFIGS[name])) + + return included, skipped + + +def resolve_group_meta(client: httpx.Client, group_id: str) -> dict[str, Any]: + """Look up a group/fleet entry in /master/groups to read its type and isFleet.""" + resp = client.get("/api/v1/master/groups") + resp.raise_for_status() + items = resp.json().get("items", []) + for entry in items: + if entry.get("id") == group_id: + return entry + available = ", ".join(e.get("id", "?") for e in items) or "(none)" + raise ValueError(f"Group or fleet '{group_id}' not found. Available: {available}") + + +def collect( + client: httpx.Client, group_id: str, *, include_sensitive: bool = False, + include_packs: bool = False, include_lookups: bool = False +) -> dict[str, Any]: + """Fetch every in-scope resource for one group/fleet into a single payload.""" + meta = resolve_group_meta(client, group_id) + is_fleet = bool(meta.get("isFleet", False)) + included, skipped = plan_resources(is_fleet, include_sensitive, include_packs, include_lookups) + + resources: dict[str, Any] = {} + exported: list[str] = [] + errors: dict[str, str] = {} + + for name, cfg in included: + try: + resources[name] = Endpoints(cfg).list(client, group_id) + exported.append(name) + except httpx.HTTPError as exc: + errors[name] = f"HTTP Error: {exc}" + except Exception as exc: # noqa: BLE001 — report per-resource, keep going + errors[name] = str(exc) + + # Routes: a single normalized table document, always part of an export. + try: + resources["routes"] = list_routes(client, group_id) + exported.append("routes") + except httpx.HTTPError as exc: + errors["routes"] = f"HTTP Error: {exc}" + except Exception as exc: # noqa: BLE001 + errors["routes"] = str(exc) + + return { + "group": group_id, + "type": meta.get("type", "stream"), + "isFleet": is_fleet, + "resources": resources, + "_meta": { + "exported": exported, + "skipped": {**skipped, "binary_content": list(BINARY_CAVEAT)}, + "errors": errors, + }, + } + + +def _items(value: Any) -> list[dict[str, Any]]: + """Normalize a list-endpoint payload to a list of resource dicts.""" + if isinstance(value, dict) and "items" in value: + return [i for i in value["items"] if isinstance(i, dict)] + if isinstance(value, list): + return [i for i in value if isinstance(i, dict)] + return [] + + +def _strip(obj: dict[str, Any]) -> dict[str, Any]: + """Drop server-computed fields before writing an object back. + + ``status`` and ``notifications`` are populated by the server and are not + user-writable; sending them on a create/update can make the API reject the + request, so they are removed from every payload we push. + """ + out = dict(obj) + for field in SERVER_FIELDS: + out.pop(field, None) + return out + + +def _upsert_item( + client: httpx.Client, target_id: str, cfg: EndpointConfig, item: dict[str, Any], + *, can_update: bool = True, +) -> str: + """Create or merge-update a single resource. + + Returns 'created', 'updated', or 'skipped' (the resource already exists but + its type has no update operation, so it is left untouched rather than + PATCHed against an endpoint that would reject it). + """ + endpoints = Endpoints(cfg) + resource_id = item.get("id") + incoming = _strip(item) + + if resource_id: + try: + existing = endpoints.get(client, target_id, resource_id) + except httpx.HTTPStatusError as exc: + if exc.response is not None and exc.response.status_code == 404: + endpoints.create(client, target_id, incoming) + return "created" + raise + if not can_update: + return "skipped" + if isinstance(existing, dict) and "items" in existing: + existing = existing["items"][0] if existing["items"] else {} + merged = deep_merge(_strip(existing) if isinstance(existing, dict) else {}, incoming) + endpoints.update(client, target_id, resource_id, merged) + return "updated" + + endpoints.create(client, target_id, incoming) + return "created" + + +def _apply_routes( + client: httpx.Client, target_id: str, routes_value: Any +) -> int: + """Replace the target route table's items with the imported routes (opt-in).""" + new_items = _items(routes_value) + replace_route_table(client, target_id, new_items) + return len(new_items) + + +def apply( + client: httpx.Client, + target_id: str, + payload: dict[str, Any], + *, + with_routes: bool = False, + with_packs: bool = False, + with_lookups: bool = False, +) -> dict[str, Any]: + """Upsert every resource in *payload* into the target group/fleet.""" + resources: dict[str, Any] = payload.get("resources", {}) + report: dict[str, Any] = { + "target": target_id, + "created": [], + "updated": [], + "failed": {}, + "skipped": {}, + } + + meta = resolve_group_meta(client, target_id) + is_fleet = bool(meta.get("isFleet", False)) + + for name in sorted(resources, key=_order_key): + value = resources[name] + if name in STREAM_ONLY and is_fleet: + report["skipped"][name] = f"skipped stream-only resource type for fleet '{target_id}'" + continue + + if name == "routes": + if not with_routes: + report["skipped"]["routes"] = "route table is wholesale-replaced; pass --with-routes" + continue + try: + count = _apply_routes(client, target_id, value) + report["updated"].append(f"routes ({count} routes)") + except httpx.HTTPError as exc: + report["failed"]["routes"] = f"HTTP Error: {exc}" + except Exception as exc: # noqa: BLE001 + report["failed"]["routes"] = str(exc) + continue + + if name in {"packs", "lookups"} and not (with_packs if name == "packs" else with_lookups): + report["skipped"][name] = f"{name} skipped; pass --with-{name} to import their JSON configs" + continue + + cfg = RESOURCE_CONFIGS.get(name) + if cfg is None: + report["skipped"][name] = "unknown resource type" + continue + + ops = RESOURCE_OPERATIONS.get(name, frozenset(ALL_OPS)) + if "create" not in ops and "update" not in ops: + report["skipped"][name] = "read-only resource type (not writable via API)" + continue + can_update = "update" in ops + + for item in _items(value): + ref = f"{name}/{item.get('id', '?')}" + try: + outcome = _upsert_item(client, target_id, cfg, item, can_update=can_update) + if outcome == "skipped": + report["skipped"][ref] = "exists; resource type has no update operation" + else: + report[outcome].append(ref) + except httpx.HTTPError as exc: + report["failed"][ref] = f"HTTP Error: {exc}" + except Exception as exc: # noqa: BLE001 + report["failed"][ref] = str(exc) + + return report + + +def write_dir(result: dict[str, Any], out_dir: str | Path) -> Path: + """Write one file per resource type under //, plus _meta.json. + + The directory and files are locked to owner-only permissions (0700/0600) so + exports taken with --include-sensitive are not left world-readable, matching + how the CLI protects ~/.criblrc. + """ + base = Path(out_dir) / str(result.get("group", "group")) + base.mkdir(parents=True, exist_ok=True) + base.chmod(0o700) + for name, value in result.get("resources", {}).items(): + f = base / f"{name}.json" + f.write_text(json.dumps(value, indent=2)) + f.chmod(0o600) + meta = {k: v for k, v in result.items() if k != "resources"} + meta_file = base / "_meta.json" + meta_file.write_text(json.dumps(meta, indent=2)) + meta_file.chmod(0o600) + return base + + +def read_input(path: str | Path) -> dict[str, Any]: + """Read an export payload from a single JSON file or a write_dir() directory.""" + p = Path(path) + if p.is_file(): + return json.loads(p.read_text()) + if not p.is_dir(): + raise ValueError(f"Input path not found: {path}") + + group_dir = p + if not (p / "_meta.json").exists(): + subdirs = [c for c in p.iterdir() if c.is_dir()] + if len(subdirs) == 1: + group_dir = subdirs[0] + elif len(subdirs) > 1: + names = ", ".join(sorted(c.name for c in subdirs)) + raise ValueError( + f"Ambiguous import directory '{path}': no _meta.json and multiple " + f"group subdirectories ({names}). Point --in at a specific " + f"/ directory." + ) + + resources: dict[str, Any] = {} + meta: dict[str, Any] = {} + for f in sorted(group_dir.glob("*.json")): + data = json.loads(f.read_text()) + if f.name == "_meta.json": + meta = data + else: + resources[f.stem] = data + + payload = {"resources": resources} + payload.update({k: v for k, v in meta.items() if k != "resources"}) + return payload + + +def format_caveat(result: dict[str, Any]) -> str: + """Build a concise human-readable caveat summary for stderr.""" + m = result.get("_meta", {}) + skipped = m.get("skipped", {}) + lines = [f"Exported {len(m.get('exported', []))} resource type(s) for '{result.get('group')}'."] + if skipped.get("sensitive"): + lines.append(f" Excluded (sensitive, use --include-sensitive): {', '.join(skipped['sensitive'])}") + if skipped.get("omitted_by_default"): + lines.append(f" Excluded (default): {', '.join(skipped['omitted_by_default'])} (use --include-packs/--include-lookups)") + if skipped.get("stream_only"): + lines.append(f" Skipped (Stream-only, not on this fleet): {', '.join(skipped['stream_only'])}") + if skipped.get("binary_content"): + lines.append(f" Not captured: {'; '.join(skipped['binary_content'])}") + if m.get("errors"): + lines.append(f" Failed to fetch: {', '.join(m['errors'])}") + return "\n".join(lines) diff --git a/tests/unit/test_group_transfer.py b/tests/unit/test_group_transfer.py new file mode 100644 index 0000000..fe21b93 --- /dev/null +++ b/tests/unit/test_group_transfer.py @@ -0,0 +1,113 @@ +"""Tests for whole-group export/import (cribl_cli.utils.group_transfer).""" +from __future__ import annotations + +from unittest.mock import MagicMock, patch +import httpx +from click.testing import CliRunner + +from cribl_cli.cli import cli +from cribl_cli.utils.group_transfer import SENSITIVE, STREAM_ONLY, apply, collect, plan_resources, read_input, write_dir + +def _mock_response(json_data, status_code=200): + resp = MagicMock(spec=httpx.Response) + resp.json.return_value = json_data + resp.status_code = status_code + resp.raise_for_status.return_value = None + return resp + +def _routing_get(group_entry, error_paths=None): + error_paths = error_paths or {} + def _get(url, *args, **kwargs): + if url == "/api/v1/master/groups": return _mock_response({"items": [group_entry]}) + for needle, exc in error_paths.items(): + if url.endswith(needle): raise exc + if url.endswith("/routes"): return _mock_response({"id": "default", "items": [{"id": "r1"}]}) + return _mock_response({"items": [{"id": "x"}]}) + return _get + +def test_plan_resources_filtering(): + """Verify sensitive, stream-only, packs, and lookups are properly filtered/included.""" + inc, skip = plan_resources(is_fleet=False, include_sensitive=False) + names = {n for n, _ in inc} + assert not (names & SENSITIVE) and set(skip["sensitive"]) == set(SENSITIVE) + assert "packs" not in names and "packs" in skip["omitted_by_default"] + + inc, skip = plan_resources(is_fleet=True, include_sensitive=True, include_packs=True) + names = {n for n, _ in inc} + assert SENSITIVE <= names and skip["sensitive"] == [] + assert not (names & STREAM_ONLY) and set(skip["stream_only"]) == set(STREAM_ONLY) + assert "packs" in names and "packs" not in skip["omitted_by_default"] + +def test_collect_aggregates_and_handles_errors(): + client = MagicMock(spec=httpx.Client) + err = httpx.HTTPStatusError("500", request=MagicMock(), response=_mock_response({}, 500)) + client.get.side_effect = _routing_get({"id": "grp", "type": "stream", "isFleet": False}, {"lib/parsers": err}) + + result = collect(client, "grp", include_packs=True) + + assert result["group"] == "grp" and "sources" in result["resources"] and "routes" in result["resources"] + assert "parsers" in result["_meta"]["errors"] and "parsers" not in result["resources"] + assert "packs" in result["resources"] + +def test_write_read_roundtrip(tmp_path): + result = { + "group": "grp", "type": "stream", "isFleet": False, + "resources": {"sources": {"items": [{"id": "s1"}]}}, + "_meta": {"exported": ["sources"], "skipped": {}, "errors": {}} + } + write_dir(result, tmp_path) + payload = read_input(tmp_path) + assert payload["group"] == "grp" and payload["resources"]["sources"] == {"items": [{"id": "s1"}]} + +def test_apply_upserts_resources(): + client = MagicMock(spec=httpx.Client) + not_found = _mock_response({}, 404) + not_found.raise_for_status.side_effect = httpx.HTTPStatusError("404", request=MagicMock(), response=not_found) + + def _mock_get(url, *args, **kwargs): + if url == "/api/v1/master/groups": + return _mock_response({"items": [{"id": "target", "isFleet": False}]}) + return not_found + + client.get.side_effect = _mock_get + client.post.return_value = _mock_response({"id": "p1"}) + + payload = {"resources": {"parsers": {"items": [{"id": "p1", "type": "x"}]}, "packs": {"items": [{"id": "pack1"}]}}} + + # 1. Packs skipped by default + report = apply(client, "target", payload) + assert "parsers/p1" in report["created"] and client.post.call_count == 1 + assert "packs" in report["skipped"] + + # 2. Packs included with flag + report2 = apply(client, "target", payload, with_packs=True) + assert "packs/pack1" in report2["created"] + +def test_apply_respects_resource_operations(): + """Read-only types are skipped; create-only types never PATCH an existing item.""" + client = MagicMock(spec=httpx.Client) + + def _mock_get(url, *args, **kwargs): + if url == "/api/v1/master/groups": + return _mock_response({"items": [{"id": "target", "isFleet": False}]}) + return _mock_response({"id": "s1"}) # every resource GET reports "exists" + + client.get.side_effect = _mock_get + + payload = {"resources": { + "executors": {"items": [{"id": "e1"}]}, # list/get only -> read-only + "scripts": {"items": [{"id": "s1"}]}, # create-only, already exists + }} + + report = apply(client, "target", payload) + + assert "executors" in report["skipped"] # whole type skipped + assert "scripts/s1" in report["skipped"] # existing create-only item left alone + assert report["updated"] == [] and report["failed"] == {} + client.patch.assert_not_called() + + +@patch("cribl_cli.commands.groups.get_client") +def test_import_deploy_requires_yes(mock_get_client): + mock_get_client.return_value = MagicMock(spec=httpx.Client) + assert CliRunner().invoke(cli, ["groups", "import", "grp", "--deploy"], input="{}").exit_code != 0 From 013904d107f2021de701cad9b9a41b87a23ce6df Mon Sep 17 00:00:00 2001 From: JacobPEvans <20714140+JacobPEvans-personal@users.noreply.github.com> Date: Sun, 21 Jun 2026 17:59:08 -0400 Subject: [PATCH 3/4] fix: address Copilot review on group export/import - write_dir() clears stale *.json from a prior export before writing, so a secrets.json left by an earlier --include-sensitive run can't linger and be re-imported by read_input(). - Narrow the "never replace the route table wholesale" rule in CLAUDE.md to note that `groups import --with-routes` is the deliberate, gated exception. - Add unit tests for replace_route_table() (stream + edge wrapper) and for write_dir() stale-file cleanup. --- CLAUDE.md | 5 +++- cribl_cli/utils/group_transfer.py | 6 +++++ tests/unit/test_group_transfer.py | 10 ++++++++ tests/unit/test_routes_commands.py | 39 ++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 514433b..679f3c2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -68,7 +68,10 @@ usage-groups, users, workspaces ## Safety Rules -- **Never replace the route table wholesale.** `routes create` fetches existing routes, inserts before the catch-all, then updates. +- **Never replace the route table wholesale from `routes`.** `routes create` + fetches existing routes, inserts before the catch-all, then updates. The one + deliberate exception is `groups import --with-routes` — a high-risk, explicit + opt-in that replaces the whole table via `replace_route_table()`. - **Always confirm before deploying.** `version deploy` pushes config to live workers. - **`groups import` is staged, never deployed.** It upserts config but never auto-commits or deploys; routes are skipped unless `--with-routes`, and diff --git a/cribl_cli/utils/group_transfer.py b/cribl_cli/utils/group_transfer.py index d811388..fb9b2f1 100644 --- a/cribl_cli/utils/group_transfer.py +++ b/cribl_cli/utils/group_transfer.py @@ -338,10 +338,16 @@ def write_dir(result: dict[str, Any], out_dir: str | Path) -> Path: The directory and files are locked to owner-only permissions (0700/0600) so exports taken with --include-sensitive are not left world-readable, matching how the CLI protects ~/.criblrc. + + Any JSON left from a previous export of the same group is removed first, so a + file written by an earlier --include-sensitive run (e.g. secrets.json) can't + linger and get re-imported once the user re-exports without that flag. """ base = Path(out_dir) / str(result.get("group", "group")) base.mkdir(parents=True, exist_ok=True) base.chmod(0o700) + for stale in base.glob("*.json"): + stale.unlink() for name, value in result.get("resources", {}).items(): f = base / f"{name}.json" f.write_text(json.dumps(value, indent=2)) diff --git a/tests/unit/test_group_transfer.py b/tests/unit/test_group_transfer.py index fe21b93..d7f2397 100644 --- a/tests/unit/test_group_transfer.py +++ b/tests/unit/test_group_transfer.py @@ -59,6 +59,16 @@ def test_write_read_roundtrip(tmp_path): payload = read_input(tmp_path) assert payload["group"] == "grp" and payload["resources"]["sources"] == {"items": [{"id": "s1"}]} +def test_write_dir_clears_stale_files(tmp_path): + """Re-exporting removes JSON from a prior export (e.g. leftover secrets.json).""" + result = {"group": "grp", "resources": {"sources": {"items": [{"id": "s1"}]}}, "_meta": {}} + base = tmp_path / "grp" + base.mkdir() + (base / "secrets.json").write_text("{}") # leftover from an earlier --include-sensitive run + write_dir(result, tmp_path) + assert not (base / "secrets.json").exists() + assert (base / "sources.json").exists() + def test_apply_upserts_resources(): client = MagicMock(spec=httpx.Client) not_found = _mock_response({}, 404) diff --git a/tests/unit/test_routes_commands.py b/tests/unit/test_routes_commands.py index e996f6c..d7633af 100644 --- a/tests/unit/test_routes_commands.py +++ b/tests/unit/test_routes_commands.py @@ -17,6 +17,7 @@ delete_route, get_route, list_routes, + replace_route_table, update_route, ) @@ -209,3 +210,41 @@ def test_list_routes_returns_table(): assert result["id"] == "default" assert len(result["items"]) == 3 + + +# --------------------------------------------------------------------------- +# replace_route_table (wholesale swap used by `groups import --with-routes`) +# --------------------------------------------------------------------------- + + +def test_replace_route_table_stream(): + """Stream groups: the whole items array is PATCHed back to /routes.""" + client = _make_client() + client.get.return_value = _mock_response({"id": "default", "items": [{"id": "old"}]}) + client.patch.return_value = _mock_response({"ok": True}) + + new_items = [{"id": "a"}, {"id": "b"}] + replace_route_table(client, "grp", new_items) + + url = client.patch.call_args[0][0] + payload = client.patch.call_args[1]["json"] + assert url == "/api/v1/m/grp/routes" + assert payload["items"] == new_items + + +def test_replace_route_table_edge(): + """Edge groups: items are re-wrapped as routes and PATCHed to /routes/{id}.""" + client = _make_client() + client.get.return_value = _mock_response( + {"items": [{"id": "default", "routes": [{"id": "old"}]}], "count": 1} + ) + client.patch.return_value = _mock_response({"ok": True}) + + new_items = [{"id": "a"}] + replace_route_table(client, "grp", new_items) + + url = client.patch.call_args[0][0] + payload = client.patch.call_args[1]["json"] + assert url == "/api/v1/m/grp/routes/default" + assert payload["routes"] == new_items + assert "_edge_format" not in payload From 6d75f827964349d914fe799c8443b7efe29420ca Mon Sep 17 00:00:00 2001 From: Ahendrix9624 <33384698+Ahendrix9624@users.noreply.github.com> Date: Wed, 24 Jun 2026 23:27:58 -0600 Subject: [PATCH 4/4] feat(groups): drop built-in and pack-owned resources from export Whole-group export captured Cribl-shipped library content (lib == "cribl"), built-in system objects (destroyable false), and pack-owned items (id prefixed "pack:") because they appear in list responses. None are writable as standalone group config, so importing them produced hundreds of 4xx/5xx errors. Filter them at export so the payload is only user-authored config. Verified live: a real group's import failures dropped from ~300 to ~14 (the remainder being pack archives, which need the .crbl, and read-only system conditions). The dropped count is reported in _meta.skipped.builtin. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 3 ++ cribl_cli/utils/group_transfer.py | 48 +++++++++++++++++++++++++++++-- tests/unit/test_group_transfer.py | 33 +++++++++++++++++++++ 3 files changed, 82 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 679f3c2..3b169bb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -98,6 +98,9 @@ usage-groups, users, workspaces - `groups export`/`groups import` move a whole group's config; the resource list is derived from `commands/registry.py` group-scoped entries plus the hand-written sources/destinations/pipelines/packs/routes, so it never drifts. + Export drops Cribl-shipped built-ins (`lib == "cribl"` or `destroyable` false) + and pack-owned items (`id` prefixed `pack:`) — they list but 4xx/5xx on import, + so only user-authored config is kept; the count is reported in `_meta.skipped.builtin`. - When running CLI commands to read data, use default JSON output (no `--table`). JSON is structured and easier to parse. Only use `--table` if the user explicitly asks for it. diff --git a/cribl_cli/utils/group_transfer.py b/cribl_cli/utils/group_transfer.py index fb9b2f1..35c13ee 100644 --- a/cribl_cli/utils/group_transfer.py +++ b/cribl_cli/utils/group_transfer.py @@ -151,6 +151,42 @@ def resolve_group_meta(client: httpx.Client, group_id: str) -> dict[str, Any]: raise ValueError(f"Group or fleet '{group_id}' not found. Available: {available}") +def _is_builtin(item: dict[str, Any]) -> bool: + """True for content that lists but cannot be re-imported as standalone config. + + - Cribl-shipped library items carry ``lib == "cribl"`` (regex, parsers, + breakers, SDS rules, …); built-in system objects carry + ``destroyable == False`` (e.g. sample collectors). + - Pack-owned items have an ``id`` prefixed ``pack:`` — they ship with their + pack, not as loose resources, and 400/500 on a standalone write. + + Re-importing any of these only yields 4xx/5xx errors, so they are dropped + from an export — only user-authored, group-level config is kept. + """ + if item.get("lib") == "cribl" or item.get("destroyable") is False: + return True + rid = item.get("id") + return isinstance(rid, str) and rid.startswith("pack:") + + +def _drop_builtins(value: Any) -> tuple[Any, int]: + """Remove built-in items from a list payload, preserving its wrapper shape. + + Returns ``(filtered_value, dropped_count)``. + """ + if isinstance(value, dict) and "items" in value: + kept = [i for i in value["items"] if not (isinstance(i, dict) and _is_builtin(i))] + out = dict(value) + out["items"] = kept + if "count" in out: + out["count"] = len(kept) + return out, len(value["items"]) - len(kept) + if isinstance(value, list): + kept = [i for i in value if not (isinstance(i, dict) and _is_builtin(i))] + return kept, len(value) - len(kept) + return value, 0 + + def collect( client: httpx.Client, group_id: str, *, include_sensitive: bool = False, include_packs: bool = False, include_lookups: bool = False @@ -163,10 +199,14 @@ def collect( resources: dict[str, Any] = {} exported: list[str] = [] errors: dict[str, str] = {} + builtins: dict[str, int] = {} for name, cfg in included: try: - resources[name] = Endpoints(cfg).list(client, group_id) + filtered, dropped = _drop_builtins(Endpoints(cfg).list(client, group_id)) + resources[name] = filtered + if dropped: + builtins[name] = dropped exported.append(name) except httpx.HTTPError as exc: errors[name] = f"HTTP Error: {exc}" @@ -189,7 +229,7 @@ def collect( "resources": resources, "_meta": { "exported": exported, - "skipped": {**skipped, "binary_content": list(BINARY_CAVEAT)}, + "skipped": {**skipped, "builtin": builtins, "binary_content": list(BINARY_CAVEAT)}, "errors": errors, }, } @@ -405,6 +445,10 @@ def format_caveat(result: dict[str, Any]) -> str: lines.append(f" Excluded (default): {', '.join(skipped['omitted_by_default'])} (use --include-packs/--include-lookups)") if skipped.get("stream_only"): lines.append(f" Skipped (Stream-only, not on this fleet): {', '.join(skipped['stream_only'])}") + if skipped.get("builtin"): + b = skipped["builtin"] + detail = ", ".join(f"{name}×{count}" for name, count in b.items()) + lines.append(f" Dropped {sum(b.values())} built-in/pack-owned item(s) (not importable): {detail}") if skipped.get("binary_content"): lines.append(f" Not captured: {'; '.join(skipped['binary_content'])}") if m.get("errors"): diff --git a/tests/unit/test_group_transfer.py b/tests/unit/test_group_transfer.py index d7f2397..deaafb1 100644 --- a/tests/unit/test_group_transfer.py +++ b/tests/unit/test_group_transfer.py @@ -49,6 +49,39 @@ def test_collect_aggregates_and_handles_errors(): assert "parsers" in result["_meta"]["errors"] and "parsers" not in result["resources"] assert "packs" in result["resources"] +def test_collect_drops_builtin_items(): + """Cribl-shipped content (lib=='cribl' or destroyable is False) is filtered + from an export so import doesn't choke on non-writable built-ins.""" + client = MagicMock(spec=httpx.Client) + + def _get(url, *args, **kwargs): + if url == "/api/v1/master/groups": + return _mock_response({"items": [{"id": "grp", "type": "stream", "isFleet": False}]}) + if url.endswith("/routes"): + return _mock_response({"id": "default", "items": [{"id": "r1"}]}) + if url.endswith("/lib/parsers"): + return _mock_response({"count": 4, "items": [ + {"id": "builtin", "lib": "cribl"}, # Cribl-shipped -> dropped + {"id": "pack:foo"}, # pack-owned -> dropped + {"id": "mine", "lib": "custom"}, + {"id": "also-mine"}, # no lib -> user content + ]}) + if url.endswith("/collectors"): + return _mock_response({"items": [ + {"id": "sys", "destroyable": False}, # built-in system object -> dropped + {"id": "user", "destroyable": True}, + ]}) + return _mock_response({"items": []}) + + client.get.side_effect = _get + result = collect(client, "grp") + + parsers = result["resources"]["parsers"] + assert {p["id"] for p in parsers["items"]} == {"mine", "also-mine"} + assert parsers["count"] == 2 # wrapper count is corrected + assert {c["id"] for c in result["resources"]["collectors"]["items"]} == {"user"} + assert result["_meta"]["skipped"]["builtin"] == {"parsers": 2, "collectors": 1} + def test_write_read_roundtrip(tmp_path): result = { "group": "grp", "type": "stream", "isFleet": False,