From 98d3092d5f18df2048d2cc47613436541736cd53 Mon Sep 17 00:00:00 2001 From: Matthew Andersen Date: Wed, 13 May 2026 19:21:48 -0700 Subject: [PATCH 1/5] bug fix for container startup --- main.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 9637535..bec17f3 100644 --- a/main.py +++ b/main.py @@ -118,13 +118,19 @@ def main(): threading.Thread(target=periodic_update_loop, args=(docker_host,), daemon=True).start() for event in client.events(decode=True): + if event.get("Type") != "container": + continue action = event.get("Action") if action not in WATCHED_DOCKER_ACTIONS: continue - container_id = event.get("id") + container_id = event.get("Actor", {}).get("ID") or event.get("id") + if not container_id: + continue try: container = client.containers.get(container_id) handle_container_event(container, docker_host, action=action) + except docker.errors.NotFound: + pass except Exception as e: logger.error(f"Failed to handle {action} event for {container_id}: {e}") From 83d3ee1f4f863544913a04d509e31e4612768299 Mon Sep 17 00:00:00 2001 From: Matthew Andersen Date: Wed, 13 May 2026 19:23:11 -0700 Subject: [PATCH 2/5] changelog update --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d23e81..32c6714 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- Docker event loop no longer logs `Failed to handle event for + None: Resource ID was not provided` errors. The container ID is now + read from `event["Actor"]["ID"]` (newer Docker daemons no longer + populate the legacy top-level `id` field), non-container events + (network/volume/service) are filtered out before lookup, and the + expected `NotFound` raised when querying an already-removed container + on a `destroy` event is swallowed silently. + --- ## [0.3.0] — 2026-05-12 From d4e825212c926b3d6f7ddc60f911a2f31a2b048c Mon Sep 17 00:00:00 2001 From: Matthew Andersen Date: Wed, 13 May 2026 23:09:00 -0700 Subject: [PATCH 3/5] updates --- CHANGELOG.md | 14 +++++++ README.md | 9 +++-- docs/PRD.md | 107 ++++++++++++++++++++++++++++++++++++++++++++------- main.py | 54 +++++++++++++++++++++++--- 4 files changed, 161 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32c6714..3f0a7f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- New env var `STD_REPORT_ALL_CONTAINERS`. When set to a truthy value + (`true`, `1`, `yes` — case-insensitive), the notifier reports every + running container on the host to STD, regardless of whether the + container has the `dockernotifier.notifiers=service-tracker-dashboard` + opt-in label. Default off — existing per-container opt-in behavior + is unchanged for operators who don't set the variable. **Only STD + dispatch is affected**: the DNS notifier still requires explicit + per-container opt-in via labels, because DNS records are an external + side effect that shouldn't fire for containers that didn't ask. + Unrecognized values (e.g. `maybe`) log a warning at startup and are + treated as off. The wire payload to STD is identical whether the + trigger came from a label or from this env var. + ### Fixed - Docker event loop no longer logs `Failed to handle event for None: Resource ID was not provided` errors. The container ID is now diff --git a/README.md b/README.md index 91a96bf..415b1f0 100644 --- a/README.md +++ b/README.md @@ -81,10 +81,11 @@ notifier targets can be added without touching the core event loop. > canonical schema. Earlier STD versions do not expose that endpoint > and will return 404. -| Variable | Required | Description | -|-------------------|----------|-------------| -| `STD_URL` | Yes (for STD) | Base URL of the STD instance, e.g. `http://std.example.com:8815`. | -| `STD_API_TOKEN` | Yes (for STD) | Bearer token configured on the STD side. | +| Variable | Required | Default | Description | +|------------------------------|----------|---------|-------------| +| `STD_URL` | Yes (for STD) | — | Base URL of the STD instance, e.g. `http://std.example.com:8815`. | +| `STD_API_TOKEN` | Yes (for STD) | — | Bearer token configured on the STD side. | +| `STD_REPORT_ALL_CONTAINERS` | No | `false` | When truthy (`true`, `1`, `yes` — case-insensitive), report **every running container on this host** to STD regardless of whether it has the `dockernotifier.notifiers=service-tracker-dashboard` opt-in label. Default off preserves per-container opt-in behavior. **Only affects STD** — the DNS notifier still requires explicit per-container opt-in via labels. Unrecognized values log a warning at startup and are treated as off. | If a notifier's required env vars are missing, that notifier silently no-ops — the container won't fail to start. This is intentional so you diff --git a/docs/PRD.md b/docs/PRD.md index 3bdbee6..602a792 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -9,6 +9,7 @@ | Version | Date | Changes | |---------|------------|---------| | 0.1 | 2026-05-10 | Initial PRD. Documents current shipped behavior at v0.2.3 and the planned v0.3.0 cleanup. | +| 0.2 | 2026-05-13 | v0.3.1 — STD reporting opt-out mode via `STD_REPORT_ALL_CONTAINERS` env var. §1.3 softened to reflect per-host opt-out scope. | --- @@ -18,11 +19,12 @@ 2. [Scope](#2-scope) 3. [Architecture](#3-architecture) 4. [Configuration Model](#4-configuration-model) -5. [Current State (v0.2.3)](#5-current-state-v023) +5. [Current State (v0.3.0)](#5-current-state-v030) 6. [v0.3.0 — Cleanup Release](#6-v030--cleanup-release) -7. [Versioning, Branches, and Releases](#7-versioning-branches-and-releases) -8. [Cross-Repo Coordination](#8-cross-repo-coordination) -9. [Open Questions](#9-open-questions) +7. [v0.3.1 — STD Reporting Opt-Out Mode](#7-v031--std-reporting-opt-out-mode) +8. [Versioning, Branches, and Releases](#8-versioning-branches-and-releases) +9. [Cross-Repo Coordination](#9-cross-repo-coordination) +10. [Open Questions](#10-open-questions) --- @@ -49,8 +51,13 @@ independently against current container metadata. ### 1.3 Design principles -- **Opt-in per container.** No labels means no notification. Run safely - alongside containers that don't know or care about this notifier. +- **Opt-in per container by default.** No labels means no notification — + run safely alongside containers that don't know or care about this + notifier. STD reporting can be flipped to opt-out on a per-host basis + via the `STD_REPORT_ALL_CONTAINERS` env var (see §7); other notifier + targets remain per-container opt-in regardless, because they create + external side effects (DNS records, etc.) that should not fire for + containers that didn't ask. - **Independent notifier modules.** Each downstream system is its own module under `notifiers/` with its own auth, retry, and payload shape. - **No state.** No database, no cache, no queue. Everything is derived @@ -233,9 +240,11 @@ Environment variables are documented in the `README.md`. --- -## 5. Current State (v0.2.3) +## 5. Current State (v0.3.0) -Tags shipped on `main`: v0.1.0 → v0.2.3. +Tags shipped on `main`: v0.1.0 → v0.3.0 (v0.3.0 released 2026-05-12). +All issues listed in §5.2 below were resolved in v0.3.0. The next +release in flight is v0.3.1 (see §7). ### 5.1 What works today @@ -308,11 +317,81 @@ window. these easier later, but none ship in v0.3.0. - Multi-host coordination. - A config file. Env vars + labels remain the only inputs. -- Test suite. Worth doing eventually (see §9), not in v0.3.0. +- Test suite. Worth doing eventually (see §10), not in v0.3.0. --- -## 7. Versioning, Branches, and Releases +## 7. v0.3.1 — STD Reporting Opt-Out Mode + +A small, additive release that introduces a single env var, +`STD_REPORT_ALL_CONTAINERS`, which flips STD reporting from +per-container opt-in to per-host opt-out. + +### 7.1 Goals + +- Let operators who want a complete inventory of a host's running + containers in STD avoid labelling every container individually. +- Preserve today's behavior as the default: unset env var → unchanged. + +### 7.2 Scope + +- **STD only.** The env var affects only the STD notifier dispatch. + The DNS notifier (and any future notifier that creates external + side effects) continues to require explicit per-container opt-in + via `dockernotifier.notifiers=...`. +- **Per-host, not per-container.** The env var is read once at + startup on each notifier instance. There is intentionally no + per-container override label — that would defeat the purpose. +- **Running containers only.** Behavior matches existing dispatch: + the boot pass and periodic refresh loop iterate + `client.containers.list()`. Stopped containers are not retroactively + reported. + +### 7.3 Semantics + +- `STD_REPORT_ALL_CONTAINERS` truthy values: `true`, `1`, `yes` + (case-insensitive). Anything else (including unrecognized strings + like `maybe`) is treated as off; unrecognized values log a single + warning at startup. +- When on, every running container on the host is reported to STD + on boot, on watched Docker events, and on each periodic refresh + tick — regardless of whether the container's + `dockernotifier.notifiers` label includes `service-tracker-dashboard` + (or even exists at all). +- `dockernotifier.std.*` labels on individual containers are still + honored. Containers without those labels are reported with the + minimum information available; STD's wire contract makes most + fields optional and applies its own defaults. +- A container with `dockernotifier.notifiers=dns` only (no STD opt-in) + AND the env var set: STD fires (env-var path) **and** DNS fires + (label path). The env var adds STD; it does not subtract anything. + +### 7.4 Wire contract + +Unchanged. STD receives identical `/api/v1/register` payloads +regardless of whether the trigger came from a label or from the env +var. STD has no way to distinguish the two paths and does not need to. + +### 7.5 Design principle reconciliation + +§1.3's "opt-in per container" principle is softened, not abandoned. +With the env var unset (default), per-container opt-in remains the +only path. The env var is an explicit, deliberate per-host stance +taken by the operator running the notifier instance — it does not +change what other operators experience. + +### 7.6 Out of scope for v0.3.1 + +- Per-container opt-out (a label like `dockernotifier.std.skip=true` + to suppress reporting even when the env var is set). Defer until + there is a concrete request. +- Changing DNS opt-in semantics. +- Reporting non-running containers. +- Network/port capture (planned for v0.3.2). + +--- + +## 8. Versioning, Branches, and Releases - `main` is the default branch and the source of truth for releases. - All work happens on `dev`. PR `dev` → `main` when ready to release. @@ -326,18 +405,18 @@ window. --- -## 8. Cross-Repo Coordination +## 9. Cross-Repo Coordination This project is paired with [service-tracker-dashboard](https://github.com/crzykidd/service-tracker-dashboard). -### 8.1 Contract ownership +### 9.1 Contract ownership STD owns the wire contract for the register endpoint. The notifier is a producer — it sends what STD documents. Wire-format changes start in STD; the notifier follows. -### 8.2 Release ordering for the v0.5.0 / v0.3.0 cycle +### 9.2 Release ordering for the v0.5.0 / v0.3.0 cycle 1. STD v0.5.0 ships with `/api/v1/register` (canonical keys) and the compat shim on `/api/register` (legacy keys, deprecated). @@ -349,7 +428,7 @@ Operators must upgrade the notifier to v0.3.0+ before STD v0.6.0. --- -## 9. Open Questions +## 10. Open Questions - **Test coverage.** No tests exist today. Worth investing in a small suite that fakes the Docker client and asserts dispatch behavior? diff --git a/main.py b/main.py index bec17f3..b18b11f 100644 --- a/main.py +++ b/main.py @@ -12,6 +12,34 @@ logger.debug("main.py is running") STD_REFRESH_SECONDS = int(os.environ.get("STD_REFRESH_SECONDS", "60")) # Default to 60 seconds + +def _parse_bool_env(name, default=False): + raw = os.environ.get(name) + if raw is None: + return default + normalized = raw.strip().lower() + if normalized in ("true", "1", "yes"): + return True + if normalized in ("false", "0", "no", ""): + return False + logger.warning( + f"Unrecognized boolean value for {name}={raw!r}; treating as off" + ) + return False + + +# When True, the STD notifier fires for every running container on this +# host regardless of whether the container has the +# `dockernotifier.notifiers=service-tracker-dashboard` opt-in label. +# Scope is intentionally limited to STD — other notifiers (DNS) still +# require explicit per-container opt-in. +STD_REPORT_ALL_CONTAINERS = _parse_bool_env("STD_REPORT_ALL_CONTAINERS") +if STD_REPORT_ALL_CONTAINERS: + logger.info( + "STD_REPORT_ALL_CONTAINERS is on — every running container on this host " + "will be reported to STD regardless of opt-in label" + ) + # Real Docker events the notifier subscribes to. WATCHED_DOCKER_ACTIONS = frozenset({ "start", "stop", "die", "pause", "unpause", @@ -54,12 +82,23 @@ def get_host_name(): def handle_container_event(container, docker_host, action): - labels = container.attrs["Config"]["Labels"] + labels = container.attrs["Config"]["Labels"] or {} notifier_list_raw = labels.get("dockernotifier.notifiers", "").strip() - if not notifier_list_raw: - return notifier_list = [n.strip() for n in notifier_list_raw.split(",") if n.strip()] + std_via_label = "service-tracker-dashboard" in notifier_list + std_via_env = STD_REPORT_ALL_CONTAINERS + std_should_fire = ( + (std_via_label or std_via_env) + and action in NOTIFIER_TRIGGERS["service-tracker-dashboard"] + ) + dns_should_fire = ( + "dns" in notifier_list and action in NOTIFIER_TRIGGERS["dns"] + ) + + if not std_should_fire and not dns_should_fire: + return + base_kwargs = { "container_name": container.name, "container_id": container.id, @@ -73,7 +112,7 @@ def handle_container_event(container, docker_host, action): logger.info(f"[MATCH] Container {action.upper()}: {container.name}") - if action in NOTIFIER_TRIGGERS["dns"] and "dns" in notifier_list: + if dns_should_fire: container_hostname = labels.get("dockernotifier.dns.containerhostname") zone_label = labels.get("dockernotifier.dns.containerzone") docker_domain = labels.get("dockernotifier.dns.dockerdomain") @@ -94,7 +133,12 @@ def handle_container_event(container, docker_host, action): else: logger.warning(f"Missing DNS label info for {container.name}, skipping DNS registration") - if "service-tracker-dashboard" in notifier_list and action in NOTIFIER_TRIGGERS["service-tracker-dashboard"]: + if std_should_fire: + if std_via_env and not std_via_label: + logger.debug( + f"STD notifier firing for {container.name} via " + f"STD_REPORT_ALL_CONTAINERS (no opt-in label)" + ) logger.info(f"STD notifier triggered for {container.name} on {action}") std_extras = { key.replace("dockernotifier.std.", ""): value From 5af7f4f9e680b11eef12f4d490eb7392378ccb6b Mon Sep 17 00:00:00 2001 From: Matthew Andersen Date: Wed, 13 May 2026 23:27:39 -0700 Subject: [PATCH 4/5] feat: capture container networks and ports for STD (v0.3.2) Reads networks, exposed_ports, and published_ports off container.attrs and forwards them in the STD register payload. Empty values are explicit empty lists so STD can distinguish "nothing to report" from "not yet reported". Requires STD v0.6.0. --- CHANGELOG.md | 11 ++ README.md | 12 +++ docs/PRD.md | 139 ++++++++++++++++++++++--- main.py | 51 +++++++++ notifiers/service_tracker_dashboard.py | 3 + 5 files changed, 202 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f0a7f4..dfb6dad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Unrecognized values (e.g. `maybe`) log a warning at startup and are treated as off. The wire payload to STD is identical whether the trigger came from a label or from this env var. +- STD payloads now include `networks`, `exposed_ports`, and + `published_ports` read directly from the Docker API. `networks` is a + list of `{"name", "aliases"}` objects, one per Docker network the + container is on. `exposed_ports` is a list of `"/"` + strings from the container's `ExposedPorts` config. `published_ports` + is a list of `{"container_port", "protocol", "host_ip", "host_port"}` + objects, one per port binding. Empty values are emitted as explicit + empty lists so STD can distinguish "nothing to report" from "not yet + reported". No new env vars or labels — capture is automatic for every + container reported to STD. **Requires STD v0.6.0 or later**; STD + v0.5.x's strict validator will reject payloads carrying these keys. ### Fixed - Docker event loop no longer logs `Failed to handle event for diff --git a/README.md b/README.md index 415b1f0..a0cad85 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,12 @@ notifier targets can be added without touching the core event loop. > notifier posts to STD's `/api/v1/register` endpoint using STD's > canonical schema. Earlier STD versions do not expose that endpoint > and will return 404. +> +> **Notifier v0.3.2 requires STD v0.6.0 or later.** Starting in v0.3.2 +> the notifier emits `networks`, `exposed_ports`, and `published_ports` +> on every STD payload. STD v0.5.x's strict pydantic validator rejects +> unknown keys and will return 422 for these payloads — upgrade STD +> first. | Variable | Required | Default | Description | |------------------------------|----------|---------|-------------| @@ -139,6 +145,12 @@ applies its own defaults for anything you don't. > coercion happens at the same boundary, so the values STD receives > are actual `bool`/`int` rather than strings. +> **Network and port data.** As of notifier v0.3.2, every STD payload +> also carries `networks`, `exposed_ports`, and `published_ports` +> read straight from the Docker API. No new labels or env vars are +> required to enable this — it is automatic for every container +> reported to STD. Requires STD v0.6.0+. + --- ## Docker Compose Example diff --git a/docs/PRD.md b/docs/PRD.md index 602a792..88e647b 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -10,6 +10,7 @@ |---------|------------|---------| | 0.1 | 2026-05-10 | Initial PRD. Documents current shipped behavior at v0.2.3 and the planned v0.3.0 cleanup. | | 0.2 | 2026-05-13 | v0.3.1 — STD reporting opt-out mode via `STD_REPORT_ALL_CONTAINERS` env var. §1.3 softened to reflect per-host opt-out scope. | +| 0.3 | 2026-05-13 | v0.3.2 — capture container network membership and port information from the Docker API and forward to STD. §3.3 base kwargs contract grows three rows (`networks`, `exposed_ports`, `published_ports`). | --- @@ -19,12 +20,13 @@ 2. [Scope](#2-scope) 3. [Architecture](#3-architecture) 4. [Configuration Model](#4-configuration-model) -5. [Current State (v0.3.0)](#5-current-state-v030) +5. [Current State (v0.3.1)](#5-current-state-v031) 6. [v0.3.0 — Cleanup Release](#6-v030--cleanup-release) 7. [v0.3.1 — STD Reporting Opt-Out Mode](#7-v031--std-reporting-opt-out-mode) -8. [Versioning, Branches, and Releases](#8-versioning-branches-and-releases) -9. [Cross-Repo Coordination](#9-cross-repo-coordination) -10. [Open Questions](#10-open-questions) +8. [v0.3.2 — Network & Port Capture](#8-v032--network--port-capture) +9. [Versioning, Branches, and Releases](#9-versioning-branches-and-releases) +10. [Cross-Repo Coordination](#10-cross-repo-coordination) +11. [Open Questions](#11-open-questions) --- @@ -184,6 +186,15 @@ arguments guaranteed present: | `stack_name` | Optional[str] | `com.docker.compose.project` label or `None` | | `started_at` | str | ISO timestamp from container state | | `action` | str | The action that triggered this call (e.g. "start", "boot", "refresh") | +| `networks` | list[dict] | One entry per Docker network the container is on: `{"name": str, "aliases": [str, ...]}`. Empty list if the container is on no networks. Added in v0.3.2. | +| `exposed_ports` | list[str] | Container's `ExposedPorts` config as a list of `"/"` strings (e.g. `"5173/tcp"`). Empty list if none. Added in v0.3.2. | +| `published_ports` | list[dict] | One entry per `(container_port, host_port)` mapping: `{"container_port": int, "protocol": str, "host_ip": str, "host_port": int}`. Empty list if no published ports. Added in v0.3.2. | + +The last three (`networks`, `exposed_ports`, `published_ports`) live +in `base_kwargs` because they are inherent container facts read off +the Docker API, not per-target extras derived from labels. STD is +the only consumer today; other notifiers receive them via +`**kwargs` and may ignore them. Modules may additionally receive notifier-specific extras (typically from stripped label namespaces). A module reading any extra should @@ -240,11 +251,13 @@ Environment variables are documented in the `README.md`. --- -## 5. Current State (v0.3.0) +## 5. Current State (v0.3.1) -Tags shipped on `main`: v0.1.0 → v0.3.0 (v0.3.0 released 2026-05-12). -All issues listed in §5.2 below were resolved in v0.3.0. The next -release in flight is v0.3.1 (see §7). +Tags shipped on `main`: v0.1.0 → v0.3.1. v0.3.0 (2026-05-12) resolved +every issue listed in §5.2 below. v0.3.1 followed as a small additive +release introducing the `STD_REPORT_ALL_CONTAINERS` env var for +per-host opt-out reporting to STD (see §7). The next release in +flight is v0.3.2 — network & port capture (see §8). ### 5.1 What works today @@ -317,7 +330,7 @@ window. these easier later, but none ship in v0.3.0. - Multi-host coordination. - A config file. Env vars + labels remain the only inputs. -- Test suite. Worth doing eventually (see §10), not in v0.3.0. +- Test suite. Worth doing eventually (see §11), not in v0.3.0. --- @@ -391,7 +404,95 @@ change what other operators experience. --- -## 8. Versioning, Branches, and Releases +## 8. v0.3.2 — Network & Port Capture + +An additive release that forwards container network membership and +port information from the Docker API to STD. STD v0.6.0 consumes +these fields; STD v0.5.x will reject the payload (strict pydantic +validation), so STD v0.6.0 must be deployed first. + +### 8.1 Goals + +- Capture inherent container facts the notifier already has access to + via the Docker API: which networks the container is on, what ports + it exposes, and what ports it publishes to the host. +- Forward all three to STD as canonical fields so STD's UI can render + badges/links without re-reading the Docker socket itself. +- Stay pure-capture. No interpretation, no derived semantics. STD + v0.7.0 will layer interpretation on top. + +### 8.2 Captured fields + +Added to the base kwargs contract (see §3.3 table): + +- `networks` — list of `{"name": str, "aliases": [str, ...]}`. One + entry per Docker network the container is on. Read from + `container.attrs["NetworkSettings"]["Networks"]`. Aliases is an + empty list (not null) when a network has no aliases. +- `exposed_ports` — list of `"/"` strings. Read from + `container.attrs["Config"]["ExposedPorts"]`. Just the keys. +- `published_ports` — list of `{"container_port": int, "protocol": + str, "host_ip": str, "host_port": int}`. One entry per + `(container_port, host_port)` binding. Read from + `container.attrs["NetworkSettings"]["Ports"]`. Entries with a null + binding list (exposed-but-not-published) are skipped. + +### 8.3 Coercion at the boundary + +- `host_port` arrives from Docker as a string (`"5173"`); coerced to + `int`. +- `container_port` is parsed from the `"/"` key and + cast to `int`. +- `protocol` is the string after the slash (typically `"tcp"` or + `"udp"`), kept as-is. + +### 8.4 Empty vs. missing + +Empty values are emitted as explicit empty lists, not null. This +lets STD's UI distinguish "we know there's nothing" from "the +notifier hasn't reported yet" (where the field is absent / null). + +- Container on no networks (`network_mode: none`): `networks: []`. +- Container with no exposed ports: `exposed_ports: []`. +- Container with no published ports: `published_ports: []`. + +### 8.5 Wire contract + +Three new fields on the canonical payload to `/api/v1/register`. +The STD notifier's `_PASSTHROUGH` set covers them; no translation +needed because they are already in canonical shape. + +### 8.6 Scope of consumption + +Only the STD notifier consumes these fields today. The DNS notifier +receives them through `**kwargs` and ignores them. Storing them in +`base_kwargs` rather than as STD-specific extras keeps them +available for any future notifier (e.g. a Traefik-config emitter) +without rerunning the Docker API call. + +### 8.7 Out of scope for v0.3.2 + +- Network IPs, gateways, MAC addresses, or any per-network detail + beyond name and aliases. +- Any interpretation of network names (e.g. "container on `proxy` + network → mark as Traefik-exposed"). That is v0.4.0 interpreter + work. +- Per-host configuration of which networks to report. All networks + the container is on get reported. +- Filtering or redaction. If a future operator wants it, that's a + separate feature. +- Sending this data to anywhere besides STD. + +### 8.8 Dependencies + +- Hard: STD v0.6.0 must be deployed first. v0.5.x rejects unknown + keys. +- Soft: notifier v0.3.1 on `main` (clean version sequencing only — + v0.3.2 does not depend on v0.3.1's behavior). + +--- + +## 9. Versioning, Branches, and Releases - `main` is the default branch and the source of truth for releases. - All work happens on `dev`. PR `dev` → `main` when ready to release. @@ -405,18 +506,18 @@ change what other operators experience. --- -## 9. Cross-Repo Coordination +## 10. Cross-Repo Coordination This project is paired with [service-tracker-dashboard](https://github.com/crzykidd/service-tracker-dashboard). -### 9.1 Contract ownership +### 10.1 Contract ownership STD owns the wire contract for the register endpoint. The notifier is a producer — it sends what STD documents. Wire-format changes start in STD; the notifier follows. -### 9.2 Release ordering for the v0.5.0 / v0.3.0 cycle +### 10.2 Release ordering for the v0.5.0 / v0.3.0 cycle 1. STD v0.5.0 ships with `/api/v1/register` (canonical keys) and the compat shim on `/api/register` (legacy keys, deprecated). @@ -426,9 +527,19 @@ STD; the notifier follows. Operators must upgrade the notifier to v0.3.0+ before STD v0.6.0. +### 10.3 Release ordering for the v0.6.0 / v0.3.2 cycle + +1. STD v0.6.0 ships with `networks`, `exposed_ports`, and + `published_ports` accepted on `/api/v1/register`. +2. Notifier v0.3.2 ships emitting those three fields. + +If notifier v0.3.2 is deployed against STD v0.5.x, STD's strict +pydantic validator rejects the payload (unknown keys). Operators +must upgrade STD before the notifier. + --- -## 10. Open Questions +## 11. Open Questions - **Test coverage.** No tests exist today. Worth investing in a small suite that fakes the Docker client and asserts dispatch behavior? diff --git a/main.py b/main.py index b18b11f..f15d3dc 100644 --- a/main.py +++ b/main.py @@ -81,6 +81,54 @@ def get_host_name(): return os.uname()[1] +def _extract_networks(container_attrs): + network_settings = container_attrs.get("NetworkSettings") or {} + networks_raw = network_settings.get("Networks") or {} + return [ + {"name": name, "aliases": (data.get("Aliases") or []) if isinstance(data, dict) else []} + for name, data in networks_raw.items() + ] + + +def _extract_exposed_ports(container_attrs): + config = container_attrs.get("Config") or {} + exposed = config.get("ExposedPorts") or {} + return list(exposed.keys()) + + +def _extract_published_ports(container_attrs): + network_settings = container_attrs.get("NetworkSettings") or {} + ports_raw = network_settings.get("Ports") or {} + out = [] + for port_key, bindings in ports_raw.items(): + if not bindings: + continue + try: + container_port_str, protocol = port_key.split("/", 1) + container_port = int(container_port_str) + except (ValueError, AttributeError): + logger.debug(f"Skipping malformed port key {port_key!r}") + continue + for binding in bindings: + if not isinstance(binding, dict): + continue + host_port_raw = binding.get("HostPort") + try: + host_port = int(host_port_raw) + except (TypeError, ValueError): + logger.debug( + f"Skipping binding with non-integer HostPort={host_port_raw!r} for {port_key}" + ) + continue + out.append({ + "container_port": container_port, + "protocol": protocol, + "host_ip": binding.get("HostIp", "") or "", + "host_port": host_port, + }) + return out + + def handle_container_event(container, docker_host, action): labels = container.attrs["Config"]["Labels"] or {} notifier_list_raw = labels.get("dockernotifier.notifiers", "").strip() @@ -108,6 +156,9 @@ def handle_container_event(container, docker_host, action): "stack_name": labels.get("com.docker.compose.project"), "started_at": container.attrs["State"]["StartedAt"], "action": action, + "networks": _extract_networks(container.attrs), + "exposed_ports": _extract_exposed_ports(container.attrs), + "published_ports": _extract_published_ports(container.attrs), } logger.info(f"[MATCH] Container {action.upper()}: {container.name}") diff --git a/notifiers/service_tracker_dashboard.py b/notifiers/service_tracker_dashboard.py index 89111e5..927a869 100644 --- a/notifiers/service_tracker_dashboard.py +++ b/notifiers/service_tracker_dashboard.py @@ -29,6 +29,9 @@ "container_name", "container_id", "docker_status", "stack_name", "started_at", "image_name", "internalurl", "externalurl", "timestamp", + # v0.3.2: network/port capture fields. Already in canonical shape; + # consumed by STD v0.6.0+. + "networks", "exposed_ports", "published_ports", } # Keys in the canonical schema that need type coercion from string. From d3a7a02728d7e56c57a3abe79e2e42cff8a19643 Mon Sep 17 00:00:00 2001 From: Matthew Andersen Date: Thu, 14 May 2026 00:06:18 -0700 Subject: [PATCH 5/5] updates for v0.4.0 release --- CHANGELOG.md | 36 +- Dockerfile | 3 +- README.md | 92 +++- docs/PRD.md | 482 +++++++++++++++------ docs/community-interpreters/README.md | 92 ++++ docs/community-interpreters/dockflare.yml | 26 ++ docs/community-interpreters/template.yml | 77 ++++ docs/community-interpreters/traefik.yml | 27 ++ interpreter_loader.py | 487 ++++++++++++++++++++++ interpreters/builtin/dockflare.yml | 26 ++ interpreters/builtin/traefik.yml | 27 ++ main.py | 28 ++ notifiers/service_tracker_dashboard.py | 6 + requirements.txt | 1 + 14 files changed, 1265 insertions(+), 145 deletions(-) create mode 100644 docs/community-interpreters/README.md create mode 100644 docs/community-interpreters/dockflare.yml create mode 100644 docs/community-interpreters/template.yml create mode 100644 docs/community-interpreters/traefik.yml create mode 100644 interpreter_loader.py create mode 100644 interpreters/builtin/dockflare.yml create mode 100644 interpreters/builtin/traefik.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index dfb6dad..969f247 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.4.0] — 2026-05-14 + ### Added +- **YAML interpreter mechanism (v0.4.0).** The notifier now reads small + YAML files that teach it how to interpret labels written by + third-party tools (Traefik, Dockflare, ...) and forward them to STD + as structured exposure observations on a new + `exposure_observations` field. Two interpreters ship baked into the + image (`traefik.yml`, `dockflare.yml`) and fire automatically for + any container reported to STD — no opt-in label required. Operators + who want to extend or override the built-ins mount a directory of + their own YAMLs at `/app/interpreters/user/`; a user file whose + `name:` matches a builtin overrides the builtin. Bad YAMLs log a + warning and are skipped; the notifier continues with whatever + loaded successfully. A new `docs/community-interpreters/` directory + holds contributed reference YAMLs (PRs welcome — they are examples, + not curated products). Full format reference is in `docs/PRD.md` + §11. **Requires STD v0.6.0 or later**; earlier STD versions reject + the unknown `exposure_observations` key. +- Debug-only env var `INTERPRETER_RELOAD_ON_EACH_EVENT`. When truthy, + re-reads the YAML interpreter directories on every dispatch instead + of once at startup. For iterating on a YAML; not for production + use. - New env var `STD_REPORT_ALL_CONTAINERS`. When set to a truthy value (`true`, `1`, `yes` — case-insensitive), the notifier reports every running container on the host to STD, regardless of whether the @@ -32,6 +54,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 container reported to STD. **Requires STD v0.6.0 or later**; STD v0.5.x's strict validator will reject payloads carrying these keys. +### Changed +- **Design principles softened (v0.4.0).** Previously, PRD §1.3 stated + "no state" and "all configuration is via environment variables. + There is no config file." Both are now narrower. The notifier still + holds no per-event state and still has no central config file, but + it now loads YAML interpreter files at startup. Operators upgrading + from v0.3.x do not need to take any action — the YAML loader works + out of the box with the two baked-in interpreters, and the + filesystem footprint is fully contained inside the notifier + container. + ### Fixed - Docker event loop no longer logs `Failed to handle event for None: Resource ID was not provided` errors. The container ID is now @@ -135,7 +168,8 @@ Released. Initial public release. -[Unreleased]: https://github.com/crzykidd/docker-api-notifier/compare/v0.3.0...HEAD +[Unreleased]: https://github.com/crzykidd/docker-api-notifier/compare/v0.4.0...HEAD +[0.4.0]: https://github.com/crzykidd/docker-api-notifier/releases/tag/v0.4.0 [0.3.0]: https://github.com/crzykidd/docker-api-notifier/releases/tag/v0.3.0 [0.2.3]: https://github.com/crzykidd/docker-api-notifier/releases/tag/v0.2.3 [0.2.2]: https://github.com/crzykidd/docker-api-notifier/releases/tag/v0.2.2 diff --git a/Dockerfile b/Dockerfile index 791153c..7ab4ee8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,7 @@ FROM python:3.11-slim WORKDIR /app COPY . /app -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir -r requirements.txt \ + && mkdir -p /app/interpreters/user ENV PYTHONUNBUFFERED=1 CMD ["python", "main.py"] \ No newline at end of file diff --git a/README.md b/README.md index a0cad85..f4a531c 100644 --- a/README.md +++ b/README.md @@ -26,9 +26,10 @@ every Docker host without it touching things you didn't ask it to touch. 1. [What It Does](#what-it-does) 2. [Environment Variables](#environment-variables) 3. [Container Labels](#container-labels) -4. [Docker Compose Example](#docker-compose-example) -5. [How It Works](#how-it-works) -6. [Building Locally](#building-locally) +4. [Interpreters](#interpreters) +5. [Docker Compose Example](#docker-compose-example) +6. [How It Works](#how-it-works) +7. [Building Locally](#building-locally) --- @@ -81,17 +82,18 @@ notifier targets can be added without touching the core event loop. > canonical schema. Earlier STD versions do not expose that endpoint > and will return 404. > -> **Notifier v0.3.2 requires STD v0.6.0 or later.** Starting in v0.3.2 -> the notifier emits `networks`, `exposed_ports`, and `published_ports` -> on every STD payload. STD v0.5.x's strict pydantic validator rejects -> unknown keys and will return 422 for these payloads — upgrade STD -> first. +> **Notifier v0.4.0 requires STD v0.6.0 or later.** Starting in v0.4.0 +> the notifier emits `networks`, `exposed_ports`, `published_ports`, +> and `exposure_observations` on every STD payload. STD v0.5.x's +> strict pydantic validator rejects unknown keys and will return 422 +> for these payloads — upgrade STD first. | Variable | Required | Default | Description | |------------------------------|----------|---------|-------------| | `STD_URL` | Yes (for STD) | — | Base URL of the STD instance, e.g. `http://std.example.com:8815`. | | `STD_API_TOKEN` | Yes (for STD) | — | Bearer token configured on the STD side. | | `STD_REPORT_ALL_CONTAINERS` | No | `false` | When truthy (`true`, `1`, `yes` — case-insensitive), report **every running container on this host** to STD regardless of whether it has the `dockernotifier.notifiers=service-tracker-dashboard` opt-in label. Default off preserves per-container opt-in behavior. **Only affects STD** — the DNS notifier still requires explicit per-container opt-in via labels. Unrecognized values log a warning at startup and are treated as off. | +| `INTERPRETER_RELOAD_ON_EACH_EVENT` | No | `false` | Debug-only. When truthy, re-reads YAML interpreters from disk on every dispatch instead of once at startup. Use while iterating on a new YAML; do not leave on in production. | If a notifier's required env vars are missing, that notifier silently no-ops — the container won't fail to start. This is intentional so you @@ -145,7 +147,7 @@ applies its own defaults for anything you don't. > coercion happens at the same boundary, so the values STD receives > are actual `bool`/`int` rather than strings. -> **Network and port data.** As of notifier v0.3.2, every STD payload +> **Network and port data.** As of notifier v0.4.0, every STD payload > also carries `networks`, `exposed_ports`, and `published_ports` > read straight from the Docker API. No new labels or env vars are > required to enable this — it is automatic for every container @@ -153,6 +155,73 @@ applies its own defaults for anything you don't. --- +## Interpreters + +> **New in v0.4.0. Requires STD v0.6.0 or later.** STD v0.5.x's strict +> validator will reject payloads carrying `exposure_observations`. + +Interpreters are small YAML files that teach the notifier how to read +labels written by third-party tools (Traefik, Dockflare, Caddy, ...) +and forward them to STD as structured **exposure observations**. The +goal is to stop operators from having to duplicate hostnames into +`dockernotifier.std.internalurl` when the same fact is already +encoded in their Traefik/Dockflare labels. + +### What ships built in + +Two interpreters live inside the container image at +`/app/interpreters/builtin/`: + +- `traefik.yml` — reads `traefik.http.routers..rule` + (hostname), `.tls`, and `.entrypoints`. Emits one observation per + router on the container. +- `dockflare.yml` — fires when `dockflare.enable=true`; reads + `dockflare.hostname` and optional Access policy labels. Emits a + single observation with `tls: true` (Cloudflare Tunnel implies + HTTPS). + +Both fire automatically for any container reported to STD. There is +no opt-in label — if the labels are there, the interpreter reads +them. + +### Adding your own + +Mount a directory of YAML files into the container at +`/app/interpreters/user/`: + +```yaml +volumes: + - ./my-interpreters:/app/interpreters/user:ro +``` + +User files load alongside builtins. A user file whose `name:` matches +a builtin **overrides** the builtin — drop in a tweaked `traefik.yml` +without rebuilding the image. + +### Format + +Every interpreter has three sections: `match` (which containers +fire), `extract` (which labels to read), `emit` (what to send to +STD). See [`docs/PRD.md` §11](docs/PRD.md) for the full reference, +or `docs/community-interpreters/template.yml` for an annotated +skeleton. + +### Community reference + +`docs/community-interpreters/` collects contributed YAMLs for tools +the maintainer doesn't necessarily run. Examples there may or may +not match your environment — read before mounting. PRs welcome. + +### Empty list vs. absent on the wire + +The notifier sends `exposure_observations` as a list when any +interpreter is loaded, even if no interpreter matched (empty list +tells STD to clear existing exposure rows for the container). If no +interpreters are loaded at all, the field is omitted, which STD +treats as "no update" — existing exposure rows are preserved. + +--- + ## Docker Compose Example ```yaml @@ -172,6 +241,8 @@ services: - /var/run/docker.sock:/var/run/docker.sock - /etc/hostname:/etc/host_hostname:ro - /var/docker/docker-api-notifier:/config + # Optional — drop your own interpreter YAMLs in here. + # - /etc/docker-api-notifier/interpreters:/app/interpreters/user:ro restart: unless-stopped ``` @@ -183,6 +254,9 @@ Volumes: notifier reports the **host's** hostname, not the container's, when posting to downstream notifiers. - `/config` — log file lives here (`notifier.log`, rotated at 10 MB). +- `/app/interpreters/user` — optional. Mount a directory of operator + YAMLs here to extend or override the built-in interpreters + (Traefik, Dockflare). See [Interpreters](#interpreters) above. --- diff --git a/docs/PRD.md b/docs/PRD.md index 88e647b..3384dba 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -11,6 +11,7 @@ | 0.1 | 2026-05-10 | Initial PRD. Documents current shipped behavior at v0.2.3 and the planned v0.3.0 cleanup. | | 0.2 | 2026-05-13 | v0.3.1 — STD reporting opt-out mode via `STD_REPORT_ALL_CONTAINERS` env var. §1.3 softened to reflect per-host opt-out scope. | | 0.3 | 2026-05-13 | v0.3.2 — capture container network membership and port information from the Docker API and forward to STD. §3.3 base kwargs contract grows three rows (`networks`, `exposed_ports`, `published_ports`). | +| 0.4 | 2026-05-14 | v0.4.0 — YAML interpreter mechanism, STD opt-out env var (`STD_REPORT_ALL_CONTAINERS`), network/ports capture, and design-principle softening. Originally planned as v0.3.1 / v0.3.2 / v0.4.0; consolidated into a single v0.4.0 release. §1.3 softens "no state" and "env vars only" to reflect YAML configuration. §3 architecture grows an interpreter component. §4 documents the interpreter loader paths and volume-mount convention. §11 fully documents the YAML format and wire emission. | --- @@ -20,13 +21,13 @@ 2. [Scope](#2-scope) 3. [Architecture](#3-architecture) 4. [Configuration Model](#4-configuration-model) -5. [Current State (v0.3.1)](#5-current-state-v031) +5. [Current State (v0.4.0)](#5-current-state-v040) 6. [v0.3.0 — Cleanup Release](#6-v030--cleanup-release) -7. [v0.3.1 — STD Reporting Opt-Out Mode](#7-v031--std-reporting-opt-out-mode) -8. [v0.3.2 — Network & Port Capture](#8-v032--network--port-capture) -9. [Versioning, Branches, and Releases](#9-versioning-branches-and-releases) -10. [Cross-Repo Coordination](#10-cross-repo-coordination) -11. [Open Questions](#11-open-questions) +7. [Delivered in v0.4.0](#7-delivered-in-v040) +8. [Versioning, Branches, and Releases](#8-versioning-branches-and-releases) +9. [Cross-Repo Coordination](#9-cross-repo-coordination) +10. [Open Questions](#10-open-questions) +11. [YAML Interpreter Format Reference](#11-yaml-interpreter-format-reference) --- @@ -48,22 +49,35 @@ systems so they stay in sync with reality without a human in the loop. container wants. - Fan out to one or more notifier modules (DNS, dashboard, future). -The notifier deliberately holds no state. Each event is processed -independently against current container metadata. +The notifier deliberately holds no per-event runtime state. Each +event is processed independently against current container metadata. +The only on-disk inputs are YAML interpreter files loaded once at +startup (see §1.3 and §11). ### 1.3 Design principles - **Opt-in per container by default.** No labels means no notification — run safely alongside containers that don't know or care about this notifier. STD reporting can be flipped to opt-out on a per-host basis - via the `STD_REPORT_ALL_CONTAINERS` env var (see §7); other notifier + via the `STD_REPORT_ALL_CONTAINERS` env var (see §7.2); other notifier targets remain per-container opt-in regardless, because they create external side effects (DNS records, etc.) that should not fire for containers that didn't ask. - **Independent notifier modules.** Each downstream system is its own module under `notifiers/` with its own auth, retry, and payload shape. -- **No state.** No database, no cache, no queue. Everything is derived - from the live Docker socket plus environment variables. +- **No runtime state.** No database, no cache, no queue. Per-event work + is derived from the live Docker socket. As of v0.4.0 the notifier + does load YAML interpreter files at startup (see §11) — this is + configuration, not per-event memory, and it is read once into a + module-level structure that does not change as events flow through. + The "no per-event state" half of the principle still holds. +- **Configuration via env vars and labels, plus YAML for interpreters.** + Through v0.3.x, env vars and container labels were the only inputs. + v0.4.0 adds optional YAML interpreter files mounted into the + container; they exist because expressing match/extract logic for + third-party label schemes (Traefik, Dockflare, ...) as Python forks + is heavier than necessary. See §4 for the loader layout and §11 for + the full format. - **One instance per host.** Multi-host coordination is out of scope. --- @@ -110,6 +124,10 @@ independently against current container metadata. │ │ ├─ event subscription│ │ │ │ └─ periodic re-scan │ │ │ │ │ │ +│ │ interpreter_loader │ │ +│ │ ├─ builtin YAMLs │ │ +│ │ └─ user YAMLs (mount)│ │ +│ │ │ │ │ │ notifiers/ │ │ │ │ ├─ technitium_dns │ │ │ │ └─ service_tracker_ │ │ @@ -131,6 +149,12 @@ independently against current container metadata. - **Common concerns** that should live outside individual notifier modules: logging configuration, retry helpers, label-to-payload mapping. (Today these are partially duplicated; see §5.) +- **`interpreter_loader.py`** — loads and evaluates the YAML + interpreters introduced in v0.4.0. Runs once at startup to load + YAMLs from `/app/interpreters/builtin/` and `/app/interpreters/user/` + into a module-level structure; runs once per dispatch event to + produce the list of `ExposureObservation` dicts forwarded to STD as + `exposure_observations`. See §11 for the full design. ### 3.2 Event flow @@ -186,9 +210,9 @@ arguments guaranteed present: | `stack_name` | Optional[str] | `com.docker.compose.project` label or `None` | | `started_at` | str | ISO timestamp from container state | | `action` | str | The action that triggered this call (e.g. "start", "boot", "refresh") | -| `networks` | list[dict] | One entry per Docker network the container is on: `{"name": str, "aliases": [str, ...]}`. Empty list if the container is on no networks. Added in v0.3.2. | -| `exposed_ports` | list[str] | Container's `ExposedPorts` config as a list of `"/"` strings (e.g. `"5173/tcp"`). Empty list if none. Added in v0.3.2. | -| `published_ports` | list[dict] | One entry per `(container_port, host_port)` mapping: `{"container_port": int, "protocol": str, "host_ip": str, "host_port": int}`. Empty list if no published ports. Added in v0.3.2. | +| `networks` | list[dict] | One entry per Docker network the container is on: `{"name": str, "aliases": [str, ...]}`. Empty list if the container is on no networks. Added in v0.4.0. | +| `exposed_ports` | list[str] | Container's `ExposedPorts` config as a list of `"/"` strings (e.g. `"5173/tcp"`). Empty list if none. Added in v0.4.0. | +| `published_ports` | list[dict] | One entry per `(container_port, host_port)` mapping: `{"container_port": int, "protocol": str, "host_ip": str, "host_port": int}`. Empty list if no published ports. Added in v0.4.0. | The last three (`networks`, `exposed_ports`, `published_ports`) live in `base_kwargs` because they are inherent container facts read off @@ -237,36 +261,71 @@ A reference implementation lives at `notifiers/_template.py`. ## 4. Configuration Model -All configuration is via environment variables. There is no config file. +Most configuration is via environment variables and container labels. +Through v0.3.x there was no config file at all. As of v0.4.0 there is +one narrow exception: **YAML interpreter files** loaded from two +on-disk paths inside the container. -Two reasons: +Two reasons env vars + labels remain the default: 1. The notifier is meant to be one-line-deployable on every host. A - shared config file would be one more thing to template and sync. + shared config file for general behavior would be one more thing to + template and sync. 2. Per-container behavior comes from labels on those containers, which is the right place for it — the people writing the compose files know what they want. Environment variables are documented in the `README.md`. +### 4.1 Interpreter YAML loader (v0.4.0+) + +The notifier reads YAML files from two paths at startup: + +- `/app/interpreters/builtin/` — baked into the container image. Ships + `traefik.yml` and `dockflare.yml`. +- `/app/interpreters/user/` — empty by default; operators mount their + own YAMLs here: + + ```yaml + volumes: + - ./my-interpreters:/app/interpreters/user:ro + ``` + +A user file whose `name:` matches a builtin **overrides** the builtin +(useful for tweaking the shipped Traefik/Dockflare logic without +forking). Files that fail to parse or validate are logged at warning +level and skipped; the notifier continues with whatever loaded +successfully. + +The set of loaded interpreters is read once at startup into a +module-level structure. No reload-on-change. The debug-only +`INTERPRETER_RELOAD_ON_EACH_EVENT` env var bypasses the cache and +reloads on every dispatch — useful when iterating on a YAML, not +intended for production. + +See §11 for the YAML format and emission semantics. + --- -## 5. Current State (v0.3.1) +## 5. Current State (v0.4.0) -Tags shipped on `main`: v0.1.0 → v0.3.1. v0.3.0 (2026-05-12) resolved -every issue listed in §5.2 below. v0.3.1 followed as a small additive -release introducing the `STD_REPORT_ALL_CONTAINERS` env var for -per-host opt-out reporting to STD (see §7). The next release in -flight is v0.3.2 — network & port capture (see §8). +Tags shipped on `main`: v0.1.0 → v0.4.0. v0.3.0 (2026-05-12) resolved +every issue listed in §5.2 below. v0.4.0 (2026-05-14) shipped the +work originally scoped across three separate releases +(v0.3.1 / v0.3.2 / v0.4.0); the consolidation is summarized in §7. ### 5.1 What works today - Docker event subscription with a fixed action whitelist. - Boot-time full scan. - Periodic re-scan thread. -- DNS notifier with no retry, requests-based, fire-and-forget. +- DNS notifier with retry, requests-based, raises on HTTP 4xx/5xx. - STD notifier with `tenacity`-backed retry, bearer-token auth. - Label-driven notifier opt-in via `dockernotifier.notifiers`. +- Per-host STD opt-out via `STD_REPORT_ALL_CONTAINERS` (see §7.2). +- Network and port capture forwarded to STD on every payload (see §7.3). +- YAML-driven interpreter layer producing `exposure_observations` for + STD (see §7.4 for the v0.4.0 summary, §11 for the format reference). - Per-notifier label namespaces (`dockernotifier.dns.*`, `dockernotifier.std.*`). @@ -330,23 +389,46 @@ window. these easier later, but none ship in v0.3.0. - Multi-host coordination. - A config file. Env vars + labels remain the only inputs. -- Test suite. Worth doing eventually (see §11), not in v0.3.0. +- Test suite. Worth doing eventually (see §10), not in v0.3.0. --- -## 7. v0.3.1 — STD Reporting Opt-Out Mode +## 7. Delivered in v0.4.0 + +Originally planned as three separate releases — v0.3.1 +(STD opt-out env var), v0.3.2 (network & port capture), and v0.4.0 +(YAML interpreter mechanism) — consolidated into a single v0.4.0 +release. The full operator-facing changelog lives in `CHANGELOG.md`; +this section captures the design intent and scope decisions for each +piece so they remain part of the PRD record. + +The whole bundle is **paired with STD v0.6.0**, which adds the +consumer side: it accepts `networks`, `exposed_ports`, +`published_ports`, and `exposure_observations` on `/api/v1/register` +and ships the synthesizer that turns observations into rendered +exposure rows. STD v0.5.x's strict pydantic validator rejects all +four keys, so operators must upgrade STD before the notifier. -A small, additive release that introduces a single env var, -`STD_REPORT_ALL_CONTAINERS`, which flips STD reporting from -per-container opt-in to per-host opt-out. +### 7.1 Design principle reconciliation -### 7.1 Goals +This release deliberately softens principles previously stated in +§1.3: -- Let operators who want a complete inventory of a host's running - containers in STD avoid labelling every container individually. -- Preserve today's behavior as the default: unset env var → unchanged. +- "No state." → "No runtime state." Configuration is now loaded from + YAML at startup. Per-event state still does not exist. +- "All configuration is via environment variables." → no longer + literally true. Env vars + labels remain the default, with a narrow + exception for interpreter YAMLs. +- "Opt-in per container." → still the default, but STD reporting can + be flipped to per-host opt-out via `STD_REPORT_ALL_CONTAINERS`. DNS + and other side-effect notifiers remain per-container opt-in. -### 7.2 Scope +§1.3 has been updated to reflect all three softenings. + +### 7.2 STD reporting opt-out mode (`STD_REPORT_ALL_CONTAINERS`) + +A single env var flips STD reporting from per-container opt-in to +per-host opt-out. - **STD only.** The env var affects only the STD notifier dispatch. The DNS notifier (and any future notifier that creates external @@ -359,71 +441,40 @@ per-container opt-in to per-host opt-out. the boot pass and periodic refresh loop iterate `client.containers.list()`. Stopped containers are not retroactively reported. - -### 7.3 Semantics - -- `STD_REPORT_ALL_CONTAINERS` truthy values: `true`, `1`, `yes` - (case-insensitive). Anything else (including unrecognized strings - like `maybe`) is treated as off; unrecognized values log a single - warning at startup. +- Truthy values: `true`, `1`, `yes` (case-insensitive). Anything + else (including unrecognized strings like `maybe`) is treated as + off; unrecognized values log a single warning at startup. - When on, every running container on the host is reported to STD on boot, on watched Docker events, and on each periodic refresh tick — regardless of whether the container's - `dockernotifier.notifiers` label includes `service-tracker-dashboard` - (or even exists at all). + `dockernotifier.notifiers` label includes + `service-tracker-dashboard` (or even exists at all). - `dockernotifier.std.*` labels on individual containers are still honored. Containers without those labels are reported with the minimum information available; STD's wire contract makes most fields optional and applies its own defaults. -- A container with `dockernotifier.notifiers=dns` only (no STD opt-in) - AND the env var set: STD fires (env-var path) **and** DNS fires - (label path). The env var adds STD; it does not subtract anything. - -### 7.4 Wire contract +- A container with `dockernotifier.notifiers=dns` only (no STD + opt-in) AND the env var set: STD fires (env-var path) **and** DNS + fires (label path). The env var adds STD; it does not subtract + anything. +- Wire contract unchanged. STD receives identical + `/api/v1/register` payloads regardless of whether the trigger came + from a label or from the env var. -Unchanged. STD receives identical `/api/v1/register` payloads -regardless of whether the trigger came from a label or from the env -var. STD has no way to distinguish the two paths and does not need to. +Out of scope: per-container opt-out (a label like +`dockernotifier.std.skip=true` to suppress reporting even when the +env var is set), changing DNS opt-in semantics, reporting +non-running containers. -### 7.5 Design principle reconciliation +### 7.3 Network & port capture -§1.3's "opt-in per container" principle is softened, not abandoned. -With the env var unset (default), per-container opt-in remains the -only path. The env var is an explicit, deliberate per-host stance -taken by the operator running the notifier instance — it does not -change what other operators experience. - -### 7.6 Out of scope for v0.3.1 - -- Per-container opt-out (a label like `dockernotifier.std.skip=true` - to suppress reporting even when the env var is set). Defer until - there is a concrete request. -- Changing DNS opt-in semantics. -- Reporting non-running containers. -- Network/port capture (planned for v0.3.2). - ---- +Container network membership and port information are read directly +from the Docker API and forwarded to STD as canonical fields, so +STD's UI can render badges/links without re-reading the Docker +socket itself. Pure capture — no interpretation, no derived +semantics. -## 8. v0.3.2 — Network & Port Capture - -An additive release that forwards container network membership and -port information from the Docker API to STD. STD v0.6.0 consumes -these fields; STD v0.5.x will reject the payload (strict pydantic -validation), so STD v0.6.0 must be deployed first. - -### 8.1 Goals - -- Capture inherent container facts the notifier already has access to - via the Docker API: which networks the container is on, what ports - it exposes, and what ports it publishes to the host. -- Forward all three to STD as canonical fields so STD's UI can render - badges/links without re-reading the Docker socket itself. -- Stay pure-capture. No interpretation, no derived semantics. STD - v0.7.0 will layer interpretation on top. - -### 8.2 Captured fields - -Added to the base kwargs contract (see §3.3 table): +Captured fields (added to the base kwargs contract; see §3.3 table): - `networks` — list of `{"name": str, "aliases": [str, ...]}`. One entry per Docker network the container is on. Read from @@ -437,32 +488,21 @@ Added to the base kwargs contract (see §3.3 table): `container.attrs["NetworkSettings"]["Ports"]`. Entries with a null binding list (exposed-but-not-published) are skipped. -### 8.3 Coercion at the boundary +Coercion at the boundary: -- `host_port` arrives from Docker as a string (`"5173"`); coerced to - `int`. +- `host_port` arrives from Docker as a string (`"5173"`); coerced + to `int`. - `container_port` is parsed from the `"/"` key and cast to `int`. - `protocol` is the string after the slash (typically `"tcp"` or `"udp"`), kept as-is. -### 8.4 Empty vs. missing - Empty values are emitted as explicit empty lists, not null. This lets STD's UI distinguish "we know there's nothing" from "the notifier hasn't reported yet" (where the field is absent / null). - -- Container on no networks (`network_mode: none`): `networks: []`. -- Container with no exposed ports: `exposed_ports: []`. -- Container with no published ports: `published_ports: []`. - -### 8.5 Wire contract - -Three new fields on the canonical payload to `/api/v1/register`. -The STD notifier's `_PASSTHROUGH` set covers them; no translation -needed because they are already in canonical shape. - -### 8.6 Scope of consumption +The STD notifier's `_PASSTHROUGH` set covers all three new fields; +no translation is needed because they are already in canonical +shape. Only the STD notifier consumes these fields today. The DNS notifier receives them through `**kwargs` and ignores them. Storing them in @@ -470,29 +510,114 @@ receives them through `**kwargs` and ignores them. Storing them in available for any future notifier (e.g. a Traefik-config emitter) without rerunning the Docker API call. -### 8.7 Out of scope for v0.3.2 - -- Network IPs, gateways, MAC addresses, or any per-network detail - beyond name and aliases. -- Any interpretation of network names (e.g. "container on `proxy` - network → mark as Traefik-exposed"). That is v0.4.0 interpreter - work. -- Per-host configuration of which networks to report. All networks - the container is on get reported. -- Filtering or redaction. If a future operator wants it, that's a - separate feature. -- Sending this data to anywhere besides STD. - -### 8.8 Dependencies - -- Hard: STD v0.6.0 must be deployed first. v0.5.x rejects unknown - keys. -- Soft: notifier v0.3.1 on `main` (clean version sequencing only — - v0.3.2 does not depend on v0.3.1's behavior). +Out of scope: per-network detail beyond name and aliases (no IPs, +gateways, MAC addresses); per-host filtering of which networks to +report; sending the data anywhere besides STD. + +### 7.4 YAML interpreter mechanism + +A YAML-driven interpreter layer reads labels written by third-party +tools (Traefik, Dockflare, ...) and emits structured exposure +observations to STD as `exposure_observations`. Eliminates the need +for operators to duplicate hostnames into +`dockernotifier.std.internalurl` when the same fact is already +encoded in their Traefik/Dockflare labels. + +Goals: + +- Translate third-party label schemes into a uniform shape STD + understands, without operators having to fork the notifier for + each new tool. +- Ship sensible defaults for the two tools the maintainer actually + runs (Traefik, Dockflare). +- Offer operators a path to add new interpreters without rebuilding + the image: drop a YAML in a mounted directory. +- Maintain a community-reference directory in the repo for sharing + contributed interpreters. + +Loader (`interpreter_loader.py`): + +- `load_interpreters()` returns a `LoadResult(interpreters, ...)` + containing the compiled interpreters and a flag indicating whether + any directories were even found. +- `evaluate(interpreters, labels)` runs every interpreter against a + container's labels and returns the concatenated list of emitted + observations. +- `/app/interpreters/builtin/` is read first. Then + `/app/interpreters/user/` — user files with a `name:` matching a + builtin override the builtin and the override is logged. +- Files that fail YAML parsing, validation, or regex compilation + log a warning and are skipped. The loader does not raise; bad + files do not block startup. +- Loaded interpreters are stored in a module-level dict keyed by + name. Not re-read per event. The debug-only + `INTERPRETER_RELOAD_ON_EACH_EVENT=true` env var re-reads both + directories on every dispatch — useful when iterating on a YAML, + not for production. + +Wire emission: the STD notifier passes `exposure_observations` +through unchanged via `_PASSTHROUGH`. The value is one of: + +- A **list** (possibly empty) — emitted when at least one + interpreter is loaded. An empty list means "interpreters ran and + nothing matched"; STD interprets this as "clear all exposure rows + for this container." +- **`None`** — emitted when no interpreters are loaded (empty dirs + or all failed validation). STD treats null as "no update; + preserve existing exposure rows." This distinction matters when + an operator disables interpreters at runtime — STD doesn't + suddenly forget exposure data. + +The STD notifier's `_to_canonical` filter drops `None` values from +outgoing payloads, so `None` becomes "field absent" on the wire. + +Baked-in interpreters (`/app/interpreters/builtin/`): + +- `traefik.yml` — regex-match flavor. Captures router names, reads + the rule for `Host(...)`, reads `tls` and `entrypoints`. Emits + one observation per router. +- `dockflare.yml` — fixed-key match (`dockflare.enable=true`). + Reads `dockflare.hostname`, optional `dockflare.access.policy` + and `dockflare.access.group`. Emits a single observation with + `tls: true` (Cloudflare Tunnel implies HTTPS) and an `auth` + string of the form `cloudflare_access:` (or `null` if no + policy is set, thanks to the null-propagation rule). + +Community-reference directory (`docs/community-interpreters/`): + +- `README.md` — explains the directory, the format, and the + explicit non-guarantee. +- `traefik.yml`, `dockflare.yml` — reference copies of the + builtins. Operators starting a new interpreter from scratch read + these to see the format in practice. +- `template.yml` — heavily annotated skeleton. + +PRs that add new interpreters are welcome. The maintainer does not +QA every contribution — examples may or may not work for a given +operator's setup. Operators adapt and mount as needed. + +The full YAML format reference (match flavors, extract semantics, +emit substitution rules, null propagation) lives in §11. + +Out of scope: reloading interpreters at runtime without notifier +restart (`INTERPRETER_RELOAD_ON_EACH_EVENT` is a debugging +affordance, not a feature); per-container interpreter selection; +per-host curation of which YAMLs apply (mount different files on +different hosts is the answer); network-membership-based matching +inside the notifier (STD's synthesizer handles that on its side); +a web UI for managing interpreters; validating emit outputs against +STD's `ExposureObservation` schema beyond basic structure. + +### 7.5 Dependencies + +- Hard pairing: **STD v0.6.0** must be deployed before notifier + v0.4.0 is rolled out. STD v0.5.x's strict pydantic validator + rejects payloads carrying `networks`, `exposed_ports`, + `published_ports`, or `exposure_observations`. --- -## 9. Versioning, Branches, and Releases +## 8. Versioning, Branches, and Releases - `main` is the default branch and the source of truth for releases. - All work happens on `dev`. PR `dev` → `main` when ready to release. @@ -506,18 +631,18 @@ without rerunning the Docker API call. --- -## 10. Cross-Repo Coordination +## 9. Cross-Repo Coordination This project is paired with [service-tracker-dashboard](https://github.com/crzykidd/service-tracker-dashboard). -### 10.1 Contract ownership +### 9.1 Contract ownership STD owns the wire contract for the register endpoint. The notifier is a producer — it sends what STD documents. Wire-format changes start in STD; the notifier follows. -### 10.2 Release ordering for the v0.5.0 / v0.3.0 cycle +### 9.2 Release ordering for the v0.5.0 / v0.3.0 cycle 1. STD v0.5.0 ships with `/api/v1/register` (canonical keys) and the compat shim on `/api/register` (legacy keys, deprecated). @@ -527,19 +652,22 @@ STD; the notifier follows. Operators must upgrade the notifier to v0.3.0+ before STD v0.6.0. -### 10.3 Release ordering for the v0.6.0 / v0.3.2 cycle +### 9.3 Release ordering for the v0.6.0 / v0.4.0 cycle -1. STD v0.6.0 ships with `networks`, `exposed_ports`, and - `published_ports` accepted on `/api/v1/register`. -2. Notifier v0.3.2 ships emitting those three fields. +1. STD v0.6.0 ships with `networks`, `exposed_ports`, + `published_ports`, and `exposure_observations` accepted on + `/api/v1/register`, plus the synthesizer that turns observations + into rendered exposure rows. +2. Notifier v0.4.0 ships emitting all four fields and loading the + YAML interpreter layer that produces `exposure_observations`. -If notifier v0.3.2 is deployed against STD v0.5.x, STD's strict +If notifier v0.4.0 is deployed against STD v0.5.x, STD's strict pydantic validator rejects the payload (unknown keys). Operators must upgrade STD before the notifier. --- -## 11. Open Questions +## 10. Open Questions - **Test coverage.** No tests exist today. Worth investing in a small suite that fakes the Docker client and asserts dispatch behavior? @@ -552,3 +680,89 @@ must upgrade STD before the notifier. - **New notifier targets.** Likely candidates if needed: Slack, Discord, ntfy, generic webhook. Each one is ~1 module under `notifiers/` plus env vars. + +--- + +## 11. YAML Interpreter Format Reference + +This is the format contract for YAML interpreter files loaded by +`interpreter_loader.py`. Community contributors and operators +writing their own interpreters should treat this section as +authoritative. See §7.4 for the surrounding release notes and +design intent. + +### 11.1 File structure + +One file per interpreter, one interpreter per file. Top-level keys: + +```yaml +name: # required +description: # optional +match: # required, exactly one flavor + any_label_key_matches: '' + # or + label_key: '' + label_value_equals: '' # optional, case-insensitive +extract: # required (may be empty) + : + from_label: '' + value_pattern: '' # optional + capture: '' # optional + coerce: bool | int # optional + default: # optional +emit: # required + layer: + : '' + details: + : '' +``` + +### 11.2 Match + +- `any_label_key_matches` — regex applied to every label key on the + container with `fullmatch`. Named captures `(?P...)` become + available in `extract` via `{name}` substitution. If multiple + label keys match, the interpreter fires once per match — useful + for tools that namespace per-router (Traefik). +- `label_key` + optional `label_value_equals` — fires once if the + exact label key exists. With `label_value_equals`, the value is + compared case-insensitively after stripping. + +A single interpreter must pick exactly one flavor. + +### 11.3 Extract + +Each entry defines a local variable. The notifier: + +1. Substitutes `{capture}` placeholders in `from_label` with values + from the match step. +2. Looks up that label key on the container. +3. If a `value_pattern` is set, runs `re.search` over the value. If + `capture` is set, the named group is used; otherwise the whole + match is the result. +4. Applies `coerce` (`bool` or `int`). Boolean truthy strings are + `true`, `1`, `yes` (case-insensitive). Failed int coercion falls + back to `default`. +5. If anything in steps 2–4 fails (label missing, pattern doesn't + match, capture missing), the variable takes its `default` (or + `None` if no default was set). + +### 11.4 Emit + +Each key in `emit` becomes a field on the resulting observation. +String values may reference `{local_var}` placeholders from the +extract step. + +Null propagation rules: + +- A **bare** placeholder like `'{var}'` resolves to the variable's + value verbatim — bools stay bools, ints stay ints, lists stay + lists, `None` passes through as `None`. +- A **mixed** template like `'cloudflare_access:{policy}'` is built + by string concatenation. If any referenced variable is `None`, + the whole field resolves to `None` (rather than substituting the + literal string `"None"`). + +`emit.layer` is required and identifies the source tool on the wire. +All other emit fields are optional; STD treats missing fields as +"no information." diff --git a/docs/community-interpreters/README.md b/docs/community-interpreters/README.md new file mode 100644 index 0000000..c0b1402 --- /dev/null +++ b/docs/community-interpreters/README.md @@ -0,0 +1,92 @@ +# Community Interpreters + +This directory holds **reference YAML interpreters** contributed by +operators of `docker-api-notifier`. Each file teaches the notifier how +to read labels written by a third-party tool (Traefik, Dockflare, +Caddy, Pangolin, NPM, etc.) and emit them as structured exposure +observations to STD. + +## Status: examples, not products + +Files here are community-contributed. They may or may not work for +your specific environment. The maintainer does not QA every +contribution — operators are expected to read the YAML, understand +what it captures, and adapt it before mounting it into a notifier +instance. + +If a file here works for your setup as-is, great. If it doesn't, +edit it locally; PRs that improve an example are welcome. + +## Using an interpreter + +1. Copy the `.yml` file to a directory on your Docker host + (e.g. `/etc/docker-api-notifier/interpreters/`). +2. Mount that directory into the notifier container at + `/app/interpreters/user`: + + ```yaml + volumes: + - /etc/docker-api-notifier/interpreters:/app/interpreters/user:ro + ``` + +3. Restart the notifier. On startup it will log which interpreters + were loaded. + +User-supplied files with the same `name:` as a builtin **override** +the builtin. That is intentional — drop in a tweaked `traefik.yml` +to customise capture without forking the notifier. + +## Writing a new interpreter + +See `docs/PRD.md` §12 for the full YAML format reference. The two +builtin interpreters are also mirrored here as references: + +- `traefik.yml` — regex-match flavor, captures router names with a + named group and reads multiple labels per router. +- `dockflare.yml` — fixed-key match flavor, simpler structure. + +At a glance, every interpreter has three sections: + +```yaml +name: +description: + +match: + # Exactly one of: + any_label_key_matches: '...) captures>' + # or + label_key: '' + label_value_equals: '' + +extract: + : + from_label: '