diff --git a/.env.prod.example b/.env.prod.example
index 7b8ce8a..a834cd7 100644
--- a/.env.prod.example
+++ b/.env.prod.example
@@ -114,6 +114,7 @@ LOG_ROLE=telemetry
LOG_HOST=telemetry-01
LOG_COLLECTOR_ID=telemetry-01
LOKI_ENDPOINT=https://telemetry.ops.example.com:8444
+LOG_AUTH_TOKEN=replace-with-a-unique-log-ingest-token
LOG_BUFFER_BYTES=2147483648
LOG_METRICS_BIND=127.0.0.1:9599
# Loki defaults to 14 days in production; query range is independently bounded.
diff --git a/compose.prod.yml b/compose.prod.yml
index 8a346ae..85a10d0 100644
--- a/compose.prod.yml
+++ b/compose.prod.yml
@@ -401,6 +401,7 @@ services:
LOG_BUFFER_BYTES: ${LOG_BUFFER_BYTES}
LOG_METRICS_ADDRESS: 0.0.0.0:9599
LOKI_ENDPOINT: ${LOKI_ENDPOINT:?LOKI_ENDPOINT is required for the logs profile}
+ LOG_AUTH_TOKEN: ${LOG_AUTH_TOKEN:?LOG_AUTH_TOKEN is required for the logs profile}
volumes:
- ./docker/vector/operational.yaml:/etc/vector/operational.yaml:ro
- /var/run/docker.sock:/var/run/docker.sock:ro
diff --git a/core/app/Support/NetworkAddress.php b/core/app/Support/NetworkAddress.php
index ecb3a9c..c1f965a 100644
--- a/core/app/Support/NetworkAddress.php
+++ b/core/app/Support/NetworkAddress.php
@@ -6,8 +6,10 @@ final class NetworkAddress
{
private const UNSAFE_NETWORKS = [
'0.0.0.0/8', '10.0.0.0/8', '100.64.0.0/10', '127.0.0.0/8', '169.254.0.0/16', '172.16.0.0/12',
- '192.0.0.0/24', '192.168.0.0/16', '198.18.0.0/15', '224.0.0.0/4', '240.0.0.0/4',
+ '192.0.0.0/24', '192.0.2.0/24', '192.88.99.0/24', '192.168.0.0/16', '198.18.0.0/15',
+ '198.51.100.0/24', '203.0.113.0/24', '224.0.0.0/4', '240.0.0.0/4',
'::/128', '::1/128', '64:ff9b::/96', '64:ff9b:1::/48', 'fc00::/7', 'fe80::/10', 'fec0::/10', 'ff00::/8',
+ '2001:db8::/32',
];
private const PRIVATE_NETWORKS = ['10.0.0.0/8', '100.64.0.0/10', '172.16.0.0/12', '192.168.0.0/16', 'fc00::/7'];
diff --git a/core/tests/Feature/SystemIdentityApiTest.php b/core/tests/Feature/SystemIdentityApiTest.php
index a5667b3..197a6d7 100644
--- a/core/tests/Feature/SystemIdentityApiTest.php
+++ b/core/tests/Feature/SystemIdentityApiTest.php
@@ -196,8 +196,8 @@ private function validPayload(): array
'platform_domain' => 'cdnf.test',
'proxy_hostname' => 'proxy.cdnf.test',
'nameservers' => [
- ['hostname' => 'ns1.cdnf.test', 'ipv4' => '192.0.2.10', 'ipv6' => '2001:db8::10'],
- ['hostname' => 'ns2.cdnf.test', 'ipv4' => '192.0.2.11', 'ipv6' => '2001:db8::11'],
+ ['hostname' => 'ns1.cdnf.test', 'ipv4' => '8.8.8.8', 'ipv6' => '2001:4860:4860::8888'],
+ ['hostname' => 'ns2.cdnf.test', 'ipv4' => '1.1.1.1', 'ipv6' => '2606:4700:4700::1111'],
],
'soa_primary' => 'ns1.cdnf.test',
'soa_mailbox' => 'hostmaster.cdnf.test',
diff --git a/core/tests/Unit/NetworkAddressTest.php b/core/tests/Unit/NetworkAddressTest.php
index 249ae13..3b5b48c 100644
--- a/core/tests/Unit/NetworkAddressTest.php
+++ b/core/tests/Unit/NetworkAddressTest.php
@@ -13,7 +13,7 @@ public function test_destination_safety_distinguishes_allowlistable_private_spac
$this->assertTrue(NetworkAddress::isPrivate('10.20.30.40'));
$this->assertTrue(NetworkAddress::inCidr('10.20.30.40', '10.0.0.0/8'));
- foreach (['127.0.0.1', '169.254.169.254', '::1', '::ffff:127.0.0.1', '64:ff9b::7f00:1'] as $address) {
+ foreach (['0.1.2.3', '127.0.0.1', '169.254.169.254', '192.0.2.1', '192.88.99.1', '198.51.100.1', '203.0.113.1', '239.1.2.3', '240.0.0.1', '::1', '::ffff:127.0.0.1', '64:ff9b::7f00:1', '2001:db8::1'] as $address) {
$this->assertTrue(NetworkAddress::isUnsafe($address), "$address must be blocked");
$this->assertFalse(NetworkAddress::isPrivate($address), "$address must never be private-allowlist eligible");
}
diff --git a/deploy/production/Caddyfile b/deploy/production/Caddyfile
index e8e9545..5ba1051 100644
--- a/deploy/production/Caddyfile
+++ b/deploy/production/Caddyfile
@@ -6,20 +6,19 @@
{$CONTROL_HOSTNAME} {
encode zstd gzip
reverse_proxy web:8080 {
- header_up X-Forwarded-Proto https
- header_up X-Forwarded-Port 443
- header_up X-Forwarded-Host {$CONTROL_HOSTNAME}
- header_down Location "http://{$CONTROL_HOSTNAME}" "https://{$CONTROL_HOSTNAME}"
-}
-
+ header_up X-Forwarded-Proto https
+ header_up X-Forwarded-Port 443
+ header_up X-Forwarded-Host {$CONTROL_HOSTNAME}
+ header_down Location "http://{$CONTROL_HOSTNAME}" "https://{$CONTROL_HOSTNAME}"
+ }
header {
Strict-Transport-Security "max-age=31536000"
X-Content-Type-Options "nosniff"
X-Frame-Options "SAMEORIGIN"
Referrer-Policy "strict-origin-when-cross-origin"
- Content-Security-Policy "upgrade-insecure-requests"
- -Server
+ Content-Security-Policy "upgrade-insecure-requests"
+ -Server
}
}
@@ -44,7 +43,11 @@ https://{$TELEMETRY_HOSTNAME}:8444 {
{$GRAFANA_HOSTNAME} {
reverse_proxy grafana:3000
- header { Strict-Transport-Security "max-age=31536000"; X-Content-Type-Options "nosniff"; -Server }
+ header {
+ Strict-Transport-Security "max-age=31536000"
+ X-Content-Type-Options "nosniff"
+ -Server
+ }
}
http://127.0.0.1:2019 {
diff --git a/deploy/production/Caddyfile.telemetry b/deploy/production/Caddyfile.telemetry
index a927524..fe3bb31 100644
--- a/deploy/production/Caddyfile.telemetry
+++ b/deploy/production/Caddyfile.telemetry
@@ -23,7 +23,11 @@ https://{$TELEMETRY_HOSTNAME}:8444 {
{$GRAFANA_HOSTNAME} {
reverse_proxy grafana:3000
- header { Strict-Transport-Security "max-age=31536000"; X-Content-Type-Options "nosniff"; -Server }
+ header {
+ Strict-Transport-Security "max-age=31536000"
+ X-Content-Type-Options "nosniff"
+ -Server
+ }
}
http://127.0.0.1:2019 {
diff --git a/docker/openresty/runtime.lua b/docker/openresty/runtime.lua
index 31c4dcd..bce6ed0 100644
--- a/docker/openresty/runtime.lua
+++ b/docker/openresty/runtime.lua
@@ -110,16 +110,18 @@ local function blocked(ip, networks, blocked_networks, denied)
end
if allowed(ip, blocked_networks) then return true end
if ip:lower():match("^::ffff:") then return true end
- if ip == "0.0.0.0" or ip:match("^127%.") or ip:match("^169%.254%.") or ip:match("^224%.") then return true end
- local a, b = ip:match("^(%d+)%.(%d+)%.")
- a, b = tonumber(a), tonumber(b)
+ local a, b, c = ip:match("^(%d+)%.(%d+)%.(%d+)%.")
+ a, b, c = tonumber(a), tonumber(b), tonumber(c)
+ if a == 0 or a == 127 or a == 169 and b == 254 or (a and a >= 224) then return true end
if (a == 10 or a == 192 and b == 168 or a == 172 and b and b >= 16 and b <= 31) and not allowed(ip, networks) then return true end
if a == 100 and b and b >= 64 and b <= 127 then return true end
- if a == 192 and b == 0 or a == 198 and b and (b == 18 or b == 19) then return true end
+ if a == 192 and (b == 0 or b == 2 or b == 88 and c == 99)
+ or a == 198 and (b == 18 or b == 19 or b == 51 and c == 100)
+ or a == 203 and b == 0 and c == 113 then return true end
local lower = ip:lower()
local hard_v6 = lower == "::" or lower == "::1" or lower:match("^fe[89ab]") ~= nil
or lower:match("^fe[c-f]") ~= nil or lower:match("^ff") ~= nil
- or lower:match("^64:ff9b:") ~= nil
+ or lower:match("^64:ff9b:") ~= nil or lower:match("^2001:db8:") ~= nil
if hard_v6 then return true end
local private_v6 = lower:match("^f[cd]") ~= nil
if private_v6 and allowed(ip, networks) then return false end
diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts
index fe9b1d4..c55b69a 100644
--- a/docs/.vitepress/config.mts
+++ b/docs/.vitepress/config.mts
@@ -248,6 +248,11 @@ export default defineConfig({
items: [
{ text: 'Production deployment', link: '/deployment/' },
{ text: 'Production quick start', link: '/deployment/production-quick-start' },
+ { text: 'Multi-region quick start', link: '/deployment/production-quick-start-multi-region' },
+ { text: 'Fleet operator guide', link: '/deployment/production-fleet-operator-guide' },
+ { text: 'Fleet configuration', link: '/deployment/production-fleet-config-reference' },
+ { text: 'Fleet architecture', link: '/deployment/production-fleet' },
+ { text: 'Manual Docker Compose deployment', link: '/deployment/manual-compose' },
{ text: 'Topology', link: '/deployment/topology' },
{ text: 'Certificates', link: '/deployment/certificates' },
{ text: 'Upgrade', link: '/deployment/upgrade' }
@@ -264,6 +269,7 @@ export default defineConfig({
{ text: 'Monitoring', link: '/operations/monitoring' },
{ text: 'Grafana command centers', link: '/operations/grafana' },
{ text: 'Operational logging', link: '/operations/operational-logging' },
+ { text: 'Laravel operations dashboard', link: '/operations/laravel-operations-dashboard' },
{ text: 'Edge gateway ingress', link: '/operations/gateway-ingress' },
{ text: 'Bounded cell inventory', link: '/operations/cell-inventory' },
{ text: 'Multi-cell pools', link: '/operations/multi-cell-pools' },
@@ -271,6 +277,8 @@ export default defineConfig({
{ text: 'Simple Anycast', link: '/operations/simple-anycast' },
{ text: 'Anycast qualification', link: '/operations/simple-anycast-qualification' },
{ text: 'Fleet rollouts', link: '/operations/fleet-rollouts' },
+ { text: 'Runtime generations', link: '/operations/runtime-generations' },
+ { text: 'Software supply chain', link: '/operations/software-supply-chain' },
{ text: 'Backup and recovery', link: '/operations/backup-and-recovery' },
{ text: 'Incident runbooks', link: '/operations/runbooks' },
{ text: 'Scaling', link: '/operations/scaling' },
diff --git a/docs/architecture/components.md b/docs/architecture/components.md
index 5dbd950..37b0c30 100644
--- a/docs/architecture/components.md
+++ b/docs/architecture/components.md
@@ -34,7 +34,7 @@ startup creates writable directories but deliberately does not migrate.
| `pdns-auth` | Private PowerDNS authoritative service |
| `pdns-db` | Rebuildable PowerDNS runtime schema |
| `pdns-migrate` | Explicit PowerDNS runtime migration tool |
-| `dns-api` overlay | Source-restricted TLS proxy for the private PowerDNS API |
+| `dns-api` | Source-restricted TLS proxy for the private PowerDNS API in the `dns` profile |
PowerAdmin exists only in the development-tools profile and is diagnostic.
Direct edits are drift.
diff --git a/docs/architecture/index.md b/docs/architecture/index.md
index c780455..c4a17dd 100644
--- a/docs/architecture/index.md
+++ b/docs/architecture/index.md
@@ -12,6 +12,10 @@ or rebuildable.
```mermaid
flowchart TB
+ ExternalDNS["Independent external DNS
management hostnames"] --> UI
+ ExternalDNS --> EdgeControl["edge-control
mTLS ingress"]
+ ExternalDNS --> DNSAPI["dns-api-N
restricted TLS"]
+ ExternalDNS --> TelemetryIngress["telemetry ingress"]
subgraph Management["Management plane"]
UI["Filament panels and Sanctum API"] --> Laravel["Laravel monolith"]
Laravel --> Horizon["Horizon workers"]
@@ -24,6 +28,7 @@ flowchart TB
subgraph DNS["Authoritative DNS plane"]
DNSdist["DNSdist public ingress"] --> PowerDNS["Private PowerDNS"]
PowerDNS --> PDNSDB[("Derived PowerDNS DB")]
+ DNSAPI --> PowerDNS
end
subgraph Edge["HTTP edge plane"]
Agent["Edge agent"] --> Gateway["Destination + Host/SNI gateway"]
@@ -43,13 +48,31 @@ flowchart TB
Laravel --> Valkey
Horizon --> Valkey
PG -->|"sanitized read-only metadata"| Grafana
- Horizon -->|"versioned reconciliation"| PowerDNS
- Agent -->|"outbound mTLS: pull artifacts/tasks, acknowledge"| Laravel
+ Horizon -->|"versioned reconciliation"| DNSAPI
+ Agent -->|"outbound mTLS: pull artifacts/tasks, acknowledge"| EdgeControl
+ EdgeControl --> Laravel
DNSdist -.-> Vector
Cell1 -.-> Vector
Cell2 -.-> Vector
```
+## DNS namespaces and addresses
+
+| Namespace or address | Owner and purpose | CDNFoundry PowerDNS? |
+| --- | --- | --- |
+| `control.`, `edge-control.`, `telemetry.`, `grafana.`, `dns-api-N.` | Independent external DNS provider; management and recovery reachability | Never |
+| `ns1.`, `ns2.` and glue | Parent/registrar delegation to public DNSdist addresses | Served by DNSdist from derived PowerDNS state after bootstrap |
+| Enrolled customer zones | PostgreSQL desired state reconciled into private PowerDNS databases | Yes |
+| Edge pool service addresses | Public HTTP/HTTPS addresses selected through platform/customer DNS | Stored as platform desired state, not management addresses |
+| Private PowerDNS, PostgreSQL, Valkey, ClickHouse, agent/status addresses | Host or private service networks | No public DNS required |
+
+::: danger Avoid a DNS bootstrap loop
+Management records must remain resolvable while CDNFoundry DNS is empty,
+degraded, or being restored. Hosting them in the platform's own PowerDNS can
+leave the DNS API and control plane unreachable precisely when operators need
+them for repair.
+:::
+
| Plane | Components | Responsibility |
| --- | --- | --- |
| Management | Laravel, Filament, Horizon, scheduler | Authorization, validation, desired state, operations, reconciliation |
@@ -60,7 +83,7 @@ flowchart TB
Only DNSdist, mapped edge-gateway service listeners, and the browser/API reverse proxy
belong on public ingress. Edge control uses mutual TLS. Telemetry and PowerDNS
-API gateways are source restricted in the production overlays. Internal
+API gateways are source restricted by the production Caddy configuration. Internal
databases, Valkey, ClickHouse, raw metrics, Grafana port 3000, and PowerDNS
itself remain private. Remote Grafana access uses a deployment-owned
authenticated HTTPS proxy or trusted tunnel.
diff --git a/docs/architecture/production-reference-architectures.md b/docs/architecture/production-reference-architectures.md
index 635d67d..2ffb173 100644
--- a/docs/architecture/production-reference-architectures.md
+++ b/docs/architecture/production-reference-architectures.md
@@ -42,18 +42,23 @@ domains.
```mermaid
flowchart LR
+ ExternalDNS["Independent external DNS
management records"] --> CONTROL
+ ExternalDNS --> DNSAPI["dns-api-N"]
+ ExternalDNS --> EC["edge-control"]
Operators["Operators"] --> CONTROL["CONTROL
control plane"]
CONTROL --> State[("PostgreSQL + Valkey")]
Resolver["Resolvers"] --> EDGE1DNS["EDGE_1
DNSdist + PowerDNS"]
Resolver --> EDGE2DNS["EDGE_2
DNSdist + PowerDNS"]
Clients["HTTP clients"] --> EDGE1GW["EDGE_1
gateway + cells"]
Clients --> EDGE2GW["EDGE_2
gateway + cells"]
- CONTROL -. "restricted DNS API" .-> EDGE1DNS
- CONTROL -. "restricted DNS API" .-> EDGE2DNS
+ CONTROL -. "revisioned reconciliation" .-> DNSAPI
+ DNSAPI --> EDGE1DNS
+ DNSAPI --> EDGE2DNS
EDGE1GW --> Origins["Validated origins"]
EDGE2GW --> Origins
- EDGE1GW -->|"outbound mTLS"| CONTROL
- EDGE2GW -->|"outbound mTLS"| CONTROL
+ EDGE1GW -->|"edge agent: outbound mTLS"| EC
+ EDGE2GW -->|"edge agent: outbound mTLS"| EC
+ EC --> CONTROL
```
Best for:
@@ -116,7 +121,7 @@ Best for:
- edge bandwidth or DNS load that should not compete on one host;
- a dedicated database or observability team;
- independent maintenance windows;
-- deployments using the external control-data or telemetry-data overlays.
+- deployments using owner-operated external control or telemetry data services.
Tradeoffs:
@@ -125,7 +130,7 @@ Tradeoffs:
- separating roles without separate failure domains mainly improves resource
isolation, not site resilience.
-CDNFoundry supplies role overlays and external endpoints. It does not supply a
+CDNFoundry supplies role profiles, generated node bundles, and external endpoint settings. It does not supply a
PostgreSQL, Valkey, or ClickHouse clustering product. The operator owns those
systems' quorum, fencing, failover, consistency, and restore qualification.
diff --git a/docs/contributing/documentation-audit.md b/docs/contributing/documentation-audit.md
index 1cffa27..83e272c 100644
--- a/docs/contributing/documentation-audit.md
+++ b/docs/contributing/documentation-audit.md
@@ -12,9 +12,10 @@ current automated and owner-run qualification evidence.
:::
This documentation system was reconstructed on 2026-07-26 and re-audited on
-2026-08-01 against release `v0.9.1` plus the current working tree. The previous
-corpus is preserved verbatim under `docs/legacy/` and in Git history. It is
-excluded from current navigation, search, lint, build, and link guarantees.
+2026-08-08 against the current working tree. The superseded corpus remains
+under `docs/legacy/` for historical context, with prohibited vendor comparisons
+removed. It is excluded from current navigation, search, lint, build, and link
+guarantees and is not an operational instruction set.
## Audited implementation surfaces
@@ -32,16 +33,18 @@ The audit covered every tracked project area:
| TLS/cache/security | controllers, support validators, jobs, runtime, feature and E2E tests |
| Telemetry | Vector transforms/sinks, ClickHouse DDL, analytics queries, metrics and alerts |
| Configuration | Laravel configs, both example environments, Compose interpolation, scripts |
-| Infrastructure | development/production Compose, every production overlay, Dockerfiles, Caddy |
+| Infrastructure | development/production Compose, generated role bundles, Dockerfiles, Caddy |
| Development and CI | Make targets, shell scripts, PHP/Go/Python tests, GitHub workflow/forms |
| Existing documentation | all legacy Markdown, generated OpenAPI, root guides, pull-request template |
The audit used file inventories, route registry output, environment-key
extraction, class/function indexes, migration constraints, Compose-rendered
-services, and targeted full-file review. The 2026-08-01 pass additionally
+services, and targeted full-file review. The 2026-08-08 pass additionally
checked bounded cells, gateway ingress, Geo-Unicast and Simple Anycast pools,
cache/compression/origin failover, managed WAF, fleet rollout, Grafana, Loki,
-production overlays, the environment generator, and every quick-start command.
+the single production Compose file, generated role bundles, the Fleet
+environment generator, management-DNS bootstrap independence, every current
+Mermaid diagram, and every quick-start command.
Generated dependency lockfiles were treated as dependency evidence, not prose
to paraphrase.
@@ -66,7 +69,7 @@ does not expand the implemented product boundary.
| Historical phase test counts were current qualification status. | Counts such as 118 or 140 tests apply only to their recorded commits; the current suite contains more tests and must be rerun. |
| Roadmap future stages, current operation, agent rules, and qualification evidence belonged in one public guide. | Governance and owner qualification are repository concerns, not public product documentation; legacy keeps the original proposal history. |
| Telemetry retention settings automatically define runtime TTLs. | Masking/finalization are active application policy, while shipped ClickHouse TTLs are static in `docker/clickhouse/init.sql` and require an operator migration to change. |
-| A public address must exist on each host so Docker can bind it. | Public/NAT addresses are advertised identities. Shared production overlays bind `HOST_BIND_IPV4`/`HOST_BIND_IPV6`; the edge gateway requires a complete advertised-to-private `EDGE_GATEWAY_ADDRESS_MAP` behind NAT or a layer-4 load balancer. |
+| A public address must exist on each host so Docker can bind it. | Public/NAT addresses are advertised identities. Generated bundles set `HOST_BIND_IPV4`/`HOST_BIND_IPV6`; the edge gateway requires a complete advertised-to-private `EDGE_GATEWAY_ADDRESS_MAP` behind NAT or a layer-4 load balancer. |
| Restic is required to render or start the control profile. | Built-in Restic backup is optional. Empty settings skip the daily job and fail backup requests explicitly while leaving serving available and backup health degraded. |
| One-shot migrations can bootstrap their own databases during first install. | The supported first-install sequence starts and health-checks PostgreSQL/Valkey or PowerDNS PostgreSQL before running its explicit migration container. |
diff --git a/docs/deployment/index.md b/docs/deployment/index.md
index fe3c8c3..c27c951 100644
--- a/docs/deployment/index.md
+++ b/docs/deployment/index.md
@@ -12,6 +12,12 @@ on production hosts and never migrates a database during container startup.
::: tip Recommended starting point
For a new installation, use the [starter Fleet quick start](production-quick-start.md). It copies a JSON topology, validates it, and generates complete per-node bundles without editing deployment scripts.
+Advanced operators who intentionally do not want Fleet can use the
+[manual Docker Compose deployment](manual-compose.md). It documents the same
+three-host outcome with hand-managed environments, secrets, PKI, migrations,
+enrollment, qualification, upgrades, and recovery, and invokes no repository
+scripts or Make targets.
+
Use the [Production quick start](production-quick-start.md) for the
complete three-host sequence: bootstrap DNS, private PKI, explicit migrations,
cluster qualification, edge enrollment, acceptance checks, and diagnosis.
@@ -19,7 +25,8 @@ cluster qualification, edge enrollment, acceptance checks, and diagnosis.
The minimum documented layout is one control/telemetry host plus two combined
DNS/edge hosts. The base file also supports colocated development-like
-qualification, while overlays expose split roles with restricted TLS gateways.
+qualification, while Compose profiles and generated bundles expose split roles
+with restricted TLS gateways.
Before deploying, read:
@@ -28,12 +35,14 @@ Before deploying, read:
2. [Production best practices](../operations/production-best-practices.md) for
the readiness and change contract.
3. [Production quick start](production-quick-start.md) for an end-to-end first installation.
-4. [Topology](topology.md) for networks, profiles, and public ports.
-5. [Certificates](certificates.md) for the edge-control and DNS API PKI.
-6. [Configuration](../reference/configuration.md) for every `.env.prod` key.
-7. [Upgrade](upgrade.md) for schema, worker, DNS, and edge sequencing.
+4. [Manual Docker Compose deployment](manual-compose.md) when deliberately
+ operating without Fleet, scripts, or Make.
+5. [Topology](topology.md) for networks, profiles, and public ports.
+6. [Certificates](certificates.md) for the edge-control and DNS API PKI.
+7. [Configuration](../reference/configuration.md) for every `.env.prod` key.
+8. [Upgrade](upgrade.md) for schema, worker, DNS, and edge sequencing.
-For separated roles across several failure domains, continue with the [multi-region Fleet quick start](production-quick-start-multi-region.md). The [Fleet operator guide](production-fleet-operator-guide.md) and [configuration reference](production-fleet-config-reference.md) cover lifecycle operations and the JSON schema.
+For separated roles across several failure domains, continue with the [multi-region Fleet quick start](production-quick-start-multi-region.md). The [Fleet operator guide](production-fleet-operator-guide.md), [configuration reference](production-fleet-config-reference.md), and [architecture reference](production-fleet.md) cover lifecycle operations, the JSON schema, and role boundaries.
The [Production quick start](production-quick-start.md) is the
authoritative first-install procedure. The remaining deployment pages explain
@@ -52,4 +61,4 @@ relevant.
See [Production quick start](production-quick-start.md) for the
verified command sequence and [Topology](topology.md) for the role and
-overlay model.
+profile and generated-bundle model.
diff --git a/docs/deployment/manual-compose.md b/docs/deployment/manual-compose.md
new file mode 100644
index 0000000..7b67467
--- /dev/null
+++ b/docs/deployment/manual-compose.md
@@ -0,0 +1,611 @@
+---
+title: Manual production deployment with Docker Compose
+description: Build and operate a CDNFoundry installation step by step with the checked-in production Compose file, without Fleet, Make, or repository scripts.
+---
+
+# Manual production deployment with Docker Compose
+
+This is the advanced, script-free installation path. It uses the checked-in
+`compose.prod.yml` and its checked-in configuration files directly. It does not
+use CDNFoundry Fleet, generated bundles, `make`, or any file under `scripts/`.
+
+You remain responsible for inventory, secret generation and custody, private
+PKI, host firewalls, DNS, transferring configuration between hosts, upgrades,
+backups, and recording what is installed. Compose starts the declared services;
+it does not replace those operator responsibilities.
+
+::: warning Meaning of "Compose only"
+The production Compose file pulls immutable published images. It does not build
+application images on a production host. Standard host tools such as `git`,
+`install`, `openssl`, `curl`, and `dig` are used to prepare and verify the
+installation, but every CDNFoundry service and migration is run through
+`docker compose`.
+:::
+
+## Resulting topology
+
+Use at least three Linux hosts in separate failure domains:
+
+| Host | Compose profiles | Public listeners |
+| --- | --- | --- |
+| `control-1` | `control`, optionally `telemetry` and `logs` | TCP 80/443, UDP 443, TCP 8443; TCP 8444 when telemetry is colocated |
+| `pop-1` | `dns`, `edge`, optionally `logs` | UDP/TCP 53, TCP 80/443 on mapped service addresses, TCP 8444 from control only |
+| `pop-2` | `dns`, `edge`, optionally `logs` | Same as `pop-1` |
+
+Do not deploy a single public DNS node. The control database and Valkey are
+single-host dependencies in this base topology; their availability and backups
+remain explicit operator concerns. PostgreSQL is desired state. PowerDNS data,
+edge snapshots, and telemetry are derived or rebuildable.
+
+Use two unrelated DNS zones:
+
+- an operator zone, such as `ops.example.com`, for control, Grafana, telemetry,
+ edge-control, and DNS API names;
+- a platform/customer zone, such as `example.net`, for nameservers, delegated
+ customer zones, and proxied hostnames.
+
+Keep the operator zone at your existing authoritative provider. Never delegate
+it to CDNFoundry.
+
+## 1. Record the installation plan
+
+Before touching a host, record:
+
+- the exact release tag or 40-character commit SHA;
+- hostnames, public and private IPv4/IPv6 addresses, NAT mappings, and failure
+ domains;
+- `control.ops.example.com`, `telemetry.ops.example.com`,
+ `grafana.ops.example.com`, and one `dns-api-N.ops.example.com` per DNS host;
+- `ns1.example.net` and `ns2.example.net` plus registrar glue addresses;
+- the advertised-to-local address map for each edge;
+- firewall sources for edge control, DNS API, telemetry ingestion, operational
+ logs, gateway metrics, and administration;
+- backup repository, retention, restoration owner, and recovery location for
+ application keys and private PKI.
+
+The local side of every `EDGE_GATEWAY_ADDRESS_MAP` entry must be a distinct
+private address that exists on that edge host. A firewall, router, or layer-4
+load balancer must map the advertised address one-to-one to it. Do not use the
+host wildcard, loopback, or the public/NAT address as the local value.
+
+## 2. Prepare every host
+
+Install a supported Linux distribution, Docker Engine, Docker Compose v2, Git,
+OpenSSL, curl, and DNS diagnostic tools. Configure clock synchronization and a
+host firewall. Permit outbound HTTPS for image pulls, ACME, MMDB updates, and
+origin access.
+
+Check out the same immutable revision on all three hosts:
+
+```bash
+git clone https://github.com/vaheed/CDNFoundry.git /opt/cdnfoundry
+cd /opt/cdnfoundry
+git checkout v1.0.0
+git rev-parse --verify HEAD
+docker version
+docker compose version
+```
+
+Replace `v1.0.0` with the selected immutable release. If GHCR requires
+authentication, use a read-only package token with `docker login ghcr.io`.
+
+Create protected configuration directories on each host:
+
+```bash
+sudo install -d -m 0750 /etc/cdnfoundry
+sudo install -d -m 0700 /etc/cdnfoundry/secrets
+sudo install -d -m 0700 /etc/cdnfoundry/pki
+cp .env.prod.example .env.prod
+chmod 0600 .env.prod
+```
+
+Never commit `.env.prod`, secret files, private keys, edge bootstrap tokens, or
+database dumps.
+
+## 3. Generate independent secrets
+
+Generate every value independently. Run these commands in a protected shell;
+do not paste their output into tickets or chat:
+
+```bash
+openssl rand -base64 32
+openssl rand -hex 32
+```
+
+Use the base64 result prefixed with `base64:` for `APP_KEY`. Use a new hex
+result for each of:
+
+- `EDGE_ARTIFACT_SIGNING_KEY`;
+- `CONTROL_DB_PASSWORD`;
+- `REDIS_PASSWORD`;
+- `PDNS_DB_PASSWORD`;
+- `PDNS_API_KEY`;
+- `CLICKHOUSE_PASSWORD`;
+- `GRAFANA_ADMIN_PASSWORD`;
+- `GRAFANA_CLICKHOUSE_PASSWORD`;
+- `GRAFANA_POSTGRES_PASSWORD`;
+- each edge host's `EDGE_STATUS_TOKEN`.
+
+Create the metrics token without printing it:
+
+```bash
+umask 077
+openssl rand -hex 32 | sudo tee /etc/cdnfoundry/secrets/metrics-token >/dev/null
+sudo chmod 0600 /etc/cdnfoundry/secrets/metrics-token
+```
+
+Use the same `APP_KEY`, artifact signing key, database credentials, metrics
+token, and telemetry credentials wherever the same logical installation needs
+them. Use a different edge status token on each edge. When backups are disabled,
+keep `RESTIC_REPOSITORY` and its credentials empty and
+`RESTIC_PASSWORD_FILE=/dev/null`. When enabled, create a different mode-`0600`
+Restic password file and keep the encrypted repository off-host.
+
+## 4. Create the private PKI manually
+
+Perform CA operations on a protected administration host. The following is a
+minimal OpenSSL procedure. Replace all example hostnames before running it.
+
+Create the edge-identity CA, which signs agent client certificates after
+one-time enrollment:
+
+```bash
+umask 077
+mkdir cdnfoundry-pki
+cd cdnfoundry-pki
+openssl ecparam -name prime256v1 -genkey -noout -out edge-identity-ca.key
+openssl req -x509 -new -sha256 -days 3650 \
+ -key edge-identity-ca.key -out edge-identity-ca.crt \
+ -subj '/CN=CDNFoundry edge identity CA' \
+ -addext 'basicConstraints=critical,CA:TRUE' \
+ -addext 'keyUsage=critical,keyCertSign,cRLSign'
+```
+
+Create a separate server CA:
+
+```bash
+openssl ecparam -name prime256v1 -genkey -noout -out edge-server-ca.key
+openssl req -x509 -new -sha256 -days 3650 \
+ -key edge-server-ca.key -out edge-server-ca.crt \
+ -subj '/CN=CDNFoundry private server CA' \
+ -addext 'basicConstraints=critical,CA:TRUE' \
+ -addext 'keyUsage=critical,keyCertSign,cRLSign'
+```
+
+For each private server certificate, create a key and CSR, then sign it with the
+server CA. This example creates the edge-control certificate:
+
+```bash
+openssl ecparam -name prime256v1 -genkey -noout \
+ -out edge-control-server.key
+openssl req -new -sha256 -key edge-control-server.key \
+ -out edge-control-server.csr \
+ -subj '/CN=control.ops.example.com' \
+ -addext 'subjectAltName=DNS:control.ops.example.com' \
+ -addext 'extendedKeyUsage=serverAuth'
+openssl x509 -req -sha256 -days 825 \
+ -in edge-control-server.csr \
+ -CA edge-server-ca.crt -CAkey edge-server-ca.key -CAcreateserial \
+ -copy_extensions copy -out edge-control-server.crt
+```
+
+Repeat that three-command sequence with these output names and SANs:
+
+| Files | Required SAN |
+| --- | --- |
+| `edge-runtime.crt/.key` | a bootstrap hostname owned by the edge deployment |
+| `dns-api-1.crt/.key` | `dns-api-1.ops.example.com` |
+| `dns-api-2.crt/.key` | `dns-api-2.ops.example.com` |
+
+Remove CSRs after verifying every certificate:
+
+```bash
+openssl verify -CAfile edge-server-ca.crt \
+ edge-control-server.crt edge-runtime.crt dns-api-1.crt dns-api-2.crt
+openssl x509 -in edge-control-server.crt -noout -subject -issuer -dates \
+ -ext subjectAltName
+```
+
+Distribute material as follows:
+
+| Host | Files |
+| --- | --- |
+| control | identity CA certificate and key, server CA certificate, edge-control certificate and key |
+| each edge | server CA certificate, edge-runtime certificate and key, that host's DNS API certificate and key |
+| offline recovery custody | both CA keys, both CA certificates, all server keys/certificates |
+
+Never put `edge-server-ca.key` or `edge-identity-ca.key` on an edge. On control,
+the shipped PHP-FPM process must read the identity CA key:
+
+```bash
+sudo chown root:82 /etc/cdnfoundry/pki/edge-identity-ca.key
+sudo chmod 0640 /etc/cdnfoundry/pki/edge-identity-ca.key
+sudo chmod 0600 /etc/cdnfoundry/pki/*-server.key
+```
+
+Keep all other private keys mode `0600`. Preserve the CA keys in encrypted,
+off-host recovery storage.
+
+## 5. Configure `.env.prod` on every host
+
+Edit the copied template rather than creating a shortened environment. Compose
+interpolates the complete model, including required values belonging to inactive
+profiles. Therefore every `.env.prod` must contain syntactically valid values
+for all required substitutions, while paths only need to exist on hosts that
+start the corresponding service.
+
+### Shared control values
+
+Set the exact same release and installation-wide credentials on all hosts:
+
+```dotenv
+CDNF_RELEASE=v1.0.0
+APP_KEY=base64:REPLACE_WITH_32_BYTE_BASE64_VALUE
+EDGE_ARTIFACT_SIGNING_KEY=REPLACE_WITH_UNIQUE_VALUE
+APP_URL=https://control.ops.example.com
+SESSION_SECURE_COOKIE=true
+CONTROL_HOSTNAME=control.ops.example.com
+TELEMETRY_HOSTNAME=telemetry.ops.example.com
+GRAFANA_HOSTNAME=grafana.ops.example.com
+EDGE_CONTROL_URL=https://control.ops.example.com:8443
+METRICS_TOKEN_FILE=/etc/cdnfoundry/secrets/metrics-token
+```
+
+Set all database, Valkey, PowerDNS, ClickHouse, Grafana, ACME, and backup values
+in the template. Do not reuse credentials. The base Compose topology expects
+the control host's local `control-db` and `redis`; leave `DB_HOST=control-db`,
+`REDIS_HOST=redis`, and their URL fields empty.
+
+### Control host
+
+Use these certificate paths and binds:
+
+```dotenv
+CONTROL_BIND=127.0.0.1:8080
+HOST_BIND_IPV4=0.0.0.0
+EDGE_CONTROL_BIND=0.0.0.0:8443
+EDGE_CONTROL_SERVER_CERTIFICATE=/etc/cdnfoundry/pki/edge-control-server.crt
+EDGE_CONTROL_SERVER_PRIVATE_KEY=/etc/cdnfoundry/pki/edge-control-server.key
+EDGE_IDENTITY_CA_CERTIFICATE=/etc/cdnfoundry/pki/edge-identity-ca.crt
+EDGE_IDENTITY_CA_PRIVATE_KEY=/etc/cdnfoundry/pki/edge-identity-ca.key
+PDNS_CA_CERTIFICATE=/etc/cdnfoundry/pki/edge-server-ca.crt
+```
+
+Set `EDGE_PUBLIC_*_ALLOWLIST` to the exact edge egress addresses allowed to
+submit telemetry. Set `LOG_SOURCE_*_ALLOWLIST` to the exact node sources allowed
+to push logs. Empty allowlists deny those remote paths.
+
+### DNS and edge host
+
+Give each host its own DNS API identity and registration fields:
+
+```dotenv
+DNS_BIND_V4=0.0.0.0
+HOST_BIND_IPV4=0.0.0.0
+DNS_API_HOSTNAME=dns-api-1.ops.example.com
+DNS_API_SERVER_CERTIFICATE=/etc/cdnfoundry/pki/dns-api-1.crt
+DNS_API_SERVER_PRIVATE_KEY=/etc/cdnfoundry/pki/dns-api-1.key
+EDGE_CONTROL_CA_CERTIFICATE=/etc/cdnfoundry/pki/edge-server-ca.crt
+EDGE_RUNTIME_TLS_CERTIFICATE=/etc/cdnfoundry/pki/edge-runtime.crt
+EDGE_RUNTIME_TLS_PRIVATE_KEY=/etc/cdnfoundry/pki/edge-runtime.key
+EDGE_GATEWAY_ADDRESS_MAP={"198.51.100.40":"10.20.0.40"}
+EDGE_ID=
+EDGE_BOOTSTRAP_TOKEN=
+```
+
+Use the real advertised and local addresses. Set
+`CONTROL_PUBLIC_*_ALLOWLIST` to the control worker's exact source address. JSON
+values in `.env.prod` remain on one line. `pop-2` uses its own API hostname,
+certificate, private local mapping, status token, log identity, edge UUID, and
+bootstrap token.
+
+Keep IPv6 absent until routing, firewalling, DNS, and external reachability all
+work. The base Compose file publishes IPv4; do not claim IPv6 service merely by
+setting `HOST_BIND_IPV6`.
+
+## 6. Validate and pull without helper commands
+
+Run on every host from `/opt/cdnfoundry`:
+
+```bash
+docker compose --env-file .env.prod -f compose.prod.yml config --quiet
+docker compose --env-file .env.prod -f compose.prod.yml config --profiles
+docker compose --env-file .env.prod -f compose.prod.yml pull
+```
+
+Review the fully rendered model without saving it to a shared location because
+it contains secrets:
+
+```bash
+docker compose --env-file .env.prod -f compose.prod.yml config
+```
+
+Confirm all image tags use the chosen immutable release and all bind-mounted
+files resolve to the intended absolute paths. A successful `config --quiet`
+checks Compose structure and interpolation; it does not check firewalls, file
+contents, certificate chains, or remote reachability.
+
+## 7. Start the control plane in dependency order
+
+Start and wait for PostgreSQL, Valkey, and the MMDB updater:
+
+```bash
+docker compose --env-file .env.prod -f compose.prod.yml \
+ --profile control up -d --wait --wait-timeout 180 \
+ control-db redis mmdb-updater
+```
+
+Run the application migration explicitly. Container startup never migrates the
+database:
+
+```bash
+docker compose --env-file .env.prod -f compose.prod.yml \
+ --profile tools run --rm migrate
+```
+
+Start the control services:
+
+```bash
+docker compose --env-file .env.prod -f compose.prod.yml \
+ --profile control up -d
+docker compose --env-file .env.prod -f compose.prod.yml \
+ --profile control ps
+```
+
+Create the first administrator interactively; the password is prompted and is
+not placed in shell history:
+
+```bash
+docker compose --env-file .env.prod -f compose.prod.yml \
+ --profile control run --rm core php artisan cdnf:admin:create \
+ --name='Operations Administrator' --email='admin@example.com'
+```
+
+Verify locally and externally:
+
+```bash
+curl --fail http://127.0.0.1:8080/api/health
+curl --fail http://127.0.0.1:8080/api/ready
+curl --fail https://control.ops.example.com/api/health
+```
+
+If a service is not healthy, inspect only that service first:
+
+```bash
+docker compose --env-file .env.prod -f compose.prod.yml ps
+docker compose --env-file .env.prod -f compose.prod.yml logs --tail=200 core
+docker compose --env-file .env.prod -f compose.prod.yml logs --tail=200 horizon
+```
+
+## 8. Start authoritative DNS on both PoPs
+
+On `pop-1`, start the PowerDNS database and MMDB updater first:
+
+```bash
+docker compose --env-file .env.prod -f compose.prod.yml \
+ --profile dns up -d --wait --wait-timeout 180 pdns-db mmdb-updater
+docker compose --env-file .env.prod -f compose.prod.yml \
+ --profile tools run --rm pdns-migrate
+docker compose --env-file .env.prod -f compose.prod.yml \
+ --profile dns up -d
+docker compose --env-file .env.prod -f compose.prod.yml \
+ --profile dns ps
+```
+
+Repeat on `pop-2`. The initial schema is mounted into PostgreSQL's init
+directory and applies only to a new `pdns-db` volume. The explicit
+`pdns-migrate` service applies maintained runtime migrations and is safe to run
+again. Never delete the volume to force initialization.
+
+Before configuring clusters, confirm DNSdist answers on UDP and TCP and the DNS
+API certificate matches its hostname:
+
+```bash
+dig @127.0.0.1 version.bind TXT CH +short
+dig +tcp @127.0.0.1 version.bind TXT CH +short
+openssl s_client -connect dns-api-1.ops.example.com:8444 \
+ -servername dns-api-1.ops.example.com \
+ -CAfile /etc/cdnfoundry/pki/edge-server-ca.crt STATE[Protected desired state]
- STATE --> C[Control bundle]
- STATE --> D[DNS bundles]
- STATE --> E[Edge bundles]
- STATE --> M[Monitoring bundle]
+ JSON["Fleet JSON
roles, addresses, nullable IPv6"] --> STATE["Protected Fleet authority
secrets + private PKI + topology"]
+ STATE --> C["CONTROL bundle
Laravel + PostgreSQL + Valkey"]
+ STATE --> D["DNS bundles
DNSdist + private PowerDNS + DNS API"]
+ STATE --> E["Edge bundles
agent + gateway + OpenResty cells"]
+ STATE --> M["Telemetry bundle
Vector + ClickHouse + Grafana"]
```
Copy `deploy/production/examples/starter-fleet.json` or `multi-region-fleet.json` to a protected local `fleet.json`, then change deployment data there. Checked-in examples are templates; repository scripts and generated Compose manifests are not configuration surfaces.
@@ -24,7 +24,7 @@ These options work before or after a subcommand:
| --- | --- | --- |
| `--state-dir` | `/var/lib/cdnfoundry-fleet` | Protected authoritative fleet state |
| `--output-dir` | `/var/lib/cdnfoundry-fleet/bundles` | Generated per-node bundles |
-| `--repo-root` | Repository containing the script | Base Compose and production overlays |
+| `--repo-root` | Repository containing the script | Base production Compose file and deployment assets |
| `--config` | none | JSON input for setup or node commands |
| `--non-interactive` | false | Never prompt; fail when required input is absent |
| `--dry-run` | false | Validate intent without writing state or bundles |
@@ -216,11 +216,11 @@ DNS nodes may additionally receive `reconcile-pdns-password.sh` and a pending pa
## Security properties
- State directories use mode `0700`.
-- State, secrets, environment files, manifests, and private keys use mode `0600`.
+- State, secrets, environment files, manifests, and private keys use mode `0600` at render and transfer time. During control activation, generated `start.sh` changes only `pki/edge-identity-ca.key` to owner `root`, numeric group `82`, mode `0640`, so the core image's PHP-FPM worker can read the signing key.
- Node bundles are assembled in temporary directories and activated atomically.
- Normal rendering does not rotate secrets.
- DNS database credentials are node-scoped.
- CA private keys remain in authoritative fleet state, except the edge identity CA key required by the control service in the control bundle.
- Bundle metadata contains hashes and non-secret inventory only.
-- Every operator-controlled Compose interpolation value is present in the node's generated `.env.prod`; production Compose and role overlays provide no fallback deployment values.
+- Every operator-controlled Compose interpolation value is present in the node's generated `.env.prod`; production Compose provides no fallback deployment values.
- Compose `environment` mappings remain explicit per-service allowlists. Replacing them with a shared `env_file` entry would expose unrelated database, PKI, and API credentials to every container, so containers receive only the variables they own while Compose reads values through `--env-file .env.prod`.
diff --git a/docs/deployment/production-fleet-operator-guide.md b/docs/deployment/production-fleet-operator-guide.md
index 1d61cfd..73ae993 100644
--- a/docs/deployment/production-fleet-operator-guide.md
+++ b/docs/deployment/production-fleet-operator-guide.md
@@ -7,10 +7,10 @@ description: Complete lifecycle guide for CDNFoundry production fleets including
```mermaid
flowchart LR
- A[Fleet authority] --> C[Control-only env and PKI]
- A --> D[DNS-only env and PKI]
- A --> E1[Edge A env and identity]
- A --> E2[Edge B env and identity]
+ A["Protected Fleet authority
topology + secrets + private PKI"] --> C["CONTROL bundle
control profile"]
+ A --> D["DNS bundle
DNSdist + private PowerDNS + DNS API"]
+ A --> E1["Edge A bundle
agent + gateway + cells"]
+ A --> E2["Edge B bundle
agent + gateway + cells"]
```
Each bundle is a security boundary. Never use a shared `env_file`: database, PowerDNS, bootstrap, identity, backup, and telemetry credentials are emitted only when that node's filtered services require them. Never copy a bundle, `.env.prod`, `pki/`, or `secrets/` directory between nodes.
@@ -202,7 +202,7 @@ Run before setup or after updating the repository:
./scripts/cdnfoundry-fleet --repo-root "$PWD" doctor
```
-`doctor` verifies the base Compose file, role overlays, Python, OpenSSL, and existing fleet state. Docker is reported separately because rendering can occur on a generator machine without starting containers, but Docker Compose is required on deployment hosts and for the final host-side validation.
+`doctor` verifies the production Compose file, deployment assets, Python, OpenSSL, and existing fleet state. Docker is reported separately because rendering can occur on a generator machine without starting containers, but Docker Compose is required on deployment hosts and for the final host-side validation.
## Node roles
@@ -233,7 +233,7 @@ The generator follows the production repository’s two-CA model:
- `edge-identity-ca`: used by the control plane for edge identity issuance and verification.
- `edge-server-ca`: signs edge-control, edge runtime, and DNS API TLS certificates.
-CA private keys stay in the protected fleet state directory. Every node bundle receives the edge server CA certificate plus its own certificate and private key. Only the control bundle receives the edge identity CA private key because the control service requires it.
+CA private keys stay in the protected fleet state directory. Every node bundle receives the edge server CA certificate plus its own certificate and private key. Only the control bundle receives the edge identity CA private key because the control service requires it. The transferred key begins root-only; the generated control `start.sh` must run as root and changes only this key to owner `root`, numeric group `82`, mode `0640`, allowing the immutable image's PHP-FPM worker to read it without making it public.
Important generated environment paths include:
@@ -390,10 +390,14 @@ cd /opt
mv cdnfoundry cdnfoundry.previous 2>/dev/null || true
mv cdnfoundry.new cdnfoundry
cd /opt/cdnfoundry
-./start.sh
+sudo ./start.sh
```
-Never transfer the entire fleet state or another node’s bundle.
+Never transfer the entire fleet state or another node’s bundle. Do not replace the generated control activation with a direct `docker compose up`: the activation applies the restricted PHP-worker access required for `pki/edge-identity-ca.key`. If `core` reports that the key is not readable, rerun `sudo ./start.sh` and verify `stat -c '%u:%g %a %n' pki/edge-identity-ca.key` reports `0:82 640`.
+
+`validate.sh` runs the pinned Caddy image's adapter against each Caddyfile present in the bundle. Treat any adapter error as a failed bundle and do not activate it. The validation container is short-lived and does not start dependencies; Docker may create the bundle's declared network or empty named-volume metadata while preparing the container.
+
+If `log-collector` exits with code `78` and reports that `LOG_AUTH_TOKEN` is missing, do not put the token directly in the rendered Compose manifest. Verify that `.env.prod` contains a non-empty `LOG_AUTH_TOKEN` and that the rendered `log-collector.environment` maps `LOG_AUTH_TOKEN` from Compose interpolation, then rerender and transfer the corrected node bundle. Recreate only `log-collector`; its bounded `operational-vector-data` volume preserves buffered logs.
## Updating the fleet
@@ -470,9 +474,9 @@ The full setup command reuses existing state. It does not rotate secrets. To ins
The renderer derives required variables from the final filtered Compose file. Add project-specific values to a node’s `extra_env` only when the repository introduces a new required variable that the generator does not yet know. Do not place secrets in version-controlled config files.
-### Compose overlay contains `!reset` or `!override`
+### Compose input contains `!reset` or `!override`
-The fleet loader supports both Docker Compose tags. Older generator versions used `yaml.safe_load` directly and failed on these overlays.
+The Fleet loader supports both Docker Compose tags. Older generator versions used `yaml.safe_load` directly and failed on Compose inputs containing them.
### Docker is unavailable on the generator machine
diff --git a/docs/deployment/production-fleet.md b/docs/deployment/production-fleet.md
index 22d6f11..942775e 100644
--- a/docs/deployment/production-fleet.md
+++ b/docs/deployment/production-fleet.md
@@ -21,28 +21,33 @@ For the complete command workflow and interactive setup, see [Production fleet o
```mermaid
flowchart TB
- CP[Control-plane host\nGenerator + application DB + Valkey]
- M[Optional monitoring host\nPrometheus + Grafana + ClickHouse/Loki]
- D1[DNS host A\nPowerDNS + local PostgreSQL]
- D2[DNS host B\nPowerDNS + local PostgreSQL]
- DN[DNS host N\nPowerDNS + local PostgreSQL]
- E1[Edge host A]
- E2[Edge host B]
- EN[Edge host N]
- CP -->|signed desired state / DNS API| D1
- CP -->|signed desired state / DNS API| D2
- CP -->|signed desired state / DNS API| DN
- CP -->|signed edge configuration| E1
- CP -->|signed edge configuration| E2
- CP -->|signed edge configuration| EN
- D1 -. metrics/logs .-> M
- D2 -. metrics/logs .-> M
- DN -. metrics/logs .-> M
- E1 -. metrics/logs .-> M
- E2 -. metrics/logs .-> M
- EN -. metrics/logs .-> M
+ ExternalDNS["Independent external DNS
operator-zone management names"] --> CONTROL["CONTROL
Laravel + Horizon + Scheduler"]
+ ExternalDNS --> EDGE_CONTROL["edge-control
mTLS ingress"]
+ ExternalDNS --> DNS_API["dns-api-N
restricted TLS"]
+ ExternalDNS --> TELEMETRY["telemetry/Grafana ingress"]
+ CONTROL --> PG[("PostgreSQL
desired state")]
+ CONTROL --> V[("Valkey
queues and locks")]
+ CONTROL -->|"revisioned reconciliation"| DNS_API
+ DNS_API --> PDNS["Private PowerDNS
one derived DB per DNS host"]
+ Resolvers["Recursive resolvers"] -->|"UDP/TCP 53"| DNSDIST["DNSdist
public authoritative ingress"]
+ DNSDIST --> PDNS
+ Agents["Edge agents"] -->|"outbound mTLS"| EDGE_CONTROL
+ EDGE_CONTROL --> CONTROL
+ Agents --> CELLS["Gateway + bounded OpenResty cells"]
+ Clients["HTTP/HTTPS clients"] --> CELLS
+ CELLS --> Origins["Validated origins"]
+ DNSDIST -. "bounded telemetry" .-> TELEMETRY
+ CELLS -. "bounded telemetry" .-> TELEMETRY
+ TELEMETRY --> CH[("ClickHouse + metrics + logs")]
```
+::: danger Keep management DNS independent
+The operator-zone records shown above must be published by an independent
+external DNS provider. CDNFoundry PowerDNS is private derived runtime state for
+platform and customer zones; using it for management names creates a bootstrap
+and recovery dependency.
+:::
+
## DNS and geo-routing flow
```mermaid
diff --git a/docs/deployment/production-quick-start-multi-region.md b/docs/deployment/production-quick-start-multi-region.md
index bc39e25..f4dc5c9 100644
--- a/docs/deployment/production-quick-start-multi-region.md
+++ b/docs/deployment/production-quick-start-multi-region.md
@@ -7,15 +7,30 @@ description: Deploy a separated-role CDNFoundry fleet across multiple regions fr
```mermaid
flowchart TB
- CF[Cloudflare: ops.example.com] --> CP[Control and Grafana]
- REG[example.net delegation] --> D1[DNS region A]
- REG --> D2[DNS region B]
- CP --> D1
- CP --> D2
- CP --> E1[Edge region A]
- CP --> E2[Edge region B]
+ ExternalDNS["Independent external DNS provider
operator zone"] --> CONTROL["CONTROL
Laravel + workers"]
+ ExternalDNS --> EDGEAPI["edge-control.operator-zone
mTLS ingress"]
+ ExternalDNS --> DNSAPIS["dns-api-N.operator-zone
restricted DNS APIs"]
+ ExternalDNS --> TELEMETRY["telemetry.operator-zone
Vector + ClickHouse + Grafana"]
+ CONTROL --> PG[("PostgreSQL
desired state")]
+ CONTROL -->|"revisioned reconciliation"| DNSAPIS
+ DNSAPIS --> PDNS["Private PowerDNS on each DNS host"]
+ Resolvers["Recursive resolvers"] --> DNSDIST["DNSdist in each region"]
+ DNSDIST --> PDNS
+ Clients["HTTP clients"] --> GATEWAYS["Regional gateways + bounded OpenResty cells"]
+ GATEWAYS --> Origins["Validated origins"]
+ Agents["Edge agents"] -->|"outbound mTLS"| EDGEAPI
+ DNSDIST -. "bounded telemetry" .-> TELEMETRY
+ GATEWAYS -. "bounded telemetry" .-> TELEMETRY
```
+::: danger Keep management DNS outside CDNFoundry
+Publish `control.`, `edge-control.`,
+`telemetry.`, and every `dns-api-N.` through an
+independent external DNS provider. Do not put the operator zone in CDNFoundry
+PowerDNS. CDNFoundry PowerDNS contains derived platform/customer runtime state
+and must not be required to find the services that repair or manage it.
+:::
+
This example models one control node, four authoritative DNS nodes, ten edge nodes, and three monitoring-role nodes. “Multi-region” describes its failure-domain design; it is not a special runtime mode or a fixed scale limit.
Read and complete the [starter fleet quick start](production-quick-start.md) first. The same security, PKI, transfer, migration, enrollment, last-valid-state, backup, and acceptance rules apply.
diff --git a/docs/deployment/production-quick-start.md b/docs/deployment/production-quick-start.md
index ad76802..ab5033c 100644
--- a/docs/deployment/production-quick-start.md
+++ b/docs/deployment/production-quick-start.md
@@ -6,18 +6,40 @@ description: Deploy CDNFoundry with one control node and two combined DNS and ed
# Production quick start: starter fleet
```mermaid
-flowchart LR
- CF[Cloudflare DNS: ops.example.com] --> C[control.ops.example.com]
- CF --> G[grafana.ops.example.com]
- CF --> P1[pop-1.ops.example.com]
- CF --> P2[pop-2.ops.example.com]
- R[example.net delegation] --> P1
- R --> P2
- C -->|mTLS control| P1
- C -->|mTLS control| P2
+flowchart TB
+ ExternalDNS["Independent external DNS provider
ops.example.com"] --> CONTROL["CONTROL
control.ops.example.com
Laravel + workers"]
+ ExternalDNS --> EC["edge-control.ops.example.com
edge-agent mTLS ingress"]
+ ExternalDNS --> TI["telemetry.ops.example.com
restricted telemetry ingress"]
+ ExternalDNS --> API1["dns-api-1.ops.example.com
restricted DNS API"]
+ ExternalDNS --> API2["dns-api-2.ops.example.com
restricted DNS API"]
+ CONTROL --> PG[("PostgreSQL
desired state")]
+ CONTROL -->|"asynchronous DNS reconciliation"| API1
+ CONTROL -->|"asynchronous DNS reconciliation"| API2
+ API1 --> PDNS1["POP 1: private PowerDNS"]
+ API2 --> PDNS2["POP 2: private PowerDNS"]
+ Resolver["Recursive resolvers"] -->|"UDP/TCP 53"| DD1["POP 1: DNSdist"]
+ Resolver -->|"UDP/TCP 53"| DD2["POP 2: DNSdist"]
+ DD1 --> PDNS1
+ DD2 --> PDNS2
+ Client["HTTP clients"] --> GW1["POP 1: gateway + OpenResty cells"]
+ Client --> GW2["POP 2: gateway + OpenResty cells"]
+ GW1 --> Origins["Validated origins"]
+ GW2 --> Origins
+ GW1 -->|"edge agent: outbound mTLS"| EC
+ GW2 -->|"edge agent: outbound mTLS"| EC
```
-`ops.example.com` and `example.net` are intentionally unrelated zones. Cloudflare remains authoritative for the operational zone: create DNS-only A and optional AAAA records for control, Grafana, telemetry, and every node. PowerDNS owns `example.net` and enrolled customer zones; never delegate `ops.example.com` to CDNFoundry.
+::: danger Keep management DNS independent
+`ops.example.com` and `example.net` are intentionally separate zones. Host
+`control`, `edge-control`, `telemetry`, `grafana`, and every `dns-api-N` record
+for the operator zone with an independent external DNS provider. Never host or
+delegate the operator zone in CDNFoundry's own PowerDNS: that database is
+derived runtime state, so using it for management names creates a bootstrap
+dependency and can break control, recovery, and DNS reconciliation.
+:::
+
+CDNFoundry owns the platform zone (`example.net`) and enrolled customer zones.
+Only DNSdist is public on port 53; PowerDNS and its database remain private.
This runbook creates the smallest practical production CDNFoundry fleet:
@@ -53,14 +75,14 @@ install -m 0600 deploy/production/examples/starter-fleet.json ./fleet.json
Edit `fleet.json` and replace every example value:
-- `operator_domain`: private operator DNS suffix for control, node, and telemetry names;
+- `operator_domain`: independently hosted management DNS suffix for control, DNS API, edge-control, node, and telemetry names;
- `platform_domain`: customer-facing CDN platform suffix;
- `release`: the exact checked-out tag or 40-character commit SHA;
- `acme_email`: monitored certificate contact;
- every `hostname`, `public_ipv4`, region, and location;
- `public_ipv6` and `bind_ipv6` when deploying dual stack.
-Keep `public_ipv6`, `bind_ipv6`, `monitor_ipv6`, and `log_ipv6` in every node object and set unavailable paths to JSON `null`. Set global `ipv6` to `true` only after Cloudflare AAAA records, host routes, firewalls, and external reachability are ready.
+Keep `public_ipv6`, `bind_ipv6`, `monitor_ipv6`, and `log_ipv6` in every node object and set unavailable paths to JSON `null`. Set global `ipv6` to `true` only after the independent DNS provider's AAAA records, host routes, firewalls, and external reachability are ready.
The checked-in addresses are RFC documentation ranges and cannot serve production traffic. Keep `bind_ipv4` as `0.0.0.0` for normal routed/NAT hosts unless a specific local interface address is required.
@@ -104,7 +126,7 @@ sudo ./scripts/cdnfoundry-fleet \
show-start-order
```
-For every bundle, verify `SHA256SUMS`, review `README.md`, and run `./validate.sh`. Production Compose has no deployment-value defaults: all interpolation comes from that bundle's generated `.env.prod`.
+For every bundle, verify `SHA256SUMS`, review `README.md`, and run `./validate.sh`. Validation uses the pinned Caddy images to parse every Caddyfile included in that node before activation, in addition to checking Compose interpolation, permissions, and certificate chains. It may pull a missing pinned image and create a short-lived validation container, but it does not start the application services. Production Compose has no deployment-value defaults: all interpolation comes from that bundle's generated `.env.prod`.
## 5. Start the control plane
@@ -114,10 +136,12 @@ Transfer `bundles/control-1` over an authenticated channel to `/opt/cdnfoundry`
cd /opt/cdnfoundry
sha256sum -c SHA256SUMS
./validate.sh
-./start.sh
+sudo ./start.sh
docker compose --env-file .env.prod ps
```
+Run the control bundle's `start.sh` as root. Before starting Compose, it keeps the edge identity CA signing key restricted while changing it from the transfer-safe root-only mode to owner `root`, numeric group `82`, mode `0640`; group `82` is the PHP-FPM worker in the immutable core image. Without this activation step, `core` deliberately refuses to start because its worker cannot read the signing key. Other private keys remain mode `0600`.
+
The control bundle starts `mmdb-updater` before services that consume GeoIP data. Run migrations only through the generated `start.sh`/tools workflow; container startup never migrates the database.
## 6. Configure DNS desired state
diff --git a/docs/deployment/topology.md b/docs/deployment/topology.md
index 14759fa..18c232b 100644
--- a/docs/deployment/topology.md
+++ b/docs/deployment/topology.md
@@ -1,6 +1,6 @@
---
title: Production topology and Compose roles
-description: Understand CDNFoundry control, DNS, edge, and telemetry roles, networks, listeners, and Compose overlays.
+description: Understand CDNFoundry control, DNS, edge, and telemetry roles, networks, listeners, and Compose profiles.
---
# Production topology and Compose roles
@@ -16,8 +16,8 @@ diagnostics.
The smallest serving layout documented by the repository is:
-1. `CONTROL`, running `control` plus the control Caddy overlay;
-2. `EDGE_1` and `EDGE_2`, each running `dns` and `edge` plus the DNS API overlay.
+1. `CONTROL`, running the `control` profile and public Caddy ingress;
+2. `EDGE_1` and `EDGE_2`, each running the `dns` and `edge` profiles.
After DNS and HTTP serving qualify, the operator may enable the `telemetry`
profile on `CONTROL` and one `logs` collector per host. This adds
@@ -29,9 +29,13 @@ addresses. This layout is not automatic high availability: PostgreSQL, Valkey,
ClickHouse, backup storage, external firewalls, routing, and failure procedures
remain operator responsibilities.
-Use one operator-owned DNS zone for control, edge-control, telemetry, and DNS API
-hostnames. Use a separate platform DNS zone for CDNFoundry nameservers, proxy
-hostnames, and pool records.
+::: danger Management DNS is an external dependency
+Use an independently hosted operator DNS zone for `control`, `edge-control`,
+`telemetry`, `grafana`, and `dns-api-N` management hostnames. Never serve this
+zone from CDNFoundry PowerDNS. Use a separate platform DNS zone for CDNFoundry
+nameservers, proxy hostnames, and pool records; those records are desired state
+that CDNFoundry derives into its private PowerDNS databases.
+:::
## Deployment sequence overview
@@ -109,10 +113,10 @@ make config-check
docker compose --env-file .env.prod -f compose.prod.yml config --quiet
```
-Validate each exact overlay combination before starting it. The repository's
-`scripts/validate-production-overrides.sh` qualifies control, combined DNS/edge,
-DNS-only, edge-only, telemetry-only, opt-in IPv6, and external control-data
-configurations with documentation addresses.
+Validate each exact profile combination before starting it. `compose.prod.yml`
+defines the `control`, `dns`, `edge`, `telemetry`, `logs`, and one-shot `tools`
+profiles; generated Fleet bundles retain only the services assigned to that
+node.
### 7. Start `CONTROL`
@@ -121,15 +125,12 @@ and only then start application processes:
```sh
docker compose --env-file .env.prod \
- -f compose.prod.yml \
-f compose.prod.yml \
--profile control up -d --wait --wait-timeout 120 control-db redis
docker compose --env-file .env.prod \
- -f compose.prod.yml \
-f compose.prod.yml \
--profile tools run --rm migrate
docker compose --env-file .env.prod \
- -f compose.prod.yml \
-f compose.prod.yml \
--profile control up -d
```
@@ -151,15 +152,12 @@ migration, then start the profiles:
```sh
docker compose --env-file .env.prod \
- -f compose.prod.yml \
-f compose.prod.yml \
--profile dns up -d --wait --wait-timeout 120 pdns-db
docker compose --env-file .env.prod \
- -f compose.prod.yml \
-f compose.prod.yml \
--profile tools run --rm pdns-migrate
docker compose --env-file .env.prod \
- -f compose.prod.yml \
-f compose.prod.yml \
--profile dns --profile edge up -d
```
@@ -202,15 +200,23 @@ From external networks, verify:
Run the current non-browser qualification suite. Record the exact revision,
topology, operation IDs, certificate fingerprints, checks, and deviations.
-## Split-role overlays
-
-- `compose.prod.yml` adds the DNS API gateway to a DNS-only host.
-- `compose.prod.yml` documents the base edge-only role; the base file owns its listeners.
-- `compose.prod.yml` adds public, source-restricted telemetry TLS.
-- `compose.prod.yml` disables local PostgreSQL and Valkey.
-- `compose.prod.yml` disables local ClickHouse while Grafana
- and Vector use the configured external telemetry endpoint.
-- `*-ipv6.yml` files explicitly add IPv6 publications.
+## Split-role profiles
+
+- `control` runs Laravel, web ingress, Horizon, Scheduler, edge-control, and its
+ default local PostgreSQL and Valkey dependencies.
+- `dns` runs DNSdist, private PowerDNS, its local PostgreSQL runtime database,
+ and the restricted DNS API gateway.
+- `edge` runs the agent, destination/Host/SNI gateway, bounded cells, and edge
+ traffic Vector.
+- `telemetry` runs ClickHouse, Prometheus, Alertmanager, Grafana, Loki, and
+ telemetry ingress; `logs` runs one host log collector.
+- `tools` contains explicit migrations and other one-shot helpers. It is not a
+ long-running role.
+
+Fleet can render separated-role hosts and typed external data endpoints without
+requiring operators to edit Compose. Review each generated Compose manifest and
+`.env.prod` for its node rather than assuming services from another role are
+present.
Use [Scaling](../operations/scaling.md) before splitting data services or adding
workers.
diff --git a/docs/deployment/upgrade.md b/docs/deployment/upgrade.md
index 59dfea1..1f72597 100644
--- a/docs/deployment/upgrade.md
+++ b/docs/deployment/upgrade.md
@@ -17,7 +17,7 @@ the proven schema compatibility envelope.
2. Create and verify an encrypted off-host control backup.
3. Retain `.env.prod`, `APP_KEY`, artifact signing key, both CA keys,
listener identities, Restic password, and externally held custom TLS material.
-4. Validate target Compose and production overlays.
+4. Validate the target production Compose file and every generated node bundle.
5. Review migrations for expand/contract compatibility.
6. Run the target's automated and real-runtime qualification.
diff --git a/docs/development/project-layout.md b/docs/development/project-layout.md
index 039d7e9..aea88be 100644
--- a/docs/development/project-layout.md
+++ b/docs/development/project-layout.md
@@ -30,7 +30,7 @@ remain rebuildable rather than becoming hidden sources of truth.
| `docker/prometheus`, `docker/alertmanager` | Metrics and alerts |
| `compose.dev.yml` | Persistent full development topology |
| `compose.prod.yml` | Role-based production service definitions |
-| `deploy/production/` | Public/split-host/IPv6/external-data overlays |
+| `deploy/production/` | Fleet examples and production deployment assets |
| `scripts/` | PKI, environment, and Compose validation helpers |
| `tests/e2e/` | Agent-owned non-UI real-runtime qualification |
| `tests/docs/` | Documentation validation |
diff --git a/docs/development/scripts-and-ci.md b/docs/development/scripts-and-ci.md
index 74b360b..2a8f49e 100644
--- a/docs/development/scripts-and-ci.md
+++ b/docs/development/scripts-and-ci.md
@@ -43,7 +43,7 @@ log collector.
| Job | Checks |
| --- | --- |
| `core` | PHP 8.4 setup, Composer install/validate/audit, Node 24 npm install/audit, Pint, frontend build, Laravel tests, OpenAPI |
-| `compose` | development/production/overlay config, docs validation, Python syntax, every production image, read-only core smoke |
+| `compose` | development/production Compose config, docs validation, Python syntax, every production image, read-only core smoke |
| `backend-e2e` | full real development stack, migrations, cumulative Python non-UI E2E |
| `scale-e2e` | bounded control stack and 500,000-zone/1,000,000-record mutation qualification |
| `go` | Go 1.24 formatting, vet, tests, build |
diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md
index 330b007..f06e6f5 100644
--- a/docs/getting-started/index.md
+++ b/docs/getting-started/index.md
@@ -27,15 +27,21 @@ clients authenticate with Laravel Sanctum bearer tokens.
## Choose your path
-- If CDN concepts are new, begin with [CDN fundamentals](../concepts/cdn-fundamentals.md).
-- To understand the product end to end, read [How CDNFoundry works](../concepts/how-cdnfoundry-works.md).
-- To evaluate a company or ISP deployment, read [How to build a private CDN](private-cdn-design.md).
-- To run the repository locally, follow [Installation](installation.md).
-- To onboard a domain, follow [First domain](first-domain.md).
-- For common administrator, domain-user, and API tasks, use [Using CDNFoundry](using-cdnfoundry.md).
-- To understand state and failure guarantees, read [Desired state](../concepts/desired-state.md).
-- To deploy real hosts, start with the [Production quick start](../deployment/production-quick-start.md).
-- To integrate over HTTP, use the [API reference](../reference/api/index.md).
+| Audience | Start here | Continue with |
+| --- | --- | --- |
+| New to CDNs | [CDN fundamentals](../concepts/cdn-fundamentals.md) | [How CDNFoundry works](../concepts/how-cdnfoundry-works.md), then [Desired state](../concepts/desired-state.md) |
+| Developer or contributor | [Local installation](installation.md) | [Developer setup](../development/index.md) and [Testing](../development/testing.md) |
+| Administrator or domain user | [Using CDNFoundry](using-cdnfoundry.md) | [First domain](first-domain.md), then the [feature guides](../guides/index.md) |
+| Hosting provider or ISP architect | [Private CDN design](private-cdn-design.md) | [Production reference architectures](../architecture/production-reference-architectures.md) |
+| Production operator | [Production quick start](../deployment/production-quick-start.md) | [Production best practices](../operations/production-best-practices.md), then [Operations](../operations/index.md) |
+| API integrator | [API conventions](../reference/api/index.md) | [Endpoint catalog](../reference/api/endpoints.md) and [Errors](../reference/api/errors.md) |
+
+::: tip Recommended learning order
+For a first deployment, follow concepts → local installation → first domain →
+production quick start → operations. The quick start owns installation order;
+reference pages explain individual settings and should not be assembled into a
+second deployment procedure.
+:::
## Current product boundary
diff --git a/docs/getting-started/private-cdn-design.md b/docs/getting-started/private-cdn-design.md
index abb9ded..ecf3ef3 100644
--- a/docs/getting-started/private-cdn-design.md
+++ b/docs/getting-started/private-cdn-design.md
@@ -208,7 +208,7 @@ not enter serving, ingestion, or reconciliation paths.
This is a reliability minimum, not a scale maximum. Larger ISPs can separate
DNS, edge, telemetry, control workers, PostgreSQL, and Valkey using the shipped
-role overlays and external endpoints.
+role profiles, generated bundles, and external endpoints.
## Capacity planning for an ISP
diff --git a/docs/index.md b/docs/index.md
index 270ce61..8cd652d 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -51,15 +51,22 @@ asynchronously through revisioned reconciliation, and invalid candidates never
replace the last valid runtime state.
```mermaid
-flowchart LR
+flowchart TB
+ ExternalDNS["Independent external DNS
management hostnames"] --> Control
+ ExternalDNS --> DNSAPI["Restricted DNS API"]
+ ExternalDNS --> EdgeControl["Edge-control mTLS ingress"]
Users["Internet users"] -->|"DNS"| DNS["DNSdist + PowerDNS"]
Users -->|"HTTP/HTTPS"| Gateway["Edge gateway"]
Gateway --> Edge["Bounded OpenResty cells"]
Edge --> Origin["Validated customer origins"]
Admins["Administrators"] --> Control["Laravel + Filament"]
Control --> State[("PostgreSQL desired state")]
- Control -->|"asynchronous reconciliation"| DNS
- Control -->|"signed snapshots"| Edge
+ Control -->|"asynchronous reconciliation"| DNSAPI
+ DNSAPI --> DNS
+ EdgeAgent["Edge agent"] -->|"outbound mTLS"| EdgeControl
+ EdgeControl --> Control
+ Control -->|"signed snapshots through agent"| EdgeAgent
+ EdgeAgent --> Edge
DNS -. "best-effort telemetry" .-> Vector["Vector"]
Edge -. "best-effort telemetry" .-> Vector
Vector --> ClickHouse[("ClickHouse telemetry")]
@@ -69,6 +76,14 @@ flowchart LR
Admins -->|"separate operator access"| Grafana
```
+::: danger Management DNS must be independent
+Management names such as `control.`,
+`edge-control.`, `telemetry.`, and
+`dns-api-N.` must use an independent external DNS provider.
+Never place the operator zone in CDNFoundry PowerDNS; that runtime is derived
+from PostgreSQL desired state and must not become its own bootstrap dependency.
+:::
+
Grafana is a read-only operations component in the telemetry role. It has no
request-path or reconciliation responsibility: an observability outage cannot
change desired state or stop DNS and HTTP serving.
diff --git a/docs/legacy/roadmap.md b/docs/legacy/roadmap.md
index a93981e..c9a9a5a 100644
--- a/docs/legacy/roadmap.md
+++ b/docs/legacy/roadmap.md
@@ -17,7 +17,7 @@ Part One is not an MVP. Features listed in Part Two are optional long-term addit
### 1. Product Definition
-CDNFoundry is a self-hosted CDN and authoritative DNS platform for companies and local providers that need a private CDN without the size, feature count, or operational complexity of Cloudflare, Akamai, or Fastly.
+CDNFoundry is a self-hosted CDN and authoritative DNS platform for companies and local providers that need a private CDN without hyperscale feature count or operational complexity.
The system intentionally supports a small set of capabilities. Every capability that is included must be predictable, recoverable, observable, bounded, and safe under production load.
@@ -1134,7 +1134,7 @@ No origin is requested.
##### Domain DNS Page
-- Cloudflare-like table density and editing flow without copying branding
+- dense operational tables and a direct editing flow without copying another product's branding
- Record type, name, content, TTL, and status
- Bulk selection and bulk action
- Import and export
@@ -3708,7 +3708,7 @@ This roadmap contains only the new work discussed after that baseline.
CDNFoundry remains a simple but solid private CDN. It must stay understandable,
bounded, production-safe, and easy to operate. It is not intended to become a
-general cloud platform or a Cloudflare replacement.
+general cloud platform or a replacement for a hyperscale public CDN.
## Project boundaries
diff --git a/docs/operations/operational-logging.md b/docs/operations/operational-logging.md
index 71bd83c..ebd231d 100644
--- a/docs/operations/operational-logging.md
+++ b/docs/operations/operational-logging.md
@@ -38,15 +38,14 @@ adding the `logs` profile to that host's normal role command:
```sh
docker compose --env-file .env.prod \
- -f compose.prod.yml \
-f compose.prod.yml \
--profile edge --profile logs up -d
```
Set a stable `LOG_HOST`, `LOG_ROLE`, and globally unique `LOG_COLLECTOR_ID` in
each host's environment copy. Set `LOKI_ENDPOINT` to the source-restricted
-telemetry gateway, for example `https://telemetry.example.com:8444`. The
-telemetry-host overlay uses private `http://loki:3100` locally. Do not run both
+telemetry gateway, for example `https://telemetry.ops.example.com:8444`. A
+collector colocated with telemetry may use private `http://loki:3100`. Do not run both
the base collector and a second role-specific collector on one host.
The default disk buffer is 2 GiB per production host and 256 MiB in development.
@@ -54,22 +53,11 @@ The default disk buffer is 2 GiB per production host and 256 MiB in development.
Expose `LOG_METRICS_BIND` only on a private monitoring address and put remote
`host:9599` targets in the file selected by `PROMETHEUS_LOG_TARGETS_FILE`.
-## Optional host journal
-
-On hosts with persistent systemd journals, add the overlay:
-
-```sh
-docker compose --env-file .env.prod \
- -f compose.prod.yml \
- -f compose.prod.yml \
- -f compose.prod.yml \
- --profile edge --profile logs up -d
-```
-
-It reads Docker/containerd units and kernel events for daemon failure, restart,
-OOM, disk/filesystem, and service-crash evidence. Omit this overlay on hosts
-without journald. Container lifecycle events not written to stdout or journald
-cannot be reconstructed by the base Docker log source.
+The committed collector reads Docker container logs. Host journal ingestion is
+not mounted or configured by `compose.prod.yml`; if an operator adds it, that is
+deployment-owned customization and must retain the same redaction, label, and
+bounded-buffer contract. Container lifecycle events absent from Docker logs
+cannot be reconstructed by the committed collector.
The Docker socket is a privileged host trust boundary even when mounted
read-only. Only the collector gets `/var/run/docker.sock`; never expose it over
diff --git a/docs/operations/production-best-practices.md b/docs/operations/production-best-practices.md
index 00fe471..2d5df76 100644
--- a/docs/operations/production-best-practices.md
+++ b/docs/operations/production-best-practices.md
@@ -21,7 +21,7 @@ Never use destructive volume or database refresh commands as an upgrade step.
- Deploy the same exact successful commit SHA or semantic-version tag on every
role participating in one rollout.
- Never deploy mutable `latest`, major-only, or minor-only image aliases.
-- Validate every exact Compose overlay combination before pulling or starting.
+- Validate `compose.prod.yml` and every exact generated node bundle before pulling or starting.
- Use bounded canaries and readiness gates; do not restart every failure domain
simultaneously.
- Keep host clocks synchronized. Certificates, DNS, telemetry, and operation
diff --git a/docs/operations/scaling.md b/docs/operations/scaling.md
index 174be71..a92219e 100644
--- a/docs/operations/scaling.md
+++ b/docs/operations/scaling.md
@@ -41,7 +41,7 @@ Adding an edge does not require a new DNS cluster.
1. prepare a distinct public IP, private PowerDNS data, valid MMDB, and DNS API certificate;
2. run the separate PowerDNS migration;
-3. start DNSdist and PowerDNS with the DNS-only overlay;
+3. start DNSdist, private PowerDNS, and DNS API with the `dns` profile;
4. register/test the cluster through its source-restricted HTTPS API;
5. enable and reconcile;
6. verify UDP/TCP, SOA, delegation, and Geo-DNS from outside.
@@ -50,15 +50,16 @@ Adding DNS capacity does not create an edge runtime.
## Move telemetry
-Use `compose.prod.yml` with an exact edge-source allowlist. Set control
+Use the `telemetry` profile or a generated monitoring-role bundle with an exact
+edge-source allowlist. Set control
`CLICKHOUSE_URL` and edge Vector endpoints to verified TLS. Keep ClickHouse and
Prometheus private. Prove that telemetry outage and backlog drain do not affect
serving.
## External control data
-`compose.prod.yml` disables local PostgreSQL and Valkey. Set
-`DB_URL` and `REDIS_URL` to owner-operated replicated services with verified
+Set `DB_URL` and `REDIS_URL` in the generated control bundle to owner-operated
+replicated services with verified
TLS, exact-source firewalls, backup, failover, and capacity plans. The repository
does not configure database replication or automatic failover.
diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md
index 454ff53..db407e6 100644
--- a/docs/reference/configuration.md
+++ b/docs/reference/configuration.md
@@ -28,8 +28,8 @@ Runtime product policy is not an environment variable. Manage it through
| `APP_URL` | control | Canonical public control-panel URL |
| `SESSION_SECURE_COOKIE` | HTTPS control | Secure-cookie flag; production default `true` |
| `CONTROL_BIND` | control | Host publication for web; default `127.0.0.1:8080` |
-| `CONTROL_HOSTNAME` | control overlay | Public browser/API hostname |
-| `TELEMETRY_HOSTNAME` | control overlay | Public telemetry-ingest hostname |
+| `CONTROL_HOSTNAME` | control | Public browser/API hostname in independent management DNS |
+| `TELEMETRY_HOSTNAME` | control/telemetry | Public telemetry-ingest hostname in independent management DNS |
| `CONTROL_PUBLIC_IPV4_ALLOWLIST` | DNS API gateway | Exact control/worker sources allowed to call PowerDNS |
| `CONTROL_PUBLIC_IPV6_ALLOWLIST` | DNS API gateway | Optional IPv6 control/worker sources allowed to call PowerDNS |
| `EDGE_PUBLIC_IPV6_ALLOWLIST` | telemetry gateway | Optional IPv6 edge sources allowed to submit telemetry |
@@ -106,9 +106,9 @@ object storage; other Restic backends need their own credential/mount wiring.
| `EDGE_GATEWAY_STATUS_URL` | edge agent | Gateway metrics URL used for heartbeat readiness |
| `EDGE_GATEWAY_METRICS_ADDRESS` | edge gateway | Restricted metrics listener; production default `0.0.0.0:9105` |
| `EDGE_GATEWAY_MAX_CONNECTIONS` | edge gateway | Global accepted-connection bound, `128`–`65536` (default `8192`) |
-| `DNS_API_HOSTNAME` | DNS overlay | DNS API TLS hostname |
-| `DNS_API_SERVER_CERTIFICATE` | DNS overlay | Absolute server certificate path |
-| `DNS_API_SERVER_PRIVATE_KEY` | DNS overlay | Absolute mode-`0600` key path |
+| `DNS_API_HOSTNAME` | DNS | DNS API TLS hostname in independent management DNS |
+| `DNS_API_SERVER_CERTIFICATE` | DNS | Absolute server certificate path |
+| `DNS_API_SERVER_PRIVATE_KEY` | DNS | Absolute mode-`0600` key path |
## Telemetry and GeoIP
@@ -150,7 +150,7 @@ Grafana telemetry variables are:
| `PROMETHEUS_EDGE_TARGETS_FILE` | telemetry | Private file_sd target file; production default is empty |
| `PROMETHEUS_LOG_TARGETS_FILE` | telemetry | Private file_sd targets for remote collector metrics; production default is empty |
| `GRAFANA_EXPLORE_URL` | control | Optional deployment fallback for the admin-only Live Logs link. The PostgreSQL-backed **Platform settings → Observability links → Grafana Explore URL** overrides it. Laravel supplies Loki, a safe selector, and a one-hour range when the chosen URL has no query; both empty hides the link |
-| `GRAFANA_HOSTNAME` | control/telemetry | Public Grafana hostname under the Cloudflare-managed operational domain |
+| `GRAFANA_HOSTNAME` | control/telemetry | Public Grafana hostname in the independently hosted operator DNS zone |
| `GRAFANA_LOKI_URL` | telemetry | Private Grafana-to-Loki endpoint; default `http://loki:3100` |
| `LOKI_RETENTION_PERIOD` | telemetry | Loki retention; production default `336h` |
| `LOKI_MAX_QUERY_LENGTH` | telemetry | Maximum query range; production default `336h` |
@@ -158,6 +158,7 @@ Grafana telemetry variables are:
| `LOG_ROLE` | logs | Stable host role: `control`, `dns`, `edge`, or `telemetry` |
| `LOG_HOST` | logs | Stable deployment host name |
| `LOG_COLLECTOR_ID` | logs | Globally unique stable collector identity |
+| `LOG_AUTH_TOKEN` | logs | Secret bearer credential used by the per-host Vector collector when pushing to the source-restricted Loki gateway |
| `LOG_BUFFER_BYTES` | logs | Per-host disk-buffer bytes; production default `2147483648` |
| `LOG_METRICS_BIND` | logs | Host metrics bind; loopback default `127.0.0.1:9599` |
| `LOG_SOURCE_IPV4_ALLOWLIST` | telemetry gateway | Exact non-edge host sources allowed to push logs |
@@ -175,8 +176,8 @@ the supported host collector.
| Variable | Required | Meaning and default |
| --- | --- | --- |
| `CDNF_RELEASE` | every production host | Exact commit SHA or exact release tag |
-| `HOST_BIND_IPV4` | multi-host overlay | Local listener address; default `0.0.0.0`, independent of public/NAT DNS addresses |
-| `HOST_BIND_IPV6` | IPv6 overlay | Local IPv6 listener; default `::`, consumed only when an IPv6 overlay is included |
+| `HOST_BIND_IPV4` | generated multi-host bundle | Local listener address; default `0.0.0.0`, independent of public/NAT DNS addresses |
+| `HOST_BIND_IPV6` | generated dual-stack bundle | Local IPv6 listener; default `::`; publish only after end-to-end IPv6 qualification |
| `EDGE_QUARANTINE_HTTP_BIND` | edge | Quarantine HTTP, default `127.0.0.1:18080` |
| `EDGE_QUARANTINE_HTTPS_BIND` | edge | Quarantine HTTPS, default `127.0.0.1:18443` |
| `EDGE_RUNTIME_TLS_CERTIFICATE` | edge | Bootstrap listener certificate path |
diff --git a/docs/reference/services-and-ports.md b/docs/reference/services-and-ports.md
index 19e6f88..24b2b09 100644
--- a/docs/reference/services-and-ports.md
+++ b/docs/reference/services-and-ports.md
@@ -22,18 +22,16 @@ telemetry internals off public networks.
| `logs` | one `log-collector` on the current host; combine once with its role profile |
| `tools` | explicit `migrate` and `pdns-migrate` one-shot services |
-`compose.prod.yml` replaces local
-`control-db` and `redis` with configured external endpoints. The control, DNS,
-edge, and telemetry host overlays add only role-specific publication and
-gateways.
-`compose.prod.yml` disables local
-ClickHouse when a verified external endpoint is configured.
+`compose.prod.yml` is the only production Compose source. Profiles select
+long-running roles; Fleet-generated per-node manifests filter those services
+and can point Laravel, Valkey clients, Vector, or Grafana at typed external
+data endpoints.
## Production listeners
| Listener | Default host bind | Exposure |
| --- | --- | --- |
-| Browser/API web | `127.0.0.1:8080` | Publish through the control Caddy overlay |
+| Browser/API web | `127.0.0.1:8080` | The `control` profile's Caddy service publishes HTTPS |
| Edge control mTLS | `0.0.0.0:8443` | Restrict to registered edge sources |
| DNSdist | `${DNS_BIND_V4}:53` TCP and UDP | Public authoritative DNS |
| Cell slot host diagnostics | loopback `18081`–`18088`, `18444`–`18451`, `19081`–`19088` | HTTP, HTTPS, and status; never public |
@@ -48,9 +46,9 @@ ClickHouse when a verified external endpoint is configured.
Host binds are deliberately separate from the public, routed, or NAT addresses
advertised in DNS. The default IPv4 wildcard works when those addresses exist
-only on an external firewall or load balancer. IPv6 publication exists only
-when the matching `compose.*-host-ipv6.yml` overlay is supplied; omit the
-overlay on IPv4-only hosts.
+only on an external firewall or load balancer. Configure IPv6 only when the
+host has a working route, firewall policy, local bind, and externally published
+AAAA/service address; otherwise retain the documented nullable IPv6 values.
The edge gateway is stricter than the shared host publications: every
control-plane service endpoint must be mapped one-to-one to a distinct local
@@ -80,7 +78,7 @@ firewalls.
The base production file defines `core-storage`, `control-db`, `redis`,
`pdns-db`, `clickhouse`, `vector-data`, `operational-vector-data`, `loki-data`, `prometheus`, `grafana-data`, `edge-state`,
-`edge-agent-state`, and `mmdb`. Caddy overlays add their data/config volumes.
+`edge-agent-state`, `mmdb`, and Caddy data/config volumes.
Do not remove these volumes during routine stop, upgrade, or testing. Recovery
requires the control database plus its encryption/signing keys and external TLS
diff --git a/edge-agent/main.go b/edge-agent/main.go
index db600e8..3b82138 100644
--- a/edge-agent/main.go
+++ b/edge-agent/main.go
@@ -606,7 +606,11 @@ func blockedIP(address string, allowlist, blockedNetworks []string) bool {
}
private := ip.IsPrivate() || inNetworks(ip, []string{"100.64.0.0/10", "fec0::/10"})
if !private {
- return inNetworks(ip, []string{"192.0.0.0/24", "198.18.0.0/15", "224.0.0.0/4", "240.0.0.0/4", "64:ff9b::/96", "64:ff9b:1::/48"})
+ return inNetworks(ip, []string{
+ "0.0.0.0/8", "192.0.0.0/24", "192.0.2.0/24", "192.88.99.0/24", "198.18.0.0/15",
+ "198.51.100.0/24", "203.0.113.0/24", "224.0.0.0/4", "240.0.0.0/4",
+ "64:ff9b::/96", "64:ff9b:1::/48", "2001:db8::/32",
+ })
}
for _, cidr := range allowlist {
_, network, err := net.ParseCIDR(cidr)
diff --git a/edge-agent/main_test.go b/edge-agent/main_test.go
index 113f3b9..da2fb78 100644
--- a/edge-agent/main_test.go
+++ b/edge-agent/main_test.go
@@ -523,6 +523,23 @@ func TestOriginTaskNeverAllowsLoopbackThroughPrivateAllowlist(t *testing.T) {
}
}
+func TestOriginTaskRejectsReservedDestinations(t *testing.T) {
+ for _, address := range []string{
+ "0.1.2.3", "192.0.2.1", "192.88.99.1", "198.51.100.1", "203.0.113.1",
+ "239.1.2.3", "240.0.0.1", "64:ff9b::7f00:1", "2001:db8::1",
+ } {
+ task := edgeTask{}
+ task.Payload.Addresses = []string{address}
+ task.Payload.Origin.Scheme = "http"
+ task.Payload.Origin.HostHeader = "origin.example"
+ task.Payload.Origin.Port = 80
+ result := runOriginTest(task)
+ if result["failure_reason"] != "blocked_destination" {
+ t.Fatalf("reserved destination %s was accepted: %#v", address, result)
+ }
+ }
+}
+
func TestOriginTaskAppliesPostgresqlBackedBlockedNetworks(t *testing.T) {
task := edgeTask{}
task.Payload.Addresses = []string{"203.0.113.10"}
diff --git a/scripts/cdnfoundry_fleet/cli.py b/scripts/cdnfoundry_fleet/cli.py
index bf8b19f..ff2dff6 100644
--- a/scripts/cdnfoundry_fleet/cli.py
+++ b/scripts/cdnfoundry_fleet/cli.py
@@ -460,7 +460,7 @@ def _setup(args: argparse.Namespace, store: FleetState, output_dir: Path, config
name = payload.get("name")
if not name:
raise ValidationError("Every setup node requires a name")
- current = store.load()
+ current = state
if name in current["nodes"]:
state = store.update_node(current, name, payload)
print(f"Updated node: {name}")
@@ -481,23 +481,23 @@ def _setup(args: argparse.Namespace, store: FleetState, output_dir: Path, config
control_payload = _interactive_node(state, role="control", defaults=control_payload)
elif not args.control_ipv4:
raise ValidationError("--control-ipv4 or a config nodes list is required in non-interactive setup")
- state = store.add_node(store.load(), control_payload)
+ state = store.add_node(state, control_payload)
print(f"Added control node: {control_payload['name']}")
if preset == "dedicated-monitoring":
if args.non_interactive:
raise ValidationError("Dedicated monitoring in non-interactive mode requires a monitoring node in --config")
monitor_payload = _interactive_node(state, role="monitoring", defaults={"name": "monitoring-1"})
- state = store.add_node(store.load(), monitor_payload)
+ state = store.add_node(state, monitor_payload)
print(f"Added monitoring node: {monitor_payload['name']}")
if not args.non_interactive:
while _prompt_yes_no("Add a DNS or edge node now?", default=False):
payload = _interactive_node(state)
- state = store.add_node(store.load(), payload)
+ state = store.add_node(state, payload)
print(f"Added node: {payload['name']}")
- state = _apply_setup_features(store, store.load(), config, preset)
+ state = _apply_setup_features(store, state, config, preset)
store.validate(state, require_secrets=not args.dry_run)
print(f"Fleet validation passed ({len(state['nodes'])} node(s)).")
diff --git a/scripts/cdnfoundry_fleet/render.py b/scripts/cdnfoundry_fleet/render.py
index 1ade086..f5e98d1 100644
--- a/scripts/cdnfoundry_fleet/render.py
+++ b/scripts/cdnfoundry_fleet/render.py
@@ -70,7 +70,7 @@ def _render_node(self, state: dict[str, Any], node: dict[str, Any]) -> Path:
atomic_write(tmp / ".env.prod", self._format_env(env), 0o600)
self._write_generated_configs(state, node, tmp, monitoring_host)
atomic_write(tmp / "README.md", self._node_readme(state, node, filtered), 0o600)
- atomic_write(tmp / "validate.sh", self._validate_script(), 0o700)
+ atomic_write(tmp / "validate.sh", self._validate_script(node, filtered), 0o700)
atomic_write(tmp / "start.sh", self._start_script(node), 0o700)
self._write_manifest(tmp, state, node)
previous = destination.with_name(destination.name + ".previous")
@@ -166,15 +166,15 @@ def _environment(
"CONTROL_HOSTNAME": f"control.{operator_domain}",
"TELEMETRY_HOSTNAME": f"telemetry.{operator_domain}",
"GRAFANA_HOSTNAME": f"grafana.{operator_domain}",
- "APP_KEY": self.store.read_secret("app-key"),
- "EDGE_ARTIFACT_SIGNING_KEY": self.store.read_secret("artifact-signing-key"),
- "CONTROL_DB_PASSWORD": self.store.read_secret("control-db-password"),
- "REDIS_PASSWORD": self.store.read_secret("valkey-password"),
- "CLICKHOUSE_PASSWORD": self.store.read_secret("clickhouse-password"),
+ "APP_KEY": self._secret("app-key"),
+ "EDGE_ARTIFACT_SIGNING_KEY": self._secret("artifact-signing-key"),
+ "CONTROL_DB_PASSWORD": self._secret("control-db-password"),
+ "REDIS_PASSWORD": self._secret("valkey-password"),
+ "CLICKHOUSE_PASSWORD": self._secret("clickhouse-password"),
"CLICKHOUSE_URL": self._clickhouse_url(state),
- "GRAFANA_ADMIN_PASSWORD": self.store.read_secret("grafana-admin-password"),
- "GRAFANA_CLICKHOUSE_PASSWORD": self.store.read_secret("grafana-clickhouse-password"),
- "GRAFANA_POSTGRES_PASSWORD": self.store.read_secret("grafana-postgres-password"),
+ "GRAFANA_ADMIN_PASSWORD": self._secret("grafana-admin-password"),
+ "GRAFANA_CLICKHOUSE_PASSWORD": self._secret("grafana-clickhouse-password"),
+ "GRAFANA_POSTGRES_PASSWORD": self._secret("grafana-postgres-password"),
"METRICS_TOKEN_FILE": "./secrets/metrics-token",
# Production Compose PKI contract (control, edge and DNS roles).
"EDGE_IDENTITY_CA_CERTIFICATE": "./pki/edge-identity-ca.crt",
@@ -200,7 +200,7 @@ def _environment(
"LOG_COLLECTOR_ID": node["name"],
"LOKI_ENDPOINT": self._loki_url(state),
"LOG_AUTH_TOKEN": self._optional_node_secret("log-auth-token", node),
- "NODE_EXPORTER_TOKEN": self.store.read_secret("node-exporter-token", node=node["name"]),
+ "NODE_EXPORTER_TOKEN": self._secret("node-exporter-token", node=node["name"]),
"EDGE_STATUS_TOKEN": self._optional_node_secret("edge-status-token", node),
"EDGE_ID": node.get("edge_id") or "",
"EDGE_BOOTSTRAP_TOKEN": self._optional_node_secret("edge-bootstrap-token", node),
@@ -208,8 +208,8 @@ def _environment(
"PDNS_API_KEY": self._optional_node_secret("pdns-api-key", node),
"RESTIC_REPOSITORY": state["features"]["backups"].get("repository") or "",
"RESTIC_PASSWORD_FILE": "./secrets/restic-password",
- "BACKUP_ACCESS_KEY_ID": self.store.read_secret("backup-access-key"),
- "BACKUP_SECRET_ACCESS_KEY": self.store.read_secret("backup-secret-key"),
+ "BACKUP_ACCESS_KEY_ID": self._secret("backup-access-key"),
+ "BACKUP_SECRET_ACCESS_KEY": self._secret("backup-secret-key"),
"BACKUP_DEFAULT_REGION": state["features"]["backups"].get("region") or "us-east-1",
"ACME_CONTACT_EMAIL": state["global"].get("acme_email", ""),
"SESSION_SECURE_COOKIE": "true",
@@ -306,7 +306,14 @@ def _environment(
if key in values and (values[key] != "" or key in needed)
}
+ def _secret(self, name: str, *, node: str | None = None) -> str:
+ if self.dry_run:
+ return f"dry-run-{name}-placeholder"
+ return self.store.read_secret(name, node=node)
+
def _optional_node_secret(self, name: str, node: dict[str, Any]) -> str:
+ if self.dry_run:
+ return f"dry-run-{name}-placeholder"
path = self.store.secret_path(name, node=node["name"])
return self.store.read_secret(name, node=node["name"]) if path.exists() else ""
@@ -608,7 +615,7 @@ def _node_readme(self, state: dict[str, Any], node: dict[str, Any], compose: dic
## Requirements
-Docker Engine, Docker Compose v2, accurate system time, CA certificates, and sufficient disk for stateful volumes. Keep the directory mode `0700` and `.env.prod`, private keys, and secret files mode `0600`.
+Docker Engine, Docker Compose v2, accurate system time, CA certificates, and sufficient disk for stateful volumes. Keep the directory mode `0700` and `.env.prod`, private keys, and secret files mode `0600`. On control nodes, `start.sh` restricts the edge identity CA key to mode `0640`, owner `root`, and numeric group `82` so only the PHP worker can read it.
## Validate and start
@@ -674,20 +681,57 @@ def _node_start_order(self, node: dict[str, Any]) -> str:
)
return "# No database migration is required for this role."
- def _validate_script(self) -> str:
- return """#!/usr/bin/env sh
+ def _validate_script(self, node: dict[str, Any], compose: dict[str, Any]) -> str:
+ identity_key_validation = ""
+ if node["role"] == "control":
+ identity_key_validation = """identity_key_mode="$(stat -c '%a' pki/edge-identity-ca.key)"
+case "$identity_key_mode" in
+ 600) ;;
+ 640)
+ test "$(stat -c '%u:%g' pki/edge-identity-ca.key)" = "0:82"
+ ;;
+ *)
+ echo "pki/edge-identity-ca.key must be transfer-safe mode 600 or activated as root:82 mode 640." >&2
+ exit 1
+ ;;
+esac
+"""
+ caddy_validation = ""
+ caddy_configs = {
+ "caddy": "/etc/caddy/Caddyfile",
+ "dns-api": "/etc/caddy/Caddyfile",
+ "telemetry-gateway": "/etc/caddy/Caddyfile",
+ }
+ for service, config in caddy_configs.items():
+ if service in compose.get("services", {}):
+ caddy_validation += (
+ f"docker compose --env-file .env.prod run --rm --no-deps {service} "
+ f"caddy adapt --adapter caddyfile --config {config} >/dev/null\n"
+ )
+ return f"""#!/usr/bin/env sh
set -eu
umask 077
test "$(stat -c '%a' .env.prod)" = 600
test "$(stat -c '%a' pki/node.key)" = 600
-docker compose --env-file .env.prod config --quiet
+{identity_key_validation}docker compose --env-file .env.prod config --quiet
+{caddy_validation}
openssl verify -CAfile pki/edge-server-ca.crt pki/node.crt
"""
def _start_script(self, node: dict[str, Any]) -> str:
migration = self._node_start_order(node)
+ key_permissions = ""
+ if node["role"] == "control":
+ key_permissions = """if [ "$(id -u)" != "0" ]; then
+ echo "Control activation must run as root so the edge identity CA key can be restricted to the PHP worker group." >&2
+ exit 1
+fi
+chown 0:82 pki/edge-identity-ca.key
+chmod 0640 pki/edge-identity-ca.key
+"""
return f"""#!/usr/bin/env sh
set -eu
+{key_permissions}
./validate.sh
{migration}
docker compose --env-file .env.prod up -d
diff --git a/tests/e2e/e2e.py b/tests/e2e/e2e.py
index 1ecd569..dd6ac5b 100644
--- a/tests/e2e/e2e.py
+++ b/tests/e2e/e2e.py
@@ -201,8 +201,8 @@ def main() -> None:
"platform_domain": "cdnf.test",
"proxy_hostname": "proxy.cdnf.test",
"nameservers": [
- {"hostname": "ns1.cdnf.test", "ipv4": "192.0.2.10", "ipv6": "2001:db8::10"},
- {"hostname": "ns2.cdnf.test", "ipv4": "192.0.2.11", "ipv6": "2001:db8::11"},
+ {"hostname": "ns1.cdnf.test", "ipv4": "8.8.8.8", "ipv6": "2001:4860:4860::8888"},
+ {"hostname": "ns2.cdnf.test", "ipv4": "1.1.1.1", "ipv6": "2606:4700:4700::1111"},
],
"soa_primary": "ns1.cdnf.test",
"soa_mailbox": "hostmaster.cdnf.test",
diff --git a/tests/e2e/phase4_runtime.py b/tests/e2e/phase4_runtime.py
index 3c9d574..4633e20 100644
--- a/tests/e2e/phase4_runtime.py
+++ b/tests/e2e/phase4_runtime.py
@@ -235,6 +235,11 @@ def main() -> None:
initial["hosts"]["blocked.example"] = initial["hosts"]["runtime.example"] | {
"origin": initial["hosts"]["runtime.example"]["origin"] | {"host": "127.0.0.1", "private_allowlist": []},
}
+ for index, address in enumerate(("0.1.2.3", "192.0.2.1", "192.88.99.1", "198.51.100.1", "203.0.113.1", "239.1.2.3", "240.0.0.1", "2001:db8::1")):
+ hostname = f"reserved-{index}.example"
+ initial["hosts"][hostname] = initial["hosts"]["runtime.example"] | {
+ "origin": initial["hosts"]["runtime.example"]["origin"] | {"host": address, "private_allowlist": []},
+ }
initial["hosts"]["policy-blocked.example"] = initial["hosts"]["runtime.example"] | {
"origin": initial["hosts"]["runtime.example"]["origin"] | {"blocked_networks": ["172.16.0.0/12"]},
}
@@ -499,6 +504,9 @@ def main() -> None:
assert bad_tls_failure["failure_count"] == bad_tls_attempts, bad_tls_failure
blocked = request("blocked.example")
assert "502 Bad Gateway" in blocked.stderr, blocked.stderr
+ for index in range(8):
+ reserved = request(f"reserved-{index}.example")
+ assert "502 Bad Gateway" in reserved.stderr, reserved.stderr
policy_blocked = request("policy-blocked.example")
assert "502 Bad Gateway" in policy_blocked.stderr, policy_blocked.stderr
unknown = request("unknown.example")
diff --git a/tests/fleet/test_fleet.py b/tests/fleet/test_fleet.py
index b98fa92..f7b4f34 100644
--- a/tests/fleet/test_fleet.py
+++ b/tests/fleet/test_fleet.py
@@ -338,6 +338,30 @@ def test_file_permissions_and_redacted_metadata(store: FleetState, source_repo:
assert pdns_password not in metadata
+def test_control_start_restricts_identity_ca_key_for_php_worker(store: FleetState, source_repo: Path, tmp_path: Path) -> None:
+ add(store, node("control-1", "control", "192.0.2.10"))
+ output = tmp_path / "bundles"
+ Renderer(source_repo, store, output).render(store.load())
+
+ start = (output / "control-1/start.sh").read_text(encoding="utf-8")
+ assert 'if [ "$(id -u)" != "0" ]' in start
+ assert "chown 0:82 pki/edge-identity-ca.key" in start
+ assert "chmod 0640 pki/edge-identity-ca.key" in start
+ assert start.index("chmod 0640 pki/edge-identity-ca.key") < start.index("./validate.sh")
+ validate = (output / "control-1/validate.sh").read_text(encoding="utf-8")
+ assert 'test "$(stat -c \'%u:%g\' pki/edge-identity-ca.key)" = "0:82"' in validate
+
+
+def test_production_control_validation_parses_caddyfile(store: FleetState, tmp_path: Path) -> None:
+ from cdnfoundry_fleet.compose import load_yaml
+
+ add(store, node("control-1", "control", "192.0.2.10"))
+ renderer = Renderer(REPO_PATCH, store, tmp_path / "bundles")
+ compose = load_yaml(REPO_PATCH / "compose.prod.yml")
+ validate = renderer._validate_script(store.load()["nodes"]["control-1"], compose)
+ assert "run --rm --no-deps caddy caddy adapt --adapter caddyfile" in validate
+
+
def test_dry_run_does_not_create_state_or_secrets(tmp_path: Path) -> None:
store = FleetState(tmp_path / "dry-state", dry_run=True)
state = store.init({"operator_domain": "ops.example.com", "platform_domain": "example.net", "release": "v1.0.0"})
@@ -345,6 +369,48 @@ def test_dry_run_does_not_create_state_or_secrets(tmp_path: Path) -> None:
assert not store.state_dir.exists()
+def test_setup_dry_run_uses_in_memory_state(source_repo: Path, tmp_path: Path) -> None:
+ import subprocess
+
+ config = tmp_path / "starter.json"
+ config.write_text(
+ json.dumps(
+ {
+ "preset": "control-monitoring",
+ "global": {
+ "operator_domain": "ops.example.com",
+ "platform_domain": "example.net",
+ "release": "v1.0.0",
+ },
+ "nodes": [node("control-1", "control", "192.0.2.10")],
+ }
+ ),
+ encoding="utf-8",
+ )
+ state_dir = tmp_path / "dry-state"
+ result = subprocess.run(
+ [
+ str(REPO_PATCH / "scripts/cdnfoundry-fleet"),
+ "--config",
+ str(config),
+ "--state-dir",
+ str(state_dir),
+ "--output-dir",
+ str(tmp_path / "dry-bundles"),
+ "--repo-root",
+ str(source_repo),
+ "--non-interactive",
+ "--dry-run",
+ "setup",
+ ],
+ check=True,
+ text=True,
+ capture_output=True,
+ )
+ assert "Fleet validation passed (1 node(s))." in result.stdout
+ assert not state_dir.exists()
+
+
def test_explicit_rotation_changes_only_target_dns_node(store: FleetState) -> None:
add(store, node("dns-one", "dns", "192.0.2.101"))
add(store, node("dns-two", "dns", "192.0.2.102"))
@@ -634,6 +700,11 @@ def test_control_monitoring_bundle_uses_project_pki_contract(store: FleetState,
add(store, node("control-1", "control", "192.0.2.140"))
with store.locked():
store.configure_feature(store.load(), "monitoring", {"mode": "colocated", "host": None})
+ store.configure_feature(
+ store.load(),
+ "logs",
+ {"mode": "centralized", "host": "control-1", "endpoint": None},
+ )
output = tmp_path / "bundles"
Renderer(source_repo, store, output).render(store.load())
bundle = output / "control-1"
@@ -649,6 +720,17 @@ def test_control_monitoring_bundle_uses_project_pki_contract(store: FleetState,
assert "core" in compose["services"]
assert "clickhouse" in compose["services"]
assert "prometheus" in compose["services"]
+ assert "LOG_AUTH_TOKEN" in compose["services"]["log-collector"]["environment"]
+ assert env["LOG_AUTH_TOKEN"]
+
+
+def test_production_log_collector_passes_auth_token_to_vector() -> None:
+ from cdnfoundry_fleet.compose import load_yaml
+
+ compose = load_yaml(REPO_PATCH / "compose.prod.yml")
+ assert compose["services"]["log-collector"]["environment"]["LOG_AUTH_TOKEN"] == (
+ "${LOG_AUTH_TOKEN:?LOG_AUTH_TOKEN is required for the logs profile}"
+ )
def test_edge_bundle_has_control_url_and_server_ca(store: FleetState, source_repo: Path, tmp_path: Path) -> None: