From 202b58a1660769e590b90c3ac4e698e44671fa82 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 05:03:20 +1100 Subject: [PATCH 1/3] fix(compose): serve TLS ingest via dfe-proxy and mount file input An external pusher could not verify a deployment CA against the receiver on Docker because /ingest answers plain http only. A `sources.file` source had no files to read because dfe-fetcher mounts only its spool and cursor volumes. `DFE_RECEIVER_HTTPS_PORT` publishes a TLS listener on dfe-proxy (container port 8443). It serves the console's certificate with the console's TLS settings. It forwards to dfe-receiver:8080 on the compose network. The listener exists only under `DFE_PROXY_TLS=true`. It binds `DFE_INGRESS_BIND_HOST`. With TLS off, `make` refuses `DFE_RECEIVER_HTTPS_PORT`. In-network callers keep posting plain http. `DFE_FETCHER_INPUT_DIR` mounts a host directory read only at `/mnt/input` in dfe-fetcher. A missing path stops startup. With neither key set the stack renders as before. --- .env.example | 2 + Makefile | 16 +++ config/proxy/envoy.tls.yaml | 40 +++++++ docs/configuration.md | 6 + scripts/check_compose.py | 4 +- scripts/tests/test_check_compose.py | 2 +- scripts/tests/test_hyperdx_embed_exposure.py | 116 +++++++++++++++++++ scripts/tests/test_proxy_routes.py | 31 +++++ scripts/tests/test_proxy_tls_config.py | 42 ++++++- scripts/tests/test_proxy_tls_guard.py | 36 ++++++ 10 files changed, 287 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index 1069ccaa..b1202193 100644 --- a/.env.example +++ b/.env.example @@ -141,6 +141,7 @@ ## ============================================================= ## DFE Fetcher ## ============================================================= +# DFE_FETCHER_INPUT_DIR= # DFE_FETCHER_VERSION= ## ============================================================= @@ -157,6 +158,7 @@ # DFE_RECEIVER_GRPC_PORT= # DFE_RECEIVER_HEC_PORT= # DFE_RECEIVER_HTTP_PORT=8080 +# DFE_RECEIVER_HTTPS_PORT= # DFE_RECEIVER_IPFIX_PORT= # DFE_RECEIVER_MAX_BODY_SIZE=8000000 # DFE_RECEIVER_NETFLOW_PORT= diff --git a/Makefile b/Makefile index 9670cce9..e0a47605 100644 --- a/Makefile +++ b/Makefile @@ -281,6 +281,12 @@ ifneq ($(RECEIVER_PORTS_SET),) $(eval $(call chain_fragment,.tmp/docker-compose.receiver-ports.yml)) endif +# A host directory mounts read only into dfe-fetcher once DFE_FETCHER_INPUT_DIR is set, for a `sources.file` source to read. `create_host_path: false` makes a mistyped path fail the start instead of mounting an empty directory Docker made +ifneq ($(strip $(DFE_FETCHER_INPUT_DIR)),) + $(file > .tmp/docker-compose.fetcher-input.yml,# Generated by make$(newline)services:$(newline) dfe-fetcher:$(newline) volumes:$(newline) - type: bind$(newline) source: $${DFE_FETCHER_INPUT_DIR}$(newline) target: /mnt/input$(newline) read_only: true$(newline) bind:$(newline) create_host_path: false) + $(eval $(call chain_fragment,.tmp/docker-compose.fetcher-input.yml)) +endif + # One service per app instance dfe-engine has rendered a config for, generated # with the profile (scripts/instances.py). Empty for the bootstrap goals, which # reads as off -- the safe direction for a target that only stops a stack. @@ -356,6 +362,16 @@ ifneq ($(PROXY_TLS_START),) endif endif +# dfe-proxy's TLS ingest listener (config/proxy/envoy.tls.yaml) publishes once its key is set. The plain config has no such listener, so the key without the dial is refused on the goals that start a stack +ifneq ($(strip $(DFE_RECEIVER_HTTPS_PORT)),) + ifeq ($(strip $(DFE_PROXY_TLS)),true) + $(file > .tmp/docker-compose.receiver-https.yml,# Generated by make$(newline)services:$(newline) dfe-proxy:$(newline) ports:$(newline) - $${DFE_INGRESS_BIND_HOST:-0.0.0.0}:$${DFE_RECEIVER_HTTPS_PORT}:8443) + $(eval $(call chain_fragment,.tmp/docker-compose.receiver-https.yml)) + else ifneq (,$(filter $(ORIGIN_GOALS),$(MAKECMDGOALS))) + $(error DFE_RECEIVER_HTTPS_PORT='$(strip $(DFE_RECEIVER_HTTPS_PORT))' publishes dfe-proxy's TLS ingest listener, which only exists under DFE_PROXY_TLS=true. Set DFE_PROXY_TLS=true and run `make certs`, or unset DFE_RECEIVER_HTTPS_PORT. Plain http ingest stays on DFE_RECEIVER_HTTP_PORT) + endif +endif + # These goals recreate services on the network that already exists with no `make down` first; the e2e suites keep ClickHouse, and so its network, between tests. Docker cannot re-address a live network, so a stack whose network predates a DFE_PROXY_TLS flip would leave every recreated service Exited. NETWORK_REUSE_GOALS := apply apply-services infra $(E2E_GOALS) PROXY_NETWORK_REUSE := $(filter $(NETWORK_REUSE_GOALS),$(or $(MAKECMDGOALS),help)) diff --git a/config/proxy/envoy.tls.yaml b/config/proxy/envoy.tls.yaml index 3523a5df..228f4af0 100644 --- a/config/proxy/envoy.tls.yaml +++ b/config/proxy/envoy.tls.yaml @@ -8,6 +8,7 @@ # # Mounted instead of envoy.yaml when DFE_PROXY_TLS=true # Must equal envoy.yaml plus the marked TLS blocks (test_proxy_tls_config.py) +# The receiver-ingest listener exists only here: TLS is its whole purpose admin: address: @@ -45,6 +46,18 @@ static_resources: address: socket_address: { address: dfe-ui, port_value: 3000 } type: STRICT_DNS + # BEGIN TLS + - name: dfe-receiver + dns_lookup_family: V4_ONLY + load_assignment: + cluster_name: dfe-receiver + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: { address: dfe-receiver, port_value: 8080 } + type: STRICT_DNS + # END TLS listeners: - name: ingress address: @@ -149,3 +162,30 @@ static_resources: # An http embed inside the https console is blocked as mixed content transport_socket: *console_tls # END TLS + # BEGIN TLS + # External pushers send to the receiver here, over the console's certificate + - name: receiver-ingest + address: + socket_address: { address: 0.0.0.0, port_value: 8443 } + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + route_config: + name: receiver_ingest + virtual_hosts: + - name: receiver + domains: ["*"] + routes: + # The receiver's own body limit (DFE_RECEIVER_MAX_BODY_SIZE) decides what is too big, and 120 s leaves room for a slow upload + - match: { prefix: / } + route: { cluster: dfe-receiver, timeout: 120s } + stat_prefix: receiver_ingest + use_remote_address: true + transport_socket: *console_tls + # END TLS diff --git a/docs/configuration.md b/docs/configuration.md index bdb7b136..edcfd0d6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -241,6 +241,7 @@ Only `single-kafka` runs dfe-archiver, so archiving needs a Kafka profile ([prof | Variable | Use | Default | |--------------------------------------------------|--------------------------------------------------------------------------------------|---------------------------------------------------------------------| | `DFE_FETCHER_VERSION` | Version of dfe-fetcher to use | none -- `make stack` pins it; unset is a hard-fail | +| `DFE_FETCHER_INPUT_DIR` | Absolute host directory mounted read only into dfe-fetcher at `/mnt/input` | unset, nothing mounted | | `AWS_ACCESS_KEY_ID` | AWS access key. Goes in `env/fetcher.env`, NOT `.env` -- the fetcher config reads it via `env:` | - | | `AWS_SECRET_ACCESS_KEY` | AWS secret key. Same file as above | - | | `AZURE_CLIENT_ID` | In `env/fetcher.env`. Client ID an Azure source names with `env:AZURE_CLIENT_ID` | - | @@ -248,6 +249,8 @@ Only `single-kafka` runs dfe-archiver, so archiving needs a Kafka profile ([prof | `AZURE_SUBSCRIPTION_ID` | In `env/fetcher.env`. Subscription an Azure source reads | - | | `AZURE_TENANT_ID` | In `env/fetcher.env`. Tenant the client belongs to | - | +A `sources.file` source reads files from the fetcher's container, so its include globs name the container path, such as `/mnt/input/*.log`. Set `DFE_FETCHER_INPUT_DIR` to the host directory holding them. `make` mounts it only when the key is set and a path that does not exist stops the start. + The AWS **region** and poll interval are not environment variables: they are literals in each source's fetcher config (`region: us-west-2` and `interval_secs` in `resources/seed/sources/aws-*.yaml`, 30 seconds and 300 for GuardDuty, whose every poll stores all its findings again), because only the two credential fields are `env:`-interpolated. ### DFE Loader @@ -267,6 +270,7 @@ The AWS **region** and poll interval are not environment variables: they are lit | `DFE_RECEIVER_GRPC_PORT` | Host port for the receiver's Vector gRPC listener (container port 6000) | unset, unpublished | | `DFE_RECEIVER_HEC_PORT` | Host port for the receiver's HEC listener (container port 8088) | unset, unpublished | | `DFE_RECEIVER_HTTP_PORT` | Receiver HTTP port | `8080` | +| `DFE_RECEIVER_HTTPS_PORT` | Host port for TLS ingest to the receiver, which dfe-proxy serves on container port 8443. Needs `DFE_PROXY_TLS=true` | unset, unpublished | | `DFE_RECEIVER_IPFIX_PORT` | Host UDP port for the receiver's flow listener on container port 4739 (IPFIX) | unset, unpublished | | `DFE_RECEIVER_MAX_BODY_SIZE` | Largest record the receiver accepts in bytes, under every broker's 8 MiB message limit | `8000000` | | `DFE_RECEIVER_NETFLOW_PORT` | Host UDP port for the receiver's flow listener on container port 2055 (NetFlow) | unset, unpublished | @@ -280,6 +284,8 @@ The AWS **region** and poll interval are not environment variables: they are lit Setting a listener's port key publishes it through a fragment `make` writes, so a raw `docker compose` outside `make` does not publish it. The listener still has to be enabled in the receiver's config, such as `DFE_RECEIVER_GRPC__ENABLED=true` in `env/receiver.env`. Syslog takes `DFE_RECEIVER_SYSLOG__ENABLED=true` and `DFE_RECEIVER_SYSLOG__ACCEPT_UNAUTHENTICATED=true`. Its UDP and plain TCP listeners bind whenever syslog is on and have no handshake to authenticate at, so the receiver refuses to start without the second key, even when only the TLS port is published. Flow takes `DFE_RECEIVER_FLOW__ENABLED=true`. +`DFE_RECEIVER_HTTPS_PORT` publishes TLS ingest to the receiver's HTTP listener, bound to `DFE_INGRESS_BIND_HOST`. dfe-proxy terminates it with the same `console.crt` and `console.key` as the console and forwards to `dfe-receiver:8080`, so callers on the compose network keep posting plain http. It exists only under `DFE_PROXY_TLS=true` and `make` refuses the key without it. A pusher outside the stack connects to the hostname the certificate names and trusts the `ca.crt` beside the pair. + A host port under 1024, such as 514, needs a Docker daemon allowed to bind it. Rootless Docker is not, so pick a high port such as 1514 there. `make` also sets `DFE_RECEIVER_DLQ_MODE` from the profile: `file_only` on a direct gRPC profile and empty on Kafka, which keeps the receiver's `cascade` default. diff --git a/scripts/check_compose.py b/scripts/check_compose.py index 406aa8e4..8f8f3d7f 100644 --- a/scripts/check_compose.py +++ b/scripts/check_compose.py @@ -256,13 +256,15 @@ "DFE_ENGINE_API_EXTERNAL": "false", "DFE_PROXY_TLS": "true", "DFE_RECEIVER_BEATS_PORT": "5044", + "DFE_RECEIVER_HTTPS_PORT": "8443", "DFE_RECEIVER_SYSLOG_UDP_PORT": "1514", + "DFE_FETCHER_INPUT_DIR": "/srv/compose-check-input", } # Console TLS refuses to start beside the auth profile, so `make dev` runs once per side, each held to the chain its own dials produce. The TLS run also needs what that refusal and the TLS precheck look for: an https origin and a certificate pair openssl accepts. _DEV_PATH_CERT_DIR = REPO_ROOT / ".tmp" / "compose-check-certs" _DEV_PATH_BIN_DIR = REPO_ROOT / ".tmp" / "compose-check-bin" _DEV_PATH_RUNS = { - "auth": {"DFE_PROXY_TLS": "false"}, + "auth": {"DFE_PROXY_TLS": "false", "DFE_RECEIVER_HTTPS_PORT": ""}, "console TLS": { "DFE_AUTH_ENABLED": "false", "DFE_AUTH_RESOLVED": "false", diff --git a/scripts/tests/test_check_compose.py b/scripts/tests/test_check_compose.py index fda70a17..1404f7c8 100644 --- a/scripts/tests/test_check_compose.py +++ b/scripts/tests/test_check_compose.py @@ -567,7 +567,7 @@ def _proxy_text(name: str) -> str: ("hyperdx.yaml", 2), ("hyperdx.tls.yaml", 4), ("envoy.yaml", 2), - ("envoy.tls.yaml", 2), + ("envoy.tls.yaml", 3), ], ) def test_no_proxy_adds_an_identity_header(hosts: int, name: str) -> None: diff --git a/scripts/tests/test_hyperdx_embed_exposure.py b/scripts/tests/test_hyperdx_embed_exposure.py index 604ca7eb..2b74cd81 100644 --- a/scripts/tests/test_hyperdx_embed_exposure.py +++ b/scripts/tests/test_hyperdx_embed_exposure.py @@ -31,6 +31,12 @@ # Container ports dfe-receiver publishes whatever its port keys say. _RECEIVER_BASE = {"8080", "9090"} _RECEIVER_FRAGMENT = ".tmp/docker-compose.receiver-ports.yml" +# The TLS ingest listener dfe-proxy publishes for the receiver, and where dfe-fetcher mounts a host directory. +_INGEST_FRAGMENT = ".tmp/docker-compose.receiver-https.yml" +_INGEST_TARGET = "8443" +_FETCHER_FRAGMENT = ".tmp/docker-compose.fetcher-input.yml" +_FETCHER_MOUNT = "/mnt/input" +_TLS = {"DFE_PROXY_TLS": "true", "DFE_EXTERNAL_ORIGIN": "https://dfe.example.test"} _RESOLVER = '''\ import os @@ -222,3 +228,113 @@ def test_a_receiver_port_key_publishes_its_listener( for port in ports if str(port.get("target")) not in _RECEIVER_BASE } == published + + +@pytest.mark.parametrize( + ("settings", "published"), + [ + pytest.param({}, None, id="none-set"), + pytest.param({"DFE_RECEIVER_HTTPS_PORT": "8443"}, None, id="key-without-tls"), + pytest.param(dict(_TLS), None, id="tls-without-key"), + pytest.param( + {**_TLS, "DFE_RECEIVER_HTTPS_PORT": "8443"}, + ("8443", "0.0.0.0"), + id="key-with-tls", + ), + pytest.param( + { + **_TLS, + "DFE_RECEIVER_HTTPS_PORT": "443", + "DFE_INGRESS_BIND_HOST": "192.0.2.10", + }, + ("443", "192.0.2.10"), + id="binds-the-ingest-host", + ), + pytest.param( + { + **_TLS, + "DFE_RECEIVER_HTTPS_PORT": "8443", + "DFE_UI_EXTERNAL": "false", + "DFE_INFRA_UIS_EXTERNAL": "false", + }, + ("8443", "0.0.0.0"), + id="survives-the-console-opt-outs", + ), + ], +) +def test_the_https_ingest_key_publishes_the_proxy_listener_under_tls( + checkout: Path, + settings: dict[str, str], + published: tuple[str, str] | None, + tmp_path: Path, +) -> None: + if shutil.which("docker") is None: + pytest.skip("`docker compose config` renders the model this asserts on") + fragment = _chained_fragment( + checkout=checkout, name=_INGEST_FRAGMENT, settings=settings + ) + unpublish = _chained_fragment( + checkout=checkout, name=check_compose._UNPUBLISH_FRAGMENT, settings=settings + ) + environ, _ = check_compose._check_env() + + assert (fragment is None) == (published is None) + overlays = [] + for number, text in enumerate(part for part in (unpublish, fragment) if part): + overlay = tmp_path / f"overlay{number}.yml" + overlay.write_text(data=text, encoding="utf-8", newline="\n") + overlays.append(str(overlay)) + model = check_compose._config_json( + env={**environ, **settings}, + files=[check_compose.COMPOSE_FILE.name, *overlays], + ) + + assert model is not None + ports = model["services"]["dfe-proxy"].get("ports") or [] + assert { + (str(port.get("published")), str(port.get("host_ip"))) + for port in ports + if str(port.get("target")) == _INGEST_TARGET + } == ({published} if published else set()) + assert (_INGEST_TARGET in {str(port.get("target")) for port in ports}) == bool( + published + ) + + +@pytest.mark.parametrize( + ("settings", "mounted"), + [ + pytest.param({}, False, id="none-set"), + pytest.param({"DFE_FETCHER_INPUT_DIR": " "}, False, id="blank"), + pytest.param({"DFE_FETCHER_INPUT_DIR": "/srv/dfe-input"}, True, id="set"), + ], +) +def test_the_fetcher_input_key_mounts_a_host_directory_read_only( + checkout: Path, settings: dict[str, str], mounted: bool +) -> None: + if shutil.which("docker") is None: + pytest.skip("`docker compose config` renders the model this asserts on") + fragment = _chained_fragment( + checkout=checkout, name=_FETCHER_FRAGMENT, settings=settings + ) + environ, _ = check_compose._check_env() + + assert (fragment is None) == (not (mounted)) + model = check_compose._config_json( + env={**environ, **settings}, + files=[check_compose.COMPOSE_FILE.name], + fragment=fragment, + ) + + assert model is not None + volumes = model["services"]["dfe-fetcher"].get("volumes") or [] + inputs = [volume for volume in volumes if volume.get("target") == _FETCHER_MOUNT] + assert [ + ( + volume["type"], + volume["source"], + volume.get("read_only"), + volume["bind"]["create_host_path"], + ) + for volume in inputs + ] == ([("bind", "/srv/dfe-input", True, False)] if mounted else []) diff --git a/scripts/tests/test_proxy_routes.py b/scripts/tests/test_proxy_routes.py index 5f730d51..e3818d42 100644 --- a/scripts/tests/test_proxy_routes.py +++ b/scripts/tests/test_proxy_routes.py @@ -22,6 +22,9 @@ _ALL_CONFIGS = [*_CONSOLE_CONFIGS, "hyperdx.yaml", "hyperdx.tls.yaml"] _CONSOLE_LISTENER = "ingress" _NORMALISING = ("normalize_path", "merge_slashes") +# The receiver ingest listener, which only the TLS configs carry, and the one route it has: match, cluster, status, timeout +_INGEST_LISTENER = "receiver-ingest" +_INGEST_ROUTES = [({"prefix": "/"}, "dfe-receiver", "", "120s")] # Console routes in match order: match, cluster, status, timeout _CONSOLE_ROUTES = [ ({"path": "/livez"}, "", "200", ""), @@ -63,6 +66,34 @@ def test_the_console_matches_and_forwards_the_canonical_path(name: str) -> None: } +@pytest.mark.parametrize( + ("name", "routes"), + [ + ("envoy.yaml", []), + ("envoy.tls.yaml", _INGEST_ROUTES), + ("hyperdx.yaml", []), + ("hyperdx.tls.yaml", []), + ], +) +def test_only_the_console_tls_config_forwards_to_the_receiver( + name: str, routes: list[tuple[dict[str, str], str, str, str]] +) -> None: + ingest = [m for m in _managers(name) if m.listener == _INGEST_LISTENER] + forwarded = { + route.cluster + for manager in _managers(name) + for host in manager.virtual_hosts + for route in host.routes + } + + assert ("dfe-receiver" in forwarded) == bool(routes) + assert [ + [(r.match, r.cluster, r.status, r.timeout) for r in host.routes] + for manager in ingest + for host in manager.virtual_hosts + ] == ([routes] if routes else []) + + @pytest.mark.parametrize("name", _ALL_CONFIGS) def test_no_hyperdx_origin_rewrites_the_path_it_forwards(name: str) -> None: others = [m for m in _managers(name) if m.listener != _CONSOLE_LISTENER] diff --git a/scripts/tests/test_proxy_tls_config.py b/scripts/tests/test_proxy_tls_config.py index c73c0456..9836cfaa 100644 --- a/scripts/tests/test_proxy_tls_config.py +++ b/scripts/tests/test_proxy_tls_config.py @@ -17,6 +17,7 @@ from pathlib import Path import pytest +import yaml REPO_ROOT = Path(__file__).resolve().parents[2] PROXY_DIR = REPO_ROOT / "config" / "proxy" @@ -24,21 +25,26 @@ _BEGIN = "# BEGIN TLS" _END = "# END TLS" _PAIRS = [("envoy.yaml", "envoy.tls.yaml"), ("hyperdx.yaml", "hyperdx.tls.yaml")] -# Each config's listeners by name and port and whether they also answer plain HTTP on the same port. +# Each config's listeners by name and port, the ones only its TLS config carries, and whether they also answer plain HTTP on the same port. _LISTENERS = [ ( "envoy.yaml", "envoy.tls.yaml", {"hyperdx-embed": "8091", "ingress": "8080"}, + {"receiver-ingest": "8443"}, False, ), ( "hyperdx.yaml", "hyperdx.tls.yaml", {"hyperdx-api": "8000", "hyperdx-app": "8090"}, + {}, True, ), ] +# Listeners that terminate TLS with the console certificate, against the listener that defines those settings. +_CONSOLE_TLS_LISTENER = "ingress" +_CONSOLE_TLS_SHARERS = ["hyperdx-embed", "receiver-ingest"] _LISTENER_ITEM = " - " _LISTENER_NAME = re.compile(r"^ [ -] name: (\S+)$") # The key a filter chain's network filters sit under, with or without the list dash. @@ -132,22 +138,46 @@ def test_stripping_the_tls_blocks_gives_back_the_plain_config( assert _past_header(text=_strip_tls(text=tls_text)) == _past_header(text=plain_text) -@pytest.mark.parametrize(("plain", "tls", "listeners", "dual"), _LISTENERS) +@pytest.mark.parametrize(("plain", "tls", "listeners", "tls_only", "dual"), _LISTENERS) def test_every_listener_terminates_tls_once_and_only_in_the_tls_config( - dual: bool, listeners: dict[str, str], plain: str, tls: str + dual: bool, + listeners: dict[str, str], + plain: str, + tls: str, + tls_only: dict[str, str], ) -> None: encrypted = _listeners(text=(PROXY_DIR / tls).read_text(encoding="utf-8")) unencrypted = _listeners(text=(PROXY_DIR / plain).read_text(encoding="utf-8")) - assert sorted(encrypted) == sorted(listeners) + assert sorted(encrypted) == sorted({**listeners, **tls_only}) assert sorted(unencrypted) == sorted(listeners) - for name, port in listeners.items(): + for name, port in {**listeners, **tls_only}.items(): block = "\n".join(encrypted[name]) assert f"port_value: {port}" in block assert block.count("transport_socket:") == 1, name assert ("transport_protocol: tls" in block) == dual, name assert ("tls_inspector" in block) == dual, name - assert "transport_socket:" not in "\n".join(unencrypted[name]) + if name in unencrypted: + assert "transport_socket:" not in "\n".join(unencrypted[name]) + + +def _tls_socket(*, listener: str) -> dict: + """Return the downstream TLS settings one listener of envoy.tls.yaml serves, anchors resolved.""" + loaded = yaml.safe_load((PROXY_DIR / "envoy.tls.yaml").read_text(encoding="utf-8")) + named = {item["name"]: item for item in loaded["static_resources"]["listeners"]} + return named[listener]["filter_chains"][0]["transport_socket"] + + +@pytest.mark.parametrize("listener", _CONSOLE_TLS_SHARERS) +def test_a_listener_on_the_console_certificate_serves_the_console_tls_settings( + listener: str, +) -> None: + settings = _tls_socket(listener=_CONSOLE_TLS_LISTENER) + params = settings["typed_config"]["common_tls_context"]["tls_params"] + + assert _tls_socket(listener=listener) == settings + assert params["cipher_suites"] + assert params["ecdh_curves"][0] == "X25519MLKEM768" def test_each_tls_chain_repeats_the_plain_filters_after_it() -> None: diff --git a/scripts/tests/test_proxy_tls_guard.py b/scripts/tests/test_proxy_tls_guard.py index 5f2e7434..a1e920e9 100644 --- a/scripts/tests/test_proxy_tls_guard.py +++ b/scripts/tests/test_proxy_tls_guard.py @@ -29,6 +29,7 @@ _BARE_CERT_DIR = "must be a path starting with" _CLOSED_CERT_DIR = "exists but this user cannot open it" _CRASHED = "precheck failed before it could check the console TLS setup" +_HTTPS_INGEST = "publishes dfe-proxy's TLS ingest listener" _HTTP_HYPERDX = "DFE_HYPERDX_APP_URL must start with https://" _HTTP_ORIGIN = "DFE_EXTERNAL_ORIGIN must start with https://" _MISSING_CERT = "needs the certificate ./certs/console.crt" @@ -46,6 +47,7 @@ _BARE_CERT_DIR, _CLOSED_CERT_DIR, _CRASHED, + _HTTPS_INGEST, _HTTP_HYPERDX, _HTTP_ORIGIN, _MISSING_CERT, @@ -543,6 +545,40 @@ def test_an_empty_dial_reads_as_false(makefile: Path) -> None: assert "compose/tls.yml" not in chain.stdout +@pytest.mark.parametrize( + ("goal", "tls", "refused"), + [ + *[(goal, "false", True) for goal in _GUARDED_GOALS], + *[(goal, "true", False) for goal in _GUARDED_GOALS], + *[(goal, "false", False) for goal in _UNGUARDED_GOALS], + ], +) +def test_the_https_ingest_key_needs_the_tls_dial_to_start( + goal: str, makefile: Path, refused: bool, tls: str +) -> None: + result = _make( + cwd=makefile, + goal=goal, + **{**_TLS_ON, "DFE_PROXY_TLS": tls, "DFE_RECEIVER_HTTPS_PORT": "8443"}, + ) + + assert (result.returncode != 0) == refused, result.stderr + assert (_HTTPS_INGEST in result.stderr) == refused + + +@pytest.mark.parametrize(("key", "chained"), [("8443", True), ("", False)]) +def test_the_https_ingest_key_chains_its_fragment_only_under_the_dial( + chained: bool, key: str, makefile: Path +) -> None: + result = _make( + cwd=makefile, + goal="print-compose-file", + **{**_TLS_ON, "DFE_RECEIVER_HTTPS_PORT": key}, + ) + + assert (".tmp/docker-compose.receiver-https.yml" in result.stdout) == chained + + @pytest.mark.parametrize("goal", _UNGUARDED_GOALS) def test_stopping_and_checking_never_need_what_tls_needs( goal: str, makefile: Path From c2d6add69548d46b8e190ea3e3088b4aeda0f584 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 05:16:23 +1100 Subject: [PATCH 2/3] fix(compose): document ingest sender identity and file input limits The receiver sees dfe-proxy as the TCP peer on the HTTPS ingest port. Its `ip_filter` cannot tell pushers apart there. `server.trusted_proxies` has no environment variable name when the receiver runs from a config file. The docs explain that fetcher dump sources use `paths`. Tail sources use `include`. The directory must be readable by uid 1000. An unreadable directory causes a failure when the source runs. The route timeout comment in envoy.tls.yaml describes only what the timeout limits. --- config/proxy/envoy.tls.yaml | 2 +- docs/configuration.md | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config/proxy/envoy.tls.yaml b/config/proxy/envoy.tls.yaml index 228f4af0..446510ea 100644 --- a/config/proxy/envoy.tls.yaml +++ b/config/proxy/envoy.tls.yaml @@ -182,7 +182,7 @@ static_resources: - name: receiver domains: ["*"] routes: - # The receiver's own body limit (DFE_RECEIVER_MAX_BODY_SIZE) decides what is too big, and 120 s leaves room for a slow upload + # The timeout bounds the wait for the receiver's reply. The receiver enforces its own body size and request timeout - match: { prefix: / } route: { cluster: dfe-receiver, timeout: 120s } stat_prefix: receiver_ingest diff --git a/docs/configuration.md b/docs/configuration.md index edcfd0d6..29681df9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -249,7 +249,7 @@ Only `single-kafka` runs dfe-archiver, so archiving needs a Kafka profile ([prof | `AZURE_SUBSCRIPTION_ID` | In `env/fetcher.env`. Subscription an Azure source reads | - | | `AZURE_TENANT_ID` | In `env/fetcher.env`. Tenant the client belongs to | - | -A `sources.file` source reads files from the fetcher's container, so its include globs name the container path, such as `/mnt/input/*.log`. Set `DFE_FETCHER_INPUT_DIR` to the host directory holding them. `make` mounts it only when the key is set and a path that does not exist stops the start. +A `sources.file` source reads files from the fetcher's container, so its globs name the container path, such as `/mnt/input/*.log`. A dump source lists them under `paths` and a tail source under `include`. Set `DFE_FETCHER_INPUT_DIR` to the host directory holding them. `make` mounts it only when the key is set, and a path that does not exist stops the start. The fetcher image runs as uid 1000, so that user must be able to read the directory. A `0700` directory owned by root passes the start and fails when the source runs. The AWS **region** and poll interval are not environment variables: they are literals in each source's fetcher config (`region: us-west-2` and `interval_secs` in `resources/seed/sources/aws-*.yaml`, 30 seconds and 300 for GuardDuty, whose every poll stores all its findings again), because only the two credential fields are `env:`-interpolated. @@ -286,6 +286,8 @@ Setting a listener's port key publishes it through a fragment `make` writes, so `DFE_RECEIVER_HTTPS_PORT` publishes TLS ingest to the receiver's HTTP listener, bound to `DFE_INGRESS_BIND_HOST`. dfe-proxy terminates it with the same `console.crt` and `console.key` as the console and forwards to `dfe-receiver:8080`, so callers on the compose network keep posting plain http. It exists only under `DFE_PROXY_TLS=true` and `make` refuses the key without it. A pusher outside the stack connects to the hostname the certificate names and trusts the `ca.crt` beside the pair. +The receiver sees dfe-proxy as the TCP peer, so its `ip_filter` cannot tell pushers apart on this port. Its rate limit and auth-failure events key on dfe-proxy too, until `server.trusted_proxies` in the receiver's config file lists dfe-proxy's address, the last host of `DFE_NETWORK_SUBNET`. That key has no environment spelling. + A host port under 1024, such as 514, needs a Docker daemon allowed to bind it. Rootless Docker is not, so pick a high port such as 1514 there. `make` also sets `DFE_RECEIVER_DLQ_MODE` from the profile: `file_only` on a direct gRPC profile and empty on Kafka, which keeps the receiver's `cascade` default. From bef68d8dc7bdf86918d15c1e77a0aedf00288201 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 05:19:39 +1100 Subject: [PATCH 3/3] fix(docs): correct how receiver listeners are enabled on Docker The receiver runs from a config file on Docker, so it reads only a fixed set of flat env keys. `DFE_RECEIVER_FLOW_ENABLED` is one of them. The documented `DFE_RECEIVER_SYSLOG__ENABLED` and `DFE_RECEIVER_GRPC__ENABLED` keys have no effect. The docs now say to change the engine config and run `make apply` for syslog and the Vector gRPC listener. `make demo` turns syslog on this way. The docs still explain why syslog needs `accept_unauthenticated`. --- docs/architecture.md | 2 +- docs/configuration.md | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 56384910..953c0da4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -147,7 +147,7 @@ flowchart LR | Container port | Host port | Owner | Protocol | Audience | Binds | |---|---|---|---|---|---| -| 6000 | none, until `DFE_RECEIVER_GRPC_PORT` is set | dfe-receiver | Vector protocol, an ingest source that listens only with `DFE_RECEIVER_GRPC__ENABLED=true` | external clients | `DFE_INGRESS_BIND_HOST` (`0.0.0.0`) | +| 6000 | none, until `DFE_RECEIVER_GRPC_PORT` is set | dfe-receiver | Vector protocol, an ingest source that listens only once `config.grpc.enabled` is set through the engine | external clients | `DFE_INGRESS_BIND_HOST` (`0.0.0.0`) | | 6000 | none | dfe-loader | `DfeTransport/Push`, internal transport | receiver, fetcher, the transforms | the compose network | Every Push listener in the suite answers on 6000 and dfe-engine compiles each sender's endpoint against that port. The receiver's listener is meant to be reachable from outside the stack. The loader's is not, so it publishes no host port. diff --git a/docs/configuration.md b/docs/configuration.md index 29681df9..803a639a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -282,7 +282,9 @@ The AWS **region** and poll interval are not environment variables: they are lit | `DFE_RECEIVER_SYSLOG_TLS_PORT` | Host TCP port for the receiver's syslog over TLS listener (container port 6514) | unset, unpublished | | `DFE_RECEIVER_SYSLOG_UDP_PORT` | Host UDP port for the receiver's syslog listener (container port 514) | unset, unpublished | -Setting a listener's port key publishes it through a fragment `make` writes, so a raw `docker compose` outside `make` does not publish it. The listener still has to be enabled in the receiver's config, such as `DFE_RECEIVER_GRPC__ENABLED=true` in `env/receiver.env`. Syslog takes `DFE_RECEIVER_SYSLOG__ENABLED=true` and `DFE_RECEIVER_SYSLOG__ACCEPT_UNAUTHENTICATED=true`. Its UDP and plain TCP listeners bind whenever syslog is on and have no handshake to authenticate at, so the receiver refuses to start without the second key, even when only the TLS port is published. Flow takes `DFE_RECEIVER_FLOW__ENABLED=true`. +Setting a listener's port key publishes it through a fragment `make` writes, so a raw `docker compose` outside `make` does not publish it. The listener still has to be enabled in the receiver's config. Flow takes `DFE_RECEIVER_FLOW_ENABLED=true` in `env/receiver.env`. + +Syslog and the Vector gRPC listener have no environment key on Docker, because the receiver reads only a fixed set of flat keys when it runs from a config file. Turn them on through the engine, as `make demo` does for syslog: `PUT /api/v1/apps/dfe-receiver//config` with `{"changes": {"config.syslog.enabled": true, "config.syslog.accept_unauthenticated": true}}`, then `make apply`. The gRPC listener takes `config.grpc.enabled` the same way. The UDP and plain TCP syslog listeners bind whenever syslog is on and have no handshake to authenticate at, so the receiver refuses to start without the second key, even when only the TLS port is published. `DFE_RECEIVER_HTTPS_PORT` publishes TLS ingest to the receiver's HTTP listener, bound to `DFE_INGRESS_BIND_HOST`. dfe-proxy terminates it with the same `console.crt` and `console.key` as the console and forwards to `dfe-receiver:8080`, so callers on the compose network keep posting plain http. It exists only under `DFE_PROXY_TLS=true` and `make` refuses the key without it. A pusher outside the stack connects to the hostname the certificate names and trusts the `ca.crt` beside the pair.