From 3fd17ebeadc7b57c9074571325c967c7fd3045ce Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 04:36:46 +1100 Subject: [PATCH 1/2] fix(compose): publish syslog and flow listeners through port keys A Docker install could not take syslog or flow from outside the Compose network because the receiver published only 8080 and 9090. Six port keys publish syslog over UDP, TCP and TLS, and NetFlow, IPFIX and sFlow over UDP. They use the same make fragment as the Beats and HEC keys. Each port stays unpublished until its key is set. The configuration doc lists the receiver settings that enable the listeners. It also states that rootless Docker cannot bind a host port under 1024. --- .env.example | 6 +++ Makefile | 2 +- docs/configuration.md | 12 +++++- docs/operating.md | 2 +- scripts/check_compose.py | 1 + scripts/tests/test_hyperdx_embed_exposure.py | 40 ++++++++++++++++---- 6 files changed, 51 insertions(+), 12 deletions(-) diff --git a/.env.example b/.env.example index 53c43d88..1069ccaa 100644 --- a/.env.example +++ b/.env.example @@ -157,10 +157,16 @@ # DFE_RECEIVER_GRPC_PORT= # DFE_RECEIVER_HEC_PORT= # DFE_RECEIVER_HTTP_PORT=8080 +# DFE_RECEIVER_IPFIX_PORT= # DFE_RECEIVER_MAX_BODY_SIZE=8000000 +# DFE_RECEIVER_NETFLOW_PORT= # DFE_RECEIVER_OTLP_GRPC_PORT= # DFE_RECEIVER_OTLP_HTTP_PORT= # DFE_RECEIVER_PROMETHEUS_PORT=9090 +# DFE_RECEIVER_SFLOW_PORT= +# DFE_RECEIVER_SYSLOG_TCP_PORT= +# DFE_RECEIVER_SYSLOG_TLS_PORT= +# DFE_RECEIVER_SYSLOG_UDP_PORT= # DFE_RECEIVER_VERSION= ## ============================================================= diff --git a/Makefile b/Makefile index f7bf3a40..9670cce9 100644 --- a/Makefile +++ b/Makefile @@ -272,7 +272,7 @@ ifneq ($(strip $(UNPUBLISHED))$(EMBED_UNPUBLISHED),) endif # A receiver listener's host port publishes once its key is set. The listener itself is enabled in the receiver's config -RECEIVER_LISTENERS := BEATS:5044 GRPC:6000 HEC:8088 OTLP_GRPC:4317 OTLP_HTTP:4318 +RECEIVER_LISTENERS := BEATS:5044 GRPC:6000 HEC:8088 OTLP_GRPC:4317 OTLP_HTTP:4318 SYSLOG_UDP:514/udp SYSLOG_TCP:514 SYSLOG_TLS:6514 NETFLOW:2055/udp IPFIX:4739/udp SFLOW:6343/udp RECEIVER_PORTS_SET := $(strip $(foreach listener,$(RECEIVER_LISTENERS),$(if $(strip $(DFE_RECEIVER_$(firstword $(subst :, ,$(listener)))_PORT)),$(listener)))) ifneq ($(RECEIVER_PORTS_SET),) diff --git a/docs/configuration.md b/docs/configuration.md index 9c68ebe2..985d98d6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -267,12 +267,20 @@ The AWS **region** and poll interval are not environment variables: they are lit | `DFE_RECEIVER_GRPC_PORT` | Host port for the receiver's Vector gRPC listener (container port 6000) | unset, unpublished | | `DFE_RECEIVER_HEC_PORT` | Host port for the receiver's HEC listener (container port 8088) | unset, unpublished | | `DFE_RECEIVER_HTTP_PORT` | Receiver HTTP port | `8080` | +| `DFE_RECEIVER_IPFIX_PORT` | Host UDP port for the receiver's flow listener on container port 4739 (IPFIX) | unset, unpublished | | `DFE_RECEIVER_MAX_BODY_SIZE` | Largest record the receiver accepts in bytes, under every broker's 8 MiB message limit | `8000000` | +| `DFE_RECEIVER_NETFLOW_PORT` | Host UDP port for the receiver's flow listener on container port 2055 (NetFlow) | unset, unpublished | | `DFE_RECEIVER_OTLP_GRPC_PORT` | Host port for the receiver's OTLP gRPC listener (container port 4317) | unset, unpublished | | `DFE_RECEIVER_OTLP_HTTP_PORT` | Host port for the receiver's OTLP HTTP listener (container port 4318) | unset, unpublished | | `DFE_RECEIVER_PROMETHEUS_PORT` | Receiver Prometheus port | `9090` | +| `DFE_RECEIVER_SFLOW_PORT` | Host UDP port for the receiver's flow listener on container port 6343 (sFlow) | unset, unpublished | +| `DFE_RECEIVER_SYSLOG_TCP_PORT` | Host TCP port for the receiver's syslog listener (container port 514) | unset, unpublished | +| `DFE_RECEIVER_SYSLOG_TLS_PORT` | Host TCP port for the receiver's syslog over TLS listener (container port 6514) | unset, unpublished | +| `DFE_RECEIVER_SYSLOG_UDP_PORT` | Host UDP port for the receiver's syslog listener (container port 514) | unset, unpublished | -Setting a listener's port key publishes it through a fragment `make` writes, so a raw `docker compose` outside `make` does not publish it. The listener still has to be enabled in the receiver's config, such as `DFE_RECEIVER_GRPC__ENABLED=true` in `env/receiver.env`. +Setting a listener's port key publishes it through a fragment `make` writes, so a raw `docker compose` outside `make` does not publish it. The listener still has to be enabled in the receiver's config, such as `DFE_RECEIVER_GRPC__ENABLED=true` in `env/receiver.env`. Syslog takes `DFE_RECEIVER_SYSLOG__ENABLED=true`, and over plain UDP or TCP also `DFE_RECEIVER_SYSLOG__ACCEPT_UNAUTHENTICATED=true`, because those senders have no handshake to authenticate at. Flow takes `DFE_RECEIVER_FLOW__ENABLED=true`. + +A host port under 1024, such as 514, needs a Docker daemon allowed to bind it. Rootless Docker is not, so pick a high port such as 1514 there. `make` also sets `DFE_RECEIVER_DLQ_MODE` from the profile: `file_only` on a direct gRPC profile and empty on Kafka, which keeps the receiver's `cascade` default. @@ -473,7 +481,7 @@ The engine runs as its image's user, not yours, so the two meet on a group. `mak | 13133 | otel-collector | health_check | | 19092 | Kafka (any backend) | Plaintext host | -The receiver's other listeners publish once their port key is set: 4317 (OTLP gRPC), 4318 (OTLP HTTP), 5044 (Beats), 6000 (Vector gRPC) and 8088 (HEC). +The receiver's other listeners publish once their port key is set: 4317 (OTLP gRPC), 4318 (OTLP HTTP), 5044 (Beats), 6000 (Vector gRPC), 8088 (HEC), 514 UDP and TCP and 6514 (syslog), and 2055, 4739 and 6343 UDP (flow). dfe-fetcher and the transforms run one instance per source. An instance publishes no host port. diff --git a/docs/operating.md b/docs/operating.md index c25bb5c2..144577e2 100644 --- a/docs/operating.md +++ b/docs/operating.md @@ -133,7 +133,7 @@ dfe-fetcher runs one container for every source and the three transforms run one ports at all -- they are reached over the Docker network, dfe-hyperdx through the proxy that holds both its origins. The collector publishes only `24224`, which the DOCKER DAEMON sends container stdout to; its OTLP ports stay on the Compose -network. The receiver's OTLP (`4317`, `4318`), Beats (`5044`) and HEC (`8088`) listeners publish once their port key is set in `.env`. Those `4317`/`4318` are the receiver's *ingest* edge, not the collector's. +network. The receiver's OTLP (`4317`, `4318`), Beats (`5044`), HEC (`8088`), syslog (`514` UDP and TCP, `6514`) and flow (`2055`, `4739`, `6343` UDP) listeners publish once their port key is set in `.env`. Those `4317`/`4318` are the receiver's *ingest* edge, not the collector's. Setting `DFE_BIND_HOST=0.0.0.0` opens every operator port at once, including a ClickHouse whose `CLICKHOUSE_PASSWORD` defaults to empty and which runs with diff --git a/scripts/check_compose.py b/scripts/check_compose.py index 99a7e031..406aa8e4 100644 --- a/scripts/check_compose.py +++ b/scripts/check_compose.py @@ -256,6 +256,7 @@ "DFE_ENGINE_API_EXTERNAL": "false", "DFE_PROXY_TLS": "true", "DFE_RECEIVER_BEATS_PORT": "5044", + "DFE_RECEIVER_SYSLOG_UDP_PORT": "1514", } # Console TLS refuses to start beside the auth profile, so `make dev` runs once per side, each held to the chain its own dials produce. The TLS run also needs what that refusal and the TLS precheck look for: an https origin and a certificate pair openssl accepts. _DEV_PATH_CERT_DIR = REPO_ROOT / ".tmp" / "compose-check-certs" diff --git a/scripts/tests/test_hyperdx_embed_exposure.py b/scripts/tests/test_hyperdx_embed_exposure.py index 7ec2d427..604ca7eb 100644 --- a/scripts/tests/test_hyperdx_embed_exposure.py +++ b/scripts/tests/test_hyperdx_embed_exposure.py @@ -152,9 +152,17 @@ def test_the_hyperdx_dials_govern_the_embed_and_leave_the_console( pytest.param({}, set(), id="none-set"), pytest.param( {"DFE_RECEIVER_BEATS_PORT": "15044"}, - {("15044", "5044")}, + {("15044", "5044", "tcp")}, id="one-on-its-own-host-port", ), + pytest.param( + { + "DFE_RECEIVER_SYSLOG_UDP_PORT": "1514", + "DFE_RECEIVER_SYSLOG_TCP_PORT": "1514", + }, + {("1514", "514", "udp"), ("1514", "514", "tcp")}, + id="syslog-udp-and-tcp-share-a-host-port", + ), pytest.param( { "DFE_RECEIVER_BEATS_PORT": "5044", @@ -162,20 +170,32 @@ def test_the_hyperdx_dials_govern_the_embed_and_leave_the_console( "DFE_RECEIVER_HEC_PORT": "8088", "DFE_RECEIVER_OTLP_GRPC_PORT": "4317", "DFE_RECEIVER_OTLP_HTTP_PORT": "4318", + "DFE_RECEIVER_SYSLOG_UDP_PORT": "514", + "DFE_RECEIVER_SYSLOG_TCP_PORT": "514", + "DFE_RECEIVER_SYSLOG_TLS_PORT": "6514", + "DFE_RECEIVER_NETFLOW_PORT": "2055", + "DFE_RECEIVER_IPFIX_PORT": "4739", + "DFE_RECEIVER_SFLOW_PORT": "6343", }, { - ("4317", "4317"), - ("4318", "4318"), - ("5044", "5044"), - ("6000", "6000"), - ("8088", "8088"), + ("4317", "4317", "tcp"), + ("4318", "4318", "tcp"), + ("5044", "5044", "tcp"), + ("6000", "6000", "tcp"), + ("8088", "8088", "tcp"), + ("514", "514", "udp"), + ("514", "514", "tcp"), + ("6514", "6514", "tcp"), + ("2055", "2055", "udp"), + ("4739", "4739", "udp"), + ("6343", "6343", "udp"), }, id="all", ), ], ) def test_a_receiver_port_key_publishes_its_listener( - checkout: Path, settings: dict[str, str], published: set[tuple[str, str]] + checkout: Path, settings: dict[str, str], published: set[tuple[str, str, str]] ) -> None: if shutil.which("docker") is None: pytest.skip("`docker compose config` renders the model this asserts on") @@ -194,7 +214,11 @@ def test_a_receiver_port_key_publishes_its_listener( assert model is not None ports = model["services"]["dfe-receiver"].get("ports") or [] assert { - (str(port.get("published")), str(port.get("target"))) + ( + str(port.get("published")), + str(port.get("target")), + str(port.get("protocol", "tcp")), + ) for port in ports if str(port.get("target")) not in _RECEIVER_BASE } == published From e612875ed24c3a068757797f53e19cf9c6afdd9c Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 04:40:00 +1100 Subject: [PATCH 2/2] docs(compose): syslog needs accept_unauthenticated whenever it is on When syslog is enabled, its UDP and plain TCP listeners always bind. The receiver refuses syslog without accept_unauthenticated, even when only the TLS port is published. The architecture page's receiver row now lists syslog and flow among the listeners a port key publishes. --- docs/architecture.md | 2 +- docs/configuration.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 8373b356..56384910 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -217,7 +217,7 @@ Dev builds fetch source into a managed git cache, or from your own checkouts whe | Repo | Language | Role | Profile | |---|---|---|---| -| `dfe-receiver` | Rust | Push ingest. HTTP `:8080` published, Vector gRPC `:6000`, OTLP, Beats and HEC published once their port key is set | `dfe` | +| `dfe-receiver` | Rust | Push ingest. HTTP `:8080` published, Vector gRPC `:6000`, OTLP, Beats, HEC, syslog and flow published once their port key is set | `dfe` | | `dfe-fetcher` | Rust | Pull ingest from vendor APIs | `dfe` | | `dfe-loader` | Rust | Writes ClickHouse; hosts `DfeTransport/Push` | `dfe` | | `dfe-archiver` | Rust | Archive sink (filesystem, S3, MinIO, GCS, Azure Blob) | `dfe` | diff --git a/docs/configuration.md b/docs/configuration.md index 985d98d6..bdb7b136 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -278,7 +278,7 @@ The AWS **region** and poll interval are not environment variables: they are lit | `DFE_RECEIVER_SYSLOG_TLS_PORT` | Host TCP port for the receiver's syslog over TLS listener (container port 6514) | unset, unpublished | | `DFE_RECEIVER_SYSLOG_UDP_PORT` | Host UDP port for the receiver's syslog listener (container port 514) | unset, unpublished | -Setting a listener's port key publishes it through a fragment `make` writes, so a raw `docker compose` outside `make` does not publish it. The listener still has to be enabled in the receiver's config, such as `DFE_RECEIVER_GRPC__ENABLED=true` in `env/receiver.env`. Syslog takes `DFE_RECEIVER_SYSLOG__ENABLED=true`, and over plain UDP or TCP also `DFE_RECEIVER_SYSLOG__ACCEPT_UNAUTHENTICATED=true`, because those senders have no handshake to authenticate at. Flow takes `DFE_RECEIVER_FLOW__ENABLED=true`. +Setting a listener's port key publishes it through a fragment `make` writes, so a raw `docker compose` outside `make` does not publish it. The listener still has to be enabled in the receiver's config, such as `DFE_RECEIVER_GRPC__ENABLED=true` in `env/receiver.env`. Syslog takes `DFE_RECEIVER_SYSLOG__ENABLED=true` and `DFE_RECEIVER_SYSLOG__ACCEPT_UNAUTHENTICATED=true`. Its UDP and plain TCP listeners bind whenever syslog is on and have no handshake to authenticate at, so the receiver refuses to start without the second key, even when only the TLS port is published. Flow takes `DFE_RECEIVER_FLOW__ENABLED=true`. A host port under 1024, such as 514, needs a Docker daemon allowed to bind it. Rootless Docker is not, so pick a high port such as 1514 there.