fix(compose): publish syslog and flow listeners through port keys

.env.example

@@ -157,10 +157,16 @@
 # DFE_RECEIVER_GRPC_PORT=
 # DFE_RECEIVER_HEC_PORT=
 # DFE_RECEIVER_HTTP_PORT=8080
+# DFE_RECEIVER_IPFIX_PORT=
 # DFE_RECEIVER_MAX_BODY_SIZE=8000000
+# DFE_RECEIVER_NETFLOW_PORT=
 # DFE_RECEIVER_OTLP_GRPC_PORT=
 # DFE_RECEIVER_OTLP_HTTP_PORT=
 # DFE_RECEIVER_PROMETHEUS_PORT=9090
+# DFE_RECEIVER_SFLOW_PORT=
+# DFE_RECEIVER_SYSLOG_TCP_PORT=
+# DFE_RECEIVER_SYSLOG_TLS_PORT=
+# DFE_RECEIVER_SYSLOG_UDP_PORT=
 # DFE_RECEIVER_VERSION=
 
 ## =============================================================

Makefile

@@ -272,7 +272,7 @@ ifneq ($(strip $(UNPUBLISHED))$(EMBED_UNPUBLISHED),)
 endif
 
 # A receiver listener's host port publishes once its key is set. The listener itself is enabled in the receiver's config
-RECEIVER_LISTENERS := BEATS:5044 GRPC:6000 HEC:8088 OTLP_GRPC:4317 OTLP_HTTP:4318
+RECEIVER_LISTENERS := BEATS:5044 GRPC:6000 HEC:8088 OTLP_GRPC:4317 OTLP_HTTP:4318 SYSLOG_UDP:514/udp SYSLOG_TCP:514 SYSLOG_TLS:6514 NETFLOW:2055/udp IPFIX:4739/udp SFLOW:6343/udp
 RECEIVER_PORTS_SET := $(strip $(foreach listener,$(RECEIVER_LISTENERS),$(if $(strip $(DFE_RECEIVER_$(firstword $(subst :, ,$(listener)))_PORT)),$(listener))))
 
 ifneq ($(RECEIVER_PORTS_SET),)

docs/architecture.md

@@ -217,7 +217,7 @@ Dev builds fetch source into a managed git cache, or from your own checkouts whe
 
 | Repo | Language | Role | Profile |
 |---|---|---|---|
-| `dfe-receiver` | Rust | Push ingest. HTTP `:8080` published, Vector gRPC `:6000`, OTLP, Beats and HEC published once their port key is set | `dfe` |
+| `dfe-receiver` | Rust | Push ingest. HTTP `:8080` published, Vector gRPC `:6000`, OTLP, Beats, HEC, syslog and flow published once their port key is set | `dfe` |
 | `dfe-fetcher` | Rust | Pull ingest from vendor APIs | `dfe` |
 | `dfe-loader` | Rust | Writes ClickHouse; hosts `DfeTransport/Push` | `dfe` |
 | `dfe-archiver` | Rust | Archive sink (filesystem, S3, MinIO, GCS, Azure Blob) | `dfe` |

docs/configuration.md

@@ -267,12 +267,20 @@ The AWS **region** and poll interval are not environment variables: they are lit
 | `DFE_RECEIVER_GRPC_PORT`                         | Host port for the receiver's Vector gRPC listener (container port 6000)             | unset, unpublished                                                  |
 | `DFE_RECEIVER_HEC_PORT`                          | Host port for the receiver's HEC listener (container port 8088)                     | unset, unpublished                                                  |
 | `DFE_RECEIVER_HTTP_PORT`                         | Receiver HTTP port                                                                   | `8080`                                                              |
+| `DFE_RECEIVER_IPFIX_PORT`                        | Host UDP port for the receiver's flow listener on container port 4739 (IPFIX)        | unset, unpublished                                                  |
 | `DFE_RECEIVER_MAX_BODY_SIZE`                     | Largest record the receiver accepts in bytes, under every broker's 8 MiB message limit | `8000000`                                                           |
+| `DFE_RECEIVER_NETFLOW_PORT`                      | Host UDP port for the receiver's flow listener on container port 2055 (NetFlow)      | unset, unpublished                                                  |
 | `DFE_RECEIVER_OTLP_GRPC_PORT`                    | Host port for the receiver's OTLP gRPC listener (container port 4317)               | unset, unpublished                                                  |
 | `DFE_RECEIVER_OTLP_HTTP_PORT`                    | Host port for the receiver's OTLP HTTP listener (container port 4318)               | unset, unpublished                                                  |
 | `DFE_RECEIVER_PROMETHEUS_PORT`                   | Receiver Prometheus port                                                             | `9090`                                                              |
+| `DFE_RECEIVER_SFLOW_PORT`                        | Host UDP port for the receiver's flow listener on container port 6343 (sFlow)        | unset, unpublished                                                  |
+| `DFE_RECEIVER_SYSLOG_TCP_PORT`                   | Host TCP port for the receiver's syslog listener (container port 514)                | unset, unpublished                                                  |
+| `DFE_RECEIVER_SYSLOG_TLS_PORT`                   | Host TCP port for the receiver's syslog over TLS listener (container port 6514)      | unset, unpublished                                                  |
+| `DFE_RECEIVER_SYSLOG_UDP_PORT`                   | Host UDP port for the receiver's syslog listener (container port 514)                | unset, unpublished                                                  |
 
-Setting a listener's port key publishes it through a fragment `make` writes, so a raw `docker compose` outside `make` does not publish it. The listener still has to be enabled in the receiver's config, such as `DFE_RECEIVER_GRPC__ENABLED=true` in `env/receiver.env`.
+Setting a listener's port key publishes it through a fragment `make` writes, so a raw `docker compose` outside `make` does not publish it. The listener still has to be enabled in the receiver's config, such as `DFE_RECEIVER_GRPC__ENABLED=true` in `env/receiver.env`. Syslog takes `DFE_RECEIVER_SYSLOG__ENABLED=true` and `DFE_RECEIVER_SYSLOG__ACCEPT_UNAUTHENTICATED=true`. Its UDP and plain TCP listeners bind whenever syslog is on and have no handshake to authenticate at, so the receiver refuses to start without the second key, even when only the TLS port is published. Flow takes `DFE_RECEIVER_FLOW__ENABLED=true`.
+
+A host port under 1024, such as 514, needs a Docker daemon allowed to bind it. Rootless Docker is not, so pick a high port such as 1514 there.
 
 `make` also sets `DFE_RECEIVER_DLQ_MODE` from the profile: `file_only` on a direct gRPC profile and empty on Kafka, which keeps the receiver's `cascade` default.
 
@@ -473,7 +481,7 @@ The engine runs as its image's user, not yours, so the two meet on a group. `mak
 | 13133 | otel-collector       | health_check       |
 | 19092 | Kafka (any backend)  | Plaintext host     |
 
-The receiver's other listeners publish once their port key is set: 4317 (OTLP gRPC), 4318 (OTLP HTTP), 5044 (Beats), 6000 (Vector gRPC) and 8088 (HEC).
+The receiver's other listeners publish once their port key is set: 4317 (OTLP gRPC), 4318 (OTLP HTTP), 5044 (Beats), 6000 (Vector gRPC), 8088 (HEC), 514 UDP and TCP and 6514 (syslog), and 2055, 4739 and 6343 UDP (flow).
 
 dfe-fetcher and the transforms run one instance per source. An instance publishes no host port.
 

docs/operating.md

@@ -133,7 +133,7 @@ dfe-fetcher runs one container for every source and the three transforms run one
 ports at all -- they are reached over the Docker network, dfe-hyperdx through the
 proxy that holds both its origins. The collector publishes only `24224`, which the
 DOCKER DAEMON sends container stdout to; its OTLP ports stay on the Compose
-network. The receiver's OTLP (`4317`, `4318`), Beats (`5044`) and HEC (`8088`) listeners publish once their port key is set in `.env`. Those `4317`/`4318` are the receiver's *ingest* edge, not the collector's.
+network. The receiver's OTLP (`4317`, `4318`), Beats (`5044`), HEC (`8088`), syslog (`514` UDP and TCP, `6514`) and flow (`2055`, `4739`, `6343` UDP) listeners publish once their port key is set in `.env`. Those `4317`/`4318` are the receiver's *ingest* edge, not the collector's.
 
 Setting `DFE_BIND_HOST=0.0.0.0` opens every operator port at once, including a
 ClickHouse whose `CLICKHOUSE_PASSWORD` defaults to empty and which runs with

scripts/check_compose.py

@@ -256,6 +256,7 @@
     "DFE_ENGINE_API_EXTERNAL": "false",
     "DFE_PROXY_TLS": "true",
     "DFE_RECEIVER_BEATS_PORT": "5044",
+    "DFE_RECEIVER_SYSLOG_UDP_PORT": "1514",
 }
 # Console TLS refuses to start beside the auth profile, so `make dev` runs once per side, each held to the chain its own dials produce. The TLS run also needs what that refusal and the TLS precheck look for: an https origin and a certificate pair openssl accepts.
 _DEV_PATH_CERT_DIR = REPO_ROOT / ".tmp" / "compose-check-certs"

scripts/tests/test_hyperdx_embed_exposure.py

@@ -152,30 +152,50 @@ def test_the_hyperdx_dials_govern_the_embed_and_leave_the_console(
         pytest.param({}, set(), id="none-set"),
         pytest.param(
             {"DFE_RECEIVER_BEATS_PORT": "15044"},
-            {("15044", "5044")},
+            {("15044", "5044", "tcp")},
             id="one-on-its-own-host-port",
         ),
+        pytest.param(
+            {
+                "DFE_RECEIVER_SYSLOG_UDP_PORT": "1514",
+                "DFE_RECEIVER_SYSLOG_TCP_PORT": "1514",
+            },
+            {("1514", "514", "udp"), ("1514", "514", "tcp")},
+            id="syslog-udp-and-tcp-share-a-host-port",
+        ),
         pytest.param(
             {
                 "DFE_RECEIVER_BEATS_PORT": "5044",
                 "DFE_RECEIVER_GRPC_PORT": "6000",
                 "DFE_RECEIVER_HEC_PORT": "8088",
                 "DFE_RECEIVER_OTLP_GRPC_PORT": "4317",
                 "DFE_RECEIVER_OTLP_HTTP_PORT": "4318",
+                "DFE_RECEIVER_SYSLOG_UDP_PORT": "514",
+                "DFE_RECEIVER_SYSLOG_TCP_PORT": "514",
+                "DFE_RECEIVER_SYSLOG_TLS_PORT": "6514",
+                "DFE_RECEIVER_NETFLOW_PORT": "2055",
+                "DFE_RECEIVER_IPFIX_PORT": "4739",
+                "DFE_RECEIVER_SFLOW_PORT": "6343",
             },
             {
-                ("4317", "4317"),
-                ("4318", "4318"),
-                ("5044", "5044"),
-                ("6000", "6000"),
-                ("8088", "8088"),
+                ("4317", "4317", "tcp"),
+                ("4318", "4318", "tcp"),
+                ("5044", "5044", "tcp"),
+                ("6000", "6000", "tcp"),
+                ("8088", "8088", "tcp"),
+                ("514", "514", "udp"),
+                ("514", "514", "tcp"),
+                ("6514", "6514", "tcp"),
+                ("2055", "2055", "udp"),
+                ("4739", "4739", "udp"),
+                ("6343", "6343", "udp"),
             },
             id="all",
         ),
     ],
 )
 def test_a_receiver_port_key_publishes_its_listener(
-    checkout: Path, settings: dict[str, str], published: set[tuple[str, str]]
+    checkout: Path, settings: dict[str, str], published: set[tuple[str, str, str]]
 ) -> None:
     if shutil.which("docker") is None:
         pytest.skip("`docker compose config` renders the model this asserts on")
@@ -194,7 +214,11 @@ def test_a_receiver_port_key_publishes_its_listener(
     assert model is not None
     ports = model["services"]["dfe-receiver"].get("ports") or []
     assert {
-        (str(port.get("published")), str(port.get("target")))
+        (
+            str(port.get("published")),
+            str(port.get("target")),
+            str(port.get("protocol", "tcp")),
+        )
         for port in ports
         if str(port.get("target")) not in _RECEIVER_BASE
     } == published