diff --git a/.env.example b/.env.example index 53c43d8..1069cca 100644 --- a/.env.example +++ b/.env.example @@ -157,10 +157,16 @@ # DFE_RECEIVER_GRPC_PORT= # DFE_RECEIVER_HEC_PORT= # DFE_RECEIVER_HTTP_PORT=8080 +# DFE_RECEIVER_IPFIX_PORT= # DFE_RECEIVER_MAX_BODY_SIZE=8000000 +# DFE_RECEIVER_NETFLOW_PORT= # DFE_RECEIVER_OTLP_GRPC_PORT= # DFE_RECEIVER_OTLP_HTTP_PORT= # DFE_RECEIVER_PROMETHEUS_PORT=9090 +# DFE_RECEIVER_SFLOW_PORT= +# DFE_RECEIVER_SYSLOG_TCP_PORT= +# DFE_RECEIVER_SYSLOG_TLS_PORT= +# DFE_RECEIVER_SYSLOG_UDP_PORT= # DFE_RECEIVER_VERSION= ## ============================================================= diff --git a/Makefile b/Makefile index f7bf3a4..9670cce 100644 --- a/Makefile +++ b/Makefile @@ -272,7 +272,7 @@ ifneq ($(strip $(UNPUBLISHED))$(EMBED_UNPUBLISHED),) endif # A receiver listener's host port publishes once its key is set. The listener itself is enabled in the receiver's config -RECEIVER_LISTENERS := BEATS:5044 GRPC:6000 HEC:8088 OTLP_GRPC:4317 OTLP_HTTP:4318 +RECEIVER_LISTENERS := BEATS:5044 GRPC:6000 HEC:8088 OTLP_GRPC:4317 OTLP_HTTP:4318 SYSLOG_UDP:514/udp SYSLOG_TCP:514 SYSLOG_TLS:6514 NETFLOW:2055/udp IPFIX:4739/udp SFLOW:6343/udp RECEIVER_PORTS_SET := $(strip $(foreach listener,$(RECEIVER_LISTENERS),$(if $(strip $(DFE_RECEIVER_$(firstword $(subst :, ,$(listener)))_PORT)),$(listener)))) ifneq ($(RECEIVER_PORTS_SET),) diff --git a/docs/architecture.md b/docs/architecture.md index 8373b35..5638491 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -217,7 +217,7 @@ Dev builds fetch source into a managed git cache, or from your own checkouts whe | Repo | Language | Role | Profile | |---|---|---|---| -| `dfe-receiver` | Rust | Push ingest. HTTP `:8080` published, Vector gRPC `:6000`, OTLP, Beats and HEC published once their port key is set | `dfe` | +| `dfe-receiver` | Rust | Push ingest. HTTP `:8080` published, Vector gRPC `:6000`, OTLP, Beats, HEC, syslog and flow published once their port key is set | `dfe` | | `dfe-fetcher` | Rust | Pull ingest from vendor APIs | `dfe` | | `dfe-loader` | Rust | Writes ClickHouse; hosts `DfeTransport/Push` | `dfe` | | `dfe-archiver` | Rust | Archive sink (filesystem, S3, MinIO, GCS, Azure Blob) | `dfe` | diff --git a/docs/configuration.md b/docs/configuration.md index 9c68ebe..bdb7b13 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -267,12 +267,20 @@ The AWS **region** and poll interval are not environment variables: they are lit | `DFE_RECEIVER_GRPC_PORT` | Host port for the receiver's Vector gRPC listener (container port 6000) | unset, unpublished | | `DFE_RECEIVER_HEC_PORT` | Host port for the receiver's HEC listener (container port 8088) | unset, unpublished | | `DFE_RECEIVER_HTTP_PORT` | Receiver HTTP port | `8080` | +| `DFE_RECEIVER_IPFIX_PORT` | Host UDP port for the receiver's flow listener on container port 4739 (IPFIX) | unset, unpublished | | `DFE_RECEIVER_MAX_BODY_SIZE` | Largest record the receiver accepts in bytes, under every broker's 8 MiB message limit | `8000000` | +| `DFE_RECEIVER_NETFLOW_PORT` | Host UDP port for the receiver's flow listener on container port 2055 (NetFlow) | unset, unpublished | | `DFE_RECEIVER_OTLP_GRPC_PORT` | Host port for the receiver's OTLP gRPC listener (container port 4317) | unset, unpublished | | `DFE_RECEIVER_OTLP_HTTP_PORT` | Host port for the receiver's OTLP HTTP listener (container port 4318) | unset, unpublished | | `DFE_RECEIVER_PROMETHEUS_PORT` | Receiver Prometheus port | `9090` | +| `DFE_RECEIVER_SFLOW_PORT` | Host UDP port for the receiver's flow listener on container port 6343 (sFlow) | unset, unpublished | +| `DFE_RECEIVER_SYSLOG_TCP_PORT` | Host TCP port for the receiver's syslog listener (container port 514) | unset, unpublished | +| `DFE_RECEIVER_SYSLOG_TLS_PORT` | Host TCP port for the receiver's syslog over TLS listener (container port 6514) | unset, unpublished | +| `DFE_RECEIVER_SYSLOG_UDP_PORT` | Host UDP port for the receiver's syslog listener (container port 514) | unset, unpublished | -Setting a listener's port key publishes it through a fragment `make` writes, so a raw `docker compose` outside `make` does not publish it. The listener still has to be enabled in the receiver's config, such as `DFE_RECEIVER_GRPC__ENABLED=true` in `env/receiver.env`. +Setting a listener's port key publishes it through a fragment `make` writes, so a raw `docker compose` outside `make` does not publish it. The listener still has to be enabled in the receiver's config, such as `DFE_RECEIVER_GRPC__ENABLED=true` in `env/receiver.env`. Syslog takes `DFE_RECEIVER_SYSLOG__ENABLED=true` and `DFE_RECEIVER_SYSLOG__ACCEPT_UNAUTHENTICATED=true`. Its UDP and plain TCP listeners bind whenever syslog is on and have no handshake to authenticate at, so the receiver refuses to start without the second key, even when only the TLS port is published. Flow takes `DFE_RECEIVER_FLOW__ENABLED=true`. + +A host port under 1024, such as 514, needs a Docker daemon allowed to bind it. Rootless Docker is not, so pick a high port such as 1514 there. `make` also sets `DFE_RECEIVER_DLQ_MODE` from the profile: `file_only` on a direct gRPC profile and empty on Kafka, which keeps the receiver's `cascade` default. @@ -473,7 +481,7 @@ The engine runs as its image's user, not yours, so the two meet on a group. `mak | 13133 | otel-collector | health_check | | 19092 | Kafka (any backend) | Plaintext host | -The receiver's other listeners publish once their port key is set: 4317 (OTLP gRPC), 4318 (OTLP HTTP), 5044 (Beats), 6000 (Vector gRPC) and 8088 (HEC). +The receiver's other listeners publish once their port key is set: 4317 (OTLP gRPC), 4318 (OTLP HTTP), 5044 (Beats), 6000 (Vector gRPC), 8088 (HEC), 514 UDP and TCP and 6514 (syslog), and 2055, 4739 and 6343 UDP (flow). dfe-fetcher and the transforms run one instance per source. An instance publishes no host port. diff --git a/docs/operating.md b/docs/operating.md index c25bb5c..144577e 100644 --- a/docs/operating.md +++ b/docs/operating.md @@ -133,7 +133,7 @@ dfe-fetcher runs one container for every source and the three transforms run one ports at all -- they are reached over the Docker network, dfe-hyperdx through the proxy that holds both its origins. The collector publishes only `24224`, which the DOCKER DAEMON sends container stdout to; its OTLP ports stay on the Compose -network. The receiver's OTLP (`4317`, `4318`), Beats (`5044`) and HEC (`8088`) listeners publish once their port key is set in `.env`. Those `4317`/`4318` are the receiver's *ingest* edge, not the collector's. +network. The receiver's OTLP (`4317`, `4318`), Beats (`5044`), HEC (`8088`), syslog (`514` UDP and TCP, `6514`) and flow (`2055`, `4739`, `6343` UDP) listeners publish once their port key is set in `.env`. Those `4317`/`4318` are the receiver's *ingest* edge, not the collector's. Setting `DFE_BIND_HOST=0.0.0.0` opens every operator port at once, including a ClickHouse whose `CLICKHOUSE_PASSWORD` defaults to empty and which runs with diff --git a/scripts/check_compose.py b/scripts/check_compose.py index 99a7e03..406aa8e 100644 --- a/scripts/check_compose.py +++ b/scripts/check_compose.py @@ -256,6 +256,7 @@ "DFE_ENGINE_API_EXTERNAL": "false", "DFE_PROXY_TLS": "true", "DFE_RECEIVER_BEATS_PORT": "5044", + "DFE_RECEIVER_SYSLOG_UDP_PORT": "1514", } # Console TLS refuses to start beside the auth profile, so `make dev` runs once per side, each held to the chain its own dials produce. The TLS run also needs what that refusal and the TLS precheck look for: an https origin and a certificate pair openssl accepts. _DEV_PATH_CERT_DIR = REPO_ROOT / ".tmp" / "compose-check-certs" diff --git a/scripts/tests/test_hyperdx_embed_exposure.py b/scripts/tests/test_hyperdx_embed_exposure.py index 7ec2d42..604ca7e 100644 --- a/scripts/tests/test_hyperdx_embed_exposure.py +++ b/scripts/tests/test_hyperdx_embed_exposure.py @@ -152,9 +152,17 @@ def test_the_hyperdx_dials_govern_the_embed_and_leave_the_console( pytest.param({}, set(), id="none-set"), pytest.param( {"DFE_RECEIVER_BEATS_PORT": "15044"}, - {("15044", "5044")}, + {("15044", "5044", "tcp")}, id="one-on-its-own-host-port", ), + pytest.param( + { + "DFE_RECEIVER_SYSLOG_UDP_PORT": "1514", + "DFE_RECEIVER_SYSLOG_TCP_PORT": "1514", + }, + {("1514", "514", "udp"), ("1514", "514", "tcp")}, + id="syslog-udp-and-tcp-share-a-host-port", + ), pytest.param( { "DFE_RECEIVER_BEATS_PORT": "5044", @@ -162,20 +170,32 @@ def test_the_hyperdx_dials_govern_the_embed_and_leave_the_console( "DFE_RECEIVER_HEC_PORT": "8088", "DFE_RECEIVER_OTLP_GRPC_PORT": "4317", "DFE_RECEIVER_OTLP_HTTP_PORT": "4318", + "DFE_RECEIVER_SYSLOG_UDP_PORT": "514", + "DFE_RECEIVER_SYSLOG_TCP_PORT": "514", + "DFE_RECEIVER_SYSLOG_TLS_PORT": "6514", + "DFE_RECEIVER_NETFLOW_PORT": "2055", + "DFE_RECEIVER_IPFIX_PORT": "4739", + "DFE_RECEIVER_SFLOW_PORT": "6343", }, { - ("4317", "4317"), - ("4318", "4318"), - ("5044", "5044"), - ("6000", "6000"), - ("8088", "8088"), + ("4317", "4317", "tcp"), + ("4318", "4318", "tcp"), + ("5044", "5044", "tcp"), + ("6000", "6000", "tcp"), + ("8088", "8088", "tcp"), + ("514", "514", "udp"), + ("514", "514", "tcp"), + ("6514", "6514", "tcp"), + ("2055", "2055", "udp"), + ("4739", "4739", "udp"), + ("6343", "6343", "udp"), }, id="all", ), ], ) def test_a_receiver_port_key_publishes_its_listener( - checkout: Path, settings: dict[str, str], published: set[tuple[str, str]] + checkout: Path, settings: dict[str, str], published: set[tuple[str, str, str]] ) -> None: if shutil.which("docker") is None: pytest.skip("`docker compose config` renders the model this asserts on") @@ -194,7 +214,11 @@ def test_a_receiver_port_key_publishes_its_listener( assert model is not None ports = model["services"]["dfe-receiver"].get("ports") or [] assert { - (str(port.get("published")), str(port.get("target"))) + ( + str(port.get("published")), + str(port.get("target")), + str(port.get("protocol", "tcp")), + ) for port in ports if str(port.get("target")) not in _RECEIVER_BASE } == published